feat: working TOM agent
This commit is contained in:
0
tests/__init__.py
Normal file
0
tests/__init__.py
Normal file
5
tests/conftest.py
Normal file
5
tests/conftest.py
Normal file
@@ -0,0 +1,5 @@
|
||||
# Env var setup that must run before the engine's settings are imported.
|
||||
import os
|
||||
|
||||
# The suite must not pick up credentials from a developer's .env.
|
||||
os.environ["ENV_FILE_EXPORT"] = "false"
|
||||
7
tests/fixtures/architecture/README.md
vendored
Normal file
7
tests/fixtures/architecture/README.md
vendored
Normal file
@@ -0,0 +1,7 @@
|
||||
# Architecture Executions, as fixtures
|
||||
|
||||
`case-08-landing-zone/` is the data files of one completed Execution of the solution architecture agent, copied unaltered from its evaluation results (run `20261011T185421Z`, at its commit `8b397cd`). The case is that agent's fictional benchmark landing zone; no customer's data is in it.
|
||||
|
||||
It is the contract between the two agents: the digest's tests read it, so a change to the shape of these files is caught here. Replace it by copying a newer Execution's files, never by editing these.
|
||||
|
||||
Not copied: the prose Deliverables, the Terraform, the solution guide and the working material. The digest reads none of them.
|
||||
79
tests/fixtures/architecture/case-08-landing-zone/components.json
vendored
Normal file
79
tests/fixtures/architecture/case-08-landing-zone/components.json
vendored
Normal file
@@ -0,0 +1,79 @@
|
||||
[
|
||||
{
|
||||
"id": "C-1",
|
||||
"name": "Resource Hierarchy and Environment Boundary Manager",
|
||||
"requirements": [
|
||||
"FR-1",
|
||||
"NFR-1",
|
||||
"NFR-8",
|
||||
"NFR-9"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "C-2",
|
||||
"name": "Dedicated Private Hybrid Connectivity Gateway",
|
||||
"requirements": [
|
||||
"FR-2",
|
||||
"NFR-3",
|
||||
"NFR-14"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "C-3",
|
||||
"name": "Centralised Outbound Traffic Inspection and Egress Boundary",
|
||||
"requirements": [
|
||||
"FR-3",
|
||||
"NFR-2",
|
||||
"NFR-4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "C-4",
|
||||
"name": "Centralised Immutable Audit Event and Compliance Log Repository",
|
||||
"requirements": [
|
||||
"FR-4",
|
||||
"NFR-7",
|
||||
"NFR-8"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "C-5",
|
||||
"name": "Automated Infrastructure Orchestration and Deployment Engine",
|
||||
"requirements": [
|
||||
"FR-5",
|
||||
"NFR-10"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "C-6",
|
||||
"name": "Federated Identity and Role Governance Authority",
|
||||
"requirements": [
|
||||
"FR-6",
|
||||
"NFR-5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "C-7",
|
||||
"name": "Cryptographic Key Lifecycle and Encryption Controller",
|
||||
"requirements": [
|
||||
"NFR-6",
|
||||
"NFR-8"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "C-8",
|
||||
"name": "Platform Telemetry and Operational Alerting Engine",
|
||||
"requirements": [
|
||||
"NFR-11",
|
||||
"NFR-12",
|
||||
"NFR-15"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "C-9",
|
||||
"name": "Financial Governance and Budget Enforcement Monitor",
|
||||
"requirements": [
|
||||
"NFR-13"
|
||||
]
|
||||
}
|
||||
]
|
||||
140
tests/fixtures/architecture/case-08-landing-zone/execution.json
vendored
Normal file
140
tests/fixtures/architecture/case-08-landing-zone/execution.json
vendored
Normal file
@@ -0,0 +1,140 @@
|
||||
{
|
||||
"customer_case": "case-08-landing-zone",
|
||||
"execution_id": "case-08-landing-zone-185426",
|
||||
"session_id": "cdd6bfe7-2bd8-44af-8e15-9dcfa869eb84",
|
||||
"intent_version": 1,
|
||||
"workflow_request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.",
|
||||
"baseline": null,
|
||||
"overlay_skills": {},
|
||||
"terraform_modules": false,
|
||||
"scope": {
|
||||
"status": "BASELINE_LOCKED",
|
||||
"readiness": {
|
||||
"supplied": 44,
|
||||
"answered": 0,
|
||||
"assumed": 12,
|
||||
"unknown": 4,
|
||||
"not_applicable": 3
|
||||
}
|
||||
},
|
||||
"delivery_plan": {
|
||||
"work_packages": 9,
|
||||
"activities": 15,
|
||||
"working_days": 36.0,
|
||||
"weeks": 7.2,
|
||||
"team_size": 3,
|
||||
"window_weeks": 10,
|
||||
"within_window": true,
|
||||
"critical_path": [
|
||||
"A-6",
|
||||
"A-10",
|
||||
"A-13",
|
||||
"A-14",
|
||||
"A-15"
|
||||
]
|
||||
},
|
||||
"status": "completed",
|
||||
"warnings": [],
|
||||
"risks": [],
|
||||
"assessment": {
|
||||
"recommendations_quality": {
|
||||
"specific": 0.91,
|
||||
"actionable": 0.79,
|
||||
"trade_offs": 0.95,
|
||||
"traceable": 0.73
|
||||
},
|
||||
"mean": 0.84,
|
||||
"meets_bar": true
|
||||
},
|
||||
"phases": {
|
||||
"scoping": "approved",
|
||||
"requirements_discovery": "approved",
|
||||
"solution_architecture": "approved",
|
||||
"solution_implementation": "approved",
|
||||
"solution_validation": "approved",
|
||||
"packaging": "approved"
|
||||
},
|
||||
"solution_guide": "solution-architecture-guide.md",
|
||||
"customer_questions": "customer-questions.md",
|
||||
"changes": null,
|
||||
"deliverables": {
|
||||
"scope": {
|
||||
"document": "deliverables/scope.md",
|
||||
"attachments": [
|
||||
"scope.json",
|
||||
"scope.yaml"
|
||||
],
|
||||
"status": "submitted"
|
||||
},
|
||||
"requirements": {
|
||||
"document": "deliverables/requirements.md",
|
||||
"attachments": [
|
||||
"requirements.json"
|
||||
],
|
||||
"status": "submitted"
|
||||
},
|
||||
"technical_decomposition": {
|
||||
"document": "deliverables/technical_decomposition.md",
|
||||
"attachments": [
|
||||
"components.json"
|
||||
],
|
||||
"status": "submitted"
|
||||
},
|
||||
"product_selection": {
|
||||
"document": "deliverables/product_selection.md",
|
||||
"attachments": [
|
||||
"products.json"
|
||||
],
|
||||
"status": "submitted"
|
||||
},
|
||||
"design_recommendations": {
|
||||
"document": "deliverables/design_recommendations.md",
|
||||
"attachments": [
|
||||
"recommendations.json"
|
||||
],
|
||||
"status": "submitted"
|
||||
},
|
||||
"target_architecture": {
|
||||
"document": "deliverables/target_architecture.md",
|
||||
"attachments": [
|
||||
"architecture.mmd",
|
||||
"target-architecture.json"
|
||||
],
|
||||
"status": "submitted"
|
||||
},
|
||||
"architecture_description": {
|
||||
"document": "deliverables/architecture_description.md",
|
||||
"attachments": [],
|
||||
"status": "submitted"
|
||||
},
|
||||
"deployment_guidance": {
|
||||
"document": "deliverables/deployment_guidance.md",
|
||||
"attachments": [
|
||||
"implementation.json",
|
||||
"terraform/main.tf",
|
||||
"terraform/variables.tf",
|
||||
"terraform/versions.tf"
|
||||
],
|
||||
"status": "submitted"
|
||||
},
|
||||
"validation_results": {
|
||||
"document": "deliverables/validation_results.md",
|
||||
"attachments": [
|
||||
"verdicts.json"
|
||||
],
|
||||
"status": "submitted"
|
||||
},
|
||||
"work_breakdown": {
|
||||
"document": "deliverables/work_breakdown.md",
|
||||
"attachments": [
|
||||
"work-breakdown.json"
|
||||
],
|
||||
"status": "submitted"
|
||||
},
|
||||
"solution_guide": {
|
||||
"document": "deliverables/solution_guide.md",
|
||||
"attachments": [],
|
||||
"status": "submitted"
|
||||
}
|
||||
}
|
||||
}
|
||||
38
tests/fixtures/architecture/case-08-landing-zone/products.json
vendored
Normal file
38
tests/fixtures/architecture/case-08-landing-zone/products.json
vendored
Normal file
@@ -0,0 +1,38 @@
|
||||
[
|
||||
{
|
||||
"component": "C-1",
|
||||
"product": "Resource Manager"
|
||||
},
|
||||
{
|
||||
"component": "C-2",
|
||||
"product": "Cloud Interconnect"
|
||||
},
|
||||
{
|
||||
"component": "C-3",
|
||||
"product": "Palo Alto Networks VM-Series"
|
||||
},
|
||||
{
|
||||
"component": "C-4",
|
||||
"product": "Cloud Storage"
|
||||
},
|
||||
{
|
||||
"component": "C-5",
|
||||
"product": "GitHub Actions"
|
||||
},
|
||||
{
|
||||
"component": "C-6",
|
||||
"product": "IAM Workload Identity Federation"
|
||||
},
|
||||
{
|
||||
"component": "C-7",
|
||||
"product": "Cloud KMS"
|
||||
},
|
||||
{
|
||||
"component": "C-8",
|
||||
"product": "Cloud Monitoring"
|
||||
},
|
||||
{
|
||||
"component": "C-9",
|
||||
"product": "Cloud Billing"
|
||||
}
|
||||
]
|
||||
248
tests/fixtures/architecture/case-08-landing-zone/recommendations.json
vendored
Normal file
248
tests/fixtures/architecture/case-08-landing-zone/recommendations.json
vendored
Normal file
@@ -0,0 +1,248 @@
|
||||
[
|
||||
{
|
||||
"id": "DR-SEC-1",
|
||||
"pillar": "security",
|
||||
"text": "Implement a VPC Service Controls perimeter around all Cardholder Data Environment (CDE) projects and the centralized audit log buckets.",
|
||||
"components": [
|
||||
"C-1",
|
||||
"C-4"
|
||||
],
|
||||
"conditions": [
|
||||
"A VPC Service Controls perimeter encloses the CDE and audit log projects.",
|
||||
"The VPC Service Controls perimeter restricts ingress and egress to authorized traffic."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-SEC-2",
|
||||
"pillar": "security",
|
||||
"text": "Apply Organization Policy constraints to disable default network creation, deny external IPs, prohibit service account key creation, and restrict resource locations.",
|
||||
"components": [
|
||||
"C-1"
|
||||
],
|
||||
"conditions": [
|
||||
"The compute.skipDefaultNetworkCreation constraint is enforced.",
|
||||
"The compute.vmExternalIpAccess constraint is enforced to deny all.",
|
||||
"The iam.disableServiceAccountKeyCreation constraint is enforced.",
|
||||
"The gcp.resourceLocations constraint restricts resources to europe-west2 and europe-west1."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-SEC-3",
|
||||
"pillar": "security",
|
||||
"text": "Enable Confidential Computing on the payment engine Compute Engine instances to encrypt data in-use.",
|
||||
"components": [],
|
||||
"conditions": [
|
||||
"Confidential Computing is enabled on the core payment engine Compute Engine instances."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-SEC-4",
|
||||
"pillar": "security",
|
||||
"text": "Deploy Sensitive Data Protection (Cloud DLP) inspection templates to detect and mask PAN data.",
|
||||
"components": [
|
||||
"C-4"
|
||||
],
|
||||
"conditions": [
|
||||
"Sensitive Data Protection (Cloud DLP) inspection templates are configured to detect and mask PAN data."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-SEC-5",
|
||||
"pillar": "security",
|
||||
"text": "Enable Security Command Center at the organization level with Event and Container Threat Detection.",
|
||||
"components": [
|
||||
"C-1",
|
||||
"C-8"
|
||||
],
|
||||
"conditions": [
|
||||
"Security Command Center is activated at the organization level.",
|
||||
"Event Threat Detection and Container Threat Detection are enabled in SCC."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-SEC-6",
|
||||
"pillar": "security",
|
||||
"text": "Implement event-driven automated responses using Eventarc and Cloud Run functions for SCC findings.",
|
||||
"components": [
|
||||
"C-8"
|
||||
],
|
||||
"conditions": [
|
||||
"Eventarc is configured to route high-severity SCC findings.",
|
||||
"Cloud Run functions are configured to execute automated remediation on Eventarc triggers."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-REL-1",
|
||||
"pillar": "reliability",
|
||||
"text": "Provision redundant Dedicated Cloud Interconnect circuits across distinct physical edge colocation facilities, terminating on redundant Cloud Routers with BFD.",
|
||||
"components": [
|
||||
"C-2"
|
||||
],
|
||||
"conditions": [
|
||||
"Dedicated Cloud Interconnect circuits are provisioned across redundant edge colocation facilities.",
|
||||
"Cloud Interconnects terminate on redundant Cloud Routers in each region.",
|
||||
"Bidirectional Forwarding Detection (BFD) is enabled on the Cloud Router BGP sessions."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-REL-2",
|
||||
"pillar": "reliability",
|
||||
"text": "Deploy Palo Alto VM-Series firewalls in a regional Managed Instance Group across at least three zones, fronted by an Internal Passthrough Network Load Balancer.",
|
||||
"components": [
|
||||
"C-3"
|
||||
],
|
||||
"conditions": [
|
||||
"Palo Alto VM-Series firewalls are deployed in a regional Managed Instance Group (MIG) across at least three zones.",
|
||||
"An Internal Passthrough Network Load Balancer routes traffic to the VM-Series MIG.",
|
||||
"Health checks are configured on the internal load balancer to probe firewall health."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-REL-3",
|
||||
"pillar": "reliability",
|
||||
"text": "Provision dedicated regional Cloud KMS key rings in both europe-west2 and europe-west1, granting workload identity service accounts Decrypter roles on both.",
|
||||
"components": [
|
||||
"C-7",
|
||||
"C-6"
|
||||
],
|
||||
"conditions": [
|
||||
"Regional Cloud KMS key rings are provisioned in both europe-west2 and europe-west1.",
|
||||
"Workload Identity Federation service accounts are granted Decrypter roles on both regional key sets."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-REL-4",
|
||||
"pillar": "reliability",
|
||||
"text": "Configure the Cloud Storage buckets for audit logging as Dual-Region spanning europe-west2 and europe-west1.",
|
||||
"components": [
|
||||
"C-4"
|
||||
],
|
||||
"conditions": [
|
||||
"Audit log Cloud Storage buckets are configured as Dual-Region spanning europe-west2 and europe-west1."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-REL-5",
|
||||
"pillar": "reliability",
|
||||
"text": "Isolate the Terraform state of the primary region from the standby region using multi-region Cloud Storage buckets with Object Versioning.",
|
||||
"components": [
|
||||
"C-5"
|
||||
],
|
||||
"conditions": [
|
||||
"Terraform state for primary and standby regions is decoupled into separate failure domains.",
|
||||
"Terraform state is stored in multi-region Cloud Storage buckets with Object Versioning enabled."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-OP-1",
|
||||
"pillar": "operational-excellence",
|
||||
"text": "Integrate Policy-as-Code validation into the GitHub Actions pull request workflows.",
|
||||
"components": [
|
||||
"C-5"
|
||||
],
|
||||
"conditions": [
|
||||
"GitHub Actions workflows include automated Policy-as-Code validation steps prior to deployment execution."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-OP-2",
|
||||
"pillar": "operational-excellence",
|
||||
"text": "Establish explicit Service Level Indicators (SLIs) and configure multi-burn-rate alerting policies in Cloud Monitoring.",
|
||||
"components": [
|
||||
"C-8"
|
||||
],
|
||||
"conditions": [
|
||||
"Service Level Objectives (SLOs) and SLIs are defined in Cloud Monitoring Service Monitoring.",
|
||||
"Multi-burn-rate alerting policies are configured based on the SLO error budget."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-OP-3",
|
||||
"pillar": "operational-excellence",
|
||||
"text": "Activate the IAM Recommender to continuously evaluate role bindings.",
|
||||
"components": [
|
||||
"C-6"
|
||||
],
|
||||
"conditions": [
|
||||
"IAM Recommender is activated on the foundation projects."
|
||||
],
|
||||
"outside_code": [
|
||||
"Permissions are continuously rightsized based on IAM Recommender findings."
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "DR-COST-1",
|
||||
"pillar": "cost-optimization",
|
||||
"text": "Enforce a mandatory labeling taxonomy through Organization Policies and export granular billing data to BigQuery.",
|
||||
"components": [
|
||||
"C-9",
|
||||
"C-1"
|
||||
],
|
||||
"conditions": [
|
||||
"Cloud Billing export to BigQuery is enabled.",
|
||||
"A mandatory labeling taxonomy is enforced via Organization Policy Service tags."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-COST-2",
|
||||
"pillar": "cost-optimization",
|
||||
"text": "Implement Cloud Storage Object Lifecycle Management to transition logs to Nearline storage after 30 days and to Coldline or Archive storage after 90 days.",
|
||||
"components": [
|
||||
"C-4"
|
||||
],
|
||||
"conditions": [
|
||||
"Object Lifecycle Management rules transition Cloud Storage objects to Nearline storage after 30 days.",
|
||||
"Object Lifecycle Management rules transition Cloud Storage objects to Coldline or Archive storage after 90 days."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-PERF-1",
|
||||
"pillar": "performance-optimization",
|
||||
"text": "Configure Cloud Routers with regional dynamic routing to localize egress traffic within the warm standby region.",
|
||||
"components": [
|
||||
"C-2"
|
||||
],
|
||||
"conditions": [
|
||||
"Cloud Routers are configured with regional dynamic routing mode."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-PERF-2",
|
||||
"pillar": "performance-optimization",
|
||||
"text": "Instrument the deployment with OpenTelemetry exporting to Cloud Trace, and use Cloud Profiler on the compute instances.",
|
||||
"components": [
|
||||
"C-8"
|
||||
],
|
||||
"conditions": [
|
||||
"OpenTelemetry tracing is configured to export to Cloud Trace.",
|
||||
"Cloud Profiler agents are enabled on the core payments engine instances."
|
||||
],
|
||||
"outside_code": []
|
||||
},
|
||||
{
|
||||
"id": "DR-SUST-1",
|
||||
"pillar": "sustainability",
|
||||
"text": "Implement a Pilot Light warm standby by maintaining active data replication while scaling the standby compute instances to a minimal baseline.",
|
||||
"components": [],
|
||||
"conditions": [
|
||||
"Secondary region compute instances are scaled to a minimal baseline (Pilot Light) configuration during normal operations."
|
||||
],
|
||||
"outside_code": []
|
||||
}
|
||||
]
|
||||
280
tests/fixtures/architecture/case-08-landing-zone/requirements.json
vendored
Normal file
280
tests/fixtures/architecture/case-08-landing-zone/requirements.json
vendored
Normal file
@@ -0,0 +1,280 @@
|
||||
[
|
||||
{
|
||||
"id": "FR-1",
|
||||
"kind": "functional",
|
||||
"text": "The platform foundation isolates development, staging, and production environments across distinct Google Cloud projects and resource folders.",
|
||||
"from": [
|
||||
"REQ-02"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "FR-2",
|
||||
"kind": "functional",
|
||||
"text": "The platform foundation establishes private bidirectional network connectivity between Google Cloud and on-premises core payment systems over Cloud Interconnect.",
|
||||
"from": [
|
||||
"REQ-03"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "FR-3",
|
||||
"kind": "functional",
|
||||
"text": "The platform foundation routes all outbound internet-bound traffic from workload spoke networks through centralised network security inspection appliances.",
|
||||
"from": [
|
||||
"REQ-03"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "FR-4",
|
||||
"kind": "functional",
|
||||
"text": "The platform foundation captures and exports all administrative and data access audit logs across environments into a centralised, tamper-resistant log repository.",
|
||||
"from": [
|
||||
"REQ-02"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "FR-5",
|
||||
"kind": "functional",
|
||||
"text": "The platform foundation provisions and manages all cloud infrastructure as code via automated CI/CD pipelines executing validated pull request workflows.",
|
||||
"from": [
|
||||
"REQ-01"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "FR-6",
|
||||
"kind": "functional",
|
||||
"text": "The platform foundation federates corporate identity for single sign-on and enforces least-privilege role-based access control for platform administrators and application squads.",
|
||||
"from": [
|
||||
"REQ-04"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-1",
|
||||
"kind": "non_functional",
|
||||
"text": "Environments are partitioned into development, staging, and production tiers, enforcing corp_workload controls on lower environments and pci_regulated_workload controls on production.",
|
||||
"conditions": [
|
||||
"Separate projects or folders exist for development, staging, and production environment tiers.",
|
||||
"Production environments enforce pci_regulated_workload archetype security controls.",
|
||||
"Development and staging environments enforce corp_workload archetype security controls."
|
||||
],
|
||||
"outside_code": [],
|
||||
"from": [
|
||||
"REQ-02",
|
||||
"governance.environment_tiers"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-2",
|
||||
"kind": "non_functional",
|
||||
"text": "Network connectivity is structured as a Shared VPC hub-and-spoke topology with zero direct internet ingress to workload subnets.",
|
||||
"conditions": [
|
||||
"A Shared VPC host project is connected to service projects across environment tiers.",
|
||||
"Workload subnets have private Google access enabled and no external public IP addresses assigned.",
|
||||
"Ingress firewall rules deny uninspected public internet traffic to workload subnets."
|
||||
],
|
||||
"outside_code": [],
|
||||
"from": [
|
||||
"REQ-02",
|
||||
"networking.topology",
|
||||
"networking.exposure"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-3",
|
||||
"kind": "non_functional",
|
||||
"text": "Hybrid connectivity to on-premises core payment systems is enabled via a 10Gbps Dedicated Cloud Interconnect.",
|
||||
"conditions": [
|
||||
"Dedicated Cloud Interconnect connections are configured for hybrid connectivity.",
|
||||
"Cloud Interconnect capacity is provisioned at 10Gbps bandwidth.",
|
||||
"Interconnect VLAN attachments are associated with Cloud Routers in the Shared VPC hub."
|
||||
],
|
||||
"outside_code": [],
|
||||
"from": [
|
||||
"REQ-03",
|
||||
"networking.hybrid_connectivity.enabled",
|
||||
"networking.hybrid_connectivity.type",
|
||||
"networking.hybrid_connectivity.bandwidth_gbps"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-4",
|
||||
"kind": "non_functional",
|
||||
"text": "All outbound internet traffic from workload spokes is forced through centralised inspection appliances deployed in the hub network.",
|
||||
"conditions": [
|
||||
"Inspection appliance instances are deployed in a centralised hub network.",
|
||||
"Custom VPC route tables direct 0.0.0.0/0 egress traffic from spokes to the centralised inspection appliance instances or internal load balancer."
|
||||
],
|
||||
"outside_code": [],
|
||||
"from": [
|
||||
"REQ-03",
|
||||
"networking.egress_control.model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-5",
|
||||
"kind": "non_functional",
|
||||
"text": "Administrative access is authenticated via Entra ID single sign-on, and delivery pipelines authenticate via Workload Identity Federation without stored service account keys.",
|
||||
"conditions": [
|
||||
"Workload Identity Federation pools and providers are configured for GitHub Actions authentication.",
|
||||
"IAM bindings grant roles to federated GitHub Actions identities without downloadable service account keys.",
|
||||
"Corporate single sign-on is configured with Entra ID as the identity source of truth."
|
||||
],
|
||||
"outside_code": [],
|
||||
"from": [
|
||||
"REQ-04",
|
||||
"governance.identity_source_of_truth",
|
||||
"governance.pipeline_authentication"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-6",
|
||||
"kind": "non_functional",
|
||||
"text": "Customer confidential data is encrypted at rest using customer-managed encryption keys with a 90-day rotation schedule.",
|
||||
"conditions": [
|
||||
"Cloud KMS customer-managed encryption keys are configured for infrastructure resources storing confidential data.",
|
||||
"Cloud KMS keys have an automated rotation period set to 90 days (7776000s)."
|
||||
],
|
||||
"outside_code": [],
|
||||
"from": [
|
||||
"REQ-04",
|
||||
"compliance.encryption.type",
|
||||
"compliance.encryption.key_rotation_days"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-7",
|
||||
"kind": "non_functional",
|
||||
"text": "Audit logs across all environments are retained for 365 days in a storage bucket protected by an immutable retention policy.",
|
||||
"conditions": [
|
||||
"An organization-level or project-level log sink exports audit logs to a centralised Cloud Storage bucket.",
|
||||
"The centralised audit log storage bucket specifies a retention period of 365 days (31536000 seconds).",
|
||||
"The centralised audit log storage bucket has retention policy locking enabled to prevent modification or early deletion."
|
||||
],
|
||||
"outside_code": [],
|
||||
"from": [
|
||||
"REQ-02",
|
||||
"compliance.audit_logging.retention_days",
|
||||
"compliance.audit_logging.immutable_retention_policy"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-8",
|
||||
"kind": "non_functional",
|
||||
"text": "The platform foundation enforces DORA, ISO27001, and PCI-DSS v4.0 regulatory baselines and protects confidential data classification.",
|
||||
"conditions": [
|
||||
"Organization policy constraints enforce perimeter isolation and restrict unauthenticated resource sharing.",
|
||||
"IAM policies restrict access to confidential data resources to authorized service identities and roles."
|
||||
],
|
||||
"outside_code": [
|
||||
"The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit."
|
||||
],
|
||||
"from": [
|
||||
"REQ-02",
|
||||
"compliance.frameworks",
|
||||
"compliance.data_classification"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-9",
|
||||
"kind": "non_functional",
|
||||
"text": "Platform foundation resources are provisioned across europe-west2 (primary) and europe-west1 (secondary) in a warm standby model, restricting data residency to the UK and EU.",
|
||||
"conditions": [
|
||||
"Primary landing zone network and foundation resources are provisioned in europe-west2.",
|
||||
"Secondary landing zone network and foundation standby resources are provisioned in europe-west1.",
|
||||
"Resource location organization policies restrict resource and data storage locations to europe-west2 and europe-west1."
|
||||
],
|
||||
"outside_code": [
|
||||
"The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic.",
|
||||
"Data residency is guaranteed within the United Kingdom and European Union."
|
||||
],
|
||||
"from": [
|
||||
"REQ-02",
|
||||
"estate.regions.primary",
|
||||
"estate.regions.secondary",
|
||||
"estate.regions.resiliency_tier",
|
||||
"estate.data_residency"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-10",
|
||||
"kind": "non_functional",
|
||||
"text": "The platform foundation landing zone is delivered and operational for workload ingestion within 10 weeks.",
|
||||
"conditions": [
|
||||
"All landing zone foundation resources are codified in modular Terraform configurations."
|
||||
],
|
||||
"outside_code": [
|
||||
"Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks."
|
||||
],
|
||||
"from": [
|
||||
"REQ-01"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-11",
|
||||
"kind": "non_functional",
|
||||
"text": "The platform foundation maintains 99.9% availability with a 4-hour recovery time objective (RTO) and a 1-hour recovery point objective (RPO).",
|
||||
"conditions": [
|
||||
"Regional networking and core management services are deployed across at least three zones in the primary region."
|
||||
],
|
||||
"outside_code": [
|
||||
"Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.",
|
||||
"Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.",
|
||||
"Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour."
|
||||
],
|
||||
"from": [
|
||||
"REQ-02"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-12",
|
||||
"kind": "non_functional",
|
||||
"text": "Platform operations monitoring and alerting are implemented using Google Cloud Operations Suite with 24/7 alerting capabilities.",
|
||||
"conditions": [
|
||||
"Cloud Monitoring workspaces and notification channels are configured for the platform foundation.",
|
||||
"Alert policies are configured for Interconnect link degradation, appliance health, and egress routing failures."
|
||||
],
|
||||
"outside_code": [
|
||||
"Alert notifications trigger operational incident workflows for the 24/7 central platform operations team."
|
||||
],
|
||||
"from": []
|
||||
},
|
||||
{
|
||||
"id": "NFR-13",
|
||||
"kind": "non_functional",
|
||||
"text": "Infrastructure costs are tracked against a monthly budget of 25,000 GBP with automated alerting.",
|
||||
"conditions": [
|
||||
"A Cloud Billing budget is configured for the foundation projects with currency set to GBP.",
|
||||
"Budget alert thresholds are configured at 50%, 80%, 90%, and 100% of the 25,000 GBP monthly budget."
|
||||
],
|
||||
"outside_code": [
|
||||
"Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off."
|
||||
],
|
||||
"from": []
|
||||
},
|
||||
{
|
||||
"id": "NFR-14",
|
||||
"kind": "non_functional",
|
||||
"text": "Hybrid networking provides dedicated 10Gbps unthrottled throughput with minimal latency between Google Cloud and on-premises core systems.",
|
||||
"conditions": [
|
||||
"Cloud Routers configured for Interconnect advertise non-overlapping RFC 1918 BGP routes.",
|
||||
"Dedicated Interconnect VLAN attachments are configured with dedicated 10Gbps bandwidth allocation."
|
||||
],
|
||||
"outside_code": [
|
||||
"Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing."
|
||||
],
|
||||
"from": [
|
||||
"REQ-03"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "NFR-15",
|
||||
"kind": "non_functional",
|
||||
"text": "Landing zone resources are deployed in Google Cloud regions with strong low-carbon energy ratings.",
|
||||
"conditions": [
|
||||
"Foundation resources are constrained to europe-west2 (London) and europe-west1 (Belgium) regions."
|
||||
],
|
||||
"outside_code": [
|
||||
"Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores."
|
||||
],
|
||||
"from": []
|
||||
}
|
||||
]
|
||||
233
tests/fixtures/architecture/case-08-landing-zone/scope.json
vendored
Normal file
233
tests/fixtures/architecture/case-08-landing-zone/scope.json
vendored
Normal file
@@ -0,0 +1,233 @@
|
||||
{
|
||||
"kind": "platform_foundation",
|
||||
"stated": {
|
||||
"engagement": {
|
||||
"kind": "platform_foundation"
|
||||
},
|
||||
"intent": {
|
||||
"statement": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.\n",
|
||||
"requirements": [
|
||||
{
|
||||
"id": "REQ-01",
|
||||
"category": "Timeline",
|
||||
"statement": "Platform foundation must be ready for workload ingestion within 10 weeks."
|
||||
},
|
||||
{
|
||||
"id": "REQ-02",
|
||||
"category": "Compliance",
|
||||
"statement": "Must achieve DORA resiliency alignment and PCI-DSS v4.0 perimeter isolation."
|
||||
},
|
||||
{
|
||||
"id": "REQ-03",
|
||||
"category": "Networking",
|
||||
"statement": "10Gbps on-premises hybrid connectivity; centralised inspection appliances for egress."
|
||||
},
|
||||
{
|
||||
"id": "REQ-04",
|
||||
"category": "Identity and security",
|
||||
"statement": "Workload identity federation with GitHub Actions; Entra ID single sign-on; customer-managed encryption keys."
|
||||
}
|
||||
],
|
||||
"exclusions": [
|
||||
"Application container deployment; workloads are managed by application squads.",
|
||||
"An active-active secondary region in phase 1; warm standby is acceptable."
|
||||
]
|
||||
},
|
||||
"estate": {
|
||||
"cloud_provider": "gcp",
|
||||
"mode": "greenfield",
|
||||
"regions": {
|
||||
"primary": "europe-west2",
|
||||
"secondary": "europe-west1",
|
||||
"resiliency_tier": "warm_standby"
|
||||
},
|
||||
"data_residency": "United Kingdom and European Union",
|
||||
"existing_systems": "On-premises payment systems that stay in place and connect over the interconnect."
|
||||
},
|
||||
"workload": {
|
||||
"availability": {
|
||||
"hours": "always"
|
||||
}
|
||||
},
|
||||
"compliance": {
|
||||
"industry": "financial_services",
|
||||
"frameworks": [
|
||||
"DORA",
|
||||
"ISO27001",
|
||||
"PCI_DSS_v4"
|
||||
],
|
||||
"data_classification": "confidential",
|
||||
"encryption": {
|
||||
"type": "cmek",
|
||||
"key_rotation_days": 90
|
||||
},
|
||||
"audit_logging": {
|
||||
"retention_days": 365,
|
||||
"immutable_retention_policy": true
|
||||
}
|
||||
},
|
||||
"networking": {
|
||||
"exposure": "internal_only",
|
||||
"topology": "shared_vpc_hub_and_spoke",
|
||||
"hybrid_connectivity": {
|
||||
"enabled": true,
|
||||
"type": "cloud_interconnect",
|
||||
"bandwidth_gbps": 10
|
||||
},
|
||||
"egress_control": {
|
||||
"model": "centralized_appliance"
|
||||
}
|
||||
},
|
||||
"governance": {
|
||||
"identity_source_of_truth": "Entra ID",
|
||||
"pipeline_authentication": "workload identity federation",
|
||||
"environment_tiers": [
|
||||
{
|
||||
"name": "development",
|
||||
"archetype": "corp_workload"
|
||||
},
|
||||
{
|
||||
"name": "staging",
|
||||
"archetype": "corp_workload"
|
||||
},
|
||||
{
|
||||
"name": "production",
|
||||
"archetype": "pci_regulated_workload"
|
||||
}
|
||||
]
|
||||
},
|
||||
"operations": {
|
||||
"operating_model": "central platform team brokering access for application squads",
|
||||
"support_hours": "always"
|
||||
},
|
||||
"delivery": {
|
||||
"window_weeks": 10,
|
||||
"commercial_model": "fixed_price_milestone",
|
||||
"iac_tooling": "terraform",
|
||||
"ci_cd": "GitHub Actions",
|
||||
"team_size": 3,
|
||||
"working_days_per_week": 5
|
||||
},
|
||||
"cost": {
|
||||
"currency": "GBP"
|
||||
}
|
||||
},
|
||||
"binding": [
|
||||
"estate.regions.primary",
|
||||
"estate.regions.secondary",
|
||||
"estate.regions.resiliency_tier",
|
||||
"estate.data_residency",
|
||||
"compliance.frameworks",
|
||||
"compliance.data_classification",
|
||||
"compliance.encryption.type",
|
||||
"compliance.encryption.key_rotation_days",
|
||||
"compliance.audit_logging.retention_days",
|
||||
"compliance.audit_logging.immutable_retention_policy",
|
||||
"networking.exposure",
|
||||
"networking.topology",
|
||||
"networking.hybrid_connectivity.enabled",
|
||||
"networking.hybrid_connectivity.type",
|
||||
"networking.hybrid_connectivity.bandwidth_gbps",
|
||||
"networking.egress_control.model",
|
||||
"governance.identity_source_of_truth",
|
||||
"governance.pipeline_authentication",
|
||||
"governance.environment_tiers"
|
||||
],
|
||||
"assumed": [
|
||||
{
|
||||
"field": "intent.success_criteria",
|
||||
"value": [
|
||||
"Platform foundation ready for workload ingestion within 10 weeks",
|
||||
"Compliance achieved with DORA and PCI-DSS v4.0 requirements",
|
||||
"Hybrid connectivity established via 10Gbps Dedicated Interconnect",
|
||||
"Centralised egress inspection operational",
|
||||
"Identity federation and CMEK security controls implemented"
|
||||
],
|
||||
"why": "Assumed standard success criteria based on delivery within 10 weeks and compliance with DORA and PCI-DSS v4.0."
|
||||
},
|
||||
{
|
||||
"field": "workload.availability.target",
|
||||
"value": "99.9%",
|
||||
"why": "Assumed 99.9% availability target for platform foundation infrastructure aligned with warm standby secondary region."
|
||||
},
|
||||
{
|
||||
"field": "workload.availability.recovery_time",
|
||||
"value": "4 hours",
|
||||
"why": "Assumed 4 hours recovery time objective (RTO) for platform foundation services in warm standby."
|
||||
},
|
||||
{
|
||||
"field": "workload.availability.recovery_point",
|
||||
"value": "1 hour",
|
||||
"why": "Assumed 1 hour recovery point objective (RPO) for platform foundation configuration and state."
|
||||
},
|
||||
{
|
||||
"field": "networking.egress_control.appliance_vendor",
|
||||
"value": "Palo Alto Networks VM-Series",
|
||||
"why": "Assumed Palo Alto Networks VM-Series as enterprise inspection appliance for centralized egress control."
|
||||
},
|
||||
{
|
||||
"field": "networking.cidr_allocations",
|
||||
"value": "RFC 1918 non-overlapping IP allocation (10.0.0.0/16 partitioned across environments) sized for growth",
|
||||
"why": "Assumed RFC 1918 non-overlapping IP allocation (10.0.0.0/16 partitioned across environments) sized for growth."
|
||||
},
|
||||
{
|
||||
"field": "operations.monitoring",
|
||||
"value": "Google Cloud Operations Suite with availability and error alerting",
|
||||
"why": "Assumed native Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging) with availability and error alerting."
|
||||
},
|
||||
{
|
||||
"field": "operations.change_process",
|
||||
"value": "GitHub Actions pull request workflow with automated Terraform plan validation and peer review",
|
||||
"why": "Assumed GitHub Actions pull request workflow with automated Terraform plan validation and peer approval."
|
||||
},
|
||||
{
|
||||
"field": "integrations",
|
||||
"value": [
|
||||
{
|
||||
"criticality": "critical",
|
||||
"direction": "bidirectional",
|
||||
"name": "On-premises core payment systems"
|
||||
}
|
||||
],
|
||||
"why": "Assumed integration with existing on-premises core payment systems via Dedicated Interconnect."
|
||||
},
|
||||
{
|
||||
"field": "cost.monthly_budget",
|
||||
"value": 25000,
|
||||
"why": "Assumed indicative monthly foundation infrastructure budget of 25000 GBP for 10Gbps interconnect, inspection appliances, and core services."
|
||||
},
|
||||
{
|
||||
"field": "constraints.products_required",
|
||||
"value": [],
|
||||
"why": "No mandated third-party products beyond stated tooling (GCP, Terraform, GitHub Actions, Entra ID)."
|
||||
},
|
||||
{
|
||||
"field": "constraints.products_forbidden",
|
||||
"value": [],
|
||||
"why": "No forbidden products specified."
|
||||
}
|
||||
],
|
||||
"to_decide": [],
|
||||
"to_ask": [
|
||||
{
|
||||
"field": "metadata.engagement_reference",
|
||||
"prompt": "What is metadata.engagement_reference?",
|
||||
"severity": "LOW"
|
||||
},
|
||||
{
|
||||
"field": "delivery.target_start_date",
|
||||
"prompt": "When should work start?",
|
||||
"severity": "LOW"
|
||||
},
|
||||
{
|
||||
"field": "constraints.other",
|
||||
"prompt": "What is constraints.other?",
|
||||
"severity": "LOW"
|
||||
}
|
||||
],
|
||||
"not_applicable": [
|
||||
"workload.users.count",
|
||||
"workload.users.peak_concurrent",
|
||||
"workload.users.who"
|
||||
]
|
||||
}
|
||||
1210
tests/fixtures/architecture/case-08-landing-zone/target-architecture.json
vendored
Normal file
1210
tests/fixtures/architecture/case-08-landing-zone/target-architecture.json
vendored
Normal file
File diff suppressed because it is too large
Load Diff
529
tests/fixtures/architecture/case-08-landing-zone/verdicts.json
vendored
Normal file
529
tests/fixtures/architecture/case-08-landing-zone/verdicts.json
vendored
Normal file
@@ -0,0 +1,529 @@
|
||||
{
|
||||
"dry_run": {
|
||||
"evidence": "Success! The configuration is valid.",
|
||||
"verdict": "pass"
|
||||
},
|
||||
"checks": [
|
||||
{
|
||||
"check": "K-1",
|
||||
"evidence": "display_name = \"Dev Folder\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-2",
|
||||
"evidence": "display_name = \"Staging Folder\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-3",
|
||||
"evidence": "display_name = \"Production Folder\""
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "tag_value = google_tags_tag_value.corp_workload.id",
|
||||
"check": "K-4"
|
||||
},
|
||||
{
|
||||
"check": "K-5",
|
||||
"verdict": "pass",
|
||||
"evidence": "host_project = google_compute_shared_vpc_host_project.hub.project"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "host_project = google_compute_shared_vpc_host_project.hub.project",
|
||||
"check": "K-6"
|
||||
},
|
||||
{
|
||||
"check": "K-7",
|
||||
"evidence": "host_project = google_compute_shared_vpc_host_project.hub.project",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "private_ip_google_access = true",
|
||||
"check": "K-8"
|
||||
},
|
||||
{
|
||||
"check": "K-9",
|
||||
"evidence": "private_ip_google_access = true",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-10",
|
||||
"evidence": "direction = \"INGRESS\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-11",
|
||||
"verdict": "pass",
|
||||
"evidence": "protocol = \"all\""
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "source_ranges = [\"0.0.0.0/0\"]",
|
||||
"check": "K-12"
|
||||
},
|
||||
{
|
||||
"check": "K-13",
|
||||
"evidence": "type = \"DEDICATED\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"evidence": "type = \"DEDICATED\"",
|
||||
"verdict": "pass",
|
||||
"check": "K-14"
|
||||
},
|
||||
{
|
||||
"check": "K-15",
|
||||
"verdict": "pass",
|
||||
"evidence": "bandwidth = \"BPS_10G\""
|
||||
},
|
||||
{
|
||||
"evidence": "bandwidth = \"BPS_10G\"",
|
||||
"verdict": "pass",
|
||||
"check": "K-16"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "router = google_compute_router.primary.id",
|
||||
"check": "K-17"
|
||||
},
|
||||
{
|
||||
"evidence": "network = google_compute_network.hub.id",
|
||||
"verdict": "pass",
|
||||
"check": "K-18"
|
||||
},
|
||||
{
|
||||
"check": "K-19",
|
||||
"evidence": "dest_range = \"0.0.0.0/0\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-20",
|
||||
"evidence": "next_hop_ilb = google_compute_forwarding_rule.palo_alto_ilb.id"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-21",
|
||||
"evidence": "issuer_uri = \"https://token.actions.githubusercontent.com\""
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-22",
|
||||
"evidence": "member = \"principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.wif_pool.name}/attribute.repository/my-org/my-repo\""
|
||||
},
|
||||
{
|
||||
"evidence": "idp_metadata_xml = \"<EntityDescriptor></EntityDescriptor>\"",
|
||||
"verdict": "pass",
|
||||
"check": "K-23"
|
||||
},
|
||||
{
|
||||
"evidence": "purpose = \"ENCRYPT_DECRYPT\"",
|
||||
"verdict": "pass",
|
||||
"check": "K-24"
|
||||
},
|
||||
{
|
||||
"check": "K-25",
|
||||
"evidence": "purpose = \"ENCRYPT_DECRYPT\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-26",
|
||||
"verdict": "pass",
|
||||
"evidence": "rotation_period = \"7776000s\""
|
||||
},
|
||||
{
|
||||
"check": "K-27",
|
||||
"evidence": "rotation_period = \"7776000s\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"evidence": "destination = \"storage.googleapis.com/${google_storage_bucket.audit_logs.name}\"",
|
||||
"verdict": "pass",
|
||||
"check": "K-28"
|
||||
},
|
||||
{
|
||||
"check": "K-29",
|
||||
"verdict": "pass",
|
||||
"evidence": "retention_period = 31536000"
|
||||
},
|
||||
{
|
||||
"check": "K-30",
|
||||
"verdict": "pass",
|
||||
"evidence": "is_locked = true"
|
||||
},
|
||||
{
|
||||
"evidence": "restricted_services = [\"compute.googleapis.com\"]",
|
||||
"verdict": "pass",
|
||||
"check": "K-31"
|
||||
},
|
||||
{
|
||||
"check": "K-32",
|
||||
"evidence": "role = \"roles/cloudkms.cryptoKeyEncrypterDecrypter\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-33",
|
||||
"evidence": "region = \"europe-west2\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-34",
|
||||
"verdict": "pass",
|
||||
"evidence": "region = \"europe-west2\""
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-35",
|
||||
"evidence": "region = \"europe-west1\""
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "constraint = \"gcp.resourceLocations\"",
|
||||
"check": "K-36"
|
||||
},
|
||||
{
|
||||
"check": "K-37",
|
||||
"verdict": "pass",
|
||||
"evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]"
|
||||
},
|
||||
{
|
||||
"check": "K-38",
|
||||
"evidence": "project = var.project_id",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "distribution_policy_zones = [\"europe-west2-a\", \"europe-west2-b\", \"europe-west2-c\"]",
|
||||
"check": "K-39"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-40",
|
||||
"evidence": "notification_channels = [google_monitoring_notification_channel.email.name]"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "condition_threshold is defined",
|
||||
"check": "K-41"
|
||||
},
|
||||
{
|
||||
"evidence": "currency_code = \"GBP\"",
|
||||
"check": "K-42",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "threshold_percent includes 0.5, 0.8, 0.9, 1.0",
|
||||
"check": "K-43"
|
||||
},
|
||||
{
|
||||
"check": "K-44",
|
||||
"verdict": "pass",
|
||||
"evidence": "units = \"25000\""
|
||||
},
|
||||
{
|
||||
"evidence": "advertise_mode = \"CUSTOM\"",
|
||||
"verdict": "pass",
|
||||
"check": "K-45"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "advertise_mode = \"CUSTOM\"",
|
||||
"check": "K-46"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "bandwidth = \"BPS_10G\"",
|
||||
"check": "K-47"
|
||||
},
|
||||
{
|
||||
"evidence": "bandwidth = \"BPS_10G\"",
|
||||
"verdict": "pass",
|
||||
"check": "K-48"
|
||||
},
|
||||
{
|
||||
"check": "K-49",
|
||||
"verdict": "pass",
|
||||
"evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]"
|
||||
},
|
||||
{
|
||||
"evidence": "resources = [\"projects/${var.prod_project_id}\", \"projects/${var.project_id}\"]",
|
||||
"check": "K-50",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-51",
|
||||
"evidence": "ingress_policies and egress_policies are configured"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "constraint = \"compute.skipDefaultNetworkCreation\"",
|
||||
"check": "K-52"
|
||||
},
|
||||
{
|
||||
"evidence": "enforced = true",
|
||||
"verdict": "pass",
|
||||
"check": "K-53"
|
||||
},
|
||||
{
|
||||
"check": "K-54",
|
||||
"evidence": "constraint = \"compute.vmExternalIpAccess\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "all = true",
|
||||
"check": "K-55"
|
||||
},
|
||||
{
|
||||
"evidence": "constraint = \"iam.disableServiceAccountKeyCreation\"",
|
||||
"check": "K-56",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-57",
|
||||
"evidence": "enforced = true",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-58",
|
||||
"evidence": "constraint = \"gcp.resourceLocations\""
|
||||
},
|
||||
{
|
||||
"evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]",
|
||||
"verdict": "pass",
|
||||
"check": "K-59"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-60",
|
||||
"evidence": "enable_confidential_compute = true"
|
||||
},
|
||||
{
|
||||
"check": "K-61",
|
||||
"evidence": "name = \"CREDIT_CARD_NUMBER\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-62",
|
||||
"evidence": "event_data_content_type = \"application/json\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"evidence": "attribute = \"type\", value = \"google.cloud.securitycenter.finding.v1.saved\"",
|
||||
"check": "K-63",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"evidence": "event_type = \"google.cloud.securitycenter.finding.v1.saved\"",
|
||||
"verdict": "pass",
|
||||
"check": "K-64"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "interconnect = var.interconnect_1_url",
|
||||
"check": "K-65"
|
||||
},
|
||||
{
|
||||
"evidence": "interconnect = var.interconnect_2_url",
|
||||
"verdict": "pass",
|
||||
"check": "K-66"
|
||||
},
|
||||
{
|
||||
"check": "K-67",
|
||||
"evidence": "router = google_compute_router.primary.id",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-68",
|
||||
"evidence": "session_initialization_mode = \"ACTIVE\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-69",
|
||||
"evidence": "distribution_policy_zones = [\"europe-west2-a\", \"europe-west2-b\", \"europe-west2-c\"]"
|
||||
},
|
||||
{
|
||||
"evidence": "load_balancing_scheme = \"INTERNAL\"",
|
||||
"check": "K-70",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"evidence": "health_checks = [google_compute_health_check.palo_alto_ilb_hc.id]",
|
||||
"verdict": "pass",
|
||||
"check": "K-71"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-72",
|
||||
"evidence": "location = \"europe-west2\""
|
||||
},
|
||||
{
|
||||
"evidence": "location = \"europe-west1\"",
|
||||
"check": "K-73",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"evidence": "role = \"roles/cloudkms.cryptoKeyDecrypter\"",
|
||||
"verdict": "pass",
|
||||
"check": "K-74"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "data_locations = [\"EUROPE-WEST1\", \"EUROPE-WEST2\"]",
|
||||
"check": "K-75"
|
||||
},
|
||||
{
|
||||
"check": "K-76",
|
||||
"verdict": "pass",
|
||||
"evidence": "name = \"${var.project_id}-tfstate-primary\""
|
||||
},
|
||||
{
|
||||
"evidence": "name = \"${var.project_id}-tfstate-secondary\"",
|
||||
"check": "K-77",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"evidence": "enabled = true",
|
||||
"check": "K-78",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-79",
|
||||
"evidence": "enabled = true",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"evidence": "goal = 0.999",
|
||||
"verdict": "pass",
|
||||
"check": "K-80"
|
||||
},
|
||||
{
|
||||
"evidence": "filter = \"select_slo_burn_rate(\\\"${google_monitoring_slo.platform_slo.name}\\\", 3600s)\"",
|
||||
"verdict": "pass",
|
||||
"check": "K-81"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-82",
|
||||
"evidence": "service = \"recommender.googleapis.com\""
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "age = 30",
|
||||
"check": "K-83"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-84",
|
||||
"evidence": "type = \"SetStorageClass\", storage_class = \"NEARLINE\""
|
||||
},
|
||||
{
|
||||
"check": "K-85",
|
||||
"verdict": "pass",
|
||||
"evidence": "age = 90"
|
||||
},
|
||||
{
|
||||
"check": "K-86",
|
||||
"evidence": "type = \"SetStorageClass\", storage_class = \"ARCHIVE\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-87",
|
||||
"evidence": "routing_mode = \"REGIONAL\""
|
||||
},
|
||||
{
|
||||
"evidence": "min_replicas = 1",
|
||||
"check": "K-88",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"evidence": "short_name = \"environment\"",
|
||||
"check": "K-89",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-90",
|
||||
"evidence": "destination = \"bigquery.googleapis.com/...\""
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-91",
|
||||
"evidence": "tag_value = google_tags_tag_value.pci_regulated.id"
|
||||
},
|
||||
{
|
||||
"check": "K-92",
|
||||
"evidence": "check_interval_sec = 5",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-93",
|
||||
"evidence": "disabled = false"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"check": "K-94",
|
||||
"evidence": "disabled = false"
|
||||
},
|
||||
{
|
||||
"evidence": "display_name = \"Platform Custom Service\"",
|
||||
"check": "K-95",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-96",
|
||||
"verdict": "pass",
|
||||
"evidence": "base_instance_name = \"palo-alto-secondary\""
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "session_initialization_mode = \"ACTIVE\"",
|
||||
"check": "K-97"
|
||||
},
|
||||
{
|
||||
"evidence": "min_receive_interval = 100",
|
||||
"check": "K-99",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"check": "K-100",
|
||||
"verdict": "pass",
|
||||
"evidence": "min_receive_interval = 100"
|
||||
},
|
||||
{
|
||||
"check": "K-101",
|
||||
"evidence": "role = \"roles/cloudkms.cryptoKeyDecrypter\"",
|
||||
"verdict": "pass"
|
||||
},
|
||||
{
|
||||
"evidence": "tag_value = google_tags_tag_value.corp_workload.id",
|
||||
"verdict": "pass",
|
||||
"check": "K-102"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "router = google_compute_router.primary.id",
|
||||
"check": "K-103"
|
||||
},
|
||||
{
|
||||
"verdict": "pass",
|
||||
"evidence": "min_transmit_interval = 100",
|
||||
"check": "K-104"
|
||||
},
|
||||
{
|
||||
"check": "K-105",
|
||||
"evidence": "min_transmit_interval = 100",
|
||||
"verdict": "pass"
|
||||
}
|
||||
],
|
||||
"proposed_additions": []
|
||||
}
|
||||
332
tests/fixtures/architecture/case-08-landing-zone/work-breakdown.json
vendored
Normal file
332
tests/fixtures/architecture/case-08-landing-zone/work-breakdown.json
vendored
Normal file
@@ -0,0 +1,332 @@
|
||||
{
|
||||
"work_packages": [
|
||||
{
|
||||
"id": "WP-0",
|
||||
"name": "Platform Foundation",
|
||||
"parent": null,
|
||||
"elements": [],
|
||||
"scope": "The complete enterprise-grade landing zone.",
|
||||
"produces": [
|
||||
"an operational landing zone"
|
||||
],
|
||||
"skills": [
|
||||
"cloud architecture"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "WP-1",
|
||||
"name": "Automated Infrastructure Orchestration",
|
||||
"parent": "WP-0",
|
||||
"elements": [
|
||||
"E-42",
|
||||
"E-43"
|
||||
],
|
||||
"scope": "CI/CD orchestration, state management, and API enablement.",
|
||||
"produces": [
|
||||
"Terraform state buckets for primary and secondary regions"
|
||||
],
|
||||
"skills": [
|
||||
"cloud engineering"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "WP-2",
|
||||
"name": "Federated Identity and Role Governance",
|
||||
"parent": "WP-0",
|
||||
"elements": [
|
||||
"E-23",
|
||||
"E-24",
|
||||
"E-25",
|
||||
"E-26",
|
||||
"E-27",
|
||||
"E-45"
|
||||
],
|
||||
"scope": "Authentication for delivery pipelines and workforce SSO.",
|
||||
"produces": [
|
||||
"Workload Identity Federation for GitHub Actions and Entra ID SSO"
|
||||
],
|
||||
"skills": [
|
||||
"identity and access management"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "WP-3",
|
||||
"name": "Resource Hierarchy and Environment Boundary",
|
||||
"parent": "WP-0",
|
||||
"elements": [
|
||||
"E-1",
|
||||
"E-2",
|
||||
"E-3",
|
||||
"E-4",
|
||||
"E-5",
|
||||
"E-6",
|
||||
"E-7",
|
||||
"E-8",
|
||||
"E-9",
|
||||
"E-10",
|
||||
"E-34",
|
||||
"E-35",
|
||||
"E-36",
|
||||
"E-37",
|
||||
"E-38",
|
||||
"E-50",
|
||||
"E-51",
|
||||
"E-55",
|
||||
"E-60"
|
||||
],
|
||||
"scope": "Google Cloud resource hierarchy, shared VPC projects, networking, and organization policies.",
|
||||
"produces": [
|
||||
"Folders, projects, networking, and organization policies ensuring isolation"
|
||||
],
|
||||
"skills": [
|
||||
"cloud engineering",
|
||||
"network engineering"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "WP-4",
|
||||
"name": "Cryptographic Key Lifecycle",
|
||||
"parent": "WP-0",
|
||||
"elements": [
|
||||
"E-28",
|
||||
"E-29",
|
||||
"E-30",
|
||||
"E-31",
|
||||
"E-49",
|
||||
"E-58",
|
||||
"E-59"
|
||||
],
|
||||
"scope": "Customer-managed encryption keys (CMEK) and cryptographic isolation.",
|
||||
"produces": [
|
||||
"Primary and secondary KMS key rings and crypto keys with IAM bindings"
|
||||
],
|
||||
"skills": [
|
||||
"security engineering"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "WP-5",
|
||||
"name": "Audit, Compliance, and Financial Governance",
|
||||
"parent": "WP-0",
|
||||
"elements": [
|
||||
"E-21",
|
||||
"E-22",
|
||||
"E-33",
|
||||
"E-39",
|
||||
"E-46",
|
||||
"E-54"
|
||||
],
|
||||
"scope": "Immutable compliance logs, DLP templates, and budget enforcement.",
|
||||
"produces": [
|
||||
"Immutable audit log storage, billing budgets, and DLP inspection templates"
|
||||
],
|
||||
"skills": [
|
||||
"cloud engineering",
|
||||
"finops"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "WP-6",
|
||||
"name": "Centralised Outbound Traffic Inspection",
|
||||
"parent": "WP-0",
|
||||
"elements": [
|
||||
"E-15",
|
||||
"E-16",
|
||||
"E-17",
|
||||
"E-18",
|
||||
"E-19",
|
||||
"E-20",
|
||||
"E-47",
|
||||
"E-48",
|
||||
"E-53"
|
||||
],
|
||||
"scope": "Egress inspection and threat prevention using Palo Alto VM-Series.",
|
||||
"produces": [
|
||||
"Palo Alto VM-Series firewalls with internal load balancing and egress routing"
|
||||
],
|
||||
"skills": [
|
||||
"network security engineering"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "WP-7",
|
||||
"name": "Dedicated Private Hybrid Connectivity",
|
||||
"parent": "WP-0",
|
||||
"elements": [
|
||||
"E-11",
|
||||
"E-12",
|
||||
"E-13",
|
||||
"E-14",
|
||||
"E-56",
|
||||
"E-57"
|
||||
],
|
||||
"scope": "Dedicated Interconnect termination and dynamic BGP routing.",
|
||||
"produces": [
|
||||
"Primary and secondary Cloud Routers with Dedicated Interconnect VLAN attachments"
|
||||
],
|
||||
"skills": [
|
||||
"network engineering"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "WP-8",
|
||||
"name": "Platform Telemetry and Operational Alerting",
|
||||
"parent": "WP-0",
|
||||
"elements": [
|
||||
"E-32",
|
||||
"E-40",
|
||||
"E-41",
|
||||
"E-44",
|
||||
"E-52"
|
||||
],
|
||||
"scope": "SLO tracking, multi-burn-rate alerting, and SCC automated remediation.",
|
||||
"produces": [
|
||||
"SLOs, alerting policies, and SCC automated remediation functions"
|
||||
],
|
||||
"skills": [
|
||||
"site reliability engineering"
|
||||
]
|
||||
}
|
||||
],
|
||||
"activities": [
|
||||
{
|
||||
"id": "A-1",
|
||||
"work_package": "WP-1",
|
||||
"name": "Enable Google Cloud APIs",
|
||||
"days": 0.5,
|
||||
"after": []
|
||||
},
|
||||
{
|
||||
"id": "A-2",
|
||||
"work_package": "WP-1",
|
||||
"name": "Setup CI/CD Pipeline and State Buckets",
|
||||
"days": 1,
|
||||
"after": [
|
||||
"A-1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-3",
|
||||
"work_package": "WP-2",
|
||||
"name": "Configure Workload Identity and Entra ID SSO",
|
||||
"days": 1.5,
|
||||
"after": [
|
||||
"A-2"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-4",
|
||||
"work_package": "WP-3",
|
||||
"name": "Provision Resource Hierarchy and Enforce Org Policies",
|
||||
"days": 2.5,
|
||||
"after": [
|
||||
"A-3"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-5",
|
||||
"work_package": "WP-3",
|
||||
"name": "Provision Hub and Spoke VPC Networks",
|
||||
"days": 2,
|
||||
"after": [
|
||||
"A-4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-6",
|
||||
"work_package": "WP-7",
|
||||
"name": "Order and provision the Interconnect carrier circuits",
|
||||
"days": 30,
|
||||
"owner": "customer",
|
||||
"assumption": "The customer manages the carrier's lead time and physical cross-connects.",
|
||||
"after": []
|
||||
},
|
||||
{
|
||||
"id": "A-7",
|
||||
"work_package": "WP-4",
|
||||
"name": "Configure Cloud KMS and Key Rings",
|
||||
"days": 1.5,
|
||||
"after": [
|
||||
"A-4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-8",
|
||||
"work_package": "WP-5",
|
||||
"name": "Set up Audit Logging, DLP, and Billing Budgets",
|
||||
"days": 2,
|
||||
"after": [
|
||||
"A-4"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-9",
|
||||
"work_package": "WP-6",
|
||||
"name": "Deploy Palo Alto Egress Firewall Appliances",
|
||||
"days": 5,
|
||||
"after": [
|
||||
"A-5"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-10",
|
||||
"work_package": "WP-7",
|
||||
"name": "Configure Cloud Routers and Terminate BGP over Interconnect",
|
||||
"days": 3,
|
||||
"after": [
|
||||
"A-5",
|
||||
"A-6"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-11",
|
||||
"work_package": "WP-3",
|
||||
"name": "Implement VPC Service Controls",
|
||||
"days": 2,
|
||||
"after": [
|
||||
"A-5",
|
||||
"A-7",
|
||||
"A-8"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-12",
|
||||
"work_package": "WP-8",
|
||||
"name": "Deploy Platform Telemetry, SLOs, and SCC Auto-remediation",
|
||||
"days": 2.5,
|
||||
"after": [
|
||||
"A-8",
|
||||
"A-11"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-13",
|
||||
"work_package": "WP-1",
|
||||
"name": "Run validation in development environment",
|
||||
"days": 1,
|
||||
"after": [
|
||||
"A-9",
|
||||
"A-10",
|
||||
"A-12"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-14",
|
||||
"work_package": "WP-1",
|
||||
"name": "Run validation in staging environment",
|
||||
"days": 1,
|
||||
"after": [
|
||||
"A-13"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "A-15",
|
||||
"work_package": "WP-1",
|
||||
"name": "Run validation in production environment",
|
||||
"days": 1,
|
||||
"after": [
|
||||
"A-14"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
294
tests/support.py
Normal file
294
tests/support.py
Normal file
@@ -0,0 +1,294 @@
|
||||
"""One small, coherent operating model, for tests: a fictional landing zone run as a
|
||||
shared service. Every name in it is invented."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from execution_engine.providers import load_provider
|
||||
from execution_engine.scope import read_scope
|
||||
|
||||
from app.operating_model import catalogue as capability_catalogue
|
||||
from app.operating_model import staffing
|
||||
from app.operating_model.names import (
|
||||
CAPABILITIES_FILE,
|
||||
CAPABILITY_MODEL,
|
||||
FINDINGS_FILE,
|
||||
GRAPH_FILE,
|
||||
OPERATING_SCOPE,
|
||||
OPERATIONAL_REQUIREMENTS,
|
||||
REQUIREMENTS_FILE,
|
||||
SCOPE_DATA_FILE,
|
||||
TARGET_OPERATING_MODEL,
|
||||
VERDICTS_FILE,
|
||||
)
|
||||
from app.operating_model.workflow import ROOT, build
|
||||
|
||||
BUNDLE = load_provider("gcp", ROOT)
|
||||
CATALOGUE = capability_catalogue.load(ROOT / "skills" / capability_catalogue.SKILL)
|
||||
CARD = staffing.load(ROOT / "skills" / staffing.SKILL)
|
||||
WORKFLOW = build(ROOT)
|
||||
|
||||
SPEC = """\
|
||||
metadata:
|
||||
customer_case: northwind-landing-zone
|
||||
engagement:
|
||||
kind: platform_foundation
|
||||
intent:
|
||||
statement: |
|
||||
We want the landing zone run as a shared service from go-live, with our own
|
||||
platform team keeping the final say.
|
||||
business_outcomes:
|
||||
- {id: BO-1, outcome: Outages of the platform are short., target: Restored within 4 hours.}
|
||||
- {id: BO-2, outcome: Regulators always get the evidence they ask for.}
|
||||
requirements:
|
||||
- {id: REQ-01, statement: Our platform team approves every change to the network.}
|
||||
solution:
|
||||
cloud_provider: gcp
|
||||
items:
|
||||
- {id: S-1, name: Hub network, kind: network, product: Shared VPC, source_ref: E-1}
|
||||
- {id: S-2, name: Audit log store, kind: storage, product: Cloud Storage, source_ref: E-2}
|
||||
- {id: S-3, name: Deployment pipeline, kind: pipeline, product: Cloud Build, source_ref: E-3}
|
||||
conditions:
|
||||
- {id: K-1, "on": S-1, condition: Flow logs are retained for 400 days., source_ref: CHK-12}
|
||||
- {id: K-2, "on": S-2, condition: Audit logs cannot be deleted for seven years., source_ref: CHK-20}
|
||||
operations:
|
||||
support_hours: 24x7
|
||||
availability_hours: 24x7
|
||||
change_process: decide
|
||||
sourcing:
|
||||
model: shared
|
||||
delivery_countries: [GB, PL, IN]
|
||||
service_levels:
|
||||
priority_1_response_minutes: 15
|
||||
recovery_time_hours: 4
|
||||
compliance:
|
||||
frameworks: [DORA]
|
||||
commercial:
|
||||
currency: GBP
|
||||
monthly_run_budget: 120000
|
||||
run:
|
||||
pre_approved: true
|
||||
"""
|
||||
|
||||
|
||||
def scope_data(text: str = SPEC) -> dict[str, Any]:
|
||||
"""What every Specialist is given of an operating spec."""
|
||||
scope = read_scope(
|
||||
text, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
|
||||
)
|
||||
return scope.for_context()
|
||||
|
||||
|
||||
REQUIREMENTS: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": "OR-1",
|
||||
"kind": "service_level",
|
||||
"statement": "A fault on any Item is acknowledged within 15 minutes, at any hour.",
|
||||
"source": [
|
||||
"service_levels.priority_1_response_minutes",
|
||||
"operations.support_hours",
|
||||
],
|
||||
},
|
||||
{
|
||||
"id": "OR-2",
|
||||
"kind": "process",
|
||||
"statement": "The customer's platform team approves every change to the network.",
|
||||
"source": ["REQ-01", "sourcing.model"],
|
||||
},
|
||||
{
|
||||
"id": "OR-3",
|
||||
"kind": "compliance",
|
||||
"statement": "Logs are kept for as long as each Condition says, and cannot be removed sooner.",
|
||||
"source": ["K-1", "K-2"],
|
||||
},
|
||||
{
|
||||
"id": "OR-4",
|
||||
"kind": "service_level",
|
||||
"statement": "Service is restored within 4 hours of a major failure.",
|
||||
"source": ["service_levels.recovery_time_hours"],
|
||||
},
|
||||
{
|
||||
"id": "OR-5",
|
||||
"kind": "organisation",
|
||||
"statement": "The solution is operated only from the countries the customer allows.",
|
||||
"source": ["sourcing.delivery_countries"],
|
||||
},
|
||||
]
|
||||
|
||||
CAPABILITIES: list[dict[str, Any]] = [
|
||||
{
|
||||
"id": "C-1",
|
||||
"catalogue": "incident-management",
|
||||
"serves": ["OR-1", "OR-4"],
|
||||
"items": ["S-1", "S-2", "S-3"],
|
||||
"hours": "24x7",
|
||||
"effort_hours_per_month": 60,
|
||||
"basis": "Two serious incidents a month and daily triage.",
|
||||
},
|
||||
{
|
||||
"id": "C-2",
|
||||
"catalogue": "change-enablement",
|
||||
"serves": ["OR-2"],
|
||||
"items": ["S-1", "S-2", "S-3"],
|
||||
"hours": "business",
|
||||
"effort_hours_per_month": 40,
|
||||
"basis": "About twenty changes a month at two hours each.",
|
||||
},
|
||||
{
|
||||
"id": "C-3",
|
||||
"catalogue": "regulatory-evidence",
|
||||
"serves": ["OR-3", "OR-5"],
|
||||
"items": ["S-1", "S-2"],
|
||||
"hours": "business",
|
||||
"effort_hours_per_month": 16,
|
||||
"basis": "A monthly review of retention settings and evidence.",
|
||||
},
|
||||
]
|
||||
|
||||
GRAPH: dict[str, list[dict[str, Any]]] = {
|
||||
"parties": [
|
||||
{"id": "P-1", "name": "Northwind platform team", "kind": "customer"},
|
||||
{"id": "P-2", "name": "Managed service provider", "kind": "provider"},
|
||||
],
|
||||
"roles": [
|
||||
{
|
||||
"id": "R-1",
|
||||
"party": "P-1",
|
||||
"name": "Platform owner",
|
||||
"country": "GB",
|
||||
"level": "lead",
|
||||
"hours": "business",
|
||||
},
|
||||
{
|
||||
"id": "R-2",
|
||||
"party": "P-2",
|
||||
"name": "Platform SRE",
|
||||
"country": "PL",
|
||||
"level": "senior",
|
||||
"hours": "24x7",
|
||||
},
|
||||
{
|
||||
"id": "R-3",
|
||||
"party": "P-2",
|
||||
"name": "Service manager",
|
||||
"country": "GB",
|
||||
"level": "lead",
|
||||
"hours": "business",
|
||||
},
|
||||
],
|
||||
"responsibilities": [
|
||||
{"id": "RS-1", "capability": "C-1", "role": "R-3", "kind": "accountable"},
|
||||
{"id": "RS-2", "capability": "C-1", "role": "R-2", "kind": "responsible"},
|
||||
{"id": "RS-3", "capability": "C-1", "role": "R-1", "kind": "consulted"},
|
||||
{"id": "RS-4", "capability": "C-2", "role": "R-1", "kind": "accountable"},
|
||||
{"id": "RS-5", "capability": "C-2", "role": "R-2", "kind": "responsible"},
|
||||
{"id": "RS-6", "capability": "C-3", "role": "R-3", "kind": "accountable"},
|
||||
{"id": "RS-7", "capability": "C-3", "role": "R-3", "kind": "responsible"},
|
||||
{"id": "RS-8", "capability": "C-3", "role": "R-1", "kind": "consulted"},
|
||||
],
|
||||
"measures": [
|
||||
{
|
||||
"id": "M-1",
|
||||
"name": "Time to restore service",
|
||||
"layer": "outcome",
|
||||
"outcome": "BO-1",
|
||||
"measured_by": "C-1",
|
||||
"framework": "dora-metrics",
|
||||
"target": "Under 4 hours",
|
||||
},
|
||||
{
|
||||
"id": "M-2",
|
||||
"name": "Evidence requests met on time",
|
||||
"layer": "service_level",
|
||||
"outcome": "BO-2",
|
||||
"measured_by": "C-3",
|
||||
},
|
||||
],
|
||||
"checks": [
|
||||
{
|
||||
"id": "OC-1",
|
||||
"on": "C-1",
|
||||
"source": "OR-1",
|
||||
"condition": "A fault on the hub network is acknowledged within 15 minutes at any hour.",
|
||||
},
|
||||
{
|
||||
"id": "OC-2",
|
||||
"on": "C-3",
|
||||
"source": "K-1",
|
||||
"condition": "Flow log retention is reviewed monthly and is 400 days.",
|
||||
},
|
||||
{
|
||||
"id": "OC-3",
|
||||
"on": "C-3",
|
||||
"source": "K-2",
|
||||
"condition": "The audit log store's retention lock is reviewed monthly.",
|
||||
},
|
||||
{
|
||||
"id": "OC-4",
|
||||
"on": "RS-4",
|
||||
"source": "OR-2",
|
||||
"condition": "No change to the network is applied without the platform owner's approval.",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
VERDICTS: dict[str, Any] = {
|
||||
"checks": [
|
||||
{
|
||||
"check": check["id"],
|
||||
"verdict": "pass",
|
||||
"evidence": "The model provides for it.",
|
||||
}
|
||||
for check in GRAPH["checks"]
|
||||
],
|
||||
"proposed_additions": [],
|
||||
}
|
||||
|
||||
|
||||
def graph(**changes: Any) -> dict[str, list[dict[str, Any]]]:
|
||||
"""A copy of the graph to change."""
|
||||
copied = copy.deepcopy(GRAPH)
|
||||
copied.update(changes)
|
||||
return copied
|
||||
|
||||
|
||||
def upstream(
|
||||
*,
|
||||
spec: str = SPEC,
|
||||
requirements: list[dict[str, Any]] | None = None,
|
||||
capabilities: list[dict[str, Any]] | None = None,
|
||||
model: dict[str, Any] | None = None,
|
||||
findings: list[dict[str, Any]] | None = None,
|
||||
) -> dict[str, dict[str, Any]]:
|
||||
"""The Deliverables a submission builds on, as the engine holds them."""
|
||||
|
||||
def deliverable(name: str, data: Any) -> dict[str, Any]:
|
||||
return {"content": "# Prose\n", "attachments": {name: json.dumps(data)}}
|
||||
|
||||
built = {
|
||||
OPERATING_SCOPE: deliverable(SCOPE_DATA_FILE, scope_data(spec)),
|
||||
OPERATIONAL_REQUIREMENTS: deliverable(
|
||||
REQUIREMENTS_FILE, REQUIREMENTS if requirements is None else requirements
|
||||
),
|
||||
CAPABILITY_MODEL: deliverable(
|
||||
CAPABILITIES_FILE, CAPABILITIES if capabilities is None else capabilities
|
||||
),
|
||||
TARGET_OPERATING_MODEL: deliverable(
|
||||
GRAPH_FILE, GRAPH if model is None else model
|
||||
),
|
||||
}
|
||||
if findings is not None:
|
||||
built[TARGET_OPERATING_MODEL]["attachments"][FINDINGS_FILE] = json.dumps(
|
||||
findings
|
||||
)
|
||||
return built
|
||||
|
||||
|
||||
def as_file(name: str, data: Any) -> dict[str, str]:
|
||||
return {name: json.dumps(data)}
|
||||
|
||||
|
||||
__all__ = ["VERDICTS_FILE"]
|
||||
107
tests/test_catalogue.py
Normal file
107
tests/test_catalogue.py
Normal file
@@ -0,0 +1,107 @@
|
||||
"""The capability catalogue and its register of frameworks."""
|
||||
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from app.operating_model import catalogue as capability_catalogue
|
||||
from app.operating_model.catalogue import CatalogueError
|
||||
from app.operating_model.workflow import ROOT
|
||||
from tests.support import CATALOGUE, scope_data
|
||||
|
||||
SKILL = ROOT / "skills" / capability_catalogue.SKILL
|
||||
|
||||
|
||||
def test_every_entry_cites_a_framework_that_is_in_the_register():
|
||||
for entry in CATALOGUE.entries.values():
|
||||
assert entry.sources, entry.id
|
||||
assert all(s["framework"] in CATALOGUE.frameworks for s in entry.sources)
|
||||
assert entry.purpose and entry.typical_checks, entry.id
|
||||
|
||||
|
||||
def test_the_register_says_its_references_have_not_been_checked():
|
||||
assert CATALOGUE.verified is False
|
||||
|
||||
|
||||
def test_the_two_things_called_dora_are_kept_apart():
|
||||
names = {key: value["name"] for key, value in CATALOGUE.frameworks.items()}
|
||||
|
||||
assert "Regulation (EU) 2022/2554" in names["eu-dora"]
|
||||
assert "DevOps Research and Assessment" in names["dora-metrics"]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def skill(tmp_path):
|
||||
shutil.copytree(SKILL, tmp_path / "skill")
|
||||
return tmp_path / "skill"
|
||||
|
||||
|
||||
def edit(skill, name, change):
|
||||
path = skill / "assets" / name
|
||||
data = yaml.safe_load(path.read_text())
|
||||
change(data)
|
||||
path.write_text(yaml.safe_dump(data))
|
||||
|
||||
|
||||
def test_an_entry_that_cites_nothing_is_refused(skill):
|
||||
edit(skill, "catalogue.yaml", lambda d: d["capabilities"][0].update(sources=[]))
|
||||
|
||||
with pytest.raises(CatalogueError, match="incident-management cites no source"):
|
||||
capability_catalogue.load(skill)
|
||||
|
||||
|
||||
def test_an_entry_that_cites_an_unregistered_framework_is_refused(skill):
|
||||
edit(
|
||||
skill,
|
||||
"catalogue.yaml",
|
||||
lambda d: d["capabilities"][0]["sources"].append(
|
||||
{"framework": "folklore", "reference": "What we always did"}
|
||||
),
|
||||
)
|
||||
|
||||
with pytest.raises(
|
||||
CatalogueError, match="cites 'folklore', which is not in the register"
|
||||
):
|
||||
capability_catalogue.load(skill)
|
||||
|
||||
|
||||
def test_a_framework_registered_without_its_licence_is_refused(skill):
|
||||
edit(skill, "sources.yaml", lambda d: d["frameworks"]["itil-4"].pop("licence"))
|
||||
|
||||
with pytest.raises(CatalogueError, match="itil-4 needs licence"):
|
||||
capability_catalogue.load(skill)
|
||||
|
||||
|
||||
def test_entries_are_proposed_from_what_the_spec_says():
|
||||
proposed = CATALOGUE.applicable(scope_data())
|
||||
|
||||
# Always; for an Item of a kind; for a framework named; for a sourcing model.
|
||||
assert "incident-management" in proposed
|
||||
assert "backup-and-recovery" in proposed # the audit log store is storage
|
||||
assert "regulatory-evidence" in proposed # DORA
|
||||
assert "supplier-management" in proposed # shared
|
||||
|
||||
|
||||
def test_an_entry_is_not_proposed_when_nothing_calls_for_it():
|
||||
bare = {
|
||||
"stated": {
|
||||
"solution": {"items": [{"id": "S-1", "name": "Hub", "kind": "network"}]}
|
||||
}
|
||||
}
|
||||
|
||||
proposed = CATALOGUE.applicable(bare)
|
||||
|
||||
assert "incident-management" in proposed
|
||||
assert not {
|
||||
"backup-and-recovery",
|
||||
"regulatory-evidence",
|
||||
"supplier-management",
|
||||
} & set(proposed)
|
||||
|
||||
|
||||
def test_sources_are_cited_by_framework_and_part_never_quoted():
|
||||
lines = capability_catalogue.cited(CATALOGUE, ["incident-management"])
|
||||
|
||||
assert "ITIL 4: Incident management practice" in lines
|
||||
assert any(line.startswith("Regulation (EU) 2022/2554") for line in lines)
|
||||
142
tests/test_chain.py
Normal file
142
tests/test_chain.py
Normal file
@@ -0,0 +1,142 @@
|
||||
"""The chain's data files: what links each to the one before, and what is refused."""
|
||||
|
||||
import copy
|
||||
|
||||
import pytest
|
||||
|
||||
from app.operating_model import chain
|
||||
from app.operating_model.names import CAPABILITIES_FILE, REQUIREMENTS_FILE
|
||||
from tests.support import CAPABILITIES, CATALOGUE, REQUIREMENTS, as_file, upstream
|
||||
|
||||
check_capabilities = chain.check_capabilities(CATALOGUE)
|
||||
|
||||
|
||||
def requirements(*changes) -> list[dict]:
|
||||
entries = copy.deepcopy(REQUIREMENTS)
|
||||
for change in changes:
|
||||
change(entries)
|
||||
return entries
|
||||
|
||||
|
||||
def capabilities(*changes) -> list[dict]:
|
||||
entries = copy.deepcopy(CAPABILITIES)
|
||||
for change in changes:
|
||||
change(entries)
|
||||
return entries
|
||||
|
||||
|
||||
def test_requirements_traced_to_what_the_customer_stated_are_accepted():
|
||||
assert (
|
||||
chain.check_requirements(as_file(REQUIREMENTS_FILE, REQUIREMENTS), upstream())
|
||||
== []
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "problem"),
|
||||
[
|
||||
(lambda e: e[0].update(source=[]), "OR-1 has no source"),
|
||||
(
|
||||
lambda e: e[0].update(source=["OR-99"]),
|
||||
"source OR-99 is not a stated requirement",
|
||||
),
|
||||
(
|
||||
lambda e: e[0].update(source=["commercial.exchange_rate"]),
|
||||
"source commercial.exchange_rate is not a stated requirement",
|
||||
),
|
||||
(lambda e: e[0].update(kind="wish"), "OR-1: kind must be one of"),
|
||||
(lambda e: e[0].update(statement=" "), "OR-1 needs a statement"),
|
||||
(lambda e: e.append(dict(e[0])), "id OR-1 is used more than once"),
|
||||
(
|
||||
lambda e: e[1].update(source=["sourcing.model"]),
|
||||
"the customer's requirement REQ-01 is carried by no Operational requirement",
|
||||
),
|
||||
(
|
||||
lambda e: e.pop(4),
|
||||
"the customer stated sourcing.delivery_countries, and no Operational requirement",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_requirement_that_breaks_the_chain_is_refused(change, problem):
|
||||
faults = chain.check_requirements(
|
||||
as_file(REQUIREMENTS_FILE, requirements(change)), upstream()
|
||||
)
|
||||
|
||||
assert any(problem in fault for fault in faults), faults
|
||||
|
||||
|
||||
def test_what_was_assumed_can_be_a_source_but_does_not_bind():
|
||||
# `change_process: decide` leaves it to the agent, so nothing has to name it.
|
||||
scope = upstream()["operating_scope"]["attachments"]["scope.json"]
|
||||
|
||||
assert "operations.change_process" not in scope.split('"binding"')[1].split("]")[0]
|
||||
|
||||
|
||||
def test_an_empty_file_is_refused_and_bad_json_is_named():
|
||||
assert chain.check_requirements({REQUIREMENTS_FILE: "[]"}, upstream()) == [
|
||||
"operational-requirements.json: lists no Operational requirements"
|
||||
]
|
||||
assert (
|
||||
"not valid JSON" in chain.check_requirements({REQUIREMENTS_FILE: "{"}, None)[0]
|
||||
)
|
||||
|
||||
|
||||
def test_capabilities_from_the_catalogue_that_cover_everything_are_accepted():
|
||||
assert (
|
||||
check_capabilities(as_file(CAPABILITIES_FILE, CAPABILITIES), upstream()) == []
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "problem"),
|
||||
[
|
||||
(
|
||||
lambda e: e[0].update(catalogue="firefighting"),
|
||||
"'firefighting' is not in the capability catalogue",
|
||||
),
|
||||
(
|
||||
lambda e: e[0].update(catalogue="custom"),
|
||||
"C-1 is custom and gives no reason",
|
||||
),
|
||||
(lambda e: e[0].update(serves=[]), "C-1 serves no Operational requirement"),
|
||||
(
|
||||
lambda e: e[0].update(serves=["OR-1", "OR-9"]),
|
||||
"C-1 serves OR-9, which is not an Operational requirement",
|
||||
),
|
||||
(
|
||||
lambda e: e[0].update(items=["S-1", "S-9"]),
|
||||
"C-1 covers S-9, which is not an Item",
|
||||
),
|
||||
(
|
||||
lambda e: [c.update(items=["S-1", "S-2"]) for c in e],
|
||||
"Item S-3 is covered by no Capability",
|
||||
),
|
||||
(lambda e: e.pop(1), "Operational requirement OR-2 is served by no Capability"),
|
||||
(lambda e: e[0].update(hours="always"), "C-1: hours must be one of"),
|
||||
(
|
||||
lambda e: e[0].update(hours="business"),
|
||||
"faults are responded to 24x7, and Incident management is staffed only business",
|
||||
),
|
||||
(
|
||||
lambda e: e[0].update(effort_hours_per_month=0),
|
||||
"effort_hours_per_month must be a number above zero",
|
||||
),
|
||||
(lambda e: e[0].update(basis=""), "C-1 needs a basis for its effort"),
|
||||
],
|
||||
)
|
||||
def test_a_capability_that_breaks_the_chain_is_refused(change, problem):
|
||||
faults = check_capabilities(
|
||||
as_file(CAPABILITIES_FILE, capabilities(change)), upstream()
|
||||
)
|
||||
|
||||
assert any(problem in fault for fault in faults), faults
|
||||
|
||||
|
||||
def test_a_custom_capability_with_a_name_and_a_reason_is_accepted():
|
||||
custom = capabilities(
|
||||
lambda e: e[2].update(
|
||||
catalogue="custom", name="Evidence keeping", reason="No entry fits."
|
||||
)
|
||||
)
|
||||
|
||||
assert check_capabilities(as_file(CAPABILITIES_FILE, custom), upstream()) == []
|
||||
237
tests/test_digest.py
Normal file
237
tests/test_digest.py
Normal file
@@ -0,0 +1,237 @@
|
||||
"""Digesting a solution architecture's files into an operating spec, in code."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from execution_engine.scope import ESSENTIAL, merged, read_scope
|
||||
|
||||
from app.operating_model import digest, spec
|
||||
from app.operating_model.workflow import ROOT
|
||||
from tests.support import BUNDLE, CATALOGUE, WORKFLOW
|
||||
|
||||
FIXTURE = Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone"
|
||||
SOURCES = digest.read_folder(FIXTURE)
|
||||
KINDS = digest.item_kinds(ROOT, "gcp")
|
||||
DIGESTED = digest.architecture(SOURCES, KINDS)
|
||||
GRAPH = json.loads(SOURCES[digest.GRAPH])
|
||||
|
||||
|
||||
def test_every_file_the_digest_reads_came_with_the_fixture():
|
||||
assert set(SOURCES) == set(digest.FILES)
|
||||
assert set(DIGESTED.used) == set(digest.FILES)
|
||||
|
||||
|
||||
def test_every_component_becomes_one_item_and_every_element_is_accounted_for():
|
||||
items = DIGESTED.spec["solution"]["items"]
|
||||
components = json.loads(SOURCES[digest.COMPONENTS])
|
||||
|
||||
assert [item["source_ref"] for item in items] == [c["id"] for c in components]
|
||||
assert [item["id"] for item in items] == [f"S-{n}" for n in range(1, 10)]
|
||||
placed = [element for item in items for element in item["elements"]]
|
||||
assert sorted(placed) == sorted(e["id"] for e in GRAPH["elements"])
|
||||
assert len(placed) == 60
|
||||
|
||||
|
||||
def test_an_item_takes_its_product_and_its_kind_from_the_providers_table():
|
||||
by_ref = {item["source_ref"]: item for item in DIGESTED.spec["solution"]["items"]}
|
||||
|
||||
assert (by_ref["C-2"]["product"], by_ref["C-2"]["kind"]) == (
|
||||
"Cloud Interconnect",
|
||||
"network",
|
||||
)
|
||||
assert by_ref["C-4"]["kind"] == "storage"
|
||||
assert by_ref["C-7"]["kind"] == "keys"
|
||||
# A product the table does not know is a service; nothing is guessed.
|
||||
assert by_ref["C-3"]["kind"] == "service"
|
||||
|
||||
|
||||
def test_what_the_architecture_leaves_to_operation_becomes_the_conditions():
|
||||
conditions = DIGESTED.spec["solution"]["conditions"]
|
||||
left_to_operation = [
|
||||
(entry["id"], statement)
|
||||
for name in (digest.REQUIREMENTS, digest.RECOMMENDATIONS)
|
||||
for entry in json.loads(SOURCES[name])
|
||||
for statement in entry.get("outside_code") or []
|
||||
]
|
||||
|
||||
assert [(c["source_ref"], c["condition"]) for c in conditions] == [
|
||||
(identifier, " ".join(statement.split()))
|
||||
for identifier, statement in left_to_operation
|
||||
]
|
||||
recovery = next(
|
||||
c for c in conditions if "recovery time objective" in c["condition"]
|
||||
)
|
||||
assert recovery["source_ref"] == "NFR-11"
|
||||
items = {item["id"] for item in DIGESTED.spec["solution"]["items"]}
|
||||
assert all(c["on"] and set(c["on"]) <= items for c in conditions)
|
||||
|
||||
|
||||
def test_the_settings_checked_at_build_are_not_conditions():
|
||||
# 104 Checks on infrastructure settings were verified in validation. Keeping
|
||||
# them true is configuration management, not a Condition each.
|
||||
assert len(GRAPH["checks"]) == 104
|
||||
assert len(DIGESTED.spec["solution"]["conditions"]) == 12
|
||||
|
||||
|
||||
def test_only_what_the_customer_stated_is_carried_across_as_stated():
|
||||
carried = DIGESTED.spec
|
||||
|
||||
assert carried["operations"]["support_hours"] == "24x7" # stated: always
|
||||
assert carried["compliance"]["frameworks"] == ["DORA", "ISO27001", "PCI_DSS_v4"]
|
||||
assert carried["commercial"]["currency"] == "GBP"
|
||||
assert carried["solution"]["cloud_provider"] == "gcp"
|
||||
assert carried["engagement"]["kind"] == "platform_foundation"
|
||||
# The architecture agent assumed these for the customer; they are asked again.
|
||||
assert "change_process" not in carried["operations"]
|
||||
assert "service_levels" not in carried
|
||||
|
||||
|
||||
def test_the_build_and_what_the_customer_owns_of_it_are_carried_across():
|
||||
build = DIGESTED.spec["solution"]["build"]
|
||||
|
||||
assert build["duration_days"] == 36
|
||||
assert build["customer_owned"] == [
|
||||
"Order and provision the Interconnect carrier circuits"
|
||||
]
|
||||
|
||||
|
||||
def test_the_spec_says_which_execution_it_was_digested_from():
|
||||
assert DIGESTED.spec["metadata"] == {
|
||||
"customer_case": "case-08-landing-zone",
|
||||
"source": "architecture:case-08-landing-zone-185426",
|
||||
}
|
||||
assert "payment platform" in DIGESTED.spec["intent"]["statement"]
|
||||
|
||||
|
||||
def test_the_digested_spec_is_one_the_catalogue_accepts():
|
||||
scope = read_scope(
|
||||
DIGESTED.spec, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
|
||||
)
|
||||
|
||||
# What only the customer can say is still essential: what the business wants
|
||||
# from the solution, and who operates it.
|
||||
assert [gap["field"] for gap in scope.gaps(ESSENTIAL)] == [
|
||||
"intent.business_outcomes",
|
||||
"sourcing.model",
|
||||
]
|
||||
|
||||
|
||||
def test_every_digested_value_is_marked_and_what_the_caller_adds_is_theirs():
|
||||
scope = merged(
|
||||
DIGESTED.spec,
|
||||
"sourcing:\n model: shared\ncommercial:\n currency: EUR\n"
|
||||
"intent:\n business_outcomes: decide\n",
|
||||
BUNDLE.scope_catalogue(),
|
||||
provider_values=BUNDLE.scope_values,
|
||||
notes=DIGESTED.notes,
|
||||
)
|
||||
|
||||
assert scope.source("solution.items") == "digested"
|
||||
assert scope.notes["solution.items"].startswith("9 components")
|
||||
assert scope.source("sourcing.model") == "supplied"
|
||||
assert scope.values["commercial.currency"] == "EUR"
|
||||
assert scope.source("commercial.currency") == "supplied"
|
||||
assert scope.gaps(ESSENTIAL) == []
|
||||
|
||||
|
||||
def test_the_capabilities_proposed_follow_from_what_was_digested():
|
||||
scope = read_scope(
|
||||
DIGESTED.spec, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
|
||||
).for_context()
|
||||
|
||||
proposed = CATALOGUE.applicable(scope)
|
||||
|
||||
assert "backup-and-recovery" in proposed # the audit log store is storage
|
||||
assert "regulatory-evidence" in proposed # DORA, ISO 27001, PCI DSS
|
||||
assert len(spec.items(scope)) == 9
|
||||
|
||||
|
||||
def test_a_check_that_did_not_pass_is_an_inherited_risk():
|
||||
sources = dict(SOURCES)
|
||||
verdicts = json.loads(sources[digest.VERDICTS])
|
||||
verdicts["checks"][0].update(verdict="fail", evidence="display_name is empty")
|
||||
sources[digest.VERDICTS] = json.dumps(verdicts)
|
||||
|
||||
risks = digest.architecture(sources, KINDS).spec["solution"]["inherited_risks"]
|
||||
|
||||
assert risks == [
|
||||
{
|
||||
"on": "S-1",
|
||||
"risk": "Check K-1 failed in validation: `display_name` is set "
|
||||
"(display_name is empty)",
|
||||
"source_ref": "K-1",
|
||||
}
|
||||
]
|
||||
assert "inherited_risks" not in DIGESTED.spec["solution"] # every Check passed
|
||||
|
||||
|
||||
def test_files_passed_without_their_manifest_are_marked_unverified():
|
||||
sources = {k: v for k, v in SOURCES.items() if k != digest.MANIFEST}
|
||||
|
||||
digested = digest.architecture(sources, KINDS)
|
||||
|
||||
assert digested.spec["metadata"] == {"source": "architecture:unverified"}
|
||||
assert (
|
||||
"not known to be one completed Execution" in digested.notes["metadata.source"]
|
||||
)
|
||||
assert len(digested.spec["solution"]["items"]) == 9
|
||||
|
||||
|
||||
def test_the_graph_alone_is_enough_to_say_what_is_operated():
|
||||
digested = digest.architecture({digest.GRAPH: SOURCES[digest.GRAPH]}, KINDS)
|
||||
|
||||
items = digested.spec["solution"]["items"]
|
||||
# With no components file, they are taken from the Elements, in the order met.
|
||||
assert sorted(item["source_ref"] for item in items) == [
|
||||
f"C-{n}" for n in range(1, 10)
|
||||
]
|
||||
assert "conditions" not in digested.spec["solution"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "problem"),
|
||||
[
|
||||
(
|
||||
lambda s: s.update({digest.GRAPH: "{"}),
|
||||
"target-architecture.json: not valid JSON",
|
||||
),
|
||||
(
|
||||
lambda s: s.update({digest.GRAPH: "[]"}),
|
||||
"must be an object with a list of elements",
|
||||
),
|
||||
(
|
||||
lambda s: s.update(
|
||||
{digest.MANIFEST: json.dumps({"status": "completed_with_warnings"})}
|
||||
),
|
||||
"the Execution did not complete",
|
||||
),
|
||||
(
|
||||
lambda s: s.update(
|
||||
{digest.COMPONENTS: json.dumps([{"id": "C-1", "name": "Only one"}])}
|
||||
),
|
||||
"these Elements name a component that components.json does not have",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_an_architecture_that_cannot_be_used_is_refused_and_says_why(change, problem):
|
||||
sources = dict(SOURCES)
|
||||
change(sources)
|
||||
|
||||
with pytest.raises(ValueError, match=problem):
|
||||
digest.architecture(sources, KINDS)
|
||||
|
||||
|
||||
def test_sources_that_are_not_an_architecture_are_left_to_be_read():
|
||||
assert digest.architecture({"design.md": "# Our platform\n"}, KINDS).spec == {}
|
||||
assert WORKFLOW.digest is not None
|
||||
assert WORKFLOW.digest({"design.md": "# Our platform\n"}).used == ()
|
||||
|
||||
|
||||
def test_the_command_prints_the_spec_and_what_it_was_read_from(capsys):
|
||||
assert digest.main([str(FIXTURE)]) == 0
|
||||
|
||||
printed = capsys.readouterr()
|
||||
assert "source: architecture:case-08-landing-zone-185426" in printed.out
|
||||
assert "# solution.items: 9 components" in printed.err
|
||||
assert digest.main([str(FIXTURE.parent)]) == 1
|
||||
69
tests/test_eval.py
Normal file
69
tests/test_eval.py
Normal file
@@ -0,0 +1,69 @@
|
||||
"""The evaluation harness itself, run on the scripted runtime: no model is called."""
|
||||
|
||||
import json
|
||||
|
||||
from execution_engine.adk.fake_runtime import FakeAgentRuntime
|
||||
from execution_engine.config import Settings
|
||||
|
||||
from eval import operating_model_eval as harness
|
||||
from tests.support import SPEC
|
||||
from tests.test_execution import turns
|
||||
|
||||
|
||||
def settings() -> Settings:
|
||||
return Settings( # type: ignore[call-arg]
|
||||
_env_file=None, CLOUD_PROVIDER="gcp", CHECKPOINTER="memory", MCP_ENABLED=False
|
||||
)
|
||||
|
||||
|
||||
CASE = {"id": "scripted", "max_revisions": 2, "max_minutes": 5}
|
||||
|
||||
|
||||
async def run(tmp_path, scripted, monkeypatch):
|
||||
monkeypatch.setattr(harness, "inputs", lambda case: {"scope": SPEC})
|
||||
return await harness.run_case(
|
||||
CASE, tmp_path, settings=settings(), runtime=FakeAgentRuntime(scripted)
|
||||
)
|
||||
|
||||
|
||||
async def test_a_sound_execution_passes_every_check(tmp_path, monkeypatch):
|
||||
result = await run(tmp_path, turns(), monkeypatch)
|
||||
|
||||
assert result.status == "completed"
|
||||
failed = [check for check in result.checks if not check.passed]
|
||||
# The scripted packager writes one word; a real one must write a summary.
|
||||
assert [check.name for check in failed] == ["the packager wrote a summary"]
|
||||
assert result.figures["cost_per_month"] == 43410
|
||||
assert result.figures["scenarios"] == 10
|
||||
assert (tmp_path / "deliverables").is_dir()
|
||||
assert json.dumps(result.as_dict()) # the whole result can be written down
|
||||
|
||||
|
||||
async def test_a_run_that_cannot_finish_is_reported_not_raised(tmp_path, monkeypatch):
|
||||
scripted = turns()
|
||||
scripted["validator"] = []
|
||||
|
||||
result = await run(tmp_path, scripted, monkeypatch)
|
||||
|
||||
assert not result.passed
|
||||
assert result.status != "completed"
|
||||
|
||||
|
||||
def test_every_case_names_something_to_start_from():
|
||||
for case in harness.load_cases():
|
||||
given = harness.inputs(case)
|
||||
assert "scope" in given or "sources" in given, case["id"]
|
||||
|
||||
|
||||
def test_the_benchmark_spec_is_complete_enough_to_run_with_nobody_to_ask():
|
||||
from execution_engine.scope import ESSENTIAL, read_scope
|
||||
|
||||
from tests.support import BUNDLE
|
||||
|
||||
text = (harness.CASES.parent / "specs" / "landing-zone-shared.yaml").read_text()
|
||||
scope = read_scope(
|
||||
text, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
|
||||
)
|
||||
|
||||
assert scope.gaps(ESSENTIAL) == []
|
||||
assert scope.customer_case == "bench-landing-zone-shared"
|
||||
411
tests/test_execution.py
Normal file
411
tests/test_execution.py
Normal file
@@ -0,0 +1,411 @@
|
||||
"""Whole Executions of the target operating model Workflow, on the scripted runtime."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
|
||||
from execution_engine.adk.roles import load_roles
|
||||
from execution_engine.adk.runtime import NoDecider
|
||||
from execution_engine.agent import ExecutionEngine
|
||||
from execution_engine.nodes.router import Router, Thresholds
|
||||
from execution_engine.recorder import InMemoryRecorder
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.operating_model.names import (
|
||||
CAPABILITIES_FILE,
|
||||
CAPABILITY_MODEL,
|
||||
FINDINGS_FILE,
|
||||
GRAPH_FILE,
|
||||
GUIDE,
|
||||
OPERATING_SCOPE,
|
||||
OPERATIONAL_REQUIREMENTS,
|
||||
REQUIREMENTS_FILE,
|
||||
TARGET_OPERATING_MODEL,
|
||||
VALIDATION_RESULTS,
|
||||
VERDICTS_FILE,
|
||||
)
|
||||
from app.operating_model.workflow import ROOT
|
||||
from tests.support import (
|
||||
BUNDLE,
|
||||
CAPABILITIES,
|
||||
GRAPH,
|
||||
REQUIREMENTS,
|
||||
SPEC,
|
||||
VERDICTS,
|
||||
WORKFLOW,
|
||||
graph,
|
||||
)
|
||||
|
||||
ROLES = load_roles(ROOT / "agents", BUNDLE, WORKFLOW)
|
||||
CASE = "northwind-landing-zone"
|
||||
FINDING = {
|
||||
"id": "AF-1",
|
||||
"about": ["S-2"],
|
||||
"needed_by": ["C-3"],
|
||||
"lacks": "Nothing alerts when the audit log store's retention lock is removed.",
|
||||
"proposed": "Add an alert on a change to the retention lock.",
|
||||
}
|
||||
|
||||
|
||||
def submit(kind: str, **files: Any) -> list[Call]:
|
||||
"""One turn that submits a Deliverable with every section of its contract."""
|
||||
spec = WORKFLOW.deliverables[kind]
|
||||
content = f"# {kind}\n" + "".join(
|
||||
f"\n## {section}\n\nDetails.\n" for section in spec.sections
|
||||
)
|
||||
return [
|
||||
Call(
|
||||
"submit_deliverable",
|
||||
{
|
||||
"content": content,
|
||||
"attachments": {name: json.dumps(data) for name, data in files.items()},
|
||||
},
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def turns(model=GRAPH, verdicts=VERDICTS, findings=None) -> dict[str, list[Any]]:
|
||||
model_files = {GRAPH_FILE: model}
|
||||
if findings is not None:
|
||||
model_files[FINDINGS_FILE] = findings
|
||||
return {
|
||||
"scope-analyst": [submit(OPERATING_SCOPE)],
|
||||
"operations-analyst": [
|
||||
submit(OPERATIONAL_REQUIREMENTS, **{REQUIREMENTS_FILE: REQUIREMENTS})
|
||||
],
|
||||
"capability-modeller": [
|
||||
submit(CAPABILITY_MODEL, **{CAPABILITIES_FILE: CAPABILITIES})
|
||||
],
|
||||
"operating-model-architect": [submit(TARGET_OPERATING_MODEL, **model_files)],
|
||||
"validator": [submit(VALIDATION_RESULTS, **{VERDICTS_FILE: verdicts})],
|
||||
"packager": [submit(GUIDE)],
|
||||
}
|
||||
|
||||
|
||||
def make_engine(
|
||||
tmp_path: Path, scripted: dict[str, list[Any]]
|
||||
) -> tuple[ExecutionEngine, FakeAgentRuntime]:
|
||||
runtime = FakeAgentRuntime(scripted)
|
||||
engine = ExecutionEngine(
|
||||
workflow=WORKFLOW,
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
scope_catalogue=BUNDLE.scope_catalogue(),
|
||||
scope_values=BUNDLE.scope_values,
|
||||
scope_skill=BUNDLE.scope_skill,
|
||||
)
|
||||
return engine, runtime
|
||||
|
||||
|
||||
def folder(tmp_path: Path, execution_id: str) -> Path:
|
||||
return tmp_path / "deliverables" / "cases" / CASE / "executions" / execution_id
|
||||
|
||||
|
||||
async def test_an_operating_spec_becomes_a_priced_validated_operating_model(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, turns())
|
||||
|
||||
result = await engine.start("", scope=SPEC, execution_id="e1")
|
||||
|
||||
assert result.status == "completed", result.state.get("warnings")
|
||||
assert [run.role for run in runtime.runs] == [
|
||||
"scope-analyst",
|
||||
"operations-analyst",
|
||||
"capability-modeller",
|
||||
"operating-model-architect",
|
||||
"validator",
|
||||
"packager",
|
||||
]
|
||||
out = folder(tmp_path, "e1")
|
||||
written = {p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file()}
|
||||
assert {
|
||||
"execution.json",
|
||||
"operating-model-guide.md",
|
||||
"scope.yaml",
|
||||
"scope.json",
|
||||
"operational-requirements.json",
|
||||
"capabilities.json",
|
||||
"target-operating-model.json",
|
||||
"findings.json",
|
||||
"staffing.json",
|
||||
"organisation.mmd",
|
||||
"verdicts.json",
|
||||
"scenarios.json",
|
||||
} <= written
|
||||
record = json.loads((out / "execution.json").read_text())
|
||||
assert record["customer_case"] == CASE
|
||||
assert record["final_document"] == "operating-model-guide.md"
|
||||
assert record["staffing"] == {
|
||||
"people": 5.1,
|
||||
"cost_per_month": 43410,
|
||||
"currency": "GBP",
|
||||
"rate_card": {
|
||||
"name": "rate-card",
|
||||
"version": "draft-2026-10-13",
|
||||
"draft": True,
|
||||
},
|
||||
}
|
||||
assert record["findings"] == []
|
||||
assert record["capability_sources_verified"] is False
|
||||
|
||||
|
||||
async def test_the_views_and_the_walk_throughs_are_written_by_code(tmp_path):
|
||||
engine, _ = make_engine(tmp_path, turns())
|
||||
|
||||
result = await engine.start("", scope=SPEC)
|
||||
|
||||
model = result.state["deliverables"][TARGET_OPERATING_MODEL]["content"]
|
||||
assert "| Incident management (C-1) | C | R | A |" in model
|
||||
assert "Cost a month: 43,410 GBP." in model
|
||||
validation = result.state["deliverables"][VALIDATION_RESULTS]
|
||||
assert "## Scenario walk-throughs" in validation["content"]
|
||||
verdicts = json.loads(validation["attachments"][VERDICTS_FILE])
|
||||
assert verdicts["dry_run"]["evidence"] == "10 of 10 scenario walk-throughs pass."
|
||||
assert len(json.loads(validation["attachments"]["scenarios.json"])) == 10
|
||||
|
||||
|
||||
async def test_specialists_are_told_what_code_found_and_which_prices_apply(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, turns())
|
||||
|
||||
await engine.start("", scope=SPEC)
|
||||
|
||||
by_role = {run.role: run.task.context for run in runtime.runs}
|
||||
proposed = by_role["capability-modeller"]["capabilities_proposed"]
|
||||
assert {"incident-management", "regulatory-evidence", "supplier-management"} <= set(
|
||||
proposed
|
||||
)
|
||||
assert by_role["operating-model-architect"]["rate_card"]["draft"] is True
|
||||
assert "PL" in by_role["operating-model-architect"]["rate_card"]["countries"]
|
||||
# The architect builds the graph with the engine's tools, in this Workflow's shape.
|
||||
architect = next(
|
||||
run for run in runtime.runs if run.role == "operating-model-architect"
|
||||
)
|
||||
assert {"graph_put", "graph_read", "graph_remove"} <= set(architect.tools)
|
||||
# Every Specialist is given the agreed operating spec, with the solution in it.
|
||||
items = by_role["validator"]["scope"]["stated"]["solution"]["items"]
|
||||
assert [item["id"] for item in items] == ["S-1", "S-2", "S-3"]
|
||||
|
||||
|
||||
async def test_a_spec_with_a_currency_the_rate_card_cannot_convert_to_is_refused(
|
||||
tmp_path,
|
||||
):
|
||||
import pytest
|
||||
|
||||
engine, runtime = make_engine(tmp_path, turns())
|
||||
|
||||
with pytest.raises(ValueError, match="commercial.currency"):
|
||||
await engine.start("", scope=SPEC.replace("currency: GBP", "currency: CHF"))
|
||||
|
||||
assert runtime.runs == []
|
||||
|
||||
|
||||
async def test_a_spec_with_no_currency_and_nobody_to_ask_is_refused(tmp_path):
|
||||
import pytest
|
||||
|
||||
engine, _ = make_engine(tmp_path, turns())
|
||||
|
||||
with pytest.raises(ValueError, match="commercial.currency"):
|
||||
await engine.start("", scope=SPEC.replace(" currency: GBP\n", ""))
|
||||
|
||||
|
||||
async def test_a_model_nobody_can_run_at_night_is_refused_before_it_is_approved(
|
||||
tmp_path,
|
||||
):
|
||||
off_hours = graph()
|
||||
off_hours["roles"][1]["hours"] = "extended"
|
||||
scripted = turns()
|
||||
# Refused, and put right in the same run: the customer never sees it.
|
||||
scripted["operating-model-architect"] = [
|
||||
[
|
||||
*submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: off_hours}),
|
||||
*submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH}),
|
||||
]
|
||||
]
|
||||
engine, runtime = make_engine(tmp_path, scripted)
|
||||
|
||||
result = await engine.start("", scope=SPEC)
|
||||
|
||||
assert result.status == "completed", result.state.get("warnings")
|
||||
refused, accepted = runtime.runs[3].results
|
||||
assert (
|
||||
"Capability C-1 is needed 24x7, and no Role responsible for it works "
|
||||
"those hours" in refused[1]
|
||||
)
|
||||
assert accepted[1] == "Submitted target_operating_model."
|
||||
assert (
|
||||
result.state["ledger"]["deliverables"][TARGET_OPERATING_MODEL]["revisions"] == 0
|
||||
)
|
||||
|
||||
|
||||
async def test_a_failed_scenario_reopens_the_model_through_the_router(tmp_path):
|
||||
stranded = graph()
|
||||
stranded["responsibilities"] = [
|
||||
r for r in stranded["responsibilities"] if r["id"] not in ("RS-3", "RS-8")
|
||||
]
|
||||
scripted = turns()
|
||||
scripted["operating-model-architect"] = [
|
||||
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: stranded}),
|
||||
lambda task: (
|
||||
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH})
|
||||
if any("SC-party-P-2 failed" in item for item in task.context["feedback"])
|
||||
else []
|
||||
),
|
||||
]
|
||||
scripted["validator"] = scripted["validator"] * 2
|
||||
engine, runtime = make_engine(tmp_path, scripted)
|
||||
|
||||
result = await engine.start("", scope=SPEC)
|
||||
|
||||
assert result.status == "completed", result.state.get("warnings")
|
||||
assert [d["action"] for d in result.state["decisions"]].count("revise_code") == 1
|
||||
ledger = result.state["ledger"]["deliverables"]
|
||||
assert ledger[TARGET_OPERATING_MODEL]["revisions"] == 1
|
||||
assert ledger[CAPABILITY_MODEL]["revisions"] == 0
|
||||
assert [run.role for run in runtime.runs].count("validator") == 2
|
||||
|
||||
|
||||
async def test_a_finding_is_kept_out_of_the_design_and_drafted_as_a_change_request(
|
||||
tmp_path,
|
||||
):
|
||||
waiting = graph()
|
||||
waiting["checks"][2]["depends_on_finding"] = "AF-1"
|
||||
verdicts = json.loads(json.dumps(VERDICTS))
|
||||
verdicts["checks"][2].update(verdict="not_verifiable", evidence="Waits on AF-1.")
|
||||
digested = SPEC.replace(
|
||||
" customer_case: northwind-landing-zone\n",
|
||||
" customer_case: northwind-landing-zone\n source: architecture:exec-42\n",
|
||||
)
|
||||
engine, _ = make_engine(tmp_path, turns(waiting, verdicts, [FINDING]))
|
||||
|
||||
result = await engine.start("", scope=digested, execution_id="e1")
|
||||
|
||||
assert result.status == "completed", result.state.get("warnings")
|
||||
out = folder(tmp_path, "e1")
|
||||
record = json.loads((out / "execution.json").read_text())
|
||||
assert record["findings"] == ["AF-1"]
|
||||
assert record["change_request"] == "architecture-change-request.md"
|
||||
request = (out / "architecture-change-request.md").read_text()
|
||||
assert "digested from `architecture:exec-42`" in request
|
||||
assert (
|
||||
"**What is asked for.** Add an alert on a change to the retention lock."
|
||||
in request
|
||||
)
|
||||
assert (
|
||||
"## Findings" in result.state["deliverables"][TARGET_OPERATING_MODEL]["content"]
|
||||
)
|
||||
# The Check that waits on the finding is reported with it, and nothing was
|
||||
# revised for it: the model is not at fault, and neither are the Capabilities.
|
||||
verdicts = json.loads((out / "verdicts.json").read_text())
|
||||
waiting = [c for c in verdicts["checks"] if c["verdict"] == "waits_on_finding"]
|
||||
assert [(c["check"], c["finding"]) for c in waiting] == [("OC-3", "AF-1")]
|
||||
ledger = result.state["ledger"]["deliverables"]
|
||||
assert all(entry["revisions"] == 0 for entry in ledger.values())
|
||||
# The solution itself is as it was described: nothing was added to it.
|
||||
scope = json.loads((out / "scope.json").read_text())
|
||||
assert [item["id"] for item in scope["stated"]["solution"]["items"]] == [
|
||||
"S-1",
|
||||
"S-2",
|
||||
"S-3",
|
||||
]
|
||||
|
||||
|
||||
# --- starting from sources ---------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_an_execution_starts_from_a_solution_architectures_files(tmp_path):
|
||||
from pathlib import Path as _Path
|
||||
|
||||
from app.operating_model import digest
|
||||
|
||||
fixture = (
|
||||
_Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone"
|
||||
)
|
||||
engine, runtime = make_engine(
|
||||
tmp_path, {"scope-analyst": [submit(OPERATING_SCOPE)]}
|
||||
)
|
||||
|
||||
result = await engine.start(
|
||||
"",
|
||||
sources=digest.read_folder(fixture),
|
||||
# The one essential thing the architecture cannot say: who operates it.
|
||||
scope="sourcing:\n model: shared\nintent:\n business_outcomes:\n"
|
||||
" - {id: BO-1, outcome: Payments are never down for long.}\n",
|
||||
execution_id="e1",
|
||||
)
|
||||
|
||||
assert result.status == "awaiting_user"
|
||||
assert result.prompt["approval"]["phase"] == "scoping"
|
||||
assert result.state["customer_case"] == "case-08-landing-zone"
|
||||
assert "payment platform" in result.state["workflow_request"]
|
||||
scope = json.loads(
|
||||
result.state["deliverables"][OPERATING_SCOPE]["attachments"]["scope.json"]
|
||||
)
|
||||
assert len(scope["stated"]["solution"]["items"]) == 9
|
||||
assert len(scope["stated"]["solution"]["conditions"]) == 12
|
||||
assert scope["stated"]["sourcing"] == {"model": "shared"}
|
||||
# What was digested is shown to the customer as digested, to approve.
|
||||
register = result.state["deliverables"][OPERATING_SCOPE]["content"]
|
||||
assert "digested from the sources supplied" in register
|
||||
# Every source is on record with its hash, and was read by code.
|
||||
assert set(result.state["sources"]) == set(digest.FILES)
|
||||
assert all(entry["digested"] for entry in result.state["sources"].values())
|
||||
assert "source_documents" not in runtime.runs[0].task.context
|
||||
|
||||
|
||||
async def test_a_document_code_cannot_read_is_given_to_the_scope_analyst(tmp_path):
|
||||
engine, runtime = make_engine(tmp_path, {"scope-analyst": [[]]})
|
||||
document = "# Our platform\n\nWe run a booking service on three clusters.\n"
|
||||
|
||||
await engine.start(
|
||||
"Design the operating model for our booking platform.",
|
||||
customer_case="bookings",
|
||||
sources={"design.md": document},
|
||||
)
|
||||
|
||||
context = runtime.runs[0].task.context
|
||||
assert context["source_documents"] == {"design.md": document}
|
||||
assert runtime.runs[0].role == "scope-analyst"
|
||||
|
||||
|
||||
async def test_a_finding_the_model_could_meet_itself_sends_the_model_back(tmp_path):
|
||||
"""A gap in the operating model's own staffing is not a finding: the validator
|
||||
fails the Check that waits on it, and the architect designs it instead."""
|
||||
excuse = {
|
||||
"id": "AF-1",
|
||||
"about": ["S-2"],
|
||||
"needed_by": ["OC-3"],
|
||||
"lacks": "The retention review is only staffed in business hours.",
|
||||
"proposed": "Staff it at every hour.",
|
||||
}
|
||||
waiting = graph()
|
||||
waiting["checks"][2]["depends_on_finding"] = "AF-1"
|
||||
judged = json.loads(json.dumps(VERDICTS))
|
||||
judged["checks"][2].update(
|
||||
verdict="fail",
|
||||
evidence="Staffing is the operating model's to provide; this is not a finding.",
|
||||
)
|
||||
scripted = turns(waiting, judged, [excuse])
|
||||
scripted["operating-model-architect"].append(
|
||||
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH, FINDINGS_FILE: []})
|
||||
)
|
||||
scripted["validator"].append(
|
||||
submit(VALIDATION_RESULTS, **{VERDICTS_FILE: VERDICTS})
|
||||
)
|
||||
engine, runtime = make_engine(tmp_path, scripted)
|
||||
|
||||
result = await engine.start("", scope=SPEC)
|
||||
|
||||
assert result.status == "completed", result.state.get("warnings")
|
||||
assert "revise_code" in [d["action"] for d in result.state["decisions"]]
|
||||
second = [run for run in runtime.runs if run.role == "operating-model-architect"][1]
|
||||
assert any("not a finding" in item for item in second.task.context["feedback"])
|
||||
assert graph_findings(result) == []
|
||||
|
||||
|
||||
def graph_findings(result) -> list:
|
||||
attachments = result.state["deliverables"][TARGET_OPERATING_MODEL]["attachments"]
|
||||
return json.loads(attachments.get(FINDINGS_FILE, "[]"))
|
||||
241
tests/test_graph.py
Normal file
241
tests/test_graph.py
Normal file
@@ -0,0 +1,241 @@
|
||||
"""The Target operating model graph: what is refused, and the views code writes."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.operating_model import graph as model
|
||||
from app.operating_model.names import (
|
||||
FINDINGS_FILE,
|
||||
GRAPH_FILE,
|
||||
ORGANISATION_FILE,
|
||||
STAFFING_FILE,
|
||||
)
|
||||
from tests.support import CARD, CATALOGUE, GRAPH, as_file, graph, upstream
|
||||
|
||||
check = model.check(CARD)
|
||||
add_views = model.views(CATALOGUE, CARD)
|
||||
FINDING = {
|
||||
"id": "AF-1",
|
||||
"about": ["S-2"],
|
||||
"needed_by": ["C-3"],
|
||||
"lacks": "Nothing alerts when the audit log store's retention lock is removed.",
|
||||
"proposed": "Add an alert on a change to the retention lock.",
|
||||
}
|
||||
|
||||
|
||||
def faults(changed, findings=None) -> list[str]:
|
||||
attachments = as_file(GRAPH_FILE, changed)
|
||||
if findings is not None:
|
||||
attachments[FINDINGS_FILE] = json.dumps(findings)
|
||||
return check(attachments, upstream())
|
||||
|
||||
|
||||
def test_a_graph_that_holds_together_is_accepted():
|
||||
assert faults(GRAPH) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "problem"),
|
||||
[
|
||||
(
|
||||
lambda g: g["parties"][0].update(kind="friend"),
|
||||
"Party P-1: kind must be one of",
|
||||
),
|
||||
(
|
||||
lambda g: g["roles"][0].update(party="P-9"),
|
||||
"Role R-1 is in P-9, which is not a Party",
|
||||
),
|
||||
(
|
||||
lambda g: g["roles"][1].update(country="FR"),
|
||||
"Role R-2 sits in FR, which the rate card does not have",
|
||||
),
|
||||
(
|
||||
lambda g: g["roles"][1].update(country="US"),
|
||||
"Role R-2 sits in US; the customer allows only GB, PL, IN",
|
||||
),
|
||||
(
|
||||
lambda g: g["roles"][1].update(level="wizard"),
|
||||
"Role R-2 is at level wizard, which the rate card does not have",
|
||||
),
|
||||
(
|
||||
lambda g: g["roles"][1].update(hours="nights"),
|
||||
"Role R-2: hours must be one of",
|
||||
),
|
||||
(lambda g: g["roles"][1].pop("country"), "(R-2) needs country"),
|
||||
(
|
||||
lambda g: g["responsibilities"][0].update(kind="owner"),
|
||||
"Responsibility RS-1: kind must be one of",
|
||||
),
|
||||
(
|
||||
lambda g: g["responsibilities"][0].update(role="R-9"),
|
||||
"RS-1 names R-9, which is not a Role",
|
||||
),
|
||||
(
|
||||
lambda g: g["responsibilities"][0].update(capability="C-9"),
|
||||
"RS-1 names C-9, which is not a Capability",
|
||||
),
|
||||
(
|
||||
lambda g: g["responsibilities"].pop(0),
|
||||
"Capability C-1 has 0 accountable Roles; it needs exactly one",
|
||||
),
|
||||
(
|
||||
lambda g: g["responsibilities"][2].update(kind="accountable"),
|
||||
"Capability C-1 has 2 accountable Roles; it needs exactly one",
|
||||
),
|
||||
(
|
||||
lambda g: g["responsibilities"].pop(1),
|
||||
"Capability C-1 has no responsible Role",
|
||||
),
|
||||
(
|
||||
lambda g: g["roles"][1].update(hours="business"),
|
||||
"Capability C-1 is needed 24x7, and no Role responsible for it works those hours",
|
||||
),
|
||||
(lambda g: g["checks"].pop(1), "Condition K-1 is carried by no Check"),
|
||||
(
|
||||
lambda g: g["checks"][0].update(on="C-9"),
|
||||
"Check OC-1 is on C-9, which is not",
|
||||
),
|
||||
(
|
||||
lambda g: g["checks"][0].update(source="OR-9"),
|
||||
"Check OC-1: source OR-9 is not an Operational requirement",
|
||||
),
|
||||
(
|
||||
lambda g: g["checks"][0].update(depends_on_finding="AF-1"),
|
||||
"Check OC-1 depends on AF-1, which is not a finding",
|
||||
),
|
||||
(
|
||||
lambda g: g["roles"].append(dict(g["roles"][0])),
|
||||
"id R-1 is used more than once in roles",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_graph_that_does_not_hold_together_is_refused(change, problem):
|
||||
changed = graph()
|
||||
change(changed)
|
||||
|
||||
found = faults(changed)
|
||||
|
||||
assert any(problem in fault for fault in found), found
|
||||
|
||||
|
||||
def test_a_file_that_is_not_a_graph_is_named():
|
||||
assert "not valid JSON" in check({GRAPH_FILE: "{"}, upstream())[0]
|
||||
assert "must be an object with" in check({GRAPH_FILE: "[]"}, upstream())[0]
|
||||
|
||||
|
||||
def test_a_finding_is_recorded_and_a_check_may_wait_on_it():
|
||||
changed = graph()
|
||||
changed["checks"][2]["depends_on_finding"] = "AF-1"
|
||||
|
||||
assert faults(changed, [FINDING]) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "problem"),
|
||||
[
|
||||
(lambda f: f.update(lacks=""), "AF-1 needs lacks"),
|
||||
(lambda f: f.update(proposed=""), "AF-1 needs proposed"),
|
||||
(lambda f: f.update(needed_by=[]), "AF-1 must say what needs it"),
|
||||
(lambda f: f.update(about=["S-9"]), "AF-1 is about S-9, which is not an Item"),
|
||||
(
|
||||
lambda f: f.update(about=[]),
|
||||
"AF-1 names no Item of the solution under about",
|
||||
),
|
||||
(
|
||||
lambda f: f.update(needed_by=["C-9"]),
|
||||
"AF-1 is needed by C-9, which is not in the operating model",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_finding_that_does_not_hold_together_is_refused(change, problem):
|
||||
finding = dict(FINDING)
|
||||
change(finding)
|
||||
|
||||
found = faults(GRAPH, [finding])
|
||||
|
||||
assert any(problem in fault for fault in found), found
|
||||
|
||||
|
||||
def viewed(changed=GRAPH, findings=None, spec=None):
|
||||
attachments = as_file(GRAPH_FILE, changed)
|
||||
if findings is not None:
|
||||
attachments[FINDINGS_FILE] = json.dumps(findings)
|
||||
built = upstream() if spec is None else upstream(spec=spec)
|
||||
return add_views(
|
||||
"# Target operating model\n\n## Design rationale\n\nWhy.\n", attachments, built
|
||||
)
|
||||
|
||||
|
||||
def test_the_matrix_is_written_from_the_graph():
|
||||
content, _ = viewed()
|
||||
|
||||
assert "## Responsibility matrix" in content
|
||||
assert "| Capability | Platform owner | Platform SRE | Service manager |" in content
|
||||
assert "| Incident management (C-1) | C | R | A |" in content
|
||||
assert "| Change enablement (C-2) | A | R | |" in content
|
||||
# One Role may be answerable for a Capability and do it: both parts are shown.
|
||||
assert "| Regulatory evidence and reporting (C-3) | C | | A/R |" in content
|
||||
|
||||
|
||||
def test_staffing_and_cost_are_written_with_the_prices_they_came_from():
|
||||
content, attachments = viewed()
|
||||
|
||||
assert (
|
||||
"| Platform SRE | Managed service provider | PL | senior | 24x7 | 100.0 h | 5.0 | 41,640 GBP |"
|
||||
in content
|
||||
)
|
||||
assert (
|
||||
"| Platform owner | Northwind platform team | GB | lead | business | 0.0 h | 0.0 | not priced |"
|
||||
in content
|
||||
)
|
||||
assert "Cost a month: 43,410 GBP." in content
|
||||
assert "The monthly budget is 120,000 GBP: this is within it." in content
|
||||
assert "version `draft-2026-10-13`" in content
|
||||
assert "Its figures are placeholders" in content
|
||||
assert "Converted from USD at 0.79, a rate set on 2026-10-13." in content
|
||||
assert json.loads(attachments[STAFFING_FILE])["cost_per_month"] == 43410
|
||||
|
||||
|
||||
def test_a_cost_over_the_budget_is_shown_as_over_not_refused():
|
||||
from tests.support import SPEC
|
||||
|
||||
tight = SPEC.replace("monthly_run_budget: 120000", "monthly_run_budget: 40000")
|
||||
|
||||
content, _ = viewed(spec=tight)
|
||||
|
||||
assert "this is over it by 3,410 GBP." in content
|
||||
assert check(as_file(GRAPH_FILE, GRAPH), upstream(spec=tight)) == []
|
||||
|
||||
|
||||
def test_the_organisation_chart_is_drawn_from_the_graph():
|
||||
_, attachments = viewed()
|
||||
|
||||
chart = attachments[ORGANISATION_FILE]
|
||||
assert chart.startswith("flowchart TB\n")
|
||||
assert 'subgraph P_2["Managed service provider (provider)"]' in chart
|
||||
assert 'R_2["Platform SRE<br/>PL, senior, 24x7"]' in chart
|
||||
|
||||
|
||||
def test_findings_are_listed_for_whoever_owns_the_design():
|
||||
content, attachments = viewed(findings=[FINDING])
|
||||
|
||||
assert "## Findings" in content
|
||||
assert "**AF-1** (S-2): Nothing alerts when" in content
|
||||
assert json.loads(attachments[FINDINGS_FILE]) == [FINDING]
|
||||
|
||||
|
||||
def test_with_no_findings_the_file_is_empty_and_no_section_is_written():
|
||||
content, attachments = viewed()
|
||||
|
||||
assert "## Findings" not in content
|
||||
assert attachments[FINDINGS_FILE] == "[]"
|
||||
|
||||
|
||||
def test_a_revision_replaces_the_views_and_never_stacks_them():
|
||||
first, attachments = viewed()
|
||||
|
||||
second, _ = add_views(first, dict(attachments), upstream())
|
||||
|
||||
assert second == first
|
||||
assert second.count("## Responsibility matrix") == 1
|
||||
166
tests/test_guide.py
Normal file
166
tests/test_guide.py
Normal file
@@ -0,0 +1,166 @@
|
||||
"""The operating model guide: the packager writes the summary, code writes the rest."""
|
||||
|
||||
import json
|
||||
|
||||
from app.operating_model import guide, scenarios, spec, staffing
|
||||
from app.operating_model.names import (
|
||||
FINDINGS_FILE,
|
||||
SCENARIOS_FILE,
|
||||
STAFFING_FILE,
|
||||
TARGET_OPERATING_MODEL,
|
||||
VALIDATION_RESULTS,
|
||||
VERDICTS_FILE,
|
||||
)
|
||||
from tests.support import (
|
||||
CAPABILITIES,
|
||||
CARD,
|
||||
CATALOGUE,
|
||||
GRAPH,
|
||||
VERDICTS,
|
||||
WORKFLOW,
|
||||
upstream,
|
||||
)
|
||||
|
||||
DRAFT = "# Northwind landing zone\n\n" + "".join(
|
||||
f"## {section}\n\n"
|
||||
+ (
|
||||
"The provider runs the platform at every hour; Northwind approves changes.\n\n"
|
||||
if section == guide.SUMMARY
|
||||
else "It costs 9,999 GBP and needs 2 people.\n\n"
|
||||
)
|
||||
for section in guide.SECTIONS
|
||||
)
|
||||
|
||||
|
||||
def delivered(findings=None, model=GRAPH) -> dict:
|
||||
"""Every Deliverable the guide is assembled from, with what code added to each."""
|
||||
built = upstream(model=model, findings=findings)
|
||||
scope = spec.agreed(built)
|
||||
cost = staffing.compute(model, CAPABILITIES, CARD, "GBP")
|
||||
built[TARGET_OPERATING_MODEL]["attachments"][STAFFING_FILE] = json.dumps(cost)
|
||||
built[TARGET_OPERATING_MODEL]["attachments"].setdefault(FINDINGS_FILE, "[]")
|
||||
walked = scenarios.walk(model, CAPABILITIES, scope, CATALOGUE, cost)
|
||||
waiting = {
|
||||
str(c["id"]): str(c["depends_on_finding"])
|
||||
for c in model["checks"]
|
||||
if c.get("depends_on_finding")
|
||||
}
|
||||
judged = json.loads(json.dumps(VERDICTS))
|
||||
for verdict in judged["checks"]:
|
||||
if verdict["check"] in waiting:
|
||||
verdict.update(verdict="not_verifiable", evidence="Waits on its finding.")
|
||||
built[VALIDATION_RESULTS] = {
|
||||
"content": "# Validation\n",
|
||||
"attachments": {
|
||||
VERDICTS_FILE: scenarios.merge(json.dumps(judged), walked, waiting),
|
||||
SCENARIOS_FILE: json.dumps(walked),
|
||||
},
|
||||
}
|
||||
return built
|
||||
|
||||
|
||||
def assembled(**options) -> str:
|
||||
return guide.assemble(DRAFT, delivered(**options), CATALOGUE)
|
||||
|
||||
|
||||
def section(text: str, heading: str) -> str:
|
||||
return text.split(f"## {heading}\n")[1].split("\n## ")[0]
|
||||
|
||||
|
||||
def test_the_guide_has_every_section_once_and_in_order():
|
||||
text = assembled()
|
||||
|
||||
headings = [line[3:] for line in text.splitlines() if line.startswith("## ")]
|
||||
assert headings == list(guide.SECTIONS)
|
||||
assert text.startswith("# Northwind landing zone\n")
|
||||
assert WORKFLOW.check_submission("operating_model_guide", text, {}) == ()
|
||||
|
||||
|
||||
def test_the_packagers_summary_is_kept_and_the_key_figures_are_set_under_it():
|
||||
summary = section(assembled(), guide.SUMMARY)
|
||||
|
||||
assert "The provider runs the platform at every hour" in summary
|
||||
assert "### Key figures" in summary
|
||||
assert "- Operated: 3 Items, with 2 Conditions to keep true." in summary
|
||||
assert (
|
||||
"- People: 5.1. Cost a month: 43,410 GBP, from rate card version "
|
||||
"`draft-2026-10-13` (placeholder prices: an illustration)." in summary
|
||||
)
|
||||
assert "- Scenario walk-throughs: 10 of 10 pass." in summary
|
||||
|
||||
|
||||
def test_a_figure_the_packager_wrote_under_another_heading_is_replaced():
|
||||
text = assembled()
|
||||
|
||||
assert "9,999" not in text
|
||||
assert "needs 2 people" not in text
|
||||
assert "Cost a month: 43,410 GBP." in section(text, "Staffing and cost")
|
||||
|
||||
|
||||
def test_each_section_is_written_from_the_data():
|
||||
text = assembled()
|
||||
|
||||
assert "| S-2 | Audit log store | storage | Cloud Storage | E-2 |" in section(
|
||||
text, "What is operated"
|
||||
)
|
||||
assert "| K-1 | S-1 | Flow logs are retained for 400 days. | CHK-12 |" in text
|
||||
assert "| OR-2 | process |" in section(text, "Operational requirements")
|
||||
assert (
|
||||
"| C-1 | Incident management | OR-1, OR-4 | S-1, S-2, S-3 | 24x7 | 60 h |"
|
||||
in section(text, "Capabilities")
|
||||
)
|
||||
who = section(text, "Who does what")
|
||||
assert (
|
||||
"| Platform SRE | Managed service provider (provider) | PL | senior | 24x7 |"
|
||||
in who
|
||||
)
|
||||
assert "| Incident management (C-1) | C | R | A |" in who
|
||||
checked = section(text, "Validation")
|
||||
assert "| OC-1 | A fault on the hub network is acknowledged" in checked
|
||||
assert "| Hub network (S-1) must be changed. | pass |" in checked
|
||||
|
||||
|
||||
def test_sources_are_cited_and_said_to_be_unchecked():
|
||||
cited = section(assembled(), "Sources")
|
||||
|
||||
assert "- ITIL 4: Incident management practice; Change enablement practice" in cited
|
||||
assert "certifies nothing against them" in cited
|
||||
assert "have not been checked against the frameworks by a person" in cited
|
||||
|
||||
|
||||
def test_what_was_left_open_is_listed_and_what_nobody_asked_for_is_not():
|
||||
left_open = section(assembled(), "Findings and open questions")
|
||||
|
||||
assert "No findings" in left_open
|
||||
assert (
|
||||
"`operations.change_process`: affects change enablement and release."
|
||||
in left_open
|
||||
)
|
||||
assert "`service_management.tooling`: Which service management" in left_open
|
||||
assert "metadata.author" not in left_open
|
||||
|
||||
|
||||
def test_a_finding_and_the_check_that_waits_on_it_are_shown():
|
||||
finding = {
|
||||
"id": "AF-1",
|
||||
"about": ["S-2"],
|
||||
"needed_by": ["C-3"],
|
||||
"lacks": "Nothing alerts when the retention lock is removed.",
|
||||
"proposed": "Add an alert on a change to the retention lock.",
|
||||
}
|
||||
waiting = json.loads(json.dumps(GRAPH))
|
||||
waiting["checks"][2]["depends_on_finding"] = "AF-1"
|
||||
|
||||
text = assembled(findings=[finding], model=waiting)
|
||||
|
||||
assert "**AF-1** (S-2): Nothing alerts when" in section(
|
||||
text, "Findings and open questions"
|
||||
)
|
||||
assert "| waits on finding (AF-1) |" in section(text, "Validation")
|
||||
assert "- Findings for whoever owns the design: 1." in text
|
||||
|
||||
|
||||
def test_assembling_a_guide_again_changes_nothing():
|
||||
first = assembled()
|
||||
|
||||
assert guide.assemble(first, delivered(), CATALOGUE) == first
|
||||
218
tests/test_outcomes.py
Normal file
218
tests/test_outcomes.py
Normal file
@@ -0,0 +1,218 @@
|
||||
"""Measures tied to the customer's business outcomes."""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from app.operating_model import digest, guide
|
||||
from app.operating_model import graph as model
|
||||
from app.operating_model.names import GRAPH_FILE
|
||||
from tests.support import CAPABILITIES, CARD, CATALOGUE, SPEC, as_file, graph, upstream
|
||||
|
||||
OUTCOMES = SPEC.replace(
|
||||
" - {id: BO-1, outcome: Outages of the platform are short., target: Restored within 4 hours.}\n"
|
||||
" - {id: BO-2, outcome: Regulators always get the evidence they ask for.}\n",
|
||||
" - {id: BO-1, outcome: New products reach customers sooner., target: Daily releases.}\n"
|
||||
" - {id: BO-2, outcome: Payments are never down for long.}\n",
|
||||
)
|
||||
# The same spec with no business outcome stated: only the customer can say none.
|
||||
NO_OUTCOMES = (
|
||||
OUTCOMES.split(" business_outcomes:\n")[0]
|
||||
+ " business_outcomes: not_applicable\n"
|
||||
+ " requirements:\n"
|
||||
+ OUTCOMES.split(" requirements:\n")[1]
|
||||
)
|
||||
check = model.check(CARD, CATALOGUE.frameworks)
|
||||
|
||||
|
||||
def measured() -> dict:
|
||||
changed = graph()
|
||||
changed["measures"] = [
|
||||
{
|
||||
"id": "M-1",
|
||||
"name": "Time to restore service",
|
||||
"layer": "outcome",
|
||||
"outcome": "BO-2",
|
||||
"measured_by": "C-1",
|
||||
"framework": "dora-metrics",
|
||||
"target": "Under 4 hours",
|
||||
},
|
||||
{
|
||||
"id": "M-2",
|
||||
"name": "Change lead time",
|
||||
"layer": "outcome",
|
||||
"outcome": "BO-1",
|
||||
"measured_by": "C-2",
|
||||
"framework": "dora-metrics",
|
||||
},
|
||||
]
|
||||
return changed
|
||||
|
||||
|
||||
def faults(changed, text=OUTCOMES, capabilities=None) -> list[str]:
|
||||
return check(
|
||||
as_file(GRAPH_FILE, changed), upstream(spec=text, capabilities=capabilities)
|
||||
)
|
||||
|
||||
|
||||
def test_every_outcome_shown_by_a_measure_is_accepted():
|
||||
assert faults(measured()) == []
|
||||
|
||||
|
||||
def test_with_no_outcomes_stated_no_measures_are_needed():
|
||||
unmeasured = graph()
|
||||
unmeasured["measures"] = []
|
||||
assert faults(unmeasured, NO_OUTCOMES) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "problem"),
|
||||
[
|
||||
(
|
||||
lambda g: g["measures"].pop(1),
|
||||
"business outcome BO-1 (New products reach customers sooner.) is shown by no Measure",
|
||||
),
|
||||
(
|
||||
lambda g: g["measures"][0].update(outcome="BO-9"),
|
||||
"Measure M-1 shows BO-9, which is not a business outcome the customer stated",
|
||||
),
|
||||
(
|
||||
lambda g: g["measures"][0].update(layer="vanity"),
|
||||
"Measure M-1: layer must be one of",
|
||||
),
|
||||
(
|
||||
lambda g: g["measures"][0].update(measured_by="C-9"),
|
||||
"Measure M-1 is measured by C-9, which is not a Capability",
|
||||
),
|
||||
(
|
||||
lambda g: g["measures"][0].update(framework="okr"),
|
||||
"Measure M-1 names framework okr, which is not in the register",
|
||||
),
|
||||
(
|
||||
lambda g: g["responsibilities"].pop(1),
|
||||
"Measure M-1 is measured by C-1, and nobody is responsible for it",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_a_measure_that_serves_no_stated_outcome_or_nobody_takes_is_refused(
|
||||
change, problem
|
||||
):
|
||||
changed = measured()
|
||||
change(changed)
|
||||
|
||||
found = faults(changed)
|
||||
|
||||
assert any(problem in fault for fault in found), found
|
||||
|
||||
|
||||
def test_a_measure_is_refused_when_the_customer_stated_no_outcomes():
|
||||
found = faults(measured(), NO_OUTCOMES)
|
||||
|
||||
assert any("none were stated" in fault for fault in found), found
|
||||
|
||||
|
||||
def test_measuring_how_work_flows_measures_all_three_layers():
|
||||
capabilities = [dict(c) for c in CAPABILITIES]
|
||||
capabilities[2]["catalogue"] = "delivery-and-developer-experience-measurement"
|
||||
changed = measured()
|
||||
changed["measures"].append(
|
||||
{
|
||||
"id": "M-3",
|
||||
"name": "Deployment frequency",
|
||||
"layer": "outcome",
|
||||
"outcome": "BO-1",
|
||||
"measured_by": "C-3",
|
||||
"framework": "dora-metrics",
|
||||
}
|
||||
)
|
||||
|
||||
found = faults(changed, capabilities=capabilities)
|
||||
|
||||
assert any(
|
||||
"C-3 measures how work flows and has no Measure at the guardrail, leading layer"
|
||||
in fault
|
||||
for fault in found
|
||||
), found
|
||||
|
||||
changed["measures"] += [
|
||||
{
|
||||
"id": "M-4",
|
||||
"name": "Developer satisfaction",
|
||||
"layer": "guardrail",
|
||||
"outcome": "BO-1",
|
||||
"measured_by": "C-3",
|
||||
"framework": "space-framework",
|
||||
},
|
||||
{
|
||||
"id": "M-5",
|
||||
"name": "Perceived build wait",
|
||||
"layer": "leading",
|
||||
"outcome": "BO-1",
|
||||
"measured_by": "C-3",
|
||||
"framework": "devex-framework",
|
||||
},
|
||||
]
|
||||
assert faults(changed, capabilities=capabilities) == []
|
||||
|
||||
|
||||
def test_the_guide_shows_each_outcome_with_what_shows_it_and_who_answers():
|
||||
built = upstream(spec=OUTCOMES, model=measured())
|
||||
|
||||
text = guide.business_outcomes(built, CATALOGUE)
|
||||
|
||||
assert (
|
||||
"**BO-1: New products reach customers sooner.** Target: Daily releases." in text
|
||||
)
|
||||
assert (
|
||||
"| Time to restore service | outcome | dora-metrics | Under 4 hours | "
|
||||
"Incident management | Service manager |" in text
|
||||
)
|
||||
|
||||
|
||||
def test_the_guide_says_when_no_outcome_was_stated():
|
||||
text = guide.business_outcomes(upstream(spec=NO_OUTCOMES), CATALOGUE)
|
||||
|
||||
assert text.startswith("The customer stated no business outcomes")
|
||||
|
||||
|
||||
def test_an_operational_requirement_may_be_traced_to_a_business_outcome():
|
||||
from app.operating_model import chain
|
||||
from app.operating_model.names import REQUIREMENTS_FILE
|
||||
from tests.support import REQUIREMENTS
|
||||
|
||||
requirements = json.loads(json.dumps(REQUIREMENTS))
|
||||
requirements[3]["source"] = ["BO-2", "service_levels.recovery_time_hours"]
|
||||
|
||||
assert (
|
||||
chain.check_requirements(
|
||||
as_file(REQUIREMENTS_FILE, requirements), upstream(spec=OUTCOMES)
|
||||
)
|
||||
== []
|
||||
)
|
||||
|
||||
|
||||
def test_the_architectures_stated_success_criteria_become_business_outcomes():
|
||||
from pathlib import Path
|
||||
|
||||
fixture = (
|
||||
Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone"
|
||||
)
|
||||
sources = digest.read_folder(fixture)
|
||||
scope = json.loads(sources[digest.SCOPE])
|
||||
scope["stated"]["intent"]["success_criteria"] = [
|
||||
"Payments engine onboarded within 10 weeks.",
|
||||
"Audit findings closed before go-live.",
|
||||
]
|
||||
sources[digest.SCOPE] = json.dumps(scope)
|
||||
|
||||
digested = digest.architecture(
|
||||
sources, digest.item_kinds(digest.Path(fixture).parents[3], "gcp")
|
||||
)
|
||||
|
||||
assert digested.spec["intent"]["business_outcomes"] == [
|
||||
{"id": "BO-1", "outcome": "Payments engine onboarded within 10 weeks."},
|
||||
{"id": "BO-2", "outcome": "Audit findings closed before go-live."},
|
||||
]
|
||||
# In the real run they were assumed for the customer, so none are carried.
|
||||
real = digest.architecture(digest.read_folder(fixture), {})
|
||||
assert "business_outcomes" not in real.spec["intent"]
|
||||
219
tests/test_scenarios.py
Normal file
219
tests/test_scenarios.py
Normal file
@@ -0,0 +1,219 @@
|
||||
"""Scenario walk-throughs: generated and traced by code, in place of a dry run."""
|
||||
|
||||
import json
|
||||
|
||||
from app.operating_model import scenarios, spec, staffing
|
||||
from app.operating_model.names import VERDICTS_FILE
|
||||
from tests.support import (
|
||||
CAPABILITIES,
|
||||
CARD,
|
||||
CATALOGUE,
|
||||
GRAPH,
|
||||
SPEC,
|
||||
VERDICTS,
|
||||
graph,
|
||||
scope_data,
|
||||
)
|
||||
|
||||
|
||||
def walked(model=GRAPH, capabilities=CAPABILITIES, text=SPEC) -> dict[str, dict]:
|
||||
scope = scope_data(text)
|
||||
currency = spec.value(scope, "commercial.currency")
|
||||
cost = staffing.compute(model, capabilities, CARD, currency)
|
||||
return {
|
||||
item["id"]: item
|
||||
for item in scenarios.walk(model, capabilities, scope, CATALOGUE, cost)
|
||||
}
|
||||
|
||||
|
||||
def test_every_scenario_of_a_sound_model_has_a_path():
|
||||
found = walked()
|
||||
|
||||
assert set(found) == {
|
||||
"SC-fault-S-1",
|
||||
"SC-fault-S-2",
|
||||
"SC-fault-S-3",
|
||||
"SC-change-S-1",
|
||||
"SC-change-S-2",
|
||||
"SC-change-S-3",
|
||||
"SC-condition-K-1",
|
||||
"SC-condition-K-2",
|
||||
"SC-party-P-2",
|
||||
"SC-budget",
|
||||
}
|
||||
assert all(item["passes"] for item in found.values())
|
||||
assert found["SC-fault-S-1"]["path"] == (
|
||||
"Platform SRE responds under C-1, and Service manager is answerable."
|
||||
)
|
||||
assert found["SC-budget"]["path"] == "The cost is 43,410 GBP, within the budget."
|
||||
|
||||
|
||||
def test_a_fault_out_of_hours_fails_when_nobody_responsible_is_on():
|
||||
model = graph()
|
||||
model["roles"][1]["hours"] = "business"
|
||||
|
||||
found = walked(model)
|
||||
|
||||
assert not found["SC-fault-S-1"]["passes"]
|
||||
assert found["SC-fault-S-1"]["path"] == (
|
||||
"No Role responsible for responding works 24x7 hours."
|
||||
)
|
||||
# Changes are made in business hours: that path still holds.
|
||||
assert found["SC-change-S-1"]["passes"]
|
||||
|
||||
|
||||
def test_an_item_no_responding_capability_covers_fails():
|
||||
capabilities = [dict(c) for c in CAPABILITIES]
|
||||
capabilities[0]["items"] = ["S-1", "S-2"]
|
||||
|
||||
found = walked(capabilities=capabilities)
|
||||
|
||||
assert (
|
||||
found["SC-fault-S-3"]["path"]
|
||||
== "No Capability that responds to faults covers it."
|
||||
)
|
||||
assert found["SC-fault-S-1"]["passes"]
|
||||
|
||||
|
||||
def test_an_item_nobody_can_change_fails():
|
||||
capabilities = [dict(c) for c in CAPABILITIES]
|
||||
capabilities[1]["items"] = ["S-1", "S-2"]
|
||||
|
||||
assert not walked(capabilities=capabilities)["SC-change-S-3"]["passes"]
|
||||
|
||||
|
||||
def test_a_condition_nothing_keeps_true_fails():
|
||||
model = graph()
|
||||
model["checks"].pop(1)
|
||||
|
||||
found = walked(model)
|
||||
|
||||
assert found["SC-condition-K-1"]["path"] == "No Check carries it."
|
||||
assert found["SC-condition-K-2"]["passes"]
|
||||
|
||||
|
||||
def test_a_party_with_nobody_else_to_turn_to_fails():
|
||||
model = graph()
|
||||
model["responsibilities"] = [
|
||||
r for r in model["responsibilities"] if r["id"] not in ("RS-3", "RS-8")
|
||||
]
|
||||
|
||||
found = walked(model)
|
||||
|
||||
assert not found["SC-party-P-2"]["passes"]
|
||||
assert "C-1, C-3" in found["SC-party-P-2"]["path"]
|
||||
|
||||
|
||||
def test_a_cost_over_the_budget_fails_the_budget_scenario():
|
||||
tight = SPEC.replace("monthly_run_budget: 120000", "monthly_run_budget: 40000")
|
||||
|
||||
found = walked(text=tight)
|
||||
|
||||
assert found["SC-budget"]["path"] == "The cost is 43,410 GBP, over the budget."
|
||||
|
||||
|
||||
def test_with_no_budget_stated_there_is_no_budget_scenario():
|
||||
assert "SC-budget" not in walked(
|
||||
text=SPEC.replace(" monthly_run_budget: 120000\n", "")
|
||||
)
|
||||
|
||||
|
||||
def test_the_scenarios_verdicts_are_written_into_the_validators_file_by_code():
|
||||
found = list(walked().values())
|
||||
|
||||
merged = json.loads(scenarios.merge(json.dumps(VERDICTS), found))
|
||||
|
||||
assert merged["dry_run"] == {
|
||||
"verdict": "pass",
|
||||
"evidence": "10 of 10 scenario walk-throughs pass.",
|
||||
}
|
||||
assert [c["check"] for c in merged["checks"]][:4] == [
|
||||
"OC-1",
|
||||
"OC-2",
|
||||
"OC-3",
|
||||
"OC-4",
|
||||
]
|
||||
assert len(merged["checks"]) == 14
|
||||
# Merging again replaces the scenarios' verdicts; it never adds them twice.
|
||||
again = json.loads(scenarios.merge(json.dumps(merged), found))
|
||||
assert again == merged
|
||||
|
||||
|
||||
def test_a_failed_scenario_fails_the_run():
|
||||
model = graph()
|
||||
model["roles"][1]["hours"] = "business"
|
||||
|
||||
merged = json.loads(
|
||||
scenarios.merge(json.dumps(VERDICTS), list(walked(model).values()))
|
||||
)
|
||||
|
||||
assert merged["dry_run"]["verdict"] == "fail"
|
||||
failed = [c["check"] for c in merged["checks"] if c["verdict"] == "fail"]
|
||||
assert failed == ["SC-fault-S-1", "SC-fault-S-2", "SC-fault-S-3"]
|
||||
|
||||
|
||||
def verdict_faults(verdicts, model=GRAPH) -> list[str]:
|
||||
return scenarios.check_verdicts(
|
||||
{VERDICTS_FILE: json.dumps(verdicts)}, model, VERDICTS_FILE
|
||||
)
|
||||
|
||||
|
||||
def test_a_verdict_on_every_check_is_accepted():
|
||||
assert verdict_faults(VERDICTS) == []
|
||||
|
||||
|
||||
def test_a_check_without_a_verdict_is_refused():
|
||||
missing = {"checks": VERDICTS["checks"][:-1]}
|
||||
|
||||
assert verdict_faults(missing) == ["verdicts.json: Check OC-4 has no verdict"]
|
||||
|
||||
|
||||
def test_a_check_that_waits_on_a_finding_cannot_pass():
|
||||
model = graph()
|
||||
model["checks"][2]["depends_on_finding"] = "AF-1"
|
||||
|
||||
assert (
|
||||
"Check OC-3 depends on finding AF-1. It is not_verifiable while the finding stands"
|
||||
in verdict_faults(VERDICTS, model)[0]
|
||||
)
|
||||
|
||||
waiting = json.loads(json.dumps(VERDICTS))
|
||||
waiting["checks"][2]["verdict"] = "not_verifiable"
|
||||
assert verdict_faults(waiting, model) == []
|
||||
|
||||
|
||||
def test_a_verdict_on_something_that_is_not_a_check_is_refused():
|
||||
extra = {
|
||||
"checks": [
|
||||
*VERDICTS["checks"],
|
||||
{"check": "OC-9", "verdict": "pass", "evidence": "x"},
|
||||
]
|
||||
}
|
||||
|
||||
assert verdict_faults(extra) == [
|
||||
"verdicts.json: OC-9 is not a Check of the Target operating model"
|
||||
]
|
||||
|
||||
|
||||
def test_a_check_whose_finding_the_model_could_meet_itself_may_fail():
|
||||
model = graph()
|
||||
model["checks"][2]["depends_on_finding"] = "AF-1"
|
||||
judged = json.loads(json.dumps(VERDICTS))
|
||||
judged["checks"][2].update(
|
||||
verdict="fail", evidence="Staff the review 24x7; this is not the solution's."
|
||||
)
|
||||
|
||||
assert verdict_faults(judged, model) == []
|
||||
merged = json.loads(scenarios.merge(json.dumps(judged), [], {"OC-3": "AF-1"}))
|
||||
# The failure stands as a failure: the router sends the model back.
|
||||
assert merged["checks"][2]["verdict"] == "fail"
|
||||
|
||||
|
||||
def test_a_check_left_not_verifiable_on_its_finding_is_recorded_as_waiting():
|
||||
judged = json.loads(json.dumps(VERDICTS))
|
||||
judged["checks"][2].update(verdict="not_verifiable", evidence="Waits on AF-1.")
|
||||
|
||||
merged = json.loads(scenarios.merge(json.dumps(judged), [], {"OC-3": "AF-1"}))
|
||||
|
||||
assert merged["checks"][2]["verdict"] == "waits_on_finding"
|
||||
assert merged["checks"][2]["finding"] == "AF-1"
|
||||
161
tests/test_staffing.py
Normal file
161
tests/test_staffing.py
Normal file
@@ -0,0 +1,161 @@
|
||||
"""Staffing and cost: arithmetic on the rate card, never an agent's figure."""
|
||||
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
||||
from app.operating_model import staffing
|
||||
from app.operating_model.staffing import RateCardError
|
||||
from app.operating_model.workflow import ROOT
|
||||
from tests.support import BUNDLE, CAPABILITIES, CARD, GRAPH, graph
|
||||
|
||||
SKILL = ROOT / "skills" / staffing.SKILL
|
||||
|
||||
|
||||
def by_role(result) -> dict:
|
||||
return {row["role"]: row for row in result["roles"]}
|
||||
|
||||
|
||||
def test_the_figures_for_a_fixed_graph_and_card_are_reproduced_exactly():
|
||||
result = staffing.compute(GRAPH, CAPABILITIES, CARD, "GBP")
|
||||
rows = by_role(result)
|
||||
|
||||
# The SRE is responsible for incidents (60 h, 24x7) and changes (40 h): 100 h
|
||||
# is two thirds of one person in Poland, but every hour of every day needs a
|
||||
# rota of five. 5 x 460 a day x 220 days / 12 x 1.25 uplift x 0.79 to GBP.
|
||||
assert rows["R-2"]["effort_hours_per_month"] == 100
|
||||
assert rows["R-2"]["hours"] == "24x7"
|
||||
assert rows["R-2"]["people"] == 5.0
|
||||
assert rows["R-2"]["cost_per_month"] == 41640
|
||||
# The service manager: 16 h of a 137.5 h month in the United Kingdom.
|
||||
assert rows["R-3"]["people"] == 0.1
|
||||
assert rows["R-3"]["cost_per_month"] == 1770
|
||||
assert result["cost_per_month"] == 43410
|
||||
assert result["by_party"] == {"P-2": 43410}
|
||||
assert result["by_country"] == {"PL": 41640, "GB": 1770}
|
||||
assert result["by_capability"] == {"C-1": 24984, "C-2": 16656, "C-3": 1770}
|
||||
|
||||
|
||||
def test_the_customers_own_staff_are_counted_and_never_priced():
|
||||
rows = by_role(staffing.compute(GRAPH, CAPABILITIES, CARD, "GBP"))
|
||||
|
||||
assert rows["R-1"]["cost_per_month"] is None
|
||||
assert rows["R-1"]["people"] == 0.0
|
||||
|
||||
|
||||
def test_a_customer_role_that_does_the_work_still_carries_no_cost():
|
||||
model = graph()
|
||||
model["responsibilities"][4]["role"] = "R-1" # the customer makes the changes
|
||||
|
||||
rows = by_role(staffing.compute(model, CAPABILITIES, CARD, "GBP"))
|
||||
|
||||
assert rows["R-1"]["effort_hours_per_month"] == 40
|
||||
assert rows["R-1"]["people"] == 0.3
|
||||
assert rows["R-1"]["cost_per_month"] is None
|
||||
|
||||
|
||||
def test_a_role_on_business_hours_is_as_small_as_its_work():
|
||||
model = graph()
|
||||
model["roles"][1]["hours"] = "business"
|
||||
capabilities = [dict(c, hours="business") for c in CAPABILITIES]
|
||||
|
||||
row = by_role(staffing.compute(model, capabilities, CARD, "GBP"))["R-2"]
|
||||
|
||||
assert row["hours"] == "business"
|
||||
assert row["people"] == 0.7 # 100 h of a 146.7 h month, and no rota
|
||||
|
||||
|
||||
def test_the_result_is_in_the_currency_the_spec_states():
|
||||
pounds = staffing.compute(GRAPH, CAPABILITIES, CARD, "GBP")
|
||||
dollars = staffing.compute(GRAPH, CAPABILITIES, CARD, "USD")
|
||||
|
||||
assert dollars["cost_per_month"] == 54948
|
||||
assert pounds["conversion"] == {"from": "USD", "rate": 0.79, "as_of": "2026-10-13"}
|
||||
assert dollars["conversion"]["rate"] == 1.0
|
||||
|
||||
|
||||
def test_roles_that_share_a_capability_divide_its_effort():
|
||||
model = graph()
|
||||
model["responsibilities"].append(
|
||||
{"id": "RS-9", "capability": "C-3", "role": "R-2", "kind": "responsible"}
|
||||
)
|
||||
|
||||
equal = by_role(staffing.compute(model, CAPABILITIES, CARD, "GBP"))
|
||||
model["responsibilities"][6]["share"] = 0.75
|
||||
model["responsibilities"][-1]["share"] = 0.25
|
||||
stated = by_role(staffing.compute(model, CAPABILITIES, CARD, "GBP"))
|
||||
|
||||
assert equal["R-2"]["effort_hours_per_month"] == 108 # 100 and half of 16
|
||||
assert stated["R-2"]["effort_hours_per_month"] == 104 # 100 and a quarter of 16
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("shares", "problem"),
|
||||
[
|
||||
((0.75, None), "must all state a share"),
|
||||
((0.75, 0.5), "do not add up to 1"),
|
||||
((1.5, -0.5), "must all state a share"),
|
||||
],
|
||||
)
|
||||
def test_shares_that_do_not_hold_together_are_a_fault(shares, problem):
|
||||
entries = [
|
||||
{"capability": "C-3", "role": role, "kind": "responsible", "share": share}
|
||||
for role, share in zip(("R-2", "R-3"), shares, strict=True)
|
||||
]
|
||||
|
||||
assert problem in staffing.shares(entries)[1][0]
|
||||
|
||||
|
||||
def test_the_card_in_the_repository_is_marked_as_placeholders():
|
||||
assert CARD.is_draft
|
||||
assert CARD.prices == ("provider",)
|
||||
|
||||
|
||||
def test_the_currencies_a_spec_may_state_are_the_ones_the_card_converts_to():
|
||||
assert list(BUNDLE.scope_values["commercial.currency"]) == CARD.currencies()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def skill(tmp_path):
|
||||
shutil.copytree(SKILL, tmp_path / "skill")
|
||||
return tmp_path / "skill"
|
||||
|
||||
|
||||
def edit(skill, change):
|
||||
path = skill / "assets" / "rates.yaml"
|
||||
data = yaml.safe_load(path.read_text())
|
||||
change(data)
|
||||
path.write_text(yaml.safe_dump(data))
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "problem"),
|
||||
[
|
||||
(
|
||||
lambda d: d["prices"].append("customer"),
|
||||
"the customer's own staff are never priced",
|
||||
),
|
||||
(
|
||||
lambda d: d["countries"]["PL"]["day_rate"].pop("lead"),
|
||||
"PL has no day rate for level lead",
|
||||
),
|
||||
(lambda d: d["cover"].pop("24x7"), "cover for 24x7 needs a minimum_roster"),
|
||||
(
|
||||
lambda d: d["conversion"]["rates"].update(GBP=0),
|
||||
"conversion rate for GBP must be a number above zero",
|
||||
),
|
||||
(lambda d: d.pop("currency"), "needs currency"),
|
||||
],
|
||||
)
|
||||
def test_a_rate_card_that_does_not_hold_together_is_refused(skill, change, problem):
|
||||
edit(skill, change)
|
||||
|
||||
with pytest.raises(RateCardError, match=problem):
|
||||
staffing.load(skill)
|
||||
|
||||
|
||||
def test_a_card_whose_version_is_set_is_no_longer_a_draft(skill):
|
||||
edit(skill, lambda d: d.update(version="2026-11"))
|
||||
|
||||
assert not staffing.load(skill).is_draft
|
||||
207
tests/test_teams.py
Normal file
207
tests/test_teams.py
Normal file
@@ -0,0 +1,207 @@
|
||||
"""Teams and how they work together: Team Topologies in the operating model graph."""
|
||||
|
||||
import pytest
|
||||
|
||||
from app.operating_model import graph as model
|
||||
from app.operating_model import guide
|
||||
from app.operating_model.names import GRAPH_FILE
|
||||
from tests.support import (
|
||||
CARD,
|
||||
CATALOGUE,
|
||||
GRAPH,
|
||||
SPEC,
|
||||
as_file,
|
||||
graph,
|
||||
scope_data,
|
||||
upstream,
|
||||
)
|
||||
|
||||
check = model.check(CARD)
|
||||
|
||||
|
||||
def with_teams() -> dict:
|
||||
"""The test model, with the provider's Roles in a platform team that the
|
||||
customer's platform owner consumes as a service."""
|
||||
teamed = graph()
|
||||
teamed["teams"] = [
|
||||
{
|
||||
"id": "T-1",
|
||||
"party": "P-1",
|
||||
"name": "Northwind product squad",
|
||||
"type": "stream_aligned",
|
||||
},
|
||||
{
|
||||
"id": "T-2",
|
||||
"party": "P-2",
|
||||
"name": "Platform team",
|
||||
"type": "platform",
|
||||
"purpose": "Runs the landing zone as a product.",
|
||||
},
|
||||
{"id": "T-3", "party": "P-2", "name": "Adoption", "type": "enabling"},
|
||||
]
|
||||
teamed["roles"][0]["team"] = "T-1"
|
||||
teamed["roles"][1]["team"] = "T-2"
|
||||
teamed["roles"][2]["team"] = "T-2"
|
||||
teamed["interactions"] = [
|
||||
{
|
||||
"id": "I-1",
|
||||
"from": "T-1",
|
||||
"to": "T-2",
|
||||
"mode": "x_as_a_service",
|
||||
"purpose": "Environments through the self-service portal.",
|
||||
},
|
||||
{"id": "I-2", "from": "T-3", "to": "T-1", "mode": "facilitating"},
|
||||
{
|
||||
"id": "I-3",
|
||||
"from": "T-1",
|
||||
"to": "T-2",
|
||||
"mode": "collaboration",
|
||||
"until": "the first workload is live",
|
||||
},
|
||||
]
|
||||
return teamed
|
||||
|
||||
|
||||
def faults(changed) -> list[str]:
|
||||
return check(as_file(GRAPH_FILE, changed), upstream())
|
||||
|
||||
|
||||
def test_teams_are_optional():
|
||||
assert faults(GRAPH) == []
|
||||
|
||||
|
||||
def test_teams_of_the_four_kinds_working_in_the_three_modes_are_accepted():
|
||||
assert faults(with_teams()) == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("change", "problem"),
|
||||
[
|
||||
(
|
||||
lambda g: g["teams"][1].update(type="squad"),
|
||||
"Team T-2: type must be one of stream_aligned, platform",
|
||||
),
|
||||
(
|
||||
lambda g: g["teams"][1].update(party="P-9"),
|
||||
"Team T-2 is in P-9, which is not a Party",
|
||||
),
|
||||
(
|
||||
lambda g: g["roles"][1].update(team="T-9"),
|
||||
"Role R-2 is in T-9, which is not a Team",
|
||||
),
|
||||
(
|
||||
lambda g: g["roles"][0].update(team="T-2"),
|
||||
"Role R-1 is in P-1, and its team T-2 is in P-2",
|
||||
),
|
||||
(
|
||||
lambda g: g["interactions"][0].update(mode="ticket"),
|
||||
"Interaction I-1: mode must be one of",
|
||||
),
|
||||
(
|
||||
lambda g: g["interactions"][0].update(to="T-9"),
|
||||
"Interaction I-1: to T-9 is not a Team",
|
||||
),
|
||||
(
|
||||
lambda g: g["interactions"][0].update(to="T-1"),
|
||||
"Interaction I-1 is between a team and itself",
|
||||
),
|
||||
(
|
||||
lambda g: g["interactions"][2].pop("until"),
|
||||
"Interaction I-3 is a collaboration and does not say when it ends",
|
||||
),
|
||||
(lambda g: g["teams"][0].pop("type"), "(T-1) needs type"),
|
||||
],
|
||||
)
|
||||
def test_teams_that_do_not_hold_together_are_refused(change, problem):
|
||||
changed = with_teams()
|
||||
change(changed)
|
||||
|
||||
found = faults(changed)
|
||||
|
||||
assert any(problem in fault for fault in found), found
|
||||
|
||||
|
||||
def test_a_check_may_be_on_a_team_or_an_interaction():
|
||||
changed = with_teams()
|
||||
changed["checks"].append(
|
||||
{
|
||||
"id": "OC-9",
|
||||
"on": "I-1",
|
||||
"source": "OR-2",
|
||||
"condition": "Every environment request is met through the portal, with no ticket.",
|
||||
}
|
||||
)
|
||||
|
||||
assert faults(changed) == []
|
||||
|
||||
|
||||
def test_the_chart_draws_each_team_inside_its_party_and_how_teams_work_together():
|
||||
chart = model.render_organisation(with_teams())
|
||||
|
||||
assert ' subgraph T_2["Platform team (platform team)"]' in chart
|
||||
assert ' R_2["Platform SRE<br/>PL, senior, 24x7"]' in chart
|
||||
assert ' T_1 -- "x-as-a-service" --> T_2' in chart
|
||||
assert ' T_1 -- "collaboration until the first workload is live" --> T_2' in chart
|
||||
|
||||
|
||||
def test_the_chart_without_teams_is_as_it_was():
|
||||
chart = model.render_organisation(GRAPH)
|
||||
|
||||
assert " R_2[" in chart
|
||||
assert " -- " not in chart
|
||||
|
||||
|
||||
def test_the_guide_shows_the_teams_and_their_interactions():
|
||||
built = upstream(model=with_teams())
|
||||
text = guide.who_does_what(built, CATALOGUE)
|
||||
|
||||
assert (
|
||||
"| Platform team | platform | Managed service provider | Platform SRE, Service manager | Runs the landing zone as a product. |"
|
||||
in text
|
||||
)
|
||||
assert (
|
||||
"| Northwind product squad | Platform team | x-as-a-service | | Environments through the self-service portal. |"
|
||||
in text
|
||||
)
|
||||
# The matrix still follows.
|
||||
assert "| Incident management (C-1) |" in text
|
||||
|
||||
|
||||
def test_a_platform_foundation_is_proposed_the_platform_and_measurement_capabilities():
|
||||
proposed = CATALOGUE.applicable(scope_data(SPEC))
|
||||
|
||||
assert {
|
||||
"platform-product-management",
|
||||
"self-service-and-golden-paths",
|
||||
"enablement",
|
||||
"delivery-and-developer-experience-measurement",
|
||||
} <= set(proposed)
|
||||
|
||||
|
||||
def test_a_workload_is_not_proposed_the_platform_capabilities():
|
||||
workload = SPEC.replace("kind: platform_foundation", "kind: workload")
|
||||
|
||||
proposed = CATALOGUE.applicable(scope_data(workload))
|
||||
|
||||
assert "platform-product-management" not in proposed
|
||||
assert "delivery-and-developer-experience-measurement" in proposed
|
||||
|
||||
|
||||
def test_measurement_cites_three_layers_and_none_alone():
|
||||
entry = CATALOGUE.entries["delivery-and-developer-experience-measurement"]
|
||||
|
||||
assert [s["framework"] for s in entry.sources] == [
|
||||
"dora-metrics",
|
||||
"space-framework",
|
||||
"devex-framework",
|
||||
]
|
||||
assert len(entry.typical_checks) == 3
|
||||
|
||||
|
||||
def test_the_socio_technical_frameworks_are_registered():
|
||||
names = {key: value["name"] for key, value in CATALOGUE.frameworks.items()}
|
||||
|
||||
assert names["team-topologies"].startswith("Team Topologies")
|
||||
assert "platform as a product" in names["platform-as-a-product"]
|
||||
assert names["space-framework"] == "The SPACE of Developer Productivity"
|
||||
assert names["devex-framework"].startswith("DevEx")
|
||||
Reference in New Issue
Block a user