feat: working TOM agent

This commit is contained in:
2026-10-21 19:47:12 +01:00
commit efe543e6e2
83 changed files with 11779 additions and 0 deletions

0
tests/__init__.py Normal file
View File

5
tests/conftest.py Normal file
View File

@@ -0,0 +1,5 @@
# Env var setup that must run before the engine's settings are imported.
import os
# The suite must not pick up credentials from a developer's .env.
os.environ["ENV_FILE_EXPORT"] = "false"

7
tests/fixtures/architecture/README.md vendored Normal file
View File

@@ -0,0 +1,7 @@
# Architecture Executions, as fixtures
`case-08-landing-zone/` is the data files of one completed Execution of the solution architecture agent, copied unaltered from its evaluation results (run `20261011T185421Z`, at its commit `8b397cd`). The case is that agent's fictional benchmark landing zone; no customer's data is in it.
It is the contract between the two agents: the digest's tests read it, so a change to the shape of these files is caught here. Replace it by copying a newer Execution's files, never by editing these.
Not copied: the prose Deliverables, the Terraform, the solution guide and the working material. The digest reads none of them.

View File

@@ -0,0 +1,79 @@
[
{
"id": "C-1",
"name": "Resource Hierarchy and Environment Boundary Manager",
"requirements": [
"FR-1",
"NFR-1",
"NFR-8",
"NFR-9"
]
},
{
"id": "C-2",
"name": "Dedicated Private Hybrid Connectivity Gateway",
"requirements": [
"FR-2",
"NFR-3",
"NFR-14"
]
},
{
"id": "C-3",
"name": "Centralised Outbound Traffic Inspection and Egress Boundary",
"requirements": [
"FR-3",
"NFR-2",
"NFR-4"
]
},
{
"id": "C-4",
"name": "Centralised Immutable Audit Event and Compliance Log Repository",
"requirements": [
"FR-4",
"NFR-7",
"NFR-8"
]
},
{
"id": "C-5",
"name": "Automated Infrastructure Orchestration and Deployment Engine",
"requirements": [
"FR-5",
"NFR-10"
]
},
{
"id": "C-6",
"name": "Federated Identity and Role Governance Authority",
"requirements": [
"FR-6",
"NFR-5"
]
},
{
"id": "C-7",
"name": "Cryptographic Key Lifecycle and Encryption Controller",
"requirements": [
"NFR-6",
"NFR-8"
]
},
{
"id": "C-8",
"name": "Platform Telemetry and Operational Alerting Engine",
"requirements": [
"NFR-11",
"NFR-12",
"NFR-15"
]
},
{
"id": "C-9",
"name": "Financial Governance and Budget Enforcement Monitor",
"requirements": [
"NFR-13"
]
}
]

View File

@@ -0,0 +1,140 @@
{
"customer_case": "case-08-landing-zone",
"execution_id": "case-08-landing-zone-185426",
"session_id": "cdd6bfe7-2bd8-44af-8e15-9dcfa869eb84",
"intent_version": 1,
"workflow_request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.",
"baseline": null,
"overlay_skills": {},
"terraform_modules": false,
"scope": {
"status": "BASELINE_LOCKED",
"readiness": {
"supplied": 44,
"answered": 0,
"assumed": 12,
"unknown": 4,
"not_applicable": 3
}
},
"delivery_plan": {
"work_packages": 9,
"activities": 15,
"working_days": 36.0,
"weeks": 7.2,
"team_size": 3,
"window_weeks": 10,
"within_window": true,
"critical_path": [
"A-6",
"A-10",
"A-13",
"A-14",
"A-15"
]
},
"status": "completed",
"warnings": [],
"risks": [],
"assessment": {
"recommendations_quality": {
"specific": 0.91,
"actionable": 0.79,
"trade_offs": 0.95,
"traceable": 0.73
},
"mean": 0.84,
"meets_bar": true
},
"phases": {
"scoping": "approved",
"requirements_discovery": "approved",
"solution_architecture": "approved",
"solution_implementation": "approved",
"solution_validation": "approved",
"packaging": "approved"
},
"solution_guide": "solution-architecture-guide.md",
"customer_questions": "customer-questions.md",
"changes": null,
"deliverables": {
"scope": {
"document": "deliverables/scope.md",
"attachments": [
"scope.json",
"scope.yaml"
],
"status": "submitted"
},
"requirements": {
"document": "deliverables/requirements.md",
"attachments": [
"requirements.json"
],
"status": "submitted"
},
"technical_decomposition": {
"document": "deliverables/technical_decomposition.md",
"attachments": [
"components.json"
],
"status": "submitted"
},
"product_selection": {
"document": "deliverables/product_selection.md",
"attachments": [
"products.json"
],
"status": "submitted"
},
"design_recommendations": {
"document": "deliverables/design_recommendations.md",
"attachments": [
"recommendations.json"
],
"status": "submitted"
},
"target_architecture": {
"document": "deliverables/target_architecture.md",
"attachments": [
"architecture.mmd",
"target-architecture.json"
],
"status": "submitted"
},
"architecture_description": {
"document": "deliverables/architecture_description.md",
"attachments": [],
"status": "submitted"
},
"deployment_guidance": {
"document": "deliverables/deployment_guidance.md",
"attachments": [
"implementation.json",
"terraform/main.tf",
"terraform/variables.tf",
"terraform/versions.tf"
],
"status": "submitted"
},
"validation_results": {
"document": "deliverables/validation_results.md",
"attachments": [
"verdicts.json"
],
"status": "submitted"
},
"work_breakdown": {
"document": "deliverables/work_breakdown.md",
"attachments": [
"work-breakdown.json"
],
"status": "submitted"
},
"solution_guide": {
"document": "deliverables/solution_guide.md",
"attachments": [],
"status": "submitted"
}
}
}

View File

@@ -0,0 +1,38 @@
[
{
"component": "C-1",
"product": "Resource Manager"
},
{
"component": "C-2",
"product": "Cloud Interconnect"
},
{
"component": "C-3",
"product": "Palo Alto Networks VM-Series"
},
{
"component": "C-4",
"product": "Cloud Storage"
},
{
"component": "C-5",
"product": "GitHub Actions"
},
{
"component": "C-6",
"product": "IAM Workload Identity Federation"
},
{
"component": "C-7",
"product": "Cloud KMS"
},
{
"component": "C-8",
"product": "Cloud Monitoring"
},
{
"component": "C-9",
"product": "Cloud Billing"
}
]

View File

@@ -0,0 +1,248 @@
[
{
"id": "DR-SEC-1",
"pillar": "security",
"text": "Implement a VPC Service Controls perimeter around all Cardholder Data Environment (CDE) projects and the centralized audit log buckets.",
"components": [
"C-1",
"C-4"
],
"conditions": [
"A VPC Service Controls perimeter encloses the CDE and audit log projects.",
"The VPC Service Controls perimeter restricts ingress and egress to authorized traffic."
],
"outside_code": []
},
{
"id": "DR-SEC-2",
"pillar": "security",
"text": "Apply Organization Policy constraints to disable default network creation, deny external IPs, prohibit service account key creation, and restrict resource locations.",
"components": [
"C-1"
],
"conditions": [
"The compute.skipDefaultNetworkCreation constraint is enforced.",
"The compute.vmExternalIpAccess constraint is enforced to deny all.",
"The iam.disableServiceAccountKeyCreation constraint is enforced.",
"The gcp.resourceLocations constraint restricts resources to europe-west2 and europe-west1."
],
"outside_code": []
},
{
"id": "DR-SEC-3",
"pillar": "security",
"text": "Enable Confidential Computing on the payment engine Compute Engine instances to encrypt data in-use.",
"components": [],
"conditions": [
"Confidential Computing is enabled on the core payment engine Compute Engine instances."
],
"outside_code": []
},
{
"id": "DR-SEC-4",
"pillar": "security",
"text": "Deploy Sensitive Data Protection (Cloud DLP) inspection templates to detect and mask PAN data.",
"components": [
"C-4"
],
"conditions": [
"Sensitive Data Protection (Cloud DLP) inspection templates are configured to detect and mask PAN data."
],
"outside_code": []
},
{
"id": "DR-SEC-5",
"pillar": "security",
"text": "Enable Security Command Center at the organization level with Event and Container Threat Detection.",
"components": [
"C-1",
"C-8"
],
"conditions": [
"Security Command Center is activated at the organization level.",
"Event Threat Detection and Container Threat Detection are enabled in SCC."
],
"outside_code": []
},
{
"id": "DR-SEC-6",
"pillar": "security",
"text": "Implement event-driven automated responses using Eventarc and Cloud Run functions for SCC findings.",
"components": [
"C-8"
],
"conditions": [
"Eventarc is configured to route high-severity SCC findings.",
"Cloud Run functions are configured to execute automated remediation on Eventarc triggers."
],
"outside_code": []
},
{
"id": "DR-REL-1",
"pillar": "reliability",
"text": "Provision redundant Dedicated Cloud Interconnect circuits across distinct physical edge colocation facilities, terminating on redundant Cloud Routers with BFD.",
"components": [
"C-2"
],
"conditions": [
"Dedicated Cloud Interconnect circuits are provisioned across redundant edge colocation facilities.",
"Cloud Interconnects terminate on redundant Cloud Routers in each region.",
"Bidirectional Forwarding Detection (BFD) is enabled on the Cloud Router BGP sessions."
],
"outside_code": []
},
{
"id": "DR-REL-2",
"pillar": "reliability",
"text": "Deploy Palo Alto VM-Series firewalls in a regional Managed Instance Group across at least three zones, fronted by an Internal Passthrough Network Load Balancer.",
"components": [
"C-3"
],
"conditions": [
"Palo Alto VM-Series firewalls are deployed in a regional Managed Instance Group (MIG) across at least three zones.",
"An Internal Passthrough Network Load Balancer routes traffic to the VM-Series MIG.",
"Health checks are configured on the internal load balancer to probe firewall health."
],
"outside_code": []
},
{
"id": "DR-REL-3",
"pillar": "reliability",
"text": "Provision dedicated regional Cloud KMS key rings in both europe-west2 and europe-west1, granting workload identity service accounts Decrypter roles on both.",
"components": [
"C-7",
"C-6"
],
"conditions": [
"Regional Cloud KMS key rings are provisioned in both europe-west2 and europe-west1.",
"Workload Identity Federation service accounts are granted Decrypter roles on both regional key sets."
],
"outside_code": []
},
{
"id": "DR-REL-4",
"pillar": "reliability",
"text": "Configure the Cloud Storage buckets for audit logging as Dual-Region spanning europe-west2 and europe-west1.",
"components": [
"C-4"
],
"conditions": [
"Audit log Cloud Storage buckets are configured as Dual-Region spanning europe-west2 and europe-west1."
],
"outside_code": []
},
{
"id": "DR-REL-5",
"pillar": "reliability",
"text": "Isolate the Terraform state of the primary region from the standby region using multi-region Cloud Storage buckets with Object Versioning.",
"components": [
"C-5"
],
"conditions": [
"Terraform state for primary and standby regions is decoupled into separate failure domains.",
"Terraform state is stored in multi-region Cloud Storage buckets with Object Versioning enabled."
],
"outside_code": []
},
{
"id": "DR-OP-1",
"pillar": "operational-excellence",
"text": "Integrate Policy-as-Code validation into the GitHub Actions pull request workflows.",
"components": [
"C-5"
],
"conditions": [
"GitHub Actions workflows include automated Policy-as-Code validation steps prior to deployment execution."
],
"outside_code": []
},
{
"id": "DR-OP-2",
"pillar": "operational-excellence",
"text": "Establish explicit Service Level Indicators (SLIs) and configure multi-burn-rate alerting policies in Cloud Monitoring.",
"components": [
"C-8"
],
"conditions": [
"Service Level Objectives (SLOs) and SLIs are defined in Cloud Monitoring Service Monitoring.",
"Multi-burn-rate alerting policies are configured based on the SLO error budget."
],
"outside_code": []
},
{
"id": "DR-OP-3",
"pillar": "operational-excellence",
"text": "Activate the IAM Recommender to continuously evaluate role bindings.",
"components": [
"C-6"
],
"conditions": [
"IAM Recommender is activated on the foundation projects."
],
"outside_code": [
"Permissions are continuously rightsized based on IAM Recommender findings."
]
},
{
"id": "DR-COST-1",
"pillar": "cost-optimization",
"text": "Enforce a mandatory labeling taxonomy through Organization Policies and export granular billing data to BigQuery.",
"components": [
"C-9",
"C-1"
],
"conditions": [
"Cloud Billing export to BigQuery is enabled.",
"A mandatory labeling taxonomy is enforced via Organization Policy Service tags."
],
"outside_code": []
},
{
"id": "DR-COST-2",
"pillar": "cost-optimization",
"text": "Implement Cloud Storage Object Lifecycle Management to transition logs to Nearline storage after 30 days and to Coldline or Archive storage after 90 days.",
"components": [
"C-4"
],
"conditions": [
"Object Lifecycle Management rules transition Cloud Storage objects to Nearline storage after 30 days.",
"Object Lifecycle Management rules transition Cloud Storage objects to Coldline or Archive storage after 90 days."
],
"outside_code": []
},
{
"id": "DR-PERF-1",
"pillar": "performance-optimization",
"text": "Configure Cloud Routers with regional dynamic routing to localize egress traffic within the warm standby region.",
"components": [
"C-2"
],
"conditions": [
"Cloud Routers are configured with regional dynamic routing mode."
],
"outside_code": []
},
{
"id": "DR-PERF-2",
"pillar": "performance-optimization",
"text": "Instrument the deployment with OpenTelemetry exporting to Cloud Trace, and use Cloud Profiler on the compute instances.",
"components": [
"C-8"
],
"conditions": [
"OpenTelemetry tracing is configured to export to Cloud Trace.",
"Cloud Profiler agents are enabled on the core payments engine instances."
],
"outside_code": []
},
{
"id": "DR-SUST-1",
"pillar": "sustainability",
"text": "Implement a Pilot Light warm standby by maintaining active data replication while scaling the standby compute instances to a minimal baseline.",
"components": [],
"conditions": [
"Secondary region compute instances are scaled to a minimal baseline (Pilot Light) configuration during normal operations."
],
"outside_code": []
}
]

View File

@@ -0,0 +1,280 @@
[
{
"id": "FR-1",
"kind": "functional",
"text": "The platform foundation isolates development, staging, and production environments across distinct Google Cloud projects and resource folders.",
"from": [
"REQ-02"
]
},
{
"id": "FR-2",
"kind": "functional",
"text": "The platform foundation establishes private bidirectional network connectivity between Google Cloud and on-premises core payment systems over Cloud Interconnect.",
"from": [
"REQ-03"
]
},
{
"id": "FR-3",
"kind": "functional",
"text": "The platform foundation routes all outbound internet-bound traffic from workload spoke networks through centralised network security inspection appliances.",
"from": [
"REQ-03"
]
},
{
"id": "FR-4",
"kind": "functional",
"text": "The platform foundation captures and exports all administrative and data access audit logs across environments into a centralised, tamper-resistant log repository.",
"from": [
"REQ-02"
]
},
{
"id": "FR-5",
"kind": "functional",
"text": "The platform foundation provisions and manages all cloud infrastructure as code via automated CI/CD pipelines executing validated pull request workflows.",
"from": [
"REQ-01"
]
},
{
"id": "FR-6",
"kind": "functional",
"text": "The platform foundation federates corporate identity for single sign-on and enforces least-privilege role-based access control for platform administrators and application squads.",
"from": [
"REQ-04"
]
},
{
"id": "NFR-1",
"kind": "non_functional",
"text": "Environments are partitioned into development, staging, and production tiers, enforcing corp_workload controls on lower environments and pci_regulated_workload controls on production.",
"conditions": [
"Separate projects or folders exist for development, staging, and production environment tiers.",
"Production environments enforce pci_regulated_workload archetype security controls.",
"Development and staging environments enforce corp_workload archetype security controls."
],
"outside_code": [],
"from": [
"REQ-02",
"governance.environment_tiers"
]
},
{
"id": "NFR-2",
"kind": "non_functional",
"text": "Network connectivity is structured as a Shared VPC hub-and-spoke topology with zero direct internet ingress to workload subnets.",
"conditions": [
"A Shared VPC host project is connected to service projects across environment tiers.",
"Workload subnets have private Google access enabled and no external public IP addresses assigned.",
"Ingress firewall rules deny uninspected public internet traffic to workload subnets."
],
"outside_code": [],
"from": [
"REQ-02",
"networking.topology",
"networking.exposure"
]
},
{
"id": "NFR-3",
"kind": "non_functional",
"text": "Hybrid connectivity to on-premises core payment systems is enabled via a 10Gbps Dedicated Cloud Interconnect.",
"conditions": [
"Dedicated Cloud Interconnect connections are configured for hybrid connectivity.",
"Cloud Interconnect capacity is provisioned at 10Gbps bandwidth.",
"Interconnect VLAN attachments are associated with Cloud Routers in the Shared VPC hub."
],
"outside_code": [],
"from": [
"REQ-03",
"networking.hybrid_connectivity.enabled",
"networking.hybrid_connectivity.type",
"networking.hybrid_connectivity.bandwidth_gbps"
]
},
{
"id": "NFR-4",
"kind": "non_functional",
"text": "All outbound internet traffic from workload spokes is forced through centralised inspection appliances deployed in the hub network.",
"conditions": [
"Inspection appliance instances are deployed in a centralised hub network.",
"Custom VPC route tables direct 0.0.0.0/0 egress traffic from spokes to the centralised inspection appliance instances or internal load balancer."
],
"outside_code": [],
"from": [
"REQ-03",
"networking.egress_control.model"
]
},
{
"id": "NFR-5",
"kind": "non_functional",
"text": "Administrative access is authenticated via Entra ID single sign-on, and delivery pipelines authenticate via Workload Identity Federation without stored service account keys.",
"conditions": [
"Workload Identity Federation pools and providers are configured for GitHub Actions authentication.",
"IAM bindings grant roles to federated GitHub Actions identities without downloadable service account keys.",
"Corporate single sign-on is configured with Entra ID as the identity source of truth."
],
"outside_code": [],
"from": [
"REQ-04",
"governance.identity_source_of_truth",
"governance.pipeline_authentication"
]
},
{
"id": "NFR-6",
"kind": "non_functional",
"text": "Customer confidential data is encrypted at rest using customer-managed encryption keys with a 90-day rotation schedule.",
"conditions": [
"Cloud KMS customer-managed encryption keys are configured for infrastructure resources storing confidential data.",
"Cloud KMS keys have an automated rotation period set to 90 days (7776000s)."
],
"outside_code": [],
"from": [
"REQ-04",
"compliance.encryption.type",
"compliance.encryption.key_rotation_days"
]
},
{
"id": "NFR-7",
"kind": "non_functional",
"text": "Audit logs across all environments are retained for 365 days in a storage bucket protected by an immutable retention policy.",
"conditions": [
"An organization-level or project-level log sink exports audit logs to a centralised Cloud Storage bucket.",
"The centralised audit log storage bucket specifies a retention period of 365 days (31536000 seconds).",
"The centralised audit log storage bucket has retention policy locking enabled to prevent modification or early deletion."
],
"outside_code": [],
"from": [
"REQ-02",
"compliance.audit_logging.retention_days",
"compliance.audit_logging.immutable_retention_policy"
]
},
{
"id": "NFR-8",
"kind": "non_functional",
"text": "The platform foundation enforces DORA, ISO27001, and PCI-DSS v4.0 regulatory baselines and protects confidential data classification.",
"conditions": [
"Organization policy constraints enforce perimeter isolation and restrict unauthenticated resource sharing.",
"IAM policies restrict access to confidential data resources to authorized service identities and roles."
],
"outside_code": [
"The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit."
],
"from": [
"REQ-02",
"compliance.frameworks",
"compliance.data_classification"
]
},
{
"id": "NFR-9",
"kind": "non_functional",
"text": "Platform foundation resources are provisioned across europe-west2 (primary) and europe-west1 (secondary) in a warm standby model, restricting data residency to the UK and EU.",
"conditions": [
"Primary landing zone network and foundation resources are provisioned in europe-west2.",
"Secondary landing zone network and foundation standby resources are provisioned in europe-west1.",
"Resource location organization policies restrict resource and data storage locations to europe-west2 and europe-west1."
],
"outside_code": [
"The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic.",
"Data residency is guaranteed within the United Kingdom and European Union."
],
"from": [
"REQ-02",
"estate.regions.primary",
"estate.regions.secondary",
"estate.regions.resiliency_tier",
"estate.data_residency"
]
},
{
"id": "NFR-10",
"kind": "non_functional",
"text": "The platform foundation landing zone is delivered and operational for workload ingestion within 10 weeks.",
"conditions": [
"All landing zone foundation resources are codified in modular Terraform configurations."
],
"outside_code": [
"Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks."
],
"from": [
"REQ-01"
]
},
{
"id": "NFR-11",
"kind": "non_functional",
"text": "The platform foundation maintains 99.9% availability with a 4-hour recovery time objective (RTO) and a 1-hour recovery point objective (RPO).",
"conditions": [
"Regional networking and core management services are deployed across at least three zones in the primary region."
],
"outside_code": [
"Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.",
"Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.",
"Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour."
],
"from": [
"REQ-02"
]
},
{
"id": "NFR-12",
"kind": "non_functional",
"text": "Platform operations monitoring and alerting are implemented using Google Cloud Operations Suite with 24/7 alerting capabilities.",
"conditions": [
"Cloud Monitoring workspaces and notification channels are configured for the platform foundation.",
"Alert policies are configured for Interconnect link degradation, appliance health, and egress routing failures."
],
"outside_code": [
"Alert notifications trigger operational incident workflows for the 24/7 central platform operations team."
],
"from": []
},
{
"id": "NFR-13",
"kind": "non_functional",
"text": "Infrastructure costs are tracked against a monthly budget of 25,000 GBP with automated alerting.",
"conditions": [
"A Cloud Billing budget is configured for the foundation projects with currency set to GBP.",
"Budget alert thresholds are configured at 50%, 80%, 90%, and 100% of the 25,000 GBP monthly budget."
],
"outside_code": [
"Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off."
],
"from": []
},
{
"id": "NFR-14",
"kind": "non_functional",
"text": "Hybrid networking provides dedicated 10Gbps unthrottled throughput with minimal latency between Google Cloud and on-premises core systems.",
"conditions": [
"Cloud Routers configured for Interconnect advertise non-overlapping RFC 1918 BGP routes.",
"Dedicated Interconnect VLAN attachments are configured with dedicated 10Gbps bandwidth allocation."
],
"outside_code": [
"Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing."
],
"from": [
"REQ-03"
]
},
{
"id": "NFR-15",
"kind": "non_functional",
"text": "Landing zone resources are deployed in Google Cloud regions with strong low-carbon energy ratings.",
"conditions": [
"Foundation resources are constrained to europe-west2 (London) and europe-west1 (Belgium) regions."
],
"outside_code": [
"Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores."
],
"from": []
}
]

View File

@@ -0,0 +1,233 @@
{
"kind": "platform_foundation",
"stated": {
"engagement": {
"kind": "platform_foundation"
},
"intent": {
"statement": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.\n",
"requirements": [
{
"id": "REQ-01",
"category": "Timeline",
"statement": "Platform foundation must be ready for workload ingestion within 10 weeks."
},
{
"id": "REQ-02",
"category": "Compliance",
"statement": "Must achieve DORA resiliency alignment and PCI-DSS v4.0 perimeter isolation."
},
{
"id": "REQ-03",
"category": "Networking",
"statement": "10Gbps on-premises hybrid connectivity; centralised inspection appliances for egress."
},
{
"id": "REQ-04",
"category": "Identity and security",
"statement": "Workload identity federation with GitHub Actions; Entra ID single sign-on; customer-managed encryption keys."
}
],
"exclusions": [
"Application container deployment; workloads are managed by application squads.",
"An active-active secondary region in phase 1; warm standby is acceptable."
]
},
"estate": {
"cloud_provider": "gcp",
"mode": "greenfield",
"regions": {
"primary": "europe-west2",
"secondary": "europe-west1",
"resiliency_tier": "warm_standby"
},
"data_residency": "United Kingdom and European Union",
"existing_systems": "On-premises payment systems that stay in place and connect over the interconnect."
},
"workload": {
"availability": {
"hours": "always"
}
},
"compliance": {
"industry": "financial_services",
"frameworks": [
"DORA",
"ISO27001",
"PCI_DSS_v4"
],
"data_classification": "confidential",
"encryption": {
"type": "cmek",
"key_rotation_days": 90
},
"audit_logging": {
"retention_days": 365,
"immutable_retention_policy": true
}
},
"networking": {
"exposure": "internal_only",
"topology": "shared_vpc_hub_and_spoke",
"hybrid_connectivity": {
"enabled": true,
"type": "cloud_interconnect",
"bandwidth_gbps": 10
},
"egress_control": {
"model": "centralized_appliance"
}
},
"governance": {
"identity_source_of_truth": "Entra ID",
"pipeline_authentication": "workload identity federation",
"environment_tiers": [
{
"name": "development",
"archetype": "corp_workload"
},
{
"name": "staging",
"archetype": "corp_workload"
},
{
"name": "production",
"archetype": "pci_regulated_workload"
}
]
},
"operations": {
"operating_model": "central platform team brokering access for application squads",
"support_hours": "always"
},
"delivery": {
"window_weeks": 10,
"commercial_model": "fixed_price_milestone",
"iac_tooling": "terraform",
"ci_cd": "GitHub Actions",
"team_size": 3,
"working_days_per_week": 5
},
"cost": {
"currency": "GBP"
}
},
"binding": [
"estate.regions.primary",
"estate.regions.secondary",
"estate.regions.resiliency_tier",
"estate.data_residency",
"compliance.frameworks",
"compliance.data_classification",
"compliance.encryption.type",
"compliance.encryption.key_rotation_days",
"compliance.audit_logging.retention_days",
"compliance.audit_logging.immutable_retention_policy",
"networking.exposure",
"networking.topology",
"networking.hybrid_connectivity.enabled",
"networking.hybrid_connectivity.type",
"networking.hybrid_connectivity.bandwidth_gbps",
"networking.egress_control.model",
"governance.identity_source_of_truth",
"governance.pipeline_authentication",
"governance.environment_tiers"
],
"assumed": [
{
"field": "intent.success_criteria",
"value": [
"Platform foundation ready for workload ingestion within 10 weeks",
"Compliance achieved with DORA and PCI-DSS v4.0 requirements",
"Hybrid connectivity established via 10Gbps Dedicated Interconnect",
"Centralised egress inspection operational",
"Identity federation and CMEK security controls implemented"
],
"why": "Assumed standard success criteria based on delivery within 10 weeks and compliance with DORA and PCI-DSS v4.0."
},
{
"field": "workload.availability.target",
"value": "99.9%",
"why": "Assumed 99.9% availability target for platform foundation infrastructure aligned with warm standby secondary region."
},
{
"field": "workload.availability.recovery_time",
"value": "4 hours",
"why": "Assumed 4 hours recovery time objective (RTO) for platform foundation services in warm standby."
},
{
"field": "workload.availability.recovery_point",
"value": "1 hour",
"why": "Assumed 1 hour recovery point objective (RPO) for platform foundation configuration and state."
},
{
"field": "networking.egress_control.appliance_vendor",
"value": "Palo Alto Networks VM-Series",
"why": "Assumed Palo Alto Networks VM-Series as enterprise inspection appliance for centralized egress control."
},
{
"field": "networking.cidr_allocations",
"value": "RFC 1918 non-overlapping IP allocation (10.0.0.0/16 partitioned across environments) sized for growth",
"why": "Assumed RFC 1918 non-overlapping IP allocation (10.0.0.0/16 partitioned across environments) sized for growth."
},
{
"field": "operations.monitoring",
"value": "Google Cloud Operations Suite with availability and error alerting",
"why": "Assumed native Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging) with availability and error alerting."
},
{
"field": "operations.change_process",
"value": "GitHub Actions pull request workflow with automated Terraform plan validation and peer review",
"why": "Assumed GitHub Actions pull request workflow with automated Terraform plan validation and peer approval."
},
{
"field": "integrations",
"value": [
{
"criticality": "critical",
"direction": "bidirectional",
"name": "On-premises core payment systems"
}
],
"why": "Assumed integration with existing on-premises core payment systems via Dedicated Interconnect."
},
{
"field": "cost.monthly_budget",
"value": 25000,
"why": "Assumed indicative monthly foundation infrastructure budget of 25000 GBP for 10Gbps interconnect, inspection appliances, and core services."
},
{
"field": "constraints.products_required",
"value": [],
"why": "No mandated third-party products beyond stated tooling (GCP, Terraform, GitHub Actions, Entra ID)."
},
{
"field": "constraints.products_forbidden",
"value": [],
"why": "No forbidden products specified."
}
],
"to_decide": [],
"to_ask": [
{
"field": "metadata.engagement_reference",
"prompt": "What is metadata.engagement_reference?",
"severity": "LOW"
},
{
"field": "delivery.target_start_date",
"prompt": "When should work start?",
"severity": "LOW"
},
{
"field": "constraints.other",
"prompt": "What is constraints.other?",
"severity": "LOW"
}
],
"not_applicable": [
"workload.users.count",
"workload.users.peak_concurrent",
"workload.users.who"
]
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,529 @@
{
"dry_run": {
"evidence": "Success! The configuration is valid.",
"verdict": "pass"
},
"checks": [
{
"check": "K-1",
"evidence": "display_name = \"Dev Folder\"",
"verdict": "pass"
},
{
"check": "K-2",
"evidence": "display_name = \"Staging Folder\"",
"verdict": "pass"
},
{
"verdict": "pass",
"check": "K-3",
"evidence": "display_name = \"Production Folder\""
},
{
"verdict": "pass",
"evidence": "tag_value = google_tags_tag_value.corp_workload.id",
"check": "K-4"
},
{
"check": "K-5",
"verdict": "pass",
"evidence": "host_project = google_compute_shared_vpc_host_project.hub.project"
},
{
"verdict": "pass",
"evidence": "host_project = google_compute_shared_vpc_host_project.hub.project",
"check": "K-6"
},
{
"check": "K-7",
"evidence": "host_project = google_compute_shared_vpc_host_project.hub.project",
"verdict": "pass"
},
{
"verdict": "pass",
"evidence": "private_ip_google_access = true",
"check": "K-8"
},
{
"check": "K-9",
"evidence": "private_ip_google_access = true",
"verdict": "pass"
},
{
"check": "K-10",
"evidence": "direction = \"INGRESS\"",
"verdict": "pass"
},
{
"check": "K-11",
"verdict": "pass",
"evidence": "protocol = \"all\""
},
{
"verdict": "pass",
"evidence": "source_ranges = [\"0.0.0.0/0\"]",
"check": "K-12"
},
{
"check": "K-13",
"evidence": "type = \"DEDICATED\"",
"verdict": "pass"
},
{
"evidence": "type = \"DEDICATED\"",
"verdict": "pass",
"check": "K-14"
},
{
"check": "K-15",
"verdict": "pass",
"evidence": "bandwidth = \"BPS_10G\""
},
{
"evidence": "bandwidth = \"BPS_10G\"",
"verdict": "pass",
"check": "K-16"
},
{
"verdict": "pass",
"evidence": "router = google_compute_router.primary.id",
"check": "K-17"
},
{
"evidence": "network = google_compute_network.hub.id",
"verdict": "pass",
"check": "K-18"
},
{
"check": "K-19",
"evidence": "dest_range = \"0.0.0.0/0\"",
"verdict": "pass"
},
{
"verdict": "pass",
"check": "K-20",
"evidence": "next_hop_ilb = google_compute_forwarding_rule.palo_alto_ilb.id"
},
{
"verdict": "pass",
"check": "K-21",
"evidence": "issuer_uri = \"https://token.actions.githubusercontent.com\""
},
{
"verdict": "pass",
"check": "K-22",
"evidence": "member = \"principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.wif_pool.name}/attribute.repository/my-org/my-repo\""
},
{
"evidence": "idp_metadata_xml = \"<EntityDescriptor></EntityDescriptor>\"",
"verdict": "pass",
"check": "K-23"
},
{
"evidence": "purpose = \"ENCRYPT_DECRYPT\"",
"verdict": "pass",
"check": "K-24"
},
{
"check": "K-25",
"evidence": "purpose = \"ENCRYPT_DECRYPT\"",
"verdict": "pass"
},
{
"check": "K-26",
"verdict": "pass",
"evidence": "rotation_period = \"7776000s\""
},
{
"check": "K-27",
"evidence": "rotation_period = \"7776000s\"",
"verdict": "pass"
},
{
"evidence": "destination = \"storage.googleapis.com/${google_storage_bucket.audit_logs.name}\"",
"verdict": "pass",
"check": "K-28"
},
{
"check": "K-29",
"verdict": "pass",
"evidence": "retention_period = 31536000"
},
{
"check": "K-30",
"verdict": "pass",
"evidence": "is_locked = true"
},
{
"evidence": "restricted_services = [\"compute.googleapis.com\"]",
"verdict": "pass",
"check": "K-31"
},
{
"check": "K-32",
"evidence": "role = \"roles/cloudkms.cryptoKeyEncrypterDecrypter\"",
"verdict": "pass"
},
{
"check": "K-33",
"evidence": "region = \"europe-west2\"",
"verdict": "pass"
},
{
"check": "K-34",
"verdict": "pass",
"evidence": "region = \"europe-west2\""
},
{
"verdict": "pass",
"check": "K-35",
"evidence": "region = \"europe-west1\""
},
{
"verdict": "pass",
"evidence": "constraint = \"gcp.resourceLocations\"",
"check": "K-36"
},
{
"check": "K-37",
"verdict": "pass",
"evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]"
},
{
"check": "K-38",
"evidence": "project = var.project_id",
"verdict": "pass"
},
{
"verdict": "pass",
"evidence": "distribution_policy_zones = [\"europe-west2-a\", \"europe-west2-b\", \"europe-west2-c\"]",
"check": "K-39"
},
{
"verdict": "pass",
"check": "K-40",
"evidence": "notification_channels = [google_monitoring_notification_channel.email.name]"
},
{
"verdict": "pass",
"evidence": "condition_threshold is defined",
"check": "K-41"
},
{
"evidence": "currency_code = \"GBP\"",
"check": "K-42",
"verdict": "pass"
},
{
"verdict": "pass",
"evidence": "threshold_percent includes 0.5, 0.8, 0.9, 1.0",
"check": "K-43"
},
{
"check": "K-44",
"verdict": "pass",
"evidence": "units = \"25000\""
},
{
"evidence": "advertise_mode = \"CUSTOM\"",
"verdict": "pass",
"check": "K-45"
},
{
"verdict": "pass",
"evidence": "advertise_mode = \"CUSTOM\"",
"check": "K-46"
},
{
"verdict": "pass",
"evidence": "bandwidth = \"BPS_10G\"",
"check": "K-47"
},
{
"evidence": "bandwidth = \"BPS_10G\"",
"verdict": "pass",
"check": "K-48"
},
{
"check": "K-49",
"verdict": "pass",
"evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]"
},
{
"evidence": "resources = [\"projects/${var.prod_project_id}\", \"projects/${var.project_id}\"]",
"check": "K-50",
"verdict": "pass"
},
{
"verdict": "pass",
"check": "K-51",
"evidence": "ingress_policies and egress_policies are configured"
},
{
"verdict": "pass",
"evidence": "constraint = \"compute.skipDefaultNetworkCreation\"",
"check": "K-52"
},
{
"evidence": "enforced = true",
"verdict": "pass",
"check": "K-53"
},
{
"check": "K-54",
"evidence": "constraint = \"compute.vmExternalIpAccess\"",
"verdict": "pass"
},
{
"verdict": "pass",
"evidence": "all = true",
"check": "K-55"
},
{
"evidence": "constraint = \"iam.disableServiceAccountKeyCreation\"",
"check": "K-56",
"verdict": "pass"
},
{
"check": "K-57",
"evidence": "enforced = true",
"verdict": "pass"
},
{
"verdict": "pass",
"check": "K-58",
"evidence": "constraint = \"gcp.resourceLocations\""
},
{
"evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]",
"verdict": "pass",
"check": "K-59"
},
{
"verdict": "pass",
"check": "K-60",
"evidence": "enable_confidential_compute = true"
},
{
"check": "K-61",
"evidence": "name = \"CREDIT_CARD_NUMBER\"",
"verdict": "pass"
},
{
"check": "K-62",
"evidence": "event_data_content_type = \"application/json\"",
"verdict": "pass"
},
{
"evidence": "attribute = \"type\", value = \"google.cloud.securitycenter.finding.v1.saved\"",
"check": "K-63",
"verdict": "pass"
},
{
"evidence": "event_type = \"google.cloud.securitycenter.finding.v1.saved\"",
"verdict": "pass",
"check": "K-64"
},
{
"verdict": "pass",
"evidence": "interconnect = var.interconnect_1_url",
"check": "K-65"
},
{
"evidence": "interconnect = var.interconnect_2_url",
"verdict": "pass",
"check": "K-66"
},
{
"check": "K-67",
"evidence": "router = google_compute_router.primary.id",
"verdict": "pass"
},
{
"check": "K-68",
"evidence": "session_initialization_mode = \"ACTIVE\"",
"verdict": "pass"
},
{
"verdict": "pass",
"check": "K-69",
"evidence": "distribution_policy_zones = [\"europe-west2-a\", \"europe-west2-b\", \"europe-west2-c\"]"
},
{
"evidence": "load_balancing_scheme = \"INTERNAL\"",
"check": "K-70",
"verdict": "pass"
},
{
"evidence": "health_checks = [google_compute_health_check.palo_alto_ilb_hc.id]",
"verdict": "pass",
"check": "K-71"
},
{
"verdict": "pass",
"check": "K-72",
"evidence": "location = \"europe-west2\""
},
{
"evidence": "location = \"europe-west1\"",
"check": "K-73",
"verdict": "pass"
},
{
"evidence": "role = \"roles/cloudkms.cryptoKeyDecrypter\"",
"verdict": "pass",
"check": "K-74"
},
{
"verdict": "pass",
"evidence": "data_locations = [\"EUROPE-WEST1\", \"EUROPE-WEST2\"]",
"check": "K-75"
},
{
"check": "K-76",
"verdict": "pass",
"evidence": "name = \"${var.project_id}-tfstate-primary\""
},
{
"evidence": "name = \"${var.project_id}-tfstate-secondary\"",
"check": "K-77",
"verdict": "pass"
},
{
"evidence": "enabled = true",
"check": "K-78",
"verdict": "pass"
},
{
"check": "K-79",
"evidence": "enabled = true",
"verdict": "pass"
},
{
"evidence": "goal = 0.999",
"verdict": "pass",
"check": "K-80"
},
{
"evidence": "filter = \"select_slo_burn_rate(\\\"${google_monitoring_slo.platform_slo.name}\\\", 3600s)\"",
"verdict": "pass",
"check": "K-81"
},
{
"verdict": "pass",
"check": "K-82",
"evidence": "service = \"recommender.googleapis.com\""
},
{
"verdict": "pass",
"evidence": "age = 30",
"check": "K-83"
},
{
"verdict": "pass",
"check": "K-84",
"evidence": "type = \"SetStorageClass\", storage_class = \"NEARLINE\""
},
{
"check": "K-85",
"verdict": "pass",
"evidence": "age = 90"
},
{
"check": "K-86",
"evidence": "type = \"SetStorageClass\", storage_class = \"ARCHIVE\"",
"verdict": "pass"
},
{
"verdict": "pass",
"check": "K-87",
"evidence": "routing_mode = \"REGIONAL\""
},
{
"evidence": "min_replicas = 1",
"check": "K-88",
"verdict": "pass"
},
{
"evidence": "short_name = \"environment\"",
"check": "K-89",
"verdict": "pass"
},
{
"verdict": "pass",
"check": "K-90",
"evidence": "destination = \"bigquery.googleapis.com/...\""
},
{
"verdict": "pass",
"check": "K-91",
"evidence": "tag_value = google_tags_tag_value.pci_regulated.id"
},
{
"check": "K-92",
"evidence": "check_interval_sec = 5",
"verdict": "pass"
},
{
"verdict": "pass",
"check": "K-93",
"evidence": "disabled = false"
},
{
"verdict": "pass",
"check": "K-94",
"evidence": "disabled = false"
},
{
"evidence": "display_name = \"Platform Custom Service\"",
"check": "K-95",
"verdict": "pass"
},
{
"check": "K-96",
"verdict": "pass",
"evidence": "base_instance_name = \"palo-alto-secondary\""
},
{
"verdict": "pass",
"evidence": "session_initialization_mode = \"ACTIVE\"",
"check": "K-97"
},
{
"evidence": "min_receive_interval = 100",
"check": "K-99",
"verdict": "pass"
},
{
"check": "K-100",
"verdict": "pass",
"evidence": "min_receive_interval = 100"
},
{
"check": "K-101",
"evidence": "role = \"roles/cloudkms.cryptoKeyDecrypter\"",
"verdict": "pass"
},
{
"evidence": "tag_value = google_tags_tag_value.corp_workload.id",
"verdict": "pass",
"check": "K-102"
},
{
"verdict": "pass",
"evidence": "router = google_compute_router.primary.id",
"check": "K-103"
},
{
"verdict": "pass",
"evidence": "min_transmit_interval = 100",
"check": "K-104"
},
{
"check": "K-105",
"evidence": "min_transmit_interval = 100",
"verdict": "pass"
}
],
"proposed_additions": []
}

View File

@@ -0,0 +1,332 @@
{
"work_packages": [
{
"id": "WP-0",
"name": "Platform Foundation",
"parent": null,
"elements": [],
"scope": "The complete enterprise-grade landing zone.",
"produces": [
"an operational landing zone"
],
"skills": [
"cloud architecture"
]
},
{
"id": "WP-1",
"name": "Automated Infrastructure Orchestration",
"parent": "WP-0",
"elements": [
"E-42",
"E-43"
],
"scope": "CI/CD orchestration, state management, and API enablement.",
"produces": [
"Terraform state buckets for primary and secondary regions"
],
"skills": [
"cloud engineering"
]
},
{
"id": "WP-2",
"name": "Federated Identity and Role Governance",
"parent": "WP-0",
"elements": [
"E-23",
"E-24",
"E-25",
"E-26",
"E-27",
"E-45"
],
"scope": "Authentication for delivery pipelines and workforce SSO.",
"produces": [
"Workload Identity Federation for GitHub Actions and Entra ID SSO"
],
"skills": [
"identity and access management"
]
},
{
"id": "WP-3",
"name": "Resource Hierarchy and Environment Boundary",
"parent": "WP-0",
"elements": [
"E-1",
"E-2",
"E-3",
"E-4",
"E-5",
"E-6",
"E-7",
"E-8",
"E-9",
"E-10",
"E-34",
"E-35",
"E-36",
"E-37",
"E-38",
"E-50",
"E-51",
"E-55",
"E-60"
],
"scope": "Google Cloud resource hierarchy, shared VPC projects, networking, and organization policies.",
"produces": [
"Folders, projects, networking, and organization policies ensuring isolation"
],
"skills": [
"cloud engineering",
"network engineering"
]
},
{
"id": "WP-4",
"name": "Cryptographic Key Lifecycle",
"parent": "WP-0",
"elements": [
"E-28",
"E-29",
"E-30",
"E-31",
"E-49",
"E-58",
"E-59"
],
"scope": "Customer-managed encryption keys (CMEK) and cryptographic isolation.",
"produces": [
"Primary and secondary KMS key rings and crypto keys with IAM bindings"
],
"skills": [
"security engineering"
]
},
{
"id": "WP-5",
"name": "Audit, Compliance, and Financial Governance",
"parent": "WP-0",
"elements": [
"E-21",
"E-22",
"E-33",
"E-39",
"E-46",
"E-54"
],
"scope": "Immutable compliance logs, DLP templates, and budget enforcement.",
"produces": [
"Immutable audit log storage, billing budgets, and DLP inspection templates"
],
"skills": [
"cloud engineering",
"finops"
]
},
{
"id": "WP-6",
"name": "Centralised Outbound Traffic Inspection",
"parent": "WP-0",
"elements": [
"E-15",
"E-16",
"E-17",
"E-18",
"E-19",
"E-20",
"E-47",
"E-48",
"E-53"
],
"scope": "Egress inspection and threat prevention using Palo Alto VM-Series.",
"produces": [
"Palo Alto VM-Series firewalls with internal load balancing and egress routing"
],
"skills": [
"network security engineering"
]
},
{
"id": "WP-7",
"name": "Dedicated Private Hybrid Connectivity",
"parent": "WP-0",
"elements": [
"E-11",
"E-12",
"E-13",
"E-14",
"E-56",
"E-57"
],
"scope": "Dedicated Interconnect termination and dynamic BGP routing.",
"produces": [
"Primary and secondary Cloud Routers with Dedicated Interconnect VLAN attachments"
],
"skills": [
"network engineering"
]
},
{
"id": "WP-8",
"name": "Platform Telemetry and Operational Alerting",
"parent": "WP-0",
"elements": [
"E-32",
"E-40",
"E-41",
"E-44",
"E-52"
],
"scope": "SLO tracking, multi-burn-rate alerting, and SCC automated remediation.",
"produces": [
"SLOs, alerting policies, and SCC automated remediation functions"
],
"skills": [
"site reliability engineering"
]
}
],
"activities": [
{
"id": "A-1",
"work_package": "WP-1",
"name": "Enable Google Cloud APIs",
"days": 0.5,
"after": []
},
{
"id": "A-2",
"work_package": "WP-1",
"name": "Setup CI/CD Pipeline and State Buckets",
"days": 1,
"after": [
"A-1"
]
},
{
"id": "A-3",
"work_package": "WP-2",
"name": "Configure Workload Identity and Entra ID SSO",
"days": 1.5,
"after": [
"A-2"
]
},
{
"id": "A-4",
"work_package": "WP-3",
"name": "Provision Resource Hierarchy and Enforce Org Policies",
"days": 2.5,
"after": [
"A-3"
]
},
{
"id": "A-5",
"work_package": "WP-3",
"name": "Provision Hub and Spoke VPC Networks",
"days": 2,
"after": [
"A-4"
]
},
{
"id": "A-6",
"work_package": "WP-7",
"name": "Order and provision the Interconnect carrier circuits",
"days": 30,
"owner": "customer",
"assumption": "The customer manages the carrier's lead time and physical cross-connects.",
"after": []
},
{
"id": "A-7",
"work_package": "WP-4",
"name": "Configure Cloud KMS and Key Rings",
"days": 1.5,
"after": [
"A-4"
]
},
{
"id": "A-8",
"work_package": "WP-5",
"name": "Set up Audit Logging, DLP, and Billing Budgets",
"days": 2,
"after": [
"A-4"
]
},
{
"id": "A-9",
"work_package": "WP-6",
"name": "Deploy Palo Alto Egress Firewall Appliances",
"days": 5,
"after": [
"A-5"
]
},
{
"id": "A-10",
"work_package": "WP-7",
"name": "Configure Cloud Routers and Terminate BGP over Interconnect",
"days": 3,
"after": [
"A-5",
"A-6"
]
},
{
"id": "A-11",
"work_package": "WP-3",
"name": "Implement VPC Service Controls",
"days": 2,
"after": [
"A-5",
"A-7",
"A-8"
]
},
{
"id": "A-12",
"work_package": "WP-8",
"name": "Deploy Platform Telemetry, SLOs, and SCC Auto-remediation",
"days": 2.5,
"after": [
"A-8",
"A-11"
]
},
{
"id": "A-13",
"work_package": "WP-1",
"name": "Run validation in development environment",
"days": 1,
"after": [
"A-9",
"A-10",
"A-12"
]
},
{
"id": "A-14",
"work_package": "WP-1",
"name": "Run validation in staging environment",
"days": 1,
"after": [
"A-13"
]
},
{
"id": "A-15",
"work_package": "WP-1",
"name": "Run validation in production environment",
"days": 1,
"after": [
"A-14"
]
}
]
}

294
tests/support.py Normal file
View File

@@ -0,0 +1,294 @@
"""One small, coherent operating model, for tests: a fictional landing zone run as a
shared service. Every name in it is invented."""
from __future__ import annotations
import copy
import json
from typing import Any
from execution_engine.providers import load_provider
from execution_engine.scope import read_scope
from app.operating_model import catalogue as capability_catalogue
from app.operating_model import staffing
from app.operating_model.names import (
CAPABILITIES_FILE,
CAPABILITY_MODEL,
FINDINGS_FILE,
GRAPH_FILE,
OPERATING_SCOPE,
OPERATIONAL_REQUIREMENTS,
REQUIREMENTS_FILE,
SCOPE_DATA_FILE,
TARGET_OPERATING_MODEL,
VERDICTS_FILE,
)
from app.operating_model.workflow import ROOT, build
BUNDLE = load_provider("gcp", ROOT)
CATALOGUE = capability_catalogue.load(ROOT / "skills" / capability_catalogue.SKILL)
CARD = staffing.load(ROOT / "skills" / staffing.SKILL)
WORKFLOW = build(ROOT)
SPEC = """\
metadata:
customer_case: northwind-landing-zone
engagement:
kind: platform_foundation
intent:
statement: |
We want the landing zone run as a shared service from go-live, with our own
platform team keeping the final say.
business_outcomes:
- {id: BO-1, outcome: Outages of the platform are short., target: Restored within 4 hours.}
- {id: BO-2, outcome: Regulators always get the evidence they ask for.}
requirements:
- {id: REQ-01, statement: Our platform team approves every change to the network.}
solution:
cloud_provider: gcp
items:
- {id: S-1, name: Hub network, kind: network, product: Shared VPC, source_ref: E-1}
- {id: S-2, name: Audit log store, kind: storage, product: Cloud Storage, source_ref: E-2}
- {id: S-3, name: Deployment pipeline, kind: pipeline, product: Cloud Build, source_ref: E-3}
conditions:
- {id: K-1, "on": S-1, condition: Flow logs are retained for 400 days., source_ref: CHK-12}
- {id: K-2, "on": S-2, condition: Audit logs cannot be deleted for seven years., source_ref: CHK-20}
operations:
support_hours: 24x7
availability_hours: 24x7
change_process: decide
sourcing:
model: shared
delivery_countries: [GB, PL, IN]
service_levels:
priority_1_response_minutes: 15
recovery_time_hours: 4
compliance:
frameworks: [DORA]
commercial:
currency: GBP
monthly_run_budget: 120000
run:
pre_approved: true
"""
def scope_data(text: str = SPEC) -> dict[str, Any]:
"""What every Specialist is given of an operating spec."""
scope = read_scope(
text, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
)
return scope.for_context()
REQUIREMENTS: list[dict[str, Any]] = [
{
"id": "OR-1",
"kind": "service_level",
"statement": "A fault on any Item is acknowledged within 15 minutes, at any hour.",
"source": [
"service_levels.priority_1_response_minutes",
"operations.support_hours",
],
},
{
"id": "OR-2",
"kind": "process",
"statement": "The customer's platform team approves every change to the network.",
"source": ["REQ-01", "sourcing.model"],
},
{
"id": "OR-3",
"kind": "compliance",
"statement": "Logs are kept for as long as each Condition says, and cannot be removed sooner.",
"source": ["K-1", "K-2"],
},
{
"id": "OR-4",
"kind": "service_level",
"statement": "Service is restored within 4 hours of a major failure.",
"source": ["service_levels.recovery_time_hours"],
},
{
"id": "OR-5",
"kind": "organisation",
"statement": "The solution is operated only from the countries the customer allows.",
"source": ["sourcing.delivery_countries"],
},
]
CAPABILITIES: list[dict[str, Any]] = [
{
"id": "C-1",
"catalogue": "incident-management",
"serves": ["OR-1", "OR-4"],
"items": ["S-1", "S-2", "S-3"],
"hours": "24x7",
"effort_hours_per_month": 60,
"basis": "Two serious incidents a month and daily triage.",
},
{
"id": "C-2",
"catalogue": "change-enablement",
"serves": ["OR-2"],
"items": ["S-1", "S-2", "S-3"],
"hours": "business",
"effort_hours_per_month": 40,
"basis": "About twenty changes a month at two hours each.",
},
{
"id": "C-3",
"catalogue": "regulatory-evidence",
"serves": ["OR-3", "OR-5"],
"items": ["S-1", "S-2"],
"hours": "business",
"effort_hours_per_month": 16,
"basis": "A monthly review of retention settings and evidence.",
},
]
GRAPH: dict[str, list[dict[str, Any]]] = {
"parties": [
{"id": "P-1", "name": "Northwind platform team", "kind": "customer"},
{"id": "P-2", "name": "Managed service provider", "kind": "provider"},
],
"roles": [
{
"id": "R-1",
"party": "P-1",
"name": "Platform owner",
"country": "GB",
"level": "lead",
"hours": "business",
},
{
"id": "R-2",
"party": "P-2",
"name": "Platform SRE",
"country": "PL",
"level": "senior",
"hours": "24x7",
},
{
"id": "R-3",
"party": "P-2",
"name": "Service manager",
"country": "GB",
"level": "lead",
"hours": "business",
},
],
"responsibilities": [
{"id": "RS-1", "capability": "C-1", "role": "R-3", "kind": "accountable"},
{"id": "RS-2", "capability": "C-1", "role": "R-2", "kind": "responsible"},
{"id": "RS-3", "capability": "C-1", "role": "R-1", "kind": "consulted"},
{"id": "RS-4", "capability": "C-2", "role": "R-1", "kind": "accountable"},
{"id": "RS-5", "capability": "C-2", "role": "R-2", "kind": "responsible"},
{"id": "RS-6", "capability": "C-3", "role": "R-3", "kind": "accountable"},
{"id": "RS-7", "capability": "C-3", "role": "R-3", "kind": "responsible"},
{"id": "RS-8", "capability": "C-3", "role": "R-1", "kind": "consulted"},
],
"measures": [
{
"id": "M-1",
"name": "Time to restore service",
"layer": "outcome",
"outcome": "BO-1",
"measured_by": "C-1",
"framework": "dora-metrics",
"target": "Under 4 hours",
},
{
"id": "M-2",
"name": "Evidence requests met on time",
"layer": "service_level",
"outcome": "BO-2",
"measured_by": "C-3",
},
],
"checks": [
{
"id": "OC-1",
"on": "C-1",
"source": "OR-1",
"condition": "A fault on the hub network is acknowledged within 15 minutes at any hour.",
},
{
"id": "OC-2",
"on": "C-3",
"source": "K-1",
"condition": "Flow log retention is reviewed monthly and is 400 days.",
},
{
"id": "OC-3",
"on": "C-3",
"source": "K-2",
"condition": "The audit log store's retention lock is reviewed monthly.",
},
{
"id": "OC-4",
"on": "RS-4",
"source": "OR-2",
"condition": "No change to the network is applied without the platform owner's approval.",
},
],
}
VERDICTS: dict[str, Any] = {
"checks": [
{
"check": check["id"],
"verdict": "pass",
"evidence": "The model provides for it.",
}
for check in GRAPH["checks"]
],
"proposed_additions": [],
}
def graph(**changes: Any) -> dict[str, list[dict[str, Any]]]:
"""A copy of the graph to change."""
copied = copy.deepcopy(GRAPH)
copied.update(changes)
return copied
def upstream(
*,
spec: str = SPEC,
requirements: list[dict[str, Any]] | None = None,
capabilities: list[dict[str, Any]] | None = None,
model: dict[str, Any] | None = None,
findings: list[dict[str, Any]] | None = None,
) -> dict[str, dict[str, Any]]:
"""The Deliverables a submission builds on, as the engine holds them."""
def deliverable(name: str, data: Any) -> dict[str, Any]:
return {"content": "# Prose\n", "attachments": {name: json.dumps(data)}}
built = {
OPERATING_SCOPE: deliverable(SCOPE_DATA_FILE, scope_data(spec)),
OPERATIONAL_REQUIREMENTS: deliverable(
REQUIREMENTS_FILE, REQUIREMENTS if requirements is None else requirements
),
CAPABILITY_MODEL: deliverable(
CAPABILITIES_FILE, CAPABILITIES if capabilities is None else capabilities
),
TARGET_OPERATING_MODEL: deliverable(
GRAPH_FILE, GRAPH if model is None else model
),
}
if findings is not None:
built[TARGET_OPERATING_MODEL]["attachments"][FINDINGS_FILE] = json.dumps(
findings
)
return built
def as_file(name: str, data: Any) -> dict[str, str]:
return {name: json.dumps(data)}
__all__ = ["VERDICTS_FILE"]

107
tests/test_catalogue.py Normal file
View File

@@ -0,0 +1,107 @@
"""The capability catalogue and its register of frameworks."""
import shutil
import pytest
import yaml
from app.operating_model import catalogue as capability_catalogue
from app.operating_model.catalogue import CatalogueError
from app.operating_model.workflow import ROOT
from tests.support import CATALOGUE, scope_data
SKILL = ROOT / "skills" / capability_catalogue.SKILL
def test_every_entry_cites_a_framework_that_is_in_the_register():
for entry in CATALOGUE.entries.values():
assert entry.sources, entry.id
assert all(s["framework"] in CATALOGUE.frameworks for s in entry.sources)
assert entry.purpose and entry.typical_checks, entry.id
def test_the_register_says_its_references_have_not_been_checked():
assert CATALOGUE.verified is False
def test_the_two_things_called_dora_are_kept_apart():
names = {key: value["name"] for key, value in CATALOGUE.frameworks.items()}
assert "Regulation (EU) 2022/2554" in names["eu-dora"]
assert "DevOps Research and Assessment" in names["dora-metrics"]
@pytest.fixture
def skill(tmp_path):
shutil.copytree(SKILL, tmp_path / "skill")
return tmp_path / "skill"
def edit(skill, name, change):
path = skill / "assets" / name
data = yaml.safe_load(path.read_text())
change(data)
path.write_text(yaml.safe_dump(data))
def test_an_entry_that_cites_nothing_is_refused(skill):
edit(skill, "catalogue.yaml", lambda d: d["capabilities"][0].update(sources=[]))
with pytest.raises(CatalogueError, match="incident-management cites no source"):
capability_catalogue.load(skill)
def test_an_entry_that_cites_an_unregistered_framework_is_refused(skill):
edit(
skill,
"catalogue.yaml",
lambda d: d["capabilities"][0]["sources"].append(
{"framework": "folklore", "reference": "What we always did"}
),
)
with pytest.raises(
CatalogueError, match="cites 'folklore', which is not in the register"
):
capability_catalogue.load(skill)
def test_a_framework_registered_without_its_licence_is_refused(skill):
edit(skill, "sources.yaml", lambda d: d["frameworks"]["itil-4"].pop("licence"))
with pytest.raises(CatalogueError, match="itil-4 needs licence"):
capability_catalogue.load(skill)
def test_entries_are_proposed_from_what_the_spec_says():
proposed = CATALOGUE.applicable(scope_data())
# Always; for an Item of a kind; for a framework named; for a sourcing model.
assert "incident-management" in proposed
assert "backup-and-recovery" in proposed # the audit log store is storage
assert "regulatory-evidence" in proposed # DORA
assert "supplier-management" in proposed # shared
def test_an_entry_is_not_proposed_when_nothing_calls_for_it():
bare = {
"stated": {
"solution": {"items": [{"id": "S-1", "name": "Hub", "kind": "network"}]}
}
}
proposed = CATALOGUE.applicable(bare)
assert "incident-management" in proposed
assert not {
"backup-and-recovery",
"regulatory-evidence",
"supplier-management",
} & set(proposed)
def test_sources_are_cited_by_framework_and_part_never_quoted():
lines = capability_catalogue.cited(CATALOGUE, ["incident-management"])
assert "ITIL 4: Incident management practice" in lines
assert any(line.startswith("Regulation (EU) 2022/2554") for line in lines)

142
tests/test_chain.py Normal file
View File

@@ -0,0 +1,142 @@
"""The chain's data files: what links each to the one before, and what is refused."""
import copy
import pytest
from app.operating_model import chain
from app.operating_model.names import CAPABILITIES_FILE, REQUIREMENTS_FILE
from tests.support import CAPABILITIES, CATALOGUE, REQUIREMENTS, as_file, upstream
check_capabilities = chain.check_capabilities(CATALOGUE)
def requirements(*changes) -> list[dict]:
entries = copy.deepcopy(REQUIREMENTS)
for change in changes:
change(entries)
return entries
def capabilities(*changes) -> list[dict]:
entries = copy.deepcopy(CAPABILITIES)
for change in changes:
change(entries)
return entries
def test_requirements_traced_to_what_the_customer_stated_are_accepted():
assert (
chain.check_requirements(as_file(REQUIREMENTS_FILE, REQUIREMENTS), upstream())
== []
)
@pytest.mark.parametrize(
("change", "problem"),
[
(lambda e: e[0].update(source=[]), "OR-1 has no source"),
(
lambda e: e[0].update(source=["OR-99"]),
"source OR-99 is not a stated requirement",
),
(
lambda e: e[0].update(source=["commercial.exchange_rate"]),
"source commercial.exchange_rate is not a stated requirement",
),
(lambda e: e[0].update(kind="wish"), "OR-1: kind must be one of"),
(lambda e: e[0].update(statement=" "), "OR-1 needs a statement"),
(lambda e: e.append(dict(e[0])), "id OR-1 is used more than once"),
(
lambda e: e[1].update(source=["sourcing.model"]),
"the customer's requirement REQ-01 is carried by no Operational requirement",
),
(
lambda e: e.pop(4),
"the customer stated sourcing.delivery_countries, and no Operational requirement",
),
],
)
def test_a_requirement_that_breaks_the_chain_is_refused(change, problem):
faults = chain.check_requirements(
as_file(REQUIREMENTS_FILE, requirements(change)), upstream()
)
assert any(problem in fault for fault in faults), faults
def test_what_was_assumed_can_be_a_source_but_does_not_bind():
# `change_process: decide` leaves it to the agent, so nothing has to name it.
scope = upstream()["operating_scope"]["attachments"]["scope.json"]
assert "operations.change_process" not in scope.split('"binding"')[1].split("]")[0]
def test_an_empty_file_is_refused_and_bad_json_is_named():
assert chain.check_requirements({REQUIREMENTS_FILE: "[]"}, upstream()) == [
"operational-requirements.json: lists no Operational requirements"
]
assert (
"not valid JSON" in chain.check_requirements({REQUIREMENTS_FILE: "{"}, None)[0]
)
def test_capabilities_from_the_catalogue_that_cover_everything_are_accepted():
assert (
check_capabilities(as_file(CAPABILITIES_FILE, CAPABILITIES), upstream()) == []
)
@pytest.mark.parametrize(
("change", "problem"),
[
(
lambda e: e[0].update(catalogue="firefighting"),
"'firefighting' is not in the capability catalogue",
),
(
lambda e: e[0].update(catalogue="custom"),
"C-1 is custom and gives no reason",
),
(lambda e: e[0].update(serves=[]), "C-1 serves no Operational requirement"),
(
lambda e: e[0].update(serves=["OR-1", "OR-9"]),
"C-1 serves OR-9, which is not an Operational requirement",
),
(
lambda e: e[0].update(items=["S-1", "S-9"]),
"C-1 covers S-9, which is not an Item",
),
(
lambda e: [c.update(items=["S-1", "S-2"]) for c in e],
"Item S-3 is covered by no Capability",
),
(lambda e: e.pop(1), "Operational requirement OR-2 is served by no Capability"),
(lambda e: e[0].update(hours="always"), "C-1: hours must be one of"),
(
lambda e: e[0].update(hours="business"),
"faults are responded to 24x7, and Incident management is staffed only business",
),
(
lambda e: e[0].update(effort_hours_per_month=0),
"effort_hours_per_month must be a number above zero",
),
(lambda e: e[0].update(basis=""), "C-1 needs a basis for its effort"),
],
)
def test_a_capability_that_breaks_the_chain_is_refused(change, problem):
faults = check_capabilities(
as_file(CAPABILITIES_FILE, capabilities(change)), upstream()
)
assert any(problem in fault for fault in faults), faults
def test_a_custom_capability_with_a_name_and_a_reason_is_accepted():
custom = capabilities(
lambda e: e[2].update(
catalogue="custom", name="Evidence keeping", reason="No entry fits."
)
)
assert check_capabilities(as_file(CAPABILITIES_FILE, custom), upstream()) == []

237
tests/test_digest.py Normal file
View File

@@ -0,0 +1,237 @@
"""Digesting a solution architecture's files into an operating spec, in code."""
import json
from pathlib import Path
import pytest
from execution_engine.scope import ESSENTIAL, merged, read_scope
from app.operating_model import digest, spec
from app.operating_model.workflow import ROOT
from tests.support import BUNDLE, CATALOGUE, WORKFLOW
FIXTURE = Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone"
SOURCES = digest.read_folder(FIXTURE)
KINDS = digest.item_kinds(ROOT, "gcp")
DIGESTED = digest.architecture(SOURCES, KINDS)
GRAPH = json.loads(SOURCES[digest.GRAPH])
def test_every_file_the_digest_reads_came_with_the_fixture():
assert set(SOURCES) == set(digest.FILES)
assert set(DIGESTED.used) == set(digest.FILES)
def test_every_component_becomes_one_item_and_every_element_is_accounted_for():
items = DIGESTED.spec["solution"]["items"]
components = json.loads(SOURCES[digest.COMPONENTS])
assert [item["source_ref"] for item in items] == [c["id"] for c in components]
assert [item["id"] for item in items] == [f"S-{n}" for n in range(1, 10)]
placed = [element for item in items for element in item["elements"]]
assert sorted(placed) == sorted(e["id"] for e in GRAPH["elements"])
assert len(placed) == 60
def test_an_item_takes_its_product_and_its_kind_from_the_providers_table():
by_ref = {item["source_ref"]: item for item in DIGESTED.spec["solution"]["items"]}
assert (by_ref["C-2"]["product"], by_ref["C-2"]["kind"]) == (
"Cloud Interconnect",
"network",
)
assert by_ref["C-4"]["kind"] == "storage"
assert by_ref["C-7"]["kind"] == "keys"
# A product the table does not know is a service; nothing is guessed.
assert by_ref["C-3"]["kind"] == "service"
def test_what_the_architecture_leaves_to_operation_becomes_the_conditions():
conditions = DIGESTED.spec["solution"]["conditions"]
left_to_operation = [
(entry["id"], statement)
for name in (digest.REQUIREMENTS, digest.RECOMMENDATIONS)
for entry in json.loads(SOURCES[name])
for statement in entry.get("outside_code") or []
]
assert [(c["source_ref"], c["condition"]) for c in conditions] == [
(identifier, " ".join(statement.split()))
for identifier, statement in left_to_operation
]
recovery = next(
c for c in conditions if "recovery time objective" in c["condition"]
)
assert recovery["source_ref"] == "NFR-11"
items = {item["id"] for item in DIGESTED.spec["solution"]["items"]}
assert all(c["on"] and set(c["on"]) <= items for c in conditions)
def test_the_settings_checked_at_build_are_not_conditions():
# 104 Checks on infrastructure settings were verified in validation. Keeping
# them true is configuration management, not a Condition each.
assert len(GRAPH["checks"]) == 104
assert len(DIGESTED.spec["solution"]["conditions"]) == 12
def test_only_what_the_customer_stated_is_carried_across_as_stated():
carried = DIGESTED.spec
assert carried["operations"]["support_hours"] == "24x7" # stated: always
assert carried["compliance"]["frameworks"] == ["DORA", "ISO27001", "PCI_DSS_v4"]
assert carried["commercial"]["currency"] == "GBP"
assert carried["solution"]["cloud_provider"] == "gcp"
assert carried["engagement"]["kind"] == "platform_foundation"
# The architecture agent assumed these for the customer; they are asked again.
assert "change_process" not in carried["operations"]
assert "service_levels" not in carried
def test_the_build_and_what_the_customer_owns_of_it_are_carried_across():
build = DIGESTED.spec["solution"]["build"]
assert build["duration_days"] == 36
assert build["customer_owned"] == [
"Order and provision the Interconnect carrier circuits"
]
def test_the_spec_says_which_execution_it_was_digested_from():
assert DIGESTED.spec["metadata"] == {
"customer_case": "case-08-landing-zone",
"source": "architecture:case-08-landing-zone-185426",
}
assert "payment platform" in DIGESTED.spec["intent"]["statement"]
def test_the_digested_spec_is_one_the_catalogue_accepts():
scope = read_scope(
DIGESTED.spec, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
)
# What only the customer can say is still essential: what the business wants
# from the solution, and who operates it.
assert [gap["field"] for gap in scope.gaps(ESSENTIAL)] == [
"intent.business_outcomes",
"sourcing.model",
]
def test_every_digested_value_is_marked_and_what_the_caller_adds_is_theirs():
scope = merged(
DIGESTED.spec,
"sourcing:\n model: shared\ncommercial:\n currency: EUR\n"
"intent:\n business_outcomes: decide\n",
BUNDLE.scope_catalogue(),
provider_values=BUNDLE.scope_values,
notes=DIGESTED.notes,
)
assert scope.source("solution.items") == "digested"
assert scope.notes["solution.items"].startswith("9 components")
assert scope.source("sourcing.model") == "supplied"
assert scope.values["commercial.currency"] == "EUR"
assert scope.source("commercial.currency") == "supplied"
assert scope.gaps(ESSENTIAL) == []
def test_the_capabilities_proposed_follow_from_what_was_digested():
scope = read_scope(
DIGESTED.spec, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
).for_context()
proposed = CATALOGUE.applicable(scope)
assert "backup-and-recovery" in proposed # the audit log store is storage
assert "regulatory-evidence" in proposed # DORA, ISO 27001, PCI DSS
assert len(spec.items(scope)) == 9
def test_a_check_that_did_not_pass_is_an_inherited_risk():
sources = dict(SOURCES)
verdicts = json.loads(sources[digest.VERDICTS])
verdicts["checks"][0].update(verdict="fail", evidence="display_name is empty")
sources[digest.VERDICTS] = json.dumps(verdicts)
risks = digest.architecture(sources, KINDS).spec["solution"]["inherited_risks"]
assert risks == [
{
"on": "S-1",
"risk": "Check K-1 failed in validation: `display_name` is set "
"(display_name is empty)",
"source_ref": "K-1",
}
]
assert "inherited_risks" not in DIGESTED.spec["solution"] # every Check passed
def test_files_passed_without_their_manifest_are_marked_unverified():
sources = {k: v for k, v in SOURCES.items() if k != digest.MANIFEST}
digested = digest.architecture(sources, KINDS)
assert digested.spec["metadata"] == {"source": "architecture:unverified"}
assert (
"not known to be one completed Execution" in digested.notes["metadata.source"]
)
assert len(digested.spec["solution"]["items"]) == 9
def test_the_graph_alone_is_enough_to_say_what_is_operated():
digested = digest.architecture({digest.GRAPH: SOURCES[digest.GRAPH]}, KINDS)
items = digested.spec["solution"]["items"]
# With no components file, they are taken from the Elements, in the order met.
assert sorted(item["source_ref"] for item in items) == [
f"C-{n}" for n in range(1, 10)
]
assert "conditions" not in digested.spec["solution"]
@pytest.mark.parametrize(
("change", "problem"),
[
(
lambda s: s.update({digest.GRAPH: "{"}),
"target-architecture.json: not valid JSON",
),
(
lambda s: s.update({digest.GRAPH: "[]"}),
"must be an object with a list of elements",
),
(
lambda s: s.update(
{digest.MANIFEST: json.dumps({"status": "completed_with_warnings"})}
),
"the Execution did not complete",
),
(
lambda s: s.update(
{digest.COMPONENTS: json.dumps([{"id": "C-1", "name": "Only one"}])}
),
"these Elements name a component that components.json does not have",
),
],
)
def test_an_architecture_that_cannot_be_used_is_refused_and_says_why(change, problem):
sources = dict(SOURCES)
change(sources)
with pytest.raises(ValueError, match=problem):
digest.architecture(sources, KINDS)
def test_sources_that_are_not_an_architecture_are_left_to_be_read():
assert digest.architecture({"design.md": "# Our platform\n"}, KINDS).spec == {}
assert WORKFLOW.digest is not None
assert WORKFLOW.digest({"design.md": "# Our platform\n"}).used == ()
def test_the_command_prints_the_spec_and_what_it_was_read_from(capsys):
assert digest.main([str(FIXTURE)]) == 0
printed = capsys.readouterr()
assert "source: architecture:case-08-landing-zone-185426" in printed.out
assert "# solution.items: 9 components" in printed.err
assert digest.main([str(FIXTURE.parent)]) == 1

69
tests/test_eval.py Normal file
View File

@@ -0,0 +1,69 @@
"""The evaluation harness itself, run on the scripted runtime: no model is called."""
import json
from execution_engine.adk.fake_runtime import FakeAgentRuntime
from execution_engine.config import Settings
from eval import operating_model_eval as harness
from tests.support import SPEC
from tests.test_execution import turns
def settings() -> Settings:
return Settings( # type: ignore[call-arg]
_env_file=None, CLOUD_PROVIDER="gcp", CHECKPOINTER="memory", MCP_ENABLED=False
)
CASE = {"id": "scripted", "max_revisions": 2, "max_minutes": 5}
async def run(tmp_path, scripted, monkeypatch):
monkeypatch.setattr(harness, "inputs", lambda case: {"scope": SPEC})
return await harness.run_case(
CASE, tmp_path, settings=settings(), runtime=FakeAgentRuntime(scripted)
)
async def test_a_sound_execution_passes_every_check(tmp_path, monkeypatch):
result = await run(tmp_path, turns(), monkeypatch)
assert result.status == "completed"
failed = [check for check in result.checks if not check.passed]
# The scripted packager writes one word; a real one must write a summary.
assert [check.name for check in failed] == ["the packager wrote a summary"]
assert result.figures["cost_per_month"] == 43410
assert result.figures["scenarios"] == 10
assert (tmp_path / "deliverables").is_dir()
assert json.dumps(result.as_dict()) # the whole result can be written down
async def test_a_run_that_cannot_finish_is_reported_not_raised(tmp_path, monkeypatch):
scripted = turns()
scripted["validator"] = []
result = await run(tmp_path, scripted, monkeypatch)
assert not result.passed
assert result.status != "completed"
def test_every_case_names_something_to_start_from():
for case in harness.load_cases():
given = harness.inputs(case)
assert "scope" in given or "sources" in given, case["id"]
def test_the_benchmark_spec_is_complete_enough_to_run_with_nobody_to_ask():
from execution_engine.scope import ESSENTIAL, read_scope
from tests.support import BUNDLE
text = (harness.CASES.parent / "specs" / "landing-zone-shared.yaml").read_text()
scope = read_scope(
text, BUNDLE.scope_catalogue(), provider_values=BUNDLE.scope_values
)
assert scope.gaps(ESSENTIAL) == []
assert scope.customer_case == "bench-landing-zone-shared"

411
tests/test_execution.py Normal file
View File

@@ -0,0 +1,411 @@
"""Whole Executions of the target operating model Workflow, on the scripted runtime."""
import json
from pathlib import Path
from typing import Any
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime
from execution_engine.adk.roles import load_roles
from execution_engine.adk.runtime import NoDecider
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from langgraph.checkpoint.memory import InMemorySaver
from app.operating_model.names import (
CAPABILITIES_FILE,
CAPABILITY_MODEL,
FINDINGS_FILE,
GRAPH_FILE,
GUIDE,
OPERATING_SCOPE,
OPERATIONAL_REQUIREMENTS,
REQUIREMENTS_FILE,
TARGET_OPERATING_MODEL,
VALIDATION_RESULTS,
VERDICTS_FILE,
)
from app.operating_model.workflow import ROOT
from tests.support import (
BUNDLE,
CAPABILITIES,
GRAPH,
REQUIREMENTS,
SPEC,
VERDICTS,
WORKFLOW,
graph,
)
ROLES = load_roles(ROOT / "agents", BUNDLE, WORKFLOW)
CASE = "northwind-landing-zone"
FINDING = {
"id": "AF-1",
"about": ["S-2"],
"needed_by": ["C-3"],
"lacks": "Nothing alerts when the audit log store's retention lock is removed.",
"proposed": "Add an alert on a change to the retention lock.",
}
def submit(kind: str, **files: Any) -> list[Call]:
"""One turn that submits a Deliverable with every section of its contract."""
spec = WORKFLOW.deliverables[kind]
content = f"# {kind}\n" + "".join(
f"\n## {section}\n\nDetails.\n" for section in spec.sections
)
return [
Call(
"submit_deliverable",
{
"content": content,
"attachments": {name: json.dumps(data) for name, data in files.items()},
},
)
]
def turns(model=GRAPH, verdicts=VERDICTS, findings=None) -> dict[str, list[Any]]:
model_files = {GRAPH_FILE: model}
if findings is not None:
model_files[FINDINGS_FILE] = findings
return {
"scope-analyst": [submit(OPERATING_SCOPE)],
"operations-analyst": [
submit(OPERATIONAL_REQUIREMENTS, **{REQUIREMENTS_FILE: REQUIREMENTS})
],
"capability-modeller": [
submit(CAPABILITY_MODEL, **{CAPABILITIES_FILE: CAPABILITIES})
],
"operating-model-architect": [submit(TARGET_OPERATING_MODEL, **model_files)],
"validator": [submit(VALIDATION_RESULTS, **{VERDICTS_FILE: verdicts})],
"packager": [submit(GUIDE)],
}
def make_engine(
tmp_path: Path, scripted: dict[str, list[Any]]
) -> tuple[ExecutionEngine, FakeAgentRuntime]:
runtime = FakeAgentRuntime(scripted)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
scope_catalogue=BUNDLE.scope_catalogue(),
scope_values=BUNDLE.scope_values,
scope_skill=BUNDLE.scope_skill,
)
return engine, runtime
def folder(tmp_path: Path, execution_id: str) -> Path:
return tmp_path / "deliverables" / "cases" / CASE / "executions" / execution_id
async def test_an_operating_spec_becomes_a_priced_validated_operating_model(tmp_path):
engine, runtime = make_engine(tmp_path, turns())
result = await engine.start("", scope=SPEC, execution_id="e1")
assert result.status == "completed", result.state.get("warnings")
assert [run.role for run in runtime.runs] == [
"scope-analyst",
"operations-analyst",
"capability-modeller",
"operating-model-architect",
"validator",
"packager",
]
out = folder(tmp_path, "e1")
written = {p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file()}
assert {
"execution.json",
"operating-model-guide.md",
"scope.yaml",
"scope.json",
"operational-requirements.json",
"capabilities.json",
"target-operating-model.json",
"findings.json",
"staffing.json",
"organisation.mmd",
"verdicts.json",
"scenarios.json",
} <= written
record = json.loads((out / "execution.json").read_text())
assert record["customer_case"] == CASE
assert record["final_document"] == "operating-model-guide.md"
assert record["staffing"] == {
"people": 5.1,
"cost_per_month": 43410,
"currency": "GBP",
"rate_card": {
"name": "rate-card",
"version": "draft-2026-10-13",
"draft": True,
},
}
assert record["findings"] == []
assert record["capability_sources_verified"] is False
async def test_the_views_and_the_walk_throughs_are_written_by_code(tmp_path):
engine, _ = make_engine(tmp_path, turns())
result = await engine.start("", scope=SPEC)
model = result.state["deliverables"][TARGET_OPERATING_MODEL]["content"]
assert "| Incident management (C-1) | C | R | A |" in model
assert "Cost a month: 43,410 GBP." in model
validation = result.state["deliverables"][VALIDATION_RESULTS]
assert "## Scenario walk-throughs" in validation["content"]
verdicts = json.loads(validation["attachments"][VERDICTS_FILE])
assert verdicts["dry_run"]["evidence"] == "10 of 10 scenario walk-throughs pass."
assert len(json.loads(validation["attachments"]["scenarios.json"])) == 10
async def test_specialists_are_told_what_code_found_and_which_prices_apply(tmp_path):
engine, runtime = make_engine(tmp_path, turns())
await engine.start("", scope=SPEC)
by_role = {run.role: run.task.context for run in runtime.runs}
proposed = by_role["capability-modeller"]["capabilities_proposed"]
assert {"incident-management", "regulatory-evidence", "supplier-management"} <= set(
proposed
)
assert by_role["operating-model-architect"]["rate_card"]["draft"] is True
assert "PL" in by_role["operating-model-architect"]["rate_card"]["countries"]
# The architect builds the graph with the engine's tools, in this Workflow's shape.
architect = next(
run for run in runtime.runs if run.role == "operating-model-architect"
)
assert {"graph_put", "graph_read", "graph_remove"} <= set(architect.tools)
# Every Specialist is given the agreed operating spec, with the solution in it.
items = by_role["validator"]["scope"]["stated"]["solution"]["items"]
assert [item["id"] for item in items] == ["S-1", "S-2", "S-3"]
async def test_a_spec_with_a_currency_the_rate_card_cannot_convert_to_is_refused(
tmp_path,
):
import pytest
engine, runtime = make_engine(tmp_path, turns())
with pytest.raises(ValueError, match="commercial.currency"):
await engine.start("", scope=SPEC.replace("currency: GBP", "currency: CHF"))
assert runtime.runs == []
async def test_a_spec_with_no_currency_and_nobody_to_ask_is_refused(tmp_path):
import pytest
engine, _ = make_engine(tmp_path, turns())
with pytest.raises(ValueError, match="commercial.currency"):
await engine.start("", scope=SPEC.replace(" currency: GBP\n", ""))
async def test_a_model_nobody_can_run_at_night_is_refused_before_it_is_approved(
tmp_path,
):
off_hours = graph()
off_hours["roles"][1]["hours"] = "extended"
scripted = turns()
# Refused, and put right in the same run: the customer never sees it.
scripted["operating-model-architect"] = [
[
*submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: off_hours}),
*submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH}),
]
]
engine, runtime = make_engine(tmp_path, scripted)
result = await engine.start("", scope=SPEC)
assert result.status == "completed", result.state.get("warnings")
refused, accepted = runtime.runs[3].results
assert (
"Capability C-1 is needed 24x7, and no Role responsible for it works "
"those hours" in refused[1]
)
assert accepted[1] == "Submitted target_operating_model."
assert (
result.state["ledger"]["deliverables"][TARGET_OPERATING_MODEL]["revisions"] == 0
)
async def test_a_failed_scenario_reopens_the_model_through_the_router(tmp_path):
stranded = graph()
stranded["responsibilities"] = [
r for r in stranded["responsibilities"] if r["id"] not in ("RS-3", "RS-8")
]
scripted = turns()
scripted["operating-model-architect"] = [
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: stranded}),
lambda task: (
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH})
if any("SC-party-P-2 failed" in item for item in task.context["feedback"])
else []
),
]
scripted["validator"] = scripted["validator"] * 2
engine, runtime = make_engine(tmp_path, scripted)
result = await engine.start("", scope=SPEC)
assert result.status == "completed", result.state.get("warnings")
assert [d["action"] for d in result.state["decisions"]].count("revise_code") == 1
ledger = result.state["ledger"]["deliverables"]
assert ledger[TARGET_OPERATING_MODEL]["revisions"] == 1
assert ledger[CAPABILITY_MODEL]["revisions"] == 0
assert [run.role for run in runtime.runs].count("validator") == 2
async def test_a_finding_is_kept_out_of_the_design_and_drafted_as_a_change_request(
tmp_path,
):
waiting = graph()
waiting["checks"][2]["depends_on_finding"] = "AF-1"
verdicts = json.loads(json.dumps(VERDICTS))
verdicts["checks"][2].update(verdict="not_verifiable", evidence="Waits on AF-1.")
digested = SPEC.replace(
" customer_case: northwind-landing-zone\n",
" customer_case: northwind-landing-zone\n source: architecture:exec-42\n",
)
engine, _ = make_engine(tmp_path, turns(waiting, verdicts, [FINDING]))
result = await engine.start("", scope=digested, execution_id="e1")
assert result.status == "completed", result.state.get("warnings")
out = folder(tmp_path, "e1")
record = json.loads((out / "execution.json").read_text())
assert record["findings"] == ["AF-1"]
assert record["change_request"] == "architecture-change-request.md"
request = (out / "architecture-change-request.md").read_text()
assert "digested from `architecture:exec-42`" in request
assert (
"**What is asked for.** Add an alert on a change to the retention lock."
in request
)
assert (
"## Findings" in result.state["deliverables"][TARGET_OPERATING_MODEL]["content"]
)
# The Check that waits on the finding is reported with it, and nothing was
# revised for it: the model is not at fault, and neither are the Capabilities.
verdicts = json.loads((out / "verdicts.json").read_text())
waiting = [c for c in verdicts["checks"] if c["verdict"] == "waits_on_finding"]
assert [(c["check"], c["finding"]) for c in waiting] == [("OC-3", "AF-1")]
ledger = result.state["ledger"]["deliverables"]
assert all(entry["revisions"] == 0 for entry in ledger.values())
# The solution itself is as it was described: nothing was added to it.
scope = json.loads((out / "scope.json").read_text())
assert [item["id"] for item in scope["stated"]["solution"]["items"]] == [
"S-1",
"S-2",
"S-3",
]
# --- starting from sources ---------------------------------------------------------------
async def test_an_execution_starts_from_a_solution_architectures_files(tmp_path):
from pathlib import Path as _Path
from app.operating_model import digest
fixture = (
_Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone"
)
engine, runtime = make_engine(
tmp_path, {"scope-analyst": [submit(OPERATING_SCOPE)]}
)
result = await engine.start(
"",
sources=digest.read_folder(fixture),
# The one essential thing the architecture cannot say: who operates it.
scope="sourcing:\n model: shared\nintent:\n business_outcomes:\n"
" - {id: BO-1, outcome: Payments are never down for long.}\n",
execution_id="e1",
)
assert result.status == "awaiting_user"
assert result.prompt["approval"]["phase"] == "scoping"
assert result.state["customer_case"] == "case-08-landing-zone"
assert "payment platform" in result.state["workflow_request"]
scope = json.loads(
result.state["deliverables"][OPERATING_SCOPE]["attachments"]["scope.json"]
)
assert len(scope["stated"]["solution"]["items"]) == 9
assert len(scope["stated"]["solution"]["conditions"]) == 12
assert scope["stated"]["sourcing"] == {"model": "shared"}
# What was digested is shown to the customer as digested, to approve.
register = result.state["deliverables"][OPERATING_SCOPE]["content"]
assert "digested from the sources supplied" in register
# Every source is on record with its hash, and was read by code.
assert set(result.state["sources"]) == set(digest.FILES)
assert all(entry["digested"] for entry in result.state["sources"].values())
assert "source_documents" not in runtime.runs[0].task.context
async def test_a_document_code_cannot_read_is_given_to_the_scope_analyst(tmp_path):
engine, runtime = make_engine(tmp_path, {"scope-analyst": [[]]})
document = "# Our platform\n\nWe run a booking service on three clusters.\n"
await engine.start(
"Design the operating model for our booking platform.",
customer_case="bookings",
sources={"design.md": document},
)
context = runtime.runs[0].task.context
assert context["source_documents"] == {"design.md": document}
assert runtime.runs[0].role == "scope-analyst"
async def test_a_finding_the_model_could_meet_itself_sends_the_model_back(tmp_path):
"""A gap in the operating model's own staffing is not a finding: the validator
fails the Check that waits on it, and the architect designs it instead."""
excuse = {
"id": "AF-1",
"about": ["S-2"],
"needed_by": ["OC-3"],
"lacks": "The retention review is only staffed in business hours.",
"proposed": "Staff it at every hour.",
}
waiting = graph()
waiting["checks"][2]["depends_on_finding"] = "AF-1"
judged = json.loads(json.dumps(VERDICTS))
judged["checks"][2].update(
verdict="fail",
evidence="Staffing is the operating model's to provide; this is not a finding.",
)
scripted = turns(waiting, judged, [excuse])
scripted["operating-model-architect"].append(
submit(TARGET_OPERATING_MODEL, **{GRAPH_FILE: GRAPH, FINDINGS_FILE: []})
)
scripted["validator"].append(
submit(VALIDATION_RESULTS, **{VERDICTS_FILE: VERDICTS})
)
engine, runtime = make_engine(tmp_path, scripted)
result = await engine.start("", scope=SPEC)
assert result.status == "completed", result.state.get("warnings")
assert "revise_code" in [d["action"] for d in result.state["decisions"]]
second = [run for run in runtime.runs if run.role == "operating-model-architect"][1]
assert any("not a finding" in item for item in second.task.context["feedback"])
assert graph_findings(result) == []
def graph_findings(result) -> list:
attachments = result.state["deliverables"][TARGET_OPERATING_MODEL]["attachments"]
return json.loads(attachments.get(FINDINGS_FILE, "[]"))

241
tests/test_graph.py Normal file
View File

@@ -0,0 +1,241 @@
"""The Target operating model graph: what is refused, and the views code writes."""
import json
import pytest
from app.operating_model import graph as model
from app.operating_model.names import (
FINDINGS_FILE,
GRAPH_FILE,
ORGANISATION_FILE,
STAFFING_FILE,
)
from tests.support import CARD, CATALOGUE, GRAPH, as_file, graph, upstream
check = model.check(CARD)
add_views = model.views(CATALOGUE, CARD)
FINDING = {
"id": "AF-1",
"about": ["S-2"],
"needed_by": ["C-3"],
"lacks": "Nothing alerts when the audit log store's retention lock is removed.",
"proposed": "Add an alert on a change to the retention lock.",
}
def faults(changed, findings=None) -> list[str]:
attachments = as_file(GRAPH_FILE, changed)
if findings is not None:
attachments[FINDINGS_FILE] = json.dumps(findings)
return check(attachments, upstream())
def test_a_graph_that_holds_together_is_accepted():
assert faults(GRAPH) == []
@pytest.mark.parametrize(
("change", "problem"),
[
(
lambda g: g["parties"][0].update(kind="friend"),
"Party P-1: kind must be one of",
),
(
lambda g: g["roles"][0].update(party="P-9"),
"Role R-1 is in P-9, which is not a Party",
),
(
lambda g: g["roles"][1].update(country="FR"),
"Role R-2 sits in FR, which the rate card does not have",
),
(
lambda g: g["roles"][1].update(country="US"),
"Role R-2 sits in US; the customer allows only GB, PL, IN",
),
(
lambda g: g["roles"][1].update(level="wizard"),
"Role R-2 is at level wizard, which the rate card does not have",
),
(
lambda g: g["roles"][1].update(hours="nights"),
"Role R-2: hours must be one of",
),
(lambda g: g["roles"][1].pop("country"), "(R-2) needs country"),
(
lambda g: g["responsibilities"][0].update(kind="owner"),
"Responsibility RS-1: kind must be one of",
),
(
lambda g: g["responsibilities"][0].update(role="R-9"),
"RS-1 names R-9, which is not a Role",
),
(
lambda g: g["responsibilities"][0].update(capability="C-9"),
"RS-1 names C-9, which is not a Capability",
),
(
lambda g: g["responsibilities"].pop(0),
"Capability C-1 has 0 accountable Roles; it needs exactly one",
),
(
lambda g: g["responsibilities"][2].update(kind="accountable"),
"Capability C-1 has 2 accountable Roles; it needs exactly one",
),
(
lambda g: g["responsibilities"].pop(1),
"Capability C-1 has no responsible Role",
),
(
lambda g: g["roles"][1].update(hours="business"),
"Capability C-1 is needed 24x7, and no Role responsible for it works those hours",
),
(lambda g: g["checks"].pop(1), "Condition K-1 is carried by no Check"),
(
lambda g: g["checks"][0].update(on="C-9"),
"Check OC-1 is on C-9, which is not",
),
(
lambda g: g["checks"][0].update(source="OR-9"),
"Check OC-1: source OR-9 is not an Operational requirement",
),
(
lambda g: g["checks"][0].update(depends_on_finding="AF-1"),
"Check OC-1 depends on AF-1, which is not a finding",
),
(
lambda g: g["roles"].append(dict(g["roles"][0])),
"id R-1 is used more than once in roles",
),
],
)
def test_a_graph_that_does_not_hold_together_is_refused(change, problem):
changed = graph()
change(changed)
found = faults(changed)
assert any(problem in fault for fault in found), found
def test_a_file_that_is_not_a_graph_is_named():
assert "not valid JSON" in check({GRAPH_FILE: "{"}, upstream())[0]
assert "must be an object with" in check({GRAPH_FILE: "[]"}, upstream())[0]
def test_a_finding_is_recorded_and_a_check_may_wait_on_it():
changed = graph()
changed["checks"][2]["depends_on_finding"] = "AF-1"
assert faults(changed, [FINDING]) == []
@pytest.mark.parametrize(
("change", "problem"),
[
(lambda f: f.update(lacks=""), "AF-1 needs lacks"),
(lambda f: f.update(proposed=""), "AF-1 needs proposed"),
(lambda f: f.update(needed_by=[]), "AF-1 must say what needs it"),
(lambda f: f.update(about=["S-9"]), "AF-1 is about S-9, which is not an Item"),
(
lambda f: f.update(about=[]),
"AF-1 names no Item of the solution under about",
),
(
lambda f: f.update(needed_by=["C-9"]),
"AF-1 is needed by C-9, which is not in the operating model",
),
],
)
def test_a_finding_that_does_not_hold_together_is_refused(change, problem):
finding = dict(FINDING)
change(finding)
found = faults(GRAPH, [finding])
assert any(problem in fault for fault in found), found
def viewed(changed=GRAPH, findings=None, spec=None):
attachments = as_file(GRAPH_FILE, changed)
if findings is not None:
attachments[FINDINGS_FILE] = json.dumps(findings)
built = upstream() if spec is None else upstream(spec=spec)
return add_views(
"# Target operating model\n\n## Design rationale\n\nWhy.\n", attachments, built
)
def test_the_matrix_is_written_from_the_graph():
content, _ = viewed()
assert "## Responsibility matrix" in content
assert "| Capability | Platform owner | Platform SRE | Service manager |" in content
assert "| Incident management (C-1) | C | R | A |" in content
assert "| Change enablement (C-2) | A | R | |" in content
# One Role may be answerable for a Capability and do it: both parts are shown.
assert "| Regulatory evidence and reporting (C-3) | C | | A/R |" in content
def test_staffing_and_cost_are_written_with_the_prices_they_came_from():
content, attachments = viewed()
assert (
"| Platform SRE | Managed service provider | PL | senior | 24x7 | 100.0 h | 5.0 | 41,640 GBP |"
in content
)
assert (
"| Platform owner | Northwind platform team | GB | lead | business | 0.0 h | 0.0 | not priced |"
in content
)
assert "Cost a month: 43,410 GBP." in content
assert "The monthly budget is 120,000 GBP: this is within it." in content
assert "version `draft-2026-10-13`" in content
assert "Its figures are placeholders" in content
assert "Converted from USD at 0.79, a rate set on 2026-10-13." in content
assert json.loads(attachments[STAFFING_FILE])["cost_per_month"] == 43410
def test_a_cost_over_the_budget_is_shown_as_over_not_refused():
from tests.support import SPEC
tight = SPEC.replace("monthly_run_budget: 120000", "monthly_run_budget: 40000")
content, _ = viewed(spec=tight)
assert "this is over it by 3,410 GBP." in content
assert check(as_file(GRAPH_FILE, GRAPH), upstream(spec=tight)) == []
def test_the_organisation_chart_is_drawn_from_the_graph():
_, attachments = viewed()
chart = attachments[ORGANISATION_FILE]
assert chart.startswith("flowchart TB\n")
assert 'subgraph P_2["Managed service provider (provider)"]' in chart
assert 'R_2["Platform SRE<br/>PL, senior, 24x7"]' in chart
def test_findings_are_listed_for_whoever_owns_the_design():
content, attachments = viewed(findings=[FINDING])
assert "## Findings" in content
assert "**AF-1** (S-2): Nothing alerts when" in content
assert json.loads(attachments[FINDINGS_FILE]) == [FINDING]
def test_with_no_findings_the_file_is_empty_and_no_section_is_written():
content, attachments = viewed()
assert "## Findings" not in content
assert attachments[FINDINGS_FILE] == "[]"
def test_a_revision_replaces_the_views_and_never_stacks_them():
first, attachments = viewed()
second, _ = add_views(first, dict(attachments), upstream())
assert second == first
assert second.count("## Responsibility matrix") == 1

166
tests/test_guide.py Normal file
View File

@@ -0,0 +1,166 @@
"""The operating model guide: the packager writes the summary, code writes the rest."""
import json
from app.operating_model import guide, scenarios, spec, staffing
from app.operating_model.names import (
FINDINGS_FILE,
SCENARIOS_FILE,
STAFFING_FILE,
TARGET_OPERATING_MODEL,
VALIDATION_RESULTS,
VERDICTS_FILE,
)
from tests.support import (
CAPABILITIES,
CARD,
CATALOGUE,
GRAPH,
VERDICTS,
WORKFLOW,
upstream,
)
DRAFT = "# Northwind landing zone\n\n" + "".join(
f"## {section}\n\n"
+ (
"The provider runs the platform at every hour; Northwind approves changes.\n\n"
if section == guide.SUMMARY
else "It costs 9,999 GBP and needs 2 people.\n\n"
)
for section in guide.SECTIONS
)
def delivered(findings=None, model=GRAPH) -> dict:
"""Every Deliverable the guide is assembled from, with what code added to each."""
built = upstream(model=model, findings=findings)
scope = spec.agreed(built)
cost = staffing.compute(model, CAPABILITIES, CARD, "GBP")
built[TARGET_OPERATING_MODEL]["attachments"][STAFFING_FILE] = json.dumps(cost)
built[TARGET_OPERATING_MODEL]["attachments"].setdefault(FINDINGS_FILE, "[]")
walked = scenarios.walk(model, CAPABILITIES, scope, CATALOGUE, cost)
waiting = {
str(c["id"]): str(c["depends_on_finding"])
for c in model["checks"]
if c.get("depends_on_finding")
}
judged = json.loads(json.dumps(VERDICTS))
for verdict in judged["checks"]:
if verdict["check"] in waiting:
verdict.update(verdict="not_verifiable", evidence="Waits on its finding.")
built[VALIDATION_RESULTS] = {
"content": "# Validation\n",
"attachments": {
VERDICTS_FILE: scenarios.merge(json.dumps(judged), walked, waiting),
SCENARIOS_FILE: json.dumps(walked),
},
}
return built
def assembled(**options) -> str:
return guide.assemble(DRAFT, delivered(**options), CATALOGUE)
def section(text: str, heading: str) -> str:
return text.split(f"## {heading}\n")[1].split("\n## ")[0]
def test_the_guide_has_every_section_once_and_in_order():
text = assembled()
headings = [line[3:] for line in text.splitlines() if line.startswith("## ")]
assert headings == list(guide.SECTIONS)
assert text.startswith("# Northwind landing zone\n")
assert WORKFLOW.check_submission("operating_model_guide", text, {}) == ()
def test_the_packagers_summary_is_kept_and_the_key_figures_are_set_under_it():
summary = section(assembled(), guide.SUMMARY)
assert "The provider runs the platform at every hour" in summary
assert "### Key figures" in summary
assert "- Operated: 3 Items, with 2 Conditions to keep true." in summary
assert (
"- People: 5.1. Cost a month: 43,410 GBP, from rate card version "
"`draft-2026-10-13` (placeholder prices: an illustration)." in summary
)
assert "- Scenario walk-throughs: 10 of 10 pass." in summary
def test_a_figure_the_packager_wrote_under_another_heading_is_replaced():
text = assembled()
assert "9,999" not in text
assert "needs 2 people" not in text
assert "Cost a month: 43,410 GBP." in section(text, "Staffing and cost")
def test_each_section_is_written_from_the_data():
text = assembled()
assert "| S-2 | Audit log store | storage | Cloud Storage | E-2 |" in section(
text, "What is operated"
)
assert "| K-1 | S-1 | Flow logs are retained for 400 days. | CHK-12 |" in text
assert "| OR-2 | process |" in section(text, "Operational requirements")
assert (
"| C-1 | Incident management | OR-1, OR-4 | S-1, S-2, S-3 | 24x7 | 60 h |"
in section(text, "Capabilities")
)
who = section(text, "Who does what")
assert (
"| Platform SRE | Managed service provider (provider) | PL | senior | 24x7 |"
in who
)
assert "| Incident management (C-1) | C | R | A |" in who
checked = section(text, "Validation")
assert "| OC-1 | A fault on the hub network is acknowledged" in checked
assert "| Hub network (S-1) must be changed. | pass |" in checked
def test_sources_are_cited_and_said_to_be_unchecked():
cited = section(assembled(), "Sources")
assert "- ITIL 4: Incident management practice; Change enablement practice" in cited
assert "certifies nothing against them" in cited
assert "have not been checked against the frameworks by a person" in cited
def test_what_was_left_open_is_listed_and_what_nobody_asked_for_is_not():
left_open = section(assembled(), "Findings and open questions")
assert "No findings" in left_open
assert (
"`operations.change_process`: affects change enablement and release."
in left_open
)
assert "`service_management.tooling`: Which service management" in left_open
assert "metadata.author" not in left_open
def test_a_finding_and_the_check_that_waits_on_it_are_shown():
finding = {
"id": "AF-1",
"about": ["S-2"],
"needed_by": ["C-3"],
"lacks": "Nothing alerts when the retention lock is removed.",
"proposed": "Add an alert on a change to the retention lock.",
}
waiting = json.loads(json.dumps(GRAPH))
waiting["checks"][2]["depends_on_finding"] = "AF-1"
text = assembled(findings=[finding], model=waiting)
assert "**AF-1** (S-2): Nothing alerts when" in section(
text, "Findings and open questions"
)
assert "| waits on finding (AF-1) |" in section(text, "Validation")
assert "- Findings for whoever owns the design: 1." in text
def test_assembling_a_guide_again_changes_nothing():
first = assembled()
assert guide.assemble(first, delivered(), CATALOGUE) == first

218
tests/test_outcomes.py Normal file
View File

@@ -0,0 +1,218 @@
"""Measures tied to the customer's business outcomes."""
import json
import pytest
from app.operating_model import digest, guide
from app.operating_model import graph as model
from app.operating_model.names import GRAPH_FILE
from tests.support import CAPABILITIES, CARD, CATALOGUE, SPEC, as_file, graph, upstream
OUTCOMES = SPEC.replace(
" - {id: BO-1, outcome: Outages of the platform are short., target: Restored within 4 hours.}\n"
" - {id: BO-2, outcome: Regulators always get the evidence they ask for.}\n",
" - {id: BO-1, outcome: New products reach customers sooner., target: Daily releases.}\n"
" - {id: BO-2, outcome: Payments are never down for long.}\n",
)
# The same spec with no business outcome stated: only the customer can say none.
NO_OUTCOMES = (
OUTCOMES.split(" business_outcomes:\n")[0]
+ " business_outcomes: not_applicable\n"
+ " requirements:\n"
+ OUTCOMES.split(" requirements:\n")[1]
)
check = model.check(CARD, CATALOGUE.frameworks)
def measured() -> dict:
changed = graph()
changed["measures"] = [
{
"id": "M-1",
"name": "Time to restore service",
"layer": "outcome",
"outcome": "BO-2",
"measured_by": "C-1",
"framework": "dora-metrics",
"target": "Under 4 hours",
},
{
"id": "M-2",
"name": "Change lead time",
"layer": "outcome",
"outcome": "BO-1",
"measured_by": "C-2",
"framework": "dora-metrics",
},
]
return changed
def faults(changed, text=OUTCOMES, capabilities=None) -> list[str]:
return check(
as_file(GRAPH_FILE, changed), upstream(spec=text, capabilities=capabilities)
)
def test_every_outcome_shown_by_a_measure_is_accepted():
assert faults(measured()) == []
def test_with_no_outcomes_stated_no_measures_are_needed():
unmeasured = graph()
unmeasured["measures"] = []
assert faults(unmeasured, NO_OUTCOMES) == []
@pytest.mark.parametrize(
("change", "problem"),
[
(
lambda g: g["measures"].pop(1),
"business outcome BO-1 (New products reach customers sooner.) is shown by no Measure",
),
(
lambda g: g["measures"][0].update(outcome="BO-9"),
"Measure M-1 shows BO-9, which is not a business outcome the customer stated",
),
(
lambda g: g["measures"][0].update(layer="vanity"),
"Measure M-1: layer must be one of",
),
(
lambda g: g["measures"][0].update(measured_by="C-9"),
"Measure M-1 is measured by C-9, which is not a Capability",
),
(
lambda g: g["measures"][0].update(framework="okr"),
"Measure M-1 names framework okr, which is not in the register",
),
(
lambda g: g["responsibilities"].pop(1),
"Measure M-1 is measured by C-1, and nobody is responsible for it",
),
],
)
def test_a_measure_that_serves_no_stated_outcome_or_nobody_takes_is_refused(
change, problem
):
changed = measured()
change(changed)
found = faults(changed)
assert any(problem in fault for fault in found), found
def test_a_measure_is_refused_when_the_customer_stated_no_outcomes():
found = faults(measured(), NO_OUTCOMES)
assert any("none were stated" in fault for fault in found), found
def test_measuring_how_work_flows_measures_all_three_layers():
capabilities = [dict(c) for c in CAPABILITIES]
capabilities[2]["catalogue"] = "delivery-and-developer-experience-measurement"
changed = measured()
changed["measures"].append(
{
"id": "M-3",
"name": "Deployment frequency",
"layer": "outcome",
"outcome": "BO-1",
"measured_by": "C-3",
"framework": "dora-metrics",
}
)
found = faults(changed, capabilities=capabilities)
assert any(
"C-3 measures how work flows and has no Measure at the guardrail, leading layer"
in fault
for fault in found
), found
changed["measures"] += [
{
"id": "M-4",
"name": "Developer satisfaction",
"layer": "guardrail",
"outcome": "BO-1",
"measured_by": "C-3",
"framework": "space-framework",
},
{
"id": "M-5",
"name": "Perceived build wait",
"layer": "leading",
"outcome": "BO-1",
"measured_by": "C-3",
"framework": "devex-framework",
},
]
assert faults(changed, capabilities=capabilities) == []
def test_the_guide_shows_each_outcome_with_what_shows_it_and_who_answers():
built = upstream(spec=OUTCOMES, model=measured())
text = guide.business_outcomes(built, CATALOGUE)
assert (
"**BO-1: New products reach customers sooner.** Target: Daily releases." in text
)
assert (
"| Time to restore service | outcome | dora-metrics | Under 4 hours | "
"Incident management | Service manager |" in text
)
def test_the_guide_says_when_no_outcome_was_stated():
text = guide.business_outcomes(upstream(spec=NO_OUTCOMES), CATALOGUE)
assert text.startswith("The customer stated no business outcomes")
def test_an_operational_requirement_may_be_traced_to_a_business_outcome():
from app.operating_model import chain
from app.operating_model.names import REQUIREMENTS_FILE
from tests.support import REQUIREMENTS
requirements = json.loads(json.dumps(REQUIREMENTS))
requirements[3]["source"] = ["BO-2", "service_levels.recovery_time_hours"]
assert (
chain.check_requirements(
as_file(REQUIREMENTS_FILE, requirements), upstream(spec=OUTCOMES)
)
== []
)
def test_the_architectures_stated_success_criteria_become_business_outcomes():
from pathlib import Path
fixture = (
Path(__file__).parent / "fixtures" / "architecture" / "case-08-landing-zone"
)
sources = digest.read_folder(fixture)
scope = json.loads(sources[digest.SCOPE])
scope["stated"]["intent"]["success_criteria"] = [
"Payments engine onboarded within 10 weeks.",
"Audit findings closed before go-live.",
]
sources[digest.SCOPE] = json.dumps(scope)
digested = digest.architecture(
sources, digest.item_kinds(digest.Path(fixture).parents[3], "gcp")
)
assert digested.spec["intent"]["business_outcomes"] == [
{"id": "BO-1", "outcome": "Payments engine onboarded within 10 weeks."},
{"id": "BO-2", "outcome": "Audit findings closed before go-live."},
]
# In the real run they were assumed for the customer, so none are carried.
real = digest.architecture(digest.read_folder(fixture), {})
assert "business_outcomes" not in real.spec["intent"]

219
tests/test_scenarios.py Normal file
View File

@@ -0,0 +1,219 @@
"""Scenario walk-throughs: generated and traced by code, in place of a dry run."""
import json
from app.operating_model import scenarios, spec, staffing
from app.operating_model.names import VERDICTS_FILE
from tests.support import (
CAPABILITIES,
CARD,
CATALOGUE,
GRAPH,
SPEC,
VERDICTS,
graph,
scope_data,
)
def walked(model=GRAPH, capabilities=CAPABILITIES, text=SPEC) -> dict[str, dict]:
scope = scope_data(text)
currency = spec.value(scope, "commercial.currency")
cost = staffing.compute(model, capabilities, CARD, currency)
return {
item["id"]: item
for item in scenarios.walk(model, capabilities, scope, CATALOGUE, cost)
}
def test_every_scenario_of_a_sound_model_has_a_path():
found = walked()
assert set(found) == {
"SC-fault-S-1",
"SC-fault-S-2",
"SC-fault-S-3",
"SC-change-S-1",
"SC-change-S-2",
"SC-change-S-3",
"SC-condition-K-1",
"SC-condition-K-2",
"SC-party-P-2",
"SC-budget",
}
assert all(item["passes"] for item in found.values())
assert found["SC-fault-S-1"]["path"] == (
"Platform SRE responds under C-1, and Service manager is answerable."
)
assert found["SC-budget"]["path"] == "The cost is 43,410 GBP, within the budget."
def test_a_fault_out_of_hours_fails_when_nobody_responsible_is_on():
model = graph()
model["roles"][1]["hours"] = "business"
found = walked(model)
assert not found["SC-fault-S-1"]["passes"]
assert found["SC-fault-S-1"]["path"] == (
"No Role responsible for responding works 24x7 hours."
)
# Changes are made in business hours: that path still holds.
assert found["SC-change-S-1"]["passes"]
def test_an_item_no_responding_capability_covers_fails():
capabilities = [dict(c) for c in CAPABILITIES]
capabilities[0]["items"] = ["S-1", "S-2"]
found = walked(capabilities=capabilities)
assert (
found["SC-fault-S-3"]["path"]
== "No Capability that responds to faults covers it."
)
assert found["SC-fault-S-1"]["passes"]
def test_an_item_nobody_can_change_fails():
capabilities = [dict(c) for c in CAPABILITIES]
capabilities[1]["items"] = ["S-1", "S-2"]
assert not walked(capabilities=capabilities)["SC-change-S-3"]["passes"]
def test_a_condition_nothing_keeps_true_fails():
model = graph()
model["checks"].pop(1)
found = walked(model)
assert found["SC-condition-K-1"]["path"] == "No Check carries it."
assert found["SC-condition-K-2"]["passes"]
def test_a_party_with_nobody_else_to_turn_to_fails():
model = graph()
model["responsibilities"] = [
r for r in model["responsibilities"] if r["id"] not in ("RS-3", "RS-8")
]
found = walked(model)
assert not found["SC-party-P-2"]["passes"]
assert "C-1, C-3" in found["SC-party-P-2"]["path"]
def test_a_cost_over_the_budget_fails_the_budget_scenario():
tight = SPEC.replace("monthly_run_budget: 120000", "monthly_run_budget: 40000")
found = walked(text=tight)
assert found["SC-budget"]["path"] == "The cost is 43,410 GBP, over the budget."
def test_with_no_budget_stated_there_is_no_budget_scenario():
assert "SC-budget" not in walked(
text=SPEC.replace(" monthly_run_budget: 120000\n", "")
)
def test_the_scenarios_verdicts_are_written_into_the_validators_file_by_code():
found = list(walked().values())
merged = json.loads(scenarios.merge(json.dumps(VERDICTS), found))
assert merged["dry_run"] == {
"verdict": "pass",
"evidence": "10 of 10 scenario walk-throughs pass.",
}
assert [c["check"] for c in merged["checks"]][:4] == [
"OC-1",
"OC-2",
"OC-3",
"OC-4",
]
assert len(merged["checks"]) == 14
# Merging again replaces the scenarios' verdicts; it never adds them twice.
again = json.loads(scenarios.merge(json.dumps(merged), found))
assert again == merged
def test_a_failed_scenario_fails_the_run():
model = graph()
model["roles"][1]["hours"] = "business"
merged = json.loads(
scenarios.merge(json.dumps(VERDICTS), list(walked(model).values()))
)
assert merged["dry_run"]["verdict"] == "fail"
failed = [c["check"] for c in merged["checks"] if c["verdict"] == "fail"]
assert failed == ["SC-fault-S-1", "SC-fault-S-2", "SC-fault-S-3"]
def verdict_faults(verdicts, model=GRAPH) -> list[str]:
return scenarios.check_verdicts(
{VERDICTS_FILE: json.dumps(verdicts)}, model, VERDICTS_FILE
)
def test_a_verdict_on_every_check_is_accepted():
assert verdict_faults(VERDICTS) == []
def test_a_check_without_a_verdict_is_refused():
missing = {"checks": VERDICTS["checks"][:-1]}
assert verdict_faults(missing) == ["verdicts.json: Check OC-4 has no verdict"]
def test_a_check_that_waits_on_a_finding_cannot_pass():
model = graph()
model["checks"][2]["depends_on_finding"] = "AF-1"
assert (
"Check OC-3 depends on finding AF-1. It is not_verifiable while the finding stands"
in verdict_faults(VERDICTS, model)[0]
)
waiting = json.loads(json.dumps(VERDICTS))
waiting["checks"][2]["verdict"] = "not_verifiable"
assert verdict_faults(waiting, model) == []
def test_a_verdict_on_something_that_is_not_a_check_is_refused():
extra = {
"checks": [
*VERDICTS["checks"],
{"check": "OC-9", "verdict": "pass", "evidence": "x"},
]
}
assert verdict_faults(extra) == [
"verdicts.json: OC-9 is not a Check of the Target operating model"
]
def test_a_check_whose_finding_the_model_could_meet_itself_may_fail():
model = graph()
model["checks"][2]["depends_on_finding"] = "AF-1"
judged = json.loads(json.dumps(VERDICTS))
judged["checks"][2].update(
verdict="fail", evidence="Staff the review 24x7; this is not the solution's."
)
assert verdict_faults(judged, model) == []
merged = json.loads(scenarios.merge(json.dumps(judged), [], {"OC-3": "AF-1"}))
# The failure stands as a failure: the router sends the model back.
assert merged["checks"][2]["verdict"] == "fail"
def test_a_check_left_not_verifiable_on_its_finding_is_recorded_as_waiting():
judged = json.loads(json.dumps(VERDICTS))
judged["checks"][2].update(verdict="not_verifiable", evidence="Waits on AF-1.")
merged = json.loads(scenarios.merge(json.dumps(judged), [], {"OC-3": "AF-1"}))
assert merged["checks"][2]["verdict"] == "waits_on_finding"
assert merged["checks"][2]["finding"] == "AF-1"

161
tests/test_staffing.py Normal file
View File

@@ -0,0 +1,161 @@
"""Staffing and cost: arithmetic on the rate card, never an agent's figure."""
import shutil
import pytest
import yaml
from app.operating_model import staffing
from app.operating_model.staffing import RateCardError
from app.operating_model.workflow import ROOT
from tests.support import BUNDLE, CAPABILITIES, CARD, GRAPH, graph
SKILL = ROOT / "skills" / staffing.SKILL
def by_role(result) -> dict:
return {row["role"]: row for row in result["roles"]}
def test_the_figures_for_a_fixed_graph_and_card_are_reproduced_exactly():
result = staffing.compute(GRAPH, CAPABILITIES, CARD, "GBP")
rows = by_role(result)
# The SRE is responsible for incidents (60 h, 24x7) and changes (40 h): 100 h
# is two thirds of one person in Poland, but every hour of every day needs a
# rota of five. 5 x 460 a day x 220 days / 12 x 1.25 uplift x 0.79 to GBP.
assert rows["R-2"]["effort_hours_per_month"] == 100
assert rows["R-2"]["hours"] == "24x7"
assert rows["R-2"]["people"] == 5.0
assert rows["R-2"]["cost_per_month"] == 41640
# The service manager: 16 h of a 137.5 h month in the United Kingdom.
assert rows["R-3"]["people"] == 0.1
assert rows["R-3"]["cost_per_month"] == 1770
assert result["cost_per_month"] == 43410
assert result["by_party"] == {"P-2": 43410}
assert result["by_country"] == {"PL": 41640, "GB": 1770}
assert result["by_capability"] == {"C-1": 24984, "C-2": 16656, "C-3": 1770}
def test_the_customers_own_staff_are_counted_and_never_priced():
rows = by_role(staffing.compute(GRAPH, CAPABILITIES, CARD, "GBP"))
assert rows["R-1"]["cost_per_month"] is None
assert rows["R-1"]["people"] == 0.0
def test_a_customer_role_that_does_the_work_still_carries_no_cost():
model = graph()
model["responsibilities"][4]["role"] = "R-1" # the customer makes the changes
rows = by_role(staffing.compute(model, CAPABILITIES, CARD, "GBP"))
assert rows["R-1"]["effort_hours_per_month"] == 40
assert rows["R-1"]["people"] == 0.3
assert rows["R-1"]["cost_per_month"] is None
def test_a_role_on_business_hours_is_as_small_as_its_work():
model = graph()
model["roles"][1]["hours"] = "business"
capabilities = [dict(c, hours="business") for c in CAPABILITIES]
row = by_role(staffing.compute(model, capabilities, CARD, "GBP"))["R-2"]
assert row["hours"] == "business"
assert row["people"] == 0.7 # 100 h of a 146.7 h month, and no rota
def test_the_result_is_in_the_currency_the_spec_states():
pounds = staffing.compute(GRAPH, CAPABILITIES, CARD, "GBP")
dollars = staffing.compute(GRAPH, CAPABILITIES, CARD, "USD")
assert dollars["cost_per_month"] == 54948
assert pounds["conversion"] == {"from": "USD", "rate": 0.79, "as_of": "2026-10-13"}
assert dollars["conversion"]["rate"] == 1.0
def test_roles_that_share_a_capability_divide_its_effort():
model = graph()
model["responsibilities"].append(
{"id": "RS-9", "capability": "C-3", "role": "R-2", "kind": "responsible"}
)
equal = by_role(staffing.compute(model, CAPABILITIES, CARD, "GBP"))
model["responsibilities"][6]["share"] = 0.75
model["responsibilities"][-1]["share"] = 0.25
stated = by_role(staffing.compute(model, CAPABILITIES, CARD, "GBP"))
assert equal["R-2"]["effort_hours_per_month"] == 108 # 100 and half of 16
assert stated["R-2"]["effort_hours_per_month"] == 104 # 100 and a quarter of 16
@pytest.mark.parametrize(
("shares", "problem"),
[
((0.75, None), "must all state a share"),
((0.75, 0.5), "do not add up to 1"),
((1.5, -0.5), "must all state a share"),
],
)
def test_shares_that_do_not_hold_together_are_a_fault(shares, problem):
entries = [
{"capability": "C-3", "role": role, "kind": "responsible", "share": share}
for role, share in zip(("R-2", "R-3"), shares, strict=True)
]
assert problem in staffing.shares(entries)[1][0]
def test_the_card_in_the_repository_is_marked_as_placeholders():
assert CARD.is_draft
assert CARD.prices == ("provider",)
def test_the_currencies_a_spec_may_state_are_the_ones_the_card_converts_to():
assert list(BUNDLE.scope_values["commercial.currency"]) == CARD.currencies()
@pytest.fixture
def skill(tmp_path):
shutil.copytree(SKILL, tmp_path / "skill")
return tmp_path / "skill"
def edit(skill, change):
path = skill / "assets" / "rates.yaml"
data = yaml.safe_load(path.read_text())
change(data)
path.write_text(yaml.safe_dump(data))
@pytest.mark.parametrize(
("change", "problem"),
[
(
lambda d: d["prices"].append("customer"),
"the customer's own staff are never priced",
),
(
lambda d: d["countries"]["PL"]["day_rate"].pop("lead"),
"PL has no day rate for level lead",
),
(lambda d: d["cover"].pop("24x7"), "cover for 24x7 needs a minimum_roster"),
(
lambda d: d["conversion"]["rates"].update(GBP=0),
"conversion rate for GBP must be a number above zero",
),
(lambda d: d.pop("currency"), "needs currency"),
],
)
def test_a_rate_card_that_does_not_hold_together_is_refused(skill, change, problem):
edit(skill, change)
with pytest.raises(RateCardError, match=problem):
staffing.load(skill)
def test_a_card_whose_version_is_set_is_no_longer_a_draft(skill):
edit(skill, lambda d: d.update(version="2026-11"))
assert not staffing.load(skill).is_draft

207
tests/test_teams.py Normal file
View File

@@ -0,0 +1,207 @@
"""Teams and how they work together: Team Topologies in the operating model graph."""
import pytest
from app.operating_model import graph as model
from app.operating_model import guide
from app.operating_model.names import GRAPH_FILE
from tests.support import (
CARD,
CATALOGUE,
GRAPH,
SPEC,
as_file,
graph,
scope_data,
upstream,
)
check = model.check(CARD)
def with_teams() -> dict:
"""The test model, with the provider's Roles in a platform team that the
customer's platform owner consumes as a service."""
teamed = graph()
teamed["teams"] = [
{
"id": "T-1",
"party": "P-1",
"name": "Northwind product squad",
"type": "stream_aligned",
},
{
"id": "T-2",
"party": "P-2",
"name": "Platform team",
"type": "platform",
"purpose": "Runs the landing zone as a product.",
},
{"id": "T-3", "party": "P-2", "name": "Adoption", "type": "enabling"},
]
teamed["roles"][0]["team"] = "T-1"
teamed["roles"][1]["team"] = "T-2"
teamed["roles"][2]["team"] = "T-2"
teamed["interactions"] = [
{
"id": "I-1",
"from": "T-1",
"to": "T-2",
"mode": "x_as_a_service",
"purpose": "Environments through the self-service portal.",
},
{"id": "I-2", "from": "T-3", "to": "T-1", "mode": "facilitating"},
{
"id": "I-3",
"from": "T-1",
"to": "T-2",
"mode": "collaboration",
"until": "the first workload is live",
},
]
return teamed
def faults(changed) -> list[str]:
return check(as_file(GRAPH_FILE, changed), upstream())
def test_teams_are_optional():
assert faults(GRAPH) == []
def test_teams_of_the_four_kinds_working_in_the_three_modes_are_accepted():
assert faults(with_teams()) == []
@pytest.mark.parametrize(
("change", "problem"),
[
(
lambda g: g["teams"][1].update(type="squad"),
"Team T-2: type must be one of stream_aligned, platform",
),
(
lambda g: g["teams"][1].update(party="P-9"),
"Team T-2 is in P-9, which is not a Party",
),
(
lambda g: g["roles"][1].update(team="T-9"),
"Role R-2 is in T-9, which is not a Team",
),
(
lambda g: g["roles"][0].update(team="T-2"),
"Role R-1 is in P-1, and its team T-2 is in P-2",
),
(
lambda g: g["interactions"][0].update(mode="ticket"),
"Interaction I-1: mode must be one of",
),
(
lambda g: g["interactions"][0].update(to="T-9"),
"Interaction I-1: to T-9 is not a Team",
),
(
lambda g: g["interactions"][0].update(to="T-1"),
"Interaction I-1 is between a team and itself",
),
(
lambda g: g["interactions"][2].pop("until"),
"Interaction I-3 is a collaboration and does not say when it ends",
),
(lambda g: g["teams"][0].pop("type"), "(T-1) needs type"),
],
)
def test_teams_that_do_not_hold_together_are_refused(change, problem):
changed = with_teams()
change(changed)
found = faults(changed)
assert any(problem in fault for fault in found), found
def test_a_check_may_be_on_a_team_or_an_interaction():
changed = with_teams()
changed["checks"].append(
{
"id": "OC-9",
"on": "I-1",
"source": "OR-2",
"condition": "Every environment request is met through the portal, with no ticket.",
}
)
assert faults(changed) == []
def test_the_chart_draws_each_team_inside_its_party_and_how_teams_work_together():
chart = model.render_organisation(with_teams())
assert ' subgraph T_2["Platform team (platform team)"]' in chart
assert ' R_2["Platform SRE<br/>PL, senior, 24x7"]' in chart
assert ' T_1 -- "x-as-a-service" --> T_2' in chart
assert ' T_1 -- "collaboration until the first workload is live" --> T_2' in chart
def test_the_chart_without_teams_is_as_it_was():
chart = model.render_organisation(GRAPH)
assert " R_2[" in chart
assert " -- " not in chart
def test_the_guide_shows_the_teams_and_their_interactions():
built = upstream(model=with_teams())
text = guide.who_does_what(built, CATALOGUE)
assert (
"| Platform team | platform | Managed service provider | Platform SRE, Service manager | Runs the landing zone as a product. |"
in text
)
assert (
"| Northwind product squad | Platform team | x-as-a-service | | Environments through the self-service portal. |"
in text
)
# The matrix still follows.
assert "| Incident management (C-1) |" in text
def test_a_platform_foundation_is_proposed_the_platform_and_measurement_capabilities():
proposed = CATALOGUE.applicable(scope_data(SPEC))
assert {
"platform-product-management",
"self-service-and-golden-paths",
"enablement",
"delivery-and-developer-experience-measurement",
} <= set(proposed)
def test_a_workload_is_not_proposed_the_platform_capabilities():
workload = SPEC.replace("kind: platform_foundation", "kind: workload")
proposed = CATALOGUE.applicable(scope_data(workload))
assert "platform-product-management" not in proposed
assert "delivery-and-developer-experience-measurement" in proposed
def test_measurement_cites_three_layers_and_none_alone():
entry = CATALOGUE.entries["delivery-and-developer-experience-measurement"]
assert [s["framework"] for s in entry.sources] == [
"dora-metrics",
"space-framework",
"devex-framework",
]
assert len(entry.typical_checks) == 3
def test_the_socio_technical_frameworks_are_registered():
names = {key: value["name"] for key, value in CATALOGUE.frameworks.items()}
assert names["team-topologies"].startswith("Team Topologies")
assert "platform as a product" in names["platform-as-a-product"]
assert names["space-framework"] == "The SPACE of Developer Productivity"
assert names["devex-framework"].startswith("DevEx")