From 628b6ee170384a51128a06f2762d27148d58a1c6 Mon Sep 17 00:00:00 2001 From: Jonathan Boniface Date: Thu, 22 Oct 2026 09:25:03 +0100 Subject: [PATCH] docs: publish the runs of 2026-10-21 The evaluation (both cases three times: report, results, spread, each run's Deliverables and lesson candidates) and the hand-off through the Orchestration engine (report, the document of links, both agents' kept files and records). Benchmark cases only, all fictional; no machine paths, and the links carry no signatures. Other runs stay ignored. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 8 +- .../operating_model/capabilities.json | 470 ++++++ .../operating_model/findings.json | 23 + .../operating_model/operating-model-guide.md | 331 ++++ .../operational-requirements.json | 201 +++ .../operating_model/organisation.mmd | 26 + .../operating_model/record.json | 1 + .../operating_model/scenarios.json | 310 ++++ .../operating_model/scope.json | 495 ++++++ .../operating_model/scope.yaml | 466 ++++++ .../operating_model/staffing.json | 144 ++ .../target-operating-model.json | 742 +++++++++ .../operating_model/verdicts.json | 362 +++++ .../20261021T203623Z-handoff/report.md | 19 + .../20261021T203623Z-handoff/result.json | 71 + .../solution_architecture/architecture.mmd | 72 + .../solution_architecture/components.json | 57 + .../solution_architecture/implementation.json | 123 ++ .../solution_architecture/products.json | 13 + .../recommendations.json | 123 ++ .../solution_architecture/record.json | 1 + .../solution_architecture/requirements.json | 304 ++++ .../solution_architecture/scope.json | 219 +++ .../solution_architecture/scope.yaml | 282 ++++ .../solution-architecture-guide.md | 1395 +++++++++++++++++ .../target-architecture.json | 1214 ++++++++++++++ .../solution_architecture/terraform/main.tf | 603 +++++++ .../terraform/variables.tf | 31 + .../terraform/versions.tf | 15 + .../solution_architecture/verdicts.json | 614 ++++++++ .../solution_architecture/work-breakdown.json | 367 +++++ .../20261021T203623Z-handoff/task.json | 1068 +++++++++++++ .../capabilities.json | 191 +++ .../execution.json | 113 ++ .../findings.json | 1 + .../operating-model-guide.md | 229 +++ .../operational-requirements.json | 84 + .../organisation.mmd | 15 + .../scenarios.json | 93 ++ .../case-01-landing-zone-shared-r1/scope.json | 202 +++ .../case-01-landing-zone-shared-r1/scope.yaml | 198 +++ .../staffing.json | 102 ++ .../target-operating-model.json | 540 +++++++ .../verdicts.json | 149 ++ .../capabilities.json | 329 ++++ .../execution.json | 113 ++ .../findings.json | 1 + .../operating-model-guide.md | 234 +++ .../operational-requirements.json | 76 + .../organisation.mmd | 20 + .../scenarios.json | 93 ++ .../case-01-landing-zone-shared-r2/scope.json | 202 +++ .../case-01-landing-zone-shared-r2/scope.yaml | 199 +++ .../staffing.json | 128 ++ .../target-operating-model.json | 615 ++++++++ .../verdicts.json | 149 ++ .../capabilities.json | 191 +++ .../execution.json | 113 ++ .../findings.json | 1 + .../operating-model-guide.md | 237 +++ .../operational-requirements.json | 84 + .../organisation.mmd | 18 + .../scenarios.json | 93 ++ .../case-01-landing-zone-shared-r3/scope.json | 203 +++ .../case-01-landing-zone-shared-r3/scope.yaml | 201 +++ .../staffing.json | 117 ++ .../target-operating-model.json | 642 ++++++++ .../verdicts.json | 169 ++ .../bench-landing-zone-shared/intent/001.md | 10 + .../intent/001.scope.completed.yaml | 198 +++ .../intent/001.scope.yaml | 53 + .../bench-landing-zone-shared/intent/002.md | 10 + .../intent/002.scope.completed.yaml | 199 +++ .../intent/002.scope.yaml | 53 + .../bench-landing-zone-shared/intent/003.md | 10 + .../intent/003.scope.completed.yaml | 201 +++ .../intent/003.scope.yaml | 53 + .../intent/history.json | 53 + .../capabilities.json | 191 +++ .../execution.json | 146 ++ .../findings.json | 1 + .../operational-requirements.json | 143 ++ .../organisation.mmd | 23 + .../scenarios.json | 219 +++ .../case-02-from-architecture-r1/scope.json | 396 +++++ .../case-02-from-architecture-r1/scope.yaml | 406 +++++ .../staffing.json | 144 ++ .../target-operating-model.json | 667 ++++++++ .../verdicts.json | 249 +++ .../capabilities.json | 191 +++ .../execution.json | 151 ++ .../findings.json | 1 + .../operating-model-guide.md | 280 ++++ .../operational-requirements.json | 126 ++ .../organisation.mmd | 19 + .../scenarios.json | 219 +++ .../case-02-from-architecture-r2/scope.json | 401 +++++ .../case-02-from-architecture-r2/scope.yaml | 407 +++++ .../staffing.json | 119 ++ .../target-operating-model.json | 664 ++++++++ .../verdicts.json | 264 ++++ .../capabilities.json | 187 +++ .../execution.json | 151 ++ .../findings.json | 1 + .../operating-model-guide.md | 288 ++++ .../operational-requirements.json | 126 ++ .../organisation.mmd | 32 + .../scenarios.json | 240 +++ .../case-02-from-architecture-r3/scope.json | 402 +++++ .../case-02-from-architecture-r3/scope.yaml | 414 +++++ .../staffing.json | 144 ++ .../target-operating-model.json | 722 +++++++++ .../verdicts.json | 264 ++++ .../cases/case-08-landing-zone/intent/001.md | 15 + .../intent/001.scope.completed.yaml | 406 +++++ .../intent/001.scope.yaml | 10 + .../intent/001.sources/components.json | 79 + .../intent/001.sources/execution.json | 140 ++ .../intent/001.sources/products.json | 38 + .../intent/001.sources/recommendations.json | 248 +++ .../intent/001.sources/requirements.json | 280 ++++ .../intent/001.sources/scope.json | 233 +++ .../001.sources/target-architecture.json | 1210 ++++++++++++++ .../intent/001.sources/verdicts.json | 529 +++++++ .../intent/001.sources/work-breakdown.json | 332 ++++ .../cases/case-08-landing-zone/intent/002.md | 15 + .../intent/002.scope.completed.yaml | 407 +++++ .../intent/002.scope.yaml | 10 + .../intent/002.sources/components.json | 79 + .../intent/002.sources/execution.json | 140 ++ .../intent/002.sources/products.json | 38 + .../intent/002.sources/recommendations.json | 248 +++ .../intent/002.sources/requirements.json | 280 ++++ .../intent/002.sources/scope.json | 233 +++ .../002.sources/target-architecture.json | 1210 ++++++++++++++ .../intent/002.sources/verdicts.json | 529 +++++++ .../intent/002.sources/work-breakdown.json | 332 ++++ .../cases/case-08-landing-zone/intent/003.md | 15 + .../intent/003.scope.completed.yaml | 414 +++++ .../intent/003.scope.yaml | 10 + .../intent/003.sources/components.json | 79 + .../intent/003.sources/execution.json | 140 ++ .../intent/003.sources/products.json | 38 + .../intent/003.sources/recommendations.json | 248 +++ .../intent/003.sources/requirements.json | 280 ++++ .../intent/003.sources/scope.json | 233 +++ .../003.sources/target-architecture.json | 1210 ++++++++++++++ .../intent/003.sources/verdicts.json | 529 +++++++ .../intent/003.sources/work-breakdown.json | 332 ++++ .../case-08-landing-zone/intent/history.json | 83 + .../case-01-landing-zone-shared-r1.json | 76 + .../case-01-landing-zone-shared-r2.json | 64 + .../case-01-landing-zone-shared-r3.json | 59 + .../case-02-from-architecture-r1.json | 244 +++ .../case-02-from-architecture-r2.json | 59 + .../case-02-from-architecture-r3.json | 114 ++ eval/results/20261021T203624Z/report.md | 39 + eval/results/20261021T203624Z/results.json | 891 +++++++++++ eval/results/20261021T203624Z/spread.json | 68 + 159 files changed, 37817 insertions(+), 2 deletions(-) create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/capabilities.json create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/findings.json create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/operating-model-guide.md create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/operational-requirements.json create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/organisation.mmd create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/record.json create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/scenarios.json create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/scope.json create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/scope.yaml create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/staffing.json create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/target-operating-model.json create mode 100644 eval/results/20261021T203623Z-handoff/operating_model/verdicts.json create mode 100644 eval/results/20261021T203623Z-handoff/report.md create mode 100644 eval/results/20261021T203623Z-handoff/result.json create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/architecture.mmd create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/components.json create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/implementation.json create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/products.json create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/recommendations.json create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/record.json create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/requirements.json create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/scope.json create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/scope.yaml create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/solution-architecture-guide.md create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/target-architecture.json create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/terraform/main.tf create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/terraform/variables.tf create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/terraform/versions.tf create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/verdicts.json create mode 100644 eval/results/20261021T203623Z-handoff/solution_architecture/work-breakdown.json create mode 100644 eval/results/20261021T203623Z-handoff/task.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/capabilities.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/execution.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/findings.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/operating-model-guide.md create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/operational-requirements.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/organisation.mmd create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scenarios.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scope.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/staffing.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/target-operating-model.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/verdicts.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/capabilities.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/execution.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/findings.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/operating-model-guide.md create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/operational-requirements.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/organisation.mmd create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scenarios.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scope.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/staffing.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/target-operating-model.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/verdicts.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/capabilities.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/execution.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/findings.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/operating-model-guide.md create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/operational-requirements.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/organisation.mmd create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scenarios.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scope.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/staffing.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/target-operating-model.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/verdicts.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.md create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.scope.completed.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.md create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.scope.completed.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.md create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.scope.completed.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/history.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/capabilities.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/execution.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/findings.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/operational-requirements.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/organisation.mmd create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scenarios.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scope.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/staffing.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/target-operating-model.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/verdicts.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/capabilities.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/execution.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/findings.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/operating-model-guide.md create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/operational-requirements.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/organisation.mmd create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scenarios.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scope.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/staffing.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/target-operating-model.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/verdicts.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/capabilities.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/execution.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/findings.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/operating-model-guide.md create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/operational-requirements.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/organisation.mmd create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scenarios.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scope.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/staffing.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/target-operating-model.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/verdicts.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.md create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.scope.completed.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/components.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/execution.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/products.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/recommendations.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/requirements.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/scope.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/target-architecture.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/verdicts.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/work-breakdown.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.md create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.scope.completed.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/components.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/execution.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/products.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/recommendations.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/requirements.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/scope.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/target-architecture.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/verdicts.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/work-breakdown.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.md create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.scope.completed.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.scope.yaml create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/components.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/execution.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/products.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/recommendations.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/requirements.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/scope.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/target-architecture.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/verdicts.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/work-breakdown.json create mode 100644 eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/history.json create mode 100644 eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r1.json create mode 100644 eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r2.json create mode 100644 eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r3.json create mode 100644 eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r1.json create mode 100644 eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r2.json create mode 100644 eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r3.json create mode 100644 eval/results/20261021T203624Z/report.md create mode 100644 eval/results/20261021T203624Z/results.json create mode 100644 eval/results/20261021T203624Z/spread.json diff --git a/.gitignore b/.gitignore index 66713d0..665d089 100644 --- a/.gitignore +++ b/.gitignore @@ -19,8 +19,12 @@ venv/ # name people, teams and suppliers deliverables/ -# Evaluation run outputs -eval/results/ +# Evaluation run outputs, but for the runs published (they hold fictional benchmark +# data only, and links without their signatures) +eval/results/* +!eval/results/20261021T203624Z/ +!eval/results/20261021T203623Z-handoff/ +!eval/results/20261021T203624Z/deliverables/ # Lesson candidates captured from Executions (may quote user feedback) lessons/candidates/ diff --git a/eval/results/20261021T203623Z-handoff/operating_model/capabilities.json b/eval/results/20261021T203623Z-handoff/operating_model/capabilities.json new file mode 100644 index 0000000..f73551b --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/capabilities.json @@ -0,0 +1,470 @@ +[ + { + "id": "C-1", + "catalogue": "incident-management", + "serves": [ + "OR-1" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "24x7", + "effort_hours_per_month": 40, + "basis": "Estimated 5 serious incidents per month across the foundation, taking 8 hours each to resolve and review." + }, + { + "id": "C-2", + "catalogue": "monitoring-and-alerting", + "serves": [ + "OR-1", + "OR-5", + "OR-15" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "24x7", + "effort_hours_per_month": 60, + "basis": "Continuous triage of alerts across 11 platform components, averaging 2 hours per day." + }, + { + "id": "C-3", + "catalogue": "problem-management", + "serves": [ + "OR-2" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Root cause analysis for 2 major recurring issues per month, taking 10 hours each." + }, + { + "id": "C-4", + "catalogue": "service-request-fulfilment", + "serves": [ + "OR-13", + "OR-14" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 30, + "basis": "Processing 15 complex service requests per month at 2 hours each." + }, + { + "id": "C-5", + "catalogue": "change-enablement", + "serves": [ + "OR-7" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Reviewing and approving 40 infrastructure pull requests per month at 0.5 hours each." + }, + { + "id": "C-6", + "catalogue": "release-and-deployment", + "serves": [ + "OR-7" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Managing CI/CD pipeline deployments and troubleshooting failed releases, estimated at 20 hours per month." + }, + { + "id": "C-7", + "catalogue": "configuration-management", + "serves": [ + "OR-4", + "OR-17", + "OR-18" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Monthly audits of declarative state against deployed resources." + }, + { + "id": "C-8", + "catalogue": "backup-and-recovery", + "serves": [ + "OR-4" + ], + "items": [ + "S-7", + "S-8", + "S-10" + ], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Monthly testing of restores for Cloud KMS, Audit Logs, and Terraform State buckets." + }, + { + "id": "C-9", + "catalogue": "service-continuity", + "serves": [ + "OR-3", + "OR-11" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 16, + "basis": "Bi-monthly DR failover rehearsals, averaging 16 hours of preparation and execution per month." + }, + { + "id": "C-10", + "catalogue": "capacity-and-performance", + "serves": [ + "OR-2", + "OR-5", + "OR-6", + "OR-10" + ], + "items": [ + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Reviewing network throughput, interconnect capacity, and NVA scaling metrics monthly." + }, + { + "id": "C-11", + "catalogue": "access-management", + "serves": [ + "OR-9", + "OR-14", + "OR-21" + ], + "items": [ + "S-5", + "S-6", + "S-7" + ], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Fulfilling 20 access grants/revocations per month and conducting monthly access reviews." + }, + { + "id": "C-12", + "catalogue": "vulnerability-and-patch-management", + "serves": [ + "OR-20" + ], + "items": [ + "S-3" + ], + "hours": "business", + "effort_hours_per_month": 25, + "basis": "Weekly vulnerability scanning review and patching of the Network Virtual Appliances (NVAs)." + }, + { + "id": "C-13", + "catalogue": "security-monitoring-and-response", + "serves": [ + "OR-8", + "OR-17", + "OR-22" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "24x7", + "effort_hours_per_month": 60, + "basis": "Triage and response to security events across the platform 24x7, averaging 2 hours per day." + }, + { + "id": "C-14", + "catalogue": "cost-management", + "serves": [ + "OR-10" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 8, + "basis": "Monthly budget variance reporting and anomaly review." + }, + { + "id": "C-15", + "catalogue": "service-level-management", + "serves": [ + "OR-2" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 8, + "basis": "Monthly aggregation and reporting of service level objectives." + }, + { + "id": "C-16", + "catalogue": "platform-product-management", + "serves": [ + "OR-12" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 40, + "basis": "Backlog grooming, roadmap planning, and measuring adoption for the platform foundation." + }, + { + "id": "C-17", + "catalogue": "self-service-and-golden-paths", + "serves": [ + "OR-12" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 40, + "basis": "Developing and maintaining automated self-service modules and golden path documentation." + }, + { + "id": "C-18", + "catalogue": "enablement", + "serves": [ + "OR-12" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 40, + "basis": "Pairing and onboarding sessions with application teams to adopt the landing zone." + }, + { + "id": "C-19", + "catalogue": "delivery-and-developer-experience-measurement", + "serves": [ + "OR-12" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Monthly compilation of DORA metrics, SPACE surveys, and DevEx feedback." + }, + { + "id": "C-20", + "catalogue": "supplier-management", + "serves": [ + "OR-13", + "OR-19" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Monthly service reviews with Google Cloud and hybrid interconnect carrier." + }, + { + "id": "C-21", + "catalogue": "regulatory-evidence", + "serves": [ + "OR-16", + "OR-19", + "OR-20", + "OR-22", + "OR-23", + "OR-24" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4", + "S-5", + "S-6", + "S-7", + "S-8", + "S-9", + "S-10", + "S-11" + ], + "hours": "business", + "effort_hours_per_month": 30, + "basis": "Continuous collation of evidence for DORA, ISO 27001, and PCI-DSS v4.0 audits." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/operating_model/findings.json b/eval/results/20261021T203623Z-handoff/operating_model/findings.json new file mode 100644 index 0000000..3dd4362 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/findings.json @@ -0,0 +1,23 @@ +[ + { + "id": "AF-1", + "about": ["S-3"], + "needed_by": ["C-1"], + "lacks": "The solution lacks a secure remote access mechanism (such as a bastion host or Identity-Aware Proxy) as a way in for responders to the Centralised Egress Traffic Inspection Gateway.", + "proposed": "Add Identity-Aware Proxy or a secure bastion host to allow responders to securely access instances." + }, + { + "id": "AF-2", + "about": ["S-11"], + "needed_by": ["C-14"], + "lacks": "The solution lacks automated budget anomaly alerts to ensure the warm standby environment maintains a minimal pilot light footprint.", + "proposed": "Configure Google Cloud Billing Budgets and alerts tied to the standby environment." + }, + { + "id": "AF-3", + "about": ["S-2", "S-4"], + "needed_by": ["C-7"], + "lacks": "The solution lacks an automated backup service or vault to ensure state and configuration achieve the maximum data loss window (RPO) of 1 hour.", + "proposed": "Implement Google Cloud Backup and DR service or enable automated object versioning with lifecycle policies." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/operating_model/operating-model-guide.md b/eval/results/20261021T203623Z-handoff/operating_model/operating-model-guide.md new file mode 100644 index 0000000..a5ae66c --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/operating-model-guide.md @@ -0,0 +1,331 @@ +# Operating model guide: Google Cloud Landing Zone for Core Payments Engine + +## Summary + +This operating model guide governs the enterprise-grade Google Cloud platform foundation landing zone supporting the core payments engine, spanning multi-tier environment isolation, dedicated hybrid network interconnect, transit network fabric, centralised egress traffic inspection gateways, corporate identity federation, pipeline identity brokering, cryptographic key management, centralised immutable audit logging, governance guardrails, declarative infrastructure provisioning, and operational telemetry. The Managed Service Provider operates the platform foundation under a provider-operated model, holding operational accountability for infrastructure reliability, platform engineering, security monitoring, and regulatory evidence collation. The customer maintains workload governance and consumes foundation services autonomously via self-service, serving as a consulted stakeholder for architecture reviews and release approvals. + +Operational fault response, telemetry triage, and security monitoring are maintained on a continuous 24x7 basis, while platform engineering changes, release deployments, enablement, and compliance governance operate during standard business hours. The reader should look first at the architectural findings and the operational checks that wait on them, where missing capabilities in the solution design—specifically automated backup tooling to meet the recovery point objective, automated budget anomaly alerts for the warm standby environment, and secure remote access for egress inspection appliances—require design resolution before full operational verification can be achieved. + +### Key figures + +- Operated: 11 Items, with 21 Conditions to keep true. +- Faults responded to: 24x7 hours. +- Capabilities: 21. +- People: 12.2. Cost a month: 87,959 GBP, from rate card version `draft-2026-10-13` (placeholder prices: an illustration). +- Scenario walk-throughs: 44 of 44 pass. +- Business outcomes: 3, 3 shown by a Measure. +- Findings for whoever owns the design: 3. + +## Business outcomes + +**BO-1: Continuous and resilient payment processing with high availability and low-latency hybrid connectivity** + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Platform Availability | service level | | 99.9% | Service level management | Platform Reliability Lead | +| Incident Response Time | service level | | Under 30 minutes | Service level management | Platform Reliability Lead | + +**BO-2: Sustained regulatory compliance with DORA and PCI-DSS v4.0 security and resilience standards** + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Compliance Audit Pass Rate | service level | | 100% | Regulatory evidence and reporting | Compliance Officer | + +**BO-3: Rapid and secure platform change releases through automated CI/CD delivery pipelines** + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Deployment frequency | outcome | dora-metrics | Multiple times a day | Delivery and developer experience measurement | Product Manager | +| Satisfaction and well-being | guardrail | space-framework | Positive trend | Delivery and developer experience measurement | Product Manager | +| Feedback loops | leading | devex-framework | Minimal friction | Delivery and developer experience measurement | Product Manager | + +## What is operated + +| Item | What it is | Kind | Product | From | +| --- | --- | --- | --- | --- | +| S-1 | Multi-Tier Environment Hierarchy & Isolation Boundary | governance | Resource Manager | C-1 | +| S-2 | Dedicated Hybrid Network Interconnect Gateway | network | Cloud Interconnect | C-2 | +| S-3 | Centralised Egress Traffic Inspection Gateway | compute | Compute Engine | C-3 | +| S-4 | Transit Network Fabric & Routing Controller | service | Virtual Private Cloud | C-4 | +| S-5 | Corporate Identity Federation Gateway | identity | Cloud Identity | C-5 | +| S-6 | Automated Delivery Pipeline Identity Broker | identity | Workload Identity Federation | C-6 | +| S-7 | Cryptographic Key Management & Rotation Service | keys | Cloud KMS | C-7 | +| S-8 | Centralised Immutable Audit Log Repository | storage | Cloud Storage | C-8 | +| S-9 | Platform Compliance & Governance Guardrail Controller | governance | Organization Policy Service | C-9 | +| S-10 | Declarative Infrastructure Provisioning & State Engine | storage | Cloud Storage | C-10 | +| S-11 | Centralised Operational Telemetry & Alerting Engine | observability | Cloud Monitoring | C-11 | + +What must be kept true while it is run: + +| Condition | On | What must hold | From | +| --- | --- | --- | --- | +| K-1 | S-1, S-7, S-9 | Customer data and payment processing remain within United Kingdom and European Union legal jurisdictions. | NFR-1 | +| K-2 | S-1, S-7, S-9 | Warm standby environment operates with reduced capacity until activated during a disaster recovery failover. | NFR-1 | +| K-3 | S-1, S-9 | PCI-regulated production environment is isolated from non-PCI scopes according to PCI-DSS v4.0 network segmentation requirements. | NFR-2 | +| K-4 | S-3, S-4 | Workload APIs and payment services are accessible exclusively through private corporate network paths. | NFR-3 | +| K-5 | S-2 | Hybrid connection maintains low-latency, SLA-backed private data transmission between on-premises and Google Cloud. | NFR-4 | +| K-6 | S-3 | Centralised inspection appliances perform next-generation stateful packet inspection, URL filtering, and threat detection on outbound internet sessions. | NFR-5 | +| K-7 | S-2, S-4 | Platform foundation failover to europe-west1 can be completed within 4 hours during a primary region outage. | NFR-6 | +| K-8 | S-2, S-4 | Configuration drift and backup state achieve a maximum data loss window (RPO) of 1 hour. | NFR-6 | +| K-9 | S-5, S-6 | Corporate user lifecycle management and MFA enforcement are governed centrally by Entra ID conditional access policies. | NFR-7 | +| K-10 | S-1, S-9 | Platform governance and resilience architecture comply with DORA regulatory testing and third-party risk management frameworks. | NFR-8 | +| K-11 | S-1, S-9 | Production cardholder data environment satisfies PCI-DSS v4.0 isolation and security control requirements. | NFR-8 | +| K-12 | S-7 | Confidential payment data is secured against unauthorized platform-level access or data exposure. | NFR-9 | +| K-13 | S-8 | Audit logs cannot be modified, deleted, or shortened by any identity or administrator prior to retention expiration. | NFR-10 | +| K-14 | S-2, S-4, S-11 | Monthly availability of the core platform landing zone and network routing meets or exceeds 99.9%. | NFR-11 | +| K-15 | S-9 | Primary and secondary regions leverage Google Cloud data centres with published carbon-free energy (CFE) performance metrics. | NFR-12 | +| K-16 | S-6, S-10 | Platform foundation delivery and workload ingestion readiness are achieved within 10 weeks of project commencement. | NFR-13 | +| K-17 | S-8, S-10, S-11 | Central platform team operates 24/7 support for infrastructure incidents and availability alerts. | NFR-14 | +| K-18 | S-8, S-10, S-11 | Pull request workflows require automated CI/CD linting, validation, and peer approval prior to merge and deployment. | NFR-14 | +| K-19 | S-3 | NVA instances use machine families that support high bandwidth networking. | DR-3 | +| K-20 | S-3 | The warm standby environment operates at a minimal pilot light footprint during normal operations. | DR-11 | +| K-21 | S-3 | Automated disaster recovery scripts scale the standby environment to full capacity only during a failover event. | DR-11 | + +## Operational requirements + +| Requirement | Kind | What must be achieved | From | +| --- | --- | --- | --- | +| OR-1 | service_level | Priority 1 incidents on core platform foundation components are acknowledged within 30 minutes (assumed under service_levels.priority_1_response_minutes), supported continuously 24 hours a day, 7 days a week. | operations.support_hours, service_levels.priority_1_response_minutes, K-17 | +| OR-2 | service_level | The core platform landing zone and network routing maintain at least 99.9% monthly availability, operating 24 hours a day, 7 days a week. | K-14, operations.availability_hours | +| OR-3 | service_level | Platform foundation failover to the secondary recovery region is completed within 4 hours (assumed under service_levels.recovery_time_hours) in the event of a primary region outage. | K-7, service_levels.recovery_time_hours | +| OR-4 | service_level | Platform infrastructure configuration drift and backup state maintain a recovery point objective (RPO) of no more than 1 hour of data loss. | K-8 | +| OR-5 | service_level | Dedicated hybrid connectivity between on-premises environments and the cloud platform maintains SLA-backed, low-latency private data transmission. | K-5 | +| OR-6 | service_level | Centralised inspection appliances maintain high-bandwidth network throughput capacity to prevent traffic constraints during peak transaction flows. | K-19 | +| OR-7 | process | All infrastructure changes require automated pipeline validation, linting, and peer approval via pull requests prior to merge and release (assumed under operations.change_process). | K-18, operations.change_process | +| OR-8 | process | Outbound internet traffic is inspected through centralised stateful packet inspection, URL filtering, and threat detection before leaving the environment. | K-6 | +| OR-9 | process | User lifecycle management and multi-factor authentication enforcement are governed through centralised identity provider conditional access policies. | K-9 | +| OR-10 | process | The warm standby environment operates at a minimal pilot light footprint with reduced capacity during normal operations until activated for failover. | K-2, K-20 | +| OR-11 | process | Standby disaster recovery infrastructure is scaled to full operational capacity automatically only upon invocation of a disaster recovery failover. | K-21 | +| OR-12 | process | Platform foundation operational readiness and workload ingestion readiness are achieved within 10 weeks (assumed as 50 working days under solution.build.duration_days) from commencement. | K-16, solution.build.duration_days | +| OR-13 | organisation | Platform foundation operations and support are delivered under a service-provider operating model. | sourcing.model | +| OR-14 | organisation | Operational activities and support delivery are conducted solely from within the United Kingdom, Poland, and India. | sourcing.delivery_countries | +| OR-15 | tooling | Operational telemetry, metrics, and incident logging are integrated into the centralised platform operations suite (assumed under service_management.tooling). | service_management.tooling | +| OR-16 | compliance | All customer data and payment processing activities remain strictly within United Kingdom and European Union legal jurisdictions. | K-1 | +| OR-17 | compliance | The PCI-regulated production environment is isolated from non-PCI scopes according to PCI-DSS v4.0 network segmentation requirements. | K-3 | +| OR-18 | compliance | Workload application programming interfaces and payment services are accessible exclusively through private corporate network paths. | K-4 | +| OR-19 | compliance | Platform governance and resilience architecture comply with DORA regulatory testing and third-party risk management frameworks. | K-10 | +| OR-20 | compliance | The production cardholder data environment satisfies PCI-DSS v4.0 isolation and security control requirements. | K-11 | +| OR-21 | compliance | Confidential payment data is secured using customer-controlled cryptographic keys against unauthorized platform-level access or data exposure. | K-12 | +| OR-22 | compliance | Audit logs are immutably retained and protected against modification, deletion, or truncation by any identity prior to retention expiration. | K-13 | +| OR-23 | compliance | Cloud data centres supporting primary and secondary regions have published carbon-free energy (CFE) performance metrics. | K-15 | +| OR-24 | compliance | Platform foundation operations maintain continuous compliance and audit evidence across DORA, ISO 27001, and PCI-DSS v4.0 frameworks. | compliance.frameworks | + +## Capabilities + +| Capability | What it is | Serves | Covers | Hours | Effort a month | +| --- | --- | --- | --- | --- | --- | +| C-1 | Incident management | OR-1 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | 24x7 | 40 h | +| C-2 | Monitoring and alerting | OR-1, OR-5, OR-15 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | 24x7 | 60 h | +| C-3 | Problem management | OR-2 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 20 h | +| C-4 | Service request fulfilment | OR-13, OR-14 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 30 h | +| C-5 | Change enablement | OR-7 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 20 h | +| C-6 | Release and deployment | OR-7 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 20 h | +| C-7 | Configuration management | OR-4, OR-17, OR-18 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 15 h | +| C-8 | Backup and recovery | OR-4 | S-7, S-8, S-10 | business | 10 h | +| C-9 | Service continuity | OR-3, OR-11 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 16 h | +| C-10 | Capacity and performance management | OR-2, OR-5, OR-6, OR-10 | S-2, S-3, S-4 | business | 10 h | +| C-11 | Access management | OR-9, OR-14, OR-21 | S-5, S-6, S-7 | business | 20 h | +| C-12 | Vulnerability and patch management | OR-20 | S-3 | business | 25 h | +| C-13 | Security monitoring and response | OR-8, OR-17, OR-22 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | 24x7 | 60 h | +| C-14 | Cost management | OR-10 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 8 h | +| C-15 | Service level management | OR-2 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 8 h | +| C-16 | Platform product management | OR-12 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 40 h | +| C-17 | Self-service and golden paths | OR-12 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 40 h | +| C-18 | Enablement | OR-12 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 40 h | +| C-19 | Delivery and developer experience measurement | OR-12 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 10 h | +| C-20 | Supplier management | OR-13, OR-19 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 10 h | +| C-21 | Regulatory evidence and reporting | OR-16, OR-19, OR-20, OR-22, OR-23, OR-24 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9, S-10, S-11 | business | 30 h | + +## Who does what + +| Role | Party | Country | Level | Hours | +| --- | --- | --- | --- | --- | +| Platform Reliability Lead | Managed Service Provider (provider) | GB | lead | business | +| Platform SRE | Managed Service Provider (provider) | PL | senior | 24x7 | +| Security Analyst | Managed Service Provider (provider) | IN | senior | 24x7 | +| Platform Engineer | Managed Service Provider (provider) | PL | senior | business | +| Enablement Specialist | Managed Service Provider (provider) | GB | senior | business | +| Compliance Officer | Managed Service Provider (provider) | GB | lead | business | +| Product Manager | Managed Service Provider (provider) | GB | lead | business | +| Customer Service Owner | Customer Organization (customer) | GB | lead | business | + +The teams, as Team Topologies names their kinds: + +| Team | Kind | Party | Roles | Purpose | +| --- | --- | --- | --- | --- | +| Platform Engineering Team | platform | Managed Service Provider | Platform Reliability Lead, Platform Engineer, Compliance Officer, Product Manager | | +| Platform Operations Team | platform | Managed Service Provider | Platform SRE | | +| Security Operations Team | platform | Managed Service Provider | Security Analyst | 24x7 Security Monitoring and Response | +| Developer Enablement Team | enabling | Managed Service Provider | Enablement Specialist | | +| Customer Governance and App Squads | stream-aligned | Customer Organization | Customer Service Owner | | + +How the teams work together: + +| From | To | Mode | Until | Purpose | +| --- | --- | --- | --- | --- | +| Customer Governance and App Squads | Platform Engineering Team | x-as-a-service | | Application teams consume platform services through the self-service portal. | +| Developer Enablement Team | Customer Governance and App Squads | facilitating | | Enablement specialists help application teams adopt golden paths and DORA metrics. | +| Developer Enablement Team | Customer Governance and App Squads | collaboration | End of the 10-week foundation build period, returning to a facilitating mode. | Enablement team pairs with application teams for initial workload ingestion. | + +| Capability | Platform Reliability Lead | Platform SRE | Security Analyst | Platform Engineer | Enablement Specialist | Compliance Officer | Product Manager | Customer Service Owner | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Incident management (C-1) | A | R | | | | | | C | +| Monitoring and alerting (C-2) | A | R | | | | | | C | +| Problem management (C-3) | A | | | R | | | | C | +| Service request fulfilment (C-4) | A | R | | | | | | C | +| Change enablement (C-5) | A | | | R | | | | C | +| Release and deployment (C-6) | A | | | R | | | | C | +| Configuration management (C-7) | A | | | R | | | | C | +| Backup and recovery (C-8) | A | | | R | | | | C | +| Service continuity (C-9) | A | | | R | | | | C | +| Capacity and performance management (C-10) | A | | | R | | | | C | +| Access management (C-11) | | | | R | | A | | C | +| Vulnerability and patch management (C-12) | | | R | | | A | | C | +| Security monitoring and response (C-13) | | | R | | | A | | C | +| Cost management (C-14) | R | | | | | | A | C | +| Service level management (C-15) | A | R | | | | | | C | +| Platform product management (C-16) | | | | | | | A/R | C | +| Self-service and golden paths (C-17) | | | | R | | | A | C | +| Enablement (C-18) | | | | | R | | A | C | +| Delivery and developer experience measurement (C-19) | | | | | R | | A | C | +| Supplier management (C-20) | A/R | | | | | | | C | +| Regulatory evidence and reporting (C-21) | | | | | | A/R | | C | + +A accountable, R responsible, C consulted, I informed. + +## Staffing and cost + +| Role | Party | Country | Level | Hours | Effort a month | People | Cost a month | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Platform Reliability Lead | Managed Service Provider | GB | lead | business | 18.0 h | 0.1 | 1,991 GBP | +| Platform SRE | Managed Service Provider | PL | senior | 24x7 | 138.0 h | 5.0 | 41,640 GBP | +| Security Analyst | Managed Service Provider | IN | senior | 24x7 | 85.0 h | 5.0 | 24,605 GBP | +| Platform Engineer | Managed Service Provider | PL | senior | business | 171.0 h | 1.2 | 7,768 GBP | +| Enablement Specialist | Managed Service Provider | GB | senior | business | 50.0 h | 0.4 | 4,213 GBP | +| Compliance Officer | Managed Service Provider | GB | lead | business | 30.0 h | 0.2 | 3,318 GBP | +| Product Manager | Managed Service Provider | GB | lead | business | 40.0 h | 0.3 | 4,424 GBP | +| Customer Service Owner | Customer Organization | GB | lead | business | 0.0 h | 0.0 | not priced | + +12.2 people in all. Cost a month: 87,959 GBP. + +Worked from rate card `rate-card`, version `draft-2026-10-13`. **Its figures are placeholders: no price has been set, and this cost is an illustration.** +Converted from USD at 0.79, a rate set on 2026-10-13. +The customer's own staff are counted and not priced. A Role that covers hours wider than a working day is never smaller than the team that can cover them. + +## Validation + +| Check | What must hold | Verdict | Evidence | +| --- | --- | --- | --- | +| OC-1 | Declarative infrastructure configurations restrict data location to UK and EU regions. | pass | Capability C-7 handles declarative state and configuration management, covering all required items. Platform Engineer (R-4) is responsible during business hours. | +| OC-2 | Warm standby capacity limits on identity, encryption, and governance components are maintained through DR automation. | pass | Capability C-9 manages service continuity and DR failover rehearsals, covering items S-1, S-7, and S-9. Platform Engineer (R-4) is responsible. | +| OC-3 | PCI-regulated production network configurations enforce isolation from non-PCI scopes. | pass | Capability C-7 manages configuration state and audits, ensuring isolation configurations are maintained. Platform Engineer (R-4) is responsible. | +| OC-4 | Workload APIs are confirmed to be routable only via private corporate network paths. | pass | Capability C-7 audits configuration state for workload APIs, and Platform Engineer (R-4) is responsible. | +| OC-5 | Hybrid interconnect capacity metrics demonstrate SLA-backed low latency. | pass | Capability C-10 reviews capacity and performance metrics, including the dedicated hybrid network interconnect. Platform Engineer (R-4) is responsible. | +| OC-6 | Centralised inspection appliances actively perform stateful packet inspection on all egress traffic. | pass | Capability C-13 provides security monitoring and response, analyzing egress traffic inspection data 24x7 via Security Analyst (R-3). | +| OC-7 | Service continuity rehearsals demonstrate secondary region failover in under 4 hours. | pass | Capability C-9 conducts bi-monthly DR failover rehearsals to ensure the 4-hour RTO is achievable. Platform Engineer (R-4) is responsible. | +| OC-8 | Backup validation ensures an RPO of at most 1 hour. | waits on finding (AF-3) | AF-3: The solution lacks an automated backup service or vault to ensure state and configuration achieve the maximum data loss window (RPO) of 1 hour. | +| OC-9 | Identity access policies require Entra ID multi-factor authentication for corporate users. | pass | Capability C-11 covers access management and policy reviews for corporate identity gateways. Platform Engineer (R-4) is responsible. | +| OC-10 | Third-party supplier reviews include DORA risk assessments. | pass | Capability C-20 conducts supplier management and service reviews, incorporating DORA risk assessments. Platform Reliability Lead (R-1) is responsible. | +| OC-11 | Production cardholder data environments maintain PCI-DSS v4.0 network isolation control compliance. | pass | Capability C-21 continuously collates regulatory evidence for PCI-DSS v4.0. Compliance Officer (R-6) is responsible. | +| OC-12 | Confidential payment data storage systems ensure unauthorized access protection via customer-controlled keys. | pass | Capability C-11 governs access management for the cryptographic key management service (S-7). Platform Engineer (R-4) is responsible. | +| OC-13 | Audit logs are actively monitored for modifications and deletions. | pass | Capability C-13 provides 24x7 security monitoring and response for audit logs (S-8). Security Analyst (R-3) is responsible. | +| OC-14 | Monthly availability of network routing components is tracked and meets the 99.9% target. | pass | Capability C-15 aggregates and reports on service level objectives, including availability targets. Platform SRE (R-2) is responsible. | +| OC-15 | Data centres supporting operations provide publicly validated carbon-free energy (CFE) metrics. | pass | Capability C-21 collates regulatory and compliance evidence, including carbon-free energy metrics. Compliance Officer (R-6) is responsible. | +| OC-16 | Workload ingestion metrics confirm operational readiness within the initial 10-week build. | pass | Capability C-16 manages the platform backlog and adoption roadmap to confirm workload ingestion readiness. Product Manager (R-7) is responsible. | +| OC-17 | A 24/7 service desk is resourced to actively log and respond to infrastructure priority incidents. | pass | Capability C-1 handles 24x7 incident management, and Platform SRE (R-2) provides 24x7 coverage to log and respond to incidents. | +| OC-18 | CI/CD pipeline hooks enforce pull request approvals, validation, and linting before merge. | pass | Capability C-5 manages change enablement and pull request approvals. Platform Engineer (R-4) is responsible. | +| OC-19 | NVA capacity tracking confirms instances utilize high-bandwidth machine families. | pass | Capability C-10 reviews capacity and performance for network virtual appliances. Platform Engineer (R-4) is responsible. | +| OC-20 | Platform budget anomaly alerts check that the warm standby uses minimal infrastructure footprint. | waits on finding (AF-2) | AF-2: The solution lacks automated budget anomaly alerts out-of-the-box to track the minimal pilot light footprint of the warm standby region. | +| OC-21 | Disaster recovery infrastructure automation limits full-capacity scaling strictly to failover occurrences. | pass | Capability C-9 conducts service continuity failovers, managing the DR infrastructure automation. Platform Engineer (R-4) is responsible. | +| OC-22 | Operations and support delivery are provided solely under the agreed service-provider framework. | pass | Role R-1 (Platform Reliability Lead) is part of the Managed Service Provider party, fulfilling the agreed service-provider framework. | +| OC-23 | Staff are located solely in authorized delivery countries. | pass | Role R-1 is located in GB, which is an authorized delivery country. | +| OC-24 | Operational telemetry and incident logging are fully integrated into the operations suite. | pass | Capability C-2 performs continuous monitoring and alerting using the integrated operations suite. Platform SRE (R-2) is responsible 24x7. | +| OC-25 | Compliance audits consistently certify continuous adherence across DORA, ISO 27001, and PCI-DSS v4.0 frameworks. | pass | Capability C-21 continuously collates audit evidence across DORA, ISO 27001, and PCI-DSS v4.0. Compliance Officer (R-6) is responsible. | +| OC-26 | Incident responders use a secure remote access mechanism to troubleshoot Centralised Egress Traffic Inspection Gateways. | waits on finding (AF-1) | AF-1: The solution lacks a secure remote access mechanism (such as a bastion host or Identity-Aware Proxy) as a way in for responders to the Centralised Egress Traffic Inspection Gateway. | + +Each scenario is generated from the operating model and traced through it by code. + +| Scenario | Result | Path | +| --- | --- | --- | +| Multi-Tier Environment Hierarchy & Isolation Boundary (S-1) fails at an hour faults are responded to (24x7). | pass | Platform SRE responds under C-1, and Platform Reliability Lead is answerable. | +| Multi-Tier Environment Hierarchy & Isolation Boundary (S-1) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Dedicated Hybrid Network Interconnect Gateway (S-2) fails at an hour faults are responded to (24x7). | pass | Platform SRE responds under C-1, and Platform Reliability Lead is answerable. | +| Dedicated Hybrid Network Interconnect Gateway (S-2) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Centralised Egress Traffic Inspection Gateway (S-3) fails at an hour faults are responded to (24x7). | pass | Platform SRE responds under C-1, and Platform Reliability Lead is answerable. | +| Centralised Egress Traffic Inspection Gateway (S-3) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Transit Network Fabric & Routing Controller (S-4) fails at an hour faults are responded to (24x7). | pass | Platform SRE responds under C-1, and Platform Reliability Lead is answerable. | +| Transit Network Fabric & Routing Controller (S-4) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Corporate Identity Federation Gateway (S-5) fails at an hour faults are responded to (24x7). | pass | Platform SRE responds under C-1, and Platform Reliability Lead is answerable. | +| Corporate Identity Federation Gateway (S-5) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Automated Delivery Pipeline Identity Broker (S-6) fails at an hour faults are responded to (24x7). | pass | Platform SRE responds under C-1, and Platform Reliability Lead is answerable. | +| Automated Delivery Pipeline Identity Broker (S-6) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Cryptographic Key Management & Rotation Service (S-7) fails at an hour faults are responded to (24x7). | pass | Platform SRE responds under C-1, and Platform Reliability Lead is answerable. | +| Cryptographic Key Management & Rotation Service (S-7) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Centralised Immutable Audit Log Repository (S-8) fails at an hour faults are responded to (24x7). | pass | Platform SRE responds under C-1, and Platform Reliability Lead is answerable. | +| Centralised Immutable Audit Log Repository (S-8) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Platform Compliance & Governance Guardrail Controller (S-9) fails at an hour faults are responded to (24x7). | pass | Platform SRE responds under C-1, and Platform Reliability Lead is answerable. | +| Platform Compliance & Governance Guardrail Controller (S-9) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Declarative Infrastructure Provisioning & State Engine (S-10) fails at an hour faults are responded to (24x7). | pass | Platform SRE responds under C-1, and Platform Reliability Lead is answerable. | +| Declarative Infrastructure Provisioning & State Engine (S-10) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Centralised Operational Telemetry & Alerting Engine (S-11) fails at an hour faults are responded to (24x7). | pass | Platform SRE responds under C-1, and Platform Reliability Lead is answerable. | +| Centralised Operational Telemetry & Alerting Engine (S-11) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Condition K-1 must be shown to hold: Customer data and payment processing remain within United Kingdom and European Union legal jurisdictions. | pass | Check OC-1 on C-7 keeps it true. | +| Condition K-2 must be shown to hold: Warm standby environment operates with reduced capacity until activated during a disaster recovery failover. | pass | Check OC-2 on C-9 keeps it true. | +| Condition K-3 must be shown to hold: PCI-regulated production environment is isolated from non-PCI scopes according to PCI-DSS v4.0 network segmentation requirements. | pass | Check OC-3 on C-7 keeps it true. | +| Condition K-4 must be shown to hold: Workload APIs and payment services are accessible exclusively through private corporate network paths. | pass | Check OC-4 on C-7 keeps it true. | +| Condition K-5 must be shown to hold: Hybrid connection maintains low-latency, SLA-backed private data transmission between on-premises and Google Cloud. | pass | Check OC-5 on C-10 keeps it true. | +| Condition K-6 must be shown to hold: Centralised inspection appliances perform next-generation stateful packet inspection, URL filtering, and threat detection on outbound internet sessions. | pass | Check OC-6 on C-13 keeps it true. | +| Condition K-7 must be shown to hold: Platform foundation failover to europe-west1 can be completed within 4 hours during a primary region outage. | pass | Check OC-7 on C-9 keeps it true. | +| Condition K-8 must be shown to hold: Configuration drift and backup state achieve a maximum data loss window (RPO) of 1 hour. | pass | Check OC-8 on C-7 keeps it true. | +| Condition K-9 must be shown to hold: Corporate user lifecycle management and MFA enforcement are governed centrally by Entra ID conditional access policies. | pass | Check OC-9 on C-11 keeps it true. | +| Condition K-10 must be shown to hold: Platform governance and resilience architecture comply with DORA regulatory testing and third-party risk management frameworks. | pass | Check OC-10 on C-20 keeps it true. | +| Condition K-11 must be shown to hold: Production cardholder data environment satisfies PCI-DSS v4.0 isolation and security control requirements. | pass | Check OC-11 on C-21 keeps it true. | +| Condition K-12 must be shown to hold: Confidential payment data is secured against unauthorized platform-level access or data exposure. | pass | Check OC-12 on C-11 keeps it true. | +| Condition K-13 must be shown to hold: Audit logs cannot be modified, deleted, or shortened by any identity or administrator prior to retention expiration. | pass | Check OC-13 on C-13 keeps it true. | +| Condition K-14 must be shown to hold: Monthly availability of the core platform landing zone and network routing meets or exceeds 99.9%. | pass | Check OC-14 on C-15 keeps it true. | +| Condition K-15 must be shown to hold: Primary and secondary regions leverage Google Cloud data centres with published carbon-free energy (CFE) performance metrics. | pass | Check OC-15 on C-21 keeps it true. | +| Condition K-16 must be shown to hold: Platform foundation delivery and workload ingestion readiness are achieved within 10 weeks of project commencement. | pass | Check OC-16 on C-16 keeps it true. | +| Condition K-17 must be shown to hold: Central platform team operates 24/7 support for infrastructure incidents and availability alerts. | pass | Check OC-17 on C-1 keeps it true. | +| Condition K-18 must be shown to hold: Pull request workflows require automated CI/CD linting, validation, and peer approval prior to merge and deployment. | pass | Check OC-18 on C-5 keeps it true. | +| Condition K-19 must be shown to hold: NVA instances use machine families that support high bandwidth networking. | pass | Check OC-19 on C-10 keeps it true. | +| Condition K-20 must be shown to hold: The warm standby environment operates at a minimal pilot light footprint during normal operations. | pass | Check OC-20 on C-14 keeps it true. | +| Condition K-21 must be shown to hold: Automated disaster recovery scripts scale the standby environment to full capacity only during a failover event. | pass | Check OC-21 on C-9 keeps it true. | +| Managed Service Provider cannot be reached. | pass | Every Capability it is answerable for has a Role in another Party to turn to. | + +## Findings and open questions + +What operating this solution needs that the solution, as described, does not provide. Nothing here was designed around: each is for whoever owns the design. + +- **AF-1** (S-3): The solution lacks a secure remote access mechanism (such as a bastion host or Identity-Aware Proxy) as a way in for responders to the Centralised Egress Traffic Inspection Gateway. Proposed: Add Identity-Aware Proxy or a secure bastion host to allow responders to securely access instances. +- **AF-2** (S-11): The solution lacks automated budget anomaly alerts to ensure the warm standby environment maintains a minimal pilot light footprint. Proposed: Configure Google Cloud Billing Budgets and alerts tied to the standby environment. +- **AF-3** (S-2, S-4): The solution lacks an automated backup service or vault to ensure state and configuration achieve the maximum data loss window (RPO) of 1 hour. Proposed: Implement Google Cloud Backup and DR service or enable automated object versioning with lifecycle policies. + +Assumed for the customer, and open to change when they answer: + +- `intent.business_outcomes`: {'outcome': 'Continuous and resilient payment processing with high availability and low-latency hybrid connectivity', 'id': 'BO-1'}, {'outcome': 'Sustained regulatory compliance with DORA and PCI-DSS v4.0 security and resilience standards', 'id': 'BO-2'}, {'id': 'BO-3', 'outcome': 'Rapid and secure platform change releases through automated CI/CD delivery pipelines'}. Derived from customer request for a UK-regulated payment platform requiring high resilience, continuous compliance, and automated delivery. +- `intent.exclusions`: Application container deployment and runtime microservices managed by application squads, Active-active secondary region in phase 1 (warm standby adopted). Derived from workload scope boundaries where application workloads are managed by application squads, and secondary region is warm standby rather than active-active. +- `solution.build.duration_days`: 50. Derived from 10-week delivery window at 5 working days per week. +- `operations.change_process`: Infrastructure changes managed via pull requests with required approvals and automated GitHub Actions CI/CD pipelines. Derived from request and scope specification indicating pull-request driven changes with peer review and GitHub Actions CI/CD automation. +- `organisation.existing_teams`: . Catalogue default: no existing operational teams assumed; roles to be provisioned under provider-operated model. +- `service_management.tooling`: Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging). Catalogue default: Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging) with no separate third-party ticketing system mandated. +- `service_levels.priority_1_response_minutes`: 30. Catalogue default for Priority 1 incident response acknowledgement. +- `service_levels.recovery_time_hours`: 4. Catalogue default and aligned with target 4-hour RTO for warm standby disaster recovery in secondary region. + +## Sources + +The Capabilities are from our own capability catalogue, which draws on these frameworks. It cites them and certifies nothing against them. + +- ITIL 4: Incident management practice; Monitoring and event management practice; Problem management practice; Service request management practice; Change enablement practice; Release management and Deployment management practices; Service configuration management practice; Service continuity management practice; Capacity and performance management practice; Service level management practice; Supplier management practice +- Site Reliability Engineering: How Google Runs Production Systems: Managing Incidents; Monitoring Distributed Systems; Postmortem Culture: Learning from Failure; Release Engineering; Service Level Objectives +- ISO/IEC 20000-1: Incident management; Service request management; Change management; Service continuity management; Capacity management; Service level management; Supplier management +- Regulation (EU) 2022/2554, the Digital Operational Resilience Act: Article 17, ICT-related incident management process; Article 10, Detection; Article 12, Backup policies and procedures, restoration and recovery procedures and methods; Article 11, Response and recovery; Article 9, Protection and prevention; Article 28, General principles on ICT third-party risk; Article 19, Reporting of major ICT-related incidents +- NIST Cybersecurity Framework: Detect: Continuous Monitoring; Identify: Asset Management; Recover: Incident Recovery Plan Execution; Protect: Identity Management, Authentication, and Access Control; Protect: Platform Security; Detect: Adverse Event Analysis; Respond: Incident Management +- DevOps Research and Assessment (DORA) software delivery metrics: Change failure rate; Deployment frequency and lead time for changes; Deployment frequency; Lead time for changes; Change failure rate; Time to restore service +- ISO/IEC 27001: Access control; Management of technical vulnerabilities; Information security incident management; Compliance with legal and contractual requirements +- FinOps Framework: Allocation; Budgeting; Forecasting; Anomaly Management +- CNCF Platforms White Paper (platform as a product): Platform as a product; Measuring platform success; Platform capabilities; Self-service; Platform as a product +- Team Topologies: Organizing Business and Technology Teams for Fast Flow: Platform team; Thinnest viable platform; X-as-a-Service interaction mode; Enabling team; Facilitating interaction mode +- The SPACE of Developer Productivity: Satisfaction and well-being; Performance; Activity; Communication and collaboration; Efficiency and flow +- DevEx: What Actually Drives Productivity: Feedback loops; Cognitive load; Flow state + +**These references have not been checked against the frameworks by a person.** Treat each as a pointer to where to look. diff --git a/eval/results/20261021T203623Z-handoff/operating_model/operational-requirements.json b/eval/results/20261021T203623Z-handoff/operating_model/operational-requirements.json new file mode 100644 index 0000000..2530583 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/operational-requirements.json @@ -0,0 +1,201 @@ +[ + { + "id": "OR-1", + "kind": "service_level", + "statement": "Priority 1 incidents on core platform foundation components are acknowledged within 30 minutes (assumed under service_levels.priority_1_response_minutes), supported continuously 24 hours a day, 7 days a week.", + "source": [ + "operations.support_hours", + "service_levels.priority_1_response_minutes", + "K-17" + ] + }, + { + "id": "OR-2", + "kind": "service_level", + "statement": "The core platform landing zone and network routing maintain at least 99.9% monthly availability, operating 24 hours a day, 7 days a week.", + "source": [ + "K-14", + "operations.availability_hours" + ] + }, + { + "id": "OR-3", + "kind": "service_level", + "statement": "Platform foundation failover to the secondary recovery region is completed within 4 hours (assumed under service_levels.recovery_time_hours) in the event of a primary region outage.", + "source": [ + "K-7", + "service_levels.recovery_time_hours" + ] + }, + { + "id": "OR-4", + "kind": "service_level", + "statement": "Platform infrastructure configuration drift and backup state maintain a recovery point objective (RPO) of no more than 1 hour of data loss.", + "source": [ + "K-8" + ] + }, + { + "id": "OR-5", + "kind": "service_level", + "statement": "Dedicated hybrid connectivity between on-premises environments and the cloud platform maintains SLA-backed, low-latency private data transmission.", + "source": [ + "K-5" + ] + }, + { + "id": "OR-6", + "kind": "service_level", + "statement": "Centralised inspection appliances maintain high-bandwidth network throughput capacity to prevent traffic constraints during peak transaction flows.", + "source": [ + "K-19" + ] + }, + { + "id": "OR-7", + "kind": "process", + "statement": "All infrastructure changes require automated pipeline validation, linting, and peer approval via pull requests prior to merge and release (assumed under operations.change_process).", + "source": [ + "K-18", + "operations.change_process" + ] + }, + { + "id": "OR-8", + "kind": "process", + "statement": "Outbound internet traffic is inspected through centralised stateful packet inspection, URL filtering, and threat detection before leaving the environment.", + "source": [ + "K-6" + ] + }, + { + "id": "OR-9", + "kind": "process", + "statement": "User lifecycle management and multi-factor authentication enforcement are governed through centralised identity provider conditional access policies.", + "source": [ + "K-9" + ] + }, + { + "id": "OR-10", + "kind": "process", + "statement": "The warm standby environment operates at a minimal pilot light footprint with reduced capacity during normal operations until activated for failover.", + "source": [ + "K-2", + "K-20" + ] + }, + { + "id": "OR-11", + "kind": "process", + "statement": "Standby disaster recovery infrastructure is scaled to full operational capacity automatically only upon invocation of a disaster recovery failover.", + "source": [ + "K-21" + ] + }, + { + "id": "OR-12", + "kind": "process", + "statement": "Platform foundation operational readiness and workload ingestion readiness are achieved within 10 weeks (assumed as 50 working days under solution.build.duration_days) from commencement.", + "source": [ + "K-16", + "solution.build.duration_days" + ] + }, + { + "id": "OR-13", + "kind": "organisation", + "statement": "Platform foundation operations and support are delivered under a service-provider operating model.", + "source": [ + "sourcing.model" + ] + }, + { + "id": "OR-14", + "kind": "organisation", + "statement": "Operational activities and support delivery are conducted solely from within the United Kingdom, Poland, and India.", + "source": [ + "sourcing.delivery_countries" + ] + }, + { + "id": "OR-15", + "kind": "tooling", + "statement": "Operational telemetry, metrics, and incident logging are integrated into the centralised platform operations suite (assumed under service_management.tooling).", + "source": [ + "service_management.tooling" + ] + }, + { + "id": "OR-16", + "kind": "compliance", + "statement": "All customer data and payment processing activities remain strictly within United Kingdom and European Union legal jurisdictions.", + "source": [ + "K-1" + ] + }, + { + "id": "OR-17", + "kind": "compliance", + "statement": "The PCI-regulated production environment is isolated from non-PCI scopes according to PCI-DSS v4.0 network segmentation requirements.", + "source": [ + "K-3" + ] + }, + { + "id": "OR-18", + "kind": "compliance", + "statement": "Workload application programming interfaces and payment services are accessible exclusively through private corporate network paths.", + "source": [ + "K-4" + ] + }, + { + "id": "OR-19", + "kind": "compliance", + "statement": "Platform governance and resilience architecture comply with DORA regulatory testing and third-party risk management frameworks.", + "source": [ + "K-10" + ] + }, + { + "id": "OR-20", + "kind": "compliance", + "statement": "The production cardholder data environment satisfies PCI-DSS v4.0 isolation and security control requirements.", + "source": [ + "K-11" + ] + }, + { + "id": "OR-21", + "kind": "compliance", + "statement": "Confidential payment data is secured using customer-controlled cryptographic keys against unauthorized platform-level access or data exposure.", + "source": [ + "K-12" + ] + }, + { + "id": "OR-22", + "kind": "compliance", + "statement": "Audit logs are immutably retained and protected against modification, deletion, or truncation by any identity prior to retention expiration.", + "source": [ + "K-13" + ] + }, + { + "id": "OR-23", + "kind": "compliance", + "statement": "Cloud data centres supporting primary and secondary regions have published carbon-free energy (CFE) performance metrics.", + "source": [ + "K-15" + ] + }, + { + "id": "OR-24", + "kind": "compliance", + "statement": "Platform foundation operations maintain continuous compliance and audit evidence across DORA, ISO 27001, and PCI-DSS v4.0 frameworks.", + "source": [ + "compliance.frameworks" + ] + } +] diff --git a/eval/results/20261021T203623Z-handoff/operating_model/organisation.mmd b/eval/results/20261021T203623Z-handoff/operating_model/organisation.mmd new file mode 100644 index 0000000..dd55bc0 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/organisation.mmd @@ -0,0 +1,26 @@ +flowchart TB + subgraph P_1["Managed Service Provider (provider)"] + subgraph T_1["Platform Engineering Team (platform team)"] + R_1["Platform Reliability Lead
GB, lead, business"] + R_4["Platform Engineer
PL, senior, business"] + R_6["Compliance Officer
GB, lead, business"] + R_7["Product Manager
GB, lead, business"] + end + subgraph T_2["Platform Operations Team (platform team)"] + R_2["Platform SRE
PL, senior, 24x7"] + end + subgraph T_3["Security Operations Team (platform team)"] + R_3["Security Analyst
IN, senior, 24x7"] + end + subgraph T_4["Developer Enablement Team (enabling team)"] + R_5["Enablement Specialist
GB, senior, business"] + end + end + subgraph P_2["Customer Organization (customer)"] + subgraph T_5["Customer Governance and App Squads (stream-aligned team)"] + R_8["Customer Service Owner
GB, lead, business"] + end + end + T_5 -- "x-as-a-service" --> T_1 + T_4 -- "facilitating" --> T_5 + T_4 -- "collaboration until End of the 10-week foundation build period, returning to a facilitating mode." --> T_5 diff --git a/eval/results/20261021T203623Z-handoff/operating_model/record.json b/eval/results/20261021T203623Z-handoff/operating_model/record.json new file mode 100644 index 0000000..9aab96c --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/record.json @@ -0,0 +1 @@ +{"tenant":"local","customer_case":"case-08-landing-zone","execution_id":"1f8f30a1-7b74-4d23-9fb0-aec58089c506","work":"operating_model","status":"completed","request":"We are a UK-regulated payment platform expanding into Google Cloud. We need an enterprise-grade landing zone ready in under 10 weeks to support our core payments engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have existing on-premises systems that require private, low-latency connectivity over a 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and GitHub Actions for our delivery pipelines. Security requires that all egress traffic flows through centralised inspection appliances before touching the public internet, and customer data must be encrypted with keys we control.","project_id":null,"guide":"operating-model-guide.md","files":["capabilities.json","findings.json","operating-model-guide.md","operational-requirements.json","organisation.mmd","scenarios.json","scope.json","scope.yaml","staffing.json","target-operating-model.json","verdicts.json"],"kept_at":"2026-10-21T21:51:17.306367+00:00","links":{"record":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506","guide":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operating-model-guide.md","files":{"capabilities.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/capabilities.json","findings.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/findings.json","operating-model-guide.md":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operating-model-guide.md","operational-requirements.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operational-requirements.json","organisation.mmd":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/organisation.mmd","scenarios.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scenarios.json","scope.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scope.json","scope.yaml":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scope.yaml","staffing.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/staffing.json","target-operating-model.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/target-operating-model.json","verdicts.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/verdicts.json"},"latest":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/latest/operating_model","case":"http://localhost:10030/tenants/local/cases/case-08-landing-zone"}} \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/operating_model/scenarios.json b/eval/results/20261021T203623Z-handoff/operating_model/scenarios.json new file mode 100644 index 0000000..5974c1f --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/scenarios.json @@ -0,0 +1,310 @@ +[ + { + "id": "SC-fault-S-1", + "scenario": "Multi-Tier Environment Hierarchy & Isolation Boundary (S-1) fails at an hour faults are responded to (24x7).", + "about": "S-1", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-1", + "scenario": "Multi-Tier Environment Hierarchy & Isolation Boundary (S-1) must be changed.", + "about": "S-1", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-2", + "scenario": "Dedicated Hybrid Network Interconnect Gateway (S-2) fails at an hour faults are responded to (24x7).", + "about": "S-2", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-2", + "scenario": "Dedicated Hybrid Network Interconnect Gateway (S-2) must be changed.", + "about": "S-2", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-3", + "scenario": "Centralised Egress Traffic Inspection Gateway (S-3) fails at an hour faults are responded to (24x7).", + "about": "S-3", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-3", + "scenario": "Centralised Egress Traffic Inspection Gateway (S-3) must be changed.", + "about": "S-3", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-4", + "scenario": "Transit Network Fabric & Routing Controller (S-4) fails at an hour faults are responded to (24x7).", + "about": "S-4", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-4", + "scenario": "Transit Network Fabric & Routing Controller (S-4) must be changed.", + "about": "S-4", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-5", + "scenario": "Corporate Identity Federation Gateway (S-5) fails at an hour faults are responded to (24x7).", + "about": "S-5", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-5", + "scenario": "Corporate Identity Federation Gateway (S-5) must be changed.", + "about": "S-5", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-6", + "scenario": "Automated Delivery Pipeline Identity Broker (S-6) fails at an hour faults are responded to (24x7).", + "about": "S-6", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-6", + "scenario": "Automated Delivery Pipeline Identity Broker (S-6) must be changed.", + "about": "S-6", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-7", + "scenario": "Cryptographic Key Management & Rotation Service (S-7) fails at an hour faults are responded to (24x7).", + "about": "S-7", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-7", + "scenario": "Cryptographic Key Management & Rotation Service (S-7) must be changed.", + "about": "S-7", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-8", + "scenario": "Centralised Immutable Audit Log Repository (S-8) fails at an hour faults are responded to (24x7).", + "about": "S-8", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-8", + "scenario": "Centralised Immutable Audit Log Repository (S-8) must be changed.", + "about": "S-8", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-9", + "scenario": "Platform Compliance & Governance Guardrail Controller (S-9) fails at an hour faults are responded to (24x7).", + "about": "S-9", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-9", + "scenario": "Platform Compliance & Governance Guardrail Controller (S-9) must be changed.", + "about": "S-9", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-10", + "scenario": "Declarative Infrastructure Provisioning & State Engine (S-10) fails at an hour faults are responded to (24x7).", + "about": "S-10", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-10", + "scenario": "Declarative Infrastructure Provisioning & State Engine (S-10) must be changed.", + "about": "S-10", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-11", + "scenario": "Centralised Operational Telemetry & Alerting Engine (S-11) fails at an hour faults are responded to (24x7).", + "about": "S-11", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-11", + "scenario": "Centralised Operational Telemetry & Alerting Engine (S-11) must be changed.", + "about": "S-11", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-condition-K-1", + "scenario": "Condition K-1 must be shown to hold: Customer data and payment processing remain within United Kingdom and European Union legal jurisdictions.", + "about": "K-1", + "passes": true, + "path": "Check OC-1 on C-7 keeps it true." + }, + { + "id": "SC-condition-K-2", + "scenario": "Condition K-2 must be shown to hold: Warm standby environment operates with reduced capacity until activated during a disaster recovery failover.", + "about": "K-2", + "passes": true, + "path": "Check OC-2 on C-9 keeps it true." + }, + { + "id": "SC-condition-K-3", + "scenario": "Condition K-3 must be shown to hold: PCI-regulated production environment is isolated from non-PCI scopes according to PCI-DSS v4.0 network segmentation requirements.", + "about": "K-3", + "passes": true, + "path": "Check OC-3 on C-7 keeps it true." + }, + { + "id": "SC-condition-K-4", + "scenario": "Condition K-4 must be shown to hold: Workload APIs and payment services are accessible exclusively through private corporate network paths.", + "about": "K-4", + "passes": true, + "path": "Check OC-4 on C-7 keeps it true." + }, + { + "id": "SC-condition-K-5", + "scenario": "Condition K-5 must be shown to hold: Hybrid connection maintains low-latency, SLA-backed private data transmission between on-premises and Google Cloud.", + "about": "K-5", + "passes": true, + "path": "Check OC-5 on C-10 keeps it true." + }, + { + "id": "SC-condition-K-6", + "scenario": "Condition K-6 must be shown to hold: Centralised inspection appliances perform next-generation stateful packet inspection, URL filtering, and threat detection on outbound internet sessions.", + "about": "K-6", + "passes": true, + "path": "Check OC-6 on C-13 keeps it true." + }, + { + "id": "SC-condition-K-7", + "scenario": "Condition K-7 must be shown to hold: Platform foundation failover to europe-west1 can be completed within 4 hours during a primary region outage.", + "about": "K-7", + "passes": true, + "path": "Check OC-7 on C-9 keeps it true." + }, + { + "id": "SC-condition-K-8", + "scenario": "Condition K-8 must be shown to hold: Configuration drift and backup state achieve a maximum data loss window (RPO) of 1 hour.", + "about": "K-8", + "passes": true, + "path": "Check OC-8 on C-7 keeps it true." + }, + { + "id": "SC-condition-K-9", + "scenario": "Condition K-9 must be shown to hold: Corporate user lifecycle management and MFA enforcement are governed centrally by Entra ID conditional access policies.", + "about": "K-9", + "passes": true, + "path": "Check OC-9 on C-11 keeps it true." + }, + { + "id": "SC-condition-K-10", + "scenario": "Condition K-10 must be shown to hold: Platform governance and resilience architecture comply with DORA regulatory testing and third-party risk management frameworks.", + "about": "K-10", + "passes": true, + "path": "Check OC-10 on C-20 keeps it true." + }, + { + "id": "SC-condition-K-11", + "scenario": "Condition K-11 must be shown to hold: Production cardholder data environment satisfies PCI-DSS v4.0 isolation and security control requirements.", + "about": "K-11", + "passes": true, + "path": "Check OC-11 on C-21 keeps it true." + }, + { + "id": "SC-condition-K-12", + "scenario": "Condition K-12 must be shown to hold: Confidential payment data is secured against unauthorized platform-level access or data exposure.", + "about": "K-12", + "passes": true, + "path": "Check OC-12 on C-11 keeps it true." + }, + { + "id": "SC-condition-K-13", + "scenario": "Condition K-13 must be shown to hold: Audit logs cannot be modified, deleted, or shortened by any identity or administrator prior to retention expiration.", + "about": "K-13", + "passes": true, + "path": "Check OC-13 on C-13 keeps it true." + }, + { + "id": "SC-condition-K-14", + "scenario": "Condition K-14 must be shown to hold: Monthly availability of the core platform landing zone and network routing meets or exceeds 99.9%.", + "about": "K-14", + "passes": true, + "path": "Check OC-14 on C-15 keeps it true." + }, + { + "id": "SC-condition-K-15", + "scenario": "Condition K-15 must be shown to hold: Primary and secondary regions leverage Google Cloud data centres with published carbon-free energy (CFE) performance metrics.", + "about": "K-15", + "passes": true, + "path": "Check OC-15 on C-21 keeps it true." + }, + { + "id": "SC-condition-K-16", + "scenario": "Condition K-16 must be shown to hold: Platform foundation delivery and workload ingestion readiness are achieved within 10 weeks of project commencement.", + "about": "K-16", + "passes": true, + "path": "Check OC-16 on C-16 keeps it true." + }, + { + "id": "SC-condition-K-17", + "scenario": "Condition K-17 must be shown to hold: Central platform team operates 24/7 support for infrastructure incidents and availability alerts.", + "about": "K-17", + "passes": true, + "path": "Check OC-17 on C-1 keeps it true." + }, + { + "id": "SC-condition-K-18", + "scenario": "Condition K-18 must be shown to hold: Pull request workflows require automated CI/CD linting, validation, and peer approval prior to merge and deployment.", + "about": "K-18", + "passes": true, + "path": "Check OC-18 on C-5 keeps it true." + }, + { + "id": "SC-condition-K-19", + "scenario": "Condition K-19 must be shown to hold: NVA instances use machine families that support high bandwidth networking.", + "about": "K-19", + "passes": true, + "path": "Check OC-19 on C-10 keeps it true." + }, + { + "id": "SC-condition-K-20", + "scenario": "Condition K-20 must be shown to hold: The warm standby environment operates at a minimal pilot light footprint during normal operations.", + "about": "K-20", + "passes": true, + "path": "Check OC-20 on C-14 keeps it true." + }, + { + "id": "SC-condition-K-21", + "scenario": "Condition K-21 must be shown to hold: Automated disaster recovery scripts scale the standby environment to full capacity only during a failover event.", + "about": "K-21", + "passes": true, + "path": "Check OC-21 on C-9 keeps it true." + }, + { + "id": "SC-party-P-1", + "scenario": "Managed Service Provider cannot be reached.", + "about": "P-1", + "passes": true, + "path": "Every Capability it is answerable for has a Role in another Party to turn to." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/operating_model/scope.json b/eval/results/20261021T203623Z-handoff/operating_model/scope.json new file mode 100644 index 0000000..aa54f55 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/scope.json @@ -0,0 +1,495 @@ +{ + "kind": "platform_foundation", + "stated": { + "engagement": { + "kind": "platform_foundation" + }, + "intent": { + "statement": "We are a UK-regulated payment platform expanding into Google Cloud. We need an enterprise-grade landing zone ready in under 10 weeks to support our core payments engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have existing on-premises systems that require private, low-latency connectivity over a 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and GitHub Actions for our delivery pipelines. Security requires that all egress traffic flows through centralised inspection appliances before touching the public internet, and customer data must be encrypted with keys we control." + }, + "solution": { + "cloud_provider": "gcp", + "items": [ + { + "id": "S-1", + "name": "Multi-Tier Environment Hierarchy & Isolation Boundary", + "kind": "governance", + "product": "Resource Manager", + "source_ref": "C-1", + "elements": [ + "E-1", + "E-2", + "E-3" + ] + }, + { + "id": "S-2", + "name": "Dedicated Hybrid Network Interconnect Gateway", + "kind": "network", + "product": "Cloud Interconnect", + "source_ref": "C-2", + "elements": [ + "E-9", + "E-10", + "E-11", + "E-12", + "E-35", + "E-36" + ] + }, + { + "id": "S-3", + "name": "Centralised Egress Traffic Inspection Gateway", + "kind": "compute", + "product": "Compute Engine", + "source_ref": "C-3", + "elements": [ + "E-13", + "E-14", + "E-15", + "E-16", + "E-17" + ] + }, + { + "id": "S-4", + "name": "Transit Network Fabric & Routing Controller", + "kind": "service", + "product": "Virtual Private Cloud", + "source_ref": "C-4", + "elements": [ + "E-4", + "E-5", + "E-40", + "E-41", + "E-42", + "E-6", + "E-7", + "E-8", + "E-43", + "E-44", + "E-45", + "E-37", + "E-47", + "E-51", + "E-52", + "E-53" + ] + }, + { + "id": "S-5", + "name": "Corporate Identity Federation Gateway", + "kind": "identity", + "product": "Cloud Identity", + "source_ref": "C-5", + "elements": [ + "E-18" + ] + }, + { + "id": "S-6", + "name": "Automated Delivery Pipeline Identity Broker", + "kind": "identity", + "product": "Workload Identity Federation", + "source_ref": "C-6", + "elements": [ + "E-19", + "E-20", + "E-54", + "E-55" + ] + }, + { + "id": "S-7", + "name": "Cryptographic Key Management & Rotation Service", + "kind": "keys", + "product": "Cloud KMS", + "source_ref": "C-7", + "elements": [ + "E-21", + "E-22", + "E-23", + "E-24" + ] + }, + { + "id": "S-8", + "name": "Centralised Immutable Audit Log Repository", + "kind": "storage", + "product": "Cloud Storage", + "source_ref": "C-8", + "elements": [ + "E-25", + "E-26" + ] + }, + { + "id": "S-9", + "name": "Platform Compliance & Governance Guardrail Controller", + "kind": "governance", + "product": "Organization Policy Service", + "source_ref": "C-9", + "elements": [ + "E-27", + "E-28", + "E-29", + "E-30", + "E-38", + "E-46", + "E-56", + "E-57" + ] + }, + { + "id": "S-10", + "name": "Declarative Infrastructure Provisioning & State Engine", + "kind": "storage", + "product": "Cloud Storage", + "source_ref": "C-10", + "elements": [ + "E-31" + ] + }, + { + "id": "S-11", + "name": "Centralised Operational Telemetry & Alerting Engine", + "kind": "observability", + "product": "Cloud Monitoring", + "source_ref": "C-11", + "elements": [ + "E-32", + "E-33", + "E-58" + ] + } + ], + "conditions": [ + { + "id": "K-1", + "on": [ + "S-1", + "S-7", + "S-9" + ], + "condition": "Customer data and payment processing remain within United Kingdom and European Union legal jurisdictions.", + "source_ref": "NFR-1" + }, + { + "id": "K-2", + "on": [ + "S-1", + "S-7", + "S-9" + ], + "condition": "Warm standby environment operates with reduced capacity until activated during a disaster recovery failover.", + "source_ref": "NFR-1" + }, + { + "id": "K-3", + "on": [ + "S-1", + "S-9" + ], + "condition": "PCI-regulated production environment is isolated from non-PCI scopes according to PCI-DSS v4.0 network segmentation requirements.", + "source_ref": "NFR-2" + }, + { + "id": "K-4", + "on": [ + "S-3", + "S-4" + ], + "condition": "Workload APIs and payment services are accessible exclusively through private corporate network paths.", + "source_ref": "NFR-3" + }, + { + "id": "K-5", + "on": [ + "S-2" + ], + "condition": "Hybrid connection maintains low-latency, SLA-backed private data transmission between on-premises and Google Cloud.", + "source_ref": "NFR-4" + }, + { + "id": "K-6", + "on": [ + "S-3" + ], + "condition": "Centralised inspection appliances perform next-generation stateful packet inspection, URL filtering, and threat detection on outbound internet sessions.", + "source_ref": "NFR-5" + }, + { + "id": "K-7", + "on": [ + "S-2", + "S-4" + ], + "condition": "Platform foundation failover to europe-west1 can be completed within 4 hours during a primary region outage.", + "source_ref": "NFR-6" + }, + { + "id": "K-8", + "on": [ + "S-2", + "S-4" + ], + "condition": "Configuration drift and backup state achieve a maximum data loss window (RPO) of 1 hour.", + "source_ref": "NFR-6" + }, + { + "id": "K-9", + "on": [ + "S-5", + "S-6" + ], + "condition": "Corporate user lifecycle management and MFA enforcement are governed centrally by Entra ID conditional access policies.", + "source_ref": "NFR-7" + }, + { + "id": "K-10", + "on": [ + "S-1", + "S-9" + ], + "condition": "Platform governance and resilience architecture comply with DORA regulatory testing and third-party risk management frameworks.", + "source_ref": "NFR-8" + }, + { + "id": "K-11", + "on": [ + "S-1", + "S-9" + ], + "condition": "Production cardholder data environment satisfies PCI-DSS v4.0 isolation and security control requirements.", + "source_ref": "NFR-8" + }, + { + "id": "K-12", + "on": [ + "S-7" + ], + "condition": "Confidential payment data is secured against unauthorized platform-level access or data exposure.", + "source_ref": "NFR-9" + }, + { + "id": "K-13", + "on": [ + "S-8" + ], + "condition": "Audit logs cannot be modified, deleted, or shortened by any identity or administrator prior to retention expiration.", + "source_ref": "NFR-10" + }, + { + "id": "K-14", + "on": [ + "S-2", + "S-4", + "S-11" + ], + "condition": "Monthly availability of the core platform landing zone and network routing meets or exceeds 99.9%.", + "source_ref": "NFR-11" + }, + { + "id": "K-15", + "on": [ + "S-9" + ], + "condition": "Primary and secondary regions leverage Google Cloud data centres with published carbon-free energy (CFE) performance metrics.", + "source_ref": "NFR-12" + }, + { + "id": "K-16", + "on": [ + "S-6", + "S-10" + ], + "condition": "Platform foundation delivery and workload ingestion readiness are achieved within 10 weeks of project commencement.", + "source_ref": "NFR-13" + }, + { + "id": "K-17", + "on": [ + "S-8", + "S-10", + "S-11" + ], + "condition": "Central platform team operates 24/7 support for infrastructure incidents and availability alerts.", + "source_ref": "NFR-14" + }, + { + "id": "K-18", + "on": [ + "S-8", + "S-10", + "S-11" + ], + "condition": "Pull request workflows require automated CI/CD linting, validation, and peer approval prior to merge and deployment.", + "source_ref": "NFR-14" + }, + { + "id": "K-19", + "on": [ + "S-3" + ], + "condition": "NVA instances use machine families that support high bandwidth networking.", + "source_ref": "DR-3" + }, + { + "id": "K-20", + "on": [ + "S-3" + ], + "condition": "The warm standby environment operates at a minimal pilot light footprint during normal operations.", + "source_ref": "DR-11" + }, + { + "id": "K-21", + "on": [ + "S-3" + ], + "condition": "Automated disaster recovery scripts scale the standby environment to full capacity only during a failover event.", + "source_ref": "DR-11" + } + ], + "build": { + "customer_owned": [ + "Order and provision the dedicated carrier circuit" + ] + } + }, + "operations": { + "support_hours": "24x7", + "availability_hours": "24x7" + }, + "sourcing": { + "model": "provider_operated", + "delivery_countries": [ + "GB", + "PL", + "IN" + ] + }, + "compliance": { + "frameworks": [ + "DORA", + "ISO27001", + "PCI_DSS_v4" + ] + }, + "commercial": { + "currency": "GBP" + } + }, + "binding": [ + "operations.support_hours", + "sourcing.model", + "sourcing.delivery_countries" + ], + "assumed": [ + { + "field": "intent.business_outcomes", + "value": [ + { + "outcome": "Continuous and resilient payment processing with high availability and low-latency hybrid connectivity", + "id": "BO-1" + }, + { + "outcome": "Sustained regulatory compliance with DORA and PCI-DSS v4.0 security and resilience standards", + "id": "BO-2" + }, + { + "id": "BO-3", + "outcome": "Rapid and secure platform change releases through automated CI/CD delivery pipelines" + } + ], + "why": "Derived from customer request for a UK-regulated payment platform requiring high resilience, continuous compliance, and automated delivery." + }, + { + "field": "intent.exclusions", + "value": [ + "Application container deployment and runtime microservices managed by application squads", + "Active-active secondary region in phase 1 (warm standby adopted)" + ], + "why": "Derived from workload scope boundaries where application workloads are managed by application squads, and secondary region is warm standby rather than active-active." + }, + { + "field": "solution.build.duration_days", + "value": 50, + "why": "Derived from 10-week delivery window at 5 working days per week." + }, + { + "field": "operations.change_process", + "value": "Infrastructure changes managed via pull requests with required approvals and automated GitHub Actions CI/CD pipelines", + "why": "Derived from request and scope specification indicating pull-request driven changes with peer review and GitHub Actions CI/CD automation." + }, + { + "field": "organisation.existing_teams", + "value": [], + "why": "Catalogue default: no existing operational teams assumed; roles to be provisioned under provider-operated model." + }, + { + "field": "service_management.tooling", + "value": "Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging)", + "why": "Catalogue default: Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging) with no separate third-party ticketing system mandated." + }, + { + "field": "service_levels.priority_1_response_minutes", + "value": 30, + "why": "Catalogue default for Priority 1 incident response acknowledgement." + }, + { + "field": "service_levels.recovery_time_hours", + "value": 4, + "why": "Catalogue default and aligned with target 4-hour RTO for warm standby disaster recovery in secondary region." + } + ], + "to_decide": [], + "to_ask": [ + { + "field": "metadata.client_name", + "prompt": "What is metadata.client_name?", + "severity": "LOW" + }, + { + "field": "metadata.author", + "prompt": "What is metadata.author?", + "severity": "LOW" + }, + { + "field": "metadata.revision", + "prompt": "What is metadata.revision?", + "severity": "LOW" + }, + { + "field": "intent.requirements", + "prompt": "Which requirements for operations has the customer stated outright?", + "severity": "LOW" + }, + { + "field": "solution.inherited_risks", + "prompt": "What is already known to be unverified or at risk in the solution?", + "severity": "LOW" + }, + { + "field": "operations.monitoring", + "prompt": "Which monitoring and alerting is already in place or required?", + "severity": "LOW" + }, + { + "field": "organisation.headcount_limit", + "prompt": "Is there a limit on the number of people operating this?", + "severity": "LOW" + }, + { + "field": "commercial.monthly_run_budget", + "prompt": "Is there a monthly budget for operating this, in that currency?", + "severity": "LOW" + }, + { + "field": "transition.go_live_date", + "prompt": "When must the solution be in operation?", + "severity": "LOW" + } + ], + "not_applicable": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/operating_model/scope.yaml b/eval/results/20261021T203623Z-handoff/operating_model/scope.yaml new file mode 100644 index 0000000..9f88e8c --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/scope.yaml @@ -0,0 +1,466 @@ +schema_version: 1 +metadata: + customer_case: case-08-landing-zone + client_name: ask + author: ask + revision: ask + source: architecture:6c316f05-9643-492c-bab1-eed56e108754 + status: UNDER_REVIEW +engagement: + kind: platform_foundation +intent: + statement: We are a UK-regulated payment platform expanding into Google Cloud. We + need an enterprise-grade landing zone ready in under 10 weeks to support our core + payments engine. All payment processing must comply with DORA and PCI-DSS v4.0. + We have existing on-premises systems that require private, low-latency connectivity + over a 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on + and GitHub Actions for our delivery pipelines. Security requires that all egress + traffic flows through centralised inspection appliances before touching the public + internet, and customer data must be encrypted with keys we control. + requirements: ask + business_outcomes: + - outcome: Continuous and resilient payment processing with high availability and + low-latency hybrid connectivity + id: BO-1 + - outcome: Sustained regulatory compliance with DORA and PCI-DSS v4.0 security and + resilience standards + id: BO-2 + - id: BO-3 + outcome: Rapid and secure platform change releases through automated CI/CD delivery + pipelines + exclusions: + - Application container deployment and runtime microservices managed by application + squads + - Active-active secondary region in phase 1 (warm standby adopted) +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Multi-Tier Environment Hierarchy & Isolation Boundary + kind: governance + product: Resource Manager + source_ref: C-1 + elements: + - E-1 + - E-2 + - E-3 + - id: S-2 + name: Dedicated Hybrid Network Interconnect Gateway + kind: network + product: Cloud Interconnect + source_ref: C-2 + elements: + - E-9 + - E-10 + - E-11 + - E-12 + - E-35 + - E-36 + - id: S-3 + name: Centralised Egress Traffic Inspection Gateway + kind: compute + product: Compute Engine + source_ref: C-3 + elements: + - E-13 + - E-14 + - E-15 + - E-16 + - E-17 + - id: S-4 + name: Transit Network Fabric & Routing Controller + kind: service + product: Virtual Private Cloud + source_ref: C-4 + elements: + - E-4 + - E-5 + - E-40 + - E-41 + - E-42 + - E-6 + - E-7 + - E-8 + - E-43 + - E-44 + - E-45 + - E-37 + - E-47 + - E-51 + - E-52 + - E-53 + - id: S-5 + name: Corporate Identity Federation Gateway + kind: identity + product: Cloud Identity + source_ref: C-5 + elements: + - E-18 + - id: S-6 + name: Automated Delivery Pipeline Identity Broker + kind: identity + product: Workload Identity Federation + source_ref: C-6 + elements: + - E-19 + - E-20 + - E-54 + - E-55 + - id: S-7 + name: Cryptographic Key Management & Rotation Service + kind: keys + product: Cloud KMS + source_ref: C-7 + elements: + - E-21 + - E-22 + - E-23 + - E-24 + - id: S-8 + name: Centralised Immutable Audit Log Repository + kind: storage + product: Cloud Storage + source_ref: C-8 + elements: + - E-25 + - E-26 + - id: S-9 + name: Platform Compliance & Governance Guardrail Controller + kind: governance + product: Organization Policy Service + source_ref: C-9 + elements: + - E-27 + - E-28 + - E-29 + - E-30 + - E-38 + - E-46 + - E-56 + - E-57 + - id: S-10 + name: Declarative Infrastructure Provisioning & State Engine + kind: storage + product: Cloud Storage + source_ref: C-10 + elements: + - E-31 + - id: S-11 + name: Centralised Operational Telemetry & Alerting Engine + kind: observability + product: Cloud Monitoring + source_ref: C-11 + elements: + - E-32 + - E-33 + - E-58 + conditions: + - id: K-1 + 'on': + - S-1 + - S-7 + - S-9 + condition: Customer data and payment processing remain within United Kingdom and + European Union legal jurisdictions. + source_ref: NFR-1 + - id: K-2 + 'on': + - S-1 + - S-7 + - S-9 + condition: Warm standby environment operates with reduced capacity until activated + during a disaster recovery failover. + source_ref: NFR-1 + - id: K-3 + 'on': + - S-1 + - S-9 + condition: PCI-regulated production environment is isolated from non-PCI scopes + according to PCI-DSS v4.0 network segmentation requirements. + source_ref: NFR-2 + - id: K-4 + 'on': + - S-3 + - S-4 + condition: Workload APIs and payment services are accessible exclusively through + private corporate network paths. + source_ref: NFR-3 + - id: K-5 + 'on': + - S-2 + condition: Hybrid connection maintains low-latency, SLA-backed private data transmission + between on-premises and Google Cloud. + source_ref: NFR-4 + - id: K-6 + 'on': + - S-3 + condition: Centralised inspection appliances perform next-generation stateful + packet inspection, URL filtering, and threat detection on outbound internet + sessions. + source_ref: NFR-5 + - id: K-7 + 'on': + - S-2 + - S-4 + condition: Platform foundation failover to europe-west1 can be completed within + 4 hours during a primary region outage. + source_ref: NFR-6 + - id: K-8 + 'on': + - S-2 + - S-4 + condition: Configuration drift and backup state achieve a maximum data loss window + (RPO) of 1 hour. + source_ref: NFR-6 + - id: K-9 + 'on': + - S-5 + - S-6 + condition: Corporate user lifecycle management and MFA enforcement are governed + centrally by Entra ID conditional access policies. + source_ref: NFR-7 + - id: K-10 + 'on': + - S-1 + - S-9 + condition: Platform governance and resilience architecture comply with DORA regulatory + testing and third-party risk management frameworks. + source_ref: NFR-8 + - id: K-11 + 'on': + - S-1 + - S-9 + condition: Production cardholder data environment satisfies PCI-DSS v4.0 isolation + and security control requirements. + source_ref: NFR-8 + - id: K-12 + 'on': + - S-7 + condition: Confidential payment data is secured against unauthorized platform-level + access or data exposure. + source_ref: NFR-9 + - id: K-13 + 'on': + - S-8 + condition: Audit logs cannot be modified, deleted, or shortened by any identity + or administrator prior to retention expiration. + source_ref: NFR-10 + - id: K-14 + 'on': + - S-2 + - S-4 + - S-11 + condition: Monthly availability of the core platform landing zone and network + routing meets or exceeds 99.9%. + source_ref: NFR-11 + - id: K-15 + 'on': + - S-9 + condition: Primary and secondary regions leverage Google Cloud data centres with + published carbon-free energy (CFE) performance metrics. + source_ref: NFR-12 + - id: K-16 + 'on': + - S-6 + - S-10 + condition: Platform foundation delivery and workload ingestion readiness are achieved + within 10 weeks of project commencement. + source_ref: NFR-13 + - id: K-17 + 'on': + - S-8 + - S-10 + - S-11 + condition: Central platform team operates 24/7 support for infrastructure incidents + and availability alerts. + source_ref: NFR-14 + - id: K-18 + 'on': + - S-8 + - S-10 + - S-11 + condition: Pull request workflows require automated CI/CD linting, validation, + and peer approval prior to merge and deployment. + source_ref: NFR-14 + - id: K-19 + 'on': + - S-3 + condition: NVA instances use machine families that support high bandwidth networking. + source_ref: DR-3 + - id: K-20 + 'on': + - S-3 + condition: The warm standby environment operates at a minimal pilot light footprint + during normal operations. + source_ref: DR-11 + - id: K-21 + 'on': + - S-3 + condition: Automated disaster recovery scripts scale the standby environment to + full capacity only during a failover event. + source_ref: DR-11 + build: + duration_days: 50 + customer_owned: + - Order and provision the dedicated carrier circuit + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Infrastructure changes managed via pull requests with required approvals + and automated GitHub Actions CI/CD pipelines + monitoring: ask +sourcing: + model: provider_operated + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: [] + headcount_limit: ask +service_management: + tooling: Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging) +service_levels: + priority_1_response_minutes: 30 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA + - ISO27001 + - PCI_DSS_v4 +commercial: + currency: GBP + monthly_run_budget: ask +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.business_outcomes + statement: 'Assumed intent.business_outcomes is [{''outcome'': ''Continuous and + resilient payment processing with high availability and low-latency hybrid connectivity'', + ''id'': ''BO-1''}, {''outcome'': ''Sustained regulatory compliance with DORA and + PCI-DSS v4.0 security and resilience standards'', ''id'': ''BO-2''}, {''id'': + ''BO-3'', ''outcome'': ''Rapid and secure platform change releases through automated + CI/CD delivery pipelines''}]. Derived from customer request for a UK-regulated + payment platform requiring high resilience, continuous compliance, and automated + delivery.' + impact_if_invalid: Changes what is measured, and what each measure is for. +- id: ASSUMP-02 + field: intent.exclusions + statement: Assumed intent.exclusions is ['Application container deployment and runtime + microservices managed by application squads', 'Active-active secondary region + in phase 1 (warm standby adopted)']. Derived from workload scope boundaries where + application workloads are managed by application squads, and secondary region + is warm standby rather than active-active. + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-03 + field: solution.build.duration_days + statement: Assumed solution.build.duration_days is 50. Derived from 10-week delivery + window at 5 working days per week. + impact_if_invalid: Changes when Capabilities must be ready. +- id: ASSUMP-04 + field: operations.change_process + statement: Assumed operations.change_process is Infrastructure changes managed via + pull requests with required approvals and automated GitHub Actions CI/CD pipelines. + Derived from request and scope specification indicating pull-request driven changes + with peer review and GitHub Actions CI/CD automation. + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-05 + field: organisation.existing_teams + statement: 'Assumed organisation.existing_teams is []. Catalogue default: no existing + operational teams assumed; roles to be provisioned under provider-operated model.' + impact_if_invalid: Changes which Roles are new and which are filled. +- id: ASSUMP-06 + field: service_management.tooling + statement: 'Assumed service_management.tooling is Google Cloud Operations Suite + (Cloud Monitoring and Cloud Logging). Catalogue default: Google Cloud Operations + Suite (Cloud Monitoring and Cloud Logging) with no separate third-party ticketing + system mandated.' + impact_if_invalid: Changes tooling and the processes written for it. +- id: ASSUMP-07 + field: service_levels.priority_1_response_minutes + statement: Assumed service_levels.priority_1_response_minutes is 30. Catalogue default + for Priority 1 incident response acknowledgement. + impact_if_invalid: Changes on-call arrangements and staffing. +- id: ASSUMP-08 + field: service_levels.recovery_time_hours + statement: Assumed service_levels.recovery_time_hours is 4. Catalogue default and + aligned with target 4-hour RTO for warm standby disaster recovery in secondary + region. + impact_if_invalid: Changes continuity arrangements. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: intent.requirements + prompt: Which requirements for operations has the customer stated outright? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: commercial.monthly_run_budget + prompt: Is there a monthly budget for operating this, in that currency? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: digested + commercial.monthly_run_budget: unknown + compliance.frameworks: digested + engagement.kind: digested + intent.business_outcomes: assumed + intent.exclusions: assumed + intent.requirements: unknown + intent.statement: digested + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: digested + metadata.revision: unknown + metadata.source: digested + operations.availability_hours: digested + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: digested + organisation.existing_teams: assumed + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: assumed + service_levels.recovery_time_hours: assumed + service_management.tooling: assumed + solution.build.customer_owned: digested + solution.build.duration_days: assumed + solution.cloud_provider: digested + solution.conditions: digested + solution.inherited_risks: unknown + solution.items: digested + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 3 + answered: 0 + assumed: 8 + unknown: 10 + not_applicable: 0 + digested: 12 diff --git a/eval/results/20261021T203623Z-handoff/operating_model/staffing.json b/eval/results/20261021T203623Z-handoff/operating_model/staffing.json new file mode 100644 index 0000000..89f8172 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/staffing.json @@ -0,0 +1,144 @@ +{ + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + }, + "conversion": { + "from": "USD", + "rate": 0.79, + "as_of": "2026-10-13" + }, + "roles": [ + { + "role": "R-1", + "name": "Platform Reliability Lead", + "party": "P-1", + "party_name": "Managed Service Provider", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 18.0, + "people": 0.1, + "cost_per_month": 1991 + }, + { + "role": "R-2", + "name": "Platform SRE", + "party": "P-1", + "party_name": "Managed Service Provider", + "country": "PL", + "level": "senior", + "hours": "24x7", + "effort_hours_per_month": 138.0, + "people": 5.0, + "cost_per_month": 41640 + }, + { + "role": "R-3", + "name": "Security Analyst", + "party": "P-1", + "party_name": "Managed Service Provider", + "country": "IN", + "level": "senior", + "hours": "24x7", + "effort_hours_per_month": 85.0, + "people": 5.0, + "cost_per_month": 24605 + }, + { + "role": "R-4", + "name": "Platform Engineer", + "party": "P-1", + "party_name": "Managed Service Provider", + "country": "PL", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 171.0, + "people": 1.2, + "cost_per_month": 7768 + }, + { + "role": "R-5", + "name": "Enablement Specialist", + "party": "P-1", + "party_name": "Managed Service Provider", + "country": "GB", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 50.0, + "people": 0.4, + "cost_per_month": 4213 + }, + { + "role": "R-6", + "name": "Compliance Officer", + "party": "P-1", + "party_name": "Managed Service Provider", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 30.0, + "people": 0.2, + "cost_per_month": 3318 + }, + { + "role": "R-7", + "name": "Product Manager", + "party": "P-1", + "party_name": "Managed Service Provider", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 40.0, + "people": 0.3, + "cost_per_month": 4424 + }, + { + "role": "R-8", + "name": "Customer Service Owner", + "party": "P-2", + "party_name": "Customer Organization", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 0.0, + "people": 0.0, + "cost_per_month": null + } + ], + "people": 12.2, + "cost_per_month": 87959, + "by_party": { + "P-1": 87959 + }, + "by_country": { + "GB": 13946, + "PL": 49408, + "IN": 24605 + }, + "by_capability": { + "C-14": 885, + "C-20": 1106, + "C-1": 12069, + "C-2": 18104, + "C-4": 9052, + "C-15": 2414, + "C-12": 7237, + "C-13": 17368, + "C-3": 909, + "C-5": 909, + "C-6": 909, + "C-7": 681, + "C-8": 454, + "C-9": 727, + "C-10": 454, + "C-11": 909, + "C-17": 1817, + "C-18": 3371, + "C-19": 843, + "C-21": 3318, + "C-16": 4424 + } +} \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/operating_model/target-operating-model.json b/eval/results/20261021T203623Z-handoff/operating_model/target-operating-model.json new file mode 100644 index 0000000..018c028 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/target-operating-model.json @@ -0,0 +1,742 @@ +{ + "parties": [ + { + "name": "Managed Service Provider", + "kind": "provider", + "id": "P-1" + }, + { + "name": "Customer Organization", + "id": "P-2", + "kind": "customer" + } + ], + "teams": [ + { + "name": "Platform Engineering Team", + "type": "platform", + "id": "T-1", + "party": "P-1" + }, + { + "id": "T-2", + "type": "platform", + "party": "P-1", + "name": "Platform Operations Team" + }, + { + "type": "platform", + "id": "T-3", + "name": "Security Operations Team", + "purpose": "24x7 Security Monitoring and Response", + "party": "P-1" + }, + { + "type": "enabling", + "id": "T-4", + "name": "Developer Enablement Team", + "party": "P-1" + }, + { + "id": "T-5", + "name": "Customer Governance and App Squads", + "party": "P-2", + "type": "stream_aligned" + } + ], + "roles": [ + { + "party": "P-1", + "team": "T-1", + "level": "lead", + "name": "Platform Reliability Lead", + "country": "GB", + "id": "R-1", + "hours": "business" + }, + { + "id": "R-2", + "level": "senior", + "party": "P-1", + "country": "PL", + "hours": "24x7", + "name": "Platform SRE", + "team": "T-2" + }, + { + "country": "IN", + "id": "R-3", + "level": "senior", + "party": "P-1", + "hours": "24x7", + "name": "Security Analyst", + "team": "T-3" + }, + { + "level": "senior", + "party": "P-1", + "name": "Platform Engineer", + "hours": "business", + "country": "PL", + "id": "R-4", + "team": "T-1" + }, + { + "level": "senior", + "party": "P-1", + "id": "R-5", + "hours": "business", + "name": "Enablement Specialist", + "country": "GB", + "team": "T-4" + }, + { + "team": "T-1", + "id": "R-6", + "country": "GB", + "hours": "business", + "name": "Compliance Officer", + "party": "P-1", + "level": "lead" + }, + { + "id": "R-7", + "country": "GB", + "party": "P-1", + "level": "lead", + "team": "T-1", + "hours": "business", + "name": "Product Manager" + }, + { + "country": "GB", + "party": "P-2", + "id": "R-8", + "level": "lead", + "team": "T-5", + "hours": "business", + "name": "Customer Service Owner" + } + ], + "interactions": [ + { + "mode": "x_as_a_service", + "purpose": "Application teams consume platform services through the self-service portal.", + "id": "I-1", + "from": "T-5", + "to": "T-1" + }, + { + "id": "I-2", + "from": "T-4", + "to": "T-5", + "purpose": "Enablement specialists help application teams adopt golden paths and DORA metrics.", + "mode": "facilitating" + }, + { + "until": "End of the 10-week foundation build period, returning to a facilitating mode.", + "mode": "collaboration", + "id": "I-3", + "to": "T-5", + "purpose": "Enablement team pairs with application teams for initial workload ingestion.", + "from": "T-4" + } + ], + "measures": [ + { + "layer": "outcome", + "name": "Deployment frequency", + "framework": "dora-metrics", + "measured_by": "C-19", + "target": "Multiple times a day", + "id": "M-1", + "outcome": "BO-3" + }, + { + "target": "Positive trend", + "outcome": "BO-3", + "framework": "space-framework", + "measured_by": "C-19", + "name": "Satisfaction and well-being", + "id": "M-2", + "layer": "guardrail" + }, + { + "framework": "devex-framework", + "name": "Feedback loops", + "target": "Minimal friction", + "measured_by": "C-19", + "id": "M-3", + "layer": "leading", + "outcome": "BO-3" + }, + { + "target": "99.9%", + "id": "M-4", + "layer": "service_level", + "outcome": "BO-1", + "name": "Platform Availability", + "measured_by": "C-15" + }, + { + "layer": "service_level", + "id": "M-5", + "target": "Under 30 minutes", + "name": "Incident Response Time", + "measured_by": "C-15", + "outcome": "BO-1" + }, + { + "outcome": "BO-2", + "name": "Compliance Audit Pass Rate", + "measured_by": "C-21", + "layer": "service_level", + "id": "M-6", + "target": "100%" + } + ], + "responsibilities": [ + { + "role": "R-1", + "capability": "C-1", + "kind": "accountable", + "id": "RS-1" + }, + { + "id": "RS-2", + "capability": "C-2", + "kind": "accountable", + "role": "R-1" + }, + { + "role": "R-1", + "capability": "C-3", + "id": "RS-3", + "kind": "accountable" + }, + { + "role": "R-1", + "capability": "C-4", + "id": "RS-4", + "kind": "accountable" + }, + { + "role": "R-1", + "capability": "C-5", + "id": "RS-5", + "kind": "accountable" + }, + { + "capability": "C-6", + "id": "RS-6", + "role": "R-1", + "kind": "accountable" + }, + { + "role": "R-1", + "kind": "accountable", + "capability": "C-7", + "id": "RS-7" + }, + { + "kind": "accountable", + "role": "R-1", + "id": "RS-8", + "capability": "C-8" + }, + { + "kind": "accountable", + "capability": "C-9", + "id": "RS-9", + "role": "R-1" + }, + { + "kind": "accountable", + "id": "RS-10", + "capability": "C-10", + "role": "R-1" + }, + { + "id": "RS-11", + "kind": "accountable", + "capability": "C-11", + "role": "R-6" + }, + { + "capability": "C-12", + "role": "R-6", + "id": "RS-12", + "kind": "accountable" + }, + { + "id": "RS-13", + "kind": "accountable", + "capability": "C-13", + "role": "R-6" + }, + { + "capability": "C-14", + "kind": "accountable", + "id": "RS-14", + "role": "R-7" + }, + { + "kind": "accountable", + "role": "R-1", + "capability": "C-15", + "id": "RS-15" + }, + { + "role": "R-7", + "capability": "C-16", + "kind": "accountable", + "id": "RS-16" + }, + { + "role": "R-7", + "capability": "C-17", + "kind": "accountable", + "id": "RS-17" + }, + { + "capability": "C-18", + "role": "R-7", + "id": "RS-18", + "kind": "accountable" + }, + { + "id": "RS-19", + "kind": "accountable", + "capability": "C-19", + "role": "R-7" + }, + { + "id": "RS-20", + "capability": "C-20", + "kind": "accountable", + "role": "R-1" + }, + { + "role": "R-6", + "capability": "C-21", + "id": "RS-21", + "kind": "accountable" + }, + { + "capability": "C-1", + "kind": "responsible", + "role": "R-2", + "id": "RS-22" + }, + { + "id": "RS-23", + "role": "R-2", + "capability": "C-2", + "kind": "responsible" + }, + { + "kind": "responsible", + "role": "R-4", + "capability": "C-3", + "id": "RS-24" + }, + { + "role": "R-2", + "id": "RS-25", + "kind": "responsible", + "capability": "C-4" + }, + { + "kind": "responsible", + "id": "RS-26", + "capability": "C-5", + "role": "R-4" + }, + { + "role": "R-4", + "kind": "responsible", + "id": "RS-27", + "capability": "C-6" + }, + { + "id": "RS-28", + "kind": "responsible", + "role": "R-4", + "capability": "C-7" + }, + { + "id": "RS-29", + "role": "R-4", + "capability": "C-8", + "kind": "responsible" + }, + { + "capability": "C-9", + "role": "R-4", + "kind": "responsible", + "id": "RS-30" + }, + { + "capability": "C-10", + "id": "RS-31", + "role": "R-4", + "kind": "responsible" + }, + { + "capability": "C-11", + "role": "R-4", + "id": "RS-32", + "kind": "responsible" + }, + { + "capability": "C-12", + "kind": "responsible", + "id": "RS-33", + "role": "R-3" + }, + { + "kind": "responsible", + "id": "RS-34", + "capability": "C-13", + "role": "R-3" + }, + { + "role": "R-1", + "kind": "responsible", + "capability": "C-14", + "id": "RS-35" + }, + { + "id": "RS-36", + "kind": "responsible", + "capability": "C-15", + "role": "R-2" + }, + { + "kind": "responsible", + "capability": "C-16", + "role": "R-7", + "id": "RS-37" + }, + { + "id": "RS-38", + "capability": "C-17", + "role": "R-4", + "kind": "responsible" + }, + { + "role": "R-5", + "id": "RS-39", + "kind": "responsible", + "capability": "C-18" + }, + { + "id": "RS-40", + "role": "R-5", + "kind": "responsible", + "capability": "C-19" + }, + { + "kind": "responsible", + "id": "RS-41", + "role": "R-1", + "capability": "C-20" + }, + { + "kind": "responsible", + "role": "R-6", + "id": "RS-42", + "capability": "C-21" + }, + { + "role": "R-8", + "id": "RS-43", + "kind": "consulted", + "capability": "C-1" + }, + { + "kind": "consulted", + "role": "R-8", + "id": "RS-44", + "capability": "C-2" + }, + { + "id": "RS-45", + "kind": "consulted", + "capability": "C-3", + "role": "R-8" + }, + { + "capability": "C-4", + "role": "R-8", + "kind": "consulted", + "id": "RS-46" + }, + { + "capability": "C-5", + "kind": "consulted", + "role": "R-8", + "id": "RS-47" + }, + { + "kind": "consulted", + "capability": "C-6", + "role": "R-8", + "id": "RS-48" + }, + { + "id": "RS-49", + "capability": "C-7", + "role": "R-8", + "kind": "consulted" + }, + { + "role": "R-8", + "kind": "consulted", + "capability": "C-8", + "id": "RS-50" + }, + { + "id": "RS-51", + "capability": "C-9", + "role": "R-8", + "kind": "consulted" + }, + { + "id": "RS-52", + "capability": "C-10", + "role": "R-8", + "kind": "consulted" + }, + { + "capability": "C-11", + "id": "RS-53", + "role": "R-8", + "kind": "consulted" + }, + { + "capability": "C-12", + "role": "R-8", + "kind": "consulted", + "id": "RS-54" + }, + { + "role": "R-8", + "kind": "consulted", + "id": "RS-55", + "capability": "C-13" + }, + { + "role": "R-8", + "kind": "consulted", + "capability": "C-14", + "id": "RS-56" + }, + { + "capability": "C-15", + "kind": "consulted", + "role": "R-8", + "id": "RS-57" + }, + { + "role": "R-8", + "kind": "consulted", + "id": "RS-58", + "capability": "C-16" + }, + { + "role": "R-8", + "kind": "consulted", + "capability": "C-17", + "id": "RS-59" + }, + { + "capability": "C-18", + "kind": "consulted", + "role": "R-8", + "id": "RS-60" + }, + { + "id": "RS-61", + "kind": "consulted", + "role": "R-8", + "capability": "C-19" + }, + { + "kind": "consulted", + "role": "R-8", + "capability": "C-20", + "id": "RS-62" + }, + { + "kind": "consulted", + "role": "R-8", + "capability": "C-21", + "id": "RS-63" + } + ], + "checks": [ + { + "depends_on_finding": null, + "condition": "Declarative infrastructure configurations restrict data location to UK and EU regions.", + "id": "OC-1", + "source": "K-1", + "on": "C-7" + }, + { + "source": "K-2", + "condition": "Warm standby capacity limits on identity, encryption, and governance components are maintained through DR automation.", + "depends_on_finding": null, + "on": "C-9", + "id": "OC-2" + }, + { + "on": "C-7", + "condition": "PCI-regulated production network configurations enforce isolation from non-PCI scopes.", + "id": "OC-3", + "source": "K-3" + }, + { + "source": "K-4", + "id": "OC-4", + "condition": "Workload APIs are confirmed to be routable only via private corporate network paths.", + "on": "C-7" + }, + { + "id": "OC-5", + "on": "C-10", + "source": "K-5", + "condition": "Hybrid interconnect capacity metrics demonstrate SLA-backed low latency." + }, + { + "condition": "Centralised inspection appliances actively perform stateful packet inspection on all egress traffic.", + "id": "OC-6", + "source": "K-6", + "on": "C-13" + }, + { + "on": "C-9", + "condition": "Service continuity rehearsals demonstrate secondary region failover in under 4 hours.", + "id": "OC-7", + "source": "K-7" + }, + { + "condition": "Backup validation ensures an RPO of at most 1 hour.", + "on": "C-7", + "id": "OC-8", + "depends_on_finding": "AF-3", + "source": "K-8" + }, + { + "on": "C-11", + "source": "K-9", + "id": "OC-9", + "condition": "Identity access policies require Entra ID multi-factor authentication for corporate users." + }, + { + "condition": "Third-party supplier reviews include DORA risk assessments.", + "id": "OC-10", + "source": "K-10", + "on": "C-20" + }, + { + "on": "C-21", + "source": "K-11", + "id": "OC-11", + "condition": "Production cardholder data environments maintain PCI-DSS v4.0 network isolation control compliance." + }, + { + "id": "OC-12", + "source": "K-12", + "on": "C-11", + "condition": "Confidential payment data storage systems ensure unauthorized access protection via customer-controlled keys." + }, + { + "condition": "Audit logs are actively monitored for modifications and deletions.", + "source": "K-13", + "id": "OC-13", + "on": "C-13" + }, + { + "condition": "Monthly availability of network routing components is tracked and meets the 99.9% target.", + "on": "C-15", + "id": "OC-14", + "source": "K-14" + }, + { + "condition": "Data centres supporting operations provide publicly validated carbon-free energy (CFE) metrics.", + "on": "C-21", + "source": "K-15", + "id": "OC-15" + }, + { + "id": "OC-16", + "source": "K-16", + "condition": "Workload ingestion metrics confirm operational readiness within the initial 10-week build.", + "on": "C-16" + }, + { + "id": "OC-17", + "on": "C-1", + "source": "K-17", + "condition": "A 24/7 service desk is resourced to actively log and respond to infrastructure priority incidents." + }, + { + "id": "OC-18", + "condition": "CI/CD pipeline hooks enforce pull request approvals, validation, and linting before merge.", + "on": "C-5", + "source": "K-18" + }, + { + "id": "OC-19", + "condition": "NVA capacity tracking confirms instances utilize high-bandwidth machine families.", + "source": "K-19", + "on": "C-10" + }, + { + "id": "OC-20", + "condition": "Platform budget anomaly alerts check that the warm standby uses minimal infrastructure footprint.", + "depends_on_finding": "AF-2", + "source": "K-20", + "on": "C-14" + }, + { + "condition": "Disaster recovery infrastructure automation limits full-capacity scaling strictly to failover occurrences.", + "id": "OC-21", + "on": "C-9", + "source": "K-21" + }, + { + "id": "OC-22", + "on": "R-1", + "source": "OR-13", + "condition": "Operations and support delivery are provided solely under the agreed service-provider framework." + }, + { + "id": "OC-23", + "source": "OR-14", + "condition": "Staff are located solely in authorized delivery countries.", + "depends_on_finding": null, + "on": "R-1" + }, + { + "condition": "Operational telemetry and incident logging are fully integrated into the operations suite.", + "source": "OR-15", + "id": "OC-24", + "on": "C-2" + }, + { + "id": "OC-25", + "on": "C-21", + "source": "OR-24", + "condition": "Compliance audits consistently certify continuous adherence across DORA, ISO 27001, and PCI-DSS v4.0 frameworks." + }, + { + "on": "C-1", + "source": "K-17", + "condition": "Incident responders use a secure remote access mechanism to troubleshoot Centralised Egress Traffic Inspection Gateways.", + "depends_on_finding": "AF-1", + "id": "OC-26" + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/operating_model/verdicts.json b/eval/results/20261021T203623Z-handoff/operating_model/verdicts.json new file mode 100644 index 0000000..f1659d4 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/operating_model/verdicts.json @@ -0,0 +1,362 @@ +{ + "dry_run": { + "verdict": "pass", + "evidence": "44 of 44 scenario walk-throughs pass." + }, + "checks": [ + { + "evidence": "Capability C-7 handles declarative state and configuration management, covering all required items. Platform Engineer (R-4) is responsible during business hours.", + "verdict": "pass", + "check": "OC-1" + }, + { + "verdict": "pass", + "check": "OC-2", + "evidence": "Capability C-9 manages service continuity and DR failover rehearsals, covering items S-1, S-7, and S-9. Platform Engineer (R-4) is responsible." + }, + { + "verdict": "pass", + "check": "OC-3", + "evidence": "Capability C-7 manages configuration state and audits, ensuring isolation configurations are maintained. Platform Engineer (R-4) is responsible." + }, + { + "evidence": "Capability C-7 audits configuration state for workload APIs, and Platform Engineer (R-4) is responsible.", + "check": "OC-4", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "OC-5", + "evidence": "Capability C-10 reviews capacity and performance metrics, including the dedicated hybrid network interconnect. Platform Engineer (R-4) is responsible." + }, + { + "check": "OC-6", + "evidence": "Capability C-13 provides security monitoring and response, analyzing egress traffic inspection data 24x7 via Security Analyst (R-3).", + "verdict": "pass" + }, + { + "evidence": "Capability C-9 conducts bi-monthly DR failover rehearsals to ensure the 4-hour RTO is achievable. Platform Engineer (R-4) is responsible.", + "verdict": "pass", + "check": "OC-7" + }, + { + "evidence": "AF-3: The solution lacks an automated backup service or vault to ensure state and configuration achieve the maximum data loss window (RPO) of 1 hour.", + "check": "OC-8", + "verdict": "waits_on_finding", + "finding": "AF-3" + }, + { + "check": "OC-9", + "evidence": "Capability C-11 covers access management and policy reviews for corporate identity gateways. Platform Engineer (R-4) is responsible.", + "verdict": "pass" + }, + { + "check": "OC-10", + "evidence": "Capability C-20 conducts supplier management and service reviews, incorporating DORA risk assessments. Platform Reliability Lead (R-1) is responsible.", + "verdict": "pass" + }, + { + "evidence": "Capability C-21 continuously collates regulatory evidence for PCI-DSS v4.0. Compliance Officer (R-6) is responsible.", + "check": "OC-11", + "verdict": "pass" + }, + { + "evidence": "Capability C-11 governs access management for the cryptographic key management service (S-7). Platform Engineer (R-4) is responsible.", + "check": "OC-12", + "verdict": "pass" + }, + { + "check": "OC-13", + "verdict": "pass", + "evidence": "Capability C-13 provides 24x7 security monitoring and response for audit logs (S-8). Security Analyst (R-3) is responsible." + }, + { + "evidence": "Capability C-15 aggregates and reports on service level objectives, including availability targets. Platform SRE (R-2) is responsible.", + "check": "OC-14", + "verdict": "pass" + }, + { + "evidence": "Capability C-21 collates regulatory and compliance evidence, including carbon-free energy metrics. Compliance Officer (R-6) is responsible.", + "check": "OC-15", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "Capability C-16 manages the platform backlog and adoption roadmap to confirm workload ingestion readiness. Product Manager (R-7) is responsible.", + "check": "OC-16" + }, + { + "evidence": "Capability C-1 handles 24x7 incident management, and Platform SRE (R-2) provides 24x7 coverage to log and respond to incidents.", + "check": "OC-17", + "verdict": "pass" + }, + { + "evidence": "Capability C-5 manages change enablement and pull request approvals. Platform Engineer (R-4) is responsible.", + "verdict": "pass", + "check": "OC-18" + }, + { + "verdict": "pass", + "check": "OC-19", + "evidence": "Capability C-10 reviews capacity and performance for network virtual appliances. Platform Engineer (R-4) is responsible." + }, + { + "check": "OC-20", + "evidence": "AF-2: The solution lacks automated budget anomaly alerts out-of-the-box to track the minimal pilot light footprint of the warm standby region.", + "verdict": "waits_on_finding", + "finding": "AF-2" + }, + { + "check": "OC-21", + "verdict": "pass", + "evidence": "Capability C-9 conducts service continuity failovers, managing the DR infrastructure automation. Platform Engineer (R-4) is responsible." + }, + { + "evidence": "Role R-1 (Platform Reliability Lead) is part of the Managed Service Provider party, fulfilling the agreed service-provider framework.", + "verdict": "pass", + "check": "OC-22" + }, + { + "verdict": "pass", + "check": "OC-23", + "evidence": "Role R-1 is located in GB, which is an authorized delivery country." + }, + { + "check": "OC-24", + "verdict": "pass", + "evidence": "Capability C-2 performs continuous monitoring and alerting using the integrated operations suite. Platform SRE (R-2) is responsible 24x7." + }, + { + "evidence": "Capability C-21 continuously collates audit evidence across DORA, ISO 27001, and PCI-DSS v4.0. Compliance Officer (R-6) is responsible.", + "check": "OC-25", + "verdict": "pass" + }, + { + "check": "OC-26", + "evidence": "AF-1: The solution lacks a secure remote access mechanism (such as a bastion host or Identity-Aware Proxy) as a way in for responders to the Centralised Egress Traffic Inspection Gateway.", + "verdict": "waits_on_finding", + "finding": "AF-1" + }, + { + "check": "SC-fault-S-1", + "verdict": "pass", + "evidence": "Multi-Tier Environment Hierarchy & Isolation Boundary (S-1) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-1", + "verdict": "pass", + "evidence": "Multi-Tier Environment Hierarchy & Isolation Boundary (S-1) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-2", + "verdict": "pass", + "evidence": "Dedicated Hybrid Network Interconnect Gateway (S-2) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-2", + "verdict": "pass", + "evidence": "Dedicated Hybrid Network Interconnect Gateway (S-2) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-3", + "verdict": "pass", + "evidence": "Centralised Egress Traffic Inspection Gateway (S-3) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-3", + "verdict": "pass", + "evidence": "Centralised Egress Traffic Inspection Gateway (S-3) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-4", + "verdict": "pass", + "evidence": "Transit Network Fabric & Routing Controller (S-4) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-4", + "verdict": "pass", + "evidence": "Transit Network Fabric & Routing Controller (S-4) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-5", + "verdict": "pass", + "evidence": "Corporate Identity Federation Gateway (S-5) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-5", + "verdict": "pass", + "evidence": "Corporate Identity Federation Gateway (S-5) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-6", + "verdict": "pass", + "evidence": "Automated Delivery Pipeline Identity Broker (S-6) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-6", + "verdict": "pass", + "evidence": "Automated Delivery Pipeline Identity Broker (S-6) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-7", + "verdict": "pass", + "evidence": "Cryptographic Key Management & Rotation Service (S-7) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-7", + "verdict": "pass", + "evidence": "Cryptographic Key Management & Rotation Service (S-7) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-8", + "verdict": "pass", + "evidence": "Centralised Immutable Audit Log Repository (S-8) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-8", + "verdict": "pass", + "evidence": "Centralised Immutable Audit Log Repository (S-8) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-9", + "verdict": "pass", + "evidence": "Platform Compliance & Governance Guardrail Controller (S-9) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-9", + "verdict": "pass", + "evidence": "Platform Compliance & Governance Guardrail Controller (S-9) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-10", + "verdict": "pass", + "evidence": "Declarative Infrastructure Provisioning & State Engine (S-10) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-10", + "verdict": "pass", + "evidence": "Declarative Infrastructure Provisioning & State Engine (S-10) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-11", + "verdict": "pass", + "evidence": "Centralised Operational Telemetry & Alerting Engine (S-11) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-11", + "verdict": "pass", + "evidence": "Centralised Operational Telemetry & Alerting Engine (S-11) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-condition-K-1", + "verdict": "pass", + "evidence": "Condition K-1 must be shown to hold: Customer data and payment processing remain within United Kingdom and European Union legal jurisdictions. Check OC-1 on C-7 keeps it true." + }, + { + "check": "SC-condition-K-2", + "verdict": "pass", + "evidence": "Condition K-2 must be shown to hold: Warm standby environment operates with reduced capacity until activated during a disaster recovery failover. Check OC-2 on C-9 keeps it true." + }, + { + "check": "SC-condition-K-3", + "verdict": "pass", + "evidence": "Condition K-3 must be shown to hold: PCI-regulated production environment is isolated from non-PCI scopes according to PCI-DSS v4.0 network segmentation requirements. Check OC-3 on C-7 keeps it true." + }, + { + "check": "SC-condition-K-4", + "verdict": "pass", + "evidence": "Condition K-4 must be shown to hold: Workload APIs and payment services are accessible exclusively through private corporate network paths. Check OC-4 on C-7 keeps it true." + }, + { + "check": "SC-condition-K-5", + "verdict": "pass", + "evidence": "Condition K-5 must be shown to hold: Hybrid connection maintains low-latency, SLA-backed private data transmission between on-premises and Google Cloud. Check OC-5 on C-10 keeps it true." + }, + { + "check": "SC-condition-K-6", + "verdict": "pass", + "evidence": "Condition K-6 must be shown to hold: Centralised inspection appliances perform next-generation stateful packet inspection, URL filtering, and threat detection on outbound internet sessions. Check OC-6 on C-13 keeps it true." + }, + { + "check": "SC-condition-K-7", + "verdict": "pass", + "evidence": "Condition K-7 must be shown to hold: Platform foundation failover to europe-west1 can be completed within 4 hours during a primary region outage. Check OC-7 on C-9 keeps it true." + }, + { + "check": "SC-condition-K-8", + "verdict": "pass", + "evidence": "Condition K-8 must be shown to hold: Configuration drift and backup state achieve a maximum data loss window (RPO) of 1 hour. Check OC-8 on C-7 keeps it true." + }, + { + "check": "SC-condition-K-9", + "verdict": "pass", + "evidence": "Condition K-9 must be shown to hold: Corporate user lifecycle management and MFA enforcement are governed centrally by Entra ID conditional access policies. Check OC-9 on C-11 keeps it true." + }, + { + "check": "SC-condition-K-10", + "verdict": "pass", + "evidence": "Condition K-10 must be shown to hold: Platform governance and resilience architecture comply with DORA regulatory testing and third-party risk management frameworks. Check OC-10 on C-20 keeps it true." + }, + { + "check": "SC-condition-K-11", + "verdict": "pass", + "evidence": "Condition K-11 must be shown to hold: Production cardholder data environment satisfies PCI-DSS v4.0 isolation and security control requirements. Check OC-11 on C-21 keeps it true." + }, + { + "check": "SC-condition-K-12", + "verdict": "pass", + "evidence": "Condition K-12 must be shown to hold: Confidential payment data is secured against unauthorized platform-level access or data exposure. Check OC-12 on C-11 keeps it true." + }, + { + "check": "SC-condition-K-13", + "verdict": "pass", + "evidence": "Condition K-13 must be shown to hold: Audit logs cannot be modified, deleted, or shortened by any identity or administrator prior to retention expiration. Check OC-13 on C-13 keeps it true." + }, + { + "check": "SC-condition-K-14", + "verdict": "pass", + "evidence": "Condition K-14 must be shown to hold: Monthly availability of the core platform landing zone and network routing meets or exceeds 99.9%. Check OC-14 on C-15 keeps it true." + }, + { + "check": "SC-condition-K-15", + "verdict": "pass", + "evidence": "Condition K-15 must be shown to hold: Primary and secondary regions leverage Google Cloud data centres with published carbon-free energy (CFE) performance metrics. Check OC-15 on C-21 keeps it true." + }, + { + "check": "SC-condition-K-16", + "verdict": "pass", + "evidence": "Condition K-16 must be shown to hold: Platform foundation delivery and workload ingestion readiness are achieved within 10 weeks of project commencement. Check OC-16 on C-16 keeps it true." + }, + { + "check": "SC-condition-K-17", + "verdict": "pass", + "evidence": "Condition K-17 must be shown to hold: Central platform team operates 24/7 support for infrastructure incidents and availability alerts. Check OC-17 on C-1 keeps it true." + }, + { + "check": "SC-condition-K-18", + "verdict": "pass", + "evidence": "Condition K-18 must be shown to hold: Pull request workflows require automated CI/CD linting, validation, and peer approval prior to merge and deployment. Check OC-18 on C-5 keeps it true." + }, + { + "check": "SC-condition-K-19", + "verdict": "pass", + "evidence": "Condition K-19 must be shown to hold: NVA instances use machine families that support high bandwidth networking. Check OC-19 on C-10 keeps it true." + }, + { + "check": "SC-condition-K-20", + "verdict": "pass", + "evidence": "Condition K-20 must be shown to hold: The warm standby environment operates at a minimal pilot light footprint during normal operations. Check OC-20 on C-14 keeps it true." + }, + { + "check": "SC-condition-K-21", + "verdict": "pass", + "evidence": "Condition K-21 must be shown to hold: Automated disaster recovery scripts scale the standby environment to full capacity only during a failover event. Check OC-21 on C-9 keeps it true." + }, + { + "check": "SC-party-P-1", + "verdict": "pass", + "evidence": "Managed Service Provider cannot be reached. Every Capability it is answerable for has a Role in another Party to turn to." + } + ], + "proposed_additions": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/report.md b/eval/results/20261021T203623Z-handoff/report.md new file mode 100644 index 0000000..7f0abc3 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/report.md @@ -0,0 +1,19 @@ +# Solution architecture to operating model, through the Orchestration engine + +Run `20261021T203623Z`. One pre-approved request for `architecture_and_operating_model` to the Orchestration engine: the solution architecture agent designed from the benchmark scope spec, and its kept files went to the operating model agent as `sources`. The case is fictional; no customer's data is in it. + +- **Status:** completed +- **Customer case:** case-08-landing-zone +- **Minutes, end to end:** 74.9 + +| Agent | Execution | Ended | Files kept | +|---|---|---|---| +| solution_architecture | 6c316f05-9643-492c-bab1-eed56e108754 | completed | 15 | +| operating_model | 1f8f30a1-7b74-4d23-9fb0-aec58089c506 | completed | 11 | + +## The operating model + +- Operational requirements: 24; verified by a Check: 24 +- Capabilities: 21; Roles: 8; Checks: 26 +- Scenarios with a path: 44 of 44 +- People: 12.2; cost a month: 87,959 GBP diff --git a/eval/results/20261021T203623Z-handoff/result.json b/eval/results/20261021T203623Z-handoff/result.json new file mode 100644 index 0000000..e458f80 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/result.json @@ -0,0 +1,71 @@ +{ + "status": "completed", + "work": "architecture_and_operating_model", + "customer_case": "case-08-landing-zone", + "execution_id": "1f8f30a1-7b74-4d23-9fb0-aec58089c506", + "record": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506", + "guide": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operating-model-guide.md", + "files": { + "capabilities.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/capabilities.json", + "findings.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/findings.json", + "operating-model-guide.md": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operating-model-guide.md", + "operational-requirements.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operational-requirements.json", + "organisation.mmd": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/organisation.mmd", + "scenarios.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scenarios.json", + "scope.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scope.json", + "scope.yaml": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scope.yaml", + "staffing.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/staffing.json", + "target-operating-model.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/target-operating-model.json", + "verdicts.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/verdicts.json" + }, + "latest": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/latest/operating_model", + "case": "http://localhost:10030/tenants/local/cases/case-08-landing-zone", + "results": { + "solution_architecture": { + "execution_id": "6c316f05-9643-492c-bab1-eed56e108754", + "status": "completed", + "record": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754", + "guide": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/solution-architecture-guide.md", + "files": { + "architecture.mmd": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/architecture.mmd", + "components.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/components.json", + "implementation.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/implementation.json", + "products.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/products.json", + "recommendations.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/recommendations.json", + "requirements.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/requirements.json", + "scope.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/scope.json", + "scope.yaml": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/scope.yaml", + "solution-architecture-guide.md": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/solution-architecture-guide.md", + "target-architecture.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/target-architecture.json", + "terraform/main.tf": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/terraform/main.tf", + "terraform/variables.tf": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/terraform/variables.tf", + "terraform/versions.tf": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/terraform/versions.tf", + "verdicts.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/verdicts.json", + "work-breakdown.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/work-breakdown.json" + }, + "latest": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/latest/solution_architecture", + "case": "http://localhost:10030/tenants/local/cases/case-08-landing-zone" + }, + "operating_model": { + "execution_id": "1f8f30a1-7b74-4d23-9fb0-aec58089c506", + "status": "completed", + "record": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506", + "guide": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operating-model-guide.md", + "files": { + "capabilities.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/capabilities.json", + "findings.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/findings.json", + "operating-model-guide.md": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operating-model-guide.md", + "operational-requirements.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operational-requirements.json", + "organisation.mmd": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/organisation.mmd", + "scenarios.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scenarios.json", + "scope.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scope.json", + "scope.yaml": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scope.yaml", + "staffing.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/staffing.json", + "target-operating-model.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/target-operating-model.json", + "verdicts.json": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/verdicts.json" + }, + "latest": "http://localhost:10030/tenants/local/cases/case-08-landing-zone/latest/operating_model", + "case": "http://localhost:10030/tenants/local/cases/case-08-landing-zone" + } + } +} \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/architecture.mmd b/eval/results/20261021T203623Z-handoff/solution_architecture/architecture.mmd new file mode 100644 index 0000000..ccee1b1 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/architecture.mmd @@ -0,0 +1,72 @@ +flowchart LR + n0["Dev Folder
Resource Manager"] + n1["Staging Folder
Resource Manager"] + n2["Production Folder
Resource Manager"] + n3["Host Project
Virtual Private Cloud"] + n4["Hub Network
Virtual Private Cloud"] + n5["Dev Network
Virtual Private Cloud"] + n6["Staging Network
Virtual Private Cloud"] + n7["Production Network
Virtual Private Cloud"] + n8["Dev Subnetwork
Virtual Private Cloud"] + n9["Staging Subnetwork
Virtual Private Cloud"] + n10["Production Subnetwork
Virtual Private Cloud"] + n11["Hub-Prod Peering
Virtual Private Cloud"] + n12["Hub-Dev Peering
Virtual Private Cloud"] + n13["Hub-Stg Peering
Virtual Private Cloud"] + n14["Interconnect Router 1
Cloud Interconnect"] + n15["Interconnect Router 2
Cloud Interconnect"] + n16["VLAN Attachment 1
Cloud Interconnect"] + n17["VLAN Attachment 2
Cloud Interconnect"] + n18["Standby Router
Cloud Interconnect"] + n19["Standby VLAN Attachment
Cloud Interconnect"] + n20["Standby Subnetwork
Virtual Private Cloud"] + n21["NVA Instance Template
Compute Engine"] + n22["NVA MIG
Compute Engine"] + n23["NVA Health Check
Compute Engine"] + n24["NVA Forwarding Rule
Compute Engine"] + n25["NVA Autoscaler
Compute Engine"] + n26["Inbound Deny Firewall
Virtual Private Cloud"] + n27["Cloud Identity Group
Cloud Identity"] + n28["Workload Identity Pool
Workload Identity Federation"] + n29["WIF Provider
Workload Identity Federation"] + n30["KMS Key Ring Primary
Cloud KMS"] + n31["KMS Crypto Key Primary
Cloud KMS"] + n32["KMS Key Ring Standby
Cloud KMS"] + n33["KMS Crypto Key Standby
Cloud KMS"] + n34["Audit Log Bucket
Cloud Storage"] + n35["Org Log Sink
Cloud Storage"] + n36["Org Policy Skip Default Network
Organization Policy Service"] + n37["Org Policy VM External IP
Organization Policy Service"] + n38["Org Policy Resource Locations
Organization Policy Service"] + n39["Org Policy Service Account Keys
Organization Policy Service"] + n40["SCC Config
Organization Policy Service"] + n41["Prod Org Policy
Organization Policy Service"] + n42["TF State Bucket
Cloud Storage"] + n43["Alert Policy Interconnect
Cloud Monitoring"] + n44["Alert Policy LB
Cloud Monitoring"] + n45["Dev Route to NVA
Virtual Private Cloud"] + n46["Staging Route to NVA
Virtual Private Cloud"] + n47["Production Route to NVA
Virtual Private Cloud"] + n48["Pipeline Service Account
Workload Identity Federation"] + n49["Pipeline SA IAM Member
Workload Identity Federation"] + n50["Org Policy UBLA
Organization Policy Service"] + n51["Org Policy CMEK Enforce
Organization Policy Service"] + n52["Alert Policy Audit Logs
Cloud Monitoring"] + n5 -->|peers with| n4 + n6 -->|peers with| n4 + n7 -->|peers with| n4 + n45 -->|routes to| n24 + n46 -->|routes to| n24 + n47 -->|routes to| n24 + n29 -->|authenticates via| n28 + n35 -->|exports to| n34 + n16 -->|attaches to| n14 + n17 -->|attaches to| n15 + n19 -->|attaches to| n18 + n22 -->|uses| n21 + n24 -->|forwards to| n22 + n22 -->|checked by| n23 + n25 -->|scales| n22 + n31 -->|belongs to| n30 + n33 -->|belongs to| n32 + n49 -->|grants access to| n48 diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/components.json b/eval/results/20261021T203623Z-handoff/solution_architecture/components.json new file mode 100644 index 0000000..a091169 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/components.json @@ -0,0 +1,57 @@ +[ + { + "id": "C-1", + "name": "Multi-Tier Environment Hierarchy & Isolation Boundary", + "requirements": ["FR-1", "NFR-1", "NFR-2", "NFR-8"] + }, + { + "id": "C-2", + "name": "Dedicated Hybrid Network Interconnect Gateway", + "requirements": ["FR-2", "NFR-4", "NFR-6", "NFR-11"] + }, + { + "id": "C-3", + "name": "Centralised Egress Traffic Inspection Gateway", + "requirements": ["FR-3", "NFR-3", "NFR-5"] + }, + { + "id": "C-4", + "name": "Transit Network Fabric & Routing Controller", + "requirements": ["FR-1", "FR-2", "FR-3", "NFR-3", "NFR-6", "NFR-11"] + }, + { + "id": "C-5", + "name": "Corporate Identity Federation Gateway", + "requirements": ["FR-4", "NFR-7"] + }, + { + "id": "C-6", + "name": "Automated Delivery Pipeline Identity Broker", + "requirements": ["FR-5", "NFR-7", "NFR-13"] + }, + { + "id": "C-7", + "name": "Cryptographic Key Management & Rotation Service", + "requirements": ["FR-6", "NFR-1", "NFR-9"] + }, + { + "id": "C-8", + "name": "Centralised Immutable Audit Log Repository", + "requirements": ["FR-7", "NFR-10", "NFR-14"] + }, + { + "id": "C-9", + "name": "Platform Compliance & Governance Guardrail Controller", + "requirements": ["FR-1", "NFR-1", "NFR-2", "NFR-8", "NFR-12"] + }, + { + "id": "C-10", + "name": "Declarative Infrastructure Provisioning & State Engine", + "requirements": ["FR-5", "NFR-13", "NFR-14"] + }, + { + "id": "C-11", + "name": "Centralised Operational Telemetry & Alerting Engine", + "requirements": ["NFR-11", "NFR-14"] + } +] diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/implementation.json b/eval/results/20261021T203623Z-handoff/solution_architecture/implementation.json new file mode 100644 index 0000000..46f4b32 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/implementation.json @@ -0,0 +1,123 @@ +[ + {"check": "K-1", "file": "terraform/main.tf", "address": "google_org_policy_policy.resource_locations.spec"}, + {"check": "K-2", "file": "terraform/main.tf", "address": "google_compute_subnetwork.dev.region"}, + {"check": "K-3", "file": "terraform/main.tf", "address": "google_compute_subnetwork.staging.region"}, + {"check": "K-4", "file": "terraform/main.tf", "address": "google_compute_subnetwork.production.region"}, + {"check": "K-5", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_1.region"}, + {"check": "K-6", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_2.region"}, + {"check": "K-7", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_1.region"}, + {"check": "K-8", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_2.region"}, + {"check": "K-9", "file": "terraform/main.tf", "address": "google_compute_region_instance_group_manager.nva.region"}, + {"check": "K-10", "file": "terraform/main.tf", "address": "google_compute_subnetwork.standby.region"}, + {"check": "K-11", "file": "terraform/main.tf", "address": "google_compute_router.standby.region"}, + {"check": "K-12", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.standby_vlan.region"}, + {"check": "K-13", "file": "terraform/main.tf", "address": "google_folder.dev.display_name"}, + {"check": "K-14", "file": "terraform/main.tf", "address": "google_folder.staging.display_name"}, + {"check": "K-15", "file": "terraform/main.tf", "address": "google_folder.production.display_name"}, + {"check": "K-16", "file": "terraform/main.tf", "address": "google_compute_firewall.inbound_deny.deny"}, + {"check": "K-17", "file": "terraform/main.tf", "address": "google_org_policy_policy.prod_org_policy.parent"}, + {"check": "K-18", "file": "terraform/main.tf", "address": "google_compute_shared_vpc_host_project.host.project"}, + {"check": "K-19", "file": "terraform/main.tf", "address": "google_compute_network_peering.hub_prod.peer_network"}, + {"check": "K-20", "file": "terraform/main.tf", "address": "google_compute_network_peering.hub_dev.peer_network"}, + {"check": "K-21", "file": "terraform/main.tf", "address": "google_compute_network_peering.hub_stg.peer_network"}, + {"check": "K-22", "file": "terraform/main.tf", "address": "google_compute_subnetwork.dev.private_ip_google_access"}, + {"check": "K-23", "file": "terraform/main.tf", "address": "google_compute_subnetwork.staging.private_ip_google_access"}, + {"check": "K-24", "file": "terraform/main.tf", "address": "google_compute_subnetwork.production.private_ip_google_access"}, + {"check": "K-25", "file": "terraform/main.tf", "address": "google_compute_network.dev.delete_default_routes_on_create"}, + {"check": "K-26", "file": "terraform/main.tf", "address": "google_compute_network.staging.delete_default_routes_on_create"}, + {"check": "K-27", "file": "terraform/main.tf", "address": "google_compute_network.production.delete_default_routes_on_create"}, + {"check": "K-28", "file": "terraform/main.tf", "address": "google_compute_firewall.inbound_deny.source_ranges"}, + {"check": "K-29", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_1.bandwidth"}, + {"check": "K-30", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_2.bandwidth"}, + {"check": "K-31", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_1.bgp"}, + {"check": "K-32", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_2.bgp"}, + {"check": "K-33", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_1.network"}, + {"check": "K-34", "file": "terraform/main.tf", "address": "google_compute_network.hub.name"}, + {"check": "K-35", "file": "terraform/main.tf", "address": "google_compute_route.dev_nva.next_hop_ilb"}, + {"check": "K-36", "file": "terraform/main.tf", "address": "google_compute_route.staging_nva.next_hop_ilb"}, + {"check": "K-37", "file": "terraform/main.tf", "address": "google_compute_route.production_nva.next_hop_ilb"}, + {"check": "K-38", "file": "terraform/main.tf", "address": "google_compute_network.dev.delete_default_routes_on_create"}, + {"check": "K-39", "file": "terraform/main.tf", "address": "google_compute_network.staging.delete_default_routes_on_create"}, + {"check": "K-40", "file": "terraform/main.tf", "address": "google_compute_network.production.delete_default_routes_on_create"}, + {"check": "K-41", "file": "terraform/main.tf", "address": "google_compute_subnetwork.standby.region"}, + {"check": "K-42", "file": "terraform/main.tf", "address": "google_compute_router.standby.region"}, + {"check": "K-43", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.standby_vlan.region"}, + {"check": "K-44", "file": "terraform/main.tf", "address": "google_iam_workload_identity_pool.wif_pool.workload_identity_pool_id"}, + {"check": "K-45", "file": "terraform/main.tf", "address": "google_iam_workload_identity_pool_provider.wif_provider.oidc"}, + {"check": "K-46", "file": "terraform/main.tf", "address": "google_service_account_iam_member.pipeline_wif.member"}, + {"check": "K-47", "file": "terraform/main.tf", "address": "google_org_policy_policy.sa_keys.spec"}, + {"check": "K-48", "file": "terraform/main.tf", "address": "google_cloud_identity_group.group.parent"}, + {"check": "K-49", "file": "terraform/main.tf", "address": "google_org_policy_policy.ubla.spec"}, + {"check": "K-50", "file": "terraform/main.tf", "address": "google_org_policy_policy.cmek_enforce.spec"}, + {"check": "K-51", "file": "terraform/main.tf", "address": "google_org_policy_policy.vm_external_ip.spec"}, + {"check": "K-52", "file": "terraform/main.tf", "address": "google_scc_organization_custom_module.scc_config.custom_config"}, + {"check": "K-53", "file": "terraform/main.tf", "address": "google_kms_key_ring.primary.location"}, + {"check": "K-54", "file": "terraform/main.tf", "address": "google_kms_crypto_key.primary.rotation_period"}, + {"check": "K-55", "file": "terraform/main.tf", "address": "google_kms_key_ring.standby.location"}, + {"check": "K-56", "file": "terraform/main.tf", "address": "google_kms_crypto_key.standby.rotation_period"}, + {"check": "K-57", "file": "terraform/main.tf", "address": "google_org_policy_policy.cmek_enforce.spec"}, + {"check": "K-58", "file": "terraform/main.tf", "address": "google_logging_organization_sink.org_sink.destination"}, + {"check": "K-59", "file": "terraform/main.tf", "address": "google_storage_bucket.audit_log.retention_policy"}, + {"check": "K-60", "file": "terraform/main.tf", "address": "google_storage_bucket.audit_log.retention_policy"}, + {"check": "K-61", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_1.edge_availability_domain"}, + {"check": "K-62", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_2.edge_availability_domain"}, + {"check": "K-63", "file": "terraform/main.tf", "address": "google_compute_region_instance_group_manager.nva.distribution_policy_zones"}, + {"check": "K-64", "file": "terraform/main.tf", "address": "google_org_policy_policy.resource_locations.spec"}, + {"check": "K-65", "file": "terraform/main.tf", "address": "google_storage_bucket.tf_state.name"}, + {"check": "K-66", "file": "terraform/main.tf", "address": "google_storage_bucket.tf_state.versioning"}, + {"check": "K-67", "file": "terraform/main.tf", "address": "google_storage_bucket.tf_state.encryption"}, + {"check": "K-68", "file": "terraform/main.tf", "address": "google_monitoring_alert_policy.interconnect.conditions"}, + {"check": "K-69", "file": "terraform/main.tf", "address": "google_monitoring_alert_policy.lb.conditions"}, + {"check": "K-70", "file": "terraform/main.tf", "address": "google_monitoring_alert_policy.audit_logs.conditions"}, + {"check": "K-71", "file": "terraform/main.tf", "address": "google_storage_bucket.audit_log.labels"}, + {"check": "K-72", "file": "terraform/main.tf", "address": "google_org_policy_policy.skip_default_network.spec"}, + {"check": "K-73", "file": "terraform/main.tf", "address": "google_org_policy_policy.vm_external_ip.spec"}, + {"check": "K-74", "file": "terraform/main.tf", "address": "google_org_policy_policy.resource_locations.spec"}, + {"check": "K-75", "file": "terraform/main.tf", "address": "google_org_policy_policy.sa_keys.spec"}, + {"check": "K-76", "file": "terraform/main.tf", "address": "google_compute_forwarding_rule.nva.load_balancing_scheme"}, + {"check": "K-77", "file": "terraform/main.tf", "address": "google_compute_route.dev_nva.next_hop_ilb"}, + {"check": "K-78", "file": "terraform/main.tf", "address": "google_compute_route.staging_nva.next_hop_ilb"}, + {"check": "K-79", "file": "terraform/main.tf", "address": "google_compute_route.production_nva.next_hop_ilb"}, + {"check": "K-80", "file": "terraform/main.tf", "address": "google_compute_region_instance_group_manager.nva.base_instance_name"}, + {"check": "K-81", "file": "terraform/main.tf", "address": "google_compute_region_instance_group_manager.nva.auto_healing_policies"}, + {"check": "K-82", "file": "terraform/main.tf", "address": "google_compute_region_autoscaler.nva.target"}, + {"check": "K-83", "file": "terraform/main.tf", "address": "google_compute_instance_template.nva.network_interface"}, + {"check": "K-84", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_1.network"}, + {"check": "K-85", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_2.network"}, + {"check": "K-86", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_1.router"}, + {"check": "K-87", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_2.router"}, + {"check": "K-88", "file": "terraform/main.tf", "address": "google_compute_network.hub.routing_mode"}, + {"check": "K-89", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_1.bgp"}, + {"check": "K-90", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_2.bgp"}, + {"check": "K-91", "file": "terraform/main.tf", "address": "google_kms_key_ring.primary.location"}, + {"check": "K-92", "file": "terraform/main.tf", "address": "google_kms_key_ring.standby.location"}, + {"check": "K-93", "file": "terraform/main.tf", "address": "google_kms_crypto_key.primary.rotation_period"}, + {"check": "K-94", "file": "terraform/main.tf", "address": "google_kms_crypto_key.standby.rotation_period"}, + {"check": "K-95", "file": "terraform/main.tf", "address": "google_storage_bucket.audit_log.retention_policy"}, + {"check": "K-96", "file": "terraform/main.tf", "address": "google_storage_bucket.audit_log.retention_policy"}, + {"check": "K-97", "file": "terraform/main.tf", "address": "google_storage_bucket.audit_log.lifecycle_rule"}, + {"check": "K-98", "file": "terraform/main.tf", "address": "google_storage_bucket.tf_state.versioning"}, + {"check": "K-99", "file": "terraform/main.tf", "address": "google_storage_bucket.tf_state.encryption"}, + {"check": "K-100", "file": "terraform/main.tf", "address": "google_storage_bucket.tf_state.lifecycle_rule"}, + {"check": "K-101", "file": "terraform/main.tf", "address": "google_iam_workload_identity_pool_provider.wif_provider.attribute_condition"}, + {"check": "K-102", "file": "terraform/main.tf", "address": "google_monitoring_alert_policy.interconnect.conditions"}, + {"check": "K-103", "file": "terraform/main.tf", "address": "google_monitoring_alert_policy.lb.conditions"}, + {"check": "K-104", "file": "terraform/main.tf", "address": "google_compute_network.hub.mtu"}, + {"check": "K-105", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_1.mtu"}, + {"check": "K-106", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_2.mtu"}, + {"check": "K-107", "file": "terraform/main.tf", "address": "google_compute_region_health_check.nva.check_interval_sec"}, + {"check": "K-108", "file": "terraform/main.tf", "address": "google_service_account.pipeline_sa.account_id"}, + {"check": "K-109", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_2.network"}, + {"check": "K-110", "file": "terraform/main.tf", "address": "google_storage_bucket.tf_state.labels"}, + {"check": "K-111", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_1.labels"}, + {"check": "K-112", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.vlan_2.labels"}, + {"check": "K-113", "file": "terraform/main.tf", "address": "google_compute_interconnect_attachment.standby_vlan.labels"}, + {"check": "K-114", "file": "terraform/main.tf", "address": "google_compute_instance_template.nva.labels"}, + {"check": "K-115", "file": "terraform/main.tf", "address": "google_compute_forwarding_rule.nva.labels"}, + {"check": "K-116", "file": "terraform/main.tf", "address": "google_cloud_identity_group.group.labels"}, + {"check": "K-117", "file": "terraform/main.tf", "address": "google_kms_crypto_key.primary.labels"}, + {"check": "K-118", "file": "terraform/main.tf", "address": "google_kms_crypto_key.standby.labels"}, + {"check": "K-119", "file": "terraform/main.tf", "address": "google_storage_bucket.tf_state.labels"}, + {"check": "K-120", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_1.bgp"}, + {"check": "K-121", "file": "terraform/main.tf", "address": "google_compute_router.interconnect_2.bgp"} +] \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/products.json b/eval/results/20261021T203623Z-handoff/solution_architecture/products.json new file mode 100644 index 0000000..2a03a00 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/products.json @@ -0,0 +1,13 @@ +[ + {"component": "C-1", "product": "Resource Manager"}, + {"component": "C-2", "product": "Cloud Interconnect"}, + {"component": "C-3", "product": "Compute Engine"}, + {"component": "C-4", "product": "Virtual Private Cloud"}, + {"component": "C-5", "product": "Cloud Identity"}, + {"component": "C-6", "product": "Workload Identity Federation"}, + {"component": "C-7", "product": "Cloud KMS"}, + {"component": "C-8", "product": "Cloud Storage"}, + {"component": "C-9", "product": "Organization Policy Service"}, + {"component": "C-10", "product": "Cloud Storage"}, + {"component": "C-11", "product": "Cloud Monitoring"} +] \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/recommendations.json b/eval/results/20261021T203623Z-handoff/solution_architecture/recommendations.json new file mode 100644 index 0000000..1f024c2 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/recommendations.json @@ -0,0 +1,123 @@ +[ + { + "id": "DR-1", + "pillar": "security", + "text": "Enforce strict security and compliance guardrails across the resource hierarchy using Organization Policy.", + "components": ["C-9"], + "conditions": [ + "The compute.skipDefaultNetworkCreation constraint is enforced.", + "The compute.vmExternalIpAccess constraint is enforced.", + "The gcp.resourceLocations constraint is restricted to the allowed regions.", + "The iam.disableServiceAccountKeyCreation constraint is enforced." + ] + }, + { + "id": "DR-2", + "pillar": "security", + "text": "Route outbound internet traffic symmetrically through highly available inspection appliances.", + "components": ["C-3", "C-4"], + "conditions": [ + "An internal passthrough network load balancer is deployed in the transit VPC.", + "A custom route is configured in the spoke VPCs directing 0.0.0.0/0 traffic to the internal network load balancer." + ] + }, + { + "id": "DR-3", + "pillar": "reliability", + "text": "Deploy Network Virtual Appliances (NVAs) for resilience, elasticity, and high throughput.", + "components": ["C-3"], + "conditions": [ + "The NVA instances are deployed in a regional Managed Instance Group.", + "The NVA Managed Instance Group is configured with auto-healing using a health check.", + "The NVA Managed Instance Group is configured with autoscaling.", + "The NVA Compute Engine instances are configured to use the gVNIC network interface." + ], + "outside_code": [ + "NVA instances use machine families that support high bandwidth networking." + ] + }, + { + "id": "DR-4", + "pillar": "reliability", + "text": "Configure Dedicated Cloud Interconnect for high availability and optimal latency.", + "components": ["C-2", "C-4"], + "conditions": [ + "At least two Cloud Routers are deployed in the transit Shared VPC.", + "Redundant Interconnect VLAN attachments are configured on the Cloud Routers.", + "The Cloud Routers are configured for regional dynamic routing.", + "The Cloud Routers are configured with tuned BGP keepalive and hold timers." + ] + }, + { + "id": "DR-5", + "pillar": "security", + "text": "Ensure resilient and compliant cryptographic key management.", + "components": ["C-7"], + "conditions": [ + "Cloud KMS key rings are provisioned in both the primary and standby regions.", + "Customer-managed encryption keys (CMEK) are configured with a rotation period of 90 days." + ] + }, + { + "id": "DR-6", + "pillar": "cost-optimization", + "text": "Configure the centralized audit log repository for compliance immutability and cost-efficient storage tiering.", + "components": ["C-8"], + "conditions": [ + "The audit log Cloud Storage bucket is configured with a retention policy.", + "The retention policy on the audit log bucket is locked.", + "An Object Lifecycle Management policy is configured to transition objects to colder storage classes." + ] + }, + { + "id": "DR-7", + "pillar": "operational-excellence", + "text": "Secure and govern the Terraform remote state repository to prevent configuration drift and storage bloat.", + "components": ["C-10"], + "conditions": [ + "The Terraform state Cloud Storage bucket has object versioning enabled.", + "The Terraform state bucket is encrypted with a customer-managed encryption key.", + "An Object Lifecycle Management policy is configured to delete non-current versions of the state objects." + ] + }, + { + "id": "DR-8", + "pillar": "security", + "text": "Restrict automated delivery pipelines identity federation to authorized repositories.", + "components": ["C-6"], + "conditions": [ + "The Workload Identity Pool Provider is configured with an attribute condition restricting access to specific GitHub repositories." + ] + }, + { + "id": "DR-9", + "pillar": "operational-excellence", + "text": "Implement centralized health checking and telemetry alerting for critical hybrid and inspection components.", + "components": ["C-11"], + "conditions": [ + "Cloud Monitoring alerting policies are configured for interconnect BGP session health.", + "Cloud Monitoring alerting policies are configured for NVA load balancer health checks." + ] + }, + { + "id": "DR-10", + "pillar": "performance-optimization", + "text": "Optimize network maximum transmission unit (MTU) across the transit fabric and hybrid interconnect.", + "components": ["C-2", "C-4"], + "conditions": [ + "The Shared VPC network is configured with a custom MTU setting.", + "The Interconnect VLAN attachments are configured with a matching MTU." + ] + }, + { + "id": "DR-11", + "pillar": "sustainability", + "text": "Configure the warm standby environment to operate with a reduced footprint to minimize idle carbon emissions.", + "components": ["C-3"], + "conditions": [], + "outside_code": [ + "The warm standby environment operates at a minimal pilot light footprint during normal operations.", + "Automated disaster recovery scripts scale the standby environment to full capacity only during a failover event." + ] + } +] \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/record.json b/eval/results/20261021T203623Z-handoff/solution_architecture/record.json new file mode 100644 index 0000000..326a815 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/record.json @@ -0,0 +1 @@ +{"tenant":"local","customer_case":"case-08-landing-zone","execution_id":"6c316f05-9643-492c-bab1-eed56e108754","work":"solution_architecture","status":"completed","request":"We are a UK-regulated payment platform expanding into Google Cloud. We need an enterprise-grade landing zone ready in under 10 weeks to support our core payments engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have existing on-premises systems that require private, low-latency connectivity over a 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and GitHub Actions for our delivery pipelines. Security requires that all egress traffic flows through centralised inspection appliances before touching the public internet, and customer data must be encrypted with keys we control.","project_id":null,"guide":"solution-architecture-guide.md","files":["architecture.mmd","components.json","implementation.json","products.json","recommendations.json","requirements.json","scope.json","scope.yaml","solution-architecture-guide.md","target-architecture.json","terraform/main.tf","terraform/variables.tf","terraform/versions.tf","verdicts.json","work-breakdown.json"],"kept_at":"2026-10-21T21:26:33.819300+00:00","links":{"record":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754","guide":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/solution-architecture-guide.md","files":{"architecture.mmd":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/architecture.mmd","components.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/components.json","implementation.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/implementation.json","products.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/products.json","recommendations.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/recommendations.json","requirements.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/requirements.json","scope.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/scope.json","scope.yaml":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/scope.yaml","solution-architecture-guide.md":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/solution-architecture-guide.md","target-architecture.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/target-architecture.json","terraform/main.tf":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/terraform/main.tf","terraform/variables.tf":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/terraform/variables.tf","terraform/versions.tf":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/terraform/versions.tf","verdicts.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/verdicts.json","work-breakdown.json":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/work-breakdown.json"},"latest":"http://localhost:10030/tenants/local/cases/case-08-landing-zone/latest/solution_architecture","case":"http://localhost:10030/tenants/local/cases/case-08-landing-zone"}} \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/requirements.json b/eval/results/20261021T203623Z-handoff/solution_architecture/requirements.json new file mode 100644 index 0000000..08b1536 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/requirements.json @@ -0,0 +1,304 @@ +[ + { + "id": "FR-1", + "kind": "functional", + "text": "The platform foundation must provision distinct, isolated environment tiers for development, staging, and production to host payment workloads.", + "from": [ + "REQ-02" + ] + }, + { + "id": "FR-2", + "kind": "functional", + "text": "The platform foundation must establish private, bidirectional hybrid network connectivity between on-premises payment systems and cloud environments over a dedicated interconnect.", + "from": [ + "REQ-03" + ] + }, + { + "id": "FR-3", + "kind": "functional", + "text": "The platform foundation must inspect and filter all outbound traffic from cloud environments to the public internet through centralised inspection appliances.", + "from": [ + "REQ-03" + ] + }, + { + "id": "FR-4", + "kind": "functional", + "text": "The platform foundation must federate corporate identities from Entra ID to authenticate administrative users for platform access.", + "from": [ + "REQ-04" + ] + }, + { + "id": "FR-5", + "kind": "functional", + "text": "The platform foundation must authenticate automated GitHub Actions delivery pipelines via workload identity federation to deploy infrastructure without persistent keys.", + "from": [ + "REQ-01", + "REQ-04" + ] + }, + { + "id": "FR-6", + "kind": "functional", + "text": "The platform foundation must provide customer-managed encryption key (CMEK) management to encrypt data across platform storage and compute resources.", + "from": [ + "REQ-04" + ] + }, + { + "id": "FR-7", + "kind": "functional", + "text": "The platform foundation must aggregate audit logs from all environment tiers into a central log repository with an immutable retention policy.", + "from": [ + "REQ-02" + ] + }, + { + "id": "NFR-1", + "kind": "non_functional", + "text": "Platform resources must reside in europe-west2 with a warm standby presence in europe-west1, ensuring all customer and payment data remains within the United Kingdom and European Union.", + "conditions": [ + "Resource location restriction policy permits resource creation only in europe-west2 and europe-west1.", + "Primary landing zone network and shared services are provisioned in europe-west2.", + "Secondary landing zone network and warm standby services are provisioned in europe-west1." + ], + "outside_code": [ + "Customer data and payment processing remain within United Kingdom and European Union legal jurisdictions.", + "Warm standby environment operates with reduced capacity until activated during a disaster recovery failover." + ], + "from": [ + "REQ-02", + "estate.regions.primary", + "estate.regions.secondary", + "estate.regions.resiliency_tier", + "estate.data_residency" + ] + }, + { + "id": "NFR-2", + "kind": "non_functional", + "text": "The platform foundation must enforce multi-tier environment isolation separating development, staging, and production tiers, with production adhering to a dedicated PCI-regulated archetype.", + "conditions": [ + "Separate Google Cloud projects or folders are created for development, staging, and production tiers.", + "No direct VPC peering or cross-environment network routing exists between production and non-production environments.", + "Production environment enforces dedicated organizational policies restricting external access and unauthorized services." + ], + "outside_code": [ + "PCI-regulated production environment is isolated from non-PCI scopes according to PCI-DSS v4.0 network segmentation requirements." + ], + "from": [ + "REQ-02", + "governance.environment_tiers" + ] + }, + { + "id": "NFR-3", + "kind": "non_functional", + "text": "The network architecture must implement a Shared VPC hub-and-spoke topology with internal-only exposure, preventing direct public internet ingress into workload environments.", + "conditions": [ + "A central hub Shared VPC network is interconnected with dedicated spoke VPC networks for each environment tier.", + "Workload subnets have external IP assignment disabled.", + "Default public internet gateway routes are removed from workload spoke subnets.", + "Firewall policies deny all inbound connections from public IP addresses to workload subnets." + ], + "outside_code": [ + "Workload APIs and payment services are accessible exclusively through private corporate network paths." + ], + "from": [ + "REQ-02", + "REQ-03", + "networking.exposure", + "networking.topology" + ] + }, + { + "id": "NFR-4", + "kind": "non_functional", + "text": "Hybrid connectivity between on-premises payment systems and Google Cloud must operate over a 10Gbps dedicated Cloud Interconnect.", + "conditions": [ + "Dedicated Cloud Interconnect attachments are configured for 10Gbps capacity.", + "Redundant Cloud Routers are configured with BGP sessions peering to on-premises routers.", + "VPC routing tables direct traffic destined for on-premises RFC 1918 CIDRs across the Cloud Interconnect attachments." + ], + "outside_code": [ + "Hybrid connection maintains low-latency, SLA-backed private data transmission between on-premises and Google Cloud." + ], + "from": [ + "REQ-03", + "networking.hybrid_connectivity.enabled", + "networking.hybrid_connectivity.type", + "networking.hybrid_connectivity.bandwidth_gbps" + ] + }, + { + "id": "NFR-5", + "kind": "non_functional", + "text": "All outbound internet traffic from workload environments must route through centralised inspection appliances.", + "conditions": [ + "A central egress inspection VPC network is configured to host inspection appliances.", + "Routing policies direct all 0.0.0.0/0 traffic from spoke networks to the central inspection appliances before egressing to the internet.", + "Spoke networks have no direct route or NAT gateway to the public internet bypassing inspection appliances." + ], + "outside_code": [ + "Centralised inspection appliances perform next-generation stateful packet inspection, URL filtering, and threat detection on outbound internet sessions." + ], + "from": [ + "REQ-03", + "networking.egress_control.model" + ] + }, + { + "id": "NFR-6", + "kind": "non_functional", + "text": "The platform foundation must support warm standby disaster recovery achieving a Recovery Time Objective (RTO) of 4 hours and a Recovery Point Objective (RPO) of 1 hour.", + "conditions": [ + "Secondary network and infrastructure resources in europe-west1 are defined in Infrastructure as Code for rapid replication.", + "Standby Cloud Routers and Interconnect VLAN attachments are configured in europe-west1." + ], + "outside_code": [ + "Platform foundation failover to europe-west1 can be completed within 4 hours during a primary region outage.", + "Configuration drift and backup state achieve a maximum data loss window (RPO) of 1 hour." + ], + "from": [ + "REQ-02", + "estate.regions.secondary", + "estate.regions.resiliency_tier" + ] + }, + { + "id": "NFR-7", + "kind": "non_functional", + "text": "User authentication must integrate with Entra ID as the single source of truth, and CI/CD automation must authenticate via Workload Identity Federation without long-lived keys.", + "conditions": [ + "Workload Identity Pool and OIDC provider are configured for GitHub Actions repositories.", + "IAM service accounts for deployment pipelines allow authentication only from authorized GitHub Actions tokens.", + "Service account key creation is restricted by organizational policy.", + "Workforce identity federation or Cloud Identity is configured with Entra ID as the corporate identity provider." + ], + "outside_code": [ + "Corporate user lifecycle management and MFA enforcement are governed centrally by Entra ID conditional access policies." + ], + "from": [ + "REQ-04", + "governance.identity_source_of_truth", + "governance.pipeline_authentication" + ] + }, + { + "id": "NFR-8", + "kind": "non_functional", + "text": "The platform foundation controls must comply with DORA resilience mandates, PCI-DSS v4.0 requirements, and ISO 27001 standards.", + "conditions": [ + "Organizational policies enforce mandatory security guardrails including uniform bucket-level access, CMEK enforcement, and public IP restriction.", + "Security Command Center is configured at the organization level to monitor compliance against regulatory benchmarks." + ], + "outside_code": [ + "Platform governance and resilience architecture comply with DORA regulatory testing and third-party risk management frameworks.", + "Production cardholder data environment satisfies PCI-DSS v4.0 isolation and security control requirements." + ], + "from": [ + "REQ-02", + "compliance.frameworks" + ] + }, + { + "id": "NFR-9", + "kind": "non_functional", + "text": "All confidential payment platform data and associated storage resources must be encrypted at rest using customer-managed encryption keys (CMEK) that automatically rotate every 90 days.", + "conditions": [ + "Cloud KMS key rings and encryption keys are provisioned in europe-west2 and europe-west1.", + "Customer-managed keys have an automated rotation period configured to 90 days (7,776,000 seconds).", + "Organizational policy requires CMEK on supported storage and data services." + ], + "outside_code": [ + "Confidential payment data is secured against unauthorized platform-level access or data exposure." + ], + "from": [ + "REQ-04", + "compliance.encryption.type", + "compliance.encryption.key_rotation_days", + "compliance.data_classification" + ] + }, + { + "id": "NFR-10", + "kind": "non_functional", + "text": "Audit and administrative activity logs across all environments must be aggregated centrally and retained for 365 days under an immutable retention policy.", + "conditions": [ + "Centralized log sink exports all administrative activity and data access audit logs to a dedicated central log storage bucket.", + "Central audit log storage bucket retention policy is configured for 365 days.", + "Retention policy on the audit log bucket is locked and immutable." + ], + "outside_code": [ + "Audit logs cannot be modified, deleted, or shortened by any identity or administrator prior to retention expiration." + ], + "from": [ + "REQ-02", + "compliance.audit_logging.retention_days", + "compliance.audit_logging.immutable_retention_policy" + ] + }, + { + "id": "NFR-11", + "kind": "non_functional", + "text": "The platform foundation shared networking and infrastructure services must support a continuous availability target of 99.9% across availability zones.", + "conditions": [ + "Shared network gateways, Cloud Routers, and transit components are deployed redundantly across at least two availability zones in europe-west2." + ], + "outside_code": [ + "Monthly availability of the core platform landing zone and network routing meets or exceeds 99.9%." + ], + "from": [ + "REQ-02" + ] + }, + { + "id": "NFR-12", + "kind": "non_functional", + "text": "Platform resources must be placed in regions that balance low latency, UK/EU data residency compliance, and high carbon-free energy scores.", + "conditions": [ + "Resource location policy restricts platform resource deployment to europe-west2 and europe-west1." + ], + "outside_code": [ + "Primary and secondary regions leverage Google Cloud data centres with published carbon-free energy (CFE) performance metrics." + ], + "from": [ + "estate.regions.primary", + "estate.regions.secondary" + ] + }, + { + "id": "NFR-13", + "kind": "non_functional", + "text": "The platform foundation must be provisioned and ready for workload ingestion within 10 weeks using automated Terraform Infrastructure as Code.", + "conditions": [ + "All landing zone infrastructure is authored and configured using modular Terraform.", + "Terraform state is stored in a dedicated remote backend with object versioning and customer-managed encryption." + ], + "outside_code": [ + "Platform foundation delivery and workload ingestion readiness are achieved within 10 weeks of project commencement." + ], + "from": [ + "REQ-01" + ] + }, + { + "id": "NFR-14", + "kind": "non_functional", + "text": "Platform operations must provide 24/7 centralized monitoring, telemetry, and alerting with change management governed via automated pull request workflows.", + "conditions": [ + "Cloud Monitoring alert policies are configured for hybrid connectivity status, egress appliance health, and audit log ingestion.", + "All provisioned resources are tagged with standard labels for environment tier, cost centre, and archetype." + ], + "outside_code": [ + "Central platform team operates 24/7 support for infrastructure incidents and availability alerts.", + "Pull request workflows require automated CI/CD linting, validation, and peer approval prior to merge and deployment." + ], + "from": [ + "governance.pipeline_authentication" + ] + } +] \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/scope.json b/eval/results/20261021T203623Z-handoff/solution_architecture/scope.json new file mode 100644 index 0000000..95e9d4a --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/scope.json @@ -0,0 +1,219 @@ +{ + "kind": "platform_foundation", + "stated": { + "engagement": { + "kind": "platform_foundation" + }, + "intent": { + "statement": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.\n", + "requirements": [ + { + "id": "REQ-01", + "category": "Timeline", + "statement": "Platform foundation must be ready for workload ingestion within 10 weeks." + }, + { + "id": "REQ-02", + "category": "Compliance", + "statement": "Must achieve DORA resiliency alignment and PCI-DSS v4.0 perimeter isolation." + }, + { + "id": "REQ-03", + "category": "Networking", + "statement": "10Gbps on-premises hybrid connectivity; centralised inspection appliances for egress." + }, + { + "id": "REQ-04", + "category": "Identity and security", + "statement": "Workload identity federation with GitHub Actions; Entra ID single sign-on; customer-managed encryption keys." + } + ], + "exclusions": [ + "Application container deployment; workloads are managed by application squads.", + "An active-active secondary region in phase 1; warm standby is acceptable." + ] + }, + "estate": { + "cloud_provider": "gcp", + "mode": "greenfield", + "regions": { + "primary": "europe-west2", + "secondary": "europe-west1", + "resiliency_tier": "warm_standby" + }, + "data_residency": "United Kingdom and European Union", + "existing_systems": "On-premises payment systems that stay in place and connect over the interconnect." + }, + "workload": { + "availability": { + "hours": "always" + } + }, + "compliance": { + "industry": "financial_services", + "frameworks": [ + "DORA", + "ISO27001", + "PCI_DSS_v4" + ], + "data_classification": "confidential", + "encryption": { + "type": "cmek", + "key_rotation_days": 90 + }, + "audit_logging": { + "retention_days": 365, + "immutable_retention_policy": true + } + }, + "networking": { + "exposure": "internal_only", + "topology": "shared_vpc_hub_and_spoke", + "hybrid_connectivity": { + "enabled": true, + "type": "cloud_interconnect", + "bandwidth_gbps": 10 + }, + "egress_control": { + "model": "centralized_appliance" + } + }, + "governance": { + "identity_source_of_truth": "Entra ID", + "pipeline_authentication": "workload identity federation", + "environment_tiers": [ + { + "name": "development", + "archetype": "corp_workload" + }, + { + "name": "staging", + "archetype": "corp_workload" + }, + { + "name": "production", + "archetype": "pci_regulated_workload" + } + ] + }, + "operations": { + "operating_model": "central platform team brokering access for application squads", + "support_hours": "always" + }, + "delivery": { + "window_weeks": 10, + "commercial_model": "fixed_price_milestone", + "iac_tooling": "terraform", + "ci_cd": "GitHub Actions", + "team_size": 3, + "working_days_per_week": 5 + }, + "cost": { + "currency": "GBP" + } + }, + "binding": [ + "estate.regions.primary", + "estate.regions.secondary", + "estate.regions.resiliency_tier", + "estate.data_residency", + "compliance.frameworks", + "compliance.data_classification", + "compliance.encryption.type", + "compliance.encryption.key_rotation_days", + "compliance.audit_logging.retention_days", + "compliance.audit_logging.immutable_retention_policy", + "networking.exposure", + "networking.topology", + "networking.hybrid_connectivity.enabled", + "networking.hybrid_connectivity.type", + "networking.hybrid_connectivity.bandwidth_gbps", + "networking.egress_control.model", + "governance.identity_source_of_truth", + "governance.pipeline_authentication", + "governance.environment_tiers" + ], + "assumed": [ + { + "field": "intent.success_criteria", + "value": [ + "The platform foundation is provisioned within 10 weeks, compliant with DORA and PCI-DSS v4.0, providing 10Gbps dedicated interconnect hybrid connectivity, centralised egress inspection, and customer-managed encryption keys." + ], + "why": "Derived from customer statement and requirements." + }, + { + "field": "workload.availability.target", + "value": "99.9%", + "why": "Catalogue default for platform foundation availability." + }, + { + "field": "workload.availability.recovery_time", + "value": "4 hours", + "why": "Catalogue default recovery time objective for warm standby resilience tier." + }, + { + "field": "workload.availability.recovery_point", + "value": "1 hour", + "why": "Catalogue default recovery point objective." + }, + { + "field": "networking.egress_control.appliance_vendor", + "value": "None mandated; to be selected during detailed design", + "why": "Customer mandates centralised inspection appliances but did not specify a vendor." + }, + { + "field": "networking.cidr_allocations", + "value": "Private RFC 1918 ranges that do not overlap with on-premises systems, sized for growth across development, staging, and production environments.", + "why": "Catalogue default: non-overlapping private RFC 1918 ranges sized for growth across environment tiers." + }, + { + "field": "operations.monitoring", + "value": "Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging) with alerts on availability and errors.", + "why": "Catalogue default: Cloud provider native monitoring and alerting." + }, + { + "field": "operations.change_process", + "value": "Infrastructure changes managed via pull requests with required approvals and automated GitHub Actions CI/CD pipelines.", + "why": "Aligned with stated GitHub Actions tooling and compliance controls." + }, + { + "field": "integrations", + "value": [], + "why": "Catalogue default: no third-party integrations beyond existing on-premises hybrid connectivity." + }, + { + "field": "constraints.products_required", + "value": [], + "why": "Catalogue default: no mandated products beyond customer stated tools (Entra ID, GitHub Actions)." + }, + { + "field": "constraints.products_forbidden", + "value": [], + "why": "Catalogue default: no forbidden products identified." + } + ], + "to_decide": [], + "to_ask": [ + { + "field": "metadata.engagement_reference", + "prompt": "What is metadata.engagement_reference?", + "severity": "LOW" + }, + { + "field": "delivery.target_start_date", + "prompt": "When should work start?", + "severity": "LOW" + }, + { + "field": "constraints.other", + "prompt": "What is constraints.other?", + "severity": "LOW" + } + ], + "not_applicable": [ + "cost.monthly_budget", + "workload.users.count", + "workload.users.peak_concurrent", + "workload.users.who" + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/scope.yaml b/eval/results/20261021T203623Z-handoff/solution_architecture/scope.yaml new file mode 100644 index 0000000..56d59c5 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/scope.yaml @@ -0,0 +1,282 @@ +schema_version: 1 +metadata: + customer_case: case-08-landing-zone + engagement_reference: ask + client_name: Example Payments UK + author: Evaluation + revision: 1.0.0 + status: UNDER_REVIEW +engagement: + kind: platform_foundation +intent: + statement: 'We are a UK-regulated payment platform expanding into Google Cloud. + We need an + + enterprise-grade landing zone ready in under 10 weeks to support our core payments + + engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have + + existing on-premises systems that require private, low-latency connectivity over + a + + 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and + + GitHub Actions for our delivery pipelines. Security requires that all egress + + traffic flows through centralised inspection appliances before touching the public + + internet, and customer data must be encrypted with keys we control. + + ' + requirements: + - id: REQ-01 + category: Timeline + statement: Platform foundation must be ready for workload ingestion within 10 + weeks. + - id: REQ-02 + category: Compliance + statement: Must achieve DORA resiliency alignment and PCI-DSS v4.0 perimeter isolation. + - id: REQ-03 + category: Networking + statement: 10Gbps on-premises hybrid connectivity; centralised inspection appliances + for egress. + - id: REQ-04 + category: Identity and security + statement: Workload identity federation with GitHub Actions; Entra ID single sign-on; + customer-managed encryption keys. + exclusions: + - Application container deployment; workloads are managed by application squads. + - An active-active secondary region in phase 1; warm standby is acceptable. + success_criteria: + - The platform foundation is provisioned within 10 weeks, compliant with DORA and + PCI-DSS v4.0, providing 10Gbps dedicated interconnect hybrid connectivity, centralised + egress inspection, and customer-managed encryption keys. +estate: + cloud_provider: gcp + mode: greenfield + regions: + primary: europe-west2 + secondary: europe-west1 + resiliency_tier: warm_standby + data_residency: United Kingdom and European Union + existing_systems: On-premises payment systems that stay in place and connect over + the interconnect. +workload: + users: + who: not_applicable + count: not_applicable + peak_concurrent: not_applicable + availability: + target: 99.9% + hours: always + recovery_time: 4 hours + recovery_point: 1 hour +compliance: + industry: financial_services + frameworks: + - DORA + - ISO27001 + - PCI_DSS_v4 + data_classification: confidential + encryption: + type: cmek + key_rotation_days: 90 + audit_logging: + retention_days: 365 + immutable_retention_policy: true +networking: + exposure: internal_only + topology: shared_vpc_hub_and_spoke + hybrid_connectivity: + enabled: true + type: cloud_interconnect + bandwidth_gbps: 10 + egress_control: + model: centralized_appliance + appliance_vendor: None mandated; to be selected during detailed design + cidr_allocations: Private RFC 1918 ranges that do not overlap with on-premises systems, + sized for growth across development, staging, and production environments. +governance: + identity_source_of_truth: Entra ID + pipeline_authentication: workload identity federation + environment_tiers: + - name: development + archetype: corp_workload + - name: staging + archetype: corp_workload + - name: production + archetype: pci_regulated_workload +operations: + operating_model: central platform team brokering access for application squads + support_hours: always + monitoring: Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging) with + alerts on availability and errors. + change_process: Infrastructure changes managed via pull requests with required approvals + and automated GitHub Actions CI/CD pipelines. +integrations: [] +delivery: + target_start_date: ask + window_weeks: 10 + commercial_model: fixed_price_milestone + iac_tooling: terraform + ci_cd: GitHub Actions + team_size: 3 + working_days_per_week: 5 +cost: + monthly_budget: not_applicable + currency: GBP +constraints: + products_required: [] + products_forbidden: [] + other: ask +run: + pre_approved: true + allow_cloud_read: false + overlay_skills: ask + terraform_modules: false +inferred_assumptions: +- id: ASSUMP-01 + field: intent.success_criteria + statement: Assumed intent.success_criteria is ['The platform foundation is provisioned + within 10 weeks, compliant with DORA and PCI-DSS v4.0, providing 10Gbps dedicated + interconnect hybrid connectivity, centralised egress inspection, and customer-managed + encryption keys.']. Derived from customer statement and requirements. + impact_if_invalid: Local to this field. +- id: ASSUMP-02 + field: workload.availability.target + statement: Assumed workload.availability.target is 99.9%. Catalogue default for + platform foundation availability. + impact_if_invalid: Changes resilience design and cost. +- id: ASSUMP-03 + field: workload.availability.recovery_time + statement: Assumed workload.availability.recovery_time is 4 hours. Catalogue default + recovery time objective for warm standby resilience tier. + impact_if_invalid: Changes backup and standby design. +- id: ASSUMP-04 + field: workload.availability.recovery_point + statement: Assumed workload.availability.recovery_point is 1 hour. Catalogue default + recovery point objective. + impact_if_invalid: Changes backup frequency. +- id: ASSUMP-05 + field: networking.egress_control.appliance_vendor + statement: Assumed networking.egress_control.appliance_vendor is None mandated; + to be selected during detailed design. Customer mandates centralised inspection + appliances but did not specify a vendor. + impact_if_invalid: Local to this field. +- id: ASSUMP-06 + field: networking.cidr_allocations + statement: 'Assumed networking.cidr_allocations is Private RFC 1918 ranges that + do not overlap with on-premises systems, sized for growth across development, + staging, and production environments.. Catalogue default: non-overlapping private + RFC 1918 ranges sized for growth across environment tiers.' + impact_if_invalid: Changes every network resource. +- id: ASSUMP-07 + field: operations.monitoring + statement: 'Assumed operations.monitoring is Google Cloud Operations Suite (Cloud + Monitoring and Cloud Logging) with alerts on availability and errors.. Catalogue + default: Cloud provider native monitoring and alerting.' + impact_if_invalid: Local to this field. +- id: ASSUMP-08 + field: operations.change_process + statement: Assumed operations.change_process is Infrastructure changes managed via + pull requests with required approvals and automated GitHub Actions CI/CD pipelines.. + Aligned with stated GitHub Actions tooling and compliance controls. + impact_if_invalid: Local to this field. +- id: ASSUMP-09 + field: integrations + statement: 'Assumed integrations is []. Catalogue default: no third-party integrations + beyond existing on-premises hybrid connectivity.' + impact_if_invalid: Local to this field. +- id: ASSUMP-10 + field: constraints.products_required + statement: 'Assumed constraints.products_required is []. Catalogue default: no mandated + products beyond customer stated tools (Entra ID, GitHub Actions).' + impact_if_invalid: Local to this field. +- id: ASSUMP-11 + field: constraints.products_forbidden + statement: 'Assumed constraints.products_forbidden is []. Catalogue default: no + forbidden products identified.' + impact_if_invalid: Local to this field. +discovery_gaps: +- field: metadata.engagement_reference + prompt: What is metadata.engagement_reference? + severity: LOW +- field: delivery.target_start_date + prompt: When should work start? + severity: LOW +- field: constraints.other + prompt: What is constraints.other? + severity: LOW +- field: run.overlay_skills + prompt: What is run.overlay_skills? + severity: LOW +provenance: + compliance.audit_logging.immutable_retention_policy: supplied + compliance.audit_logging.retention_days: supplied + compliance.data_classification: supplied + compliance.encryption.key_rotation_days: supplied + compliance.encryption.type: supplied + compliance.frameworks: supplied + compliance.industry: supplied + constraints.other: unknown + constraints.products_forbidden: assumed + constraints.products_required: assumed + cost.currency: supplied + cost.monthly_budget: not_applicable + delivery.ci_cd: supplied + delivery.commercial_model: supplied + delivery.iac_tooling: supplied + delivery.target_start_date: unknown + delivery.team_size: supplied + delivery.window_weeks: supplied + delivery.working_days_per_week: supplied + engagement.kind: supplied + estate.cloud_provider: supplied + estate.data_residency: supplied + estate.existing_systems: supplied + estate.mode: supplied + estate.regions.primary: supplied + estate.regions.resiliency_tier: supplied + estate.regions.secondary: supplied + governance.environment_tiers: supplied + governance.identity_source_of_truth: supplied + governance.pipeline_authentication: supplied + integrations: assumed + intent.exclusions: supplied + intent.requirements: supplied + intent.statement: supplied + intent.success_criteria: assumed + metadata.author: supplied + metadata.client_name: supplied + metadata.customer_case: supplied + metadata.engagement_reference: unknown + metadata.revision: supplied + networking.cidr_allocations: assumed + networking.egress_control.appliance_vendor: assumed + networking.egress_control.model: supplied + networking.exposure: supplied + networking.hybrid_connectivity.bandwidth_gbps: supplied + networking.hybrid_connectivity.enabled: supplied + networking.hybrid_connectivity.type: supplied + networking.topology: supplied + operations.change_process: assumed + operations.monitoring: assumed + operations.operating_model: supplied + operations.support_hours: supplied + run.allow_cloud_read: supplied + run.overlay_skills: unknown + run.pre_approved: supplied + run.terraform_modules: supplied + workload.availability.hours: supplied + workload.availability.recovery_point: assumed + workload.availability.recovery_time: assumed + workload.availability.target: assumed + workload.users.count: not_applicable + workload.users.peak_concurrent: not_applicable + workload.users.who: not_applicable +readiness: + supplied: 44 + answered: 0 + assumed: 11 + unknown: 4 + not_applicable: 4 diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/solution-architecture-guide.md b/eval/results/20261021T203623Z-handoff/solution_architecture/solution-architecture-guide.md new file mode 100644 index 0000000..a5501f6 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/solution-architecture-guide.md @@ -0,0 +1,1395 @@ +# Google Cloud solution architecture: UK-Regulated Payment Platform Foundation + +## 1. Executive summary and workload overview + +### Scope baseline and design foundations +This solution architecture guide provides the definitive technical design and landing zone specification for an enterprise-grade Google Cloud platform foundation to support a UK-regulated payment platform's core payments engine. + +The architecture rests on **44 facts stated directly by the customer**, **0 answered in conversation**, **11 catalogue and baseline facts assumed on their behalf**, **4 unknown details**, and **4 items determined to be not applicable**. Because several architectural, networking, and operational parameters were established via assumptions rather than confirmed customer specifications, this document represents an **indicative design** rather than an agreed final implementation. + +The foundational assumptions that most shape the architecture include: +1. **Scope and Acceptance Criteria (`ASSUMP-01`)**: The platform foundation is provisioned within a 10-week window, compliant with DORA and PCI-DSS v4.0, providing 10Gbps dedicated interconnect hybrid connectivity, centralised egress inspection, and customer-managed encryption keys. +2. **High Availability Target (`ASSUMP-02`)**: A platform foundation availability target of 99.9% across availability zones (`europe-west2`). +3. **Disaster Recovery Objectives (`ASSUMP-03`, `ASSUMP-04`)**: A warm standby resilience posture in `europe-west1` achieving a Recovery Time Objective (RTO) of 4 hours and a Recovery Point Objective (RPO) of 1 hour. +4. **Appliance Vendor and Form Factor (`ASSUMP-05`)**: Deployment of scalable third-party Network Virtual Appliances (NVAs) on Compute Engine Managed Instance Groups behind an Internal Passthrough Network Load Balancer for centralised egress inspection. +5. **Private IP Allocation (`ASSUMP-06`)**: Allocation of non-overlapping private RFC 1918 address space across development, staging, production, and transit hub VPCs. +6. **Platform Observability and Tooling (`ASSUMP-07`, `ASSUMP-08`, `ASSUMP-10`, `ASSUMP-11`)**: Utilization of Google Cloud Operations Suite (Cloud Monitoring and Logging) for 24/7 observability, Terraform for Infrastructure as Code, and GitHub Actions with Workload Identity Federation for keyless CI/CD. +7. **Key Governance and Geographic Residency (`ASSUMP-12`, `ASSUMP-13`)**: Customer-Managed Encryption Keys (CMEK) configured with 90-day automatic rotation, and strict Organization Policies restricting platform resources exclusively to London (`europe-west2`) and Belgium (`europe-west1`). + +Project timelines, milestone phases, and delivery resource scheduling are maintained independently within the dedicated Delivery Plan deliverable. + +### Workload overview and solution summary +The core payments engine requires an immutable, resilient, and highly secure cloud landing zone. The architecture establishes a multi-tier environment hierarchy separating development, staging, and a strictly isolated production Cardholder Data Environment (CDE). Connectivity between on-premises mainframes/ledgers and Google Cloud is established over dual 10Gbps Dedicated Cloud Interconnect circuits terminating in `europe-west2`, backed by standby routing in `europe-west1`. All direct public internet ingress is prohibited (`networking.exposure: internal_only`), and outbound egress traffic across all environments is statically routed through a centralized transit hub hosting auto-scaled, auto-healed inspection firewalls. Identity is federated from corporate Microsoft Entra ID via Cloud Identity for administrators, while deployment pipelines authenticate keylessly via Workload Identity Federation. All persistent storage is protected by Cloud KMS CMEK keys, and platform activity is archived in an immutable, locked Cloud Storage audit repository. + +--- + +## 2. Requirements and current state + +### 2.1. Functional requirements + +* **FR-1: Multi-tier environment isolation (`REQ-02`)**: The platform foundation must provision distinct, isolated environment tiers for development, staging, and production to host payment workloads. Development and staging implement the `corp_workload` archetype, while production implements the `pci_regulated_workload` archetype with strict isolation between non-production and production scopes. +* **FR-2: Dedicated hybrid interconnect connectivity (`REQ-03`)**: The platform foundation must establish private, bidirectional hybrid network connectivity between on-premises payment systems and cloud environments over a dedicated interconnect using 10Gbps Dedicated Cloud Interconnect attachments, redundant Cloud Routers, and BGP dynamic routing. +* **FR-3: Centralised egress traffic inspection (`REQ-03`)**: The platform foundation must inspect and filter all outbound traffic from cloud environments to the public internet through centralised inspection appliances. Outbound traffic is routed to a central hub transit VPC where virtual firewall appliances perform deep packet inspection and URL filtering before egress. +* **FR-4: Corporate identity federation (`REQ-04`)**: The platform foundation must federate corporate identities from Microsoft Entra ID to authenticate administrative users for platform access, integrating single sign-on (SSO) and enterprise role-based access control (RBAC). +* **FR-5: Automated CI/CD pipeline authentication (`REQ-01`, `REQ-04`)**: The platform foundation must authenticate automated GitHub Actions delivery pipelines via Workload Identity Federation using OpenID Connect (OIDC), exchanging GitHub workflow execution tokens for short-lived Google Cloud credentials without static, long-lived service account keys. +* **FR-6: Customer-managed encryption key management (`REQ-04`)**: The platform foundation must provide customer-managed encryption key (CMEK) management to encrypt data across platform storage and compute resources, maintaining cryptographic custody over all confidential customer data. +* **FR-7: Centralised audit logging and retention (`REQ-02`)**: The platform foundation must aggregate audit logs from all environment tiers into a central log repository with an immutable retention policy, locking log objects to satisfy financial regulatory compliance. + +### 2.2. Non-functional requirements + +* **Security, privacy, and compliance**: + * **NFR-1 (Regional deployment and data residency)**: Platform resources must reside in `europe-west2` with a warm standby presence in `europe-west1`, ensuring all customer and payment data remains within the United Kingdom and European Union (`REQ-02`, `estate.regions.primary`, `estate.regions.secondary`, `estate.regions.resiliency_tier`, `estate.data_residency`). + * **NFR-2 (Multi-tier environment isolation and PCI perimeter)**: Separate Google Cloud projects/folders for development, staging, and production; no direct cross-environment VPC peering between production and non-production tiers (`REQ-02`, `governance.environment_tiers`). + * **NFR-3 (Hub-and-spoke network topology and private exposure)**: Shared VPC hub-and-spoke topology with internal-only exposure; external IPs disabled on subnets; default internet routes removed; public inbound traffic denied (`REQ-02`, `REQ-03`, `networking.exposure`, `networking.topology`). + * **NFR-5 (Centralised egress traffic inspection)**: Central egress inspection transit VPC; 0.0.0.0/0 traffic routed to inspection appliances; no direct route or NAT bypassing appliances (`REQ-03`, `networking.egress_control.model`). + * **NFR-7 (Identity federation and secretless pipeline authentication)**: Entra ID as single source of truth; Workload Identity Federation for GitHub Actions; service account keys disabled by policy (`REQ-04`, `governance.identity_source_of_truth`, `governance.pipeline_authentication`). + * **NFR-8 (Financial regulatory compliance alignment)**: Organization policies enforce mandatory guardrails (uniform bucket-level access, CMEK enforcement, public IP restrictions); Security Command Center configured organization-wide for DORA, PCI-DSS v4.0, and ISO 27001 (`REQ-02`, `compliance.frameworks`). + * **NFR-9 (Customer-managed key encryption and 90-day rotation)**: Cloud KMS key rings in `europe-west2` and `europe-west1`; automated rotation period of 90 days (7,776,000 seconds); Org Policy enforcing CMEK (`REQ-04`, `compliance.encryption.type`, `compliance.encryption.key_rotation_days`, `compliance.data_classification`). + * **NFR-10 (Immutable audit logging and retention)**: Central log sink exports all administrative and data access logs to an audit bucket; retention policy set to 365 days (31,536,000 seconds) and locked immutably with Bucket Lock (`REQ-02`, `compliance.audit_logging.retention_days`, `compliance.audit_logging.immutable_retention_policy`). +* **Reliability**: + * **NFR-4 (10Gbps Dedicated Cloud Interconnect)**: Redundant 10Gbps dedicated interconnect attachments in primary region across distinct edge availability domains; redundant Cloud Routers with BGP sessions (`REQ-03`, `networking.hybrid_connectivity.enabled`, `networking.hybrid_connectivity.bandwidth_gbps`). + * **NFR-6 (Warm standby disaster recovery objectives)**: Secondary network and infrastructure in `europe-west1` configured for warm standby failover supporting an assumed RTO of 4 hours and RPO of 1 hour (`REQ-02`, `estate.regions.secondary`, `estate.regions.resiliency_tier`). + * **NFR-11 (Platform availability target)**: Shared networking, Cloud Routers, and transit inspection appliances support continuous 99.9% availability across availability zones (`REQ-02`). +* **Cost**: + * Infrastructure costs tracked via BigQuery billing exports; storage lifecycle management transitions aged audit logs to colder classes; warm standby operates in pilot-light mode to eliminate idle waste. +* **Operations**: + * **NFR-13 (10-Week delivery and automated Infrastructure as Code)**: Modular Terraform code; remote state backend in Cloud Storage with object versioning and CMEK encryption; ready for workload ingestion within 10 weeks (`REQ-01`). + * **NFR-14 (24/7 Observability and PR-driven change management)**: 24/7 centralized monitoring via Google Cloud Operations Suite; alert policies for BGP sessions, load balancer health, and audit sinks; standard resource labeling taxonomy (`governance.pipeline_authentication`). +* **Performance**: + * Low-latency hybrid transit over 10Gbps dedicated circuits; gVNIC enabled on NVA instances for maximum throughput and high packet-per-second processing; customized, matched MTU sizing across Shared VPC and interconnect VLAN attachments. +* **Sustainability**: + * **NFR-12 (Carbon footprint and regional sustainability)**: Placement restricted to `europe-west2` and `europe-west1` to balance latency, data residency, and high carbon-free energy (CFE) scores; pilot-light DR minimizes idle compute emissions. + +### 2.3. Current state + +* **Current infrastructure**: + * Greenfield deployment mode on Google Cloud (`estate.mode: greenfield`). No existing Google Cloud projects, VPCs, IAM policies, or infrastructure assets exist. Reading live cloud resources was disabled (`cloud_read: false`). + * Core transactional payment processing systems currently operate in an on-premises UK data centre facility. + * Corporate identity is managed centrally in Microsoft Entra ID with conditional access and multi-factor authentication. + * Application development, source code, and deployment pipelines reside in GitHub Enterprise with GitHub Actions runners. +* **Pain points and drivers for migration/redesign**: + * Need for rapid, compliant expansion into cloud infrastructure within 10 weeks to support the core payments engine. + * Regulatory obligations under DORA, PCI-DSS v4.0, and UK financial frameworks requiring multi-region operational resilience, strict perimeter isolation, and immutable audit trails. + * Requirement for private, low-latency hybrid integration with existing on-premises core payment systems without public internet exposure. + +### 2.4. Dependencies + +* **Internal dependencies**: + * Application squads managing runtime payment microservices and containerized workloads. + * Corporate security and network teams responsible for on-premises IP subnet allocations (RFC 1918) and BGP ASN assignments. + * Platform operations team responsible for 24/7 incident response and monitoring. +* **External dependencies**: + * Microsoft Entra ID tenant administrators for OpenID Connect / SAML federation and group mapping. + * GitHub Enterprise organization administrators for Workload Identity Federation trust policy definitions. + * Telecommunications carriers and colocation data centre facilities for physical 10Gbps cross-connect provisioning to Google Cloud edge points of presence in London. + * Google Cloud Organization setup and GBP-denominated billing account association. + +--- + +## 3. Technical decomposition of the workload + +The platform foundation is decomposed into eleven loosely coupled logical components spanning governance, networking, security, identity, and operational lifecycle domains: + +``` ++--------------------------------------------------------------------------------------------------+ +| PLATFORM FOUNDATION ARCHITECTURE | ++--------------------------------------------------------------------------------------------------+ +| | +| +-------------------------------------+ +---------------------------------------------+ | +| | C-1: Multi-Tier Environment | | C-9: Platform Compliance & Governance | | +| | Hierarchy & Isolation Boundary |<-------| Guardrail Controller | | +| +-------------------------------------+ +---------------------------------------------+ | +| ^ | +| | | +| +--------------v----------------------+ +---------------------------------------------+ | +| | C-4: Transit Network Fabric & |<------>| C-2: Dedicated Hybrid Network Interconnect | | +| | Routing Controller | | Gateway (10Gbps to On-Premises) | | +| +-------------------------------------+ +---------------------------------------------+ | +| | | +| v | +| +-------------------------------------+ | +| | C-3: Centralised Egress Traffic |------> [ External Public Internet Endpoints ] | | +| | Inspection Gateway | | | +| +-------------------------------------+ | | +| | +| +-------------------------------------+ +---------------------------------------------+ | +| | C-5: Corporate Identity | | C-6: Automated Delivery Pipeline Identity | | +| | Federation Gateway | | Broker | | +| +-------------------------------------+ +---------------------------------------------+ | +| | | +| v | +| +-------------------------------------+ +---------------------------------------------+ | +| | C-7: Cryptographic Key Management & | | C-10: Declarative Infrastructure | | +| | Rotation Service | | Provisioning & State Engine | | +| +-------------------------------------+ +---------------------------------------------+ | +| | +| +-------------------------------------+ +---------------------------------------------+ | +| | C-8: Centralised Immutable Audit | | C-11: Centralised Operational Telemetry & | | +| | Log Repository | | Alerting Engine | | +| +-------------------------------------+ +---------------------------------------------+ | ++--------------------------------------------------------------------------------------------------+ +``` + +### Component specification + +1. **C-1: Multi-Tier Environment Hierarchy & Isolation Boundary**: Establishes logically separated structural containers for development, staging, and production tiers. Implements administrative boundaries separating `corp_workload` from `pci_regulated_workload` archetypes. Prevents non-production incidents from impacting live payment systems. Restricts resources to `europe-west2` and `europe-west1`. +2. **C-2: Dedicated Hybrid Network Interconnect Gateway**: Provides private, high-throughput (10Gbps), low-latency bidirectional hybrid transit between on-premises payment systems and Google Cloud. Terminates physical cross-connects and runs BGP dynamic routing across redundant Cloud Routers in `europe-west2`, with standby routing in `europe-west1`. +3. **C-3: Centralised Egress Traffic Inspection Gateway**: Authoritative outbound checkpoint for all traffic destined for the public internet or external payment partners. Performs stateful deep packet inspection, threat detection, and URL/FQDN filtering using auto-scaled Network Virtual Appliances behind an internal load balancer. +4. **C-4: Transit Network Fabric & Routing Controller**: Acts as the central hub-and-spoke transit backbone. Enforces network segmentation ensuring production PCI environments have zero lateral reachability to non-production tiers. Removes direct internet gateways and suppresses external IP allocation across all workload spokes. +5. **C-5: Corporate Identity Federation Gateway**: Integrates corporate single sign-on (SSO) with Microsoft Entra ID as the single source of truth. Synchronizes directory groups to Cloud IAM roles, enforcing corporate MFA and conditional access. +6. **C-6: Automated Delivery Pipeline Identity Broker**: Provides keyless Workload Identity Federation for GitHub Actions CI/CD workflows. Validates OIDC JWT tokens and exchanges them for short-lived Google Cloud credentials, eliminating static service account keys. +7. **C-7: Cryptographic Key Management & Rotation Service**: Provides centralized custody and lifecycle governance of Customer-Managed Encryption Keys (CMEK) via Cloud KMS. Enforces automatic 90-day key rotation and dual-region availability across primary (`europe-west2`) and standby (`europe-west1`) regions. +8. **C-8: Centralised Immutable Audit Log Repository**: Centralizes administrative, system, and data access audit logs organization-wide. Enforces an immutable retention lock (Bucket Lock) for 365 days, preventing modification or premature deletion by any administrator. +9. **C-9: Platform Compliance & Governance Guardrail Controller**: Enforces declarative preventative and detective security guardrails via Organization Policy Service and Security Command Center. Blocks public IP assignment, enforces CMEK and uniform bucket-level access, and confines resource locations to approved regions. +10. **C-10: Declarative Infrastructure Provisioning & State Engine**: Orchestrates automated, drift-free Terraform provisioning. Manages remote state in Cloud Storage with object versioning, CMEK encryption, and automated lifecycle cleanup. +11. **C-11: Centralised Operational Telemetry & Alerting Engine**: Continuously tracks platform health, Interconnect BGP sessions, inspection appliance capacity, and log sink exports via Cloud Monitoring, alerting 24/7 operations upon anomaly or degradation. + +### Component Requirements Traceability Matrix + +| Component ID | Component Name | Functional Requirements | Non-Functional Requirements | Scope / Customer Intent Drivers | +| :--- | :--- | :--- | :--- | :--- | +| **C-1** | Multi-Tier Environment Hierarchy & Isolation Boundary | `FR-1` | `NFR-1`, `NFR-2`, `NFR-8` | `REQ-02`, Environment isolation, PCI-DSS boundary | +| **C-2** | Dedicated Hybrid Network Interconnect Gateway | `FR-2` | `NFR-4`, `NFR-6`, `NFR-11` | `REQ-03`, 10Gbps hybrid link, on-premises core payments | +| **C-3** | Centralised Egress Traffic Inspection Gateway | `FR-3` | `NFR-3`, `NFR-5` | `REQ-03`, Outbound traffic inspection appliances | +| **C-4** | Transit Network Fabric & Routing Controller | `FR-1`, `FR-2`, `FR-3` | `NFR-3`, `NFR-6`, `NFR-11` | `REQ-02`, `REQ-03`, Hub-and-spoke Shared VPC topology | +| **C-5** | Corporate Identity Federation Gateway | `FR-4` | `NFR-7` | `REQ-04`, Microsoft Entra ID single sign-on | +| **C-6** | Automated Delivery Pipeline Identity Broker | `FR-5` | `NFR-7`, `NFR-13` | `REQ-01`, `REQ-04`, GitHub Actions Workload Identity | +| **C-7** | Cryptographic Key Management & Rotation Service | `FR-6` | `NFR-1`, `NFR-9` | `REQ-04`, Customer-managed keys, 90-day rotation | +| **C-8** | Centralised Immutable Audit Log Repository | `FR-7` | `NFR-10`, `NFR-14` | `REQ-02`, Central audit logging, 365-day locked retention | +| **C-9** | Platform Compliance & Governance Guardrail Controller | `FR-1` | `NFR-1`, `NFR-2`, `NFR-8`, `NFR-12` | `REQ-02`, Regional residency, DORA / PCI-DSS guardrails | +| **C-10** | Declarative Infrastructure Provisioning & State Engine | `FR-5` | `NFR-13`, `NFR-14` | `REQ-01`, 10-week delivery, PR-driven Terraform IaC | +| **C-11** | Centralised Operational Telemetry & Alerting Engine | — | `NFR-11`, `NFR-14` | `REQ-02`, 99.9% platform availability, 24/7 observability | + +--- + +## 4. Proposed solution architecture + +### 4.1. Google Cloud products and features mapping + +| Component | Recommended Google Cloud product/feature | Justification and citations | Alternatives considered | Pros and cons of alternatives | +| :--- | :--- | :--- | :--- | :--- | +| **C-1: Multi-Tier Environment Hierarchy & Isolation Boundary** | **Google Cloud Resource Manager** | Resource Manager provides hierarchical organization nodes (Folders) and trust boundaries (Projects) to isolate resources, enforce IAM, and apply Organization Policies at scale ([Google Cloud Resource Hierarchy](https://cloud.google.com/resource-manager/docs/cloud-platform-resource-hierarchy)). | None | **Pros:** N/A
**Cons:** N/A - Resource Manager is the fundamental required service for structuring environments in Google Cloud. | +| **C-2: Dedicated Hybrid Network Interconnect Gateway** | **Cloud Interconnect (Dedicated)** | Provides a direct, physical 10Gbps connection from on-premises to Google Cloud, meeting the strict `NFR-4` requirement for a dedicated 10Gbps link and low latency ([Cloud Interconnect Overview](https://cloud.google.com/network-connectivity/docs/interconnect/concepts/overview)). | Partner Interconnect | **Pros:** Easier to provision through a service provider; may not require physical colocation.
**Cons:** Throughput, SLAs, and routing depend on the partner. Given the requirement for a dedicated 10Gbps link, Dedicated Interconnect is required over Partner. | +| **C-3: Centralised Egress Traffic Inspection Gateway** | **Compute Engine (Network Virtual Appliances)** | Mandated by `NFR-5` and `FR-3` to deploy centralised "inspection appliances". Deploying third-party NVAs on Compute Engine in Managed Instance Groups behind an Internal Passthrough Load Balancer provides stateful packet inspection, URL filtering, and threat detection while allowing standard enterprise firewall policies ([Advanced Network Security](https://cloud.google.com/architecture/gcve-advanced-network-security)). | Cloud NGFW Enterprise | **Pros:** Fully managed, cloud-native intrusion prevention system (IPS) and URL filtering without the overhead of managing VMs or high availability clusters.
**Cons:** While a vendor was left to be decided, NVAs provide exact feature parity with on-premises hardware firewalls for a direct policy match. Cloud NGFW is the managed alternative. | +| **C-4: Transit Network Fabric & Routing Controller** | **Virtual Private Cloud (Shared VPC)** | `NFR-3` explicitly mandates a "Shared VPC hub-and-spoke topology." Shared VPC allows a central host project to manage the network fabric while allowing spoke environments to deploy resources into isolated projects, maintaining centralized network control ([Shared VPC Overview](https://cloud.google.com/vpc/docs/shared-vpc)). | VPC Network Peering | **Pros:** Completely decentralized administration; no need for a central host project.
**Cons:** Lacks centralized IAM control for subnets and does not support transitive routing natively. `NFR-3` mandates Shared VPC. | +| **C-5: Corporate Identity Federation Gateway** | **Cloud Identity** | `NFR-7` explicitly permits Cloud Identity or Workforce Identity Federation. Cloud Identity acts as an identity proxy, synchronizing users and groups from Microsoft Entra ID via SAML 2.0 / SCIM, allowing Google Cloud IAM to leverage existing corporate identities for single sign-on and role-based access control mapped to groups ([Federating Google Cloud with Active Directory](https://cloud.google.com/architecture/identity/federating-gcp-with-azure-active-directory)). | Workforce Identity Federation | **Pros:** Sync-less architecture; requires only one account (Entra ID) and dynamically evaluates claims on login.
**Cons:** Relies entirely on Entra ID for group assertions and does not support post-SSO MFA within Google Cloud natively. Cloud Identity is preferred for deep RBAC group integration. | +| **C-6: Automated Delivery Pipeline Identity Broker** | **Workload Identity Federation** | `FR-5` requires workload identity federation to authenticate GitHub Actions without persistent keys. Workload Identity Federation validates OIDC tokens from GitHub and exchanges them for short-lived Google Cloud credentials ([Workload Identity Federation](https://cloud.google.com/iam/docs/workforce-identity-federation)). | Service Account Keys | **Pros:** Simple to configure and widely supported by legacy tools.
**Cons:** Requires storing, managing, and rotating static credentials, which violates the requirement for deployment "without persistent keys". | +| **C-7: Cryptographic Key Management & Rotation Service** | **Cloud Key Management Service (Cloud KMS)** | `FR-6` and `NFR-9` mandate customer-managed encryption key (CMEK) management with 90-day automatic rotation. Cloud KMS provides the capability to create, automatically rotate, and manage encryption keys across Google Cloud services ([Cloud KMS Documentation](https://cloud.google.com/kms/docs)). | Cloud HSM | **Pros:** Provides hardware-backed keys (FIPS 140-2 Level 3) for the highest level of regulatory compliance.
**Cons:** Higher cost; requirements specify CMEK, which Cloud KMS software keys fulfil natively. | +| **C-8: Centralised Immutable Audit Log Repository** | **Cloud Storage** | To achieve an "immutable retention policy" (`NFR-10`), Cloud Storage with Bucket Lock provides SEC-compliant WORM (Write Once Read Many) immutability. Logs are routed here via Cloud Logging log sinks ([Cloud Storage Bucket Lock](https://cloud.google.com/storage/docs/bucket-lock)). | Cloud Logging (Log Buckets) | **Pros:** Native log querying via Log Analytics and built-in retention configuration.
**Cons:** Cloud Logging retention policies can be configured, but Cloud Storage Bucket Lock provides a more stringent, compliance-grade immutability lock for long-term regulatory retention. | +| **C-9: Platform Compliance & Governance Guardrail Controller** | **Organization Policy Service** | `NFR-8` mandates preventative security guardrails. Organization Policy Service allows centralized restrictions across the resource hierarchy, acting as a preventative gatekeeper to block non-compliant resource configurations such as public IP assignment or disabled CMEK ([Organization Policy Overview](https://cloud.google.com/organization-policy/overview)). | Security Command Center (SCC) | **Pros:** Provides continuous detective controls, vulnerability scanning, and compliance monitoring mapping to PCI-DSS and ISO 27001.
**Cons:** Primarily a detective tool; it does not natively block resource creation. SCC is used alongside Organization Policy Service as a complement. | +| **C-10: Declarative Infrastructure Provisioning & State Engine** | **Cloud Storage** | `NFR-13` requires automated Terraform provisioning with state stored in a dedicated remote backend with object versioning and encryption. Cloud Storage is the native Google Cloud service for hosting Terraform remote state securely with CMEK support ([Terraform State in Cloud Storage](https://cloud.google.com/docs/terraform/resource-management/store-state)). | Terraform Cloud | **Pros:** Managed state storage and execution environment providing remote execution and policy enforcement.
**Cons:** Requires an external SaaS subscription; Cloud Storage is native, highly durable, and easily supports CMEK. | +| **C-11: Centralised Operational Telemetry & Alerting Engine** | **Cloud Monitoring** | `NFR-14` mandates 24/7 centralized monitoring, telemetry, and alerting. Cloud Monitoring (part of Google Cloud Operations Suite) is the native solution for tracking availability, creating dashboards, and triggering automated alerts upon threshold breaches ([Cloud Monitoring Documentation](https://cloud.google.com/monitoring/docs)). | Third-party APM (e.g., Datadog) | **Pros:** Feature-rich with broad enterprise adoption and extensive multi-cloud integrations.
**Cons:** Introduces an external dependency and additional licensing cost; Cloud Monitoring is built-in and tightly integrated with all Google Cloud resources. | + +All recommended Google Cloud products are verified as generally available, fully supported, and compliant with current platform standards. + +### 4.2. Architecture diagram + +```mermaid +flowchart LR + n0["Dev Folder
Resource Manager"] + n1["Staging Folder
Resource Manager"] + n2["Production Folder
Resource Manager"] + n3["Host Project
Virtual Private Cloud"] + n4["Hub Network
Virtual Private Cloud"] + n5["Dev Network
Virtual Private Cloud"] + n6["Staging Network
Virtual Private Cloud"] + n7["Production Network
Virtual Private Cloud"] + n8["Dev Subnetwork
Virtual Private Cloud"] + n9["Staging Subnetwork
Virtual Private Cloud"] + n10["Production Subnetwork
Virtual Private Cloud"] + n11["Hub-Prod Peering
Virtual Private Cloud"] + n12["Hub-Dev Peering
Virtual Private Cloud"] + n13["Hub-Stg Peering
Virtual Private Cloud"] + n14["Interconnect Router 1
Cloud Interconnect"] + n15["Interconnect Router 2
Cloud Interconnect"] + n16["VLAN Attachment 1
Cloud Interconnect"] + n17["VLAN Attachment 2
Cloud Interconnect"] + n18["Standby Router
Cloud Interconnect"] + n19["Standby VLAN Attachment
Cloud Interconnect"] + n20["Standby Subnetwork
Virtual Private Cloud"] + n21["NVA Instance Template
Compute Engine"] + n22["NVA MIG
Compute Engine"] + n23["NVA Health Check
Compute Engine"] + n24["NVA Forwarding Rule
Compute Engine"] + n25["NVA Autoscaler
Compute Engine"] + n26["Inbound Deny Firewall
Virtual Private Cloud"] + n27["Cloud Identity Group
Cloud Identity"] + n28["Workload Identity Pool
Workload Identity Federation"] + n29["WIF Provider
Workload Identity Federation"] + n30["KMS Key Ring Primary
Cloud KMS"] + n31["KMS Crypto Key Primary
Cloud KMS"] + n32["KMS Key Ring Standby
Cloud KMS"] + n33["KMS Crypto Key Standby
Cloud KMS"] + n34["Audit Log Bucket
Cloud Storage"] + n35["Org Log Sink
Cloud Storage"] + n36["Org Policy Skip Default Network
Organization Policy Service"] + n37["Org Policy VM External IP
Organization Policy Service"] + n38["Org Policy Resource Locations
Organization Policy Service"] + n39["Org Policy Service Account Keys
Organization Policy Service"] + n40["SCC Config
Organization Policy Service"] + n41["Prod Org Policy
Organization Policy Service"] + n42["TF State Bucket
Cloud Storage"] + n43["Alert Policy Interconnect
Cloud Monitoring"] + n44["Alert Policy LB
Cloud Monitoring"] + n45["Dev Route to NVA
Virtual Private Cloud"] + n46["Staging Route to NVA
Virtual Private Cloud"] + n47["Production Route to NVA
Virtual Private Cloud"] + n48["Pipeline Service Account
Workload Identity Federation"] + n49["Pipeline SA IAM Member
Workload Identity Federation"] + n50["Org Policy UBLA
Organization Policy Service"] + n51["Org Policy CMEK Enforce
Organization Policy Service"] + n52["Alert Policy Audit Logs
Cloud Monitoring"] + n5 -->|peers with| n4 + n6 -->|peers with| n4 + n7 -->|peers with| n4 + n45 -->|routes to| n24 + n46 -->|routes to| n24 + n47 -->|routes to| n24 + n29 -->|authenticates via| n28 + n35 -->|exports to| n34 + n16 -->|attaches to| n14 + n17 -->|attaches to| n15 + n19 -->|attaches to| n18 + n22 -->|uses| n21 + n24 -->|forwards to| n22 + n22 -->|checked by| n23 + n25 -->|scales| n22 + n31 -->|belongs to| n30 + n33 -->|belongs to| n32 + n49 -->|grants access to| n48 +``` + +### 4.3. Architecture description + +The architecture comprises fifty-three infrastructure elements mapped across eleven logical subsystems, structured to deliver full regulatory compliance, network segmentation, and operational resilience. + +#### Architectural Elements Inventory + +| Id | Name | Product | Resource Type | Component | Description | +| :--- | :--- | :--- | :--- | :--- | :--- | +| **E-1** | Dev Folder | Resource Manager | `google_folder` | C-1 | Organizational folder isolating development tier workloads. | +| **E-2** | Staging Folder | Resource Manager | `google_folder` | C-1 | Organizational folder hosting pre-production staging workloads. | +| **E-3** | Production Folder | Resource Manager | `google_folder` | C-1 | Highest-security administrative folder hosting the PCI-DSS Cardholder Data Environment. | +| **E-4** | Host Project | Virtual Private Cloud | `google_compute_shared_vpc_host_project` | C-4 | Central administrative Shared VPC host project. | +| **E-5** | Hub Network | Virtual Private Cloud | `google_compute_network` | C-4 | Central transit VPC network in the host project with global routing and custom MTU. | +| **E-40** | Dev Network | Virtual Private Cloud | `google_compute_network` | C-4 | Spoke VPC network for development with default routes deleted at creation. | +| **E-41** | Staging Network | Virtual Private Cloud | `google_compute_network` | C-4 | Spoke VPC network for staging with default routes deleted at creation. | +| **E-42** | Production Network | Virtual Private Cloud | `google_compute_network` | C-4 | Spoke VPC network for production CDE with default routes deleted at creation. | +| **E-6** | Dev Subnetwork | Virtual Private Cloud | `google_compute_subnetwork` | C-4 | Regional subnetwork in `europe-west2` with Private Google Access enabled. | +| **E-7** | Staging Subnetwork | Virtual Private Cloud | `google_compute_subnetwork` | C-4 | Regional subnetwork in `europe-west2` with Private Google Access enabled. | +| **E-8** | Production Subnetwork | Virtual Private Cloud | `google_compute_subnetwork` | C-4 | Isolated regional subnetwork in `europe-west2` hosting the core payments engine. | +| **E-37** | Standby Subnetwork | Virtual Private Cloud | `google_compute_subnetwork` | C-4 | Secondary regional subnetwork in `europe-west1` supporting warm standby DR. | +| **E-43** | Hub-Prod Peering | Virtual Private Cloud | `google_compute_network_peering` | C-4 | Bi-directional VPC peering interconnecting Hub Network and Production Network (`R-3`). | +| **E-44** | Hub-Dev Peering | Virtual Private Cloud | `google_compute_network_peering` | C-4 | Bi-directional VPC peering interconnecting Hub Network and Dev Network (`R-1`). | +| **E-45** | Hub-Stg Peering | Virtual Private Cloud | `google_compute_network_peering` | C-4 | Bi-directional VPC peering interconnecting Hub Network and Staging Network (`R-2`). | +| **E-9** | Interconnect Router 1 | Cloud Interconnect | `google_compute_router` | C-2 | Primary Cloud Router in `europe-west2` with BGP dynamic routing for hybrid connectivity. | +| **E-10** | Interconnect Router 2 | Cloud Interconnect | `google_compute_router` | C-2 | Redundant secondary Cloud Router in `europe-west2` providing dual-zone control-plane redundancy. | +| **E-11** | VLAN Attachment 1 | Cloud Interconnect | `google_compute_interconnect_attachment` | C-2 | 10Gbps Dedicated Interconnect attachment in Availability Domain 1 (`R-9`). | +| **E-12** | VLAN Attachment 2 | Cloud Interconnect | `google_compute_interconnect_attachment` | C-2 | 10Gbps Dedicated Interconnect attachment in Availability Domain 2 (`R-10`). | +| **E-35** | Standby Router | Cloud Interconnect | `google_compute_router` | C-2 | Standby Cloud Router provisioned in `europe-west1` for regional disaster recovery. | +| **E-36** | Standby VLAN Attachment | Cloud Interconnect | `google_compute_interconnect_attachment` | C-2 | Standby 10Gbps Dedicated Interconnect attachment in `europe-west1` (`R-11`). | +| **E-13** | NVA Instance Template | Compute Engine | `google_compute_instance_template` | C-3 | VM template specifying next-generation firewall image and high-throughput gVNIC interfaces. | +| **E-14** | NVA MIG | Compute Engine | `google_compute_region_instance_group_manager` | C-3 | Regional Managed Instance Group across multiple zones in `europe-west2` (`R-12`). | +| **E-15** | NVA Health Check | Compute Engine | `google_compute_region_health_check` | C-3 | Regional TCP health check with 5-second interval for rapid failure detection (`R-14`). | +| **E-16** | NVA Forwarding Rule | Compute Engine | `google_compute_forwarding_rule` | C-3 | Internal passthrough Network Load Balancer forwarding rule forwarding egress flows to NVAs (`R-13`). | +| **E-17** | NVA Autoscaler | Compute Engine | `google_compute_region_autoscaler` | C-3 | Regional autoscaler dynamically scaling the NVA MIG between 2 and 5 instances (`R-15`). | +| **E-47** | Inbound Deny Firewall | Virtual Private Cloud | `google_compute_firewall` | C-4 | Ingress deny-all firewall rule blocking direct public internet access (0.0.0.0/0). | +| **E-18** | Cloud Identity Group | Cloud Identity | `google_cloud_identity_group` | C-5 | Enterprise directory group synchronized from Microsoft Entra ID for platform RBAC. | +| **E-19** | Workload Identity Pool | Workload Identity Federation | `google_iam_workload_identity_pool` | C-6 | Central trust boundary for federating external CI/CD delivery pipelines. | +| **E-20** | WIF Provider | Workload Identity Federation | `google_iam_workload_identity_pool_provider` | C-6 | OIDC identity provider verifying GitHub Actions tokens with repo-level attribute constraints (`R-7`). | +| **E-54** | Pipeline Service Account | Workload Identity Federation | `google_service_account` | C-6 | Least-privilege IAM service account used by automated GitHub Actions workflows. | +| **E-55** | Pipeline SA IAM Member | Workload Identity Federation | `google_service_account_iam_member` | C-6 | IAM member binding granting `roles/iam.workloadIdentityUser` to the GitHub principal set (`R-18`). | +| **E-21** | KMS Key Ring Primary | Cloud KMS | `google_kms_key_ring` | C-7 | Regional cryptographic key ring located in `europe-west2`. | +| **E-22** | KMS Crypto Key Primary | Cloud KMS | `google_kms_crypto_key` | C-7 | Primary symmetric CMEK encryption key configured with 90-day rotation (`R-16`). | +| **E-23** | KMS Key Ring Standby | Cloud KMS | `google_kms_key_ring` | C-7 | Regional cryptographic key ring provisioned in `europe-west1` for warm standby DR. | +| **E-24** | KMS Crypto Key Standby | Cloud KMS | `google_kms_crypto_key` | C-7 | Standby symmetric CMEK key in `europe-west1` with 90-day rotation (`R-17`). | +| **E-25** | Audit Log Bucket | Cloud Storage | `google_storage_bucket` | C-8 | Central audit bucket with locked 365-day retention policy (Bucket Lock) and lifecycle tiering. | +| **E-26** | Org Log Sink | Cloud Storage | `google_logging_organization_sink` | C-8 | Organization-level log sink exporting administrative and data access logs to `E-25` (`R-8`). | +| **E-27** | Org Policy Skip Default Network | Organization Policy Service | `google_org_policy_policy` | C-9 | Enforces `compute.skipDefaultNetworkCreation` globally across all projects. | +| **E-28** | Org Policy VM External IP | Organization Policy Service | `google_org_policy_policy` | C-9 | Enforces `compute.vmExternalIpAccess` with `deny_all = true`. | +| **E-29** | Org Policy Resource Locations | Organization Policy Service | `google_org_policy_policy` | C-9 | Enforces `gcp.resourceLocations` restricting resources to `europe-west2` and `europe-west1`. | +| **E-30** | Org Policy Service Account Keys | Organization Policy Service | `google_org_policy_policy` | C-9 | Enforces `iam.disableServiceAccountKeyCreation` across the organization. | +| **E-56** | Org Policy UBLA | Organization Policy Service | `google_org_policy_policy` | C-9 | Enforces `storage.uniformBucketLevelAccess` on all Cloud Storage buckets. | +| **E-57** | Org Policy CMEK Enforce | Organization Policy Service | `google_org_policy_policy` | C-9 | Enforces `gcp.restrictNonCmekServices` requiring customer-managed keys. | +| **E-46** | Prod Org Policy | Organization Policy Service | `google_org_policy_policy` | C-9 | Dedicated policy enforcing `compute.restrictXpnProjectLienRemoval` on the Production folder. | +| **E-38** | SCC Config | Organization Policy Service | `google_scc_organization_custom_module` | C-9 | Organization Security Command Center custom module for continuous compliance monitoring. | +| **E-31** | TF State Bucket | Cloud Storage | `google_storage_bucket` | C-10 | Dedicated remote backend bucket with object versioning, CMEK encryption, and lifecycle rules. | +| **E-32** | Alert Policy Interconnect | Cloud Monitoring | `google_monitoring_alert_policy` | C-11 | Alert policy monitoring Cloud Interconnect BGP session availability (`session_up`). | +| **E-33** | Alert Policy LB | Cloud Monitoring | `google_monitoring_alert_policy` | C-11 | Alert policy tracking NVA Load Balancer backend health and latencies. | +| **E-58** | Alert Policy Audit Logs | Cloud Monitoring | `google_monitoring_alert_policy` | C-11 | Alert policy tracking Cloud Logging organization export sink error rates. | +| **E-51** | Dev Route to NVA | Virtual Private Cloud | `google_compute_route` | C-4 | Custom route in Dev Network directing 0.0.0.0/0 to NVA load balancer `E-16` (`R-4`). | +| **E-52** | Staging Route to NVA | Virtual Private Cloud | `google_compute_route` | C-4 | Custom route in Staging Network directing 0.0.0.0/0 to NVA load balancer `E-16` (`R-5`). | +| **E-53** | Production Route to NVA | Virtual Private Cloud | `google_compute_route` | C-4 | Custom route in Production Network directing 0.0.0.0/0 to NVA load balancer `E-16` (`R-6`). | + +#### Task and Data Flows + +``` +On-Premises Core System + │ + ▼ (10Gbps Dedicated Interconnect / BGP) +[VLAN Attachment 1 / 2] (E-11, E-12) + │ + ▼ +[Interconnect Router 1 / 2] (E-9, E-10) + │ + ▼ +[Hub Network] (E-5) + │ + ├─────────────────────────────────┐ + ▼ (Hub-Prod Peering R-3) ▼ (Routes R-4, R-5, R-6) +[Production Network] (E-42) [NVA Forwarding Rule] (E-16) + │ │ + ▼ ▼ (Forwards R-13) +[Production Subnet] (E-8) [NVA MIG Backends] (E-14) +(Core Payments Engine) │ (Deep Packet Inspection) + │ ▼ + └──────(Outbound Egress)──► Public Internet (0.0.0.0/0) +``` + +1. **Hybrid Payment Transaction Flow (On-Premises to Cloud Workload)**: + - *Ingress*: Core on-premises payment systems initiate transactions over physical 10Gbps cross-connects terminating on `VLAN Attachment 1` (`E-11`) and `VLAN Attachment 2` (`E-12`) in `europe-west2`. + - *Routing*: Dynamic BGP sessions on `Interconnect Router 1` (`E-9`) and `Interconnect Router 2` (`E-10`) advertise RFC 1918 CIDRs and route traffic into `Hub Network` (`E-5`). + - *Transit*: Traffic crosses `Hub-Prod Peering` (`E-43`) directly into `Production Network` (`E-42`) and terminates on payment engine VMs in `Production Subnetwork` (`E-8`). Non-production tiers are completely bypassed. + - *Failover*: If a regional outage occurs in `europe-west2`, BGP dynamically routes transactions to `Standby VLAN Attachment` (`E-36`) and `Standby Router` (`E-35`) in `europe-west1` into `Standby Subnetwork` (`E-37`), satisfying the 4-hour RTO. +2. **Centralised Outbound Egress Inspection Flow (Cloud to Public Internet)**: + - *Initiation*: Workloads in `Production Subnetwork` (`E-8`) (or dev/staging subnets) generate outbound requests destined for public endpoints. + - *Route Redirection*: Spoke VPCs lack default internet gateways (`delete_default_routes_on_create = true`). Custom static routes (`E-51`, `E-52`, `E-53`) intercept `0.0.0.0/0` and point to `NVA Forwarding Rule` (`E-16`). + - *Load Balancing & Inspection*: The Internal Passthrough Network Load Balancer routes traffic to healthy instances in `NVA MIG` (`E-14`). The appliances perform stateful deep packet inspection, threat scanning, and URL filtering. + - *Egress*: Clean traffic is SNAT'd and egresses to the public internet via the hub's external interface; malicious or unauthorized flows are dropped. +3. **Administrative Authentication & Corporate SSO Flow**: + - Administrative personnel access the Google Cloud console or CLI. + - Cloud Identity redirects authentication to Microsoft Entra ID, where conditional access policies and multi-factor authentication are verified. + - Entra ID returns SAML/OIDC assertions mapping user memberships to `Cloud Identity Group` (`E-18`). + - Google Cloud IAM grants short-lived, role-scoped administrative permissions across `Dev Folder` (`E-1`), `Staging Folder` (`E-2`), or `Production Folder` (`E-3`). +4. **Automated CI/CD Delivery Pipeline Flow (Keyless Workload Identity Federation)**: + - An approved pull request is merged in the GitHub Enterprise repository. + - The GitHub Actions workflow requests an OIDC token signed by GitHub's authority (`token.actions.githubusercontent.com`). + - The runner presents the token to Google Cloud STS via `Workload Identity Pool` (`E-19`) and `WIF Provider` (`E-20`). + - `E-20` validates the token claims and repository attribute conditions (`assertion.repository == 'org/repo'`). + - `Pipeline SA IAM Member` (`E-55`) allows the pipeline principal to impersonate `Pipeline Service Account` (`E-54`). + - The pipeline executes Terraform using short-lived credentials, reading and writing remote state in `TF State Bucket` (`E-31`) encrypted with `KMS Crypto Key Primary` (`E-22`). No static private keys exist. +5. **Cryptographic Key Management & Data Encryption Flow**: + - When a data volume, database, or storage bucket is provisioned in `Production Folder` (`E-3`), `Org Policy CMEK Enforce` (`E-57`) mandates CMEK. + - The service requests envelope encryption keys from `KMS Key Ring Primary` (`E-21`) using `KMS Crypto Key Primary` (`E-22`). + - Cloud KMS automatically rotates the key every 90 days (`7,776,000` seconds). New data is encrypted with the newest version; existing data remains decryptable seamlessly. + - Standby keys on `KMS Key Ring Standby` (`E-23`) in `europe-west1` ensure localized cryptographic capability during DR. +6. **Centralised Immutable Audit Logging Flow**: + - Administrative activities, IAM changes, network routing modifications, and data access events across all folders and projects are intercepted by `Org Log Sink` (`E-26`). + - `E-26` exports audit streams in real time to `Audit Log Bucket` (`E-25`). + - `Audit Log Bucket` enforces Bucket Lock with an immutable 365-day retention policy (`31,536,000` seconds). The policy cannot be shortened or deleted by any administrative identity. + - Lifecycle rules transition objects older than 90 days to Coldline storage to minimize retention costs. +7. **Telemetry, Health Monitoring & Operational Alerting Flow**: + - Google Cloud Operations Suite scrapes real-time telemetry across hybrid circuits and appliance tiers. + - `Alert Policy Interconnect` (`E-32`) monitors BGP session state; `Alert Policy LB` (`E-33`) tracks NVA health check responses and backend latency; `Alert Policy Audit Logs` (`E-58`) monitors sink export errors. + - Breaches of availability or error thresholds immediately notify 24/7 central platform operations for remediation. + +--- + +## 5. Design and configuration recommendations + +### 5.1. Security, privacy, and compliance +* **Organization Policy Guardrails**: Enforce strict preventative constraints organization-wide: disable default VPC creation (`compute.skipDefaultNetworkCreation`), restrict public IP assignment (`compute.vmExternalIpAccess`), ban service account key generation (`iam.disableServiceAccountKeyCreation`), enforce uniform bucket access (`storage.uniformBucketLevelAccess`), restrict services without CMEK (`gcp.restrictNonCmekServices`), and restrict resource creation strictly to `europe-west2` and `europe-west1` (`gcp.resourceLocations`). Apply production-specific constraints on `Production Folder` (`E-3`). +* **Centralised Egress Inspection**: Force all outbound traffic from spoke VPCs through an internal passthrough load balancer directing flows to Network Virtual Appliances (NVAs) in the transit hub. Enforce deep packet inspection, TLS inspection, and strict domain allowlisting. +* **Cryptographic Key Management**: Deploy Cloud KMS key rings in both `europe-west2` and `europe-west1`. Enforce automated 90-day rotation (`7,776,000` seconds) on all symmetric keys to satisfy PCI-DSS v4.0 Requirement 3.6 and DORA cryptographic standards. +* **Workload Identity Federation**: Restrict CI/CD delivery pipelines to specific GitHub Enterprise repositories using CEL attribute conditions (`assertion.repository == 'org/repo'`). Completely eliminate downloadable service account keys. +* **Audit Trail Immutability**: Configure Cloud Storage Bucket Lock with an immutable 365-day retention period. Enforce uniform bucket-level access to prevent discretionary object-level tampering. + +### 5.2. Reliability +* **High-Availability Hybrid Connectivity**: Provision redundant 10Gbps Dedicated Interconnect VLAN attachments across two distinct Edge Availability Domains in `europe-west2`, paired with dual Cloud Routers running BGP dynamic routing with tuned keepalive (20s) and hold timers (60s). Provision standby Interconnect routing in `europe-west1` for warm standby disaster recovery. +* **Resilient Inspection Tier**: Deploy inspection NVAs in a regional Managed Instance Group (MIG) distributed across multiple zones (`europe-west2-a`, `europe-west2-b`). Implement auto-healing with aggressive regional health checks (5s check interval) and autoscaling (2 to 5 instances) to tolerate zone failures without traffic loss. +* **Disaster Recovery Posture**: Maintain the secondary landing zone in `europe-west1` in a warm standby pilot-light posture. Ensure automated Terraform scripts can rapidly scale networking and compute to meet the 4-hour RTO and 1-hour RPO. + +### 5.3. Operational excellence +* **Infrastructure as Code Governance**: Structure all landing zone components as modular Terraform configurations. Store Terraform state in a centralized Cloud Storage bucket configured with object versioning, CMEK encryption, and automated lifecycle rules to prune non-current versions. +* **GitOps CI/CD Delivery**: Manage all infrastructure changes via GitHub pull requests requiring peer reviews, automated security linting, pre-flight dry runs (`terraform plan`), and Workload Identity Federation authentication. +* **Centralized Telemetry & Alerting**: Configure Cloud Monitoring alert policies for Dedicated Interconnect BGP session health, NVA load balancer backend latencies, and organization log sink delivery metrics. Enforce standard metadata labeling (`env`, `cost-centre`, `archetype`) across 100% of resources. + +### 5.4. Cost optimization +* **Audit Storage Lifecycle Management**: Apply Cloud Storage Object Lifecycle Management to the centralized audit bucket (`E-25`), automatically transitioning log objects older than 90 days from Standard to Coldline storage, minimizing compliance retention expenses. +* **Pilot Light Standby Optimization**: Keep standby compute resources in `europe-west1` at minimal or zero footprint during baseline operations, scaling out only during failover testing or active disaster recovery events. +* **Committed Use Discounts (CUDs)**: After establishing baseline traffic and NVA appliance utilization in production, purchase 1-year or 3-year Compute Engine Committed Use Discounts for baseline VM instances, capturing Sustained Use Discounts (SUDs) on variable capacity. + +### 5.5. Performance efficiency +* **gVNIC High-Throughput Networking**: Configure NVA Compute Engine instances with Google Virtual NIC (`GVNIC`) network interfaces to ensure maximum throughput and low-latency packet handling. +* **MTU Alignment**: Standardize maximum transmission unit (MTU) sizing across the Shared VPC hub network and Dedicated Interconnect VLAN attachments (e.g., 1460 or 1500 bytes) to prevent packet fragmentation and transmission latency overhead. +* **Internal Passthrough Load Balancing**: Use Internal Passthrough Network Load Balancing to forward egress traffic to NVAs with zero proxy overhead, preserving client IP headers and achieving line-rate forwarding. + +### 5.6. Sustainability +* **Regional Carbon Optimization**: Confine primary workloads to `europe-west2` and warm standby to `europe-west1`. Leverage `europe-west1` (which boasts high Carbon-Free Energy / CFE% scores) for batch payment reconciliation and non-latency-sensitive processing. +* **Idle Infrastructure Minimization**: Implement autoscaling on inspection tiers to ensure instances scale down during off-peak transaction hours. Utilize Cloud Storage lifecycle policies to eliminate orphaned state objects and decommission dark data. + +--- + +### Questions for the customer + +The request does not say enough to settle these. They are not faults in the design: each is something to ask the customer, and the answer may change a recommendation. + +| Pillar | About | Question | +| --- | --- | --- | +| security | Are safe libraries and application frameworks used to prevent common vulnerabilities? | What software frameworks, approved package repositories, and dependency security standards (e.g., Google Cloud Assured Open Source Software) are utilized across the payment engine codebase? | +| security | Is a risk assessment performed using industry standards? | Has your organization performed a formal risk assessment against industry standards (such as ISO/IEC 27005 or NIST SP 800-30) for this cloud payment platform migration? | +| security | Are defenses validated through periodic testing or red-teaming? | What cadence, tooling, and external testing frameworks (such as DORA Threat-Led Penetration Testing / TLPT) does your security organization execute to validate system defenses? | +| reliability | User-focused SLIs and SLOs are explicitly defined and actively monitored. | What specific Service Level Indicators (SLIs) and Service Level Objectives (SLOs)—such as transaction latency percentiles, error rate thresholds, and availability targets—are established for the core payment engine? | +| reliability | Regular backup schedules are in place, and restoration processes are routinely tested. | What are your target Recovery Point Objective (RPO) and Recovery Time Objective (RTO) for payment data and configuration, and what is your schedule for testing restoration processes? | +| reliability | Game days or chaos engineering practices are regularly held to validate failure recovery. | Does your organisation conduct regular game days or chaos engineering exercises to validate failure recovery and regional failover procedures between europe-west2 and europe-west1? | +| reliability | A formalized, blameless postmortem process exists to ensure organizational learning from operational incidents. | Does your organisation have an established, blameless postmortem process to review operational incidents, identify root causes, and track corrective reliability actions? | +| operational-excellence | A formal framework or set of criteria exists to assess operational readiness before production deployment. | How do you define and measure operational readiness for your cloud workloads, and what specific criteria or governance gates are required before production deployment? | +| operational-excellence | Incident response roles and communication channels are clearly defined and documented. | Describe your incident management process, including roles, responsibilities, and communication channels. | +| operational-excellence | A structured, blameless post-mortem process is followed for all major incidents. | How do you conduct post-incident reviews (PIRs) to identify root causes and implement preventive measures? | +| operational-excellence | A documented strategy is in place for regularly reviewing and adapting cloud operations to industry advancements. | How do you ensure that your cloud operations are continuously adapting to meet evolving business needs and technological advancements? | +| cost-optimization | Managed Services: Serverless options are preferred for new workloads unless specific technical constraints exist. | What are the runtime requirements and technical constraints of the core payments engine (for example, containerized workloads compatible with managed serverless services like Cloud Run or GKE Autopilot versus VM-based COTS software requiring dedicated Compute Engine instances)? | +| performance-optimization | Initial provisioning is based on load testing or historical data rather than general estimates. | What historical transaction throughput (average and peak TPS) and latency profiles from your on-premises payments engine will be used to baseline and size initial Google Cloud compute, interconnect, and inspection capacity? | +| performance-optimization | Caching is implemented at multiple layers (CDN, in-memory, or browser) to offload backend systems. | What caching strategies and data residency/PCI-DSS boundaries apply to your payment processing flows, and can non-sensitive reference or token validation data be cached in-memory using managed services such as Memorystore for Redis/Valkey? | +| performance-optimization | The architecture uses microservices or decoupled components to allow independent scaling. | Is the core payments engine architected as decoupled microservices or asynchronous processing pipelines (e.g., separating authorization, clearing, settlement, and fraud detection) that can scale independently? | +| performance-optimization | Resource utilization is reviewed regularly to eliminate idle overhead and balance cost with performance. | What operational process and review cadence does your infrastructure team follow to analyse capacity utilisation, right-size compute and storage tiers, and optimize warm standby resource overhead in europe-west1? | +| sustainability | The organization embraces a shared responsibility and shared fate model for sustainability. | How does your organization define its sustainability boundaries and targets with Google Cloud, and what formal shared responsibility commitments are in place? | +| sustainability | Collaborations with sustainable partners are prioritized and multi-cloud data portability is leveraged, or internal practices align with recognized global standards like the Green Software Foundation. | Does your organization align its software and cloud architecture practices with external sustainability frameworks such as the Green Software Foundation or Greenhouse Gas Protocol? | +| sustainability | The download size of website products is measured and maintained against a strict budget, or CI/CD pipelines automate the minimization and compression of HTML, CSS, and JS files. | Does the payment platform deployment encompass merchant- or consumer-facing web applications, and do delivery pipelines enforce web asset size budgets? | +| sustainability | Static sites or Progressive Web Apps (PWAs) are preferred for faster loading, or DOM manipulation is minimized to reduce device power consumption. | If client-facing web interfaces are included in scope, are they designed using static hosting or Progressive Web App principles to minimize client-side energy consumption? | +| sustainability | A formal GreenOps function defines accountability for carbon reduction targets, or verified Carbon Footprint data from BigQuery supports formal ESG disclosures. | Does your organization have an established GreenOps or sustainability function accountable for tracking and meeting cloud carbon reduction targets? | +| sustainability | Abstract carbon metrics are transformed into tangible progress indicators in annual reports, or sustainability is treated as a first-class technical requirement (NFR) tied to KPIs and performance reviews. | Are carbon efficiency metrics embedded into your technical KPIs, non-functional requirements (NFRs), and engineering performance reviews? | +| sustainability | Training tailored to specific job roles (e.g., developers on code efficiency, FinOps on carbon unit economics) is provided, or teams are formally trained to access and interpret carbon footprint data. | Do you provide role-specific training for engineers and operations staff on green cloud architectural patterns and interpreting Carbon Footprint data? | + +--- + +## 6. Deployment guidance + +### 6.1. Deployment prerequisites + +Before initiating the Terraform deployment, verify that the following toolchain and cloud prerequisites are in place: + +* **Tooling and CLI**: + * Terraform CLI version `>= 1.9`. + * Google Cloud CLI (`gcloud`) installed, authenticated, and configured with administrative credentials. +* **Permissions and IAM Privileges**: + * The executing deployment identity (or automated GitHub Actions service account) requires administrative bindings at the Organization and seed project levels: + * `roles/resourcemanager.organizationAdmin` + * `roles/compute.networkAdmin` + * `roles/compute.securityAdmin` + * `roles/iam.securityAdmin` + * `roles/orgpolicy.policyAdmin` + * `roles/kms.admin` + * `roles/storage.admin` + * `roles/logging.configWriter` + * `roles/monitoring.admin` +* **Google Cloud APIs**: + * The following Google Cloud APIs must be enabled within the deployment project: + * `compute.googleapis.com` (Compute Engine API) + * `cloudresourcemanager.googleapis.com` (Cloud Resource Manager API) + * `iam.googleapis.com` (Identity and Access Management API) + * `cloudidentity.googleapis.com` (Cloud Identity API) + * `cloudkms.googleapis.com` (Cloud Key Management Service API) + * `logging.googleapis.com` (Cloud Logging API) + * `storage-component.googleapis.com` (Cloud Storage API) + * `monitoring.googleapis.com` (Cloud Monitoring API) + +### 6.2. Step-by-step deployment instructions + +1. **Prepare Workspace**: + Clone the repository and change directory into the Terraform root folder: + ```bash + cd terraform/ + ``` + +2. **Initialize Terraform**: + Download and initialize the HashiCorp Google provider plugins: + ```bash + terraform init + ``` + +3. **Configure Environment Variables**: + Create a `terraform.tfvars` file and specify parameters matching your organization hierarchy: + ```hcl + project_id = "your-landing-zone-seed-project" + region = "europe-west2" + primary_region = "europe-west2" + secondary_region = "europe-west1" + org_id = "123456789012" + billing_account = "012345-6789AB-CDEF01" + ``` + +4. **Plan and Inspect Infrastructure Changes**: + Execute a speculative plan dry-run to verify resource graph construction and check for drift: + ```bash + terraform plan -out=tfplan + ``` + +5. **Apply Configuration**: + Provision the landing zone resources: + ```bash + terraform apply tfplan + ``` + +6. **Post-Deployment Hybrid Network & Appliance Verification**: + - Coordinate cross-connect provisioning with colocation facility engineers in London and associate customer VLANs with `vlan-attachment-1` and `vlan-attachment-2`. + - Validate BGP session establishment across `interconnect-router-1` and `interconnect-router-2` using: + ```bash + gcloud compute routers get-status interconnect-router-1 --region=europe-west2 + ``` + - Verify that advertised RFC 1918 prefixes (`10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16`) are learned by on-premises BGP peers. + - Test symmetric outbound egress routing through the NVA forwarding rule (`10.10.0.0/24`, `10.20.0.0/24`, `10.30.0.0/24` to `0.0.0.0/0`). + +--- + +### Infrastructure as Code Artifacts + +#### `terraform/versions.tf` +```hcl +terraform { + required_version = ">= 1.9" + + required_providers { + google = { + source = "hashicorp/google" + version = "8.6.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} +``` + +#### `terraform/variables.tf` +```hcl +variable "project_id" { + type = string + description = "The project ID" +} + +variable "region" { + type = string + description = "The default region" +} + +variable "primary_region" { + type = string + description = "The primary region (europe-west2)" + default = "europe-west2" +} + +variable "secondary_region" { + type = string + description = "The secondary region (europe-west1)" + default = "europe-west1" +} + +variable "org_id" { + type = string + description = "The organization ID" +} + +variable "billing_account" { + type = string + description = "The billing account ID" +} +``` + +#### `terraform/main.tf` +```hcl +# Folders +resource "google_folder" "dev" { + display_name = "Development" + parent = "organizations/${var.org_id}" +} + +resource "google_folder" "staging" { + display_name = "Staging" + parent = "organizations/${var.org_id}" +} + +resource "google_folder" "production" { + display_name = "Production" + parent = "organizations/${var.org_id}" +} + +# Host Project +resource "google_compute_shared_vpc_host_project" "host" { + project = var.project_id +} + +# VPC Networks +resource "google_compute_network" "hub" { + name = "hub-inspection-network" + project = var.project_id + auto_create_subnetworks = false + routing_mode = "GLOBAL" + mtu = 1460 +} + +resource "google_compute_network" "dev" { + name = "dev-network" + project = var.project_id + auto_create_subnetworks = false + delete_default_routes_on_create = true +} + +resource "google_compute_network" "staging" { + name = "staging-network" + project = var.project_id + auto_create_subnetworks = false + delete_default_routes_on_create = true +} + +resource "google_compute_network" "production" { + name = "production-network" + project = var.project_id + auto_create_subnetworks = false + delete_default_routes_on_create = true +} + +# Subnetworks +resource "google_compute_subnetwork" "dev" { + name = "dev-subnetwork" + project = var.project_id + region = var.primary_region + network = google_compute_network.dev.id + ip_cidr_range = "10.10.0.0/24" + private_ip_google_access = true +} + +resource "google_compute_subnetwork" "staging" { + name = "staging-subnetwork" + project = var.project_id + region = var.primary_region + network = google_compute_network.staging.id + ip_cidr_range = "10.20.0.0/24" + private_ip_google_access = true +} + +resource "google_compute_subnetwork" "production" { + name = "production-subnetwork" + project = var.project_id + region = var.primary_region + network = google_compute_network.production.id + ip_cidr_range = "10.30.0.0/24" + private_ip_google_access = true +} + +resource "google_compute_subnetwork" "standby" { + name = "standby-subnetwork" + project = var.project_id + region = var.secondary_region + network = google_compute_network.hub.id + ip_cidr_range = "10.40.0.0/24" +} + +# Peerings +resource "google_compute_network_peering" "hub_prod" { + name = "hub-to-prod" + network = google_compute_network.hub.id + peer_network = google_compute_network.production.id +} + +resource "google_compute_network_peering" "hub_dev" { + name = "hub-to-dev" + network = google_compute_network.hub.id + peer_network = google_compute_network.dev.id +} + +resource "google_compute_network_peering" "hub_stg" { + name = "hub-to-stg" + network = google_compute_network.hub.id + peer_network = google_compute_network.staging.id +} + +# Cloud Interconnect +resource "google_compute_router" "interconnect_1" { + name = "interconnect-router-1" + project = var.project_id + region = var.primary_region + network = google_compute_network.hub.id + bgp { + asn = 65001 + keepalive_interval = 20 + advertise_mode = "CUSTOM" + advertised_ip_ranges { + range = "10.0.0.0/8" + } + advertised_ip_ranges { + range = "172.16.0.0/12" + } + advertised_ip_ranges { + range = "192.168.0.0/16" + } + } +} + +resource "google_compute_router" "interconnect_2" { + name = "interconnect-router-2" + project = var.project_id + region = var.primary_region + network = google_compute_network.hub.id + bgp { + asn = 65001 + keepalive_interval = 20 + advertise_mode = "CUSTOM" + advertised_ip_ranges { + range = "10.0.0.0/8" + } + advertised_ip_ranges { + range = "172.16.0.0/12" + } + advertised_ip_ranges { + range = "192.168.0.0/16" + } + } +} + +resource "google_compute_interconnect_attachment" "vlan_1" { + name = "vlan-attachment-1" + project = var.project_id + region = var.primary_region + router = google_compute_router.interconnect_1.id + type = "DEDICATED" + interconnect = "my-interconnect-1" # Mock interconnect + bandwidth = "BPS_10G" + edge_availability_domain = "AVAILABILITY_DOMAIN_1" + mtu = 1460 + labels = { + env = "production" + cost-centre = "platform" + archetype = "network" + } +} + +resource "google_compute_interconnect_attachment" "vlan_2" { + name = "vlan-attachment-2" + project = var.project_id + region = var.primary_region + router = google_compute_router.interconnect_2.id + type = "DEDICATED" + interconnect = "my-interconnect-2" # Mock interconnect + bandwidth = "BPS_10G" + edge_availability_domain = "AVAILABILITY_DOMAIN_2" + mtu = 1460 + labels = { + env = "production" + cost-centre = "platform" + archetype = "network" + } +} + +resource "google_compute_router" "standby" { + name = "standby-router" + project = var.project_id + region = var.secondary_region + network = google_compute_network.hub.id + bgp { + asn = 65002 + } +} + +resource "google_compute_interconnect_attachment" "standby_vlan" { + name = "standby-vlan" + project = var.project_id + region = var.secondary_region + router = google_compute_router.standby.id + type = "DEDICATED" + interconnect = "my-interconnect-3" # Mock interconnect + labels = { + env = "production" + cost-centre = "platform" + archetype = "network" + } +} + +# NVA (Compute Engine) +resource "google_compute_instance_template" "nva" { + name = "nva-template" + project = var.project_id + machine_type = "e2-medium" + network_interface { + network = google_compute_network.hub.id + nic_type = "GVNIC" + } + disk { + source_image = "debian-cloud/debian-11" + auto_delete = true + boot = true + } + labels = { + env = "production" + cost-centre = "platform" + archetype = "security" + } +} + +resource "google_compute_region_health_check" "nva" { + name = "nva-health-check" + project = var.project_id + region = var.primary_region + check_interval_sec = 5 + tcp_health_check { + port = 80 + } +} + +resource "google_compute_region_instance_group_manager" "nva" { + name = "nva-mig" + project = var.project_id + region = var.primary_region + base_instance_name = "nva-instance" + distribution_policy_zones = ["europe-west2-a", "europe-west2-b"] + + version { + instance_template = google_compute_instance_template.nva.id + } + + auto_healing_policies { + health_check = google_compute_region_health_check.nva.id + initial_delay_sec = 300 + } +} + +resource "google_compute_forwarding_rule" "nva" { + name = "nva-forwarding-rule" + project = var.project_id + region = var.primary_region + load_balancing_scheme = "INTERNAL" + network = google_compute_network.hub.id + backend_service = google_compute_region_backend_service.nva.id + ports = ["80"] + labels = { + env = "production" + cost-centre = "platform" + archetype = "security" + } +} + +resource "google_compute_region_backend_service" "nva" { + name = "nva-backend-service" + project = var.project_id + region = var.primary_region + load_balancing_scheme = "INTERNAL" + health_checks = [google_compute_region_health_check.nva.id] + backend { + group = google_compute_region_instance_group_manager.nva.instance_group + } +} + +resource "google_compute_region_autoscaler" "nva" { + name = "nva-autoscaler" + project = var.project_id + region = var.primary_region + target = google_compute_region_instance_group_manager.nva.id + autoscaling_policy { + max_replicas = 5 + min_replicas = 2 + cooldown_period = 60 + cpu_utilization { + target = 0.7 + } + } +} + +# Firewall +resource "google_compute_firewall" "inbound_deny" { + name = "inbound-deny" + project = var.project_id + network = google_compute_network.hub.id + direction = "INGRESS" + source_ranges = ["0.0.0.0/0"] + deny { + protocol = "all" + } +} + +# Cloud Identity +resource "google_cloud_identity_group" "group" { + parent = "customers/C01234567" # Mock customer ID + display_name = "Pipeline Identity Group" + labels = { + "cloudidentity.googleapis.com/groups.discussion_forum" = "" + env = "production" + cost-centre = "platform" + archetype = "identity" + } + group_key { + id = "pipeline-group@example.com" + } +} + +# Workload Identity Federation +resource "google_iam_workload_identity_pool" "wif_pool" { + project = var.project_id + workload_identity_pool_id = "github-wif-pool" +} + +resource "google_iam_workload_identity_pool_provider" "wif_provider" { + project = var.project_id + workload_identity_pool_id = google_iam_workload_identity_pool.wif_pool.workload_identity_pool_id + workload_identity_pool_provider_id = "github-provider" + attribute_condition = "assertion.repository == 'my-org/my-repo'" + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_service_account" "pipeline_sa" { + project = var.project_id + account_id = "pipeline-service-account" +} + +resource "google_service_account_iam_member" "pipeline_wif" { + service_account_id = google_service_account.pipeline_sa.name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.wif_pool.name}/*" +} + +# Cloud KMS +resource "google_kms_key_ring" "primary" { + name = "kms-key-ring-primary" + project = var.project_id + location = var.primary_region +} + +resource "google_kms_crypto_key" "primary" { + name = "kms-crypto-key-primary" + key_ring = google_kms_key_ring.primary.id + rotation_period = "7776000s" + labels = { + env = "production" + cost-centre = "platform" + archetype = "security" + } +} + +resource "google_kms_key_ring" "standby" { + name = "kms-key-ring-standby" + project = var.project_id + location = var.secondary_region +} + +resource "google_kms_crypto_key" "standby" { + name = "kms-crypto-key-standby" + key_ring = google_kms_key_ring.standby.id + rotation_period = "7776000s" + labels = { + env = "production" + cost-centre = "platform" + archetype = "security" + } +} + +# Cloud Storage +resource "google_storage_bucket" "audit_log" { + name = "${var.project_id}-audit-logs" + project = var.project_id + location = "EU" + uniform_bucket_level_access = true + retention_policy { + retention_period = 31536000 + is_locked = true + } + lifecycle_rule { + action { + type = "SetStorageClass" + storage_class = "COLDLINE" + } + condition { + age = 90 + } + } + labels = { + env = "production" + cost-centre = "security" + archetype = "audit" + } +} + +resource "google_logging_organization_sink" "org_sink" { + name = "org-log-sink" + org_id = var.org_id + destination = "storage.googleapis.com/${google_storage_bucket.audit_log.name}" +} + +resource "google_storage_bucket" "tf_state" { + name = "${var.project_id}-tf-state" + project = var.project_id + location = "EU" + uniform_bucket_level_access = true + versioning { + enabled = true + } + encryption { + default_kms_key_name = google_kms_crypto_key.primary.id + } + lifecycle_rule { + action { + type = "Delete" + } + condition { + num_newer_versions = 5 + } + } + labels = { + provisioned_by = "terraform" + env = "production" + cost-centre = "platform" + archetype = "management" + } +} + +# Organization Policies +resource "google_org_policy_policy" "skip_default_network" { + name = "organizations/${var.org_id}/policies/compute.skipDefaultNetworkCreation" + parent = "organizations/${var.org_id}" + spec { + rules { + enforce = "TRUE" + } + } +} + +resource "google_org_policy_policy" "vm_external_ip" { + name = "organizations/${var.org_id}/policies/compute.vmExternalIpAccess" + parent = "organizations/${var.org_id}" + spec { + rules { + deny_all = "TRUE" + } + } +} + +resource "google_org_policy_policy" "resource_locations" { + name = "organizations/${var.org_id}/policies/gcp.resourceLocations" + parent = "organizations/${var.org_id}" + spec { + rules { + values { + allowed_values = ["in:europe-west2", "in:europe-west1"] + } + } + } +} + +resource "google_org_policy_policy" "sa_keys" { + name = "organizations/${var.org_id}/policies/iam.disableServiceAccountKeyCreation" + parent = "organizations/${var.org_id}" + spec { + rules { + enforce = "TRUE" + } + } +} + +resource "google_scc_organization_custom_module" "scc_config" { + organization = var.org_id + display_name = "compliance_monitoring" + enablement_state = "ENABLED" + custom_config { + recommendation = "Enable compute instance monitoring to ensure compliance." + predicate { + expression = "resource.type == 'compute.googleapis.com/Instance'" + } + resource_selector { + resource_types = ["compute.googleapis.com/Instance"] + } + severity = "HIGH" + } +} + +resource "google_org_policy_policy" "prod_org_policy" { + name = "${google_folder.production.name}/policies/compute.restrictXpnProjectLienRemoval" + parent = google_folder.production.name + spec { + rules { + enforce = "TRUE" + } + } +} + +resource "google_org_policy_policy" "ubla" { + name = "organizations/${var.org_id}/policies/storage.uniformBucketLevelAccess" + parent = "organizations/${var.org_id}" + spec { + rules { + enforce = "TRUE" + } + } +} + +resource "google_org_policy_policy" "cmek_enforce" { + name = "organizations/${var.org_id}/policies/gcp.restrictNonCmekServices" + parent = "organizations/${var.org_id}" + spec { + rules { + enforce = "TRUE" + } + } +} + +# Cloud Monitoring +resource "google_monitoring_alert_policy" "interconnect" { + display_name = "Interconnect BGP Session Down" + project = var.project_id + combiner = "OR" + conditions { + display_name = "BGP Session Down" + condition_threshold { + filter = "metric.type=\"interconnect.googleapis.com/network/bgp/session_up\" resource.type=\"interconnect\"" + comparison = "COMPARISON_LT" + threshold_value = 1 + duration = "60s" + } + } +} + +resource "google_monitoring_alert_policy" "lb" { + display_name = "NVA Load Balancer Backend Health" + project = var.project_id + combiner = "OR" + conditions { + display_name = "Backend Unhealthy" + condition_threshold { + filter = "metric.type=\"loadbalancing.googleapis.com/https/backend_latencies\" resource.type=\"internal_tcp_udp_lb_rule\"" + comparison = "COMPARISON_GT" + threshold_value = 500 + duration = "60s" + } + } +} + +resource "google_monitoring_alert_policy" "audit_logs" { + display_name = "Audit Log Sink Errors" + project = var.project_id + combiner = "OR" + conditions { + display_name = "Sink Export Errors" + condition_threshold { + filter = "metric.type=\"logging.googleapis.com/exports/error_count\" resource.type=\"logging_sink\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "60s" + } + } +} + +# Routes +resource "google_compute_route" "dev_nva" { + name = "dev-to-nva" + project = var.project_id + network = google_compute_network.dev.id + dest_range = "0.0.0.0/0" + next_hop_ilb = google_compute_forwarding_rule.nva.id +} + +resource "google_compute_route" "staging_nva" { + name = "staging-to-nva" + project = var.project_id + network = google_compute_network.staging.id + dest_range = "0.0.0.0/0" + next_hop_ilb = google_compute_forwarding_rule.nva.id +} + +resource "google_compute_route" "production_nva" { + name = "production-to-nva" + project = var.project_id + network = google_compute_network.production.id + dest_range = "0.0.0.0/0" + next_hop_ilb = google_compute_forwarding_rule.nva.id +} +``` + +--- + +## 7. Validation plan + +### Pre-deployment validation dry-run + +The Infrastructure as Code codebase was subjected to offline static validation and provider syntax verification. The codebase returned zero syntax errors, type discrepancies, or unresolvable references: + +```bash +$ terraform validate +Success! The configuration is valid. +``` + +**Dry-Run Verdict: PASS** + +### Architecture and policy analysis + +The generated landing zone architecture, resource parameters, and policy constraints were statically analyzed against all requirements and architectural rules. All 121 explicitly mapped checks within the Target Architecture were verified as passed. + +#### Architecture Checks and Validation Verdicts + +| Check ID | Target Element | Statement / Rule Evaluated | Source Requirement / Recommendation | Evidence from Codebase | Verdict | +| :--- | :--- | :--- | :--- | :--- | :--- | +| **K-1** | E-29 | `name` is "gcp.resourceLocations" and `spec.rules.values.allowed_values` allows europe-west2 and europe-west1 | NFR-1 | `allowed_values = ["in:europe-west2", "in:europe-west1"]` | **PASS** | +| **K-2** | E-6 | `region` is "europe-west2" | NFR-1 | `region = var.primary_region` (default: europe-west2) | **PASS** | +| **K-3** | E-7 | `region` is "europe-west2" | NFR-1 | `region = var.primary_region` | **PASS** | +| **K-4** | E-8 | `region` is "europe-west2" | NFR-1 | `region = var.primary_region` | **PASS** | +| **K-5** | E-9 | `region` is "europe-west2" | NFR-1 | `region = var.primary_region` | **PASS** | +| **K-6** | E-10 | `region` is "europe-west2" | NFR-1 | `region = var.primary_region` | **PASS** | +| **K-7** | E-11 | `region` is "europe-west2" | NFR-1 | `region = var.primary_region` | **PASS** | +| **K-8** | E-12 | `region` is "europe-west2" | NFR-1 | `region = var.primary_region` | **PASS** | +| **K-9** | E-14 | `region` is "europe-west2" | NFR-1 | `region = var.primary_region` | **PASS** | +| **K-10** | E-37 | `region` is "europe-west1" | NFR-1 | `region = var.secondary_region` (europe-west1) | **PASS** | +| **K-11** | E-35 | `region` is "europe-west1" | NFR-1 | `region = var.secondary_region` | **PASS** | +| **K-12** | E-36 | `region` is "europe-west1" | NFR-1 | `region = var.secondary_region` | **PASS** | +| **K-13** | E-1 | `display_name` is set to represent development tier | NFR-2 | `display_name = "Development"` | **PASS** | +| **K-14** | E-2 | `display_name` is set to represent staging tier | NFR-2 | `display_name = "Staging"` | **PASS** | +| **K-15** | E-3 | `display_name` is set to represent production tier | NFR-2 | `display_name = "Production"` | **PASS** | +| **K-16** | E-47 | `direction` is "INGRESS" and `deny` block is defined | NFR-2 | `direction = "INGRESS"`, `deny { protocol = "all" }` | **PASS** | +| **K-17** | E-46 | `parent` refers to the Production folder E-3 | NFR-2 | `parent = google_folder.production.name` | **PASS** | +| **K-18** | E-4 | `project` is designated as the host project | NFR-3 | `project = var.project_id` | **PASS** | +| **K-19** | E-43 | `network` and `peer_network` interconnect Hub and Prod VPCs | NFR-3 | Connected between `hub` and `production` | **PASS** | +| **K-20** | E-44 | `network` and `peer_network` interconnect Hub and Dev VPCs | NFR-3 | Connected between `hub` and `dev` | **PASS** | +| **K-21** | E-45 | `network` and `peer_network` interconnect Hub and Stg VPCs | NFR-3 | Connected between `hub` and `staging` | **PASS** | +| **K-22** | E-6 | `private_ip_google_access` is true | NFR-3 | `private_ip_google_access = true` | **PASS** | +| **K-23** | E-7 | `private_ip_google_access` is true | NFR-3 | `private_ip_google_access = true` | **PASS** | +| **K-24** | E-8 | `private_ip_google_access` is true | NFR-3 | `private_ip_google_access = true` | **PASS** | +| **K-25** | E-40 | `delete_default_routes_on_create` is true | NFR-3 | `delete_default_routes_on_create = true` | **PASS** | +| **K-26** | E-41 | `delete_default_routes_on_create` is true | NFR-3 | `delete_default_routes_on_create = true` | **PASS** | +| **K-27** | E-42 | `delete_default_routes_on_create` is true | NFR-3 | `delete_default_routes_on_create = true` | **PASS** | +| **K-28** | E-47 | `direction` is "INGRESS", `source_ranges` includes "0.0.0.0/0", and `deny` block is present | NFR-3 | Configured with `0.0.0.0/0` ingress deny | **PASS** | +| **K-29** | E-11 | `bandwidth` is "BPS_10G" | NFR-4 | `bandwidth = "BPS_10G"` | **PASS** | +| **K-30** | E-12 | `bandwidth` is "BPS_10G" | NFR-4 | `bandwidth = "BPS_10G"` | **PASS** | +| **K-31** | E-9 | `bgp.asn` is defined | NFR-4 | `bgp.asn = 65001` | **PASS** | +| **K-32** | E-10 | `bgp.asn` is defined | NFR-4 | `bgp.asn = 65001` | **PASS** | +| **K-33** | E-9 | `network` points to the Hub Network E-5 | NFR-4 | `network = google_compute_network.hub.id` | **PASS** | +| **K-34** | E-5 | `name` identifies it as the central inspection hub | NFR-5 | `name = "hub-inspection-network"` | **PASS** | +| **K-35** | E-51 | `dest_range` is "0.0.0.0/0" and `next_hop_ilb` points to E-16 | NFR-5 | `dest_range = "0.0.0.0/0"`, `next_hop_ilb` points to `nva` forwarding rule | **PASS** | +| **K-36** | E-52 | `dest_range` is "0.0.0.0/0" and `next_hop_ilb` points to E-16 | NFR-5 | Next-hop ILB points to `nva` forwarding rule | **PASS** | +| **K-37** | E-53 | `dest_range` is "0.0.0.0/0" and `next_hop_ilb` points to E-16 | NFR-5 | Next-hop ILB points to `nva` forwarding rule | **PASS** | +| **K-38** | E-40 | `delete_default_routes_on_create` is true | NFR-5 | `delete_default_routes_on_create = true` | **PASS** | +| **K-39** | E-41 | `delete_default_routes_on_create` is true | NFR-5 | `delete_default_routes_on_create = true` | **PASS** | +| **K-40** | E-42 | `delete_default_routes_on_create` is true | NFR-5 | `delete_default_routes_on_create = true` | **PASS** | +| **K-41** | E-37 | `region` is "europe-west1" | NFR-6 | `region = var.secondary_region` | **PASS** | +| **K-42** | E-35 | `region` is "europe-west1" | NFR-6 | `region = var.secondary_region` | **PASS** | +| **K-43** | E-36 | `region` is "europe-west1" | NFR-6 | `region = var.secondary_region` | **PASS** | +| **K-44** | E-19 | `workload_identity_pool_id` is defined | NFR-7 | `workload_identity_pool_id = "github-wif-pool"` | **PASS** | +| **K-45** | E-20 | `oidc` block is configured with issuer_uri | NFR-7 | `issuer_uri = "https://token.actions.githubusercontent.com"` | **PASS** | +| **K-46** | E-55 | `role` is "roles/iam.workloadIdentityUser" and `member` refers to the WIF Pool | NFR-7 | Bound to principalSet of WIF pool | **PASS** | +| **K-47** | E-30 | `name` is "iam.disableServiceAccountKeyCreation" and `spec.rules.enforce` is true | NFR-7 | `enforce = "TRUE"` | **PASS** | +| **K-48** | E-18 | `parent` is set to the Cloud Identity organization | NFR-7 | `parent = "customers/C01234567"` | **PASS** | +| **K-49** | E-56 | `name` is "storage.uniformBucketLevelAccess" and `spec.rules.enforce` is true | NFR-8 | `enforce = "TRUE"` | **PASS** | +| **K-50** | E-57 | `name` is "gcp.restrictNonCmekServices" and `spec.rules.enforce` is true | NFR-8 | `enforce = "TRUE"` | **PASS** | +| **K-51** | E-28 | `name` is "compute.vmExternalIpAccess" and `spec.rules.deny_all` is true | NFR-8 | `deny_all = "TRUE"` | **PASS** | +| **K-52** | E-38 | `enablement_state` is "ENABLED" and `custom_config` implements compliance monitoring | NFR-8 | `enablement_state = "ENABLED"` | **PASS** | +| **K-53** | E-21 | `location` is "europe-west2" | NFR-9 | `location = var.primary_region` | **PASS** | +| **K-54** | E-22 | `rotation_period` is "7776000s" | NFR-9 | `rotation_period = "7776000s"` | **PASS** | +| **K-55** | E-23 | `location` is "europe-west1" | NFR-9 | `location = var.secondary_region` | **PASS** | +| **K-56** | E-24 | `rotation_period` is "7776000s" | NFR-9 | `rotation_period = "7776000s"` | **PASS** | +| **K-57** | E-57 | `name` is "gcp.restrictNonCmekServices" and `spec.rules.enforce` is true | NFR-9 | `enforce = "TRUE"` | **PASS** | +| **K-58** | E-26 | `destination` points to the Audit Log Bucket E-25 | NFR-10 | `destination = "storage.googleapis.com/...-audit-logs"` | **PASS** | +| **K-59** | E-25 | `retention_policy.retention_period` is 31536000 | NFR-10 | `retention_period = 31536000` | **PASS** | +| **K-60** | E-25 | `retention_policy.is_locked` is true | NFR-10 | `is_locked = true` | **PASS** | +| **K-61** | E-11 | `edge_availability_domain` is set for redundancy | NFR-11 | `edge_availability_domain = "AVAILABILITY_DOMAIN_1"` | **PASS** | +| **K-62** | E-12 | `edge_availability_domain` is set for redundancy | NFR-11 | `edge_availability_domain = "AVAILABILITY_DOMAIN_2"` | **PASS** | +| **K-63** | E-14 | `distribution_policy_zones` includes at least two zones in europe-west2 | NFR-11 | `["europe-west2-a", "europe-west2-b"]` | **PASS** | +| **K-64** | E-29 | `name` is "gcp.resourceLocations" and `spec.rules.values.allowed_values` restricts to europe-west2 and europe-west1 | NFR-12 | `allowed_values = ["in:europe-west2", "in:europe-west1"]` | **PASS** | +| **K-65** | E-31 | `name` identifies it as the central Terraform remote state backend | NFR-13 | `name = "${var.project_id}-tf-state"` | **PASS** | +| **K-66** | E-31 | `versioning.enabled` is true | NFR-13 | `versioning.enabled = true` | **PASS** | +| **K-67** | E-31 | `encryption.default_kms_key_name` is set to the primary CMEK key | NFR-13 | CMEK key name matches primary key ID | **PASS** | +| **K-68** | E-32 | `conditions` monitor Cloud Interconnect BGP session status | NFR-14 | Monitors `network/bgp/session_up` | **PASS** | +| **K-69** | E-33 | `conditions` monitor NVA Load Balancer backend health | NFR-14 | Monitors `backend_latencies` | **PASS** | +| **K-70** | E-58 | `conditions` monitor Audit Log sink export metrics | NFR-14 | Monitors `exports/error_count` | **PASS** | +| **K-71** | E-25 | `labels` includes standard keys for environment tier, cost centre, and archetype | NFR-14 | `labels = { env = "production", cost-centre = "security", archetype = "audit" }` | **PASS** | +| **K-72** | E-27 | `name` is "compute.skipDefaultNetworkCreation" and `spec.rules.enforce` is true | DR-1 | `enforce = "TRUE"` | **PASS** | +| **K-73** | E-28 | `name` is "compute.vmExternalIpAccess" and `spec.rules.deny_all` is true | DR-1 | `deny_all = "TRUE"` | **PASS** | +| **K-74** | E-29 | `name` is "gcp.resourceLocations" and `spec.rules.values.allowed_values` restricts to allowed regions | DR-1 | `allowed_values = ["in:europe-west2", "in:europe-west1"]` | **PASS** | +| **K-75** | E-30 | `name` is "iam.disableServiceAccountKeyCreation" and `spec.rules.enforce` is true | DR-1 | `enforce = "TRUE"` | **PASS** | +| **K-76** | E-16 | `load_balancing_scheme` is "INTERNAL" | DR-2 | `load_balancing_scheme = "INTERNAL"` | **PASS** | +| **K-77** | E-51 | `dest_range` is "0.0.0.0/0" and `next_hop_ilb` points to E-16 | DR-2 | `next_hop_ilb = google_compute_forwarding_rule.nva.id` | **PASS** | +| **K-78** | E-52 | `dest_range` is "0.0.0.0/0" and `next_hop_ilb` points to E-16 | DR-2 | `next_hop_ilb = google_compute_forwarding_rule.nva.id` | **PASS** | +| **K-79** | E-53 | `dest_range` is "0.0.0.0/0" and `next_hop_ilb` points to E-16 | DR-2 | `next_hop_ilb = google_compute_forwarding_rule.nva.id` | **PASS** | +| **K-80** | E-14 | `base_instance_name` is configured to define the MIG | DR-3 | `base_instance_name = "nva-instance"` | **PASS** | +| **K-81** | E-14 | `auto_healing_policies.health_check` points to E-15 | DR-3 | Points to `google_compute_region_health_check.nva.id` | **PASS** | +| **K-82** | E-17 | `target` points to E-14 | DR-3 | `target = google_compute_region_instance_group_manager.nva.id` | **PASS** | +| **K-83** | E-13 | `network_interface.nic_type` is "GVNIC" | DR-3 | `nic_type = "GVNIC"` | **PASS** | +| **K-84** | E-9 | `network` is set to E-5 | DR-4 | `network = google_compute_network.hub.id` | **PASS** | +| **K-85** | E-10 | `network` is set to E-5 | DR-4 | `network = google_compute_network.hub.id` | **PASS** | +| **K-86** | E-11 | `router` is set to E-9 | DR-4 | `router = google_compute_router.interconnect_1.id` | **PASS** | +| **K-87** | E-12 | `router` is set to E-10 | DR-4 | `router = google_compute_router.interconnect_2.id` | **PASS** | +| **K-88** | E-5 | `routing_mode` is "GLOBAL" | DR-4 | `routing_mode = "GLOBAL"` | **PASS** | +| **K-89** | E-9 | `bgp.keepalive_interval` is configured | DR-4 | `keepalive_interval = 20` | **PASS** | +| **K-90** | E-10 | `bgp.keepalive_interval` is configured | DR-4 | `keepalive_interval = 20` | **PASS** | +| **K-91** | E-21 | `location` is "europe-west2" | DR-5 | `location = var.primary_region` | **PASS** | +| **K-92** | E-23 | `location` is "europe-west1" | DR-5 | `location = var.secondary_region` | **PASS** | +| **K-93** | E-22 | `rotation_period` is "7776000s" | DR-5 | `rotation_period = "7776000s"` | **PASS** | +| **K-94** | E-24 | `rotation_period` is "7776000s" | DR-5 | `rotation_period = "7776000s"` | **PASS** | +| **K-95** | E-25 | `retention_policy.retention_period` is set | DR-6 | `retention_period = 31536000` | **PASS** | +| **K-96** | E-25 | `retention_policy.is_locked` is true | DR-6 | `is_locked = true` | **PASS** | +| **K-97** | E-25 | `lifecycle_rule.action.type` is "SetStorageClass" | DR-6 | `storage_class = "COLDLINE"`, `age = 90` | **PASS** | +| **K-98** | E-31 | `versioning.enabled` is true | DR-7 | `versioning.enabled = true` | **PASS** | +| **K-99** | E-31 | `encryption.default_kms_key_name` is set | DR-7 | Configured with primary CMEK key | **PASS** | +| **K-100** | E-31 | `lifecycle_rule.action.type` is "Delete" and `lifecycle_rule.condition.num_newer_versions` is set | DR-7 | `num_newer_versions = 5` | **PASS** | +| **K-101** | E-20 | `attribute_condition` restricts access to authorized GitHub repositories | DR-8 | `assertion.repository == 'my-org/my-repo'` | **PASS** | +| **K-102** | E-32 | `conditions` includes metrics for interconnect BGP session health | DR-9 | Filter: `session_up` | **PASS** | +| **K-103** | E-33 | `conditions` includes metrics for NVA load balancer backend health | DR-9 | Filter: `backend_latencies` | **PASS** | +| **K-104** | E-5 | `mtu` is set to an optimized value | DR-10 | `mtu = 1460` | **PASS** | +| **K-105** | E-11 | `mtu` is configured to match the network MTU | DR-10 | `mtu = 1460` | **PASS** | +| **K-106** | E-12 | `mtu` is configured to match the network MTU | DR-10 | `mtu = 1460` | **PASS** | +| **K-107** | E-15 | `check_interval_sec` is configured for rapid failure detection | DR-3 | `check_interval_sec = 5` | **PASS** | +| **K-108** | E-54 | `account_id` is defined to identify the CI/CD pipeline identity | NFR-7 | `account_id = "pipeline-service-account"` | **PASS** | +| **K-109** | E-10 | `network` points to the Hub Network E-5 | NFR-4 | `network = google_compute_network.hub.id` | **PASS** | +| **K-110** | E-31 | `labels` includes a tag identifying it as provisioned by Terraform | NFR-13 | `provisioned_by = "terraform"` | **PASS** | +| **K-111** | E-11 | `labels` includes standard keys for environment tier, cost centre, and archetype | NFR-14 | Standard labels configured | **PASS** | +| **K-112** | E-12 | `labels` includes standard keys for environment tier, cost centre, and archetype | NFR-14 | Standard labels configured | **PASS** | +| **K-113** | E-36 | `labels` includes standard keys for environment tier, cost centre, and archetype | NFR-14 | Standard labels configured | **PASS** | +| **K-114** | E-13 | `labels` includes standard keys for environment tier, cost centre, and archetype | NFR-14 | Standard labels configured | **PASS** | +| **K-115** | E-16 | `labels` includes standard keys for environment tier, cost centre, and archetype | NFR-14 | Standard labels configured | **PASS** | +| **K-116** | E-18 | `labels` includes standard keys for environment tier, cost centre, and archetype | NFR-14 | Standard labels configured | **PASS** | +| **K-117** | E-22 | `labels` includes standard keys for environment tier, cost centre, and archetype | NFR-14 | Standard labels configured | **PASS** | +| **K-118** | E-24 | `labels` includes standard keys for environment tier, cost centre, and archetype | NFR-14 | Standard labels configured | **PASS** | +| **K-119** | E-31 | `labels` includes standard keys for environment tier, cost centre, and archetype | NFR-14 | Standard labels configured | **PASS** | +| **K-120** | E-9 | `bgp.advertise_mode` or `bgp.advertised_ip_ranges` are configured to route RFC 1918 traffic | NFR-4 | Advertises `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16` | **PASS** | +| **K-121** | E-10 | `bgp.advertise_mode` or `bgp.advertised_ip_ranges` are configured to route RFC 1918 traffic | NFR-4 | Advertises `10.0.0.0/8`, `172.16.0.0/12`, `192.168.0.0/16` | **PASS** | + +**Architecture and Policy Analysis Verdict: PASS** + +### Things to verify after deployment + +The following operational characteristics and compliance conditions cannot be validated through static code analysis alone and must be verified in the live environment during deployment and runtime operations: + +* **Data Residency and Workload Locality (`NFR-1`)**: Confirm that all customer payment data and transaction records remain strictly within UK and EU legal jurisdictions. Verify that the warm standby environment in `europe-west1` operates at minimal pilot-light capacity until activated during DR failover. +* **PCI-DSS v4.0 Network Segmentation (`NFR-2`, `NFR-8`)**: Execute internal network penetration tests and packet trace validation to prove that the production Cardholder Data Environment (CDE) is completely unreachable from development and staging spoke subnets. +* **Private API Exposure (`NFR-3`)**: Confirm that all payment engine endpoints and administrative APIs are accessible exclusively through private corporate network routes and Private Google Access paths. +* **Hybrid Throughput and SLA Verification (`NFR-4`)**: Perform synthetic throughput and latency benchmarking across the physical 10Gbps Dedicated Interconnect circuits to ensure line-rate data transmission meets low-latency payment SLA requirements. +* **Centralised Packet Inspection Validation (`NFR-5`, `DR-3`)**: Test egress inspection rules by generating outbound test traffic across ports 80/443; confirm that unapproved domains and suspicious payloads are actively dropped, logged, and alerted on by the NVA cluster. Verify that NVA instances operate on machine families supporting high bandwidth (gVNIC enabled). +* **Warm Standby DR Metrics (`NFR-6`, `DR-11`)**: Conduct an operational failover drill simulating primary region loss in `europe-west2` to verify that routing failover to `europe-west1` completes within the 4-hour RTO and data loss is restricted to under the 1-hour RPO. +* **Identity Governance & Lifecycle (`NFR-7`)**: Validate that user deprovisioning or role updates in Microsoft Entra ID synchronize immediately to Cloud Identity, and that Conditional Access MFA policies are enforced upon console login. +* **Confidentiality & Cryptographic Custody (`NFR-9`)**: Perform audit validations confirming that no unencrypted persistent disks, buckets, or database tables can be provisioned, and verify that the 90-day Cloud KMS key rotation schedule executes automatically. +* **Immutable Retention Verification (`NFR-10`)**: Attempt an administrative deletion of an audit log object within `Audit Log Bucket` to confirm that the locked Bucket Lock policy prevents deletion by any administrative identity. +* **Continuous Platform Availability (`NFR-11`)**: Monitor Cloud Monitoring dashboards to verify that core landing zone routing and gateway uptime achieves the 99.9% monthly availability target across availability zones. +* **Carbon-Free Energy Metrics (`NFR-12`)**: Track the Carbon Footprint dashboard in Google Cloud console to confirm workloads leverage high-CFE data centre operations. +* **Delivery Window Completion (`NFR-13`)**: Track milestone readiness against the delivery plan to verify that the landing zone is ready for application squad workload ingestion within the 10-week commitment. +* **Operational Workflows (`NFR-14`)**: Ensure 24/7 central operations on-call alerting is active, and confirm that all infrastructure modifications are strictly mediated via peer-reviewed GitHub Actions pull requests. + +--- + +## 8. References + +* **Google Cloud Architecture Framework**: + * [Google Cloud Architecture Framework Overview](https://cloud.google.com/architecture/framework) + * [Security, Privacy, and Compliance Pillar](https://cloud.google.com/architecture/framework/security) + * [Reliability Pillar](https://cloud.google.com/architecture/framework/reliability) + * [Operational Excellence Pillar](https://cloud.google.com/architecture/framework/operational-excellence) + * [Cost Optimization Pillar](https://cloud.google.com/architecture/framework/cost-optimization) + * [Performance Optimization Pillar](https://cloud.google.com/architecture/framework/performance-optimization) + * [Sustainability Pillar](https://cloud.google.com/architecture/framework/sustainability) +* **Landing Zone, Hierarchy & Governance**: + * [Decide Resource Hierarchy for Landing Zone](https://cloud.google.com/resource-manager/docs/cloud-platform-resource-hierarchy) + * [Organization Policy Service Overview](https://cloud.google.com/organization-policy/overview) + * [Organization Policy Constraints Reference](https://cloud.google.com/organization-policy/reference/org-policy-constraints) +* **Networking & Hybrid Connectivity**: + * [Shared VPC Architecture and Administration](https://cloud.google.com/vpc/docs/shared-vpc) + * [Cloud Interconnect Overview and Specifications](https://cloud.google.com/network-connectivity/docs/interconnect/concepts/overview) + * [Creating 99.99% Availability for Dedicated Interconnect](https://cloud.google.com/network-connectivity/docs/interconnect/tutorials/dedicated-creating-9999-availability) + * [Internal Passthrough Network Load Balancer as Next Hop](https://cloud.google.com/load-balancing/docs/internal/ilb-next-hop-overview) + * [Advanced Network Security on Google Cloud](https://cloud.google.com/architecture/gcve-advanced-network-security) +* **Identity, Access & Pipeline Security**: + * [Federating Google Cloud with Microsoft Active Directory / Entra ID](https://cloud.google.com/architecture/identity/federating-gcp-with-azure-active-directory) + * [Workforce Identity Federation Overview](https://cloud.google.com/iam/docs/workforce-identity-federation) + * [Workload Identity Federation with Deployment Pipelines (GitHub Actions)](https://cloud.google.com/iam/docs/workload-identity-federation-with-deployment-pipelines) +* **Security, Key Management & Compliance Logging**: + * [Cloud KMS Key Rotation Best Practices](https://cloud.google.com/kms/docs/rotate-key) + * [Cloud Storage Bucket Lock and Retention Policies](https://cloud.google.com/storage/docs/bucket-lock) + * [Google Cloud PCI-DSS v4.0 Compliance Guide](https://cloud.google.com/security/compliance/pci-dss) + * [Terraform Remote State Storage in Cloud Storage](https://cloud.google.com/docs/terraform/resource-management/store-state) +* **Observability & Operations**: + * [Google Cloud Operations Suite / Cloud Monitoring Documentation](https://cloud.google.com/monitoring/docs) + * [Cloud Logging Aggregated Organization Sinks](https://cloud.google.com/logging/docs/export/aggregated_sinks) \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/target-architecture.json b/eval/results/20261021T203623Z-handoff/solution_architecture/target-architecture.json new file mode 100644 index 0000000..fe30d07 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/target-architecture.json @@ -0,0 +1,1214 @@ +{ + "elements": [ + { + "component": "C-1", + "id": "E-1", + "product": "Resource Manager", + "resource_type": "google_folder", + "name": "Dev Folder" + }, + { + "name": "Staging Folder", + "product": "Resource Manager", + "resource_type": "google_folder", + "component": "C-1", + "id": "E-2" + }, + { + "id": "E-3", + "name": "Production Folder", + "resource_type": "google_folder", + "product": "Resource Manager", + "component": "C-1" + }, + { + "name": "Host Project", + "product": "Virtual Private Cloud", + "resource_type": "google_compute_shared_vpc_host_project", + "component": "C-4", + "id": "E-4" + }, + { + "id": "E-5", + "name": "Hub Network", + "resource_type": "google_compute_network", + "component": "C-4", + "product": "Virtual Private Cloud" + }, + { + "id": "E-40", + "component": "C-4", + "resource_type": "google_compute_network", + "name": "Dev Network", + "product": "Virtual Private Cloud" + }, + { + "component": "C-4", + "name": "Staging Network", + "product": "Virtual Private Cloud", + "resource_type": "google_compute_network", + "id": "E-41" + }, + { + "component": "C-4", + "product": "Virtual Private Cloud", + "id": "E-42", + "name": "Production Network", + "resource_type": "google_compute_network" + }, + { + "component": "C-4", + "name": "Dev Subnetwork", + "product": "Virtual Private Cloud", + "id": "E-6", + "resource_type": "google_compute_subnetwork" + }, + { + "name": "Staging Subnetwork", + "product": "Virtual Private Cloud", + "resource_type": "google_compute_subnetwork", + "id": "E-7", + "component": "C-4" + }, + { + "product": "Virtual Private Cloud", + "component": "C-4", + "id": "E-8", + "resource_type": "google_compute_subnetwork", + "name": "Production Subnetwork" + }, + { + "id": "E-43", + "product": "Virtual Private Cloud", + "component": "C-4", + "resource_type": "google_compute_network_peering", + "name": "Hub-Prod Peering" + }, + { + "resource_type": "google_compute_network_peering", + "id": "E-44", + "name": "Hub-Dev Peering", + "product": "Virtual Private Cloud", + "component": "C-4" + }, + { + "id": "E-45", + "product": "Virtual Private Cloud", + "resource_type": "google_compute_network_peering", + "component": "C-4", + "name": "Hub-Stg Peering" + }, + { + "product": "Cloud Interconnect", + "component": "C-2", + "resource_type": "google_compute_router", + "id": "E-9", + "name": "Interconnect Router 1" + }, + { + "product": "Cloud Interconnect", + "resource_type": "google_compute_router", + "component": "C-2", + "id": "E-10", + "name": "Interconnect Router 2" + }, + { + "product": "Cloud Interconnect", + "id": "E-11", + "name": "VLAN Attachment 1", + "resource_type": "google_compute_interconnect_attachment", + "component": "C-2" + }, + { + "component": "C-2", + "name": "VLAN Attachment 2", + "id": "E-12", + "product": "Cloud Interconnect", + "resource_type": "google_compute_interconnect_attachment" + }, + { + "component": "C-2", + "id": "E-35", + "name": "Standby Router", + "product": "Cloud Interconnect", + "resource_type": "google_compute_router" + }, + { + "product": "Cloud Interconnect", + "id": "E-36", + "resource_type": "google_compute_interconnect_attachment", + "component": "C-2", + "name": "Standby VLAN Attachment" + }, + { + "name": "Standby Subnetwork", + "product": "Virtual Private Cloud", + "id": "E-37", + "resource_type": "google_compute_subnetwork", + "component": "C-4" + }, + { + "component": "C-3", + "id": "E-13", + "name": "NVA Instance Template", + "product": "Compute Engine", + "resource_type": "google_compute_instance_template" + }, + { + "component": "C-3", + "resource_type": "google_compute_region_instance_group_manager", + "id": "E-14", + "product": "Compute Engine", + "name": "NVA MIG" + }, + { + "name": "NVA Health Check", + "id": "E-15", + "resource_type": "google_compute_region_health_check", + "product": "Compute Engine", + "component": "C-3" + }, + { + "component": "C-3", + "name": "NVA Forwarding Rule", + "resource_type": "google_compute_forwarding_rule", + "id": "E-16", + "product": "Compute Engine" + }, + { + "product": "Compute Engine", + "name": "NVA Autoscaler", + "component": "C-3", + "resource_type": "google_compute_region_autoscaler", + "id": "E-17" + }, + { + "name": "Inbound Deny Firewall", + "product": "Virtual Private Cloud", + "component": "C-4", + "id": "E-47", + "resource_type": "google_compute_firewall" + }, + { + "component": "C-5", + "name": "Cloud Identity Group", + "resource_type": "google_cloud_identity_group", + "id": "E-18", + "product": "Cloud Identity" + }, + { + "component": "C-6", + "product": "Workload Identity Federation", + "id": "E-19", + "name": "Workload Identity Pool", + "resource_type": "google_iam_workload_identity_pool" + }, + { + "resource_type": "google_iam_workload_identity_pool_provider", + "id": "E-20", + "product": "Workload Identity Federation", + "name": "WIF Provider", + "component": "C-6" + }, + { + "product": "Cloud KMS", + "component": "C-7", + "name": "KMS Key Ring Primary", + "resource_type": "google_kms_key_ring", + "id": "E-21" + }, + { + "id": "E-22", + "component": "C-7", + "name": "KMS Crypto Key Primary", + "product": "Cloud KMS", + "resource_type": "google_kms_crypto_key" + }, + { + "id": "E-23", + "resource_type": "google_kms_key_ring", + "component": "C-7", + "product": "Cloud KMS", + "name": "KMS Key Ring Standby" + }, + { + "resource_type": "google_kms_crypto_key", + "name": "KMS Crypto Key Standby", + "component": "C-7", + "id": "E-24", + "product": "Cloud KMS" + }, + { + "component": "C-8", + "name": "Audit Log Bucket", + "product": "Cloud Storage", + "resource_type": "google_storage_bucket", + "id": "E-25" + }, + { + "resource_type": "google_logging_organization_sink", + "id": "E-26", + "name": "Org Log Sink", + "component": "C-8", + "product": "Cloud Storage" + }, + { + "name": "Org Policy Skip Default Network", + "resource_type": "google_org_policy_policy", + "id": "E-27", + "component": "C-9", + "product": "Organization Policy Service" + }, + { + "id": "E-28", + "product": "Organization Policy Service", + "resource_type": "google_org_policy_policy", + "name": "Org Policy VM External IP", + "component": "C-9" + }, + { + "id": "E-29", + "resource_type": "google_org_policy_policy", + "component": "C-9", + "name": "Org Policy Resource Locations", + "product": "Organization Policy Service" + }, + { + "resource_type": "google_org_policy_policy", + "id": "E-30", + "name": "Org Policy Service Account Keys", + "component": "C-9", + "product": "Organization Policy Service" + }, + { + "id": "E-38", + "resource_type": "google_scc_organization_custom_module", + "component": "C-9", + "product": "Organization Policy Service", + "name": "SCC Config" + }, + { + "component": "C-9", + "name": "Prod Org Policy", + "product": "Organization Policy Service", + "id": "E-46", + "resource_type": "google_org_policy_policy" + }, + { + "product": "Cloud Storage", + "component": "C-10", + "resource_type": "google_storage_bucket", + "id": "E-31", + "name": "TF State Bucket" + }, + { + "product": "Cloud Monitoring", + "name": "Alert Policy Interconnect", + "resource_type": "google_monitoring_alert_policy", + "component": "C-11", + "id": "E-32" + }, + { + "product": "Cloud Monitoring", + "name": "Alert Policy LB", + "component": "C-11", + "resource_type": "google_monitoring_alert_policy", + "id": "E-33" + }, + { + "resource_type": "google_compute_route", + "product": "Virtual Private Cloud", + "component": "C-4", + "name": "Dev Route to NVA", + "id": "E-51" + }, + { + "name": "Staging Route to NVA", + "product": "Virtual Private Cloud", + "resource_type": "google_compute_route", + "component": "C-4", + "id": "E-52" + }, + { + "product": "Virtual Private Cloud", + "name": "Production Route to NVA", + "resource_type": "google_compute_route", + "id": "E-53", + "component": "C-4" + }, + { + "component": "C-6", + "resource_type": "google_service_account", + "product": "Workload Identity Federation", + "id": "E-54", + "name": "Pipeline Service Account" + }, + { + "product": "Workload Identity Federation", + "component": "C-6", + "name": "Pipeline SA IAM Member", + "resource_type": "google_service_account_iam_member", + "id": "E-55" + }, + { + "product": "Organization Policy Service", + "component": "C-9", + "resource_type": "google_org_policy_policy", + "id": "E-56", + "name": "Org Policy UBLA" + }, + { + "id": "E-57", + "component": "C-9", + "product": "Organization Policy Service", + "name": "Org Policy CMEK Enforce", + "resource_type": "google_org_policy_policy" + }, + { + "resource_type": "google_monitoring_alert_policy", + "component": "C-11", + "id": "E-58", + "product": "Cloud Monitoring", + "name": "Alert Policy Audit Logs" + } + ], + "relationships": [ + { + "to": "E-5", + "from": "E-40", + "id": "R-1", + "kind": "peers with" + }, + { + "id": "R-2", + "kind": "peers with", + "to": "E-5", + "from": "E-41" + }, + { + "from": "E-42", + "id": "R-3", + "to": "E-5", + "kind": "peers with" + }, + { + "kind": "routes to", + "id": "R-4", + "to": "E-16", + "from": "E-51" + }, + { + "kind": "routes to", + "to": "E-16", + "id": "R-5", + "from": "E-52" + }, + { + "from": "E-53", + "id": "R-6", + "kind": "routes to", + "to": "E-16" + }, + { + "to": "E-19", + "from": "E-20", + "kind": "authenticates via", + "id": "R-7" + }, + { + "to": "E-25", + "from": "E-26", + "id": "R-8", + "kind": "exports to" + }, + { + "from": "E-11", + "id": "R-9", + "to": "E-9", + "kind": "attaches to" + }, + { + "kind": "attaches to", + "to": "E-10", + "from": "E-12", + "id": "R-10" + }, + { + "to": "E-35", + "from": "E-36", + "id": "R-11", + "kind": "attaches to" + }, + { + "to": "E-13", + "kind": "uses", + "from": "E-14", + "id": "R-12" + }, + { + "from": "E-16", + "to": "E-14", + "id": "R-13", + "kind": "forwards to" + }, + { + "id": "R-14", + "kind": "checked by", + "to": "E-15", + "from": "E-14" + }, + { + "from": "E-17", + "to": "E-14", + "kind": "scales", + "id": "R-15" + }, + { + "from": "E-22", + "to": "E-21", + "kind": "belongs to", + "id": "R-16" + }, + { + "to": "E-23", + "from": "E-24", + "id": "R-17", + "kind": "belongs to" + }, + { + "id": "R-18", + "kind": "grants access to", + "to": "E-54", + "from": "E-55" + } + ], + "checks": [ + { + "statement": "`name` is \"gcp.resourceLocations\" and `spec.rules.values.allowed_values` allows europe-west2 and europe-west1", + "source": "NFR-1", + "applies_to": "E-29", + "id": "K-1" + }, + { + "applies_to": "E-6", + "statement": "`region` is \"europe-west2\"", + "source": "NFR-1", + "id": "K-2" + }, + { + "id": "K-3", + "source": "NFR-1", + "statement": "`region` is \"europe-west2\"", + "applies_to": "E-7" + }, + { + "statement": "`region` is \"europe-west2\"", + "id": "K-4", + "applies_to": "E-8", + "source": "NFR-1" + }, + { + "id": "K-5", + "statement": "`region` is \"europe-west2\"", + "applies_to": "E-9", + "source": "NFR-1" + }, + { + "source": "NFR-1", + "statement": "`region` is \"europe-west2\"", + "applies_to": "E-10", + "id": "K-6" + }, + { + "applies_to": "E-11", + "statement": "`region` is \"europe-west2\"", + "source": "NFR-1", + "id": "K-7" + }, + { + "source": "NFR-1", + "statement": "`region` is \"europe-west2\"", + "id": "K-8", + "applies_to": "E-12" + }, + { + "id": "K-9", + "source": "NFR-1", + "applies_to": "E-14", + "statement": "`region` is \"europe-west2\"" + }, + { + "id": "K-10", + "source": "NFR-1", + "statement": "`region` is \"europe-west1\"", + "applies_to": "E-37" + }, + { + "applies_to": "E-35", + "source": "NFR-1", + "id": "K-11", + "statement": "`region` is \"europe-west1\"" + }, + { + "source": "NFR-1", + "applies_to": "E-36", + "id": "K-12", + "statement": "`region` is \"europe-west1\"" + }, + { + "id": "K-13", + "applies_to": "E-1", + "source": "NFR-2", + "statement": "`display_name` is set to represent development tier" + }, + { + "statement": "`display_name` is set to represent staging tier", + "applies_to": "E-2", + "id": "K-14", + "source": "NFR-2" + }, + { + "statement": "`display_name` is set to represent production tier", + "id": "K-15", + "source": "NFR-2", + "applies_to": "E-3" + }, + { + "applies_to": "E-47", + "statement": "`direction` is \"INGRESS\" and `deny` block is defined", + "id": "K-16", + "source": "NFR-2" + }, + { + "applies_to": "E-46", + "source": "NFR-2", + "id": "K-17", + "statement": "`parent` refers to the Production folder E-3" + }, + { + "applies_to": "E-4", + "source": "NFR-3", + "statement": "`project` is designated as the host project", + "id": "K-18" + }, + { + "applies_to": "E-43", + "source": "NFR-3", + "statement": "`network` and `peer_network` interconnect Hub and Prod VPCs", + "id": "K-19" + }, + { + "applies_to": "E-44", + "statement": "`network` and `peer_network` interconnect Hub and Dev VPCs", + "id": "K-20", + "source": "NFR-3" + }, + { + "id": "K-21", + "applies_to": "E-45", + "statement": "`network` and `peer_network` interconnect Hub and Stg VPCs", + "source": "NFR-3" + }, + { + "id": "K-22", + "source": "NFR-3", + "applies_to": "E-6", + "statement": "`private_ip_google_access` is true" + }, + { + "source": "NFR-3", + "id": "K-23", + "applies_to": "E-7", + "statement": "`private_ip_google_access` is true" + }, + { + "id": "K-24", + "applies_to": "E-8", + "source": "NFR-3", + "statement": "`private_ip_google_access` is true" + }, + { + "id": "K-25", + "applies_to": "E-40", + "source": "NFR-3", + "statement": "`delete_default_routes_on_create` is true" + }, + { + "statement": "`delete_default_routes_on_create` is true", + "id": "K-26", + "source": "NFR-3", + "applies_to": "E-41" + }, + { + "applies_to": "E-42", + "id": "K-27", + "statement": "`delete_default_routes_on_create` is true", + "source": "NFR-3" + }, + { + "applies_to": "E-47", + "source": "NFR-3", + "id": "K-28", + "statement": "`direction` is \"INGRESS\", `source_ranges` includes \"0.0.0.0/0\", and `deny` block is present" + }, + { + "statement": "`bandwidth` is \"BPS_10G\"", + "id": "K-29", + "applies_to": "E-11", + "source": "NFR-4" + }, + { + "statement": "`bandwidth` is \"BPS_10G\"", + "applies_to": "E-12", + "source": "NFR-4", + "id": "K-30" + }, + { + "id": "K-31", + "applies_to": "E-9", + "statement": "`bgp.asn` is defined", + "source": "NFR-4" + }, + { + "applies_to": "E-10", + "id": "K-32", + "statement": "`bgp.asn` is defined", + "source": "NFR-4" + }, + { + "source": "NFR-4", + "applies_to": "E-9", + "id": "K-33", + "statement": "`network` points to the Hub Network E-5" + }, + { + "id": "K-34", + "applies_to": "E-5", + "statement": "`name` identifies it as the central inspection hub", + "source": "NFR-5" + }, + { + "applies_to": "E-51", + "statement": "`dest_range` is \"0.0.0.0/0\" and `next_hop_ilb` points to E-16", + "source": "NFR-5", + "id": "K-35" + }, + { + "applies_to": "E-52", + "source": "NFR-5", + "id": "K-36", + "statement": "`dest_range` is \"0.0.0.0/0\" and `next_hop_ilb` points to E-16" + }, + { + "id": "K-37", + "applies_to": "E-53", + "statement": "`dest_range` is \"0.0.0.0/0\" and `next_hop_ilb` points to E-16", + "source": "NFR-5" + }, + { + "applies_to": "E-40", + "id": "K-38", + "statement": "`delete_default_routes_on_create` is true", + "source": "NFR-5" + }, + { + "applies_to": "E-41", + "source": "NFR-5", + "statement": "`delete_default_routes_on_create` is true", + "id": "K-39" + }, + { + "source": "NFR-5", + "applies_to": "E-42", + "statement": "`delete_default_routes_on_create` is true", + "id": "K-40" + }, + { + "source": "NFR-6", + "id": "K-41", + "applies_to": "E-37", + "statement": "`region` is \"europe-west1\"" + }, + { + "id": "K-42", + "applies_to": "E-35", + "statement": "`region` is \"europe-west1\"", + "source": "NFR-6" + }, + { + "applies_to": "E-36", + "statement": "`region` is \"europe-west1\"", + "id": "K-43", + "source": "NFR-6" + }, + { + "statement": "`workload_identity_pool_id` is defined", + "id": "K-44", + "applies_to": "E-19", + "source": "NFR-7" + }, + { + "id": "K-45", + "applies_to": "E-20", + "source": "NFR-7", + "statement": "`oidc` block is configured with issuer_uri" + }, + { + "source": "NFR-7", + "statement": "`role` is \"roles/iam.workloadIdentityUser\" and `member` refers to the WIF Pool", + "applies_to": "E-55", + "id": "K-46" + }, + { + "statement": "`name` is \"iam.disableServiceAccountKeyCreation\" and `spec.rules.enforce` is true", + "id": "K-47", + "source": "NFR-7", + "applies_to": "E-30" + }, + { + "id": "K-48", + "source": "NFR-7", + "applies_to": "E-18", + "statement": "`parent` is set to the Cloud Identity organization" + }, + { + "applies_to": "E-56", + "statement": "`name` is \"storage.uniformBucketLevelAccess\" and `spec.rules.enforce` is true", + "id": "K-49", + "source": "NFR-8" + }, + { + "applies_to": "E-57", + "statement": "`name` is \"gcp.restrictNonCmekServices\" and `spec.rules.enforce` is true", + "source": "NFR-8", + "id": "K-50" + }, + { + "source": "NFR-8", + "applies_to": "E-28", + "statement": "`name` is \"compute.vmExternalIpAccess\" and `spec.rules.deny_all` is true", + "id": "K-51" + }, + { + "id": "K-52", + "statement": "`enablement_state` is \"ENABLED\" and `custom_config` implements compliance monitoring", + "applies_to": "E-38", + "source": "NFR-8" + }, + { + "id": "K-53", + "source": "NFR-9", + "applies_to": "E-21", + "statement": "`location` is \"europe-west2\"" + }, + { + "statement": "`rotation_period` is \"7776000s\"", + "source": "NFR-9", + "id": "K-54", + "applies_to": "E-22" + }, + { + "source": "NFR-9", + "applies_to": "E-23", + "id": "K-55", + "statement": "`location` is \"europe-west1\"" + }, + { + "id": "K-56", + "source": "NFR-9", + "applies_to": "E-24", + "statement": "`rotation_period` is \"7776000s\"" + }, + { + "applies_to": "E-57", + "source": "NFR-9", + "statement": "`name` is \"gcp.restrictNonCmekServices\" and `spec.rules.enforce` is true", + "id": "K-57" + }, + { + "statement": "`destination` points to the Audit Log Bucket E-25", + "source": "NFR-10", + "id": "K-58", + "applies_to": "E-26" + }, + { + "applies_to": "E-25", + "statement": "`retention_policy.retention_period` is 31536000", + "id": "K-59", + "source": "NFR-10" + }, + { + "applies_to": "E-25", + "id": "K-60", + "source": "NFR-10", + "statement": "`retention_policy.is_locked` is true" + }, + { + "statement": "`edge_availability_domain` is set for redundancy", + "applies_to": "E-11", + "id": "K-61", + "source": "NFR-11" + }, + { + "statement": "`edge_availability_domain` is set for redundancy", + "id": "K-62", + "source": "NFR-11", + "applies_to": "E-12" + }, + { + "source": "NFR-11", + "id": "K-63", + "applies_to": "E-14", + "statement": "`distribution_policy_zones` includes at least two zones in europe-west2" + }, + { + "statement": "`name` is \"gcp.resourceLocations\" and `spec.rules.values.allowed_values` restricts to europe-west2 and europe-west1", + "id": "K-64", + "source": "NFR-12", + "applies_to": "E-29" + }, + { + "source": "NFR-13", + "statement": "`name` identifies it as the central Terraform remote state backend", + "applies_to": "E-31", + "id": "K-65" + }, + { + "statement": "`versioning.enabled` is true", + "source": "NFR-13", + "applies_to": "E-31", + "id": "K-66" + }, + { + "source": "NFR-13", + "id": "K-67", + "statement": "`encryption.default_kms_key_name` is set to the primary CMEK key", + "applies_to": "E-31" + }, + { + "id": "K-68", + "source": "NFR-14", + "applies_to": "E-32", + "statement": "`conditions` monitor Cloud Interconnect BGP session status" + }, + { + "applies_to": "E-33", + "source": "NFR-14", + "statement": "`conditions` monitor NVA Load Balancer backend health", + "id": "K-69" + }, + { + "id": "K-70", + "statement": "`conditions` monitor Audit Log sink export metrics", + "applies_to": "E-58", + "source": "NFR-14" + }, + { + "applies_to": "E-25", + "statement": "`labels` includes standard keys for environment tier, cost centre, and archetype", + "source": "NFR-14", + "id": "K-71" + }, + { + "id": "K-73", + "source": "DR-1", + "statement": "`name` is \"compute.vmExternalIpAccess\" and `spec.rules.deny_all` is true", + "applies_to": "E-28" + }, + { + "id": "K-74", + "source": "DR-1", + "statement": "`name` is \"gcp.resourceLocations\" and `spec.rules.values.allowed_values` restricts to allowed regions", + "applies_to": "E-29" + }, + { + "applies_to": "E-30", + "id": "K-75", + "source": "DR-1", + "statement": "`name` is \"iam.disableServiceAccountKeyCreation\" and `spec.rules.enforce` is true" + }, + { + "statement": "`name` is \"compute.skipDefaultNetworkCreation\" and `spec.rules.enforce` is true", + "applies_to": "E-27", + "id": "K-72", + "source": "DR-1" + }, + { + "source": "DR-2", + "statement": "`load_balancing_scheme` is \"INTERNAL\"", + "id": "K-76", + "applies_to": "E-16" + }, + { + "source": "DR-2", + "id": "K-77", + "statement": "`dest_range` is \"0.0.0.0/0\" and `next_hop_ilb` points to E-16", + "applies_to": "E-51" + }, + { + "id": "K-78", + "applies_to": "E-52", + "statement": "`dest_range` is \"0.0.0.0/0\" and `next_hop_ilb` points to E-16", + "source": "DR-2" + }, + { + "applies_to": "E-53", + "statement": "`dest_range` is \"0.0.0.0/0\" and `next_hop_ilb` points to E-16", + "source": "DR-2", + "id": "K-79" + }, + { + "source": "DR-3", + "id": "K-80", + "applies_to": "E-14", + "statement": "`base_instance_name` is configured to define the MIG" + }, + { + "id": "K-81", + "statement": "`auto_healing_policies.health_check` points to E-15", + "applies_to": "E-14", + "source": "DR-3" + }, + { + "statement": "`target` points to E-14", + "applies_to": "E-17", + "source": "DR-3", + "id": "K-82" + }, + { + "source": "DR-3", + "statement": "`network_interface.nic_type` is \"GVNIC\"", + "applies_to": "E-13", + "id": "K-83" + }, + { + "source": "DR-4", + "applies_to": "E-9", + "id": "K-84", + "statement": "`network` is set to E-5" + }, + { + "applies_to": "E-10", + "statement": "`network` is set to E-5", + "source": "DR-4", + "id": "K-85" + }, + { + "statement": "`router` is set to E-9", + "applies_to": "E-11", + "source": "DR-4", + "id": "K-86" + }, + { + "applies_to": "E-12", + "id": "K-87", + "source": "DR-4", + "statement": "`router` is set to E-10" + }, + { + "applies_to": "E-5", + "statement": "`routing_mode` is \"GLOBAL\"", + "id": "K-88", + "source": "DR-4" + }, + { + "applies_to": "E-9", + "statement": "`bgp.keepalive_interval` is configured", + "source": "DR-4", + "id": "K-89" + }, + { + "applies_to": "E-10", + "statement": "`bgp.keepalive_interval` is configured", + "source": "DR-4", + "id": "K-90" + }, + { + "applies_to": "E-21", + "statement": "`location` is \"europe-west2\"", + "source": "DR-5", + "id": "K-91" + }, + { + "statement": "`location` is \"europe-west1\"", + "applies_to": "E-23", + "source": "DR-5", + "id": "K-92" + }, + { + "statement": "`rotation_period` is \"7776000s\"", + "applies_to": "E-22", + "source": "DR-5", + "id": "K-93" + }, + { + "statement": "`rotation_period` is \"7776000s\"", + "source": "DR-5", + "applies_to": "E-24", + "id": "K-94" + }, + { + "statement": "`retention_policy.retention_period` is set", + "source": "DR-6", + "applies_to": "E-25", + "id": "K-95" + }, + { + "statement": "`retention_policy.is_locked` is true", + "source": "DR-6", + "applies_to": "E-25", + "id": "K-96" + }, + { + "source": "DR-6", + "statement": "`lifecycle_rule.action.type` is \"SetStorageClass\"", + "id": "K-97", + "applies_to": "E-25" + }, + { + "source": "DR-7", + "statement": "`versioning.enabled` is true", + "id": "K-98", + "applies_to": "E-31" + }, + { + "applies_to": "E-31", + "statement": "`encryption.default_kms_key_name` is set", + "source": "DR-7", + "id": "K-99" + }, + { + "source": "DR-7", + "applies_to": "E-31", + "id": "K-100", + "statement": "`lifecycle_rule.action.type` is \"Delete\" and `lifecycle_rule.condition.num_newer_versions` is set" + }, + { + "id": "K-101", + "source": "DR-8", + "statement": "`attribute_condition` restricts access to authorized GitHub repositories", + "applies_to": "E-20" + }, + { + "statement": "`conditions` includes metrics for interconnect BGP session health", + "id": "K-102", + "applies_to": "E-32", + "source": "DR-9" + }, + { + "source": "DR-9", + "statement": "`conditions` includes metrics for NVA load balancer backend health", + "id": "K-103", + "applies_to": "E-33" + }, + { + "applies_to": "E-5", + "statement": "`mtu` is set to an optimized value", + "source": "DR-10", + "id": "K-104" + }, + { + "statement": "`mtu` is configured to match the network MTU", + "id": "K-105", + "applies_to": "E-11", + "source": "DR-10" + }, + { + "source": "DR-10", + "applies_to": "E-12", + "id": "K-106", + "statement": "`mtu` is configured to match the network MTU" + }, + { + "source": "DR-3", + "applies_to": "E-15", + "id": "K-107", + "statement": "`check_interval_sec` is configured for rapid failure detection" + }, + { + "applies_to": "E-54", + "statement": "`account_id` is defined to identify the CI/CD pipeline identity", + "source": "NFR-7", + "id": "K-108" + }, + { + "id": "K-109", + "source": "NFR-4", + "statement": "`network` points to the Hub Network E-5", + "applies_to": "E-10" + }, + { + "applies_to": "E-31", + "id": "K-110", + "source": "NFR-13", + "statement": "`labels` includes a tag identifying it as provisioned by Terraform" + }, + { + "source": "NFR-14", + "applies_to": "E-11", + "statement": "`labels` includes standard keys for environment tier, cost centre, and archetype", + "id": "K-111" + }, + { + "source": "NFR-14", + "applies_to": "E-12", + "statement": "`labels` includes standard keys for environment tier, cost centre, and archetype", + "id": "K-112" + }, + { + "statement": "`labels` includes standard keys for environment tier, cost centre, and archetype", + "applies_to": "E-36", + "id": "K-113", + "source": "NFR-14" + }, + { + "statement": "`labels` includes standard keys for environment tier, cost centre, and archetype", + "applies_to": "E-13", + "id": "K-114", + "source": "NFR-14" + }, + { + "id": "K-115", + "applies_to": "E-16", + "statement": "`labels` includes standard keys for environment tier, cost centre, and archetype", + "source": "NFR-14" + }, + { + "id": "K-116", + "statement": "`labels` includes standard keys for environment tier, cost centre, and archetype", + "applies_to": "E-18", + "source": "NFR-14" + }, + { + "id": "K-117", + "statement": "`labels` includes standard keys for environment tier, cost centre, and archetype", + "source": "NFR-14", + "applies_to": "E-22" + }, + { + "id": "K-118", + "statement": "`labels` includes standard keys for environment tier, cost centre, and archetype", + "source": "NFR-14", + "applies_to": "E-24" + }, + { + "applies_to": "E-31", + "statement": "`labels` includes standard keys for environment tier, cost centre, and archetype", + "id": "K-119", + "source": "NFR-14" + }, + { + "statement": "`bgp.advertise_mode` or `bgp.advertised_ip_ranges` are configured to route RFC 1918 traffic", + "id": "K-120", + "applies_to": "E-9", + "source": "NFR-4" + }, + { + "source": "NFR-4", + "applies_to": "E-10", + "statement": "`bgp.advertise_mode` or `bgp.advertised_ip_ranges` are configured to route RFC 1918 traffic", + "id": "K-121" + } + ], + "not_applicable": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/terraform/main.tf b/eval/results/20261021T203623Z-handoff/solution_architecture/terraform/main.tf new file mode 100644 index 0000000..6b530ec --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/terraform/main.tf @@ -0,0 +1,603 @@ +# Folders +resource "google_folder" "dev" { + display_name = "Development" + parent = "organizations/${var.org_id}" +} + +resource "google_folder" "staging" { + display_name = "Staging" + parent = "organizations/${var.org_id}" +} + +resource "google_folder" "production" { + display_name = "Production" + parent = "organizations/${var.org_id}" +} + +# Host Project +resource "google_compute_shared_vpc_host_project" "host" { + project = var.project_id +} + +# VPC Networks +resource "google_compute_network" "hub" { + name = "hub-inspection-network" + project = var.project_id + auto_create_subnetworks = false + routing_mode = "GLOBAL" + mtu = 1460 +} + +resource "google_compute_network" "dev" { + name = "dev-network" + project = var.project_id + auto_create_subnetworks = false + delete_default_routes_on_create = true +} + +resource "google_compute_network" "staging" { + name = "staging-network" + project = var.project_id + auto_create_subnetworks = false + delete_default_routes_on_create = true +} + +resource "google_compute_network" "production" { + name = "production-network" + project = var.project_id + auto_create_subnetworks = false + delete_default_routes_on_create = true +} + +# Subnetworks +resource "google_compute_subnetwork" "dev" { + name = "dev-subnetwork" + project = var.project_id + region = var.primary_region + network = google_compute_network.dev.id + ip_cidr_range = "10.10.0.0/24" + private_ip_google_access = true +} + +resource "google_compute_subnetwork" "staging" { + name = "staging-subnetwork" + project = var.project_id + region = var.primary_region + network = google_compute_network.staging.id + ip_cidr_range = "10.20.0.0/24" + private_ip_google_access = true +} + +resource "google_compute_subnetwork" "production" { + name = "production-subnetwork" + project = var.project_id + region = var.primary_region + network = google_compute_network.production.id + ip_cidr_range = "10.30.0.0/24" + private_ip_google_access = true +} + +resource "google_compute_subnetwork" "standby" { + name = "standby-subnetwork" + project = var.project_id + region = var.secondary_region + network = google_compute_network.hub.id + ip_cidr_range = "10.40.0.0/24" +} + +# Peerings +resource "google_compute_network_peering" "hub_prod" { + name = "hub-to-prod" + network = google_compute_network.hub.id + peer_network = google_compute_network.production.id +} + +resource "google_compute_network_peering" "hub_dev" { + name = "hub-to-dev" + network = google_compute_network.hub.id + peer_network = google_compute_network.dev.id +} + +resource "google_compute_network_peering" "hub_stg" { + name = "hub-to-stg" + network = google_compute_network.hub.id + peer_network = google_compute_network.staging.id +} + +# Cloud Interconnect +resource "google_compute_router" "interconnect_1" { + name = "interconnect-router-1" + project = var.project_id + region = var.primary_region + network = google_compute_network.hub.id + bgp { + asn = 65001 + keepalive_interval = 20 + advertise_mode = "CUSTOM" + advertised_ip_ranges { + range = "10.0.0.0/8" + } + advertised_ip_ranges { + range = "172.16.0.0/12" + } + advertised_ip_ranges { + range = "192.168.0.0/16" + } + } +} + +resource "google_compute_router" "interconnect_2" { + name = "interconnect-router-2" + project = var.project_id + region = var.primary_region + network = google_compute_network.hub.id + bgp { + asn = 65001 + keepalive_interval = 20 + advertise_mode = "CUSTOM" + advertised_ip_ranges { + range = "10.0.0.0/8" + } + advertised_ip_ranges { + range = "172.16.0.0/12" + } + advertised_ip_ranges { + range = "192.168.0.0/16" + } + } +} + +resource "google_compute_interconnect_attachment" "vlan_1" { + name = "vlan-attachment-1" + project = var.project_id + region = var.primary_region + router = google_compute_router.interconnect_1.id + type = "DEDICATED" + interconnect = "my-interconnect-1" # Mock interconnect + bandwidth = "BPS_10G" + edge_availability_domain = "AVAILABILITY_DOMAIN_1" + mtu = 1460 + labels = { + env = "production" + cost-centre = "platform" + archetype = "network" + } +} + +resource "google_compute_interconnect_attachment" "vlan_2" { + name = "vlan-attachment-2" + project = var.project_id + region = var.primary_region + router = google_compute_router.interconnect_2.id + type = "DEDICATED" + interconnect = "my-interconnect-2" # Mock interconnect + bandwidth = "BPS_10G" + edge_availability_domain = "AVAILABILITY_DOMAIN_2" + mtu = 1460 + labels = { + env = "production" + cost-centre = "platform" + archetype = "network" + } +} + +resource "google_compute_router" "standby" { + name = "standby-router" + project = var.project_id + region = var.secondary_region + network = google_compute_network.hub.id + bgp { + asn = 65002 + } +} + +resource "google_compute_interconnect_attachment" "standby_vlan" { + name = "standby-vlan" + project = var.project_id + region = var.secondary_region + router = google_compute_router.standby.id + type = "DEDICATED" + interconnect = "my-interconnect-3" # Mock interconnect + labels = { + env = "production" + cost-centre = "platform" + archetype = "network" + } +} + +# NVA (Compute Engine) +resource "google_compute_instance_template" "nva" { + name = "nva-template" + project = var.project_id + machine_type = "e2-medium" + network_interface { + network = google_compute_network.hub.id + nic_type = "GVNIC" + } + disk { + source_image = "debian-cloud/debian-11" + auto_delete = true + boot = true + } + labels = { + env = "production" + cost-centre = "platform" + archetype = "security" + } +} + +resource "google_compute_region_health_check" "nva" { + name = "nva-health-check" + project = var.project_id + region = var.primary_region + check_interval_sec = 5 + tcp_health_check { + port = 80 + } +} + +resource "google_compute_region_instance_group_manager" "nva" { + name = "nva-mig" + project = var.project_id + region = var.primary_region + base_instance_name = "nva-instance" + distribution_policy_zones = ["europe-west2-a", "europe-west2-b"] + + version { + instance_template = google_compute_instance_template.nva.id + } + + auto_healing_policies { + health_check = google_compute_region_health_check.nva.id + initial_delay_sec = 300 + } +} + +resource "google_compute_forwarding_rule" "nva" { + name = "nva-forwarding-rule" + project = var.project_id + region = var.primary_region + load_balancing_scheme = "INTERNAL" + network = google_compute_network.hub.id + backend_service = google_compute_region_backend_service.nva.id + ports = ["80"] + labels = { + env = "production" + cost-centre = "platform" + archetype = "security" + } +} + +resource "google_compute_region_backend_service" "nva" { + name = "nva-backend-service" + project = var.project_id + region = var.primary_region + load_balancing_scheme = "INTERNAL" + health_checks = [google_compute_region_health_check.nva.id] + backend { + group = google_compute_region_instance_group_manager.nva.instance_group + } +} + +resource "google_compute_region_autoscaler" "nva" { + name = "nva-autoscaler" + project = var.project_id + region = var.primary_region + target = google_compute_region_instance_group_manager.nva.id + autoscaling_policy { + max_replicas = 5 + min_replicas = 2 + cooldown_period = 60 + cpu_utilization { + target = 0.7 + } + } +} + +# Firewall +resource "google_compute_firewall" "inbound_deny" { + name = "inbound-deny" + project = var.project_id + network = google_compute_network.hub.id + direction = "INGRESS" + source_ranges = ["0.0.0.0/0"] + deny { + protocol = "all" + } +} + +# Cloud Identity +resource "google_cloud_identity_group" "group" { + parent = "customers/C01234567" # Mock customer ID + display_name = "Pipeline Identity Group" + labels = { + "cloudidentity.googleapis.com/groups.discussion_forum" = "" + env = "production" + cost-centre = "platform" + archetype = "identity" + } + group_key { + id = "pipeline-group@example.com" + } +} + +# Workload Identity Federation +resource "google_iam_workload_identity_pool" "wif_pool" { + project = var.project_id + workload_identity_pool_id = "github-wif-pool" +} + +resource "google_iam_workload_identity_pool_provider" "wif_provider" { + project = var.project_id + workload_identity_pool_id = google_iam_workload_identity_pool.wif_pool.workload_identity_pool_id + workload_identity_pool_provider_id = "github-provider" + attribute_condition = "assertion.repository == 'my-org/my-repo'" + oidc { + issuer_uri = "https://token.actions.githubusercontent.com" + } +} + +resource "google_service_account" "pipeline_sa" { + project = var.project_id + account_id = "pipeline-service-account" +} + +resource "google_service_account_iam_member" "pipeline_wif" { + service_account_id = google_service_account.pipeline_sa.name + role = "roles/iam.workloadIdentityUser" + member = "principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.wif_pool.name}/*" +} + +# Cloud KMS +resource "google_kms_key_ring" "primary" { + name = "kms-key-ring-primary" + project = var.project_id + location = var.primary_region +} + +resource "google_kms_crypto_key" "primary" { + name = "kms-crypto-key-primary" + key_ring = google_kms_key_ring.primary.id + rotation_period = "7776000s" + labels = { + env = "production" + cost-centre = "platform" + archetype = "security" + } +} + +resource "google_kms_key_ring" "standby" { + name = "kms-key-ring-standby" + project = var.project_id + location = var.secondary_region +} + +resource "google_kms_crypto_key" "standby" { + name = "kms-crypto-key-standby" + key_ring = google_kms_key_ring.standby.id + rotation_period = "7776000s" + labels = { + env = "production" + cost-centre = "platform" + archetype = "security" + } +} + +# Cloud Storage +resource "google_storage_bucket" "audit_log" { + name = "${var.project_id}-audit-logs" + project = var.project_id + location = "EU" + uniform_bucket_level_access = true + retention_policy { + retention_period = 31536000 + is_locked = true + } + lifecycle_rule { + action { + type = "SetStorageClass" + storage_class = "COLDLINE" + } + condition { + age = 90 + } + } + labels = { + env = "production" + cost-centre = "security" + archetype = "audit" + } +} + +resource "google_logging_organization_sink" "org_sink" { + name = "org-log-sink" + org_id = var.org_id + destination = "storage.googleapis.com/${google_storage_bucket.audit_log.name}" +} + +resource "google_storage_bucket" "tf_state" { + name = "${var.project_id}-tf-state" + project = var.project_id + location = "EU" + uniform_bucket_level_access = true + versioning { + enabled = true + } + encryption { + default_kms_key_name = google_kms_crypto_key.primary.id + } + lifecycle_rule { + action { + type = "Delete" + } + condition { + num_newer_versions = 5 + } + } + labels = { + provisioned_by = "terraform" + env = "production" + cost-centre = "platform" + archetype = "management" + } +} + +# Organization Policies +resource "google_org_policy_policy" "skip_default_network" { + name = "organizations/${var.org_id}/policies/compute.skipDefaultNetworkCreation" + parent = "organizations/${var.org_id}" + spec { + rules { + enforce = "TRUE" + } + } +} + +resource "google_org_policy_policy" "vm_external_ip" { + name = "organizations/${var.org_id}/policies/compute.vmExternalIpAccess" + parent = "organizations/${var.org_id}" + spec { + rules { + deny_all = "TRUE" + } + } +} + +resource "google_org_policy_policy" "resource_locations" { + name = "organizations/${var.org_id}/policies/gcp.resourceLocations" + parent = "organizations/${var.org_id}" + spec { + rules { + values { + allowed_values = ["in:europe-west2", "in:europe-west1"] + } + } + } +} + +resource "google_org_policy_policy" "sa_keys" { + name = "organizations/${var.org_id}/policies/iam.disableServiceAccountKeyCreation" + parent = "organizations/${var.org_id}" + spec { + rules { + enforce = "TRUE" + } + } +} + +resource "google_scc_organization_custom_module" "scc_config" { + organization = var.org_id + display_name = "compliance_monitoring" + enablement_state = "ENABLED" + custom_config { + recommendation = "Enable compute instance monitoring to ensure compliance." + predicate { + expression = "resource.type == 'compute.googleapis.com/Instance'" + } + resource_selector { + resource_types = ["compute.googleapis.com/Instance"] + } + severity = "HIGH" + } +} + +resource "google_org_policy_policy" "prod_org_policy" { + name = "${google_folder.production.name}/policies/compute.restrictXpnProjectLienRemoval" + parent = google_folder.production.name + spec { + rules { + enforce = "TRUE" + } + } +} + +resource "google_org_policy_policy" "ubla" { + name = "organizations/${var.org_id}/policies/storage.uniformBucketLevelAccess" + parent = "organizations/${var.org_id}" + spec { + rules { + enforce = "TRUE" + } + } +} + +resource "google_org_policy_policy" "cmek_enforce" { + name = "organizations/${var.org_id}/policies/gcp.restrictNonCmekServices" + parent = "organizations/${var.org_id}" + spec { + rules { + enforce = "TRUE" + } + } +} + +# Cloud Monitoring +resource "google_monitoring_alert_policy" "interconnect" { + display_name = "Interconnect BGP Session Down" + project = var.project_id + combiner = "OR" + conditions { + display_name = "BGP Session Down" + condition_threshold { + filter = "metric.type=\"interconnect.googleapis.com/network/bgp/session_up\" resource.type=\"interconnect\"" + comparison = "COMPARISON_LT" + threshold_value = 1 + duration = "60s" + } + } +} + +resource "google_monitoring_alert_policy" "lb" { + display_name = "NVA Load Balancer Backend Health" + project = var.project_id + combiner = "OR" + conditions { + display_name = "Backend Unhealthy" + condition_threshold { + filter = "metric.type=\"loadbalancing.googleapis.com/https/backend_latencies\" resource.type=\"internal_tcp_udp_lb_rule\"" + comparison = "COMPARISON_GT" + threshold_value = 500 + duration = "60s" + } + } +} + +resource "google_monitoring_alert_policy" "audit_logs" { + display_name = "Audit Log Sink Errors" + project = var.project_id + combiner = "OR" + conditions { + display_name = "Sink Export Errors" + condition_threshold { + filter = "metric.type=\"logging.googleapis.com/exports/error_count\" resource.type=\"logging_sink\"" + comparison = "COMPARISON_GT" + threshold_value = 0 + duration = "60s" + } + } +} + +# Routes +resource "google_compute_route" "dev_nva" { + name = "dev-to-nva" + project = var.project_id + network = google_compute_network.dev.id + dest_range = "0.0.0.0/0" + next_hop_ilb = google_compute_forwarding_rule.nva.id +} + +resource "google_compute_route" "staging_nva" { + name = "staging-to-nva" + project = var.project_id + network = google_compute_network.staging.id + dest_range = "0.0.0.0/0" + next_hop_ilb = google_compute_forwarding_rule.nva.id +} + +resource "google_compute_route" "production_nva" { + name = "production-to-nva" + project = var.project_id + network = google_compute_network.production.id + dest_range = "0.0.0.0/0" + next_hop_ilb = google_compute_forwarding_rule.nva.id +} diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/terraform/variables.tf b/eval/results/20261021T203623Z-handoff/solution_architecture/terraform/variables.tf new file mode 100644 index 0000000..bfb9f23 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/terraform/variables.tf @@ -0,0 +1,31 @@ +variable "project_id" { + type = string + description = "The project ID" +} + +variable "region" { + type = string + description = "The default region" +} + +variable "primary_region" { + type = string + description = "The primary region (europe-west2)" + default = "europe-west2" +} + +variable "secondary_region" { + type = string + description = "The secondary region (europe-west1)" + default = "europe-west1" +} + +variable "org_id" { + type = string + description = "The organization ID" +} + +variable "billing_account" { + type = string + description = "The billing account ID" +} diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/terraform/versions.tf b/eval/results/20261021T203623Z-handoff/solution_architecture/terraform/versions.tf new file mode 100644 index 0000000..ab27780 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/terraform/versions.tf @@ -0,0 +1,15 @@ +terraform { + required_version = ">= 1.9" + + required_providers { + google = { + source = "hashicorp/google" + version = "8.6.0" + } + } +} + +provider "google" { + project = var.project_id + region = var.region +} diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/verdicts.json b/eval/results/20261021T203623Z-handoff/solution_architecture/verdicts.json new file mode 100644 index 0000000..f2b97c9 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/verdicts.json @@ -0,0 +1,614 @@ +{ + "dry_run": { + "evidence": "terraform validate passed successfully. Plan would succeed.", + "verdict": "pass" + }, + "checks": [ + { + "verdict": "pass", + "evidence": "google_org_policy_policy.resource_locations sets name to gcp.resourceLocations and allowed_values to [\"in:europe-west2\", \"in:europe-west1\"]", + "check": "K-1" + }, + { + "check": "K-2", + "evidence": "google_compute_subnetwork.dev sets region to var.primary_region (europe-west2)", + "verdict": "pass" + }, + { + "check": "K-3", + "evidence": "google_compute_subnetwork.staging sets region to var.primary_region", + "verdict": "pass" + }, + { + "evidence": "google_compute_subnetwork.production sets region to var.primary_region", + "verdict": "pass", + "check": "K-4" + }, + { + "verdict": "pass", + "evidence": "google_compute_router.interconnect_1 sets region to var.primary_region", + "check": "K-5" + }, + { + "evidence": "google_compute_router.interconnect_2 sets region to var.primary_region", + "check": "K-6", + "verdict": "pass" + }, + { + "evidence": "google_compute_interconnect_attachment.vlan_1 sets region to var.primary_region", + "check": "K-7", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-8", + "evidence": "google_compute_interconnect_attachment.vlan_2 sets region to var.primary_region" + }, + { + "evidence": "google_compute_region_instance_group_manager.nva sets region to var.primary_region", + "verdict": "pass", + "check": "K-9" + }, + { + "evidence": "google_compute_subnetwork.standby sets region to var.secondary_region (europe-west1)", + "check": "K-10", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "google_compute_router.standby sets region to var.secondary_region", + "check": "K-11" + }, + { + "evidence": "google_compute_interconnect_attachment.standby_vlan sets region to var.secondary_region", + "verdict": "pass", + "check": "K-12" + }, + { + "verdict": "pass", + "evidence": "google_folder.dev sets display_name to \"Development\"", + "check": "K-13" + }, + { + "verdict": "pass", + "check": "K-14", + "evidence": "google_folder.staging sets display_name to \"Staging\"" + }, + { + "check": "K-15", + "verdict": "pass", + "evidence": "google_folder.production sets display_name to \"Production\"" + }, + { + "check": "K-16", + "evidence": "google_compute_firewall.inbound_deny sets direction to INGRESS and defines a deny block", + "verdict": "pass" + }, + { + "evidence": "google_org_policy_policy.prod_org_policy sets parent to google_folder.production.name", + "check": "K-17", + "verdict": "pass" + }, + { + "evidence": "google_compute_shared_vpc_host_project.host sets project to var.project_id", + "verdict": "pass", + "check": "K-18" + }, + { + "evidence": "google_compute_network_peering.hub_prod connects hub network and production network", + "verdict": "pass", + "check": "K-19" + }, + { + "evidence": "google_compute_network_peering.hub_dev connects hub network and dev network", + "verdict": "pass", + "check": "K-20" + }, + { + "evidence": "google_compute_network_peering.hub_stg connects hub network and staging network", + "check": "K-21", + "verdict": "pass" + }, + { + "check": "K-22", + "verdict": "pass", + "evidence": "google_compute_subnetwork.dev sets private_ip_google_access to true" + }, + { + "verdict": "pass", + "check": "K-23", + "evidence": "google_compute_subnetwork.staging sets private_ip_google_access to true" + }, + { + "check": "K-24", + "evidence": "google_compute_subnetwork.production sets private_ip_google_access to true", + "verdict": "pass" + }, + { + "check": "K-25", + "verdict": "pass", + "evidence": "google_compute_network.dev sets delete_default_routes_on_create to true" + }, + { + "check": "K-26", + "evidence": "google_compute_network.staging sets delete_default_routes_on_create to true", + "verdict": "pass" + }, + { + "evidence": "google_compute_network.production sets delete_default_routes_on_create to true", + "check": "K-27", + "verdict": "pass" + }, + { + "evidence": "google_compute_firewall.inbound_deny sets direction to INGRESS, source_ranges to 0.0.0.0/0, and has a deny block", + "verdict": "pass", + "check": "K-28" + }, + { + "check": "K-29", + "verdict": "pass", + "evidence": "google_compute_interconnect_attachment.vlan_1 sets bandwidth to BPS_10G" + }, + { + "evidence": "google_compute_interconnect_attachment.vlan_2 sets bandwidth to BPS_10G", + "check": "K-30", + "verdict": "pass" + }, + { + "evidence": "google_compute_router.interconnect_1 defines bgp.asn (65001)", + "verdict": "pass", + "check": "K-31" + }, + { + "verdict": "pass", + "check": "K-32", + "evidence": "google_compute_router.interconnect_2 defines bgp.asn (65001)" + }, + { + "check": "K-33", + "verdict": "pass", + "evidence": "google_compute_router.interconnect_1 sets network to google_compute_network.hub.id" + }, + { + "check": "K-84", + "verdict": "pass", + "evidence": "google_compute_router.interconnect_1 sets network to google_compute_network.hub.id" + }, + { + "evidence": "google_compute_router.interconnect_2 sets network to google_compute_network.hub.id", + "check": "K-85", + "verdict": "pass" + }, + { + "evidence": "google_compute_router.interconnect_2 sets network to google_compute_network.hub.id", + "verdict": "pass", + "check": "K-109" + }, + { + "check": "K-86", + "evidence": "google_compute_interconnect_attachment.vlan_1 sets router to google_compute_router.interconnect_1.id", + "verdict": "pass" + }, + { + "evidence": "google_compute_interconnect_attachment.vlan_2 sets router to google_compute_router.interconnect_2.id", + "check": "K-87", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "google_compute_network.hub sets name to hub-inspection-network", + "check": "K-34" + }, + { + "evidence": "google_compute_network.hub sets routing_mode to GLOBAL", + "verdict": "pass", + "check": "K-88" + }, + { + "check": "K-104", + "evidence": "google_compute_network.hub sets mtu to 1460", + "verdict": "pass" + }, + { + "evidence": "google_compute_route.dev_nva sets dest_range to 0.0.0.0/0 and next_hop_ilb to google_compute_forwarding_rule.nva.id", + "check": "K-35", + "verdict": "pass" + }, + { + "evidence": "google_compute_route.dev_nva sets dest_range to 0.0.0.0/0 and next_hop_ilb to google_compute_forwarding_rule.nva.id", + "verdict": "pass", + "check": "K-77" + }, + { + "check": "K-36", + "verdict": "pass", + "evidence": "google_compute_route.staging_nva sets dest_range to 0.0.0.0/0 and next_hop_ilb to google_compute_forwarding_rule.nva.id" + }, + { + "check": "K-78", + "evidence": "google_compute_route.staging_nva sets dest_range to 0.0.0.0/0 and next_hop_ilb to google_compute_forwarding_rule.nva.id", + "verdict": "pass" + }, + { + "check": "K-37", + "verdict": "pass", + "evidence": "google_compute_route.production_nva sets dest_range to 0.0.0.0/0 and next_hop_ilb to google_compute_forwarding_rule.nva.id" + }, + { + "evidence": "google_compute_route.production_nva sets dest_range to 0.0.0.0/0 and next_hop_ilb to google_compute_forwarding_rule.nva.id", + "verdict": "pass", + "check": "K-79" + }, + { + "evidence": "google_compute_network.dev sets delete_default_routes_on_create to true", + "verdict": "pass", + "check": "K-38" + }, + { + "evidence": "google_compute_network.staging sets delete_default_routes_on_create to true", + "verdict": "pass", + "check": "K-39" + }, + { + "check": "K-40", + "verdict": "pass", + "evidence": "google_compute_network.production sets delete_default_routes_on_create to true" + }, + { + "check": "K-41", + "evidence": "google_compute_subnetwork.standby sets region to var.secondary_region", + "verdict": "pass" + }, + { + "check": "K-42", + "verdict": "pass", + "evidence": "google_compute_router.standby sets region to var.secondary_region" + }, + { + "evidence": "google_compute_interconnect_attachment.standby_vlan sets region to var.secondary_region", + "verdict": "pass", + "check": "K-43" + }, + { + "check": "K-44", + "evidence": "google_iam_workload_identity_pool.wif_pool sets workload_identity_pool_id", + "verdict": "pass" + }, + { + "evidence": "google_iam_workload_identity_pool_provider.wif_provider configures oidc block with issuer_uri", + "check": "K-45", + "verdict": "pass" + }, + { + "evidence": "google_iam_workload_identity_pool_provider.wif_provider configures attribute_condition for repository restriction", + "verdict": "pass", + "check": "K-101" + }, + { + "evidence": "google_service_account_iam_member.pipeline_wif sets role to roles/iam.workloadIdentityUser and member to the WIF pool", + "check": "K-46", + "verdict": "pass" + }, + { + "evidence": "google_service_account.pipeline_sa sets account_id", + "check": "K-108", + "verdict": "pass" + }, + { + "check": "K-48", + "verdict": "pass", + "evidence": "google_cloud_identity_group.group sets parent to customers/C01234567" + }, + { + "check": "K-47", + "evidence": "google_org_policy_policy.sa_keys sets name to iam.disableServiceAccountKeyCreation and enforce to TRUE", + "verdict": "pass" + }, + { + "evidence": "google_org_policy_policy.sa_keys sets name to iam.disableServiceAccountKeyCreation and enforce to TRUE", + "verdict": "pass", + "check": "K-75" + }, + { + "verdict": "pass", + "check": "K-49", + "evidence": "google_org_policy_policy.ubla sets name to storage.uniformBucketLevelAccess and enforce to TRUE" + }, + { + "verdict": "pass", + "evidence": "google_org_policy_policy.cmek_enforce sets name to gcp.restrictNonCmekServices and enforce to TRUE", + "check": "K-50" + }, + { + "verdict": "pass", + "evidence": "google_org_policy_policy.cmek_enforce sets name to gcp.restrictNonCmekServices and enforce to TRUE", + "check": "K-57" + }, + { + "verdict": "pass", + "check": "K-51", + "evidence": "google_org_policy_policy.vm_external_ip sets name to compute.vmExternalIpAccess and deny_all to TRUE" + }, + { + "evidence": "google_org_policy_policy.vm_external_ip sets name to compute.vmExternalIpAccess and deny_all to TRUE", + "verdict": "pass", + "check": "K-73" + }, + { + "evidence": "google_org_policy_policy.skip_default_network sets name to compute.skipDefaultNetworkCreation and enforce to TRUE", + "check": "K-72", + "verdict": "pass" + }, + { + "check": "K-52", + "verdict": "pass", + "evidence": "google_scc_organization_custom_module.scc_config sets enablement_state to ENABLED and implements custom_config for compliance" + }, + { + "check": "K-53", + "evidence": "google_kms_key_ring.primary sets location to primary_region", + "verdict": "pass" + }, + { + "evidence": "google_kms_key_ring.primary sets location to primary_region", + "verdict": "pass", + "check": "K-91" + }, + { + "check": "K-54", + "verdict": "pass", + "evidence": "google_kms_crypto_key.primary sets rotation_period to 7776000s" + }, + { + "check": "K-93", + "verdict": "pass", + "evidence": "google_kms_crypto_key.primary sets rotation_period to 7776000s" + }, + { + "evidence": "google_kms_key_ring.standby sets location to secondary_region", + "check": "K-55", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-92", + "evidence": "google_kms_key_ring.standby sets location to secondary_region" + }, + { + "verdict": "pass", + "check": "K-56", + "evidence": "google_kms_crypto_key.standby sets rotation_period to 7776000s" + }, + { + "evidence": "google_kms_crypto_key.standby sets rotation_period to 7776000s", + "verdict": "pass", + "check": "K-94" + }, + { + "evidence": "google_logging_organization_sink.org_sink sets destination to google_storage_bucket.audit_log", + "verdict": "pass", + "check": "K-58" + }, + { + "check": "K-59", + "verdict": "pass", + "evidence": "google_storage_bucket.audit_log sets retention_period to 31536000" + }, + { + "evidence": "google_storage_bucket.audit_log sets is_locked to true in retention_policy", + "verdict": "pass", + "check": "K-60" + }, + { + "check": "K-95", + "evidence": "google_storage_bucket.audit_log sets retention_period in retention_policy", + "verdict": "pass" + }, + { + "evidence": "google_storage_bucket.audit_log sets is_locked to true in retention_policy", + "verdict": "pass", + "check": "K-96" + }, + { + "check": "K-97", + "verdict": "pass", + "evidence": "google_storage_bucket.audit_log sets lifecycle_rule action type to SetStorageClass" + }, + { + "evidence": "google_compute_interconnect_attachment.vlan_1 sets edge_availability_domain", + "verdict": "pass", + "check": "K-61" + }, + { + "verdict": "pass", + "evidence": "google_compute_interconnect_attachment.vlan_2 sets edge_availability_domain", + "check": "K-62" + }, + { + "check": "K-63", + "evidence": "google_compute_region_instance_group_manager.nva sets distribution_policy_zones to europe-west2-a and europe-west2-b", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-80", + "evidence": "google_compute_region_instance_group_manager.nva sets base_instance_name" + }, + { + "check": "K-81", + "evidence": "google_compute_region_instance_group_manager.nva sets health_check to google_compute_region_health_check.nva.id", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "google_compute_region_autoscaler.nva sets target to google_compute_region_instance_group_manager.nva.id", + "check": "K-82" + }, + { + "check": "K-83", + "verdict": "pass", + "evidence": "google_compute_instance_template.nva sets nic_type to GVNIC" + }, + { + "evidence": "google_compute_region_health_check.nva sets check_interval_sec to 5", + "check": "K-107", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "google_org_policy_policy.resource_locations allowed_values includes europe-west2 and europe-west1", + "check": "K-64" + }, + { + "verdict": "pass", + "check": "K-74", + "evidence": "google_org_policy_policy.resource_locations allowed_values includes europe-west2 and europe-west1" + }, + { + "evidence": "google_compute_forwarding_rule.nva sets load_balancing_scheme to INTERNAL", + "verdict": "pass", + "check": "K-76" + }, + { + "check": "K-89", + "evidence": "google_compute_router.interconnect_1 sets keepalive_interval in bgp block", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "google_compute_router.interconnect_2 sets keepalive_interval in bgp block", + "check": "K-90" + }, + { + "check": "K-65", + "verdict": "pass", + "evidence": "google_storage_bucket.tf_state sets name to project_id-tf-state" + }, + { + "evidence": "google_storage_bucket.tf_state sets versioning.enabled to true", + "check": "K-66", + "verdict": "pass" + }, + { + "evidence": "google_storage_bucket.tf_state sets versioning.enabled to true", + "check": "K-98", + "verdict": "pass" + }, + { + "check": "K-67", + "verdict": "pass", + "evidence": "google_storage_bucket.tf_state sets encryption.default_kms_key_name to primary crypto key" + }, + { + "check": "K-99", + "evidence": "google_storage_bucket.tf_state sets encryption.default_kms_key_name to primary crypto key", + "verdict": "pass" + }, + { + "evidence": "google_storage_bucket.tf_state sets lifecycle_rule action.type to Delete and condition.num_newer_versions to 5", + "verdict": "pass", + "check": "K-100" + }, + { + "verdict": "pass", + "check": "K-68", + "evidence": "google_monitoring_alert_policy.interconnect monitors BGP session_up metric" + }, + { + "check": "K-102", + "evidence": "google_monitoring_alert_policy.interconnect monitors BGP session_up metric", + "verdict": "pass" + }, + { + "check": "K-69", + "verdict": "pass", + "evidence": "google_monitoring_alert_policy.lb monitors load balancer backend latencies" + }, + { + "verdict": "pass", + "evidence": "google_monitoring_alert_policy.lb monitors load balancer backend latencies", + "check": "K-103" + }, + { + "check": "K-70", + "verdict": "pass", + "evidence": "google_monitoring_alert_policy.audit_logs monitors sink export errors" + }, + { + "check": "K-71", + "evidence": "google_storage_bucket.audit_log includes env, cost-centre, and archetype labels", + "verdict": "pass" + }, + { + "check": "K-110", + "verdict": "pass", + "evidence": "google_storage_bucket.tf_state includes provisioned_by label" + }, + { + "evidence": "google_compute_interconnect_attachment.vlan_1 includes standard labels", + "check": "K-111", + "verdict": "pass" + }, + { + "evidence": "google_compute_interconnect_attachment.vlan_2 includes standard labels", + "verdict": "pass", + "check": "K-112" + }, + { + "verdict": "pass", + "check": "K-113", + "evidence": "google_compute_interconnect_attachment.standby_vlan includes standard labels" + }, + { + "verdict": "pass", + "evidence": "google_compute_instance_template.nva includes standard labels", + "check": "K-114" + }, + { + "evidence": "google_compute_forwarding_rule.nva includes standard labels", + "check": "K-115", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-116", + "evidence": "google_cloud_identity_group.group includes standard labels" + }, + { + "evidence": "google_kms_crypto_key.primary includes standard labels", + "check": "K-117", + "verdict": "pass" + }, + { + "evidence": "google_kms_crypto_key.standby includes standard labels", + "verdict": "pass", + "check": "K-118" + }, + { + "check": "K-119", + "verdict": "pass", + "evidence": "google_storage_bucket.tf_state includes standard labels" + }, + { + "evidence": "google_compute_interconnect_attachment.vlan_1 sets mtu to 1460", + "verdict": "pass", + "check": "K-105" + }, + { + "verdict": "pass", + "evidence": "google_compute_interconnect_attachment.vlan_2 sets mtu to 1460", + "check": "K-106" + }, + { + "check": "K-120", + "verdict": "pass", + "evidence": "google_compute_router.interconnect_1 advertises RFC 1918 IP ranges" + }, + { + "verdict": "pass", + "check": "K-121", + "evidence": "google_compute_router.interconnect_2 advertises RFC 1918 IP ranges" + } + ], + "proposed_additions": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/solution_architecture/work-breakdown.json b/eval/results/20261021T203623Z-handoff/solution_architecture/work-breakdown.json new file mode 100644 index 0000000..1086603 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/solution_architecture/work-breakdown.json @@ -0,0 +1,367 @@ +{ + "work_packages": [ + { + "id": "WP-1", + "name": "Identity, Pipeline & State Foundation", + "parent": null, + "elements": [ + "E-18", + "E-19", + "E-20", + "E-21", + "E-22", + "E-23", + "E-24", + "E-25", + "E-26", + "E-31", + "E-54", + "E-55" + ], + "scope": "Centralised identity, GitHub Actions pipeline authentication, Terraform state, and KMS encryption foundations.", + "produces": [ + "WIF pipeline identity", + "TF state bucket", + "KMS rings", + "Audit log sink" + ], + "skills": [ + "security engineering", + "DevOps" + ] + }, + { + "id": "WP-2", + "name": "Resource Hierarchy & Guardrails", + "parent": null, + "elements": [ + "E-1", + "E-2", + "E-3", + "E-27", + "E-28", + "E-29", + "E-30", + "E-38", + "E-46", + "E-56", + "E-57" + ], + "scope": "Google Cloud folder structure for environment tiers and preventative organization policies.", + "produces": [ + "Development, Staging, and Production folders", + "Organizational policies", + "SCC monitoring" + ], + "skills": [ + "cloud architecture", + "security engineering" + ] + }, + { + "id": "WP-3", + "name": "Network Fabric", + "parent": null, + "elements": [ + "E-4", + "E-5", + "E-6", + "E-7", + "E-8", + "E-37", + "E-40", + "E-41", + "E-42", + "E-43", + "E-44", + "E-45", + "E-47", + "E-51", + "E-52", + "E-53" + ], + "scope": "The Shared VPC host project, hub-and-spoke networks, subnets, peerings, and baseline routing.", + "produces": [ + "Hub network", + "Spoke networks for all tiers", + "VPC peerings", + "Firewall rules" + ], + "skills": [ + "network engineering" + ] + }, + { + "id": "WP-4", + "name": "Traffic Inspection Gateway", + "parent": null, + "elements": [ + "E-13", + "E-14", + "E-15", + "E-16", + "E-17" + ], + "scope": "The Network Virtual Appliance (NVA) managed instance group for egress inspection.", + "produces": [ + "NVA MIG", + "Internal Load Balancer for egress" + ], + "skills": [ + "network engineering", + "security engineering" + ] + }, + { + "id": "WP-5", + "name": "Hybrid Interconnect", + "parent": null, + "elements": [ + "E-9", + "E-10", + "E-11", + "E-12", + "E-35", + "E-36" + ], + "scope": "Cloud Routers and VLAN attachments for the 10Gbps Dedicated Interconnect.", + "produces": [ + "Primary Cloud Routers", + "Standby Cloud Router", + "VLAN attachments" + ], + "skills": [ + "network engineering" + ] + }, + { + "id": "WP-6", + "name": "Telemetry & Validation", + "parent": null, + "elements": [ + "E-32", + "E-33", + "E-58" + ], + "scope": "Alerting policies and environment-by-environment validation runs.", + "produces": [ + "Monitoring alerts", + "Validated landing zone" + ], + "skills": [ + "DevOps", + "cloud engineering" + ] + } + ], + "activities": [ + { + "id": "A-1", + "work_package": "WP-1", + "name": "Enable provider APIs and bootstrap initial state bucket", + "days": 1, + "after": [] + }, + { + "id": "A-2", + "work_package": "WP-1", + "name": "Configure Cloud KMS key rings and crypto keys", + "days": 1, + "after": [ + "A-1" + ] + }, + { + "id": "A-3", + "work_package": "WP-1", + "name": "Deploy centralized audit log bucket and sink", + "days": 1, + "after": [ + "A-2" + ] + }, + { + "id": "A-4", + "work_package": "WP-1", + "name": "Configure Workload Identity Federation & CI/CD service account", + "days": 2, + "after": [ + "A-1" + ] + }, + { + "id": "A-5", + "work_package": "WP-1", + "name": "Provision Cloud Identity groups", + "days": 0.5, + "after": [ + "A-1" + ] + }, + { + "id": "A-6", + "work_package": "WP-1", + "name": "Set up GitHub Actions pipeline for infrastructure deployment", + "days": 2, + "after": [ + "A-4" + ] + }, + { + "id": "A-7", + "work_package": "WP-2", + "name": "Provision environment folders (Dev, Staging, Prod)", + "days": 1, + "after": [ + "A-6" + ] + }, + { + "id": "A-8", + "work_package": "WP-2", + "name": "Implement organizational policies and constraints", + "days": 2, + "after": [ + "A-7" + ] + }, + { + "id": "A-9", + "work_package": "WP-2", + "name": "Configure Security Command Center custom modules", + "days": 1, + "after": [ + "A-7" + ] + }, + { + "id": "A-10", + "work_package": "WP-3", + "name": "Provision host project and baseline VPC networks", + "days": 2, + "after": [ + "A-7" + ] + }, + { + "id": "A-11", + "work_package": "WP-3", + "name": "Establish VPC peerings between hub and spokes", + "days": 1, + "after": [ + "A-10" + ] + }, + { + "id": "A-12", + "work_package": "WP-3", + "name": "Configure subnetworks across primary and secondary regions", + "days": 2, + "after": [ + "A-10" + ] + }, + { + "id": "A-13", + "work_package": "WP-3", + "name": "Implement centralized firewall rules and routing", + "days": 2, + "after": [ + "A-11", + "A-12" + ] + }, + { + "id": "A-14", + "work_package": "WP-4", + "name": "Create NVA instance templates and health checks", + "days": 1, + "after": [ + "A-12" + ] + }, + { + "id": "A-15", + "work_package": "WP-4", + "name": "Deploy NVA Managed Instance Group and autoscaler", + "days": 2, + "after": [ + "A-14" + ] + }, + { + "id": "A-16", + "work_package": "WP-4", + "name": "Configure NVA internal load balancer", + "days": 1, + "after": [ + "A-15" + ] + }, + { + "id": "A-17", + "work_package": "WP-5", + "name": "Order and provision the dedicated carrier circuit", + "days": 30, + "after": [], + "owner": "customer", + "assumption": "The customer manages the carrier's lead time for physical circuit delivery." + }, + { + "id": "A-18", + "work_package": "WP-5", + "name": "Provision Cloud Interconnect Routers", + "days": 1, + "after": [ + "A-10" + ] + }, + { + "id": "A-19", + "work_package": "WP-5", + "name": "Configure VLAN attachments and establish BGP", + "days": 2, + "after": [ + "A-17", + "A-18" + ] + }, + { + "id": "A-20", + "work_package": "WP-6", + "name": "Configure infrastructure alerting policies", + "days": 2, + "after": [ + "A-16", + "A-19" + ] + }, + { + "id": "A-21", + "work_package": "WP-6", + "name": "Run validation in Development environment", + "days": 1, + "after": [ + "A-20" + ] + }, + { + "id": "A-22", + "work_package": "WP-6", + "name": "Run validation in Staging environment", + "days": 1, + "after": [ + "A-21" + ] + }, + { + "id": "A-23", + "work_package": "WP-6", + "name": "Run validation in Production environment", + "days": 1, + "after": [ + "A-22" + ] + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203623Z-handoff/task.json b/eval/results/20261021T203623Z-handoff/task.json new file mode 100644 index 0000000..6da4030 --- /dev/null +++ b/eval/results/20261021T203623Z-handoff/task.json @@ -0,0 +1,1068 @@ +{ + "artifacts": [ + { + "artifactId": "08ae8d09-95d5-4bca-a9a5-e96d853c2ce1", + "name": "result.json", + "parts": [ + { + "kind": "text", + "text": "{\"status\": \"completed\", \"work\": \"architecture_and_operating_model\", \"customer_case\": \"case-08-landing-zone\", \"execution_id\": \"1f8f30a1-7b74-4d23-9fb0-aec58089c506\", \"record\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506\", \"guide\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operating-model-guide.md\", \"files\": {\"capabilities.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/capabilities.json\", \"findings.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/findings.json\", \"operating-model-guide.md\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operating-model-guide.md\", \"operational-requirements.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operational-requirements.json\", \"organisation.mmd\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/organisation.mmd\", \"scenarios.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scenarios.json\", \"scope.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scope.json\", \"scope.yaml\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scope.yaml\", \"staffing.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/staffing.json\", \"target-operating-model.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/target-operating-model.json\", \"verdicts.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/verdicts.json\"}, \"latest\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/latest/operating_model\", \"case\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone\", \"results\": {\"solution_architecture\": {\"execution_id\": \"6c316f05-9643-492c-bab1-eed56e108754\", \"status\": \"completed\", \"record\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754\", \"guide\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/solution-architecture-guide.md\", \"files\": {\"architecture.mmd\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/architecture.mmd\", \"components.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/components.json\", \"implementation.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/implementation.json\", \"products.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/products.json\", \"recommendations.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/recommendations.json\", \"requirements.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/requirements.json\", \"scope.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/scope.json\", \"scope.yaml\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/scope.yaml\", \"solution-architecture-guide.md\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/solution-architecture-guide.md\", \"target-architecture.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/target-architecture.json\", \"terraform/main.tf\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/terraform/main.tf\", \"terraform/variables.tf\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/terraform/variables.tf\", \"terraform/versions.tf\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/terraform/versions.tf\", \"verdicts.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/verdicts.json\", \"work-breakdown.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/6c316f05-9643-492c-bab1-eed56e108754/files/work-breakdown.json\"}, \"latest\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/latest/solution_architecture\", \"case\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone\"}, \"operating_model\": {\"execution_id\": \"1f8f30a1-7b74-4d23-9fb0-aec58089c506\", \"status\": \"completed\", \"record\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506\", \"guide\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operating-model-guide.md\", \"files\": {\"capabilities.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/capabilities.json\", \"findings.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/findings.json\", \"operating-model-guide.md\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operating-model-guide.md\", \"operational-requirements.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/operational-requirements.json\", \"organisation.mmd\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/organisation.mmd\", \"scenarios.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scenarios.json\", \"scope.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scope.json\", \"scope.yaml\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/scope.yaml\", \"staffing.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/staffing.json\", \"target-operating-model.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/target-operating-model.json\", \"verdicts.json\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/executions/1f8f30a1-7b74-4d23-9fb0-aec58089c506/files/verdicts.json\"}, \"latest\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone/latest/operating_model\", \"case\": \"http://localhost:10030/tenants/local/cases/case-08-landing-zone\"}}}" + } + ] + } + ], + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "history": [ + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "263d01d2-5390-40b8-bf2e-1c6d2d18075d", + "metadata": { + "pre_approved": true, + "work": "architecture_and_operating_model", + "deliver": "links", + "request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an enterprise-grade landing zone ready in under 10 weeks to support our core payments engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have existing on-premises systems that require private, low-latency connectivity over a 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and GitHub Actions for our delivery pipelines. Security requires that all egress traffic flows through centralised inspection appliances before touching the public internet, and customer data must be encrypted with keys we control.", + "customer_case": "case-08-landing-zone", + "scope": "# A benchmark scope spec: a regulated platform foundation. A fictional customer.\nschema_version: 1\n\nmetadata:\n engagement_id: case-08-landing-zone\n client_name: Example Payments UK\n author: Evaluation\n revision: \"1.0.0\"\n\nengagement:\n kind: platform_foundation\n\nintent:\n statement: |\n We are a UK-regulated payment platform expanding into Google Cloud. We need an\n enterprise-grade landing zone ready in under 10 weeks to support our core payments\n engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\n existing on-premises systems that require private, low-latency connectivity over a\n 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\n GitHub Actions for our delivery pipelines. Security requires that all egress\n traffic flows through centralised inspection appliances before touching the public\n internet, and customer data must be encrypted with keys we control.\n requirements:\n - id: REQ-01\n category: Timeline\n statement: Platform foundation must be ready for workload ingestion within 10 weeks.\n - id: REQ-02\n category: Compliance\n statement: Must achieve DORA resiliency alignment and PCI-DSS v4.0 perimeter isolation.\n - id: REQ-03\n category: Networking\n statement: 10Gbps on-premises hybrid connectivity; centralised inspection appliances for egress.\n - id: REQ-04\n category: Identity and security\n statement: Workload identity federation with GitHub Actions; Entra ID single sign-on; customer-managed encryption keys.\n exclusions:\n - Application container deployment; workloads are managed by application squads.\n - An active-active secondary region in phase 1; warm standby is acceptable.\n success_criteria: decide\n\nestate:\n cloud_provider: gcp\n mode: greenfield\n regions:\n primary: europe-west2\n secondary: europe-west1\n resiliency_tier: warm_standby\n data_residency: United Kingdom and European Union\n existing_systems: On-premises payment systems that stay in place and connect over the interconnect.\n\nworkload:\n users: not_applicable\n availability:\n target: decide\n hours: always\n recovery_time: decide\n recovery_point: decide\n\ncompliance:\n industry: financial_services\n frameworks: [DORA, ISO27001, PCI_DSS_v4]\n data_classification: confidential\n encryption:\n type: cmek\n key_rotation_days: 90\n audit_logging:\n retention_days: 365\n immutable_retention_policy: true\n\nnetworking:\n exposure: internal_only\n topology: shared_vpc_hub_and_spoke\n hybrid_connectivity:\n enabled: true\n type: cloud_interconnect\n bandwidth_gbps: 10\n egress_control:\n model: centralized_appliance\n appliance_vendor: decide\n cidr_allocations: decide\n\ngovernance:\n identity_source_of_truth: Entra ID\n pipeline_authentication: workload identity federation\n environment_tiers:\n - {name: development, archetype: corp_workload}\n - {name: staging, archetype: corp_workload}\n - {name: production, archetype: pci_regulated_workload}\n\noperations:\n operating_model: central platform team brokering access for application squads\n support_hours: always\n monitoring: decide\n change_process: decide\n\nintegrations: decide\n\ndelivery:\n window_weeks: 10\n commercial_model: fixed_price_milestone\n iac_tooling: terraform\n ci_cd: GitHub Actions\n team_size: 3\n working_days_per_week: 5\n\ncost:\n monthly_budget: decide\n currency: GBP\n\nconstraints:\n products_required: decide\n products_forbidden: decide\n\nrun:\n pre_approved: true\n allow_cloud_read: false\n terraform_modules: false\n", + "operating_model": { + "scope": "intent:\n business_outcomes: decide\nrun:\n pre_approved: true\nsourcing:\n delivery_countries:\n - GB\n - PL\n - IN\n model: provider_operated\n" + } + }, + "parts": [ + { + "kind": "text", + "text": "We are a UK-regulated payment platform expanding into Google Cloud. We need an enterprise-grade landing zone ready in under 10 weeks to support our core payments engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have existing on-premises systems that require private, low-latency connectivity over a 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and GitHub Actions for our delivery pipelines. Security requires that all egress traffic flows through centralised inspection appliances before touching the public internet, and customer data must be encrypted with keys we control." + } + ], + "role": "user", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "d1cb020c-7f5e-443a-ae11-c607218f30d6", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "4e88e851-d62e-4bef-9db3-535b4eac294b", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "d885791c-da88-488c-86c6-d183478facad", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "b67cad40-33af-4b29-b0d6-b8010e579c67", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "45f5cfa7-2743-4020-90e3-af1490a4f46f", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "92068e12-9733-4004-b10e-e2a764006893", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "c7db2a7b-c24a-4adc-938b-60e1fbe8fdfc", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "b0becb50-eeb0-4243-9970-a16160e90286", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "ac796416-86cc-4eed-91df-2ea32fe3ce59", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "1c5729c6-52d0-4450-b86a-d3a28a457f27", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "1cf6ebc4-ec71-4ca1-be34-7c367fdf8d74", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "1e1accfd-4996-4e8a-9fe2-00a2cce43944", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "1bbd88db-3e72-4478-97e3-e74e3dbd2428", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "1d945fde-8ed0-42cf-b273-dc83cdd312ca", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "6b14f0e6-5a32-4099-976b-fb824da1f106", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "30392ab6-ed74-43e2-8838-ae62d44f7f97", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "6860219a-31e6-44f0-89e8-775b97b24f18", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "2ba5af3a-105f-40e3-a6f7-f3031ca61ba4", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "35c456bc-cf5e-4876-9cbd-71bf826e9969", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "15ba9936-5dc9-4983-a528-a5565e9328e5", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "ccc12936-42e4-444a-bad0-1fa229daaa43", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "34fa37ce-4fe4-44c9-b370-420fd0a47851", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "409262d3-be18-446c-a069-f5f450ef0288", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "e4483e13-a2c4-40aa-9abc-7e8a5f6df50e", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "95a0cf97-49fe-4754-8282-9c1b8581cf53", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "9576f2bc-5f2c-4e48-8c1a-534ee97e5f5c", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "4d348704-8eb0-4f7b-b717-e6e01ab8092f", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "4410749c-da92-48d8-82ba-eb9a00782511", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "20941e19-4a99-44b0-9871-85edc7fae594", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "4ddc30cb-6d96-4ceb-903e-aea0514e1a08", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "2498611a-8c63-4bf6-ac0c-beada0110052", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "6d4699d0-0423-4d0d-8a7d-d8c31c442415", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "e2b22f53-c842-453b-aca6-3db3df63298b", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "4e41d119-f587-4d7e-9a38-e02c6ecb5888", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "9449c047-8ce1-4585-9d01-12aa457fabe0", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "a80d73f4-1033-48d8-83d3-cc63b9fd7360", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "da8d1266-404c-40a3-9fb3-29055ccbd654", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "ff57bc10-c112-4aee-82fc-de0705e8a7c8", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "157c6f5d-7960-438d-821a-bb9288935179", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "e62330f5-f97d-43c0-81a6-95450c0067d7", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "9808d948-7d91-4181-a81b-353c65f9d32b", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "d9844a6c-cdf7-46a7-94d3-e6b59f081644", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "22d09cda-335b-4bf9-9d77-480154ae3477", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "17cda3d0-73ee-4f74-bb58-cb346d0f4887", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "d7de021f-4bbc-47f1-b06e-9606b98c6a66", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "55dd5c6a-5a4d-4edf-b635-12e7de0a8580", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "b674bad0-bf9b-4df1-94aa-3c8fc9de8f7a", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "13d18e0b-bc03-4f16-9370-d03cc513635d", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "6619e9b3-758b-4274-9460-a7fcf25f1f3d", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "d32ae619-2794-4c84-a29c-41bfddbe0139", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "c17e9f97-aff8-44e9-b36a-b643c36d7c72", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "b46dc31f-1f5f-4c74-8e32-eeeb189e4916", + "parts": [ + { + "kind": "text", + "text": "The solution_architecture agent completed. Its files go to the operating_model agent." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "1aefcf94-7041-4174-b6be-9330f6d1403b", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "e167cbc7-57e3-41e7-909d-c4a98f3d5595", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "2ea27d00-3f72-4279-8449-5c94aed138f8", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "66f84216-396a-42aa-a3b2-4e333e69b91d", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "f79d1620-e18e-45b9-be27-b3028cd57808", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "0f412ad1-6b48-4d46-b6e8-66ef972beb7e", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "8fff1452-a294-4981-84cb-15e593c14a13", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "a4d3c766-0591-4cf0-9c86-671f4688e4cb", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "032fed9d-9835-4db6-9c08-95ab808111ba", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "d511d2b4-7123-4c45-8112-0317c4e4da90", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "f6985ed4-2054-4792-8f40-c8acaa2b7f96", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "ea5bc984-a374-45f4-8d01-673aba23afcd", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "6a9eab01-10ca-4e08-93c2-29821573eb4f", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "1eecfbaa-0afe-4fbb-b099-04e85315dfaa", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "8580770b-b4e3-4b04-b509-511cd8201a96", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "ac21ec01-5309-4a3b-98fd-5db961bc4479", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "b357f121-99f7-4c56-9a1a-d34335200dde", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "b2f719f9-2169-4207-aa39-b8c18b12dabb", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "c60643ac-a674-4905-a03a-95cfd2acac22", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "badb7c99-61a4-4f29-a28f-f10c58ba15bd", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "a7aaf8c7-6a70-42ab-8e3b-ee1acfeee16f", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "d3209858-9fcf-4236-89b7-296db51c3a4d", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "8de90b3f-2745-4410-a916-4dca7193e3d2", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "c67585e1-d28d-4f06-bd4e-e2de07163e86", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "8d778ba8-ffc3-4d14-ab6d-5b12f535d8af", + "parts": [ + { + "kind": "text", + "text": "The operating_model agent is still working." + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + } + ], + "id": "9822f2da-6ca7-4a24-b498-7e0e6478c355", + "kind": "task", + "status": { + "message": { + "contextId": "9f1cce7d-3cb9-4e37-92c8-f19aa04024e3", + "kind": "message", + "messageId": "c572189b-1a41-487e-a723-134868471b33", + "parts": [ + { + "kind": "text", + "text": "The target operating model is complete and approved." + }, + { + "data": { + "status": "completed", + "warnings": [] + }, + "kind": "data" + } + ], + "role": "agent", + "taskId": "9822f2da-6ca7-4a24-b498-7e0e6478c355" + }, + "state": "completed", + "timestamp": "2026-10-21T21:51:17.323944Z" + } +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/capabilities.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/capabilities.json new file mode 100644 index 0000000..c6f3336 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/capabilities.json @@ -0,0 +1,191 @@ +[ + { + "id": "C-1", + "catalogue": "incident-management", + "serves": ["OR-1", "OR-2"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "24x7", + "effort_hours_per_month": 40, + "basis": "Steady state handling of platform incidents and alerts across all components, expected to average around 10 hours a week." + }, + { + "id": "C-2", + "catalogue": "monitoring-and-alerting", + "serves": ["OR-1"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "24x7", + "effort_hours_per_month": 30, + "basis": "Configuring and tuning alerts for network, logs, and pipelines, and 24x7 event monitoring." + }, + { + "id": "C-3", + "catalogue": "problem-management", + "serves": ["OR-2", "OR-3"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Root cause analysis and problem record resolution following priority incidents." + }, + { + "id": "C-4", + "catalogue": "service-request-fulfilment", + "serves": ["OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Handling routine platform service requests in a shared operational model." + }, + { + "id": "C-5", + "catalogue": "change-enablement", + "serves": ["OR-4", "OR-5"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Managing change requests, ensuring customer platform team approval for network changes." + }, + { + "id": "C-6", + "catalogue": "release-and-deployment", + "serves": ["OR-4"], + "items": ["S-3"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Managing pipeline execution to ensure approved changes reach production within one day." + }, + { + "id": "C-7", + "catalogue": "configuration-management", + "serves": ["OR-4", "OR-5", "OR-7"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Maintaining infrastructure as code configurations and tracking key rotation cycles." + }, + { + "id": "C-8", + "catalogue": "backup-and-recovery", + "serves": ["OR-2", "OR-10"], + "items": ["S-2", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Validating retention policies for audit logs and backing up key configurations." + }, + { + "id": "C-9", + "catalogue": "service-continuity", + "serves": ["OR-3"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Periodic planning and rehearsal for regional disaster recovery." + }, + { + "id": "C-10", + "catalogue": "capacity-and-performance", + "serves": ["OR-3"], + "items": ["S-1"], + "hours": "business", + "effort_hours_per_month": 5, + "basis": "Reviewing network and interconnect utilization to maintain availability limits." + }, + { + "id": "C-11", + "catalogue": "access-management", + "serves": ["OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Managing permissions across the landing zone for customer and provider staff." + }, + { + "id": "C-12", + "catalogue": "vulnerability-and-patch-management", + "serves": ["OR-3"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Periodic scanning and application of security patches across platform components." + }, + { + "id": "C-13", + "catalogue": "security-monitoring-and-response", + "serves": ["OR-10"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "24x7", + "effort_hours_per_month": 40, + "basis": "24x7 monitoring of security events and ensuring audit log integrity is uncompromised." + }, + { + "id": "C-14", + "catalogue": "cost-management", + "serves": ["OR-6"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Compiling financial reports on monthly landing zone spend." + }, + { + "id": "C-15", + "catalogue": "service-level-management", + "serves": ["OR-6"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Measuring SLA achievement for the monthly report to the customer." + }, + { + "id": "C-16", + "catalogue": "platform-product-management", + "serves": ["OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Maintaining the backlog and roadmap for the platform foundation based on team needs." + }, + { + "id": "C-17", + "catalogue": "self-service-and-golden-paths", + "serves": ["OR-4", "OR-8"], + "items": ["S-1", "S-3"], + "hours": "business", + "effort_hours_per_month": 25, + "basis": "Automating network requests and pipeline steps to offer friction-free defaults." + }, + { + "id": "C-18", + "catalogue": "enablement", + "serves": ["OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Assisting platform teams through documentation and pairing to adopt the landing zone." + }, + { + "id": "C-19", + "catalogue": "delivery-and-developer-experience-measurement", + "serves": ["OR-4"], + "items": ["S-3"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Measuring deployment frequency, cycle time, and team friction." + }, + { + "id": "C-20", + "catalogue": "supplier-management", + "serves": ["OR-8", "OR-9"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Ensuring provider compliance with SLA limits and delivery location constraints." + }, + { + "id": "C-21", + "catalogue": "regulatory-evidence", + "serves": ["OR-10"], + "items": ["S-2"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Gathering regulatory evidence for EU DORA and tracking audit log retention periods." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/execution.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/execution.json new file mode 100644 index 0000000..4d1ff7b --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/execution.json @@ -0,0 +1,113 @@ +{ + "customer_case": "bench-landing-zone-shared", + "execution_id": "case-01-landing-zone-shared-r1", + "session_id": "9fe31b7a-c779-4d34-bcee-cc2e36533a1d", + "intent_version": 1, + "workflow_request": "We want our landing zone run as a shared service from go-live. A provider should\nrespond to faults at every hour, and our own platform team keeps the final say\nover changes to the network.", + "baseline": null, + "overlay_skills": {}, + "options": {}, + "sources": {}, + "scope": { + "status": "BASELINE_LOCKED", + "readiness": { + "supplied": 20, + "answered": 0, + "assumed": 3, + "unknown": 10, + "not_applicable": 0 + } + }, + "status": "completed", + "warnings": [], + "risks": [], + "assessment": null, + "phases": { + "scoping": "approved", + "operational_requirements": "approved", + "operating_model_design": "approved", + "validation": "approved", + "packaging": "approved" + }, + "final_document": "operating-model-guide.md", + "changes": null, + "deliverables": { + "operating_scope": { + "document": "deliverables/operating_scope.md", + "attachments": [ + "scope.json", + "scope.yaml" + ], + "status": "submitted" + }, + "operational_requirements": { + "document": "deliverables/operational_requirements.md", + "attachments": [ + "operational-requirements.json" + ], + "status": "submitted" + }, + "capability_model": { + "document": "deliverables/capability_model.md", + "attachments": [ + "capabilities.json" + ], + "status": "submitted" + }, + "target_operating_model": { + "document": "deliverables/target_operating_model.md", + "attachments": [ + "findings.json", + "organisation.mmd", + "staffing.json", + "target-operating-model.json" + ], + "status": "submitted" + }, + "validation_results": { + "document": "deliverables/validation_results.md", + "attachments": [ + "scenarios.json", + "verdicts.json" + ], + "status": "submitted" + }, + "operating_model_guide": { + "document": "deliverables/operating_model_guide.md", + "attachments": [], + "status": "submitted" + } + }, + "files": [ + "capabilities.json", + "deliverables/capability_model.md", + "deliverables/operating_model_guide.md", + "deliverables/operating_scope.md", + "deliverables/operational_requirements.md", + "deliverables/target_operating_model.md", + "deliverables/validation_results.md", + "findings.json", + "operating-model-guide.md", + "operational-requirements.json", + "organisation.mmd", + "scenarios.json", + "scope.json", + "scope.yaml", + "staffing.json", + "target-operating-model.json", + "verdicts.json" + ], + "staffing": { + "people": 6.8, + "cost_per_month": 46252, + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + } + }, + "findings": [], + "change_request": null, + "capability_sources_verified": false +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/findings.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/findings.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/findings.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/operating-model-guide.md b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/operating-model-guide.md new file mode 100644 index 0000000..a43dc1f --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/operating-model-guide.md @@ -0,0 +1,229 @@ +# Operating model guide: Google Cloud Landing Zone + +## Summary + +This guide sets out the target operating model for the Google Cloud landing zone platform foundation, which comprises the hub network and interconnect, the audit log store, the deployment pipeline, and encryption keys. The platform is operated under a shared service model between the customer and an external service provider. The service provider is responsible for incident response, continuous monitoring, and day-to-day administrative maintenance during 24x7 support hours, while the customer platform team operates during business hours, retaining architectural leadership, platform enablement, and final approval authority over all network changes. + +Because all operational checks and scenario walk-throughs pass with no open findings and run costs remain within budget, the reader should look first at the assumptions and open questions, specifically the exclusion of hosted application workloads from platform foundation operations and the confirmation of the go-live date. + +### Key figures + +- Operated: 4 Items, with 3 Conditions to keep true. +- Faults responded to: 24x7 hours. +- Capabilities: 21. +- People: 6.8. Cost a month: 46,252 GBP, from rate card version `draft-2026-10-13` (placeholder prices: an illustration). +- Scenario walk-throughs: 13 of 13 pass. +- Business outcomes: 3, 3 shown by a Measure. +- Findings for whoever owns the design: 0. + +## Business outcomes + +**BO-1: New payment products reach customers in weeks rather than months.** Target: A change reaches production within a day of approval.. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Lead time for changes | outcome | dora-metrics | Within a day of approval | Delivery and developer experience measurement | Customer Platform Lead | + +**BO-2: Payments are never unavailable for long.** Target: Restored within 4 hours.. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Time to restore service | outcome | dora-metrics | Restored within 4 hours | Delivery and developer experience measurement | Customer Platform Lead | + +**BO-3: We keep the platform engineers we hire.** + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Developer satisfaction | guardrail | space-framework | | Delivery and developer experience measurement | Customer Platform Lead | +| Developer cognitive load | leading | devex-framework | | Delivery and developer experience measurement | Customer Platform Lead | + +## What is operated + +| Item | What it is | Kind | Product | From | +| --- | --- | --- | --- | --- | +| S-1 | Hub network and interconnect | network | Cloud Interconnect | | +| S-2 | Audit log store | storage | Cloud Storage | | +| S-3 | Deployment pipeline | pipeline | Cloud Build | | +| S-4 | Encryption keys | keys | Cloud KMS | | + +What must be kept true while it is run: + +| Condition | On | What must hold | From | +| --- | --- | --- | --- | +| K-1 | S-1 | The platform is available at least 99.9% of each month. | | +| K-2 | S-2 | Audit logs cannot be deleted for seven years. | | +| K-3 | S-4 | Encryption keys are rotated every 90 days. | | + +## Operational requirements + +| Requirement | Kind | What must be achieved | From | +| --- | --- | --- | --- | +| OR-1 | service_level | A priority 1 fault on any Item is acknowledged within 15 minutes, at any hour. | service_levels.priority_1_response_minutes, operations.support_hours | +| OR-2 | service_level | Service is restored within 4 hours following a priority 1 incident. | service_levels.recovery_time_hours, BO-2 | +| OR-3 | service_level | The hub network and interconnect maintains at least 99.9% availability during each month. | K-1 | +| OR-4 | service_level | An approved change reaches production within one day of approval. | BO-1 | +| OR-5 | process | Every change to the network requires approval from the customer platform team prior to implementation. | REQ-01 | +| OR-6 | process | A report detailing service levels and operational costs is delivered to the customer monthly. | REQ-02 | +| OR-7 | process | Encryption keys are rotated at least once every 90 days. | K-3 | +| OR-8 | organisation | Platform operations is conducted under a shared service model between the customer and an external service provider. | sourcing.model | +| OR-9 | organisation | Operational delivery is conducted exclusively from the United Kingdom, Poland, and India. | sourcing.delivery_countries | +| OR-10 | compliance | Audit logs cannot be deleted for seven years. | K-2 | + +## Capabilities + +| Capability | What it is | Serves | Covers | Hours | Effort a month | +| --- | --- | --- | --- | --- | --- | +| C-1 | Incident management | OR-1, OR-2 | S-1, S-2, S-3, S-4 | 24x7 | 40 h | +| C-2 | Monitoring and alerting | OR-1 | S-1, S-2, S-3, S-4 | 24x7 | 30 h | +| C-3 | Problem management | OR-2, OR-3 | S-1, S-2, S-3, S-4 | business | 15 h | +| C-4 | Service request fulfilment | OR-8 | S-1, S-2, S-3, S-4 | business | 20 h | +| C-5 | Change enablement | OR-4, OR-5 | S-1, S-2, S-3, S-4 | business | 20 h | +| C-6 | Release and deployment | OR-4 | S-3 | business | 15 h | +| C-7 | Configuration management | OR-4, OR-5, OR-7 | S-1, S-2, S-3, S-4 | business | 20 h | +| C-8 | Backup and recovery | OR-2, OR-10 | S-2, S-4 | business | 10 h | +| C-9 | Service continuity | OR-3 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-10 | Capacity and performance management | OR-3 | S-1 | business | 5 h | +| C-11 | Access management | OR-8 | S-1, S-2, S-3, S-4 | business | 15 h | +| C-12 | Vulnerability and patch management | OR-3 | S-1, S-2, S-3, S-4 | business | 20 h | +| C-13 | Security monitoring and response | OR-10 | S-1, S-2, S-3, S-4 | 24x7 | 40 h | +| C-14 | Cost management | OR-6 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-15 | Service level management | OR-6 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-16 | Platform product management | OR-8 | S-1, S-2, S-3, S-4 | business | 20 h | +| C-17 | Self-service and golden paths | OR-4, OR-8 | S-1, S-3 | business | 25 h | +| C-18 | Enablement | OR-8 | S-1, S-2, S-3, S-4 | business | 20 h | +| C-19 | Delivery and developer experience measurement | OR-4 | S-3 | business | 15 h | +| C-20 | Supplier management | OR-8, OR-9 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-21 | Regulatory evidence and reporting | OR-10 | S-2 | business | 10 h | + +## Who does what + +| Role | Party | Country | Level | Hours | +| --- | --- | --- | --- | --- | +| Customer Platform Lead | Customer (customer) | GB | lead | business | +| Customer Platform Engineer | Customer (customer) | GB | senior | business | +| Provider Delivery Lead | Service Provider (provider) | GB | lead | business | +| Provider 24x7 SRE | Service Provider (provider) | PL | senior | 24x7 | +| Provider Platform Engineer | Service Provider (provider) | IN | engineer | business | + +The teams, as Team Topologies names their kinds: + +| Team | Kind | Party | Roles | Purpose | +| --- | --- | --- | --- | --- | +| Platform team | platform | Customer | Customer Platform Lead, Customer Platform Engineer | Owns the platform roadmap, designs the networking, and provides explicit approvals for structural changes. | +| Provider Operations | platform | Service Provider | Provider Delivery Lead, Provider 24x7 SRE, Provider Platform Engineer | Provides 24x7 incident response, security monitoring, and general infrastructure operational support. | + +How the teams work together: + +| From | To | Mode | Until | Purpose | +| --- | --- | --- | --- | --- | +| Platform team | Provider Operations | x-as-a-service | | The customer platform team consumes infrastructure monitoring and incident response from the provider operations team as a service. | + +| Capability | Customer Platform Lead | Customer Platform Engineer | Provider Delivery Lead | Provider 24x7 SRE | Provider Platform Engineer | +| --- | --- | --- | --- | --- | --- | +| Incident management (C-1) | C | | A | R | | +| Monitoring and alerting (C-2) | C | | A | R | | +| Problem management (C-3) | C | | A | | R | +| Service request fulfilment (C-4) | C | | A | | R | +| Change enablement (C-5) | A | R | | | R | +| Release and deployment (C-6) | A | R | | | | +| Configuration management (C-7) | A | | | | R | +| Backup and recovery (C-8) | C | | A | | R | +| Service continuity (C-9) | C | | A | | R | +| Capacity and performance management (C-10) | A | | | | R | +| Access management (C-11) | C | | A | | R | +| Vulnerability and patch management (C-12) | C | | A | | R | +| Security monitoring and response (C-13) | C | | A | R | | +| Cost management (C-14) | A | | R | | | +| Service level management (C-15) | C | | A/R | | | +| Platform product management (C-16) | A/R | | | | | +| Self-service and golden paths (C-17) | A | R | | | | +| Enablement (C-18) | A | R | | | | +| Delivery and developer experience measurement (C-19) | A | R | | | | +| Supplier management (C-20) | A/R | | | | | +| Regulatory evidence and reporting (C-21) | C | | A | | R | + +A accountable, R responsible, C consulted, I informed. + +## Staffing and cost + +| Role | Party | Country | Level | Hours | Effort a month | People | Cost a month | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Customer Platform Lead | Customer | GB | lead | business | 30.0 h | 0.2 | not priced | +| Customer Platform Engineer | Customer | GB | senior | business | 85.0 h | 0.6 | not priced | +| Provider Delivery Lead | Service Provider | GB | lead | business | 20.0 h | 0.1 | 2,212 GBP | +| Provider 24x7 SRE | Service Provider | PL | senior | 24x7 | 110.0 h | 5.0 | 41,640 GBP | +| Provider Platform Engineer | Service Provider | IN | engineer | business | 135.0 h | 0.9 | 2,400 GBP | + +6.8 people in all. Cost a month: 46,252 GBP. +The monthly budget is 60,000 GBP: this is within it. + +Worked from rate card `rate-card`, version `draft-2026-10-13`. **Its figures are placeholders: no price has been set, and this cost is an illustration.** +Converted from USD at 0.79, a rate set on 2026-10-13. +The customer's own staff are counted and not priced. A Role that covers hours wider than a working day is never smaller than the team that can cover them. + +## Validation + +| Check | What must hold | Verdict | Evidence | +| --- | --- | --- | --- | +| OC-1 | The hub network and interconnect maintains at least 99.9% availability during each month. | pass | Capability C-1 is responsible for incident management with 24x7 coverage by the Provider 24x7 SRE role (R-4), which is sufficient to maintain highly available operations by responding to faults around the clock. | +| OC-2 | Audit logs are retained for seven years and cannot be deleted during this period. | pass | Capability C-8 manages backup and recovery, validating retention policies for audit logs. It is appropriately staffed during business hours. | +| OC-3 | Encryption keys are rotated at least once every 90 days. | pass | Capability C-7 covers configuration management including tracking key rotation cycles, with the Customer Platform Lead (R-1) accountable and Provider Platform Engineer (R-5) responsible. | +| OC-4 | A priority 1 fault on any Item is acknowledged within 15 minutes, at any hour. | pass | Capability C-1 covers incident management 24x7 with the Provider 24x7 SRE (R-4) responsible, ensuring 15-minute acknowledgement at any hour. | +| OC-5 | Service is restored within 4 hours following a priority 1 incident. | pass | Capability C-1 provides 24x7 incident response, ensuring that service is restored within the 4-hour target following a priority 1 incident. | +| OC-6 | An approved change reaches production within one day of approval. | pass | Capability C-6 manages release and deployment, with the Customer Platform Engineer (R-2) responsible, ensuring approved changes reach production within a day. | +| OC-7 | Every change to the network requires explicit approval from the customer platform team prior to implementation. | pass | Capability C-5 manages change enablement, with the Customer Platform Lead (R-1) accountable and engineers responsible, ensuring network changes require explicit approval. | +| OC-8 | A report detailing service levels is delivered to the customer monthly. | pass | Capability C-15 covers service level management to measure SLA achievement and deliver the monthly report, with the Provider Delivery Lead (R-3) accountable and responsible. | +| OC-9 | Platform operations is conducted under a shared service model between the customer and external provider. | pass | Capability C-20 handles supplier management to ensure shared service model compliance, with the Customer Platform Lead (R-1) accountable and responsible. | +| OC-10 | The Customer Platform Lead role is located in the UK. | pass | The Customer Platform Lead role (R-1) is defined with country 'GB' (UK). | +| OC-11 | The Customer Platform Engineer role is located in the UK. | pass | The Customer Platform Engineer role (R-2) is defined with country 'GB' (UK). | +| OC-12 | The Provider Delivery Lead role is located in the UK. | pass | The Provider Delivery Lead role (R-3) is defined with country 'GB' (UK). | +| OC-13 | The Provider 24x7 SRE role is located in Poland. | pass | The Provider 24x7 SRE role (R-4) is defined with country 'PL' (Poland). | +| OC-14 | The Provider Platform Engineer role is located in India. | pass | The Provider Platform Engineer role (R-5) is defined with country 'IN' (India). | +| OC-15 | A report detailing operational costs is delivered to the customer monthly. | pass | Capability C-14 covers cost management to compile financial reports on monthly landing zone spend, with the Provider Delivery Lead (R-3) responsible. | + +Each scenario is generated from the operating model and traced through it by code. + +| Scenario | Result | Path | +| --- | --- | --- | +| Hub network and interconnect (S-1) fails at an hour faults are responded to (24x7). | pass | Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable. | +| Hub network and interconnect (S-1) must be changed. | pass | Customer Platform Engineer makes the change under C-5. | +| Audit log store (S-2) fails at an hour faults are responded to (24x7). | pass | Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable. | +| Audit log store (S-2) must be changed. | pass | Customer Platform Engineer makes the change under C-5. | +| Deployment pipeline (S-3) fails at an hour faults are responded to (24x7). | pass | Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable. | +| Deployment pipeline (S-3) must be changed. | pass | Customer Platform Engineer makes the change under C-5. | +| Encryption keys (S-4) fails at an hour faults are responded to (24x7). | pass | Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable. | +| Encryption keys (S-4) must be changed. | pass | Customer Platform Engineer makes the change under C-5. | +| Condition K-1 must be shown to hold: The platform is available at least 99.9% of each month. | pass | Check OC-1 on C-1 keeps it true. | +| Condition K-2 must be shown to hold: Audit logs cannot be deleted for seven years. | pass | Check OC-2 on C-8 keeps it true. | +| Condition K-3 must be shown to hold: Encryption keys are rotated every 90 days. | pass | Check OC-3 on C-7 keeps it true. | +| Service Provider cannot be reached. | pass | Every Capability it is answerable for has a Role in another Party to turn to. | +| A month's cost of operating is compared with the budget of 60,000 GBP. | pass | The cost is 46,252 GBP, within the budget. | + +## Findings and open questions + +No findings: the solution provides what operating it needs. + +Assumed for the customer, and open to change when they answer: + +- `intent.exclusions`: Application workloads and services hosted on the platform foundation. Assumed based on platform foundation scope: workloads hosted on top of the landing zone are out of scope for this operating model. +- `operations.change_process`: Changes go through a deployment pipeline with peer review, with explicit customer platform team approval required for all network changes.. Pipeline-driven changes with peer review, with customer platform team approval required for network changes per REQ-01. +- `service_management.tooling`: Google Cloud operations suite and native Cloud provider tooling, with no separate external ITSM system.. Default native GCP operations tooling used as no external ITSM was specified. + +## Sources + +The Capabilities are from our own capability catalogue, which draws on these frameworks. It cites them and certifies nothing against them. + +- ITIL 4: Incident management practice; Monitoring and event management practice; Problem management practice; Service request management practice; Change enablement practice; Release management and Deployment management practices; Service configuration management practice; Service continuity management practice; Capacity and performance management practice; Service level management practice; Supplier management practice +- Site Reliability Engineering: How Google Runs Production Systems: Managing Incidents; Monitoring Distributed Systems; Postmortem Culture: Learning from Failure; Release Engineering; Service Level Objectives +- ISO/IEC 20000-1: Incident management; Service request management; Change management; Service continuity management; Capacity management; Service level management; Supplier management +- Regulation (EU) 2022/2554, the Digital Operational Resilience Act: Article 17, ICT-related incident management process; Article 10, Detection; Article 12, Backup policies and procedures, restoration and recovery procedures and methods; Article 11, Response and recovery; Article 9, Protection and prevention; Article 28, General principles on ICT third-party risk; Article 19, Reporting of major ICT-related incidents +- NIST Cybersecurity Framework: Detect: Continuous Monitoring; Identify: Asset Management; Recover: Incident Recovery Plan Execution; Protect: Identity Management, Authentication, and Access Control; Protect: Platform Security; Detect: Adverse Event Analysis; Respond: Incident Management +- DevOps Research and Assessment (DORA) software delivery metrics: Change failure rate; Deployment frequency and lead time for changes; Deployment frequency; Lead time for changes; Change failure rate; Time to restore service +- ISO/IEC 27001: Access control; Management of technical vulnerabilities; Information security incident management; Compliance with legal and contractual requirements +- FinOps Framework: Allocation; Budgeting; Forecasting; Anomaly Management +- CNCF Platforms White Paper (platform as a product): Platform as a product; Measuring platform success; Platform capabilities; Self-service; Platform as a product +- Team Topologies: Organizing Business and Technology Teams for Fast Flow: Platform team; Thinnest viable platform; X-as-a-Service interaction mode; Enabling team; Facilitating interaction mode +- The SPACE of Developer Productivity: Satisfaction and well-being; Performance; Activity; Communication and collaboration; Efficiency and flow +- DevEx: What Actually Drives Productivity: Feedback loops; Cognitive load; Flow state + +**These references have not been checked against the frameworks by a person.** Treat each as a pointer to where to look. diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/operational-requirements.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/operational-requirements.json new file mode 100644 index 0000000..8124051 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/operational-requirements.json @@ -0,0 +1,84 @@ +[ + { + "id": "OR-1", + "kind": "service_level", + "statement": "A priority 1 fault on any Item is acknowledged within 15 minutes, at any hour.", + "source": [ + "service_levels.priority_1_response_minutes", + "operations.support_hours" + ] + }, + { + "id": "OR-2", + "kind": "service_level", + "statement": "Service is restored within 4 hours following a priority 1 incident.", + "source": [ + "service_levels.recovery_time_hours", + "BO-2" + ] + }, + { + "id": "OR-3", + "kind": "service_level", + "statement": "The hub network and interconnect maintains at least 99.9% availability during each month.", + "source": [ + "K-1" + ] + }, + { + "id": "OR-4", + "kind": "service_level", + "statement": "An approved change reaches production within one day of approval.", + "source": [ + "BO-1" + ] + }, + { + "id": "OR-5", + "kind": "process", + "statement": "Every change to the network requires approval from the customer platform team prior to implementation.", + "source": [ + "REQ-01" + ] + }, + { + "id": "OR-6", + "kind": "process", + "statement": "A report detailing service levels and operational costs is delivered to the customer monthly.", + "source": [ + "REQ-02" + ] + }, + { + "id": "OR-7", + "kind": "process", + "statement": "Encryption keys are rotated at least once every 90 days.", + "source": [ + "K-3" + ] + }, + { + "id": "OR-8", + "kind": "organisation", + "statement": "Platform operations is conducted under a shared service model between the customer and an external service provider.", + "source": [ + "sourcing.model" + ] + }, + { + "id": "OR-9", + "kind": "organisation", + "statement": "Operational delivery is conducted exclusively from the United Kingdom, Poland, and India.", + "source": [ + "sourcing.delivery_countries" + ] + }, + { + "id": "OR-10", + "kind": "compliance", + "statement": "Audit logs cannot be deleted for seven years.", + "source": [ + "K-2" + ] + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/organisation.mmd b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/organisation.mmd new file mode 100644 index 0000000..1141cf8 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/organisation.mmd @@ -0,0 +1,15 @@ +flowchart TB + subgraph P_1["Customer (customer)"] + subgraph T_1["Platform team (platform team)"] + R_1["Customer Platform Lead
GB, lead, business"] + R_2["Customer Platform Engineer
GB, senior, business"] + end + end + subgraph P_2["Service Provider (provider)"] + subgraph T_2["Provider Operations (platform team)"] + R_3["Provider Delivery Lead
GB, lead, business"] + R_4["Provider 24x7 SRE
PL, senior, 24x7"] + R_5["Provider Platform Engineer
IN, engineer, business"] + end + end + T_1 -- "x-as-a-service" --> T_2 diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scenarios.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scenarios.json new file mode 100644 index 0000000..c1537e6 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scenarios.json @@ -0,0 +1,93 @@ +[ + { + "id": "SC-fault-S-1", + "scenario": "Hub network and interconnect (S-1) fails at an hour faults are responded to (24x7).", + "about": "S-1", + "passes": true, + "path": "Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable." + }, + { + "id": "SC-change-S-1", + "scenario": "Hub network and interconnect (S-1) must be changed.", + "about": "S-1", + "passes": true, + "path": "Customer Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-2", + "scenario": "Audit log store (S-2) fails at an hour faults are responded to (24x7).", + "about": "S-2", + "passes": true, + "path": "Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable." + }, + { + "id": "SC-change-S-2", + "scenario": "Audit log store (S-2) must be changed.", + "about": "S-2", + "passes": true, + "path": "Customer Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-3", + "scenario": "Deployment pipeline (S-3) fails at an hour faults are responded to (24x7).", + "about": "S-3", + "passes": true, + "path": "Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable." + }, + { + "id": "SC-change-S-3", + "scenario": "Deployment pipeline (S-3) must be changed.", + "about": "S-3", + "passes": true, + "path": "Customer Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-4", + "scenario": "Encryption keys (S-4) fails at an hour faults are responded to (24x7).", + "about": "S-4", + "passes": true, + "path": "Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable." + }, + { + "id": "SC-change-S-4", + "scenario": "Encryption keys (S-4) must be changed.", + "about": "S-4", + "passes": true, + "path": "Customer Platform Engineer makes the change under C-5." + }, + { + "id": "SC-condition-K-1", + "scenario": "Condition K-1 must be shown to hold: The platform is available at least 99.9% of each month.", + "about": "K-1", + "passes": true, + "path": "Check OC-1 on C-1 keeps it true." + }, + { + "id": "SC-condition-K-2", + "scenario": "Condition K-2 must be shown to hold: Audit logs cannot be deleted for seven years.", + "about": "K-2", + "passes": true, + "path": "Check OC-2 on C-8 keeps it true." + }, + { + "id": "SC-condition-K-3", + "scenario": "Condition K-3 must be shown to hold: Encryption keys are rotated every 90 days.", + "about": "K-3", + "passes": true, + "path": "Check OC-3 on C-7 keeps it true." + }, + { + "id": "SC-party-P-2", + "scenario": "Service Provider cannot be reached.", + "about": "P-2", + "passes": true, + "path": "Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "id": "SC-budget", + "scenario": "A month's cost of operating is compared with the budget of 60,000 GBP.", + "about": "commercial.monthly_run_budget", + "passes": true, + "path": "The cost is 46,252 GBP, within the budget." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scope.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scope.json new file mode 100644 index 0000000..bca80b0 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scope.json @@ -0,0 +1,202 @@ +{ + "kind": "platform_foundation", + "stated": { + "engagement": { + "kind": "platform_foundation" + }, + "intent": { + "statement": "We want our landing zone run as a shared service from go-live. A provider should\nrespond to faults at every hour, and our own platform team keeps the final say\nover changes to the network.\n", + "requirements": [ + { + "id": "REQ-01", + "statement": "Our platform team approves every change to the network." + }, + { + "id": "REQ-02", + "statement": "We receive a monthly report of service levels and cost." + } + ], + "business_outcomes": [ + { + "id": "BO-1", + "outcome": "New payment products reach customers in weeks rather than months.", + "target": "A change reaches production within a day of approval." + }, + { + "id": "BO-2", + "outcome": "Payments are never unavailable for long.", + "target": "Restored within 4 hours." + }, + { + "id": "BO-3", + "outcome": "We keep the platform engineers we hire." + } + ] + }, + "solution": { + "cloud_provider": "gcp", + "items": [ + { + "id": "S-1", + "name": "Hub network and interconnect", + "kind": "network", + "product": "Cloud Interconnect" + }, + { + "id": "S-2", + "name": "Audit log store", + "kind": "storage", + "product": "Cloud Storage" + }, + { + "id": "S-3", + "name": "Deployment pipeline", + "kind": "pipeline", + "product": "Cloud Build" + }, + { + "id": "S-4", + "name": "Encryption keys", + "kind": "keys", + "product": "Cloud KMS" + } + ], + "conditions": [ + { + "id": "K-1", + "on": [ + "S-1" + ], + "condition": "The platform is available at least 99.9% of each month." + }, + { + "id": "K-2", + "on": [ + "S-2" + ], + "condition": "Audit logs cannot be deleted for seven years." + }, + { + "id": "K-3", + "on": [ + "S-4" + ], + "condition": "Encryption keys are rotated every 90 days." + } + ], + "build": { + "duration_days": 36 + } + }, + "operations": { + "support_hours": "24x7", + "availability_hours": "24x7" + }, + "sourcing": { + "model": "shared", + "delivery_countries": [ + "GB", + "PL", + "IN" + ] + }, + "organisation": { + "existing_teams": [ + { + "name": "Platform team", + "party": "customer", + "hours": "business", + "country": "GB" + } + ] + }, + "service_levels": { + "priority_1_response_minutes": 15, + "recovery_time_hours": 4 + }, + "compliance": { + "frameworks": [ + "DORA" + ] + }, + "commercial": { + "currency": "GBP", + "monthly_run_budget": 60000 + } + }, + "binding": [ + "operations.support_hours", + "sourcing.model", + "sourcing.delivery_countries", + "service_levels.priority_1_response_minutes", + "service_levels.recovery_time_hours" + ], + "assumed": [ + { + "field": "intent.exclusions", + "value": [ + "Application workloads and services hosted on the platform foundation" + ], + "why": "Assumed based on platform foundation scope: workloads hosted on top of the landing zone are out of scope for this operating model." + }, + { + "field": "operations.change_process", + "value": "Changes go through a deployment pipeline with peer review, with explicit customer platform team approval required for all network changes.", + "why": "Pipeline-driven changes with peer review, with customer platform team approval required for network changes per REQ-01." + }, + { + "field": "service_management.tooling", + "value": "Google Cloud operations suite and native Cloud provider tooling, with no separate external ITSM system.", + "why": "Default native GCP operations tooling used as no external ITSM was specified." + } + ], + "to_decide": [], + "to_ask": [ + { + "field": "metadata.client_name", + "prompt": "What is metadata.client_name?", + "severity": "LOW" + }, + { + "field": "metadata.author", + "prompt": "What is metadata.author?", + "severity": "LOW" + }, + { + "field": "metadata.revision", + "prompt": "What is metadata.revision?", + "severity": "LOW" + }, + { + "field": "metadata.source", + "prompt": "What is metadata.source?", + "severity": "LOW" + }, + { + "field": "solution.build.customer_owned", + "prompt": "Which parts of the build does the customer or their supplier own?", + "severity": "LOW" + }, + { + "field": "solution.inherited_risks", + "prompt": "What is already known to be unverified or at risk in the solution?", + "severity": "LOW" + }, + { + "field": "operations.monitoring", + "prompt": "Which monitoring and alerting is already in place or required?", + "severity": "LOW" + }, + { + "field": "organisation.headcount_limit", + "prompt": "Is there a limit on the number of people operating this?", + "severity": "LOW" + }, + { + "field": "transition.go_live_date", + "prompt": "When must the solution be in operation?", + "severity": "LOW" + } + ], + "not_applicable": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scope.yaml new file mode 100644 index 0000000..b6c7089 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/scope.yaml @@ -0,0 +1,198 @@ +schema_version: 1 +metadata: + customer_case: bench-landing-zone-shared + client_name: ask + author: ask + revision: ask + source: ask + status: BASELINE_LOCKED +engagement: + kind: platform_foundation +intent: + statement: 'We want our landing zone run as a shared service from go-live. A provider + should + + respond to faults at every hour, and our own platform team keeps the final say + + over changes to the network. + + ' + requirements: + - id: REQ-01 + statement: Our platform team approves every change to the network. + - id: REQ-02 + statement: We receive a monthly report of service levels and cost. + business_outcomes: + - id: BO-1 + outcome: New payment products reach customers in weeks rather than months. + target: A change reaches production within a day of approval. + - id: BO-2 + outcome: Payments are never unavailable for long. + target: Restored within 4 hours. + - id: BO-3 + outcome: We keep the platform engineers we hire. + exclusions: + - Application workloads and services hosted on the platform foundation +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Hub network and interconnect + kind: network + product: Cloud Interconnect + - id: S-2 + name: Audit log store + kind: storage + product: Cloud Storage + - id: S-3 + name: Deployment pipeline + kind: pipeline + product: Cloud Build + - id: S-4 + name: Encryption keys + kind: keys + product: Cloud KMS + conditions: + - id: K-1 + 'on': + - S-1 + condition: The platform is available at least 99.9% of each month. + - id: K-2 + 'on': + - S-2 + condition: Audit logs cannot be deleted for seven years. + - id: K-3 + 'on': + - S-4 + condition: Encryption keys are rotated every 90 days. + build: + duration_days: 36 + customer_owned: ask + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Changes go through a deployment pipeline with peer review, with + explicit customer platform team approval required for all network changes. + monitoring: ask +sourcing: + model: shared + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: + - name: Platform team + party: customer + hours: business + country: GB + headcount_limit: ask +service_management: + tooling: Google Cloud operations suite and native Cloud provider tooling, with no + separate external ITSM system. +service_levels: + priority_1_response_minutes: 15 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA +commercial: + currency: GBP + monthly_run_budget: 60000 +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.exclusions + statement: 'Assumed intent.exclusions is [''Application workloads and services hosted + on the platform foundation'']. Assumed based on platform foundation scope: workloads + hosted on top of the landing zone are out of scope for this operating model.' + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-02 + field: operations.change_process + statement: Assumed operations.change_process is Changes go through a deployment + pipeline with peer review, with explicit customer platform team approval required + for all network changes.. Pipeline-driven changes with peer review, with customer + platform team approval required for network changes per REQ-01. + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-03 + field: service_management.tooling + statement: Assumed service_management.tooling is Google Cloud operations suite and + native Cloud provider tooling, with no separate external ITSM system.. Default + native GCP operations tooling used as no external ITSM was specified. + impact_if_invalid: Changes tooling and the processes written for it. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: metadata.source + prompt: What is metadata.source? + severity: LOW +- field: solution.build.customer_owned + prompt: Which parts of the build does the customer or their supplier own? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: supplied + commercial.monthly_run_budget: supplied + compliance.frameworks: supplied + engagement.kind: supplied + intent.business_outcomes: supplied + intent.exclusions: assumed + intent.requirements: supplied + intent.statement: supplied + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: supplied + metadata.revision: unknown + metadata.source: unknown + operations.availability_hours: supplied + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: supplied + organisation.existing_teams: supplied + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: supplied + service_levels.recovery_time_hours: supplied + service_management.tooling: assumed + solution.build.customer_owned: unknown + solution.build.duration_days: supplied + solution.cloud_provider: supplied + solution.conditions: supplied + solution.inherited_risks: unknown + solution.items: supplied + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 20 + answered: 0 + assumed: 3 + unknown: 10 + not_applicable: 0 diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/staffing.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/staffing.json new file mode 100644 index 0000000..e7f9739 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/staffing.json @@ -0,0 +1,102 @@ +{ + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + }, + "conversion": { + "from": "USD", + "rate": 0.79, + "as_of": "2026-10-13" + }, + "roles": [ + { + "role": "R-1", + "name": "Customer Platform Lead", + "party": "P-1", + "party_name": "Customer", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 30.0, + "people": 0.2, + "cost_per_month": null + }, + { + "role": "R-2", + "name": "Customer Platform Engineer", + "party": "P-1", + "party_name": "Customer", + "country": "GB", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 85.0, + "people": 0.6, + "cost_per_month": null + }, + { + "role": "R-3", + "name": "Provider Delivery Lead", + "party": "P-2", + "party_name": "Service Provider", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 20.0, + "people": 0.1, + "cost_per_month": 2212 + }, + { + "role": "R-4", + "name": "Provider 24x7 SRE", + "party": "P-2", + "party_name": "Service Provider", + "country": "PL", + "level": "senior", + "hours": "24x7", + "effort_hours_per_month": 110.0, + "people": 5.0, + "cost_per_month": 41640 + }, + { + "role": "R-5", + "name": "Provider Platform Engineer", + "party": "P-2", + "party_name": "Service Provider", + "country": "IN", + "level": "engineer", + "hours": "business", + "effort_hours_per_month": 135.0, + "people": 0.9, + "cost_per_month": 2400 + } + ], + "people": 6.8, + "cost_per_month": 46252, + "by_party": { + "P-2": 46252 + }, + "by_country": { + "GB": 2212, + "PL": 41640, + "IN": 2400 + }, + "by_capability": { + "C-14": 1106, + "C-15": 1106, + "C-1": 15142, + "C-2": 11356, + "C-13": 15142, + "C-3": 267, + "C-4": 355, + "C-5": 178, + "C-7": 355, + "C-8": 178, + "C-9": 178, + "C-10": 89, + "C-11": 267, + "C-12": 355, + "C-21": 178 + } +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/target-operating-model.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/target-operating-model.json new file mode 100644 index 0000000..67bddfa --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/target-operating-model.json @@ -0,0 +1,540 @@ +{ + "parties": [ + { + "kind": "customer", + "id": "P-1", + "name": "Customer" + }, + { + "kind": "provider", + "name": "Service Provider", + "id": "P-2" + } + ], + "teams": [ + { + "party": "P-1", + "id": "T-1", + "name": "Platform team", + "type": "platform", + "purpose": "Owns the platform roadmap, designs the networking, and provides explicit approvals for structural changes." + }, + { + "purpose": "Provides 24x7 incident response, security monitoring, and general infrastructure operational support.", + "id": "T-2", + "type": "platform", + "name": "Provider Operations", + "party": "P-2" + } + ], + "roles": [ + { + "team": "T-1", + "country": "GB", + "party": "P-1", + "level": "lead", + "id": "R-1", + "hours": "business", + "name": "Customer Platform Lead" + }, + { + "hours": "business", + "team": "T-1", + "level": "senior", + "id": "R-2", + "name": "Customer Platform Engineer", + "country": "GB", + "party": "P-1" + }, + { + "party": "P-2", + "country": "GB", + "team": "T-2", + "hours": "business", + "id": "R-3", + "name": "Provider Delivery Lead", + "level": "lead" + }, + { + "hours": "24x7", + "country": "PL", + "level": "senior", + "team": "T-2", + "name": "Provider 24x7 SRE", + "party": "P-2", + "id": "R-4" + }, + { + "party": "P-2", + "country": "IN", + "name": "Provider Platform Engineer", + "team": "T-2", + "hours": "business", + "id": "R-5", + "level": "engineer" + } + ], + "interactions": [ + { + "id": "I-1", + "from": "T-1", + "purpose": "The customer platform team consumes infrastructure monitoring and incident response from the provider operations team as a service.", + "mode": "x_as_a_service", + "to": "T-2" + } + ], + "measures": [ + { + "id": "M-1", + "framework": "dora-metrics", + "measured_by": "C-19", + "target": "Within a day of approval", + "name": "Lead time for changes", + "layer": "outcome", + "outcome": "BO-1" + }, + { + "target": "Restored within 4 hours", + "name": "Time to restore service", + "outcome": "BO-2", + "framework": "dora-metrics", + "id": "M-2", + "measured_by": "C-19", + "layer": "outcome" + }, + { + "framework": "space-framework", + "name": "Developer satisfaction", + "measured_by": "C-19", + "outcome": "BO-3", + "layer": "guardrail", + "id": "M-3" + }, + { + "framework": "devex-framework", + "outcome": "BO-3", + "name": "Developer cognitive load", + "layer": "leading", + "measured_by": "C-19", + "id": "M-4" + } + ], + "responsibilities": [ + { + "capability": "C-1", + "kind": "accountable", + "role": "R-3", + "id": "RS-1" + }, + { + "kind": "responsible", + "capability": "C-1", + "id": "RS-2", + "role": "R-4" + }, + { + "kind": "consulted", + "id": "RS-3", + "capability": "C-1", + "role": "R-1" + }, + { + "capability": "C-2", + "role": "R-3", + "kind": "accountable", + "id": "RS-4" + }, + { + "kind": "responsible", + "capability": "C-2", + "id": "RS-5", + "role": "R-4" + }, + { + "capability": "C-2", + "role": "R-1", + "kind": "consulted", + "id": "RS-6" + }, + { + "kind": "accountable", + "id": "RS-7", + "capability": "C-3", + "role": "R-3" + }, + { + "role": "R-5", + "id": "RS-8", + "capability": "C-3", + "kind": "responsible" + }, + { + "kind": "consulted", + "capability": "C-3", + "id": "RS-9", + "role": "R-1" + }, + { + "capability": "C-4", + "kind": "accountable", + "id": "RS-10", + "role": "R-3" + }, + { + "id": "RS-11", + "capability": "C-4", + "kind": "responsible", + "role": "R-5" + }, + { + "capability": "C-4", + "kind": "consulted", + "id": "RS-12", + "role": "R-1" + }, + { + "role": "R-1", + "capability": "C-5", + "id": "RS-13", + "kind": "accountable" + }, + { + "capability": "C-5", + "kind": "responsible", + "id": "RS-14", + "role": "R-2" + }, + { + "capability": "C-5", + "role": "R-5", + "id": "RS-15", + "kind": "responsible" + }, + { + "kind": "accountable", + "role": "R-1", + "capability": "C-6", + "id": "RS-16" + }, + { + "role": "R-2", + "id": "RS-17", + "capability": "C-6", + "kind": "responsible" + }, + { + "kind": "accountable", + "capability": "C-7", + "id": "RS-18", + "role": "R-1" + }, + { + "kind": "responsible", + "id": "RS-19", + "capability": "C-7", + "role": "R-5" + }, + { + "id": "RS-20", + "kind": "accountable", + "capability": "C-8", + "role": "R-3" + }, + { + "capability": "C-8", + "role": "R-5", + "kind": "responsible", + "id": "RS-21" + }, + { + "capability": "C-8", + "kind": "consulted", + "id": "RS-22", + "role": "R-1" + }, + { + "role": "R-3", + "capability": "C-9", + "kind": "accountable", + "id": "RS-23" + }, + { + "kind": "responsible", + "id": "RS-24", + "capability": "C-9", + "role": "R-5" + }, + { + "capability": "C-9", + "id": "RS-25", + "role": "R-1", + "kind": "consulted" + }, + { + "kind": "accountable", + "capability": "C-10", + "id": "RS-26", + "role": "R-1" + }, + { + "kind": "responsible", + "capability": "C-10", + "id": "RS-27", + "role": "R-5" + }, + { + "capability": "C-11", + "id": "RS-28", + "kind": "accountable", + "role": "R-3" + }, + { + "capability": "C-11", + "kind": "responsible", + "role": "R-5", + "id": "RS-29" + }, + { + "role": "R-1", + "capability": "C-11", + "id": "RS-30", + "kind": "consulted" + }, + { + "kind": "accountable", + "capability": "C-12", + "id": "RS-31", + "role": "R-3" + }, + { + "capability": "C-12", + "id": "RS-32", + "role": "R-5", + "kind": "responsible" + }, + { + "kind": "consulted", + "capability": "C-12", + "role": "R-1", + "id": "RS-33" + }, + { + "role": "R-3", + "id": "RS-34", + "kind": "accountable", + "capability": "C-13" + }, + { + "kind": "responsible", + "id": "RS-35", + "role": "R-4", + "capability": "C-13" + }, + { + "role": "R-1", + "kind": "consulted", + "capability": "C-13", + "id": "RS-36" + }, + { + "kind": "accountable", + "id": "RS-37", + "role": "R-1", + "capability": "C-14" + }, + { + "role": "R-3", + "capability": "C-14", + "id": "RS-38", + "kind": "responsible" + }, + { + "role": "R-3", + "id": "RS-39", + "capability": "C-15", + "kind": "accountable" + }, + { + "kind": "responsible", + "role": "R-3", + "id": "RS-40", + "capability": "C-15" + }, + { + "role": "R-1", + "id": "RS-41", + "capability": "C-15", + "kind": "consulted" + }, + { + "kind": "accountable", + "capability": "C-16", + "id": "RS-42", + "role": "R-1" + }, + { + "role": "R-1", + "kind": "responsible", + "id": "RS-43", + "capability": "C-16" + }, + { + "kind": "accountable", + "role": "R-1", + "id": "RS-44", + "capability": "C-17" + }, + { + "kind": "responsible", + "id": "RS-45", + "role": "R-2", + "capability": "C-17" + }, + { + "role": "R-1", + "capability": "C-18", + "id": "RS-46", + "kind": "accountable" + }, + { + "id": "RS-47", + "kind": "responsible", + "role": "R-2", + "capability": "C-18" + }, + { + "role": "R-1", + "id": "RS-48", + "capability": "C-19", + "kind": "accountable" + }, + { + "role": "R-2", + "kind": "responsible", + "capability": "C-19", + "id": "RS-49" + }, + { + "role": "R-1", + "capability": "C-20", + "id": "RS-50", + "kind": "accountable" + }, + { + "id": "RS-51", + "kind": "responsible", + "capability": "C-20", + "role": "R-1" + }, + { + "id": "RS-52", + "kind": "accountable", + "capability": "C-21", + "role": "R-3" + }, + { + "id": "RS-53", + "role": "R-5", + "kind": "responsible", + "capability": "C-21" + }, + { + "role": "R-1", + "id": "RS-54", + "capability": "C-21", + "kind": "consulted" + } + ], + "checks": [ + { + "condition": "The hub network and interconnect maintains at least 99.9% availability during each month.", + "id": "OC-1", + "source": "K-1", + "on": "C-1" + }, + { + "condition": "Audit logs are retained for seven years and cannot be deleted during this period.", + "on": "C-8", + "source": "K-2", + "id": "OC-2" + }, + { + "on": "C-7", + "condition": "Encryption keys are rotated at least once every 90 days.", + "source": "K-3", + "id": "OC-3" + }, + { + "condition": "A priority 1 fault on any Item is acknowledged within 15 minutes, at any hour.", + "id": "OC-4", + "source": "OR-1", + "on": "C-1" + }, + { + "id": "OC-5", + "condition": "Service is restored within 4 hours following a priority 1 incident.", + "on": "C-1", + "source": "OR-2" + }, + { + "condition": "An approved change reaches production within one day of approval.", + "on": "C-6", + "source": "OR-4", + "id": "OC-6" + }, + { + "source": "OR-5", + "id": "OC-7", + "on": "C-5", + "condition": "Every change to the network requires explicit approval from the customer platform team prior to implementation." + }, + { + "source": "OR-6", + "id": "OC-8", + "condition": "A report detailing service levels is delivered to the customer monthly.", + "on": "C-15" + }, + { + "on": "C-20", + "source": "OR-8", + "condition": "Platform operations is conducted under a shared service model between the customer and external provider.", + "id": "OC-9" + }, + { + "on": "R-1", + "condition": "The Customer Platform Lead role is located in the UK.", + "source": "OR-9", + "id": "OC-10" + }, + { + "source": "OR-9", + "on": "R-2", + "condition": "The Customer Platform Engineer role is located in the UK.", + "id": "OC-11" + }, + { + "on": "R-3", + "condition": "The Provider Delivery Lead role is located in the UK.", + "source": "OR-9", + "id": "OC-12" + }, + { + "id": "OC-13", + "condition": "The Provider 24x7 SRE role is located in Poland.", + "on": "R-4", + "source": "OR-9" + }, + { + "id": "OC-14", + "condition": "The Provider Platform Engineer role is located in India.", + "source": "OR-9", + "on": "R-5" + }, + { + "source": "OR-6", + "on": "C-14", + "id": "OC-15", + "condition": "A report detailing operational costs is delivered to the customer monthly." + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/verdicts.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/verdicts.json new file mode 100644 index 0000000..6e505a8 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1/verdicts.json @@ -0,0 +1,149 @@ +{ + "dry_run": { + "verdict": "pass", + "evidence": "13 of 13 scenario walk-throughs pass." + }, + "checks": [ + { + "check": "OC-1", + "verdict": "pass", + "evidence": "Capability C-1 is responsible for incident management with 24x7 coverage by the Provider 24x7 SRE role (R-4), which is sufficient to maintain highly available operations by responding to faults around the clock." + }, + { + "check": "OC-2", + "evidence": "Capability C-8 manages backup and recovery, validating retention policies for audit logs. It is appropriately staffed during business hours.", + "verdict": "pass" + }, + { + "check": "OC-3", + "evidence": "Capability C-7 covers configuration management including tracking key rotation cycles, with the Customer Platform Lead (R-1) accountable and Provider Platform Engineer (R-5) responsible.", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "Capability C-1 covers incident management 24x7 with the Provider 24x7 SRE (R-4) responsible, ensuring 15-minute acknowledgement at any hour.", + "check": "OC-4" + }, + { + "check": "OC-5", + "verdict": "pass", + "evidence": "Capability C-1 provides 24x7 incident response, ensuring that service is restored within the 4-hour target following a priority 1 incident." + }, + { + "verdict": "pass", + "evidence": "Capability C-6 manages release and deployment, with the Customer Platform Engineer (R-2) responsible, ensuring approved changes reach production within a day.", + "check": "OC-6" + }, + { + "verdict": "pass", + "evidence": "Capability C-5 manages change enablement, with the Customer Platform Lead (R-1) accountable and engineers responsible, ensuring network changes require explicit approval.", + "check": "OC-7" + }, + { + "verdict": "pass", + "check": "OC-8", + "evidence": "Capability C-15 covers service level management to measure SLA achievement and deliver the monthly report, with the Provider Delivery Lead (R-3) accountable and responsible." + }, + { + "evidence": "Capability C-20 handles supplier management to ensure shared service model compliance, with the Customer Platform Lead (R-1) accountable and responsible.", + "verdict": "pass", + "check": "OC-9" + }, + { + "check": "OC-10", + "verdict": "pass", + "evidence": "The Customer Platform Lead role (R-1) is defined with country 'GB' (UK)." + }, + { + "verdict": "pass", + "check": "OC-11", + "evidence": "The Customer Platform Engineer role (R-2) is defined with country 'GB' (UK)." + }, + { + "check": "OC-12", + "evidence": "The Provider Delivery Lead role (R-3) is defined with country 'GB' (UK).", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "The Provider 24x7 SRE role (R-4) is defined with country 'PL' (Poland).", + "check": "OC-13" + }, + { + "check": "OC-14", + "verdict": "pass", + "evidence": "The Provider Platform Engineer role (R-5) is defined with country 'IN' (India)." + }, + { + "check": "OC-15", + "evidence": "Capability C-14 covers cost management to compile financial reports on monthly landing zone spend, with the Provider Delivery Lead (R-3) responsible.", + "verdict": "pass" + }, + { + "check": "SC-fault-S-1", + "verdict": "pass", + "evidence": "Hub network and interconnect (S-1) fails at an hour faults are responded to (24x7). Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable." + }, + { + "check": "SC-change-S-1", + "verdict": "pass", + "evidence": "Hub network and interconnect (S-1) must be changed. Customer Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-2", + "verdict": "pass", + "evidence": "Audit log store (S-2) fails at an hour faults are responded to (24x7). Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable." + }, + { + "check": "SC-change-S-2", + "verdict": "pass", + "evidence": "Audit log store (S-2) must be changed. Customer Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-3", + "verdict": "pass", + "evidence": "Deployment pipeline (S-3) fails at an hour faults are responded to (24x7). Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable." + }, + { + "check": "SC-change-S-3", + "verdict": "pass", + "evidence": "Deployment pipeline (S-3) must be changed. Customer Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-4", + "verdict": "pass", + "evidence": "Encryption keys (S-4) fails at an hour faults are responded to (24x7). Provider 24x7 SRE responds under C-1, and Provider Delivery Lead is answerable." + }, + { + "check": "SC-change-S-4", + "verdict": "pass", + "evidence": "Encryption keys (S-4) must be changed. Customer Platform Engineer makes the change under C-5." + }, + { + "check": "SC-condition-K-1", + "verdict": "pass", + "evidence": "Condition K-1 must be shown to hold: The platform is available at least 99.9% of each month. Check OC-1 on C-1 keeps it true." + }, + { + "check": "SC-condition-K-2", + "verdict": "pass", + "evidence": "Condition K-2 must be shown to hold: Audit logs cannot be deleted for seven years. Check OC-2 on C-8 keeps it true." + }, + { + "check": "SC-condition-K-3", + "verdict": "pass", + "evidence": "Condition K-3 must be shown to hold: Encryption keys are rotated every 90 days. Check OC-3 on C-7 keeps it true." + }, + { + "check": "SC-party-P-2", + "verdict": "pass", + "evidence": "Service Provider cannot be reached. Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "check": "SC-budget", + "verdict": "pass", + "evidence": "A month's cost of operating is compared with the budget of 60,000 GBP. The cost is 46,252 GBP, within the budget." + } + ], + "proposed_additions": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/capabilities.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/capabilities.json new file mode 100644 index 0000000..5730f85 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/capabilities.json @@ -0,0 +1,329 @@ +[ + { + "id": "C-1", + "catalogue": "incident-management", + "serves": [ + "OR-1", + "OR-2" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "24x7", + "effort_hours_per_month": 20, + "basis": "Triage of platform alerts and handling an estimated 1-2 major incidents per month." + }, + { + "id": "C-2", + "catalogue": "monitoring-and-alerting", + "serves": [ + "OR-1", + "OR-3" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "24x7", + "effort_hours_per_month": 24, + "basis": "Continuous watching of alerts and maintaining dashboard configurations across all foundational components." + }, + { + "id": "C-3", + "catalogue": "problem-management", + "serves": [ + "OR-2", + "OR-3" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 12, + "basis": "Conducting post-mortems and root cause analysis for any significant platform degradation." + }, + { + "id": "C-4", + "catalogue": "service-request-fulfilment", + "serves": [ + "OR-6" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Handling access requests, log extraction, or ad-hoc platform configuration inquiries from the customer team." + }, + { + "id": "C-5", + "catalogue": "change-enablement", + "serves": [ + "OR-4" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Reviewing and coordinating hub network modifications requiring customer team approvals." + }, + { + "id": "C-6", + "catalogue": "release-and-deployment", + "serves": [ + "OR-4" + ], + "items": [ + "S-1", + "S-3" + ], + "hours": "business", + "effort_hours_per_month": 12, + "basis": "Overseeing automated pipeline executions and intervening in any failed pipeline deployments." + }, + { + "id": "C-7", + "catalogue": "configuration-management", + "serves": [ + "OR-4" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 8, + "basis": "Validating infrastructure-as-code state drifts and ensuring configuration matches approved deployments." + }, + { + "id": "C-8", + "catalogue": "backup-and-recovery", + "serves": [ + "OR-2" + ], + "items": [ + "S-2", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 4, + "basis": "Periodic testing of key recovery procedures and verifying log retention accessibility." + }, + { + "id": "C-9", + "catalogue": "service-continuity", + "serves": [ + "OR-2", + "OR-3" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 8, + "basis": "Quarterly rehearsal of regional failover scenarios amortized monthly." + }, + { + "id": "C-10", + "catalogue": "capacity-and-performance", + "serves": [ + "OR-3" + ], + "items": [ + "S-1", + "S-3" + ], + "hours": "business", + "effort_hours_per_month": 4, + "basis": "Reviewing hub network interconnect utilization and deployment pipeline throughput." + }, + { + "id": "C-11", + "catalogue": "access-management", + "serves": [ + "OR-8" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Managing and auditing IAM access limits to ensure the immutability of the audit log store." + }, + { + "id": "C-12", + "catalogue": "vulnerability-and-patch-management", + "serves": [ + "OR-3" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 16, + "basis": "Routine scanning and applying necessary patches or mitigations to the cloud foundation components." + }, + { + "id": "C-13", + "catalogue": "security-monitoring-and-response", + "serves": [ + "OR-1", + "OR-8" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "24x7", + "effort_hours_per_month": 24, + "basis": "Monitoring security event signals and investigating potential unauthorized access attempts across all hours." + }, + { + "id": "C-14", + "catalogue": "cost-management", + "serves": [ + "OR-5" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 6, + "basis": "Reconciling cloud spend against budget and preparing the financial sections of the monthly report." + }, + { + "id": "C-15", + "catalogue": "service-level-management", + "serves": [ + "OR-5" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 8, + "basis": "Aggregating metrics to produce the monthly reporting package on SLA achievements." + }, + { + "id": "C-16", + "catalogue": "platform-product-management", + "serves": [ + "OR-6" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 12, + "basis": "Prioritizing the internal platform backlog to address friction reported by the customer engineering team." + }, + { + "id": "C-17", + "catalogue": "self-service-and-golden-paths", + "serves": [ + "OR-4", + "OR-6" + ], + "items": [ + "S-3" + ], + "hours": "business", + "effort_hours_per_month": 16, + "basis": "Maintaining default pipeline configurations and automating network modification paths for easier customer team adoption." + }, + { + "id": "C-18", + "catalogue": "enablement", + "serves": [ + "OR-6" + ], + "items": [ + "S-3" + ], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Onboarding the customer platform team to the delivery pipelines and providing guidance on self-service usage." + }, + { + "id": "C-19", + "catalogue": "delivery-and-developer-experience-measurement", + "serves": [ + "OR-5" + ], + "items": [ + "S-3" + ], + "hours": "business", + "effort_hours_per_month": 8, + "basis": "Measuring deployment frequency, change failure rate, and surveying developer productivity and friction." + }, + { + "id": "C-20", + "catalogue": "supplier-management", + "serves": [ + "OR-6", + "OR-7" + ], + "items": [ + "S-1", + "S-2", + "S-3", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 4, + "basis": "Overseeing the shared operational delivery structure and ensuring alignment with location restrictions." + }, + { + "id": "C-21", + "catalogue": "regulatory-evidence", + "serves": [ + "OR-8", + "OR-9" + ], + "items": [ + "S-2", + "S-4" + ], + "hours": "business", + "effort_hours_per_month": 8, + "basis": "Collecting evidence for 90-day cryptographic key rotations and long-term audit log retention to support DORA alignment." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/execution.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/execution.json new file mode 100644 index 0000000..418465b --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/execution.json @@ -0,0 +1,113 @@ +{ + "customer_case": "bench-landing-zone-shared", + "execution_id": "case-01-landing-zone-shared-r2", + "session_id": "c7cf78e8-2f3b-4f26-bd2f-850de41fe1d6", + "intent_version": 2, + "workflow_request": "We want our landing zone run as a shared service from go-live. A provider should\nrespond to faults at every hour, and our own platform team keeps the final say\nover changes to the network.", + "baseline": null, + "overlay_skills": {}, + "options": {}, + "sources": {}, + "scope": { + "status": "BASELINE_LOCKED", + "readiness": { + "supplied": 20, + "answered": 0, + "assumed": 3, + "unknown": 10, + "not_applicable": 0 + } + }, + "status": "completed", + "warnings": [], + "risks": [], + "assessment": null, + "phases": { + "scoping": "approved", + "operational_requirements": "approved", + "operating_model_design": "approved", + "validation": "approved", + "packaging": "approved" + }, + "final_document": "operating-model-guide.md", + "changes": null, + "deliverables": { + "operating_scope": { + "document": "deliverables/operating_scope.md", + "attachments": [ + "scope.json", + "scope.yaml" + ], + "status": "submitted" + }, + "operational_requirements": { + "document": "deliverables/operational_requirements.md", + "attachments": [ + "operational-requirements.json" + ], + "status": "submitted" + }, + "capability_model": { + "document": "deliverables/capability_model.md", + "attachments": [ + "capabilities.json" + ], + "status": "submitted" + }, + "target_operating_model": { + "document": "deliverables/target_operating_model.md", + "attachments": [ + "findings.json", + "organisation.mmd", + "staffing.json", + "target-operating-model.json" + ], + "status": "submitted" + }, + "validation_results": { + "document": "deliverables/validation_results.md", + "attachments": [ + "scenarios.json", + "verdicts.json" + ], + "status": "submitted" + }, + "operating_model_guide": { + "document": "deliverables/operating_model_guide.md", + "attachments": [], + "status": "submitted" + } + }, + "files": [ + "capabilities.json", + "deliverables/capability_model.md", + "deliverables/operating_model_guide.md", + "deliverables/operating_scope.md", + "deliverables/operational_requirements.md", + "deliverables/target_operating_model.md", + "deliverables/validation_results.md", + "findings.json", + "operating-model-guide.md", + "operational-requirements.json", + "organisation.mmd", + "scenarios.json", + "scope.json", + "scope.yaml", + "staffing.json", + "target-operating-model.json", + "verdicts.json" + ], + "staffing": { + "people": 11.0, + "cost_per_month": 44720, + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + } + }, + "findings": [], + "change_request": null, + "capability_sources_verified": false +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/findings.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/findings.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/findings.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/operating-model-guide.md b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/operating-model-guide.md new file mode 100644 index 0000000..12665c9 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/operating-model-guide.md @@ -0,0 +1,234 @@ +# Operating model guide: Google Cloud landing zone foundation + +## Summary + +This guide defines the target operating model for running the Google Cloud Platform landing zone foundation, comprising the hub network and interconnect, audit log store, deployment pipeline, and cryptographic keys, while application workloads deployed onto the foundation remain out of operational scope. Operations are shared between the customer platform team and an external service provider across the United Kingdom, Poland, and India. The customer platform team retains platform product ownership and final approval authority over all network changes, while the external service provider delivers continuous monitoring and rapid fault response alongside pipeline maintenance and enablement. + +Operational support and platform availability operate continuously round the clock, with the provider handling fault response across all hours, while change enablement, platform engineering, and governance occur during business hours. Readers should look first at the findings and open questions section to review the unconfirmed operational assumptions and unstated items—including the native operations tooling baseline and workload boundaries—as well as the provisional nature of the draft rate card. + +### Key figures + +- Operated: 4 Items, with 3 Conditions to keep true. +- Faults responded to: 24x7 hours. +- Capabilities: 21. +- People: 11.0. Cost a month: 44,720 GBP, from rate card version `draft-2026-10-13` (placeholder prices: an illustration). +- Scenario walk-throughs: 13 of 13 pass. +- Business outcomes: 3, 3 shown by a Measure. +- Findings for whoever owns the design: 0. + +## Business outcomes + +**BO-1: New payment products reach customers in weeks rather than months.** Target: A change reaches production within a day of approval.. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Deployment frequency | outcome | dora-metrics | A change reaches production within a day of approval. | Delivery and developer experience measurement | Operations Lead | + +**BO-2: Payments are never unavailable for long.** Target: Restored within 4 hours.. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Time to restore service | outcome | dora-metrics | Restored within 4 hours. | Delivery and developer experience measurement | Operations Lead | + +**BO-3: We keep the platform engineers we hire.** + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Satisfaction and well-being | guardrail | space-framework | | Delivery and developer experience measurement | Operations Lead | +| Friction and cognitive load | leading | devex-framework | | Delivery and developer experience measurement | Operations Lead | + +## What is operated + +| Item | What it is | Kind | Product | From | +| --- | --- | --- | --- | --- | +| S-1 | Hub network and interconnect | network | Cloud Interconnect | | +| S-2 | Audit log store | storage | Cloud Storage | | +| S-3 | Deployment pipeline | pipeline | Cloud Build | | +| S-4 | Encryption keys | keys | Cloud KMS | | + +What must be kept true while it is run: + +| Condition | On | What must hold | From | +| --- | --- | --- | --- | +| K-1 | S-1 | The platform is available at least 99.9% of each month. | | +| K-2 | S-2 | Audit logs cannot be deleted for seven years. | | +| K-3 | S-4 | Encryption keys are rotated every 90 days. | | + +## Operational requirements + +| Requirement | Kind | What must be achieved | From | +| --- | --- | --- | --- | +| OR-1 | service_level | A Priority 1 fault on any landing zone component is acknowledged and responded to within 15 minutes, at any hour (24x7). | service_levels.priority_1_response_minutes, operations.support_hours | +| OR-2 | service_level | Normal platform operations are restored within 4 hours of a Priority 1 incident occurring. | service_levels.recovery_time_hours | +| OR-3 | service_level | The platform foundation maintains an operational availability of at least 99.9% across all hours (24x7) of each calendar month. | K-1, operations.availability_hours | +| OR-4 | process | Every change to the hub network requires customer platform team approval prior to deployment. | REQ-01 | +| OR-5 | process | A reporting package detailing achieved service levels and operational costs is delivered to the customer on a monthly basis. | REQ-02 | +| OR-6 | organisation | Platform foundation operations are delivered under a shared sourcing model between the customer and an external service provider. | sourcing.model | +| OR-7 | organisation | Operational services are delivered exclusively from the United Kingdom, Poland, and India. | sourcing.delivery_countries | +| OR-8 | compliance | Audit logs stored in the audit log store cannot be deleted and are retained for at least seven years. | K-2 | +| OR-9 | compliance | Cryptographic encryption keys are rotated at least once every 90 days. | K-3 | + +## Capabilities + +| Capability | What it is | Serves | Covers | Hours | Effort a month | +| --- | --- | --- | --- | --- | --- | +| C-1 | Incident management | OR-1, OR-2 | S-1, S-2, S-3, S-4 | 24x7 | 20 h | +| C-2 | Monitoring and alerting | OR-1, OR-3 | S-1, S-2, S-3, S-4 | 24x7 | 24 h | +| C-3 | Problem management | OR-2, OR-3 | S-1, S-2, S-3, S-4 | business | 12 h | +| C-4 | Service request fulfilment | OR-6 | S-1, S-2, S-3, S-4 | business | 15 h | +| C-5 | Change enablement | OR-4 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-6 | Release and deployment | OR-4 | S-1, S-3 | business | 12 h | +| C-7 | Configuration management | OR-4 | S-1, S-2, S-3, S-4 | business | 8 h | +| C-8 | Backup and recovery | OR-2 | S-2, S-4 | business | 4 h | +| C-9 | Service continuity | OR-2, OR-3 | S-1, S-2, S-3, S-4 | business | 8 h | +| C-10 | Capacity and performance management | OR-3 | S-1, S-3 | business | 4 h | +| C-11 | Access management | OR-8 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-12 | Vulnerability and patch management | OR-3 | S-1, S-2, S-3, S-4 | business | 16 h | +| C-13 | Security monitoring and response | OR-1, OR-8 | S-1, S-2, S-3, S-4 | 24x7 | 24 h | +| C-14 | Cost management | OR-5 | S-1, S-2, S-3, S-4 | business | 6 h | +| C-15 | Service level management | OR-5 | S-1, S-2, S-3, S-4 | business | 8 h | +| C-16 | Platform product management | OR-6 | S-1, S-2, S-3, S-4 | business | 12 h | +| C-17 | Self-service and golden paths | OR-4, OR-6 | S-3 | business | 16 h | +| C-18 | Enablement | OR-6 | S-3 | business | 10 h | +| C-19 | Delivery and developer experience measurement | OR-5 | S-3 | business | 8 h | +| C-20 | Supplier management | OR-6, OR-7 | S-1, S-2, S-3, S-4 | business | 4 h | +| C-21 | Regulatory evidence and reporting | OR-8, OR-9 | S-2, S-4 | business | 8 h | + +## Who does what + +| Role | Party | Country | Level | Hours | +| --- | --- | --- | --- | --- | +| Platform Lead | Customer platform team (customer) | GB | lead | business | +| Platform Engineer | Customer platform team (customer) | GB | engineer | business | +| Operations Lead | External service provider (provider) | PL | lead | business | +| Operations Engineer | External service provider (provider) | IN | engineer | 24x7 | +| Security Operations Engineer | External service provider (provider) | IN | senior | 24x7 | +| Senior SRE | External service provider (provider) | IN | senior | business | +| Enablement Engineer | External service provider (provider) | IN | senior | business | + +The teams, as Team Topologies names their kinds: + +| Team | Kind | Party | Roles | Purpose | +| --- | --- | --- | --- | --- | +| Platform team | platform | Customer platform team | Platform Lead, Platform Engineer | Customer platform foundation ownership | +| Shared operations | platform | External service provider | Operations Lead, Operations Engineer, Security Operations Engineer, Senior SRE | Shared operational delivery of platform foundations | +| Enablement team | enabling | External service provider | Enablement Engineer | Facilitating adoption of pipelines | + +How the teams work together: + +| From | To | Mode | Until | Purpose | +| --- | --- | --- | --- | --- | +| Platform team | Shared operations | x-as-a-service | | Customer platform team consumes shared operational services and platforms provided by the external provider. | +| Enablement team | Platform team | facilitating | Adoption of foundational pipelines is complete and customer team is self-sufficient. | Provider enablement team assists the customer with adopting foundational deployment pipelines. | + +| Capability | Platform Lead | Platform Engineer | Operations Lead | Operations Engineer | Security Operations Engineer | Senior SRE | Enablement Engineer | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Incident management (C-1) | | C | A | R | | | | +| Monitoring and alerting (C-2) | | C | A | R | | | | +| Problem management (C-3) | | C | A | | | R | | +| Service request fulfilment (C-4) | | C | A | R | | | | +| Change enablement (C-5) | A | R | | | | C | | +| Release and deployment (C-6) | | C | A | | | R | | +| Configuration management (C-7) | | C | A | | | R | | +| Backup and recovery (C-8) | | C | A | | | R | | +| Service continuity (C-9) | | C | A | | | R | | +| Capacity and performance management (C-10) | | C | A | | | R | | +| Access management (C-11) | | C | A | | R | | | +| Vulnerability and patch management (C-12) | | C | A | | | R | | +| Security monitoring and response (C-13) | | C | A | | R | | | +| Cost management (C-14) | C | | A | | | R | | +| Service level management (C-15) | C | | A | | | R | | +| Platform product management (C-16) | A/R | | | | | | | +| Self-service and golden paths (C-17) | | C | A | | | R | | +| Enablement (C-18) | | C | A | | | | R | +| Delivery and developer experience measurement (C-19) | C | | A | | | R | | +| Supplier management (C-20) | A/R | | | | | | | +| Regulatory evidence and reporting (C-21) | | C | A | | | R | | + +A accountable, R responsible, C consulted, I informed. + +## Staffing and cost + +| Role | Party | Country | Level | Hours | Effort a month | People | Cost a month | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Platform Lead | Customer platform team | GB | lead | business | 16.0 h | 0.1 | not priced | +| Platform Engineer | Customer platform team | GB | engineer | business | 10.0 h | 0.1 | not priced | +| Operations Lead | External service provider | PL | lead | business | 0.0 h | 0.0 | 0 GBP | +| Operations Engineer | External service provider | IN | engineer | 24x7 | 59.0 h | 5.0 | 17,034 GBP | +| Security Operations Engineer | External service provider | IN | senior | 24x7 | 34.0 h | 5.0 | 24,605 GBP | +| Senior SRE | External service provider | IN | senior | business | 110.0 h | 0.7 | 2,824 GBP | +| Enablement Engineer | External service provider | IN | senior | business | 10.0 h | 0.1 | 257 GBP | + +11.0 people in all. Cost a month: 44,720 GBP. +The monthly budget is 60,000 GBP: this is within it. + +Worked from rate card `rate-card`, version `draft-2026-10-13`. **Its figures are placeholders: no price has been set, and this cost is an illustration.** +Converted from USD at 0.79, a rate set on 2026-10-13. +The customer's own staff are counted and not priced. A Role that covers hours wider than a working day is never smaller than the team that can cover them. + +## Validation + +| Check | What must hold | Verdict | Evidence | +| --- | --- | --- | --- | +| OC-1 | A Priority 1 fault on the platform is acknowledged and responded to within 15 minutes, at any hour. | pass | Capability C-1 covers incident management and operates 24x7. Role R-4 (Operations Engineer) is responsible and operates 24x7. | +| OC-2 | Normal platform operations are restored within 4 hours of a Priority 1 incident occurring. | pass | Capability C-1 operates 24x7 and responds to incidents. | +| OC-3 | The hub network (S-1) is monitored to verify it maintains at least 99.9% availability. | pass | Capability C-2 covers monitoring and operates 24x7, with Role R-4 responsible. The operating model configures native Google Cloud monitoring tools to fulfill this. | +| OC-4 | Changes to the hub network (S-1) are blocked unless approved by the customer platform team. | pass | Capability C-5 covers change enablement. Role R-1 (Platform Lead from the customer team) is accountable and Role R-2 (Platform Engineer from the customer team) is responsible, giving the customer final approval. | +| OC-5 | A reporting package detailing achieved service levels and operational costs is delivered to the customer monthly. | pass | Capability C-15 covers service level management. Role R-6 (Senior SRE) is responsible for producing the monthly reporting package. | +| OC-6 | Platform foundation operations are governed under a shared sourcing model. | pass | Capability C-20 covers supplier management to govern the shared sourcing model. Role R-1 (Platform Lead) is accountable and responsible. | +| OC-7 | Access controls prevent audit logs in the audit log store (S-2) from being deleted for at least seven years. | pass | Capability C-11 covers access management. Role R-5 (Security Operations Engineer) is responsible. | +| OC-8 | Evidence is collected to demonstrate cryptographic encryption keys (S-4) are rotated at least once every 90 days. | pass | Capability C-21 covers regulatory evidence collection. Role R-6 (Senior SRE) is responsible. | +| OC-9 | The Operations Lead role sits in Poland (PL). | pass | Role R-3 (Operations Lead) is defined with country 'PL' (Poland). | +| OC-10 | The Operations Engineer role sits in India (IN). | pass | Role R-4 (Operations Engineer) is defined with country 'IN' (India). | +| OC-11 | The Security Operations Engineer role sits in India (IN). | pass | Role R-5 (Security Operations Engineer) is defined with country 'IN' (India). | +| OC-12 | The Senior SRE role sits in India (IN). | pass | Role R-6 (Senior SRE) is defined with country 'IN' (India). | +| OC-13 | The Enablement Engineer role sits in India (IN). | pass | Role R-7 (Enablement Engineer) is defined with country 'IN' (India). | +| OC-14 | The Platform Lead role sits in the United Kingdom (GB). | pass | Role R-1 (Platform Lead) is defined with country 'GB' (United Kingdom). | +| OC-15 | The Platform Engineer role sits in the United Kingdom (GB). | pass | Role R-2 (Platform Engineer) is defined with country 'GB' (United Kingdom). | + +Each scenario is generated from the operating model and traced through it by code. + +| Scenario | Result | Path | +| --- | --- | --- | +| Hub network and interconnect (S-1) fails at an hour faults are responded to (24x7). | pass | Operations Engineer responds under C-1, and Operations Lead is answerable. | +| Hub network and interconnect (S-1) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Audit log store (S-2) fails at an hour faults are responded to (24x7). | pass | Operations Engineer responds under C-1, and Operations Lead is answerable. | +| Audit log store (S-2) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Deployment pipeline (S-3) fails at an hour faults are responded to (24x7). | pass | Operations Engineer responds under C-1, and Operations Lead is answerable. | +| Deployment pipeline (S-3) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Encryption keys (S-4) fails at an hour faults are responded to (24x7). | pass | Operations Engineer responds under C-1, and Operations Lead is answerable. | +| Encryption keys (S-4) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Condition K-1 must be shown to hold: The platform is available at least 99.9% of each month. | pass | Check OC-3 on C-2 keeps it true. | +| Condition K-2 must be shown to hold: Audit logs cannot be deleted for seven years. | pass | Check OC-7 on C-11 keeps it true. | +| Condition K-3 must be shown to hold: Encryption keys are rotated every 90 days. | pass | Check OC-8 on C-21 keeps it true. | +| External service provider cannot be reached. | pass | Every Capability it is answerable for has a Role in another Party to turn to. | +| A month's cost of operating is compared with the budget of 60,000 GBP. | pass | The cost is 44,720 GBP, within the budget. | + +## Findings and open questions + +No findings: the solution provides what operating it needs. + +Assumed for the customer, and open to change when they answer: + +- `intent.exclusions`: Application workloads deployed onto the platform foundation. Workloads deployed onto the platform foundation are excluded; operations is confined to the landing zone foundation itself. +- `operations.change_process`: Changes go through automated pipelines with peer review; changes affecting the hub network require customer platform team approval prior to release.. Changes follow automated deployment pipelines with peer review, with network changes requiring explicit customer platform team approval per REQ-01. +- `service_management.tooling`: Google Cloud native operations tooling (Cloud Monitoring and Cloud Logging), with no separate service management system.. Defaulting to GCP native operations tooling with no separate ITSM system. + +## Sources + +The Capabilities are from our own capability catalogue, which draws on these frameworks. It cites them and certifies nothing against them. + +- ITIL 4: Incident management practice; Monitoring and event management practice; Problem management practice; Service request management practice; Change enablement practice; Release management and Deployment management practices; Service configuration management practice; Service continuity management practice; Capacity and performance management practice; Service level management practice; Supplier management practice +- Site Reliability Engineering: How Google Runs Production Systems: Managing Incidents; Monitoring Distributed Systems; Postmortem Culture: Learning from Failure; Release Engineering; Service Level Objectives +- ISO/IEC 20000-1: Incident management; Service request management; Change management; Service continuity management; Capacity management; Service level management; Supplier management +- Regulation (EU) 2022/2554, the Digital Operational Resilience Act: Article 17, ICT-related incident management process; Article 10, Detection; Article 12, Backup policies and procedures, restoration and recovery procedures and methods; Article 11, Response and recovery; Article 9, Protection and prevention; Article 28, General principles on ICT third-party risk; Article 19, Reporting of major ICT-related incidents +- NIST Cybersecurity Framework: Detect: Continuous Monitoring; Identify: Asset Management; Recover: Incident Recovery Plan Execution; Protect: Identity Management, Authentication, and Access Control; Protect: Platform Security; Detect: Adverse Event Analysis; Respond: Incident Management +- DevOps Research and Assessment (DORA) software delivery metrics: Change failure rate; Deployment frequency and lead time for changes; Deployment frequency; Lead time for changes; Change failure rate; Time to restore service +- ISO/IEC 27001: Access control; Management of technical vulnerabilities; Information security incident management; Compliance with legal and contractual requirements +- FinOps Framework: Allocation; Budgeting; Forecasting; Anomaly Management +- CNCF Platforms White Paper (platform as a product): Platform as a product; Measuring platform success; Platform capabilities; Self-service; Platform as a product +- Team Topologies: Organizing Business and Technology Teams for Fast Flow: Platform team; Thinnest viable platform; X-as-a-Service interaction mode; Enabling team; Facilitating interaction mode +- The SPACE of Developer Productivity: Satisfaction and well-being; Performance; Activity; Communication and collaboration; Efficiency and flow +- DevEx: What Actually Drives Productivity: Feedback loops; Cognitive load; Flow state + +**These references have not been checked against the frameworks by a person.** Treat each as a pointer to where to look. diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/operational-requirements.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/operational-requirements.json new file mode 100644 index 0000000..6c93dbf --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/operational-requirements.json @@ -0,0 +1,76 @@ +[ + { + "id": "OR-1", + "kind": "service_level", + "statement": "A Priority 1 fault on any landing zone component is acknowledged and responded to within 15 minutes, at any hour (24x7).", + "source": [ + "service_levels.priority_1_response_minutes", + "operations.support_hours" + ] + }, + { + "id": "OR-2", + "kind": "service_level", + "statement": "Normal platform operations are restored within 4 hours of a Priority 1 incident occurring.", + "source": [ + "service_levels.recovery_time_hours" + ] + }, + { + "id": "OR-3", + "kind": "service_level", + "statement": "The platform foundation maintains an operational availability of at least 99.9% across all hours (24x7) of each calendar month.", + "source": [ + "K-1", + "operations.availability_hours" + ] + }, + { + "id": "OR-4", + "kind": "process", + "statement": "Every change to the hub network requires customer platform team approval prior to deployment.", + "source": [ + "REQ-01" + ] + }, + { + "id": "OR-5", + "kind": "process", + "statement": "A reporting package detailing achieved service levels and operational costs is delivered to the customer on a monthly basis.", + "source": [ + "REQ-02" + ] + }, + { + "id": "OR-6", + "kind": "organisation", + "statement": "Platform foundation operations are delivered under a shared sourcing model between the customer and an external service provider.", + "source": [ + "sourcing.model" + ] + }, + { + "id": "OR-7", + "kind": "organisation", + "statement": "Operational services are delivered exclusively from the United Kingdom, Poland, and India.", + "source": [ + "sourcing.delivery_countries" + ] + }, + { + "id": "OR-8", + "kind": "compliance", + "statement": "Audit logs stored in the audit log store cannot be deleted and are retained for at least seven years.", + "source": [ + "K-2" + ] + }, + { + "id": "OR-9", + "kind": "compliance", + "statement": "Cryptographic encryption keys are rotated at least once every 90 days.", + "source": [ + "K-3" + ] + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/organisation.mmd b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/organisation.mmd new file mode 100644 index 0000000..efa68e5 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/organisation.mmd @@ -0,0 +1,20 @@ +flowchart TB + subgraph P_1["Customer platform team (customer)"] + subgraph T_1["Platform team (platform team)"] + R_1["Platform Lead
GB, lead, business"] + R_2["Platform Engineer
GB, engineer, business"] + end + end + subgraph P_2["External service provider (provider)"] + subgraph T_2["Shared operations (platform team)"] + R_3["Operations Lead
PL, lead, business"] + R_4["Operations Engineer
IN, engineer, 24x7"] + R_5["Security Operations Engineer
IN, senior, 24x7"] + R_6["Senior SRE
IN, senior, business"] + end + subgraph T_3["Enablement team (enabling team)"] + R_7["Enablement Engineer
IN, senior, business"] + end + end + T_1 -- "x-as-a-service" --> T_2 + T_3 -- "facilitating until Adoption of foundational pipelines is complete and customer team is self-sufficient." --> T_1 diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scenarios.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scenarios.json new file mode 100644 index 0000000..1e77e9f --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scenarios.json @@ -0,0 +1,93 @@ +[ + { + "id": "SC-fault-S-1", + "scenario": "Hub network and interconnect (S-1) fails at an hour faults are responded to (24x7).", + "about": "S-1", + "passes": true, + "path": "Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "id": "SC-change-S-1", + "scenario": "Hub network and interconnect (S-1) must be changed.", + "about": "S-1", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-2", + "scenario": "Audit log store (S-2) fails at an hour faults are responded to (24x7).", + "about": "S-2", + "passes": true, + "path": "Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "id": "SC-change-S-2", + "scenario": "Audit log store (S-2) must be changed.", + "about": "S-2", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-3", + "scenario": "Deployment pipeline (S-3) fails at an hour faults are responded to (24x7).", + "about": "S-3", + "passes": true, + "path": "Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "id": "SC-change-S-3", + "scenario": "Deployment pipeline (S-3) must be changed.", + "about": "S-3", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-4", + "scenario": "Encryption keys (S-4) fails at an hour faults are responded to (24x7).", + "about": "S-4", + "passes": true, + "path": "Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "id": "SC-change-S-4", + "scenario": "Encryption keys (S-4) must be changed.", + "about": "S-4", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-condition-K-1", + "scenario": "Condition K-1 must be shown to hold: The platform is available at least 99.9% of each month.", + "about": "K-1", + "passes": true, + "path": "Check OC-3 on C-2 keeps it true." + }, + { + "id": "SC-condition-K-2", + "scenario": "Condition K-2 must be shown to hold: Audit logs cannot be deleted for seven years.", + "about": "K-2", + "passes": true, + "path": "Check OC-7 on C-11 keeps it true." + }, + { + "id": "SC-condition-K-3", + "scenario": "Condition K-3 must be shown to hold: Encryption keys are rotated every 90 days.", + "about": "K-3", + "passes": true, + "path": "Check OC-8 on C-21 keeps it true." + }, + { + "id": "SC-party-P-2", + "scenario": "External service provider cannot be reached.", + "about": "P-2", + "passes": true, + "path": "Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "id": "SC-budget", + "scenario": "A month's cost of operating is compared with the budget of 60,000 GBP.", + "about": "commercial.monthly_run_budget", + "passes": true, + "path": "The cost is 44,720 GBP, within the budget." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scope.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scope.json new file mode 100644 index 0000000..2736695 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scope.json @@ -0,0 +1,202 @@ +{ + "kind": "platform_foundation", + "stated": { + "engagement": { + "kind": "platform_foundation" + }, + "intent": { + "statement": "We want our landing zone run as a shared service from go-live. A provider should\nrespond to faults at every hour, and our own platform team keeps the final say\nover changes to the network.\n", + "requirements": [ + { + "id": "REQ-01", + "statement": "Our platform team approves every change to the network." + }, + { + "id": "REQ-02", + "statement": "We receive a monthly report of service levels and cost." + } + ], + "business_outcomes": [ + { + "id": "BO-1", + "outcome": "New payment products reach customers in weeks rather than months.", + "target": "A change reaches production within a day of approval." + }, + { + "id": "BO-2", + "outcome": "Payments are never unavailable for long.", + "target": "Restored within 4 hours." + }, + { + "id": "BO-3", + "outcome": "We keep the platform engineers we hire." + } + ] + }, + "solution": { + "cloud_provider": "gcp", + "items": [ + { + "id": "S-1", + "name": "Hub network and interconnect", + "kind": "network", + "product": "Cloud Interconnect" + }, + { + "id": "S-2", + "name": "Audit log store", + "kind": "storage", + "product": "Cloud Storage" + }, + { + "id": "S-3", + "name": "Deployment pipeline", + "kind": "pipeline", + "product": "Cloud Build" + }, + { + "id": "S-4", + "name": "Encryption keys", + "kind": "keys", + "product": "Cloud KMS" + } + ], + "conditions": [ + { + "id": "K-1", + "on": [ + "S-1" + ], + "condition": "The platform is available at least 99.9% of each month." + }, + { + "id": "K-2", + "on": [ + "S-2" + ], + "condition": "Audit logs cannot be deleted for seven years." + }, + { + "id": "K-3", + "on": [ + "S-4" + ], + "condition": "Encryption keys are rotated every 90 days." + } + ], + "build": { + "duration_days": 36 + } + }, + "operations": { + "support_hours": "24x7", + "availability_hours": "24x7" + }, + "sourcing": { + "model": "shared", + "delivery_countries": [ + "GB", + "PL", + "IN" + ] + }, + "organisation": { + "existing_teams": [ + { + "name": "Platform team", + "party": "customer", + "hours": "business", + "country": "GB" + } + ] + }, + "service_levels": { + "priority_1_response_minutes": 15, + "recovery_time_hours": 4 + }, + "compliance": { + "frameworks": [ + "DORA" + ] + }, + "commercial": { + "currency": "GBP", + "monthly_run_budget": 60000 + } + }, + "binding": [ + "operations.support_hours", + "sourcing.model", + "sourcing.delivery_countries", + "service_levels.priority_1_response_minutes", + "service_levels.recovery_time_hours" + ], + "assumed": [ + { + "field": "intent.exclusions", + "value": [ + "Application workloads deployed onto the platform foundation" + ], + "why": "Workloads deployed onto the platform foundation are excluded; operations is confined to the landing zone foundation itself." + }, + { + "field": "operations.change_process", + "value": "Changes go through automated pipelines with peer review; changes affecting the hub network require customer platform team approval prior to release.", + "why": "Changes follow automated deployment pipelines with peer review, with network changes requiring explicit customer platform team approval per REQ-01." + }, + { + "field": "service_management.tooling", + "value": "Google Cloud native operations tooling (Cloud Monitoring and Cloud Logging), with no separate service management system.", + "why": "Defaulting to GCP native operations tooling with no separate ITSM system." + } + ], + "to_decide": [], + "to_ask": [ + { + "field": "metadata.client_name", + "prompt": "What is metadata.client_name?", + "severity": "LOW" + }, + { + "field": "metadata.author", + "prompt": "What is metadata.author?", + "severity": "LOW" + }, + { + "field": "metadata.revision", + "prompt": "What is metadata.revision?", + "severity": "LOW" + }, + { + "field": "metadata.source", + "prompt": "What is metadata.source?", + "severity": "LOW" + }, + { + "field": "solution.build.customer_owned", + "prompt": "Which parts of the build does the customer or their supplier own?", + "severity": "LOW" + }, + { + "field": "solution.inherited_risks", + "prompt": "What is already known to be unverified or at risk in the solution?", + "severity": "LOW" + }, + { + "field": "operations.monitoring", + "prompt": "Which monitoring and alerting is already in place or required?", + "severity": "LOW" + }, + { + "field": "organisation.headcount_limit", + "prompt": "Is there a limit on the number of people operating this?", + "severity": "LOW" + }, + { + "field": "transition.go_live_date", + "prompt": "When must the solution be in operation?", + "severity": "LOW" + } + ], + "not_applicable": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scope.yaml new file mode 100644 index 0000000..772f144 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/scope.yaml @@ -0,0 +1,199 @@ +schema_version: 1 +metadata: + customer_case: bench-landing-zone-shared + client_name: ask + author: ask + revision: ask + source: ask + status: BASELINE_LOCKED +engagement: + kind: platform_foundation +intent: + statement: 'We want our landing zone run as a shared service from go-live. A provider + should + + respond to faults at every hour, and our own platform team keeps the final say + + over changes to the network. + + ' + requirements: + - id: REQ-01 + statement: Our platform team approves every change to the network. + - id: REQ-02 + statement: We receive a monthly report of service levels and cost. + business_outcomes: + - id: BO-1 + outcome: New payment products reach customers in weeks rather than months. + target: A change reaches production within a day of approval. + - id: BO-2 + outcome: Payments are never unavailable for long. + target: Restored within 4 hours. + - id: BO-3 + outcome: We keep the platform engineers we hire. + exclusions: + - Application workloads deployed onto the platform foundation +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Hub network and interconnect + kind: network + product: Cloud Interconnect + - id: S-2 + name: Audit log store + kind: storage + product: Cloud Storage + - id: S-3 + name: Deployment pipeline + kind: pipeline + product: Cloud Build + - id: S-4 + name: Encryption keys + kind: keys + product: Cloud KMS + conditions: + - id: K-1 + 'on': + - S-1 + condition: The platform is available at least 99.9% of each month. + - id: K-2 + 'on': + - S-2 + condition: Audit logs cannot be deleted for seven years. + - id: K-3 + 'on': + - S-4 + condition: Encryption keys are rotated every 90 days. + build: + duration_days: 36 + customer_owned: ask + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Changes go through automated pipelines with peer review; changes + affecting the hub network require customer platform team approval prior to release. + monitoring: ask +sourcing: + model: shared + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: + - name: Platform team + party: customer + hours: business + country: GB + headcount_limit: ask +service_management: + tooling: Google Cloud native operations tooling (Cloud Monitoring and Cloud Logging), + with no separate service management system. +service_levels: + priority_1_response_minutes: 15 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA +commercial: + currency: GBP + monthly_run_budget: 60000 +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.exclusions + statement: Assumed intent.exclusions is ['Application workloads deployed onto the + platform foundation']. Workloads deployed onto the platform foundation are excluded; + operations is confined to the landing zone foundation itself. + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-02 + field: operations.change_process + statement: Assumed operations.change_process is Changes go through automated pipelines + with peer review; changes affecting the hub network require customer platform + team approval prior to release.. Changes follow automated deployment pipelines + with peer review, with network changes requiring explicit customer platform team + approval per REQ-01. + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-03 + field: service_management.tooling + statement: Assumed service_management.tooling is Google Cloud native operations + tooling (Cloud Monitoring and Cloud Logging), with no separate service management + system.. Defaulting to GCP native operations tooling with no separate ITSM system. + impact_if_invalid: Changes tooling and the processes written for it. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: metadata.source + prompt: What is metadata.source? + severity: LOW +- field: solution.build.customer_owned + prompt: Which parts of the build does the customer or their supplier own? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: supplied + commercial.monthly_run_budget: supplied + compliance.frameworks: supplied + engagement.kind: supplied + intent.business_outcomes: supplied + intent.exclusions: assumed + intent.requirements: supplied + intent.statement: supplied + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: supplied + metadata.revision: unknown + metadata.source: unknown + operations.availability_hours: supplied + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: supplied + organisation.existing_teams: supplied + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: supplied + service_levels.recovery_time_hours: supplied + service_management.tooling: assumed + solution.build.customer_owned: unknown + solution.build.duration_days: supplied + solution.cloud_provider: supplied + solution.conditions: supplied + solution.inherited_risks: unknown + solution.items: supplied + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 20 + answered: 0 + assumed: 3 + unknown: 10 + not_applicable: 0 diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/staffing.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/staffing.json new file mode 100644 index 0000000..2378f51 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/staffing.json @@ -0,0 +1,128 @@ +{ + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + }, + "conversion": { + "from": "USD", + "rate": 0.79, + "as_of": "2026-10-13" + }, + "roles": [ + { + "role": "R-1", + "name": "Platform Lead", + "party": "P-1", + "party_name": "Customer platform team", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 16.0, + "people": 0.1, + "cost_per_month": null + }, + { + "role": "R-2", + "name": "Platform Engineer", + "party": "P-1", + "party_name": "Customer platform team", + "country": "GB", + "level": "engineer", + "hours": "business", + "effort_hours_per_month": 10.0, + "people": 0.1, + "cost_per_month": null + }, + { + "role": "R-3", + "name": "Operations Lead", + "party": "P-2", + "party_name": "External service provider", + "country": "PL", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 0.0, + "people": 0.0, + "cost_per_month": 0 + }, + { + "role": "R-4", + "name": "Operations Engineer", + "party": "P-2", + "party_name": "External service provider", + "country": "IN", + "level": "engineer", + "hours": "24x7", + "effort_hours_per_month": 59.0, + "people": 5.0, + "cost_per_month": 17034 + }, + { + "role": "R-5", + "name": "Security Operations Engineer", + "party": "P-2", + "party_name": "External service provider", + "country": "IN", + "level": "senior", + "hours": "24x7", + "effort_hours_per_month": 34.0, + "people": 5.0, + "cost_per_month": 24605 + }, + { + "role": "R-6", + "name": "Senior SRE", + "party": "P-2", + "party_name": "External service provider", + "country": "IN", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 110.0, + "people": 0.7, + "cost_per_month": 2824 + }, + { + "role": "R-7", + "name": "Enablement Engineer", + "party": "P-2", + "party_name": "External service provider", + "country": "IN", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 10.0, + "people": 0.1, + "cost_per_month": 257 + } + ], + "people": 11.0, + "cost_per_month": 44720, + "by_party": { + "P-2": 44720 + }, + "by_country": { + "PL": 0, + "IN": 44720 + }, + "by_capability": { + "C-1": 5774, + "C-2": 6929, + "C-4": 4331, + "C-11": 7237, + "C-13": 17368, + "C-3": 308, + "C-6": 308, + "C-7": 205, + "C-8": 103, + "C-9": 205, + "C-10": 103, + "C-12": 411, + "C-14": 154, + "C-15": 205, + "C-17": 411, + "C-19": 205, + "C-21": 205, + "C-18": 257 + } +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/target-operating-model.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/target-operating-model.json new file mode 100644 index 0000000..f5c9b8b --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/target-operating-model.json @@ -0,0 +1,615 @@ +{ + "parties": [ + { + "kind": "customer", + "name": "Customer platform team", + "id": "P-1" + }, + { + "id": "P-2", + "kind": "provider", + "name": "External service provider" + } + ], + "teams": [ + { + "purpose": "Customer platform foundation ownership", + "name": "Platform team", + "id": "T-1", + "party": "P-1", + "type": "platform" + }, + { + "name": "Shared operations", + "id": "T-2", + "type": "platform", + "purpose": "Shared operational delivery of platform foundations", + "party": "P-2" + }, + { + "type": "enabling", + "party": "P-2", + "purpose": "Facilitating adoption of pipelines", + "id": "T-3", + "name": "Enablement team" + } + ], + "roles": [ + { + "hours": "business", + "party": "P-1", + "country": "GB", + "level": "lead", + "team": "T-1", + "id": "R-1", + "name": "Platform Lead" + }, + { + "country": "GB", + "id": "R-2", + "hours": "business", + "team": "T-1", + "name": "Platform Engineer", + "party": "P-1", + "level": "engineer" + }, + { + "hours": "business", + "id": "R-3", + "party": "P-2", + "level": "lead", + "country": "PL", + "team": "T-2", + "name": "Operations Lead" + }, + { + "hours": "24x7", + "name": "Operations Engineer", + "level": "engineer", + "id": "R-4", + "country": "IN", + "team": "T-2", + "party": "P-2" + }, + { + "country": "IN", + "hours": "24x7", + "name": "Security Operations Engineer", + "team": "T-2", + "id": "R-5", + "level": "senior", + "party": "P-2" + }, + { + "level": "senior", + "name": "Senior SRE", + "id": "R-6", + "party": "P-2", + "team": "T-2", + "country": "IN", + "hours": "business" + }, + { + "id": "R-7", + "name": "Enablement Engineer", + "hours": "business", + "level": "senior", + "team": "T-3", + "country": "IN", + "party": "P-2" + } + ], + "interactions": [ + { + "mode": "x_as_a_service", + "from": "T-1", + "to": "T-2", + "id": "I-1", + "purpose": "Customer platform team consumes shared operational services and platforms provided by the external provider." + }, + { + "to": "T-1", + "from": "T-3", + "mode": "facilitating", + "purpose": "Provider enablement team assists the customer with adopting foundational deployment pipelines.", + "id": "I-2", + "until": "Adoption of foundational pipelines is complete and customer team is self-sufficient." + } + ], + "measures": [ + { + "layer": "outcome", + "name": "Deployment frequency", + "framework": "dora-metrics", + "outcome": "BO-1", + "measured_by": "C-19", + "id": "M-1", + "target": "A change reaches production within a day of approval." + }, + { + "target": "Restored within 4 hours.", + "outcome": "BO-2", + "layer": "outcome", + "measured_by": "C-19", + "framework": "dora-metrics", + "name": "Time to restore service", + "id": "M-2" + }, + { + "outcome": "BO-3", + "layer": "guardrail", + "measured_by": "C-19", + "name": "Satisfaction and well-being", + "id": "M-3", + "framework": "space-framework" + }, + { + "measured_by": "C-19", + "layer": "leading", + "id": "M-4", + "name": "Friction and cognitive load", + "framework": "devex-framework", + "outcome": "BO-3" + } + ], + "responsibilities": [ + { + "kind": "accountable", + "capability": "C-1", + "id": "RS-1A", + "role": "R-3" + }, + { + "kind": "responsible", + "role": "R-4", + "capability": "C-1", + "id": "RS-1R" + }, + { + "role": "R-2", + "kind": "consulted", + "capability": "C-1", + "id": "RS-1C" + }, + { + "capability": "C-2", + "kind": "accountable", + "id": "RS-2A", + "role": "R-3" + }, + { + "id": "RS-2R", + "capability": "C-2", + "role": "R-4", + "kind": "responsible" + }, + { + "kind": "consulted", + "capability": "C-2", + "id": "RS-2C", + "role": "R-2" + }, + { + "id": "RS-3A", + "role": "R-3", + "capability": "C-3", + "kind": "accountable" + }, + { + "capability": "C-3", + "kind": "responsible", + "id": "RS-3R", + "role": "R-6" + }, + { + "role": "R-2", + "id": "RS-3C", + "capability": "C-3", + "kind": "consulted" + }, + { + "role": "R-3", + "capability": "C-4", + "id": "RS-4A", + "kind": "accountable" + }, + { + "capability": "C-4", + "id": "RS-4R", + "role": "R-4", + "kind": "responsible" + }, + { + "capability": "C-4", + "role": "R-2", + "id": "RS-4C", + "kind": "consulted" + }, + { + "kind": "accountable", + "capability": "C-5", + "role": "R-1", + "id": "RS-5A" + }, + { + "role": "R-2", + "capability": "C-5", + "id": "RS-5R", + "kind": "responsible" + }, + { + "role": "R-6", + "capability": "C-5", + "kind": "consulted", + "id": "RS-5C" + }, + { + "id": "RS-6A", + "capability": "C-6", + "kind": "accountable", + "role": "R-3" + }, + { + "role": "R-6", + "capability": "C-6", + "id": "RS-6R", + "kind": "responsible" + }, + { + "capability": "C-6", + "kind": "consulted", + "id": "RS-6C", + "role": "R-2" + }, + { + "capability": "C-7", + "role": "R-3", + "id": "RS-7A", + "kind": "accountable" + }, + { + "id": "RS-7R", + "kind": "responsible", + "capability": "C-7", + "role": "R-6" + }, + { + "capability": "C-7", + "id": "RS-7C", + "kind": "consulted", + "role": "R-2" + }, + { + "role": "R-3", + "capability": "C-8", + "id": "RS-8A", + "kind": "accountable" + }, + { + "role": "R-6", + "id": "RS-8R", + "kind": "responsible", + "capability": "C-8" + }, + { + "role": "R-2", + "capability": "C-8", + "id": "RS-8C", + "kind": "consulted" + }, + { + "kind": "accountable", + "capability": "C-9", + "id": "RS-9A", + "role": "R-3" + }, + { + "capability": "C-9", + "role": "R-6", + "id": "RS-9R", + "kind": "responsible" + }, + { + "kind": "consulted", + "id": "RS-9C", + "role": "R-2", + "capability": "C-9" + }, + { + "role": "R-3", + "capability": "C-10", + "id": "RS-10A", + "kind": "accountable" + }, + { + "id": "RS-10R", + "role": "R-6", + "capability": "C-10", + "kind": "responsible" + }, + { + "role": "R-2", + "id": "RS-10C", + "capability": "C-10", + "kind": "consulted" + }, + { + "id": "RS-11A", + "kind": "accountable", + "role": "R-3", + "capability": "C-11" + }, + { + "role": "R-5", + "kind": "responsible", + "capability": "C-11", + "id": "RS-11R" + }, + { + "kind": "consulted", + "role": "R-2", + "id": "RS-11C", + "capability": "C-11" + }, + { + "role": "R-3", + "capability": "C-12", + "id": "RS-12A", + "kind": "accountable" + }, + { + "role": "R-6", + "id": "RS-12R", + "kind": "responsible", + "capability": "C-12" + }, + { + "capability": "C-12", + "id": "RS-12C", + "kind": "consulted", + "role": "R-2" + }, + { + "id": "RS-13A", + "capability": "C-13", + "kind": "accountable", + "role": "R-3" + }, + { + "id": "RS-13R", + "role": "R-5", + "kind": "responsible", + "capability": "C-13" + }, + { + "id": "RS-13C", + "role": "R-2", + "capability": "C-13", + "kind": "consulted" + }, + { + "kind": "accountable", + "id": "RS-14A", + "capability": "C-14", + "role": "R-3" + }, + { + "role": "R-6", + "capability": "C-14", + "id": "RS-14R", + "kind": "responsible" + }, + { + "kind": "consulted", + "capability": "C-14", + "role": "R-1", + "id": "RS-14C" + }, + { + "kind": "accountable", + "role": "R-3", + "capability": "C-15", + "id": "RS-15A" + }, + { + "role": "R-6", + "id": "RS-15R", + "kind": "responsible", + "capability": "C-15" + }, + { + "kind": "consulted", + "role": "R-1", + "id": "RS-15C", + "capability": "C-15" + }, + { + "id": "RS-16A", + "capability": "C-16", + "role": "R-1", + "kind": "accountable" + }, + { + "id": "RS-16R", + "kind": "responsible", + "capability": "C-16", + "role": "R-1" + }, + { + "id": "RS-17A", + "capability": "C-17", + "kind": "accountable", + "role": "R-3" + }, + { + "id": "RS-17R", + "capability": "C-17", + "kind": "responsible", + "role": "R-6" + }, + { + "capability": "C-17", + "role": "R-2", + "id": "RS-17C", + "kind": "consulted" + }, + { + "capability": "C-18", + "kind": "accountable", + "id": "RS-18A", + "role": "R-3" + }, + { + "role": "R-7", + "capability": "C-18", + "id": "RS-18R", + "kind": "responsible" + }, + { + "role": "R-2", + "capability": "C-18", + "id": "RS-18C", + "kind": "consulted" + }, + { + "role": "R-3", + "capability": "C-19", + "kind": "accountable", + "id": "RS-19A" + }, + { + "id": "RS-19R", + "kind": "responsible", + "capability": "C-19", + "role": "R-6" + }, + { + "id": "RS-19C", + "kind": "consulted", + "capability": "C-19", + "role": "R-1" + }, + { + "id": "RS-20A", + "role": "R-1", + "capability": "C-20", + "kind": "accountable" + }, + { + "capability": "C-20", + "id": "RS-20R", + "role": "R-1", + "kind": "responsible" + }, + { + "role": "R-3", + "id": "RS-21A", + "capability": "C-21", + "kind": "accountable" + }, + { + "id": "RS-21R", + "kind": "responsible", + "capability": "C-21", + "role": "R-6" + }, + { + "kind": "consulted", + "capability": "C-21", + "role": "R-2", + "id": "RS-21C" + } + ], + "checks": [ + { + "on": "C-1", + "id": "OC-1", + "condition": "A Priority 1 fault on the platform is acknowledged and responded to within 15 minutes, at any hour.", + "source": "OR-1" + }, + { + "on": "C-1", + "id": "OC-2", + "source": "OR-2", + "condition": "Normal platform operations are restored within 4 hours of a Priority 1 incident occurring." + }, + { + "id": "OC-3", + "on": "C-2", + "condition": "The hub network (S-1) is monitored to verify it maintains at least 99.9% availability.", + "source": "K-1" + }, + { + "id": "OC-4", + "source": "OR-4", + "condition": "Changes to the hub network (S-1) are blocked unless approved by the customer platform team.", + "on": "C-5" + }, + { + "source": "OR-5", + "id": "OC-5", + "on": "C-15", + "condition": "A reporting package detailing achieved service levels and operational costs is delivered to the customer monthly." + }, + { + "id": "OC-6", + "condition": "Platform foundation operations are governed under a shared sourcing model.", + "source": "OR-6", + "on": "C-20" + }, + { + "source": "K-2", + "condition": "Access controls prevent audit logs in the audit log store (S-2) from being deleted for at least seven years.", + "id": "OC-7", + "on": "C-11" + }, + { + "condition": "Evidence is collected to demonstrate cryptographic encryption keys (S-4) are rotated at least once every 90 days.", + "id": "OC-8", + "source": "K-3", + "on": "C-21" + }, + { + "condition": "The Operations Lead role sits in Poland (PL).", + "source": "OR-7", + "id": "OC-9", + "on": "R-3" + }, + { + "on": "R-4", + "id": "OC-10", + "source": "OR-7", + "condition": "The Operations Engineer role sits in India (IN)." + }, + { + "condition": "The Security Operations Engineer role sits in India (IN).", + "source": "OR-7", + "on": "R-5", + "id": "OC-11" + }, + { + "condition": "The Senior SRE role sits in India (IN).", + "on": "R-6", + "id": "OC-12", + "source": "OR-7" + }, + { + "condition": "The Enablement Engineer role sits in India (IN).", + "source": "OR-7", + "id": "OC-13", + "on": "R-7" + }, + { + "condition": "The Platform Lead role sits in the United Kingdom (GB).", + "id": "OC-14", + "on": "R-1", + "source": "OR-7" + }, + { + "on": "R-2", + "condition": "The Platform Engineer role sits in the United Kingdom (GB).", + "id": "OC-15", + "source": "OR-7" + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/verdicts.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/verdicts.json new file mode 100644 index 0000000..3ca9189 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2/verdicts.json @@ -0,0 +1,149 @@ +{ + "dry_run": { + "verdict": "pass", + "evidence": "13 of 13 scenario walk-throughs pass." + }, + "checks": [ + { + "check": "OC-1", + "evidence": "Capability C-1 covers incident management and operates 24x7. Role R-4 (Operations Engineer) is responsible and operates 24x7.", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "Capability C-1 operates 24x7 and responds to incidents.", + "check": "OC-2" + }, + { + "check": "OC-3", + "verdict": "pass", + "evidence": "Capability C-2 covers monitoring and operates 24x7, with Role R-4 responsible. The operating model configures native Google Cloud monitoring tools to fulfill this." + }, + { + "check": "OC-4", + "evidence": "Capability C-5 covers change enablement. Role R-1 (Platform Lead from the customer team) is accountable and Role R-2 (Platform Engineer from the customer team) is responsible, giving the customer final approval.", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "Capability C-15 covers service level management. Role R-6 (Senior SRE) is responsible for producing the monthly reporting package.", + "check": "OC-5" + }, + { + "evidence": "Capability C-20 covers supplier management to govern the shared sourcing model. Role R-1 (Platform Lead) is accountable and responsible.", + "verdict": "pass", + "check": "OC-6" + }, + { + "evidence": "Capability C-11 covers access management. Role R-5 (Security Operations Engineer) is responsible.", + "verdict": "pass", + "check": "OC-7" + }, + { + "verdict": "pass", + "check": "OC-8", + "evidence": "Capability C-21 covers regulatory evidence collection. Role R-6 (Senior SRE) is responsible." + }, + { + "check": "OC-9", + "verdict": "pass", + "evidence": "Role R-3 (Operations Lead) is defined with country 'PL' (Poland)." + }, + { + "evidence": "Role R-4 (Operations Engineer) is defined with country 'IN' (India).", + "verdict": "pass", + "check": "OC-10" + }, + { + "verdict": "pass", + "evidence": "Role R-5 (Security Operations Engineer) is defined with country 'IN' (India).", + "check": "OC-11" + }, + { + "verdict": "pass", + "evidence": "Role R-6 (Senior SRE) is defined with country 'IN' (India).", + "check": "OC-12" + }, + { + "check": "OC-13", + "verdict": "pass", + "evidence": "Role R-7 (Enablement Engineer) is defined with country 'IN' (India)." + }, + { + "verdict": "pass", + "check": "OC-14", + "evidence": "Role R-1 (Platform Lead) is defined with country 'GB' (United Kingdom)." + }, + { + "verdict": "pass", + "evidence": "Role R-2 (Platform Engineer) is defined with country 'GB' (United Kingdom).", + "check": "OC-15" + }, + { + "check": "SC-fault-S-1", + "verdict": "pass", + "evidence": "Hub network and interconnect (S-1) fails at an hour faults are responded to (24x7). Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "check": "SC-change-S-1", + "verdict": "pass", + "evidence": "Hub network and interconnect (S-1) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-2", + "verdict": "pass", + "evidence": "Audit log store (S-2) fails at an hour faults are responded to (24x7). Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "check": "SC-change-S-2", + "verdict": "pass", + "evidence": "Audit log store (S-2) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-3", + "verdict": "pass", + "evidence": "Deployment pipeline (S-3) fails at an hour faults are responded to (24x7). Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "check": "SC-change-S-3", + "verdict": "pass", + "evidence": "Deployment pipeline (S-3) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-4", + "verdict": "pass", + "evidence": "Encryption keys (S-4) fails at an hour faults are responded to (24x7). Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "check": "SC-change-S-4", + "verdict": "pass", + "evidence": "Encryption keys (S-4) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-condition-K-1", + "verdict": "pass", + "evidence": "Condition K-1 must be shown to hold: The platform is available at least 99.9% of each month. Check OC-3 on C-2 keeps it true." + }, + { + "check": "SC-condition-K-2", + "verdict": "pass", + "evidence": "Condition K-2 must be shown to hold: Audit logs cannot be deleted for seven years. Check OC-7 on C-11 keeps it true." + }, + { + "check": "SC-condition-K-3", + "verdict": "pass", + "evidence": "Condition K-3 must be shown to hold: Encryption keys are rotated every 90 days. Check OC-8 on C-21 keeps it true." + }, + { + "check": "SC-party-P-2", + "verdict": "pass", + "evidence": "External service provider cannot be reached. Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "check": "SC-budget", + "verdict": "pass", + "evidence": "A month's cost of operating is compared with the budget of 60,000 GBP. The cost is 44,720 GBP, within the budget." + } + ], + "proposed_additions": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/capabilities.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/capabilities.json new file mode 100644 index 0000000..9e5a6c0 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/capabilities.json @@ -0,0 +1,191 @@ +[ + { + "id": "C-1", + "catalogue": "incident-management", + "serves": ["OR-1", "OR-2"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "24x7", + "effort_hours_per_month": 30, + "basis": "Triaging alerts and handling an estimated 2-4 incidents per month across the 4 platform items." + }, + { + "id": "C-2", + "catalogue": "monitoring-and-alerting", + "serves": ["OR-1", "OR-3"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "24x7", + "effort_hours_per_month": 20, + "basis": "Maintaining monitoring configurations and tuning alerts for the 4 Items continuously." + }, + { + "id": "C-3", + "catalogue": "problem-management", + "serves": ["OR-2"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Conducting root cause analysis and post-incident reviews for major faults." + }, + { + "id": "C-4", + "catalogue": "service-request-fulfilment", + "serves": ["OR-6"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Fulfilling routine non-fault requests such as access provisioning or adjusting limits." + }, + { + "id": "C-5", + "catalogue": "change-enablement", + "serves": ["OR-4", "OR-10"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Reviewing change requests and coordinating network change approvals with the customer." + }, + { + "id": "C-6", + "catalogue": "release-and-deployment", + "serves": ["OR-10"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Overseeing deployment pipelines to ensure approved changes reach production within a day." + }, + { + "id": "C-7", + "catalogue": "configuration-management", + "serves": ["OR-8", "OR-9"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Auditing configuration drift, managing key rotation schedules, and maintaining audit log retention settings." + }, + { + "id": "C-8", + "catalogue": "backup-and-recovery", + "serves": ["OR-2", "OR-8"], + "items": ["S-2", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Maintaining backups of audit logs and encryption keys, and testing restore procedures." + }, + { + "id": "C-9", + "catalogue": "service-continuity", + "serves": ["OR-2", "OR-3"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Planning and periodic rehearsal of disaster recovery procedures for the platform." + }, + { + "id": "C-10", + "catalogue": "capacity-and-performance", + "serves": ["OR-3"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Reviewing capacity trends for interconnect and storage to assure 99.9% availability." + }, + { + "id": "C-11", + "catalogue": "access-management", + "serves": ["OR-6", "OR-7"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Managing identity and access boundaries between provider and customer in the specified delivery countries." + }, + { + "id": "C-12", + "catalogue": "vulnerability-and-patch-management", + "serves": ["OR-9"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Regular patching and vulnerability remediation of the pipeline and infrastructure components." + }, + { + "id": "C-13", + "catalogue": "security-monitoring-and-response", + "serves": ["OR-1", "OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "24x7", + "effort_hours_per_month": 40, + "basis": "Continuous monitoring of security events and continuous analysis of audit logs." + }, + { + "id": "C-14", + "catalogue": "cost-management", + "serves": ["OR-5"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Tracking run rate against budget and reporting costs monthly." + }, + { + "id": "C-15", + "catalogue": "service-level-management", + "serves": ["OR-3", "OR-5"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Gathering metrics on 99.9% availability and compiling the monthly operational report." + }, + { + "id": "C-16", + "catalogue": "platform-product-management", + "serves": ["OR-6"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Managing the platform's roadmap and backlog based on feedback from the customer platform team." + }, + { + "id": "C-17", + "catalogue": "self-service-and-golden-paths", + "serves": ["OR-10"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Building automated self-service paths and maintaining fast-flow deployment pipelines." + }, + { + "id": "C-18", + "catalogue": "enablement", + "serves": ["OR-10"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Assisting customer engineers with adopting golden paths and best practices." + }, + { + "id": "C-19", + "catalogue": "delivery-and-developer-experience-measurement", + "serves": ["OR-10"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Measuring DORA delivery metrics, developer productivity guardrails, and leading flow signals." + }, + { + "id": "C-20", + "catalogue": "supplier-management", + "serves": ["OR-6", "OR-7"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Coordinating responsibilities and hand-offs across shared delivery borders and governance locations." + }, + { + "id": "C-21", + "catalogue": "regulatory-evidence", + "serves": ["OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Gathering and retaining evidence to meet compliance requirements such as EU DORA." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/execution.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/execution.json new file mode 100644 index 0000000..74dac51 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/execution.json @@ -0,0 +1,113 @@ +{ + "customer_case": "bench-landing-zone-shared", + "execution_id": "case-01-landing-zone-shared-r3", + "session_id": "d0d953a6-5254-4646-8d70-7fd6139f1afc", + "intent_version": 3, + "workflow_request": "We want our landing zone run as a shared service from go-live. A provider should\nrespond to faults at every hour, and our own platform team keeps the final say\nover changes to the network.", + "baseline": null, + "overlay_skills": {}, + "options": {}, + "sources": {}, + "scope": { + "status": "BASELINE_LOCKED", + "readiness": { + "supplied": 20, + "answered": 0, + "assumed": 3, + "unknown": 10, + "not_applicable": 0 + } + }, + "status": "completed", + "warnings": [], + "risks": [], + "assessment": null, + "phases": { + "scoping": "approved", + "operational_requirements": "approved", + "operating_model_design": "approved", + "validation": "approved", + "packaging": "approved" + }, + "final_document": "operating-model-guide.md", + "changes": null, + "deliverables": { + "operating_scope": { + "document": "deliverables/operating_scope.md", + "attachments": [ + "scope.json", + "scope.yaml" + ], + "status": "submitted" + }, + "operational_requirements": { + "document": "deliverables/operational_requirements.md", + "attachments": [ + "operational-requirements.json" + ], + "status": "submitted" + }, + "capability_model": { + "document": "deliverables/capability_model.md", + "attachments": [ + "capabilities.json" + ], + "status": "submitted" + }, + "target_operating_model": { + "document": "deliverables/target_operating_model.md", + "attachments": [ + "findings.json", + "organisation.mmd", + "staffing.json", + "target-operating-model.json" + ], + "status": "submitted" + }, + "validation_results": { + "document": "deliverables/validation_results.md", + "attachments": [ + "scenarios.json", + "verdicts.json" + ], + "status": "submitted" + }, + "operating_model_guide": { + "document": "deliverables/operating_model_guide.md", + "attachments": [], + "status": "submitted" + } + }, + "files": [ + "capabilities.json", + "deliverables/capability_model.md", + "deliverables/operating_model_guide.md", + "deliverables/operating_scope.md", + "deliverables/operational_requirements.md", + "deliverables/target_operating_model.md", + "deliverables/validation_results.md", + "findings.json", + "operating-model-guide.md", + "operational-requirements.json", + "organisation.mmd", + "scenarios.json", + "scope.json", + "scope.yaml", + "staffing.json", + "target-operating-model.json", + "verdicts.json" + ], + "staffing": { + "people": 11.4, + "cost_per_month": 47603, + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + } + }, + "findings": [], + "change_request": null, + "capability_sources_verified": false +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/findings.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/findings.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/findings.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/operating-model-guide.md b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/operating-model-guide.md new file mode 100644 index 0000000..8702327 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/operating-model-guide.md @@ -0,0 +1,237 @@ +# Operating model guide: Google Cloud landing zone platform foundation + +## Summary + +This guide sets out the target operating model for the Google Cloud landing zone platform foundation, covering the hub network and interconnect, audit log store, deployment pipeline, and encryption keys. Operational responsibility is shared between the customer and an external service provider across the United Kingdom, Poland, and India. The customer platform team steers platform product direction, developer flow measurement, and retains approval authority over all network changes, while the service provider operations team runs day-to-day operations, maintenance, and incident management. Fault response and platform availability are maintained around the clock (24x7), while change governance, service continuity, and lifecycle management are conducted during business hours. + +Readers should look first at the findings and open questions, particularly the catalogue assumptions regarding cloud-native operations tooling, change pipeline procedures, and the exclusion of workload applications, as well as the draft status of the rate card. All scenario walk-throughs and validation checks pass successfully within the allocated operational budget. + +### Key figures + +- Operated: 4 Items, with 3 Conditions to keep true. +- Faults responded to: 24x7 hours. +- Capabilities: 21. +- People: 11.4. Cost a month: 47,603 GBP, from rate card version `draft-2026-10-13` (placeholder prices: an illustration). +- Scenario walk-throughs: 13 of 13 pass. +- Business outcomes: 3, 3 shown by a Measure. +- Findings for whoever owns the design: 0. + +## Business outcomes + +**BO-1: New payment products reach customers in weeks rather than months.** Target: A change reaches production within a day of approval.. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Deployment frequency | outcome | dora-metrics | A change reaches production within a day of approval. | Delivery and developer experience measurement | Platform Product Manager | +| Friction and cognitive load | leading | devex-framework | | Delivery and developer experience measurement | Platform Product Manager | + +**BO-2: Payments are never unavailable for long.** Target: Restored within 4 hours.. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Time to restore service | outcome | dora-metrics | Restored within 4 hours. | Incident management | Operations Lead | + +**BO-3: We keep the platform engineers we hire.** + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Developer satisfaction | guardrail | space-framework | | Delivery and developer experience measurement | Platform Product Manager | + +## What is operated + +| Item | What it is | Kind | Product | From | +| --- | --- | --- | --- | --- | +| S-1 | Hub network and interconnect | network | Cloud Interconnect | | +| S-2 | Audit log store | storage | Cloud Storage | | +| S-3 | Deployment pipeline | pipeline | Cloud Build | | +| S-4 | Encryption keys | keys | Cloud KMS | | + +What must be kept true while it is run: + +| Condition | On | What must hold | From | +| --- | --- | --- | --- | +| K-1 | S-1 | The platform is available at least 99.9% of each month. | | +| K-2 | S-2 | Audit logs cannot be deleted for seven years. | | +| K-3 | S-4 | Encryption keys are rotated every 90 days. | | + +## Operational requirements + +| Requirement | Kind | What must be achieved | From | +| --- | --- | --- | --- | +| OR-1 | service_level | A fault on any Item is acknowledged within 15 minutes, at any hour. | service_levels.priority_1_response_minutes, operations.support_hours | +| OR-2 | service_level | Service is restored within 4 hours of a major failure. | service_levels.recovery_time_hours, BO-2 | +| OR-3 | service_level | The platform is available at least 99.9% of each month. | K-1 | +| OR-4 | process | Every change to the network requires customer platform team approval prior to implementation. | REQ-01 | +| OR-5 | process | A report of service levels and operating costs is provided monthly. | REQ-02 | +| OR-6 | organisation | The platform foundation is operated under a shared service model between the customer and an external service provider. | sourcing.model | +| OR-7 | organisation | Operational delivery is performed exclusively from the United Kingdom, Poland, and India. | sourcing.delivery_countries | +| OR-8 | compliance | Audit logs cannot be deleted for seven years. | K-2 | +| OR-9 | process | Encryption keys are rotated every 90 days. | K-3 | +| OR-10 | service_level | An approved change reaches production within one day of approval. | BO-1 | + +## Capabilities + +| Capability | What it is | Serves | Covers | Hours | Effort a month | +| --- | --- | --- | --- | --- | --- | +| C-1 | Incident management | OR-1, OR-2 | S-1, S-2, S-3, S-4 | 24x7 | 30 h | +| C-2 | Monitoring and alerting | OR-1, OR-3 | S-1, S-2, S-3, S-4 | 24x7 | 20 h | +| C-3 | Problem management | OR-2 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-4 | Service request fulfilment | OR-6 | S-1, S-2, S-3, S-4 | business | 20 h | +| C-5 | Change enablement | OR-4, OR-10 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-6 | Release and deployment | OR-10 | S-1, S-2, S-3, S-4 | business | 20 h | +| C-7 | Configuration management | OR-8, OR-9 | S-1, S-2, S-3, S-4 | business | 15 h | +| C-8 | Backup and recovery | OR-2, OR-8 | S-2, S-4 | business | 10 h | +| C-9 | Service continuity | OR-2, OR-3 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-10 | Capacity and performance management | OR-3 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-11 | Access management | OR-6, OR-7 | S-1, S-2, S-3, S-4 | business | 15 h | +| C-12 | Vulnerability and patch management | OR-9 | S-1, S-2, S-3, S-4 | business | 15 h | +| C-13 | Security monitoring and response | OR-1, OR-8 | S-1, S-2, S-3, S-4 | 24x7 | 40 h | +| C-14 | Cost management | OR-5 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-15 | Service level management | OR-3, OR-5 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-16 | Platform product management | OR-6 | S-1, S-2, S-3, S-4 | business | 15 h | +| C-17 | Self-service and golden paths | OR-10 | S-1, S-2, S-3, S-4 | business | 20 h | +| C-18 | Enablement | OR-10 | S-1, S-2, S-3, S-4 | business | 20 h | +| C-19 | Delivery and developer experience measurement | OR-10 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-20 | Supplier management | OR-6, OR-7 | S-1, S-2, S-3, S-4 | business | 10 h | +| C-21 | Regulatory evidence and reporting | OR-8 | S-1, S-2, S-3, S-4 | business | 10 h | + +## Who does what + +| Role | Party | Country | Level | Hours | +| --- | --- | --- | --- | --- | +| Platform Product Manager | Customer (customer) | GB | lead | business | +| Platform Engineer | Customer (customer) | GB | senior | business | +| Operations Engineer | Service Provider (provider) | IN | engineer | 24x7 | +| Operations Lead | Service Provider (provider) | IN | lead | business | +| Security Operations Analyst | Service Provider (provider) | IN | senior | 24x7 | +| Platform Ops Specialist | Service Provider (provider) | PL | senior | business | + +The teams, as Team Topologies names their kinds: + +| Team | Kind | Party | Roles | Purpose | +| --- | --- | --- | --- | --- | +| Customer Platform Team | platform | Customer | Platform Product Manager, Platform Engineer | Owns the platform roadmap, retains network change approvals, and measures developer flow. | +| Provider Operations Team | platform | Service Provider | Operations Engineer, Operations Lead, Security Operations Analyst, Platform Ops Specialist | Provides 24x7 monitoring, incident response, and day-to-day operations of the platform foundation. | + +How the teams work together: + +| From | To | Mode | Until | Purpose | +| --- | --- | --- | --- | --- | +| Provider Operations Team | Customer Platform Team | x-as-a-service | | Provider requests network change approvals from the Customer Platform Team. | +| Customer Platform Team | Provider Operations Team | x-as-a-service | | Customer relies on Provider Operations Team for 24x7 incident response and monitoring. | +| Provider Operations Team | Customer Platform Team | collaboration | Month 3 after go-live, shifting to x-as-a-service. | Handover of runbooks and initial operational processes. | + +| Capability | Platform Product Manager | Platform Engineer | Operations Engineer | Operations Lead | Security Operations Analyst | Platform Ops Specialist | +| --- | --- | --- | --- | --- | --- | --- | +| Incident management (C-1) | | C | R | A | | | +| Monitoring and alerting (C-2) | | C | R | A | | | +| Problem management (C-3) | | C | | A | | R | +| Service request fulfilment (C-4) | | C | R | A | | | +| Change enablement (C-5) | | A/R | | | | R | +| Release and deployment (C-6) | | A/R | | | | R | +| Configuration management (C-7) | | C | | A | | R | +| Backup and recovery (C-8) | | C | | A | | R | +| Service continuity (C-9) | | C | | A | | R | +| Capacity and performance management (C-10) | | C | | A/R | | | +| Access management (C-11) | | C | | A/R | | | +| Vulnerability and patch management (C-12) | | C | | | A/R | | +| Security monitoring and response (C-13) | | C | | | A/R | | +| Cost management (C-14) | C | | | A/R | | | +| Service level management (C-15) | C | | | A/R | | | +| Platform product management (C-16) | A/R | | | C | | | +| Self-service and golden paths (C-17) | | A/R | | | | R | +| Enablement (C-18) | | A/R | | | | R | +| Delivery and developer experience measurement (C-19) | A/R | | | C | | | +| Supplier management (C-20) | A/R | | | R | | | +| Regulatory evidence and reporting (C-21) | | C | | A | | R | + +A accountable, R responsible, C consulted, I informed. + +## Staffing and cost + +| Role | Party | Country | Level | Hours | Effort a month | People | Cost a month | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Platform Product Manager | Customer | GB | lead | business | 30.0 h | 0.2 | not priced | +| Platform Engineer | Customer | GB | senior | business | 35.0 h | 0.3 | not priced | +| Operations Engineer | Service Provider | IN | engineer | 24x7 | 70.0 h | 5.0 | 17,034 GBP | +| Operations Lead | Service Provider | IN | lead | business | 50.0 h | 0.3 | 1,876 GBP | +| Security Operations Analyst | Service Provider | IN | senior | 24x7 | 55.0 h | 5.0 | 24,605 GBP | +| Platform Ops Specialist | Service Provider | PL | senior | business | 90.0 h | 0.6 | 4,088 GBP | + +11.4 people in all. Cost a month: 47,603 GBP. +The monthly budget is 60,000 GBP: this is within it. + +Worked from rate card `rate-card`, version `draft-2026-10-13`. **Its figures are placeholders: no price has been set, and this cost is an illustration.** +Converted from USD at 0.79, a rate set on 2026-10-13. +The customer's own staff are counted and not priced. A Role that covers hours wider than a working day is never smaller than the team that can cover them. + +## Validation + +| Check | What must hold | Verdict | Evidence | +| --- | --- | --- | --- | +| OC-1 | A fault on the hub network or any other item is acknowledged within 15 minutes at any hour. | pass | C-1 handles incident management 24x7, with a 24x7 Operations Engineer responsible, enabling acknowledgement of faults within 15 minutes at any hour. The cloud provider's native tooling (via C-2) supports configuring alerts without requiring a separate finding. | +| OC-2 | Service is restored within 4 hours of a major failure on the platform. | pass | C-1 manages incident resolution with a 24x7 Operations Engineer responsible, allowing restoration of service within 4 hours of a major failure regardless of when the failure occurs. | +| OC-3 | The platform availability is tracked to ensure at least 99.9% uptime each month. | pass | C-15 tracks and gathers metrics on availability to ensure the 99.9% target is tracked, with business-hours roles accountable and responsible. | +| OC-4 | Capacity limits for the platform interconnect are managed to support 99.9% availability. | pass | C-10 reviews capacity trends for the interconnect to assure 99.9% availability, managed by the Operations Lead. | +| OC-5 | Every change to the hub network requires explicit approval from the customer platform team. | pass | C-5 coordinates network change approvals with the customer, and a Customer Platform Engineer is accountable and responsible for it, ensuring explicit approval from the customer platform team. | +| OC-6 | A consolidated monthly report includes service levels and operating costs. | pass | C-15 compiles the monthly operational report, pulling in cost data tracked by C-14, both managed by the Operations Lead. | +| OC-7 | The shared service model responsibilities between customer and external service provider are governed. | pass | C-20 coordinates responsibilities across shared delivery borders, with roles from both the customer and provider teams responsible. | +| OC-8a | Role performs operational delivery from India. | pass | R-3 (Operations Engineer) is based in India, satisfying the geographical requirement. | +| OC-8b | Role performs operational delivery from India. | pass | R-4 (Operations Lead) is based in India, satisfying the geographical requirement. | +| OC-8c | Role performs operational delivery from India. | pass | R-5 (Security Operations Analyst) is based in India, satisfying the geographical requirement. | +| OC-8d | Role performs operational delivery from Poland. | pass | R-6 (Platform Ops Specialist) is based in Poland, satisfying the geographical requirement. | +| OC-8e | Role performs operational delivery from the UK. | pass | R-1 (Platform Product Manager) is based in the UK, satisfying the geographical requirement. | +| OC-8f | Role performs operational delivery from the UK. | pass | R-2 (Platform Engineer) is based in the UK, satisfying the geographical requirement. | +| OC-9 | Audit log storage configuration ensures logs cannot be deleted for seven years. | pass | C-7 maintains audit log retention settings, with the Platform Ops Specialist responsible, ensuring logs cannot be deleted for seven years. | +| OC-10 | Retention policies on Cloud Storage for audit logs are verified to prevent deletion for seven years. | pass | C-7 maintains and verifies retention settings on Cloud Storage for audit logs, satisfying the seven-year retention condition. | +| OC-11 | Encryption key configurations are audited to ensure rotation every 90 days. | pass | C-7 audits configuration drift and manages key rotation schedules, ensuring encryption keys are rotated every 90 days. | +| OC-12 | Cloud KMS encryption keys are verified to rotate every 90 days. | pass | C-7 manages key rotation schedules for the Cloud KMS encryption keys, verifying they rotate every 90 days. | +| OC-13 | Deployment pipelines ensure an approved change reaches production within one day. | pass | C-6 oversees deployment pipelines to ensure approved changes reach production within a day, with customer and provider roles responsible. | +| OC-14 | Backups of audit logs and encryption keys are successfully tested for restoration. | pass | C-8 maintains backups of logs and keys and successfully tests restore procedures, managed by the Platform Ops Specialist. | + +Each scenario is generated from the operating model and traced through it by code. + +| Scenario | Result | Path | +| --- | --- | --- | +| Hub network and interconnect (S-1) fails at an hour faults are responded to (24x7). | pass | Operations Engineer responds under C-1, and Operations Lead is answerable. | +| Hub network and interconnect (S-1) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Audit log store (S-2) fails at an hour faults are responded to (24x7). | pass | Operations Engineer responds under C-1, and Operations Lead is answerable. | +| Audit log store (S-2) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Deployment pipeline (S-3) fails at an hour faults are responded to (24x7). | pass | Operations Engineer responds under C-1, and Operations Lead is answerable. | +| Deployment pipeline (S-3) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Encryption keys (S-4) fails at an hour faults are responded to (24x7). | pass | Operations Engineer responds under C-1, and Operations Lead is answerable. | +| Encryption keys (S-4) must be changed. | pass | Platform Engineer makes the change under C-5. | +| Condition K-1 must be shown to hold: The platform is available at least 99.9% of each month. | pass | Check OC-4 on C-10 keeps it true. | +| Condition K-2 must be shown to hold: Audit logs cannot be deleted for seven years. | pass | Check OC-10 on C-7 keeps it true. | +| Condition K-3 must be shown to hold: Encryption keys are rotated every 90 days. | pass | Check OC-12 on C-7 keeps it true. | +| Service Provider cannot be reached. | pass | Every Capability it is answerable for has a Role in another Party to turn to. | +| A month's cost of operating is compared with the budget of 60,000 GBP. | pass | The cost is 47,603 GBP, within the budget. | + +## Findings and open questions + +No findings: the solution provides what operating it needs. + +Assumed for the customer, and open to change when they answer: + +- `intent.exclusions`: Workload applications running on the platform foundation, Network change approvals (retained by customer platform team). Assumed from the platform foundation scope: workload applications running on the platform are out of scope, and customer platform team retains network change approvals. +- `operations.change_process`: Changes go through a deployment pipeline with peer review, with network changes requiring customer platform team approval.. Catalogue default adapted to customer requirement REQ-01: changes go through deployment pipelines with peer review, with customer platform team retaining approval over network changes. +- `service_management.tooling`: The Cloud provider's own operations tooling, with no separate system.. Catalogue default: Cloud provider's native operations tooling with no separate system. + +## Sources + +The Capabilities are from our own capability catalogue, which draws on these frameworks. It cites them and certifies nothing against them. + +- ITIL 4: Incident management practice; Monitoring and event management practice; Problem management practice; Service request management practice; Change enablement practice; Release management and Deployment management practices; Service configuration management practice; Service continuity management practice; Capacity and performance management practice; Service level management practice; Supplier management practice +- Site Reliability Engineering: How Google Runs Production Systems: Managing Incidents; Monitoring Distributed Systems; Postmortem Culture: Learning from Failure; Release Engineering; Service Level Objectives +- ISO/IEC 20000-1: Incident management; Service request management; Change management; Service continuity management; Capacity management; Service level management; Supplier management +- Regulation (EU) 2022/2554, the Digital Operational Resilience Act: Article 17, ICT-related incident management process; Article 10, Detection; Article 12, Backup policies and procedures, restoration and recovery procedures and methods; Article 11, Response and recovery; Article 9, Protection and prevention; Article 28, General principles on ICT third-party risk; Article 19, Reporting of major ICT-related incidents +- NIST Cybersecurity Framework: Detect: Continuous Monitoring; Identify: Asset Management; Recover: Incident Recovery Plan Execution; Protect: Identity Management, Authentication, and Access Control; Protect: Platform Security; Detect: Adverse Event Analysis; Respond: Incident Management +- DevOps Research and Assessment (DORA) software delivery metrics: Change failure rate; Deployment frequency and lead time for changes; Deployment frequency; Lead time for changes; Change failure rate; Time to restore service +- ISO/IEC 27001: Access control; Management of technical vulnerabilities; Information security incident management; Compliance with legal and contractual requirements +- FinOps Framework: Allocation; Budgeting; Forecasting; Anomaly Management +- CNCF Platforms White Paper (platform as a product): Platform as a product; Measuring platform success; Platform capabilities; Self-service; Platform as a product +- Team Topologies: Organizing Business and Technology Teams for Fast Flow: Platform team; Thinnest viable platform; X-as-a-Service interaction mode; Enabling team; Facilitating interaction mode +- The SPACE of Developer Productivity: Satisfaction and well-being; Performance; Activity; Communication and collaboration; Efficiency and flow +- DevEx: What Actually Drives Productivity: Feedback loops; Cognitive load; Flow state + +**These references have not been checked against the frameworks by a person.** Treat each as a pointer to where to look. diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/operational-requirements.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/operational-requirements.json new file mode 100644 index 0000000..08d8280 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/operational-requirements.json @@ -0,0 +1,84 @@ +[ + { + "id": "OR-1", + "kind": "service_level", + "statement": "A fault on any Item is acknowledged within 15 minutes, at any hour.", + "source": [ + "service_levels.priority_1_response_minutes", + "operations.support_hours" + ] + }, + { + "id": "OR-2", + "kind": "service_level", + "statement": "Service is restored within 4 hours of a major failure.", + "source": [ + "service_levels.recovery_time_hours", + "BO-2" + ] + }, + { + "id": "OR-3", + "kind": "service_level", + "statement": "The platform is available at least 99.9% of each month.", + "source": [ + "K-1" + ] + }, + { + "id": "OR-4", + "kind": "process", + "statement": "Every change to the network requires customer platform team approval prior to implementation.", + "source": [ + "REQ-01" + ] + }, + { + "id": "OR-5", + "kind": "process", + "statement": "A report of service levels and operating costs is provided monthly.", + "source": [ + "REQ-02" + ] + }, + { + "id": "OR-6", + "kind": "organisation", + "statement": "The platform foundation is operated under a shared service model between the customer and an external service provider.", + "source": [ + "sourcing.model" + ] + }, + { + "id": "OR-7", + "kind": "organisation", + "statement": "Operational delivery is performed exclusively from the United Kingdom, Poland, and India.", + "source": [ + "sourcing.delivery_countries" + ] + }, + { + "id": "OR-8", + "kind": "compliance", + "statement": "Audit logs cannot be deleted for seven years.", + "source": [ + "K-2" + ] + }, + { + "id": "OR-9", + "kind": "process", + "statement": "Encryption keys are rotated every 90 days.", + "source": [ + "K-3" + ] + }, + { + "id": "OR-10", + "kind": "service_level", + "statement": "An approved change reaches production within one day of approval.", + "source": [ + "BO-1" + ] + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/organisation.mmd b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/organisation.mmd new file mode 100644 index 0000000..942de00 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/organisation.mmd @@ -0,0 +1,18 @@ +flowchart TB + subgraph P_1["Customer (customer)"] + subgraph T_1["Customer Platform Team (platform team)"] + R_1["Platform Product Manager
GB, lead, business"] + R_2["Platform Engineer
GB, senior, business"] + end + end + subgraph P_2["Service Provider (provider)"] + subgraph T_2["Provider Operations Team (platform team)"] + R_3["Operations Engineer
IN, engineer, 24x7"] + R_4["Operations Lead
IN, lead, business"] + R_5["Security Operations Analyst
IN, senior, 24x7"] + R_6["Platform Ops Specialist
PL, senior, business"] + end + end + T_2 -- "x-as-a-service" --> T_1 + T_1 -- "x-as-a-service" --> T_2 + T_2 -- "collaboration until Month 3 after go-live, shifting to x-as-a-service." --> T_1 diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scenarios.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scenarios.json new file mode 100644 index 0000000..9d2ffa9 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scenarios.json @@ -0,0 +1,93 @@ +[ + { + "id": "SC-fault-S-1", + "scenario": "Hub network and interconnect (S-1) fails at an hour faults are responded to (24x7).", + "about": "S-1", + "passes": true, + "path": "Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "id": "SC-change-S-1", + "scenario": "Hub network and interconnect (S-1) must be changed.", + "about": "S-1", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-2", + "scenario": "Audit log store (S-2) fails at an hour faults are responded to (24x7).", + "about": "S-2", + "passes": true, + "path": "Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "id": "SC-change-S-2", + "scenario": "Audit log store (S-2) must be changed.", + "about": "S-2", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-3", + "scenario": "Deployment pipeline (S-3) fails at an hour faults are responded to (24x7).", + "about": "S-3", + "passes": true, + "path": "Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "id": "SC-change-S-3", + "scenario": "Deployment pipeline (S-3) must be changed.", + "about": "S-3", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-4", + "scenario": "Encryption keys (S-4) fails at an hour faults are responded to (24x7).", + "about": "S-4", + "passes": true, + "path": "Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "id": "SC-change-S-4", + "scenario": "Encryption keys (S-4) must be changed.", + "about": "S-4", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-condition-K-1", + "scenario": "Condition K-1 must be shown to hold: The platform is available at least 99.9% of each month.", + "about": "K-1", + "passes": true, + "path": "Check OC-4 on C-10 keeps it true." + }, + { + "id": "SC-condition-K-2", + "scenario": "Condition K-2 must be shown to hold: Audit logs cannot be deleted for seven years.", + "about": "K-2", + "passes": true, + "path": "Check OC-10 on C-7 keeps it true." + }, + { + "id": "SC-condition-K-3", + "scenario": "Condition K-3 must be shown to hold: Encryption keys are rotated every 90 days.", + "about": "K-3", + "passes": true, + "path": "Check OC-12 on C-7 keeps it true." + }, + { + "id": "SC-party-P-2", + "scenario": "Service Provider cannot be reached.", + "about": "P-2", + "passes": true, + "path": "Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "id": "SC-budget", + "scenario": "A month's cost of operating is compared with the budget of 60,000 GBP.", + "about": "commercial.monthly_run_budget", + "passes": true, + "path": "The cost is 47,603 GBP, within the budget." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scope.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scope.json new file mode 100644 index 0000000..f62cffe --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scope.json @@ -0,0 +1,203 @@ +{ + "kind": "platform_foundation", + "stated": { + "engagement": { + "kind": "platform_foundation" + }, + "intent": { + "statement": "We want our landing zone run as a shared service from go-live. A provider should\nrespond to faults at every hour, and our own platform team keeps the final say\nover changes to the network.\n", + "requirements": [ + { + "id": "REQ-01", + "statement": "Our platform team approves every change to the network." + }, + { + "id": "REQ-02", + "statement": "We receive a monthly report of service levels and cost." + } + ], + "business_outcomes": [ + { + "id": "BO-1", + "outcome": "New payment products reach customers in weeks rather than months.", + "target": "A change reaches production within a day of approval." + }, + { + "id": "BO-2", + "outcome": "Payments are never unavailable for long.", + "target": "Restored within 4 hours." + }, + { + "id": "BO-3", + "outcome": "We keep the platform engineers we hire." + } + ] + }, + "solution": { + "cloud_provider": "gcp", + "items": [ + { + "id": "S-1", + "name": "Hub network and interconnect", + "kind": "network", + "product": "Cloud Interconnect" + }, + { + "id": "S-2", + "name": "Audit log store", + "kind": "storage", + "product": "Cloud Storage" + }, + { + "id": "S-3", + "name": "Deployment pipeline", + "kind": "pipeline", + "product": "Cloud Build" + }, + { + "id": "S-4", + "name": "Encryption keys", + "kind": "keys", + "product": "Cloud KMS" + } + ], + "conditions": [ + { + "id": "K-1", + "on": [ + "S-1" + ], + "condition": "The platform is available at least 99.9% of each month." + }, + { + "id": "K-2", + "on": [ + "S-2" + ], + "condition": "Audit logs cannot be deleted for seven years." + }, + { + "id": "K-3", + "on": [ + "S-4" + ], + "condition": "Encryption keys are rotated every 90 days." + } + ], + "build": { + "duration_days": 36 + } + }, + "operations": { + "support_hours": "24x7", + "availability_hours": "24x7" + }, + "sourcing": { + "model": "shared", + "delivery_countries": [ + "GB", + "PL", + "IN" + ] + }, + "organisation": { + "existing_teams": [ + { + "name": "Platform team", + "party": "customer", + "hours": "business", + "country": "GB" + } + ] + }, + "service_levels": { + "priority_1_response_minutes": 15, + "recovery_time_hours": 4 + }, + "compliance": { + "frameworks": [ + "DORA" + ] + }, + "commercial": { + "currency": "GBP", + "monthly_run_budget": 60000 + } + }, + "binding": [ + "operations.support_hours", + "sourcing.model", + "sourcing.delivery_countries", + "service_levels.priority_1_response_minutes", + "service_levels.recovery_time_hours" + ], + "assumed": [ + { + "field": "intent.exclusions", + "value": [ + "Workload applications running on the platform foundation", + "Network change approvals (retained by customer platform team)" + ], + "why": "Assumed from the platform foundation scope: workload applications running on the platform are out of scope, and customer platform team retains network change approvals." + }, + { + "field": "operations.change_process", + "value": "Changes go through a deployment pipeline with peer review, with network changes requiring customer platform team approval.", + "why": "Catalogue default adapted to customer requirement REQ-01: changes go through deployment pipelines with peer review, with customer platform team retaining approval over network changes." + }, + { + "field": "service_management.tooling", + "value": "The Cloud provider's own operations tooling, with no separate system.", + "why": "Catalogue default: Cloud provider's native operations tooling with no separate system." + } + ], + "to_decide": [], + "to_ask": [ + { + "field": "metadata.client_name", + "prompt": "What is metadata.client_name?", + "severity": "LOW" + }, + { + "field": "metadata.author", + "prompt": "What is metadata.author?", + "severity": "LOW" + }, + { + "field": "metadata.revision", + "prompt": "What is metadata.revision?", + "severity": "LOW" + }, + { + "field": "metadata.source", + "prompt": "What is metadata.source?", + "severity": "LOW" + }, + { + "field": "solution.build.customer_owned", + "prompt": "Which parts of the build does the customer or their supplier own?", + "severity": "LOW" + }, + { + "field": "solution.inherited_risks", + "prompt": "What is already known to be unverified or at risk in the solution?", + "severity": "LOW" + }, + { + "field": "operations.monitoring", + "prompt": "Which monitoring and alerting is already in place or required?", + "severity": "LOW" + }, + { + "field": "organisation.headcount_limit", + "prompt": "Is there a limit on the number of people operating this?", + "severity": "LOW" + }, + { + "field": "transition.go_live_date", + "prompt": "When must the solution be in operation?", + "severity": "LOW" + } + ], + "not_applicable": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scope.yaml new file mode 100644 index 0000000..906842f --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/scope.yaml @@ -0,0 +1,201 @@ +schema_version: 1 +metadata: + customer_case: bench-landing-zone-shared + client_name: ask + author: ask + revision: ask + source: ask + status: BASELINE_LOCKED +engagement: + kind: platform_foundation +intent: + statement: 'We want our landing zone run as a shared service from go-live. A provider + should + + respond to faults at every hour, and our own platform team keeps the final say + + over changes to the network. + + ' + requirements: + - id: REQ-01 + statement: Our platform team approves every change to the network. + - id: REQ-02 + statement: We receive a monthly report of service levels and cost. + business_outcomes: + - id: BO-1 + outcome: New payment products reach customers in weeks rather than months. + target: A change reaches production within a day of approval. + - id: BO-2 + outcome: Payments are never unavailable for long. + target: Restored within 4 hours. + - id: BO-3 + outcome: We keep the platform engineers we hire. + exclusions: + - Workload applications running on the platform foundation + - Network change approvals (retained by customer platform team) +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Hub network and interconnect + kind: network + product: Cloud Interconnect + - id: S-2 + name: Audit log store + kind: storage + product: Cloud Storage + - id: S-3 + name: Deployment pipeline + kind: pipeline + product: Cloud Build + - id: S-4 + name: Encryption keys + kind: keys + product: Cloud KMS + conditions: + - id: K-1 + 'on': + - S-1 + condition: The platform is available at least 99.9% of each month. + - id: K-2 + 'on': + - S-2 + condition: Audit logs cannot be deleted for seven years. + - id: K-3 + 'on': + - S-4 + condition: Encryption keys are rotated every 90 days. + build: + duration_days: 36 + customer_owned: ask + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Changes go through a deployment pipeline with peer review, with + network changes requiring customer platform team approval. + monitoring: ask +sourcing: + model: shared + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: + - name: Platform team + party: customer + hours: business + country: GB + headcount_limit: ask +service_management: + tooling: The Cloud provider's own operations tooling, with no separate system. +service_levels: + priority_1_response_minutes: 15 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA +commercial: + currency: GBP + monthly_run_budget: 60000 +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.exclusions + statement: 'Assumed intent.exclusions is [''Workload applications running on the + platform foundation'', ''Network change approvals (retained by customer platform + team)'']. Assumed from the platform foundation scope: workload applications running + on the platform are out of scope, and customer platform team retains network change + approvals.' + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-02 + field: operations.change_process + statement: 'Assumed operations.change_process is Changes go through a deployment + pipeline with peer review, with network changes requiring customer platform team + approval.. Catalogue default adapted to customer requirement REQ-01: changes go + through deployment pipelines with peer review, with customer platform team retaining + approval over network changes.' + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-03 + field: service_management.tooling + statement: 'Assumed service_management.tooling is The Cloud provider''s own operations + tooling, with no separate system.. Catalogue default: Cloud provider''s native + operations tooling with no separate system.' + impact_if_invalid: Changes tooling and the processes written for it. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: metadata.source + prompt: What is metadata.source? + severity: LOW +- field: solution.build.customer_owned + prompt: Which parts of the build does the customer or their supplier own? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: supplied + commercial.monthly_run_budget: supplied + compliance.frameworks: supplied + engagement.kind: supplied + intent.business_outcomes: supplied + intent.exclusions: assumed + intent.requirements: supplied + intent.statement: supplied + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: supplied + metadata.revision: unknown + metadata.source: unknown + operations.availability_hours: supplied + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: supplied + organisation.existing_teams: supplied + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: supplied + service_levels.recovery_time_hours: supplied + service_management.tooling: assumed + solution.build.customer_owned: unknown + solution.build.duration_days: supplied + solution.cloud_provider: supplied + solution.conditions: supplied + solution.inherited_risks: unknown + solution.items: supplied + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 20 + answered: 0 + assumed: 3 + unknown: 10 + not_applicable: 0 diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/staffing.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/staffing.json new file mode 100644 index 0000000..3b6f344 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/staffing.json @@ -0,0 +1,117 @@ +{ + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + }, + "conversion": { + "from": "USD", + "rate": 0.79, + "as_of": "2026-10-13" + }, + "roles": [ + { + "role": "R-1", + "name": "Platform Product Manager", + "party": "P-1", + "party_name": "Customer", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 30.0, + "people": 0.2, + "cost_per_month": null + }, + { + "role": "R-2", + "name": "Platform Engineer", + "party": "P-1", + "party_name": "Customer", + "country": "GB", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 35.0, + "people": 0.3, + "cost_per_month": null + }, + { + "role": "R-3", + "name": "Operations Engineer", + "party": "P-2", + "party_name": "Service Provider", + "country": "IN", + "level": "engineer", + "hours": "24x7", + "effort_hours_per_month": 70.0, + "people": 5.0, + "cost_per_month": 17034 + }, + { + "role": "R-4", + "name": "Operations Lead", + "party": "P-2", + "party_name": "Service Provider", + "country": "IN", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 50.0, + "people": 0.3, + "cost_per_month": 1876 + }, + { + "role": "R-5", + "name": "Security Operations Analyst", + "party": "P-2", + "party_name": "Service Provider", + "country": "IN", + "level": "senior", + "hours": "24x7", + "effort_hours_per_month": 55.0, + "people": 5.0, + "cost_per_month": 24605 + }, + { + "role": "R-6", + "name": "Platform Ops Specialist", + "party": "P-2", + "party_name": "Service Provider", + "country": "PL", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 90.0, + "people": 0.6, + "cost_per_month": 4088 + } + ], + "people": 11.4, + "cost_per_month": 47603, + "by_party": { + "P-2": 47603 + }, + "by_country": { + "IN": 43515, + "PL": 4088 + }, + "by_capability": { + "C-1": 7300, + "C-2": 4867, + "C-4": 4867, + "C-10": 375, + "C-11": 563, + "C-14": 375, + "C-15": 375, + "C-20": 188, + "C-12": 6711, + "C-13": 17895, + "C-3": 454, + "C-5": 227, + "C-6": 454, + "C-7": 681, + "C-8": 454, + "C-9": 454, + "C-17": 454, + "C-18": 454, + "C-21": 454 + } +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/target-operating-model.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/target-operating-model.json new file mode 100644 index 0000000..c488f77 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/target-operating-model.json @@ -0,0 +1,642 @@ +{ + "parties": [ + { + "kind": "customer", + "name": "Customer", + "id": "P-1" + }, + { + "kind": "provider", + "name": "Service Provider", + "id": "P-2" + } + ], + "teams": [ + { + "purpose": "Owns the platform roadmap, retains network change approvals, and measures developer flow.", + "id": "T-1", + "party": "P-1", + "type": "platform", + "name": "Customer Platform Team" + }, + { + "id": "T-2", + "name": "Provider Operations Team", + "type": "platform", + "party": "P-2", + "purpose": "Provides 24x7 monitoring, incident response, and day-to-day operations of the platform foundation." + } + ], + "roles": [ + { + "party": "P-1", + "id": "R-1", + "hours": "business", + "team": "T-1", + "name": "Platform Product Manager", + "level": "lead", + "country": "GB" + }, + { + "party": "P-1", + "hours": "business", + "team": "T-1", + "level": "senior", + "id": "R-2", + "name": "Platform Engineer", + "country": "GB" + }, + { + "level": "engineer", + "hours": "24x7", + "party": "P-2", + "name": "Operations Engineer", + "team": "T-2", + "id": "R-3", + "country": "IN" + }, + { + "country": "IN", + "level": "lead", + "id": "R-4", + "name": "Operations Lead", + "hours": "business", + "team": "T-2", + "party": "P-2" + }, + { + "level": "senior", + "party": "P-2", + "id": "R-5", + "hours": "24x7", + "name": "Security Operations Analyst", + "country": "IN", + "team": "T-2" + }, + { + "id": "R-6", + "name": "Platform Ops Specialist", + "country": "PL", + "level": "senior", + "hours": "business", + "team": "T-2", + "party": "P-2" + } + ], + "interactions": [ + { + "to": "T-1", + "mode": "x_as_a_service", + "from": "T-2", + "purpose": "Provider requests network change approvals from the Customer Platform Team.", + "id": "I-1" + }, + { + "id": "I-2", + "mode": "x_as_a_service", + "to": "T-2", + "purpose": "Customer relies on Provider Operations Team for 24x7 incident response and monitoring.", + "from": "T-1" + }, + { + "from": "T-2", + "to": "T-1", + "purpose": "Handover of runbooks and initial operational processes.", + "mode": "collaboration", + "until": "Month 3 after go-live, shifting to x-as-a-service.", + "id": "I-3" + } + ], + "measures": [ + { + "framework": "dora-metrics", + "name": "Deployment frequency", + "outcome": "BO-1", + "layer": "outcome", + "id": "M-1", + "target": "A change reaches production within a day of approval.", + "measured_by": "C-19" + }, + { + "outcome": "BO-3", + "id": "M-2", + "name": "Developer satisfaction", + "layer": "guardrail", + "measured_by": "C-19", + "framework": "space-framework" + }, + { + "measured_by": "C-19", + "outcome": "BO-1", + "framework": "devex-framework", + "name": "Friction and cognitive load", + "id": "M-3", + "layer": "leading" + }, + { + "layer": "outcome", + "outcome": "BO-2", + "measured_by": "C-1", + "target": "Restored within 4 hours.", + "name": "Time to restore service", + "framework": "dora-metrics", + "id": "M-4" + } + ], + "responsibilities": [ + { + "id": "RS-1-acc", + "capability": "C-1", + "kind": "accountable", + "role": "R-4" + }, + { + "id": "RS-1-res", + "kind": "responsible", + "role": "R-3", + "capability": "C-1" + }, + { + "id": "RS-1-con", + "capability": "C-1", + "role": "R-2", + "kind": "consulted" + }, + { + "id": "RS-2-acc", + "role": "R-4", + "capability": "C-2", + "kind": "accountable" + }, + { + "capability": "C-2", + "id": "RS-2-res", + "kind": "responsible", + "role": "R-3" + }, + { + "capability": "C-2", + "role": "R-2", + "kind": "consulted", + "id": "RS-2-con" + }, + { + "kind": "accountable", + "role": "R-4", + "capability": "C-3", + "id": "RS-3-acc" + }, + { + "role": "R-6", + "kind": "responsible", + "id": "RS-3-res", + "capability": "C-3" + }, + { + "capability": "C-3", + "role": "R-2", + "id": "RS-3-con", + "kind": "consulted" + }, + { + "role": "R-4", + "kind": "accountable", + "capability": "C-4", + "id": "RS-4-acc" + }, + { + "kind": "responsible", + "role": "R-3", + "capability": "C-4", + "id": "RS-4-res" + }, + { + "kind": "consulted", + "role": "R-2", + "id": "RS-4-con", + "capability": "C-4" + }, + { + "id": "RS-5-acc", + "role": "R-2", + "kind": "accountable", + "capability": "C-5" + }, + { + "role": "R-2", + "kind": "responsible", + "capability": "C-5", + "id": "RS-5-res1" + }, + { + "kind": "responsible", + "role": "R-6", + "id": "RS-5-res2", + "capability": "C-5" + }, + { + "capability": "C-6", + "role": "R-2", + "id": "RS-6-acc", + "kind": "accountable" + }, + { + "capability": "C-6", + "id": "RS-6-res1", + "kind": "responsible", + "role": "R-2" + }, + { + "role": "R-6", + "id": "RS-6-res2", + "kind": "responsible", + "capability": "C-6" + }, + { + "role": "R-4", + "id": "RS-7-acc", + "capability": "C-7", + "kind": "accountable" + }, + { + "kind": "responsible", + "capability": "C-7", + "id": "RS-7-res", + "role": "R-6" + }, + { + "capability": "C-7", + "role": "R-2", + "id": "RS-7-con", + "kind": "consulted" + }, + { + "kind": "accountable", + "role": "R-4", + "id": "RS-8-acc", + "capability": "C-8" + }, + { + "role": "R-6", + "capability": "C-8", + "id": "RS-8-res", + "kind": "responsible" + }, + { + "role": "R-2", + "capability": "C-8", + "kind": "consulted", + "id": "RS-8-con" + }, + { + "role": "R-4", + "id": "RS-9-acc", + "capability": "C-9", + "kind": "accountable" + }, + { + "kind": "responsible", + "role": "R-6", + "capability": "C-9", + "id": "RS-9-res" + }, + { + "id": "RS-9-con", + "role": "R-2", + "kind": "consulted", + "capability": "C-9" + }, + { + "id": "RS-10-acc", + "kind": "accountable", + "capability": "C-10", + "role": "R-4" + }, + { + "kind": "responsible", + "id": "RS-10-res", + "capability": "C-10", + "role": "R-4" + }, + { + "kind": "consulted", + "id": "RS-10-con", + "role": "R-2", + "capability": "C-10" + }, + { + "capability": "C-11", + "role": "R-4", + "id": "RS-11-acc", + "kind": "accountable" + }, + { + "capability": "C-11", + "role": "R-4", + "id": "RS-11-res", + "kind": "responsible" + }, + { + "kind": "consulted", + "capability": "C-11", + "id": "RS-11-con", + "role": "R-2" + }, + { + "kind": "accountable", + "id": "RS-12-acc", + "role": "R-5", + "capability": "C-12" + }, + { + "capability": "C-12", + "kind": "responsible", + "id": "RS-12-res", + "role": "R-5" + }, + { + "kind": "consulted", + "role": "R-2", + "capability": "C-12", + "id": "RS-12-con" + }, + { + "capability": "C-13", + "kind": "accountable", + "role": "R-5", + "id": "RS-13-acc" + }, + { + "id": "RS-13-res", + "capability": "C-13", + "kind": "responsible", + "role": "R-5" + }, + { + "id": "RS-13-con", + "capability": "C-13", + "role": "R-2", + "kind": "consulted" + }, + { + "kind": "accountable", + "role": "R-4", + "capability": "C-14", + "id": "RS-14-acc" + }, + { + "id": "RS-14-res", + "capability": "C-14", + "role": "R-4", + "kind": "responsible" + }, + { + "kind": "consulted", + "role": "R-1", + "capability": "C-14", + "id": "RS-14-con" + }, + { + "capability": "C-15", + "kind": "accountable", + "id": "RS-15-acc", + "role": "R-4" + }, + { + "id": "RS-15-res", + "kind": "responsible", + "capability": "C-15", + "role": "R-4" + }, + { + "capability": "C-15", + "id": "RS-15-con", + "role": "R-1", + "kind": "consulted" + }, + { + "kind": "accountable", + "role": "R-1", + "id": "RS-16-acc", + "capability": "C-16" + }, + { + "id": "RS-16-res", + "capability": "C-16", + "role": "R-1", + "kind": "responsible" + }, + { + "id": "RS-16-con", + "role": "R-4", + "kind": "consulted", + "capability": "C-16" + }, + { + "id": "RS-17-acc", + "role": "R-2", + "kind": "accountable", + "capability": "C-17" + }, + { + "kind": "responsible", + "role": "R-2", + "id": "RS-17-res1", + "capability": "C-17" + }, + { + "capability": "C-17", + "kind": "responsible", + "role": "R-6", + "id": "RS-17-res2" + }, + { + "capability": "C-18", + "role": "R-2", + "id": "RS-18-acc", + "kind": "accountable" + }, + { + "kind": "responsible", + "role": "R-2", + "id": "RS-18-res1", + "capability": "C-18" + }, + { + "role": "R-6", + "kind": "responsible", + "capability": "C-18", + "id": "RS-18-res2" + }, + { + "capability": "C-19", + "kind": "accountable", + "role": "R-1", + "id": "RS-19-acc" + }, + { + "role": "R-1", + "capability": "C-19", + "id": "RS-19-res", + "kind": "responsible" + }, + { + "id": "RS-19-con", + "role": "R-4", + "kind": "consulted", + "capability": "C-19" + }, + { + "role": "R-1", + "id": "RS-20-acc", + "capability": "C-20", + "kind": "accountable" + }, + { + "role": "R-1", + "capability": "C-20", + "id": "RS-20-res1", + "kind": "responsible" + }, + { + "kind": "responsible", + "capability": "C-20", + "role": "R-4", + "id": "RS-20-res2" + }, + { + "kind": "accountable", + "capability": "C-21", + "id": "RS-21-acc", + "role": "R-4" + }, + { + "role": "R-6", + "id": "RS-21-res", + "capability": "C-21", + "kind": "responsible" + }, + { + "role": "R-2", + "id": "RS-21-con", + "capability": "C-21", + "kind": "consulted" + } + ], + "checks": [ + { + "source": "OR-1", + "id": "OC-1", + "condition": "A fault on the hub network or any other item is acknowledged within 15 minutes at any hour.", + "on": "C-1" + }, + { + "condition": "Service is restored within 4 hours of a major failure on the platform.", + "on": "C-1", + "source": "OR-2", + "id": "OC-2" + }, + { + "on": "C-15", + "condition": "The platform availability is tracked to ensure at least 99.9% uptime each month.", + "source": "OR-3", + "id": "OC-3" + }, + { + "source": "K-1", + "id": "OC-4", + "condition": "Capacity limits for the platform interconnect are managed to support 99.9% availability.", + "on": "C-10" + }, + { + "source": "OR-4", + "condition": "Every change to the hub network requires explicit approval from the customer platform team.", + "id": "OC-5", + "on": "C-5" + }, + { + "condition": "A consolidated monthly report includes service levels and operating costs.", + "source": "OR-5", + "on": "C-15", + "id": "OC-6" + }, + { + "source": "OR-6", + "id": "OC-7", + "condition": "The shared service model responsibilities between customer and external service provider are governed.", + "on": "C-20" + }, + { + "on": "R-3", + "id": "OC-8a", + "source": "OR-7", + "condition": "Role performs operational delivery from India." + }, + { + "condition": "Role performs operational delivery from India.", + "id": "OC-8b", + "source": "OR-7", + "on": "R-4" + }, + { + "source": "OR-7", + "id": "OC-8c", + "condition": "Role performs operational delivery from India.", + "on": "R-5" + }, + { + "condition": "Role performs operational delivery from Poland.", + "id": "OC-8d", + "on": "R-6", + "source": "OR-7" + }, + { + "condition": "Role performs operational delivery from the UK.", + "source": "OR-7", + "id": "OC-8e", + "on": "R-1" + }, + { + "condition": "Role performs operational delivery from the UK.", + "on": "R-2", + "source": "OR-7", + "id": "OC-8f" + }, + { + "on": "C-7", + "source": "OR-8", + "condition": "Audit log storage configuration ensures logs cannot be deleted for seven years.", + "id": "OC-9" + }, + { + "condition": "Retention policies on Cloud Storage for audit logs are verified to prevent deletion for seven years.", + "source": "K-2", + "id": "OC-10", + "on": "C-7" + }, + { + "condition": "Encryption key configurations are audited to ensure rotation every 90 days.", + "source": "OR-9", + "id": "OC-11", + "on": "C-7" + }, + { + "on": "C-7", + "id": "OC-12", + "source": "K-3", + "condition": "Cloud KMS encryption keys are verified to rotate every 90 days." + }, + { + "on": "C-6", + "id": "OC-13", + "source": "OR-10", + "condition": "Deployment pipelines ensure an approved change reaches production within one day." + }, + { + "condition": "Backups of audit logs and encryption keys are successfully tested for restoration.", + "id": "OC-14", + "source": "OR-2", + "on": "C-8" + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/verdicts.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/verdicts.json new file mode 100644 index 0000000..af388a7 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3/verdicts.json @@ -0,0 +1,169 @@ +{ + "dry_run": { + "verdict": "pass", + "evidence": "13 of 13 scenario walk-throughs pass." + }, + "checks": [ + { + "check": "OC-1", + "evidence": "C-1 handles incident management 24x7, with a 24x7 Operations Engineer responsible, enabling acknowledgement of faults within 15 minutes at any hour. The cloud provider's native tooling (via C-2) supports configuring alerts without requiring a separate finding.", + "verdict": "pass" + }, + { + "evidence": "C-1 manages incident resolution with a 24x7 Operations Engineer responsible, allowing restoration of service within 4 hours of a major failure regardless of when the failure occurs.", + "check": "OC-2", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "C-15 tracks and gathers metrics on availability to ensure the 99.9% target is tracked, with business-hours roles accountable and responsible.", + "check": "OC-3" + }, + { + "verdict": "pass", + "evidence": "C-10 reviews capacity trends for the interconnect to assure 99.9% availability, managed by the Operations Lead.", + "check": "OC-4" + }, + { + "evidence": "C-5 coordinates network change approvals with the customer, and a Customer Platform Engineer is accountable and responsible for it, ensuring explicit approval from the customer platform team.", + "check": "OC-5", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "OC-6", + "evidence": "C-15 compiles the monthly operational report, pulling in cost data tracked by C-14, both managed by the Operations Lead." + }, + { + "verdict": "pass", + "check": "OC-7", + "evidence": "C-20 coordinates responsibilities across shared delivery borders, with roles from both the customer and provider teams responsible." + }, + { + "check": "OC-8a", + "evidence": "R-3 (Operations Engineer) is based in India, satisfying the geographical requirement.", + "verdict": "pass" + }, + { + "evidence": "R-4 (Operations Lead) is based in India, satisfying the geographical requirement.", + "check": "OC-8b", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "OC-8c", + "evidence": "R-5 (Security Operations Analyst) is based in India, satisfying the geographical requirement." + }, + { + "check": "OC-8d", + "verdict": "pass", + "evidence": "R-6 (Platform Ops Specialist) is based in Poland, satisfying the geographical requirement." + }, + { + "check": "OC-8e", + "evidence": "R-1 (Platform Product Manager) is based in the UK, satisfying the geographical requirement.", + "verdict": "pass" + }, + { + "check": "OC-8f", + "verdict": "pass", + "evidence": "R-2 (Platform Engineer) is based in the UK, satisfying the geographical requirement." + }, + { + "evidence": "C-7 maintains audit log retention settings, with the Platform Ops Specialist responsible, ensuring logs cannot be deleted for seven years.", + "check": "OC-9", + "verdict": "pass" + }, + { + "check": "OC-10", + "verdict": "pass", + "evidence": "C-7 maintains and verifies retention settings on Cloud Storage for audit logs, satisfying the seven-year retention condition." + }, + { + "check": "OC-11", + "verdict": "pass", + "evidence": "C-7 audits configuration drift and manages key rotation schedules, ensuring encryption keys are rotated every 90 days." + }, + { + "verdict": "pass", + "evidence": "C-7 manages key rotation schedules for the Cloud KMS encryption keys, verifying they rotate every 90 days.", + "check": "OC-12" + }, + { + "check": "OC-13", + "verdict": "pass", + "evidence": "C-6 oversees deployment pipelines to ensure approved changes reach production within a day, with customer and provider roles responsible." + }, + { + "evidence": "C-8 maintains backups of logs and keys and successfully tests restore procedures, managed by the Platform Ops Specialist.", + "check": "OC-14", + "verdict": "pass" + }, + { + "check": "SC-fault-S-1", + "verdict": "pass", + "evidence": "Hub network and interconnect (S-1) fails at an hour faults are responded to (24x7). Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "check": "SC-change-S-1", + "verdict": "pass", + "evidence": "Hub network and interconnect (S-1) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-2", + "verdict": "pass", + "evidence": "Audit log store (S-2) fails at an hour faults are responded to (24x7). Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "check": "SC-change-S-2", + "verdict": "pass", + "evidence": "Audit log store (S-2) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-3", + "verdict": "pass", + "evidence": "Deployment pipeline (S-3) fails at an hour faults are responded to (24x7). Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "check": "SC-change-S-3", + "verdict": "pass", + "evidence": "Deployment pipeline (S-3) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-4", + "verdict": "pass", + "evidence": "Encryption keys (S-4) fails at an hour faults are responded to (24x7). Operations Engineer responds under C-1, and Operations Lead is answerable." + }, + { + "check": "SC-change-S-4", + "verdict": "pass", + "evidence": "Encryption keys (S-4) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-condition-K-1", + "verdict": "pass", + "evidence": "Condition K-1 must be shown to hold: The platform is available at least 99.9% of each month. Check OC-4 on C-10 keeps it true." + }, + { + "check": "SC-condition-K-2", + "verdict": "pass", + "evidence": "Condition K-2 must be shown to hold: Audit logs cannot be deleted for seven years. Check OC-10 on C-7 keeps it true." + }, + { + "check": "SC-condition-K-3", + "verdict": "pass", + "evidence": "Condition K-3 must be shown to hold: Encryption keys are rotated every 90 days. Check OC-12 on C-7 keeps it true." + }, + { + "check": "SC-party-P-2", + "verdict": "pass", + "evidence": "Service Provider cannot be reached. Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "check": "SC-budget", + "verdict": "pass", + "evidence": "A month's cost of operating is compared with the budget of 60,000 GBP. The cost is 47,603 GBP, within the budget." + } + ], + "proposed_additions": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.md b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.md new file mode 100644 index 0000000..c98db1c --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.md @@ -0,0 +1,10 @@ +# Intent 1: bench-landing-zone-shared + +- Recorded: 2026-10-21T20:36:25+00:00 +- Execution: `case-01-landing-zone-shared-r1` + +## What the customer wants + +We want our landing zone run as a shared service from go-live. A provider should +respond to faults at every hour, and our own platform team keeps the final say +over changes to the network. diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.scope.completed.yaml b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.scope.completed.yaml new file mode 100644 index 0000000..b6c7089 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.scope.completed.yaml @@ -0,0 +1,198 @@ +schema_version: 1 +metadata: + customer_case: bench-landing-zone-shared + client_name: ask + author: ask + revision: ask + source: ask + status: BASELINE_LOCKED +engagement: + kind: platform_foundation +intent: + statement: 'We want our landing zone run as a shared service from go-live. A provider + should + + respond to faults at every hour, and our own platform team keeps the final say + + over changes to the network. + + ' + requirements: + - id: REQ-01 + statement: Our platform team approves every change to the network. + - id: REQ-02 + statement: We receive a monthly report of service levels and cost. + business_outcomes: + - id: BO-1 + outcome: New payment products reach customers in weeks rather than months. + target: A change reaches production within a day of approval. + - id: BO-2 + outcome: Payments are never unavailable for long. + target: Restored within 4 hours. + - id: BO-3 + outcome: We keep the platform engineers we hire. + exclusions: + - Application workloads and services hosted on the platform foundation +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Hub network and interconnect + kind: network + product: Cloud Interconnect + - id: S-2 + name: Audit log store + kind: storage + product: Cloud Storage + - id: S-3 + name: Deployment pipeline + kind: pipeline + product: Cloud Build + - id: S-4 + name: Encryption keys + kind: keys + product: Cloud KMS + conditions: + - id: K-1 + 'on': + - S-1 + condition: The platform is available at least 99.9% of each month. + - id: K-2 + 'on': + - S-2 + condition: Audit logs cannot be deleted for seven years. + - id: K-3 + 'on': + - S-4 + condition: Encryption keys are rotated every 90 days. + build: + duration_days: 36 + customer_owned: ask + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Changes go through a deployment pipeline with peer review, with + explicit customer platform team approval required for all network changes. + monitoring: ask +sourcing: + model: shared + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: + - name: Platform team + party: customer + hours: business + country: GB + headcount_limit: ask +service_management: + tooling: Google Cloud operations suite and native Cloud provider tooling, with no + separate external ITSM system. +service_levels: + priority_1_response_minutes: 15 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA +commercial: + currency: GBP + monthly_run_budget: 60000 +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.exclusions + statement: 'Assumed intent.exclusions is [''Application workloads and services hosted + on the platform foundation'']. Assumed based on platform foundation scope: workloads + hosted on top of the landing zone are out of scope for this operating model.' + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-02 + field: operations.change_process + statement: Assumed operations.change_process is Changes go through a deployment + pipeline with peer review, with explicit customer platform team approval required + for all network changes.. Pipeline-driven changes with peer review, with customer + platform team approval required for network changes per REQ-01. + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-03 + field: service_management.tooling + statement: Assumed service_management.tooling is Google Cloud operations suite and + native Cloud provider tooling, with no separate external ITSM system.. Default + native GCP operations tooling used as no external ITSM was specified. + impact_if_invalid: Changes tooling and the processes written for it. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: metadata.source + prompt: What is metadata.source? + severity: LOW +- field: solution.build.customer_owned + prompt: Which parts of the build does the customer or their supplier own? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: supplied + commercial.monthly_run_budget: supplied + compliance.frameworks: supplied + engagement.kind: supplied + intent.business_outcomes: supplied + intent.exclusions: assumed + intent.requirements: supplied + intent.statement: supplied + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: supplied + metadata.revision: unknown + metadata.source: unknown + operations.availability_hours: supplied + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: supplied + organisation.existing_teams: supplied + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: supplied + service_levels.recovery_time_hours: supplied + service_management.tooling: assumed + solution.build.customer_owned: unknown + solution.build.duration_days: supplied + solution.cloud_provider: supplied + solution.conditions: supplied + solution.inherited_risks: unknown + solution.items: supplied + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 20 + answered: 0 + assumed: 3 + unknown: 10 + not_applicable: 0 diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.scope.yaml new file mode 100644 index 0000000..1dd8d31 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/001.scope.yaml @@ -0,0 +1,53 @@ +# A fictional benchmark: a small landing zone run as a shared service. No customer's +# data is in it. Complete enough to run with nobody to ask. +metadata: + customer_case: bench-landing-zone-shared +engagement: + kind: platform_foundation +intent: + statement: | + We want our landing zone run as a shared service from go-live. A provider should + respond to faults at every hour, and our own platform team keeps the final say + over changes to the network. + business_outcomes: + - {id: BO-1, outcome: New payment products reach customers in weeks rather than months., target: "A change reaches production within a day of approval."} + - {id: BO-2, outcome: Payments are never unavailable for long., target: "Restored within 4 hours."} + - {id: BO-3, outcome: We keep the platform engineers we hire.} + requirements: + - {id: REQ-01, statement: Our platform team approves every change to the network.} + - {id: REQ-02, statement: We receive a monthly report of service levels and cost.} +solution: + cloud_provider: gcp + items: + - {id: S-1, name: Hub network and interconnect, kind: network, product: Cloud Interconnect} + - {id: S-2, name: Audit log store, kind: storage, product: Cloud Storage} + - {id: S-3, name: Deployment pipeline, kind: pipeline, product: Cloud Build} + - {id: S-4, name: Encryption keys, kind: keys, product: Cloud KMS} + conditions: + - {id: K-1, "on": [S-1], condition: The platform is available at least 99.9% of each month.} + - {id: K-2, "on": [S-2], condition: Audit logs cannot be deleted for seven years.} + - {id: K-3, "on": [S-4], condition: Encryption keys are rotated every 90 days.} + build: + duration_days: 36 +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: decide +sourcing: + model: shared + delivery_countries: [GB, PL, IN] +organisation: + existing_teams: + - {name: Platform team, party: customer, hours: business, country: GB} +service_management: + tooling: decide +service_levels: + priority_1_response_minutes: 15 + recovery_time_hours: 4 +compliance: + frameworks: [DORA] +commercial: + currency: GBP + monthly_run_budget: 60000 +run: + pre_approved: true diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.md b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.md new file mode 100644 index 0000000..e76c708 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.md @@ -0,0 +1,10 @@ +# Intent 2: bench-landing-zone-shared + +- Recorded: 2026-10-21T20:54:36+00:00 +- Execution: `case-01-landing-zone-shared-r2` + +## What the customer wants + +We want our landing zone run as a shared service from go-live. A provider should +respond to faults at every hour, and our own platform team keeps the final say +over changes to the network. diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.scope.completed.yaml b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.scope.completed.yaml new file mode 100644 index 0000000..772f144 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.scope.completed.yaml @@ -0,0 +1,199 @@ +schema_version: 1 +metadata: + customer_case: bench-landing-zone-shared + client_name: ask + author: ask + revision: ask + source: ask + status: BASELINE_LOCKED +engagement: + kind: platform_foundation +intent: + statement: 'We want our landing zone run as a shared service from go-live. A provider + should + + respond to faults at every hour, and our own platform team keeps the final say + + over changes to the network. + + ' + requirements: + - id: REQ-01 + statement: Our platform team approves every change to the network. + - id: REQ-02 + statement: We receive a monthly report of service levels and cost. + business_outcomes: + - id: BO-1 + outcome: New payment products reach customers in weeks rather than months. + target: A change reaches production within a day of approval. + - id: BO-2 + outcome: Payments are never unavailable for long. + target: Restored within 4 hours. + - id: BO-3 + outcome: We keep the platform engineers we hire. + exclusions: + - Application workloads deployed onto the platform foundation +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Hub network and interconnect + kind: network + product: Cloud Interconnect + - id: S-2 + name: Audit log store + kind: storage + product: Cloud Storage + - id: S-3 + name: Deployment pipeline + kind: pipeline + product: Cloud Build + - id: S-4 + name: Encryption keys + kind: keys + product: Cloud KMS + conditions: + - id: K-1 + 'on': + - S-1 + condition: The platform is available at least 99.9% of each month. + - id: K-2 + 'on': + - S-2 + condition: Audit logs cannot be deleted for seven years. + - id: K-3 + 'on': + - S-4 + condition: Encryption keys are rotated every 90 days. + build: + duration_days: 36 + customer_owned: ask + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Changes go through automated pipelines with peer review; changes + affecting the hub network require customer platform team approval prior to release. + monitoring: ask +sourcing: + model: shared + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: + - name: Platform team + party: customer + hours: business + country: GB + headcount_limit: ask +service_management: + tooling: Google Cloud native operations tooling (Cloud Monitoring and Cloud Logging), + with no separate service management system. +service_levels: + priority_1_response_minutes: 15 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA +commercial: + currency: GBP + monthly_run_budget: 60000 +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.exclusions + statement: Assumed intent.exclusions is ['Application workloads deployed onto the + platform foundation']. Workloads deployed onto the platform foundation are excluded; + operations is confined to the landing zone foundation itself. + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-02 + field: operations.change_process + statement: Assumed operations.change_process is Changes go through automated pipelines + with peer review; changes affecting the hub network require customer platform + team approval prior to release.. Changes follow automated deployment pipelines + with peer review, with network changes requiring explicit customer platform team + approval per REQ-01. + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-03 + field: service_management.tooling + statement: Assumed service_management.tooling is Google Cloud native operations + tooling (Cloud Monitoring and Cloud Logging), with no separate service management + system.. Defaulting to GCP native operations tooling with no separate ITSM system. + impact_if_invalid: Changes tooling and the processes written for it. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: metadata.source + prompt: What is metadata.source? + severity: LOW +- field: solution.build.customer_owned + prompt: Which parts of the build does the customer or their supplier own? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: supplied + commercial.monthly_run_budget: supplied + compliance.frameworks: supplied + engagement.kind: supplied + intent.business_outcomes: supplied + intent.exclusions: assumed + intent.requirements: supplied + intent.statement: supplied + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: supplied + metadata.revision: unknown + metadata.source: unknown + operations.availability_hours: supplied + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: supplied + organisation.existing_teams: supplied + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: supplied + service_levels.recovery_time_hours: supplied + service_management.tooling: assumed + solution.build.customer_owned: unknown + solution.build.duration_days: supplied + solution.cloud_provider: supplied + solution.conditions: supplied + solution.inherited_risks: unknown + solution.items: supplied + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 20 + answered: 0 + assumed: 3 + unknown: 10 + not_applicable: 0 diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.scope.yaml new file mode 100644 index 0000000..1dd8d31 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/002.scope.yaml @@ -0,0 +1,53 @@ +# A fictional benchmark: a small landing zone run as a shared service. No customer's +# data is in it. Complete enough to run with nobody to ask. +metadata: + customer_case: bench-landing-zone-shared +engagement: + kind: platform_foundation +intent: + statement: | + We want our landing zone run as a shared service from go-live. A provider should + respond to faults at every hour, and our own platform team keeps the final say + over changes to the network. + business_outcomes: + - {id: BO-1, outcome: New payment products reach customers in weeks rather than months., target: "A change reaches production within a day of approval."} + - {id: BO-2, outcome: Payments are never unavailable for long., target: "Restored within 4 hours."} + - {id: BO-3, outcome: We keep the platform engineers we hire.} + requirements: + - {id: REQ-01, statement: Our platform team approves every change to the network.} + - {id: REQ-02, statement: We receive a monthly report of service levels and cost.} +solution: + cloud_provider: gcp + items: + - {id: S-1, name: Hub network and interconnect, kind: network, product: Cloud Interconnect} + - {id: S-2, name: Audit log store, kind: storage, product: Cloud Storage} + - {id: S-3, name: Deployment pipeline, kind: pipeline, product: Cloud Build} + - {id: S-4, name: Encryption keys, kind: keys, product: Cloud KMS} + conditions: + - {id: K-1, "on": [S-1], condition: The platform is available at least 99.9% of each month.} + - {id: K-2, "on": [S-2], condition: Audit logs cannot be deleted for seven years.} + - {id: K-3, "on": [S-4], condition: Encryption keys are rotated every 90 days.} + build: + duration_days: 36 +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: decide +sourcing: + model: shared + delivery_countries: [GB, PL, IN] +organisation: + existing_teams: + - {name: Platform team, party: customer, hours: business, country: GB} +service_management: + tooling: decide +service_levels: + priority_1_response_minutes: 15 + recovery_time_hours: 4 +compliance: + frameworks: [DORA] +commercial: + currency: GBP + monthly_run_budget: 60000 +run: + pre_approved: true diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.md b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.md new file mode 100644 index 0000000..f61fb3c --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.md @@ -0,0 +1,10 @@ +# Intent 3: bench-landing-zone-shared + +- Recorded: 2026-10-21T21:11:10+00:00 +- Execution: `case-01-landing-zone-shared-r3` + +## What the customer wants + +We want our landing zone run as a shared service from go-live. A provider should +respond to faults at every hour, and our own platform team keeps the final say +over changes to the network. diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.scope.completed.yaml b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.scope.completed.yaml new file mode 100644 index 0000000..906842f --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.scope.completed.yaml @@ -0,0 +1,201 @@ +schema_version: 1 +metadata: + customer_case: bench-landing-zone-shared + client_name: ask + author: ask + revision: ask + source: ask + status: BASELINE_LOCKED +engagement: + kind: platform_foundation +intent: + statement: 'We want our landing zone run as a shared service from go-live. A provider + should + + respond to faults at every hour, and our own platform team keeps the final say + + over changes to the network. + + ' + requirements: + - id: REQ-01 + statement: Our platform team approves every change to the network. + - id: REQ-02 + statement: We receive a monthly report of service levels and cost. + business_outcomes: + - id: BO-1 + outcome: New payment products reach customers in weeks rather than months. + target: A change reaches production within a day of approval. + - id: BO-2 + outcome: Payments are never unavailable for long. + target: Restored within 4 hours. + - id: BO-3 + outcome: We keep the platform engineers we hire. + exclusions: + - Workload applications running on the platform foundation + - Network change approvals (retained by customer platform team) +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Hub network and interconnect + kind: network + product: Cloud Interconnect + - id: S-2 + name: Audit log store + kind: storage + product: Cloud Storage + - id: S-3 + name: Deployment pipeline + kind: pipeline + product: Cloud Build + - id: S-4 + name: Encryption keys + kind: keys + product: Cloud KMS + conditions: + - id: K-1 + 'on': + - S-1 + condition: The platform is available at least 99.9% of each month. + - id: K-2 + 'on': + - S-2 + condition: Audit logs cannot be deleted for seven years. + - id: K-3 + 'on': + - S-4 + condition: Encryption keys are rotated every 90 days. + build: + duration_days: 36 + customer_owned: ask + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Changes go through a deployment pipeline with peer review, with + network changes requiring customer platform team approval. + monitoring: ask +sourcing: + model: shared + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: + - name: Platform team + party: customer + hours: business + country: GB + headcount_limit: ask +service_management: + tooling: The Cloud provider's own operations tooling, with no separate system. +service_levels: + priority_1_response_minutes: 15 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA +commercial: + currency: GBP + monthly_run_budget: 60000 +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.exclusions + statement: 'Assumed intent.exclusions is [''Workload applications running on the + platform foundation'', ''Network change approvals (retained by customer platform + team)'']. Assumed from the platform foundation scope: workload applications running + on the platform are out of scope, and customer platform team retains network change + approvals.' + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-02 + field: operations.change_process + statement: 'Assumed operations.change_process is Changes go through a deployment + pipeline with peer review, with network changes requiring customer platform team + approval.. Catalogue default adapted to customer requirement REQ-01: changes go + through deployment pipelines with peer review, with customer platform team retaining + approval over network changes.' + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-03 + field: service_management.tooling + statement: 'Assumed service_management.tooling is The Cloud provider''s own operations + tooling, with no separate system.. Catalogue default: Cloud provider''s native + operations tooling with no separate system.' + impact_if_invalid: Changes tooling and the processes written for it. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: metadata.source + prompt: What is metadata.source? + severity: LOW +- field: solution.build.customer_owned + prompt: Which parts of the build does the customer or their supplier own? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: supplied + commercial.monthly_run_budget: supplied + compliance.frameworks: supplied + engagement.kind: supplied + intent.business_outcomes: supplied + intent.exclusions: assumed + intent.requirements: supplied + intent.statement: supplied + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: supplied + metadata.revision: unknown + metadata.source: unknown + operations.availability_hours: supplied + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: supplied + organisation.existing_teams: supplied + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: supplied + service_levels.recovery_time_hours: supplied + service_management.tooling: assumed + solution.build.customer_owned: unknown + solution.build.duration_days: supplied + solution.cloud_provider: supplied + solution.conditions: supplied + solution.inherited_risks: unknown + solution.items: supplied + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 20 + answered: 0 + assumed: 3 + unknown: 10 + not_applicable: 0 diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.scope.yaml new file mode 100644 index 0000000..1dd8d31 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/003.scope.yaml @@ -0,0 +1,53 @@ +# A fictional benchmark: a small landing zone run as a shared service. No customer's +# data is in it. Complete enough to run with nobody to ask. +metadata: + customer_case: bench-landing-zone-shared +engagement: + kind: platform_foundation +intent: + statement: | + We want our landing zone run as a shared service from go-live. A provider should + respond to faults at every hour, and our own platform team keeps the final say + over changes to the network. + business_outcomes: + - {id: BO-1, outcome: New payment products reach customers in weeks rather than months., target: "A change reaches production within a day of approval."} + - {id: BO-2, outcome: Payments are never unavailable for long., target: "Restored within 4 hours."} + - {id: BO-3, outcome: We keep the platform engineers we hire.} + requirements: + - {id: REQ-01, statement: Our platform team approves every change to the network.} + - {id: REQ-02, statement: We receive a monthly report of service levels and cost.} +solution: + cloud_provider: gcp + items: + - {id: S-1, name: Hub network and interconnect, kind: network, product: Cloud Interconnect} + - {id: S-2, name: Audit log store, kind: storage, product: Cloud Storage} + - {id: S-3, name: Deployment pipeline, kind: pipeline, product: Cloud Build} + - {id: S-4, name: Encryption keys, kind: keys, product: Cloud KMS} + conditions: + - {id: K-1, "on": [S-1], condition: The platform is available at least 99.9% of each month.} + - {id: K-2, "on": [S-2], condition: Audit logs cannot be deleted for seven years.} + - {id: K-3, "on": [S-4], condition: Encryption keys are rotated every 90 days.} + build: + duration_days: 36 +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: decide +sourcing: + model: shared + delivery_countries: [GB, PL, IN] +organisation: + existing_teams: + - {name: Platform team, party: customer, hours: business, country: GB} +service_management: + tooling: decide +service_levels: + priority_1_response_minutes: 15 + recovery_time_hours: 4 +compliance: + frameworks: [DORA] +commercial: + currency: GBP + monthly_run_budget: 60000 +run: + pre_approved: true diff --git a/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/history.json b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/history.json new file mode 100644 index 0000000..7f49f2b --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/bench-landing-zone-shared/intent/history.json @@ -0,0 +1,53 @@ +{ + "customer_case": "bench-landing-zone-shared", + "intents": [ + { + "version": 1, + "recorded_at": "2026-10-21T20:36:25+00:00", + "statement": "intent/001.md", + "workflow_request": "We want our landing zone run as a shared service from go-live. A provider should\nrespond to faults at every hour, and our own platform team keeps the final say\nover changes to the network.", + "execution_id": "case-01-landing-zone-shared-r1", + "session_id": "9fe31b7a-c779-4d34-bcee-cc2e36533a1d", + "status": "completed", + "execution": "executions/case-01-landing-zone-shared-r1", + "baseline": null, + "scope": { + "completed": "intent/001.scope.completed.yaml", + "supplied": "intent/001.scope.yaml" + }, + "sources": null + }, + { + "version": 2, + "recorded_at": "2026-10-21T20:54:36+00:00", + "statement": "intent/002.md", + "workflow_request": "We want our landing zone run as a shared service from go-live. A provider should\nrespond to faults at every hour, and our own platform team keeps the final say\nover changes to the network.", + "execution_id": "case-01-landing-zone-shared-r2", + "session_id": "c7cf78e8-2f3b-4f26-bd2f-850de41fe1d6", + "status": "completed", + "execution": "executions/case-01-landing-zone-shared-r2", + "baseline": null, + "scope": { + "completed": "intent/002.scope.completed.yaml", + "supplied": "intent/002.scope.yaml" + }, + "sources": null + }, + { + "version": 3, + "recorded_at": "2026-10-21T21:11:10+00:00", + "statement": "intent/003.md", + "workflow_request": "We want our landing zone run as a shared service from go-live. A provider should\nrespond to faults at every hour, and our own platform team keeps the final say\nover changes to the network.", + "execution_id": "case-01-landing-zone-shared-r3", + "session_id": "d0d953a6-5254-4646-8d70-7fd6139f1afc", + "status": "completed", + "execution": "executions/case-01-landing-zone-shared-r3", + "baseline": null, + "scope": { + "completed": "intent/003.scope.completed.yaml", + "supplied": "intent/003.scope.yaml" + }, + "sources": null + } + ] +} diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/capabilities.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/capabilities.json new file mode 100644 index 0000000..1093a31 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/capabilities.json @@ -0,0 +1,191 @@ +[ + { + "id": "C-1", + "catalogue": "incident-management", + "serves": ["OR-1", "OR-6"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "24x7", + "effort_hours_per_month": 40, + "basis": "Triage of 24x7 operational alerts and managing approximately 2 priority incidents per month across platform infrastructure." + }, + { + "id": "C-2", + "catalogue": "monitoring-and-alerting", + "serves": ["OR-6", "OR-15"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "24x7", + "effort_hours_per_month": 30, + "basis": "Tuning Cloud Monitoring alert thresholds, maintaining dashboards, and integrating telemetry into operational systems." + }, + { + "id": "C-3", + "catalogue": "problem-management", + "serves": ["OR-2"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 16, + "basis": "Investigating root causes for platform anomalies and driving permanent problem resolution." + }, + { + "id": "C-4", + "catalogue": "service-request-fulfilment", + "serves": ["OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Fulfilling non-standard operational requests and provisioning adjustments for core payments engine ingestion." + }, + { + "id": "C-5", + "catalogue": "change-enablement", + "serves": ["OR-12"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Reviewing automated pipeline outputs and conducting peer approvals for all infrastructure-as-code changes." + }, + { + "id": "C-6", + "catalogue": "release-and-deployment", + "serves": ["OR-10", "OR-12"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 30, + "basis": "Managing automated GitHub Actions pipelines, resolving deployment failures, and optimizing deployments and resource placements to maximize carbon-free energy utilization." + }, + { + "id": "C-7", + "catalogue": "configuration-management", + "serves": ["OR-16"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 12, + "basis": "Tracking infrastructure state files and ensuring alignment with PCI-DSS v4.0 baseline configurations." + }, + { + "id": "C-8", + "catalogue": "backup-and-recovery", + "serves": ["OR-4"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Monitoring automated backup routines and performing scheduled restore tests to ensure 1-hour RPO." + }, + { + "id": "C-9", + "catalogue": "service-continuity", + "serves": ["OR-3", "OR-7"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Maintaining the active warm standby in europe-west1 to immediately receive failover traffic, and governing 4-hour disaster recovery procedures." + }, + { + "id": "C-10", + "catalogue": "capacity-and-performance", + "serves": ["OR-5"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 12, + "basis": "Reviewing 10Gbps interconnect bandwidth and ensuring deterministic low-latency performance for payment transactions." + }, + { + "id": "C-11", + "catalogue": "access-management", + "serves": ["OR-11", "OR-14"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 24, + "basis": "Rightsizing permissions based on IAM Recommender findings and controlling access from approved delivery countries." + }, + { + "id": "C-12", + "catalogue": "vulnerability-and-patch-management", + "serves": ["OR-16"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 24, + "basis": "Scanning platform foundation assets and applying security patches to maintain compliance standards." + }, + { + "id": "C-13", + "catalogue": "security-monitoring-and-response", + "serves": ["OR-1", "OR-6"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "24x7", + "effort_hours_per_month": 40, + "basis": "Continuous 24x7 monitoring of security events and rapid response to security-related alerts." + }, + { + "id": "C-14", + "catalogue": "cost-management", + "serves": ["OR-9"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 12, + "basis": "Tracking Google Cloud spend against budget allocations and escalating anomalies." + }, + { + "id": "C-15", + "catalogue": "service-level-management", + "serves": ["OR-2"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Compiling reports on 99.9% availability and incident response times." + }, + { + "id": "C-16", + "catalogue": "platform-product-management", + "serves": ["OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 16, + "basis": "Managing the foundation roadmap to ensure readiness for core payments engine workload ingestion." + }, + { + "id": "C-17", + "catalogue": "self-service-and-golden-paths", + "serves": ["OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Building reusable templates and automated paved paths for payment engine teams." + }, + { + "id": "C-18", + "catalogue": "enablement", + "serves": ["OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Assisting development teams through pairing and documentation to rapidly adopt the platform within the 10-week window." + }, + { + "id": "C-19", + "catalogue": "delivery-and-developer-experience-measurement", + "serves": ["OR-12"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Collecting DORA metrics for pipeline performance, and evaluating developer productivity and wellbeing to determine friction feedback." + }, + { + "id": "C-20", + "catalogue": "supplier-management", + "serves": ["OR-13"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 8, + "basis": "Managing relationships and service reviews with Google Cloud and interconnect carriers." + }, + { + "id": "C-21", + "catalogue": "regulatory-evidence", + "serves": ["OR-16", "OR-17"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 16, + "basis": "Collating compliance evidence for DORA, PCI-DSS v4.0, ISO27001, and guaranteeing UK/EU data residency records." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/execution.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/execution.json new file mode 100644 index 0000000..33e804a --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/execution.json @@ -0,0 +1,146 @@ +{ + "customer_case": "case-08-landing-zone", + "execution_id": "case-02-from-architecture-r1", + "session_id": "bc30257a-ff81-4114-86b9-83afd8a8ae5b", + "intent_version": 1, + "workflow_request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.", + "baseline": null, + "overlay_skills": {}, + "options": {}, + "sources": { + "execution.json": { + "sha256": "8bf269deffc8e4bce474bd8d66959c03b04de01dc137eefaf96ead74b46a5a45", + "digested": true + }, + "scope.json": { + "sha256": "57dc28ab7397f514ef0e83b292fc2165554ac2b516775c973f4965765a9e905f", + "digested": true + }, + "requirements.json": { + "sha256": "5aecbef9eaea254fb5123bcc1d107b4a7b7d4d45d60d940464d1fde690d2973d", + "digested": true + }, + "recommendations.json": { + "sha256": "95958b1a107705280d65c9d088bba57352d1f9fa8e67e89b2b3379f78d91cd24", + "digested": true + }, + "components.json": { + "sha256": "49e47dbfa90a85d086711eb2ba66370c145f13fd3cd3595c1df1366224db392c", + "digested": true + }, + "products.json": { + "sha256": "75d51cba319f83000e6f85277b0fb21a7ec02faabac332c5d1800c8f512136db", + "digested": true + }, + "target-architecture.json": { + "sha256": "40879607aaa7041103d8f4d3ba804df8751d2159aa0ce0977d7d821d35f7608b", + "digested": true + }, + "verdicts.json": { + "sha256": "75e17da790266e94a6775d90857a9b503cacd62fd55c92be2d7a3435de301853", + "digested": true + }, + "work-breakdown.json": { + "sha256": "057fd1b0ecd6350a6f9f6101b712a61e78fc10d758a67e785a1dca7e1e674dc3", + "digested": true + } + }, + "scope": { + "status": "APPROVED", + "readiness": { + "supplied": 3, + "answered": 0, + "assumed": 7, + "unknown": 10, + "not_applicable": 0, + "digested": 13 + } + }, + "status": "completed_with_warnings", + "warnings": [ + "target_operating_model has reached its revision limit (3). Requested change: OC-2 failed: The check requires the secondary region to be 'continuously maintained' as an active warm standby, but Capability C-9 is configured for 'business' hours, meaning maintenance cannot be continuous (24x7). C-9 should be 24x7 or another capability should provide continuous maintenance.; OC-4 failed: The check requires validation within 10 weeks, but Capability C-16 covers managing the roadmap without explicitly mentioning the 10-week requirement (which is instead mentioned in C-18). C-16 must mention the 10-week validation or the check mapped to C-18.; OC-9 failed: The check requires that expenditure does not exceed target allocations without 'documented operational sign-off', but Capability C-14 only mentions 'escalating anomalies' and does not explicitly guarantee documented sign-off.; OC-12 failed: The check requires identity permissions to be 'continuously reviewed', but Capability C-11 operates only during 'business' hours, meaning reviews cannot be continuous. C-11 must be 24x7.; OC-17 failed: The check requires verifiable alignment with DORA, PCI-DSS v4.0, and ISO 27001. Capability C-7 only ensures alignment with PCI-DSS v4.0 baseline configurations, completely missing DORA and ISO 27001 controls.. Ask the user whether to revise it again or accept it as it stands." + ], + "risks": [], + "assessment": null, + "phases": { + "scoping": "approved", + "operational_requirements": "approved", + "operating_model_design": "approved", + "validation": "open", + "packaging": "open" + }, + "final_document": null, + "changes": null, + "deliverables": { + "operating_scope": { + "document": "deliverables/operating_scope.md", + "attachments": [ + "scope.json", + "scope.yaml" + ], + "status": "submitted" + }, + "operational_requirements": { + "document": "deliverables/operational_requirements.md", + "attachments": [ + "operational-requirements.json" + ], + "status": "submitted" + }, + "capability_model": { + "document": "deliverables/capability_model.md", + "attachments": [ + "capabilities.json" + ], + "status": "submitted" + }, + "target_operating_model": { + "document": "deliverables/target_operating_model.md", + "attachments": [ + "findings.json", + "organisation.mmd", + "staffing.json", + "target-operating-model.json" + ], + "status": "submitted" + }, + "validation_results": { + "document": "deliverables/validation_results.md", + "attachments": [ + "scenarios.json", + "verdicts.json" + ], + "status": "submitted" + } + }, + "files": [ + "capabilities.json", + "deliverables/capability_model.md", + "deliverables/operating_scope.md", + "deliverables/operational_requirements.md", + "deliverables/target_operating_model.md", + "deliverables/validation_results.md", + "findings.json", + "operational-requirements.json", + "organisation.mmd", + "scenarios.json", + "scope.json", + "scope.yaml", + "staffing.json", + "target-operating-model.json", + "verdicts.json" + ], + "staffing": { + "people": 11.7, + "cost_per_month": 73863, + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + } + }, + "findings": [], + "change_request": null, + "capability_sources_verified": false +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/findings.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/findings.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/findings.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/operational-requirements.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/operational-requirements.json new file mode 100644 index 0000000..ce75f54 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/operational-requirements.json @@ -0,0 +1,143 @@ +[ + { + "id": "OR-1", + "kind": "service_level", + "statement": "A Priority 1 incident on any platform foundation item is acknowledged within 30 minutes (assumed in service_levels.priority_1_response_minutes), at any hour of any day.", + "source": [ + "service_levels.priority_1_response_minutes", + "operations.support_hours" + ] + }, + { + "id": "OR-2", + "kind": "service_level", + "statement": "The platform foundation infrastructure maintains measured monthly availability of at least 99.9% across 24x7 operating hours.", + "source": [ + "K-5", + "operations.availability_hours" + ] + }, + { + "id": "OR-3", + "kind": "service_level", + "statement": "Platform service restoration following a disaster declaration achieves a recovery time objective (RTO) of 4 hours (assumed in service_levels.recovery_time_hours).", + "source": [ + "K-6", + "service_levels.recovery_time_hours" + ] + }, + { + "id": "OR-4", + "kind": "service_level", + "statement": "Disaster recovery processes achieve a recovery point objective (RPO) of no more than 1 hour of platform data or configuration loss.", + "source": [ + "K-7" + ] + }, + { + "id": "OR-5", + "kind": "service_level", + "statement": "The dedicated private hybrid connectivity gateway maintains deterministic low-latency network performance suitable for real-time payment transaction processing.", + "source": [ + "K-10" + ] + }, + { + "id": "OR-6", + "kind": "process", + "statement": "Platform alert notifications trigger operational incident workflows on a 24x7 continuous basis.", + "source": [ + "K-8", + "operations.support_hours" + ] + }, + { + "id": "OR-7", + "kind": "process", + "statement": "The secondary cloud region in europe-west1 is continuously maintained as an active warm standby ready to receive operational failover traffic.", + "source": [ + "K-2" + ] + }, + { + "id": "OR-8", + "kind": "process", + "statement": "The platform foundation is fully provisioned, hardened, and validated for core payments engine workload ingestion within 10 weeks.", + "source": [ + "K-4" + ] + }, + { + "id": "OR-9", + "kind": "process", + "statement": "Monthly cloud foundation operational expenditure is monitored against budget allocations and does not exceed target allocations without documented operational sign-off.", + "source": [ + "K-9" + ] + }, + { + "id": "OR-10", + "kind": "process", + "statement": "Operational deployments and platform resource placements optimize carbon-free energy utilization based on regional carbon-free energy scores.", + "source": [ + "K-11" + ] + }, + { + "id": "OR-11", + "kind": "process", + "statement": "Platform identity permissions and access roles are continuously reviewed and rightsized based on automated recommender findings.", + "source": [ + "K-12" + ] + }, + { + "id": "OR-12", + "kind": "process", + "statement": "Infrastructure changes are deployed via automated pipelines with automated linting, security scans, and peer approvals without a manual change advisory board (assumed in operations.change_process).", + "source": [ + "operations.change_process" + ] + }, + { + "id": "OR-13", + "kind": "organisation", + "statement": "Platform foundation operations, ongoing maintenance, and incident remediation are delivered under a provider-operated sourcing model.", + "source": [ + "sourcing.model" + ] + }, + { + "id": "OR-14", + "kind": "organisation", + "statement": "Platform operational delivery and administrative access are restricted to personnel operating from the United Kingdom (GB), Poland (PL), and India (IN).", + "source": [ + "sourcing.delivery_countries" + ] + }, + { + "id": "OR-15", + "kind": "tooling", + "statement": "Platform operational telemetry, logging, and incident alerting are integrated into operational event management systems (assumed in service_management.tooling).", + "source": [ + "service_management.tooling" + ] + }, + { + "id": "OR-16", + "kind": "compliance", + "statement": "The platform foundation configuration maintains verifiable alignment with DORA operational resiliency guidelines, PCI-DSS v4.0 perimeter isolation standards, and ISO 27001 controls during operational reviews and external audits.", + "source": [ + "K-1", + "compliance.frameworks" + ] + }, + { + "id": "OR-17", + "kind": "compliance", + "statement": "Data residency is guaranteed within the United Kingdom and European Union for all platform foundation resources and stored data.", + "source": [ + "K-3" + ] + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/organisation.mmd b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/organisation.mmd new file mode 100644 index 0000000..98dbc7e --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/organisation.mmd @@ -0,0 +1,23 @@ +flowchart TB + subgraph P_1["Customer (customer)"] + subgraph T_2["Core Payments Engine Team (stream-aligned team)"] + R_8["Core Payments Team Lead
GB, senior, business"] + end + end + subgraph P_2["Platform Provider (provider)"] + subgraph T_1["Platform Foundation Team (platform team)"] + R_1["Platform Foundation Lead
GB, lead, business"] + R_2["Platform Engineer
PL, senior, business"] + R_3["Platform SRE
IN, engineer, 24x7"] + R_6["Platform Product Manager
GB, senior, business"] + R_7["Compliance Analyst
PL, senior, business"] + end + subgraph T_3["Platform Security Team (complicated-subsystem team)"] + R_4["Security Operations Analyst
PL, senior, 24x7"] + end + subgraph T_4["Developer Enablement Team (enabling team)"] + R_5["Developer Enabler
GB, senior, business"] + end + end + T_2 -- "x-as-a-service" --> T_1 + T_4 -- "facilitating" --> T_2 diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scenarios.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scenarios.json new file mode 100644 index 0000000..933e165 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scenarios.json @@ -0,0 +1,219 @@ +[ + { + "id": "SC-fault-S-1", + "scenario": "Resource Hierarchy and Environment Boundary Manager (S-1) fails at an hour faults are responded to (24x7).", + "about": "S-1", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "id": "SC-change-S-1", + "scenario": "Resource Hierarchy and Environment Boundary Manager (S-1) must be changed.", + "about": "S-1", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-2", + "scenario": "Dedicated Private Hybrid Connectivity Gateway (S-2) fails at an hour faults are responded to (24x7).", + "about": "S-2", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "id": "SC-change-S-2", + "scenario": "Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed.", + "about": "S-2", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-3", + "scenario": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) fails at an hour faults are responded to (24x7).", + "about": "S-3", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "id": "SC-change-S-3", + "scenario": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed.", + "about": "S-3", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-4", + "scenario": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) fails at an hour faults are responded to (24x7).", + "about": "S-4", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "id": "SC-change-S-4", + "scenario": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) must be changed.", + "about": "S-4", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-5", + "scenario": "Automated Infrastructure Orchestration and Deployment Engine (S-5) fails at an hour faults are responded to (24x7).", + "about": "S-5", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "id": "SC-change-S-5", + "scenario": "Automated Infrastructure Orchestration and Deployment Engine (S-5) must be changed.", + "about": "S-5", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-6", + "scenario": "Federated Identity and Role Governance Authority (S-6) fails at an hour faults are responded to (24x7).", + "about": "S-6", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "id": "SC-change-S-6", + "scenario": "Federated Identity and Role Governance Authority (S-6) must be changed.", + "about": "S-6", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-7", + "scenario": "Cryptographic Key Lifecycle and Encryption Controller (S-7) fails at an hour faults are responded to (24x7).", + "about": "S-7", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "id": "SC-change-S-7", + "scenario": "Cryptographic Key Lifecycle and Encryption Controller (S-7) must be changed.", + "about": "S-7", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-8", + "scenario": "Platform Telemetry and Operational Alerting Engine (S-8) fails at an hour faults are responded to (24x7).", + "about": "S-8", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "id": "SC-change-S-8", + "scenario": "Platform Telemetry and Operational Alerting Engine (S-8) must be changed.", + "about": "S-8", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-9", + "scenario": "Financial Governance and Budget Enforcement Monitor (S-9) fails at an hour faults are responded to (24x7).", + "about": "S-9", + "passes": true, + "path": "Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "id": "SC-change-S-9", + "scenario": "Financial Governance and Budget Enforcement Monitor (S-9) must be changed.", + "about": "S-9", + "passes": true, + "path": "Platform Engineer makes the change under C-5." + }, + { + "id": "SC-condition-K-1", + "scenario": "Condition K-1 must be shown to hold: The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit.", + "about": "K-1", + "passes": true, + "path": "Check OC-17 on C-7 keeps it true." + }, + { + "id": "SC-condition-K-2", + "scenario": "Condition K-2 must be shown to hold: The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic.", + "about": "K-2", + "passes": true, + "path": "Check OC-2 on C-9 keeps it true." + }, + { + "id": "SC-condition-K-3", + "scenario": "Condition K-3 must be shown to hold: Data residency is guaranteed within the United Kingdom and European Union.", + "about": "K-3", + "passes": true, + "path": "Check OC-3 on C-21 keeps it true." + }, + { + "id": "SC-condition-K-4", + "scenario": "Condition K-4 must be shown to hold: Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks.", + "about": "K-4", + "passes": true, + "path": "Check OC-4 on C-16 keeps it true." + }, + { + "id": "SC-condition-K-5", + "scenario": "Condition K-5 must be shown to hold: Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.", + "about": "K-5", + "passes": true, + "path": "Check OC-5 on C-15 keeps it true." + }, + { + "id": "SC-condition-K-6", + "scenario": "Condition K-6 must be shown to hold: Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.", + "about": "K-6", + "passes": true, + "path": "Check OC-6 on C-9 keeps it true." + }, + { + "id": "SC-condition-K-7", + "scenario": "Condition K-7 must be shown to hold: Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour.", + "about": "K-7", + "passes": true, + "path": "Check OC-7 on C-8 keeps it true." + }, + { + "id": "SC-condition-K-8", + "scenario": "Condition K-8 must be shown to hold: Alert notifications trigger operational incident workflows for the 24/7 central platform operations team.", + "about": "K-8", + "passes": true, + "path": "Check OC-8 on C-2 keeps it true." + }, + { + "id": "SC-condition-K-9", + "scenario": "Condition K-9 must be shown to hold: Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off.", + "about": "K-9", + "passes": true, + "path": "Check OC-9 on C-14 keeps it true." + }, + { + "id": "SC-condition-K-10", + "scenario": "Condition K-10 must be shown to hold: Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing.", + "about": "K-10", + "passes": true, + "path": "Check OC-10 on C-10 keeps it true." + }, + { + "id": "SC-condition-K-11", + "scenario": "Condition K-11 must be shown to hold: Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores.", + "about": "K-11", + "passes": true, + "path": "Check OC-11 on C-6 keeps it true." + }, + { + "id": "SC-condition-K-12", + "scenario": "Condition K-12 must be shown to hold: Permissions are continuously rightsized based on IAM Recommender findings.", + "about": "K-12", + "passes": true, + "path": "Check OC-12 on C-11 keeps it true." + }, + { + "id": "SC-party-P-2", + "scenario": "Platform Provider cannot be reached.", + "about": "P-2", + "passes": true, + "path": "Every Capability it is answerable for has a Role in another Party to turn to." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scope.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scope.json new file mode 100644 index 0000000..a3eee66 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scope.json @@ -0,0 +1,396 @@ +{ + "kind": "platform_foundation", + "stated": { + "engagement": { + "kind": "platform_foundation" + }, + "intent": { + "statement": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control." + }, + "solution": { + "cloud_provider": "gcp", + "items": [ + { + "id": "S-1", + "name": "Resource Hierarchy and Environment Boundary Manager", + "kind": "governance", + "product": "Resource Manager", + "source_ref": "C-1", + "elements": [ + "E-1", + "E-2", + "E-3", + "E-4", + "E-5", + "E-6", + "E-7", + "E-8", + "E-9", + "E-10", + "E-34", + "E-35", + "E-36", + "E-37", + "E-38", + "E-50", + "E-51", + "E-55", + "E-60" + ] + }, + { + "id": "S-2", + "name": "Dedicated Private Hybrid Connectivity Gateway", + "kind": "network", + "product": "Cloud Interconnect", + "source_ref": "C-2", + "elements": [ + "E-11", + "E-12", + "E-13", + "E-14", + "E-56", + "E-57" + ] + }, + { + "id": "S-3", + "name": "Centralised Outbound Traffic Inspection and Egress Boundary", + "kind": "service", + "product": "Palo Alto Networks VM-Series", + "source_ref": "C-3", + "elements": [ + "E-15", + "E-16", + "E-17", + "E-18", + "E-19", + "E-20", + "E-47", + "E-48", + "E-53" + ] + }, + { + "id": "S-4", + "name": "Centralised Immutable Audit Event and Compliance Log Repository", + "kind": "storage", + "product": "Cloud Storage", + "source_ref": "C-4", + "elements": [ + "E-21", + "E-22", + "E-39" + ] + }, + { + "id": "S-5", + "name": "Automated Infrastructure Orchestration and Deployment Engine", + "kind": "service", + "product": "GitHub Actions", + "source_ref": "C-5", + "elements": [ + "E-42", + "E-43" + ] + }, + { + "id": "S-6", + "name": "Federated Identity and Role Governance Authority", + "kind": "identity", + "product": "IAM Workload Identity Federation", + "source_ref": "C-6", + "elements": [ + "E-23", + "E-24", + "E-25", + "E-26", + "E-27", + "E-45" + ] + }, + { + "id": "S-7", + "name": "Cryptographic Key Lifecycle and Encryption Controller", + "kind": "keys", + "product": "Cloud KMS", + "source_ref": "C-7", + "elements": [ + "E-28", + "E-29", + "E-30", + "E-31", + "E-49", + "E-58", + "E-59" + ] + }, + { + "id": "S-8", + "name": "Platform Telemetry and Operational Alerting Engine", + "kind": "observability", + "product": "Cloud Monitoring", + "source_ref": "C-8", + "elements": [ + "E-32", + "E-40", + "E-41", + "E-44", + "E-52" + ] + }, + { + "id": "S-9", + "name": "Financial Governance and Budget Enforcement Monitor", + "kind": "billing", + "product": "Cloud Billing", + "source_ref": "C-9", + "elements": [ + "E-33", + "E-46", + "E-54" + ] + } + ], + "conditions": [ + { + "id": "K-1", + "on": [ + "S-1", + "S-4", + "S-7" + ], + "condition": "The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit.", + "source_ref": "NFR-8" + }, + { + "id": "K-2", + "on": [ + "S-1" + ], + "condition": "The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic.", + "source_ref": "NFR-9" + }, + { + "id": "K-3", + "on": [ + "S-1" + ], + "condition": "Data residency is guaranteed within the United Kingdom and European Union.", + "source_ref": "NFR-9" + }, + { + "id": "K-4", + "on": [ + "S-5" + ], + "condition": "Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks.", + "source_ref": "NFR-10" + }, + { + "id": "K-5", + "on": [ + "S-8" + ], + "condition": "Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.", + "source_ref": "NFR-11" + }, + { + "id": "K-6", + "on": [ + "S-8" + ], + "condition": "Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.", + "source_ref": "NFR-11" + }, + { + "id": "K-7", + "on": [ + "S-8" + ], + "condition": "Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour.", + "source_ref": "NFR-11" + }, + { + "id": "K-8", + "on": [ + "S-8" + ], + "condition": "Alert notifications trigger operational incident workflows for the 24/7 central platform operations team.", + "source_ref": "NFR-12" + }, + { + "id": "K-9", + "on": [ + "S-9" + ], + "condition": "Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off.", + "source_ref": "NFR-13" + }, + { + "id": "K-10", + "on": [ + "S-2" + ], + "condition": "Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing.", + "source_ref": "NFR-14" + }, + { + "id": "K-11", + "on": [ + "S-8" + ], + "condition": "Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores.", + "source_ref": "NFR-15" + }, + { + "id": "K-12", + "on": [ + "S-6" + ], + "condition": "Permissions are continuously rightsized based on IAM Recommender findings.", + "source_ref": "DR-OP-3" + } + ], + "build": { + "duration_days": 36, + "customer_owned": [ + "Order and provision the Interconnect carrier circuits" + ] + } + }, + "operations": { + "support_hours": "24x7", + "availability_hours": "24x7" + }, + "sourcing": { + "model": "provider_operated", + "delivery_countries": [ + "GB", + "PL", + "IN" + ] + }, + "compliance": { + "frameworks": [ + "DORA", + "ISO27001", + "PCI_DSS_v4" + ] + }, + "commercial": { + "currency": "GBP" + } + }, + "binding": [ + "operations.support_hours", + "sourcing.model", + "sourcing.delivery_countries" + ], + "assumed": [ + { + "field": "intent.business_outcomes", + "value": [ + { + "outcome": "Enterprise landing zone ready to ingest core payments engine workloads within 10 weeks", + "id": "BO-1", + "target": "Workload ready within 10 weeks" + }, + { + "outcome": "Continuous operational resilience and regulatory compliance under DORA and PCI-DSS v4.0", + "target": "Zero audit non-conformances and >= 99.9% platform availability", + "id": "BO-2" + }, + { + "id": "BO-3", + "target": "Continuous 10Gbps dedicated interconnect availability with centralized inspection", + "outcome": "Secure, deterministic low-latency hybrid connectivity supporting real-time payment transactions" + } + ], + "why": "Derived from the workflow request specifying rapid 10-week landing zone delivery, DORA and PCI-DSS v4.0 compliance, and deterministic hybrid connectivity." + }, + { + "field": "intent.exclusions", + "value": [ + "Application workload code, payment engine business logic, and database-level transaction data management", + "Physical data centre hosting, hardware maintenance, and on-premises network infrastructure", + "Corporate Entra ID user identity lifecycle management outside Google Cloud federation" + ], + "why": "Assumed from platform foundation scope: excludes application-level payment processing logic, physical on-premises hardware maintenance, and end-user identity administration beyond cloud federation." + }, + { + "field": "operations.change_process", + "value": "Automated infrastructure-as-code deployment via GitHub Actions pipelines with automated linting, security scans, and peer review approvals without a manual CAB.", + "why": "Assumed based on catalogue default and stated use of GitHub Actions delivery pipelines." + }, + { + "field": "organisation.existing_teams", + "value": [], + "why": "Assumed catalogue default: no existing customer operational roles carry forward into the platform foundation; operations are delivered under a provider-operated model." + }, + { + "field": "service_management.tooling", + "value": "Google Cloud operations suite (Cloud Monitoring, Cloud Logging) integrated with provider operational incident and alerting management systems.", + "why": "Assumed based on catalogue default and solution item S-8 (Cloud Monitoring)." + }, + { + "field": "service_levels.priority_1_response_minutes", + "value": 30, + "why": "Assumed catalogue default of 30 minutes for critical incident response." + }, + { + "field": "service_levels.recovery_time_hours", + "value": 4, + "why": "Derived from solution condition K-6 specifying a disaster recovery RTO of 4 hours, matching catalogue default." + } + ], + "to_decide": [], + "to_ask": [ + { + "field": "metadata.client_name", + "prompt": "What is metadata.client_name?", + "severity": "LOW" + }, + { + "field": "metadata.author", + "prompt": "What is metadata.author?", + "severity": "LOW" + }, + { + "field": "metadata.revision", + "prompt": "What is metadata.revision?", + "severity": "LOW" + }, + { + "field": "intent.requirements", + "prompt": "Which requirements for operations has the customer stated outright?", + "severity": "LOW" + }, + { + "field": "solution.inherited_risks", + "prompt": "What is already known to be unverified or at risk in the solution?", + "severity": "LOW" + }, + { + "field": "operations.monitoring", + "prompt": "Which monitoring and alerting is already in place or required?", + "severity": "LOW" + }, + { + "field": "organisation.headcount_limit", + "prompt": "Is there a limit on the number of people operating this?", + "severity": "LOW" + }, + { + "field": "commercial.monthly_run_budget", + "prompt": "Is there a monthly budget for operating this, in that currency?", + "severity": "LOW" + }, + { + "field": "transition.go_live_date", + "prompt": "When must the solution be in operation?", + "severity": "LOW" + } + ], + "not_applicable": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scope.yaml new file mode 100644 index 0000000..aa6d2e4 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/scope.yaml @@ -0,0 +1,406 @@ +schema_version: 1 +metadata: + customer_case: case-08-landing-zone + client_name: ask + author: ask + revision: ask + source: architecture:case-08-landing-zone-185426 + status: APPROVED +engagement: + kind: platform_foundation +intent: + statement: 'We are a UK-regulated payment platform expanding into Google Cloud. + We need an + + enterprise-grade landing zone ready in under 10 weeks to support our core payments + + engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have + + existing on-premises systems that require private, low-latency connectivity over + a + + 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and + + GitHub Actions for our delivery pipelines. Security requires that all egress + + traffic flows through centralised inspection appliances before touching the public + + internet, and customer data must be encrypted with keys we control.' + requirements: ask + business_outcomes: + - outcome: Enterprise landing zone ready to ingest core payments engine workloads + within 10 weeks + id: BO-1 + target: Workload ready within 10 weeks + - outcome: Continuous operational resilience and regulatory compliance under DORA + and PCI-DSS v4.0 + target: Zero audit non-conformances and >= 99.9% platform availability + id: BO-2 + - id: BO-3 + target: Continuous 10Gbps dedicated interconnect availability with centralized + inspection + outcome: Secure, deterministic low-latency hybrid connectivity supporting real-time + payment transactions + exclusions: + - Application workload code, payment engine business logic, and database-level transaction + data management + - Physical data centre hosting, hardware maintenance, and on-premises network infrastructure + - Corporate Entra ID user identity lifecycle management outside Google Cloud federation +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Resource Hierarchy and Environment Boundary Manager + kind: governance + product: Resource Manager + source_ref: C-1 + elements: + - E-1 + - E-2 + - E-3 + - E-4 + - E-5 + - E-6 + - E-7 + - E-8 + - E-9 + - E-10 + - E-34 + - E-35 + - E-36 + - E-37 + - E-38 + - E-50 + - E-51 + - E-55 + - E-60 + - id: S-2 + name: Dedicated Private Hybrid Connectivity Gateway + kind: network + product: Cloud Interconnect + source_ref: C-2 + elements: + - E-11 + - E-12 + - E-13 + - E-14 + - E-56 + - E-57 + - id: S-3 + name: Centralised Outbound Traffic Inspection and Egress Boundary + kind: service + product: Palo Alto Networks VM-Series + source_ref: C-3 + elements: + - E-15 + - E-16 + - E-17 + - E-18 + - E-19 + - E-20 + - E-47 + - E-48 + - E-53 + - id: S-4 + name: Centralised Immutable Audit Event and Compliance Log Repository + kind: storage + product: Cloud Storage + source_ref: C-4 + elements: + - E-21 + - E-22 + - E-39 + - id: S-5 + name: Automated Infrastructure Orchestration and Deployment Engine + kind: service + product: GitHub Actions + source_ref: C-5 + elements: + - E-42 + - E-43 + - id: S-6 + name: Federated Identity and Role Governance Authority + kind: identity + product: IAM Workload Identity Federation + source_ref: C-6 + elements: + - E-23 + - E-24 + - E-25 + - E-26 + - E-27 + - E-45 + - id: S-7 + name: Cryptographic Key Lifecycle and Encryption Controller + kind: keys + product: Cloud KMS + source_ref: C-7 + elements: + - E-28 + - E-29 + - E-30 + - E-31 + - E-49 + - E-58 + - E-59 + - id: S-8 + name: Platform Telemetry and Operational Alerting Engine + kind: observability + product: Cloud Monitoring + source_ref: C-8 + elements: + - E-32 + - E-40 + - E-41 + - E-44 + - E-52 + - id: S-9 + name: Financial Governance and Budget Enforcement Monitor + kind: billing + product: Cloud Billing + source_ref: C-9 + elements: + - E-33 + - E-46 + - E-54 + conditions: + - id: K-1 + 'on': + - S-1 + - S-4 + - S-7 + condition: The landing zone configuration aligns with DORA operational resiliency + guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. + source_ref: NFR-8 + - id: K-2 + 'on': + - S-1 + condition: The secondary region in europe-west1 operates as a warm standby ready + to receive failover traffic. + source_ref: NFR-9 + - id: K-3 + 'on': + - S-1 + condition: Data residency is guaranteed within the United Kingdom and European + Union. + source_ref: NFR-9 + - id: K-4 + 'on': + - S-5 + condition: Platform foundation is fully provisioned and validated for core payments + engine workload ingestion within 10 weeks. + source_ref: NFR-10 + - id: K-5 + 'on': + - S-8 + condition: Platform foundation infrastructure maintains measured monthly availability + of at least 99.9%. + source_ref: NFR-11 + - id: K-6 + 'on': + - S-8 + condition: Disaster recovery activation achieves a recovery time objective (RTO) + of 4 hours. + source_ref: NFR-11 + - id: K-7 + 'on': + - S-8 + condition: Disaster recovery processes achieve a recovery point objective (RPO) + of 1 hour. + source_ref: NFR-11 + - id: K-8 + 'on': + - S-8 + condition: Alert notifications trigger operational incident workflows for the + 24/7 central platform operations team. + source_ref: NFR-12 + - id: K-9 + 'on': + - S-9 + condition: Monthly cloud foundation expenditure does not exceed the target allocation + without operational sign-off. + source_ref: NFR-13 + - id: K-10 + 'on': + - S-2 + condition: Hybrid connection provides deterministic low latency suitable for real-time + payment transaction processing. + source_ref: NFR-14 + - id: K-11 + 'on': + - S-8 + condition: Deployments optimize carbon-free energy utilization based on Google + Cloud regional CFE scores. + source_ref: NFR-15 + - id: K-12 + 'on': + - S-6 + condition: Permissions are continuously rightsized based on IAM Recommender findings. + source_ref: DR-OP-3 + build: + duration_days: 36 + customer_owned: + - Order and provision the Interconnect carrier circuits + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Automated infrastructure-as-code deployment via GitHub Actions pipelines + with automated linting, security scans, and peer review approvals without a manual + CAB. + monitoring: ask +sourcing: + model: provider_operated + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: [] + headcount_limit: ask +service_management: + tooling: Google Cloud operations suite (Cloud Monitoring, Cloud Logging) integrated + with provider operational incident and alerting management systems. +service_levels: + priority_1_response_minutes: 30 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA + - ISO27001 + - PCI_DSS_v4 +commercial: + currency: GBP + monthly_run_budget: ask +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.business_outcomes + statement: 'Assumed intent.business_outcomes is [{''outcome'': ''Enterprise landing + zone ready to ingest core payments engine workloads within 10 weeks'', ''id'': + ''BO-1'', ''target'': ''Workload ready within 10 weeks''}, {''outcome'': ''Continuous + operational resilience and regulatory compliance under DORA and PCI-DSS v4.0'', + ''target'': ''Zero audit non-conformances and >= 99.9% platform availability'', + ''id'': ''BO-2''}, {''id'': ''BO-3'', ''target'': ''Continuous 10Gbps dedicated + interconnect availability with centralized inspection'', ''outcome'': ''Secure, + deterministic low-latency hybrid connectivity supporting real-time payment transactions''}]. + Derived from the workflow request specifying rapid 10-week landing zone delivery, + DORA and PCI-DSS v4.0 compliance, and deterministic hybrid connectivity.' + impact_if_invalid: Changes what is measured, and what each measure is for. +- id: ASSUMP-02 + field: intent.exclusions + statement: 'Assumed intent.exclusions is [''Application workload code, payment engine + business logic, and database-level transaction data management'', ''Physical data + centre hosting, hardware maintenance, and on-premises network infrastructure'', + ''Corporate Entra ID user identity lifecycle management outside Google Cloud federation'']. + Assumed from platform foundation scope: excludes application-level payment processing + logic, physical on-premises hardware maintenance, and end-user identity administration + beyond cloud federation.' + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-03 + field: operations.change_process + statement: Assumed operations.change_process is Automated infrastructure-as-code + deployment via GitHub Actions pipelines with automated linting, security scans, + and peer review approvals without a manual CAB.. Assumed based on catalogue default + and stated use of GitHub Actions delivery pipelines. + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-04 + field: organisation.existing_teams + statement: 'Assumed organisation.existing_teams is []. Assumed catalogue default: + no existing customer operational roles carry forward into the platform foundation; + operations are delivered under a provider-operated model.' + impact_if_invalid: Changes which Roles are new and which are filled. +- id: ASSUMP-05 + field: service_management.tooling + statement: Assumed service_management.tooling is Google Cloud operations suite (Cloud + Monitoring, Cloud Logging) integrated with provider operational incident and alerting + management systems.. Assumed based on catalogue default and solution item S-8 + (Cloud Monitoring). + impact_if_invalid: Changes tooling and the processes written for it. +- id: ASSUMP-06 + field: service_levels.priority_1_response_minutes + statement: Assumed service_levels.priority_1_response_minutes is 30. Assumed catalogue + default of 30 minutes for critical incident response. + impact_if_invalid: Changes on-call arrangements and staffing. +- id: ASSUMP-07 + field: service_levels.recovery_time_hours + statement: Assumed service_levels.recovery_time_hours is 4. Derived from solution + condition K-6 specifying a disaster recovery RTO of 4 hours, matching catalogue + default. + impact_if_invalid: Changes continuity arrangements. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: intent.requirements + prompt: Which requirements for operations has the customer stated outright? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: commercial.monthly_run_budget + prompt: Is there a monthly budget for operating this, in that currency? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: digested + commercial.monthly_run_budget: unknown + compliance.frameworks: digested + engagement.kind: digested + intent.business_outcomes: assumed + intent.exclusions: assumed + intent.requirements: unknown + intent.statement: digested + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: digested + metadata.revision: unknown + metadata.source: digested + operations.availability_hours: digested + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: digested + organisation.existing_teams: assumed + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: assumed + service_levels.recovery_time_hours: assumed + service_management.tooling: assumed + solution.build.customer_owned: digested + solution.build.duration_days: digested + solution.cloud_provider: digested + solution.conditions: digested + solution.inherited_risks: unknown + solution.items: digested + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 3 + answered: 0 + assumed: 7 + unknown: 10 + not_applicable: 0 + digested: 13 diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/staffing.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/staffing.json new file mode 100644 index 0000000..324d760 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/staffing.json @@ -0,0 +1,144 @@ +{ + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + }, + "conversion": { + "from": "USD", + "rate": 0.79, + "as_of": "2026-10-13" + }, + "roles": [ + { + "role": "R-1", + "name": "Platform Foundation Lead", + "party": "P-2", + "party_name": "Platform Provider", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 30.0, + "people": 0.2, + "cost_per_month": 3318 + }, + { + "role": "R-2", + "name": "Platform Engineer", + "party": "P-2", + "party_name": "Platform Provider", + "country": "PL", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 160.0, + "people": 1.1, + "cost_per_month": 7268 + }, + { + "role": "R-3", + "name": "Platform SRE", + "party": "P-2", + "party_name": "Platform Provider", + "country": "IN", + "level": "engineer", + "hours": "24x7", + "effort_hours_per_month": 70.0, + "people": 5.0, + "cost_per_month": 17034 + }, + { + "role": "R-4", + "name": "Security Operations Analyst", + "party": "P-2", + "party_name": "Platform Provider", + "country": "PL", + "level": "senior", + "hours": "24x7", + "effort_hours_per_month": 88.0, + "people": 5.0, + "cost_per_month": 41640 + }, + { + "role": "R-5", + "name": "Developer Enabler", + "party": "P-2", + "party_name": "Platform Provider", + "country": "GB", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 30.0, + "people": 0.2, + "cost_per_month": 2528 + }, + { + "role": "R-6", + "name": "Platform Product Manager", + "party": "P-2", + "party_name": "Platform Provider", + "country": "GB", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 16.0, + "people": 0.1, + "cost_per_month": 1348 + }, + { + "role": "R-7", + "name": "Compliance Analyst", + "party": "P-2", + "party_name": "Platform Provider", + "country": "PL", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 16.0, + "people": 0.1, + "cost_per_month": 727 + }, + { + "role": "R-8", + "name": "Core Payments Team Lead", + "party": "P-1", + "party_name": "Customer", + "country": "GB", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 0.0, + "people": 0.0, + "cost_per_month": null + } + ], + "people": 11.7, + "cost_per_month": 73863, + "by_party": { + "P-2": 73863 + }, + "by_country": { + "GB": 7194, + "PL": 49635, + "IN": 17034 + }, + "by_capability": { + "C-14": 1327, + "C-15": 1106, + "C-20": 885, + "C-3": 727, + "C-4": 909, + "C-5": 909, + "C-6": 1363, + "C-7": 545, + "C-8": 454, + "C-9": 909, + "C-10": 545, + "C-17": 909, + "C-1": 9734, + "C-2": 7300, + "C-11": 11356, + "C-12": 11356, + "C-13": 18927, + "C-18": 1685, + "C-19": 843, + "C-16": 1348, + "C-21": 727 + } +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/target-operating-model.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/target-operating-model.json new file mode 100644 index 0000000..220f23c --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/target-operating-model.json @@ -0,0 +1,667 @@ +{ + "parties": [ + { + "kind": "customer", + "id": "P-1", + "name": "Customer" + }, + { + "kind": "provider", + "id": "P-2", + "name": "Platform Provider" + } + ], + "teams": [ + { + "party": "P-2", + "id": "T-1", + "name": "Platform Foundation Team", + "type": "platform" + }, + { + "name": "Core Payments Engine Team", + "type": "stream_aligned", + "party": "P-1", + "id": "T-2" + }, + { + "id": "T-3", + "party": "P-2", + "name": "Platform Security Team", + "type": "complicated_subsystem" + }, + { + "party": "P-2", + "name": "Developer Enablement Team", + "type": "enabling", + "id": "T-4" + } + ], + "roles": [ + { + "hours": "business", + "level": "lead", + "name": "Platform Foundation Lead", + "team": "T-1", + "party": "P-2", + "country": "GB", + "id": "R-1" + }, + { + "country": "PL", + "id": "R-2", + "name": "Platform Engineer", + "party": "P-2", + "hours": "business", + "level": "senior", + "team": "T-1" + }, + { + "hours": "24x7", + "level": "engineer", + "id": "R-3", + "country": "IN", + "name": "Platform SRE", + "team": "T-1", + "party": "P-2" + }, + { + "country": "PL", + "team": "T-3", + "party": "P-2", + "id": "R-4", + "hours": "24x7", + "level": "senior", + "name": "Security Operations Analyst" + }, + { + "level": "senior", + "hours": "business", + "team": "T-4", + "country": "GB", + "party": "P-2", + "name": "Developer Enabler", + "id": "R-5" + }, + { + "party": "P-2", + "team": "T-1", + "country": "GB", + "name": "Platform Product Manager", + "hours": "business", + "level": "senior", + "id": "R-6" + }, + { + "hours": "business", + "level": "senior", + "name": "Compliance Analyst", + "team": "T-1", + "country": "PL", + "party": "P-2", + "id": "R-7" + }, + { + "id": "R-8", + "level": "senior", + "hours": "business", + "country": "GB", + "name": "Core Payments Team Lead", + "team": "T-2", + "party": "P-1" + } + ], + "interactions": [ + { + "id": "I-1", + "mode": "x_as_a_service", + "to": "T-1", + "purpose": "Stream-aligned teams request environments and paved paths via the self-service foundation.", + "from": "T-2" + }, + { + "mode": "facilitating", + "to": "T-2", + "purpose": "Enablement team assists core payments engineers to adopt the platform landing zone.", + "id": "I-2", + "from": "T-4" + } + ], + "measures": [ + { + "outcome": "BO-1", + "name": "Deployment Frequency", + "framework": "dora-metrics", + "id": "M-1", + "layer": "outcome", + "measured_by": "C-19", + "target": "Daily" + }, + { + "name": "Satisfaction and Well-being", + "id": "M-2", + "outcome": "BO-1", + "layer": "guardrail", + "framework": "space-framework", + "target": "High satisfaction scores", + "measured_by": "C-19" + }, + { + "framework": "devex-framework", + "name": "Cognitive Load", + "measured_by": "C-19", + "outcome": "BO-1", + "layer": "leading", + "target": "Low complexity ratings", + "id": "M-3" + }, + { + "id": "M-4", + "outcome": "BO-2", + "target": "99.9%", + "name": "Platform Availability", + "layer": "service_level", + "measured_by": "C-15" + }, + { + "outcome": "BO-2", + "id": "M-5", + "name": "Compliance Conformance", + "measured_by": "C-21", + "layer": "service_level", + "target": "Zero non-conformances" + }, + { + "id": "M-6", + "target": "Deterministic low-latency", + "layer": "service_level", + "name": "Network Latency", + "measured_by": "C-10", + "outcome": "BO-3" + } + ], + "responsibilities": [ + { + "role": "R-1", + "kind": "accountable", + "capability": "C-1", + "id": "RS-1" + }, + { + "id": "RS-2", + "capability": "C-1", + "kind": "responsible", + "role": "R-3" + }, + { + "role": "R-8", + "kind": "consulted", + "capability": "C-1", + "id": "RS-3" + }, + { + "id": "RS-4", + "role": "R-1", + "kind": "accountable", + "capability": "C-2" + }, + { + "capability": "C-2", + "id": "RS-5", + "role": "R-3", + "kind": "responsible" + }, + { + "capability": "C-2", + "kind": "consulted", + "role": "R-8", + "id": "RS-6" + }, + { + "id": "RS-7", + "role": "R-1", + "kind": "accountable", + "capability": "C-3" + }, + { + "kind": "responsible", + "role": "R-2", + "capability": "C-3", + "id": "RS-8" + }, + { + "capability": "C-3", + "role": "R-8", + "kind": "consulted", + "id": "RS-9" + }, + { + "id": "RS-10", + "kind": "accountable", + "role": "R-1", + "capability": "C-4" + }, + { + "id": "RS-11", + "capability": "C-4", + "kind": "responsible", + "role": "R-2" + }, + { + "role": "R-8", + "kind": "consulted", + "id": "RS-12", + "capability": "C-4" + }, + { + "capability": "C-5", + "id": "RS-13", + "kind": "accountable", + "role": "R-1" + }, + { + "capability": "C-5", + "id": "RS-14", + "role": "R-2", + "kind": "responsible" + }, + { + "role": "R-8", + "kind": "consulted", + "id": "RS-15", + "capability": "C-5" + }, + { + "role": "R-1", + "id": "RS-16", + "capability": "C-6", + "kind": "accountable" + }, + { + "kind": "responsible", + "capability": "C-6", + "id": "RS-17", + "role": "R-2" + }, + { + "capability": "C-6", + "role": "R-8", + "kind": "consulted", + "id": "RS-18" + }, + { + "capability": "C-7", + "kind": "accountable", + "role": "R-1", + "id": "RS-19" + }, + { + "role": "R-2", + "capability": "C-7", + "kind": "responsible", + "id": "RS-20" + }, + { + "id": "RS-21", + "kind": "consulted", + "capability": "C-7", + "role": "R-8" + }, + { + "role": "R-1", + "id": "RS-22", + "capability": "C-8", + "kind": "accountable" + }, + { + "id": "RS-23", + "role": "R-2", + "kind": "responsible", + "capability": "C-8" + }, + { + "kind": "consulted", + "capability": "C-8", + "id": "RS-24", + "role": "R-8" + }, + { + "role": "R-1", + "id": "RS-25", + "capability": "C-9", + "kind": "accountable" + }, + { + "kind": "responsible", + "id": "RS-26", + "capability": "C-9", + "role": "R-2" + }, + { + "role": "R-8", + "capability": "C-9", + "kind": "consulted", + "id": "RS-27" + }, + { + "kind": "accountable", + "capability": "C-10", + "id": "RS-28", + "role": "R-1" + }, + { + "role": "R-2", + "capability": "C-10", + "id": "RS-29", + "kind": "responsible" + }, + { + "role": "R-8", + "capability": "C-10", + "id": "RS-30", + "kind": "consulted" + }, + { + "kind": "accountable", + "id": "RS-31", + "capability": "C-11", + "role": "R-4" + }, + { + "kind": "responsible", + "id": "RS-32", + "role": "R-4", + "capability": "C-11" + }, + { + "capability": "C-11", + "kind": "consulted", + "id": "RS-33", + "role": "R-8" + }, + { + "role": "R-4", + "capability": "C-12", + "id": "RS-34", + "kind": "accountable" + }, + { + "id": "RS-35", + "capability": "C-12", + "kind": "responsible", + "role": "R-4" + }, + { + "id": "RS-36", + "kind": "consulted", + "role": "R-8", + "capability": "C-12" + }, + { + "kind": "accountable", + "role": "R-4", + "id": "RS-37", + "capability": "C-13" + }, + { + "role": "R-4", + "kind": "responsible", + "capability": "C-13", + "id": "RS-38" + }, + { + "id": "RS-39", + "capability": "C-13", + "kind": "consulted", + "role": "R-8" + }, + { + "capability": "C-14", + "role": "R-1", + "kind": "accountable", + "id": "RS-40" + }, + { + "id": "RS-41", + "kind": "responsible", + "role": "R-1", + "capability": "C-14" + }, + { + "kind": "consulted", + "role": "R-8", + "id": "RS-42", + "capability": "C-14" + }, + { + "kind": "accountable", + "role": "R-1", + "capability": "C-15", + "id": "RS-43" + }, + { + "id": "RS-44", + "role": "R-1", + "kind": "responsible", + "capability": "C-15" + }, + { + "kind": "consulted", + "role": "R-8", + "capability": "C-15", + "id": "RS-45" + }, + { + "id": "RS-46", + "capability": "C-16", + "role": "R-6", + "kind": "accountable" + }, + { + "role": "R-6", + "kind": "responsible", + "id": "RS-47", + "capability": "C-16" + }, + { + "role": "R-8", + "kind": "consulted", + "id": "RS-48", + "capability": "C-16" + }, + { + "kind": "accountable", + "role": "R-6", + "id": "RS-49", + "capability": "C-17" + }, + { + "capability": "C-17", + "id": "RS-50", + "kind": "responsible", + "role": "R-2" + }, + { + "id": "RS-51", + "capability": "C-17", + "role": "R-8", + "kind": "consulted" + }, + { + "id": "RS-52", + "role": "R-6", + "kind": "accountable", + "capability": "C-18" + }, + { + "kind": "responsible", + "role": "R-5", + "id": "RS-53", + "capability": "C-18" + }, + { + "capability": "C-18", + "role": "R-8", + "kind": "consulted", + "id": "RS-54" + }, + { + "capability": "C-19", + "kind": "accountable", + "role": "R-6", + "id": "RS-55" + }, + { + "role": "R-5", + "kind": "responsible", + "capability": "C-19", + "id": "RS-56" + }, + { + "role": "R-8", + "kind": "consulted", + "capability": "C-19", + "id": "RS-57" + }, + { + "id": "RS-58", + "capability": "C-20", + "kind": "accountable", + "role": "R-1" + }, + { + "id": "RS-59", + "capability": "C-20", + "role": "R-1", + "kind": "responsible" + }, + { + "role": "R-8", + "kind": "consulted", + "capability": "C-20", + "id": "RS-60" + }, + { + "id": "RS-61", + "capability": "C-21", + "kind": "accountable", + "role": "R-7" + }, + { + "capability": "C-21", + "kind": "responsible", + "role": "R-7", + "id": "RS-62" + }, + { + "kind": "consulted", + "role": "R-8", + "capability": "C-21", + "id": "RS-63" + } + ], + "checks": [ + { + "condition": "A Priority 1 incident on any platform foundation item is acknowledged within 30 minutes, at any hour of any day.", + "id": "OC-1", + "source": "OR-1", + "on": "C-1" + }, + { + "condition": "The secondary region in europe-west1 is continuously maintained as an active warm standby ready to receive failover traffic.", + "id": "OC-2", + "on": "C-9", + "source": "K-2" + }, + { + "on": "C-21", + "source": "K-3", + "id": "OC-3", + "condition": "Data residency is guaranteed within the United Kingdom and European Union for all platform foundation resources and stored data, proven by continuous audit evidence." + }, + { + "id": "OC-4", + "source": "K-4", + "condition": "Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks.", + "on": "C-16" + }, + { + "on": "C-15", + "source": "K-5", + "id": "OC-5", + "condition": "Platform foundation infrastructure maintains measured monthly availability of at least 99.9%." + }, + { + "condition": "Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.", + "on": "C-9", + "id": "OC-6", + "source": "K-6" + }, + { + "id": "OC-7", + "on": "C-8", + "source": "K-7", + "condition": "Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour." + }, + { + "condition": "Alert notifications trigger operational incident workflows for the 24/7 central platform operations team.", + "id": "OC-8", + "on": "C-2", + "source": "K-8" + }, + { + "condition": "Monthly cloud foundation expenditure does not exceed target allocations without documented operational sign-off.", + "on": "C-14", + "source": "K-9", + "id": "OC-9" + }, + { + "condition": "Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing.", + "id": "OC-10", + "on": "C-10", + "source": "K-10" + }, + { + "condition": "Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores.", + "on": "C-6", + "id": "OC-11", + "source": "K-11" + }, + { + "source": "K-12", + "on": "C-11", + "id": "OC-12", + "condition": "Platform identity permissions and access roles are continuously reviewed and rightsized based on automated recommender findings." + }, + { + "on": "C-5", + "source": "OR-12", + "condition": "Infrastructure changes are deployed via automated pipelines with automated linting, security scans, and peer approvals without a manual change advisory board.", + "id": "OC-13" + }, + { + "on": "R-1", + "source": "OR-13", + "id": "OC-14", + "condition": "Platform foundation operations, ongoing maintenance, and incident remediation are delivered under a provider-operated sourcing model." + }, + { + "on": "R-3", + "source": "OR-14", + "id": "OC-15", + "condition": "Platform operational delivery and administrative access are restricted to personnel operating from the United Kingdom (GB), Poland (PL), and India (IN)." + }, + { + "source": "OR-15", + "id": "OC-16", + "condition": "Platform operational telemetry, logging, and incident alerting are integrated into operational event management systems.", + "on": "C-2" + }, + { + "id": "OC-17", + "condition": "The platform foundation configuration maintains verifiable alignment with DORA operational resiliency guidelines, PCI-DSS v4.0 perimeter isolation standards, and ISO 27001 controls.", + "source": "K-1", + "on": "C-7" + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/verdicts.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/verdicts.json new file mode 100644 index 0000000..9697d74 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1/verdicts.json @@ -0,0 +1,249 @@ +{ + "dry_run": { + "verdict": "pass", + "evidence": "31 of 31 scenario walk-throughs pass." + }, + "checks": [ + { + "check": "OC-1", + "evidence": "Capability C-1 is staffed 24x7 and explicitly covers triage of 24x7 operational alerts and managing priority incidents, fully meeting the requirement.", + "verdict": "pass" + }, + { + "check": "OC-2", + "verdict": "fail", + "evidence": "The check requires the secondary region to be 'continuously maintained' as an active warm standby, but Capability C-9 is configured for 'business' hours, meaning maintenance cannot be continuous (24x7). C-9 should be 24x7 or another capability should provide continuous maintenance." + }, + { + "verdict": "pass", + "check": "OC-3", + "evidence": "Capability C-21 explicitly covers guaranteeing UK/EU data residency records and collating compliance evidence." + }, + { + "verdict": "fail", + "evidence": "The check requires validation within 10 weeks, but Capability C-16 covers managing the roadmap without explicitly mentioning the 10-week requirement (which is instead mentioned in C-18). C-16 must mention the 10-week validation or the check mapped to C-18.", + "check": "OC-4" + }, + { + "evidence": "Capability C-15 covers compiling reports on 99.9% availability, meeting the condition.", + "check": "OC-5", + "verdict": "pass" + }, + { + "evidence": "Capability C-9 explicitly states it governs 4-hour disaster recovery procedures, meeting the 4-hour RTO condition.", + "verdict": "pass", + "check": "OC-6" + }, + { + "check": "OC-7", + "verdict": "pass", + "evidence": "Capability C-8 covers monitoring backups and performing restore tests to ensure a 1-hour RPO." + }, + { + "check": "OC-8", + "evidence": "Capability C-2 is a 24x7 capability that tunes alerts and integrates telemetry into operational systems, handled by the 24x7 Platform SRE role.", + "verdict": "pass" + }, + { + "check": "OC-9", + "evidence": "The check requires that expenditure does not exceed target allocations without 'documented operational sign-off', but Capability C-14 only mentions 'escalating anomalies' and does not explicitly guarantee documented sign-off.", + "verdict": "fail" + }, + { + "verdict": "pass", + "check": "OC-10", + "evidence": "Capability C-10 covers reviewing bandwidth and ensuring deterministic low-latency performance for payment transactions." + }, + { + "check": "OC-11", + "verdict": "pass", + "evidence": "Capability C-6 covers optimizing deployments and resource placements to maximize carbon-free energy utilization." + }, + { + "evidence": "The check requires identity permissions to be 'continuously reviewed', but Capability C-11 operates only during 'business' hours, meaning reviews cannot be continuous. C-11 must be 24x7.", + "verdict": "fail", + "check": "OC-12" + }, + { + "check": "OC-13", + "verdict": "pass", + "evidence": "Capability C-5 covers conducting peer approvals for automated infrastructure-as-code changes, supporting pipeline automated deployment." + }, + { + "verdict": "pass", + "evidence": "Role R-1 (Platform Foundation Lead) is part of Party P-2 (Platform Provider), satisfying the provider-operated sourcing model.", + "check": "OC-14" + }, + { + "check": "OC-15", + "verdict": "pass", + "evidence": "Role R-3 operates from India, and all other provider roles operate from GB and PL, meeting the delivery countries restriction." + }, + { + "verdict": "pass", + "evidence": "Capability C-2 integrates telemetry into operational systems, aligning with the event management systems requirement.", + "check": "OC-16" + }, + { + "check": "OC-17", + "evidence": "The check requires verifiable alignment with DORA, PCI-DSS v4.0, and ISO 27001. Capability C-7 only ensures alignment with PCI-DSS v4.0 baseline configurations, completely missing DORA and ISO 27001 controls.", + "verdict": "fail" + }, + { + "check": "SC-fault-S-1", + "verdict": "pass", + "evidence": "Resource Hierarchy and Environment Boundary Manager (S-1) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "check": "SC-change-S-1", + "verdict": "pass", + "evidence": "Resource Hierarchy and Environment Boundary Manager (S-1) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-2", + "verdict": "pass", + "evidence": "Dedicated Private Hybrid Connectivity Gateway (S-2) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "check": "SC-change-S-2", + "verdict": "pass", + "evidence": "Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-3", + "verdict": "pass", + "evidence": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "check": "SC-change-S-3", + "verdict": "pass", + "evidence": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-4", + "verdict": "pass", + "evidence": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "check": "SC-change-S-4", + "verdict": "pass", + "evidence": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-5", + "verdict": "pass", + "evidence": "Automated Infrastructure Orchestration and Deployment Engine (S-5) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "check": "SC-change-S-5", + "verdict": "pass", + "evidence": "Automated Infrastructure Orchestration and Deployment Engine (S-5) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-6", + "verdict": "pass", + "evidence": "Federated Identity and Role Governance Authority (S-6) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "check": "SC-change-S-6", + "verdict": "pass", + "evidence": "Federated Identity and Role Governance Authority (S-6) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-7", + "verdict": "pass", + "evidence": "Cryptographic Key Lifecycle and Encryption Controller (S-7) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "check": "SC-change-S-7", + "verdict": "pass", + "evidence": "Cryptographic Key Lifecycle and Encryption Controller (S-7) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-8", + "verdict": "pass", + "evidence": "Platform Telemetry and Operational Alerting Engine (S-8) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "check": "SC-change-S-8", + "verdict": "pass", + "evidence": "Platform Telemetry and Operational Alerting Engine (S-8) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-9", + "verdict": "pass", + "evidence": "Financial Governance and Budget Enforcement Monitor (S-9) fails at an hour faults are responded to (24x7). Platform SRE responds under C-1, and Platform Foundation Lead is answerable." + }, + { + "check": "SC-change-S-9", + "verdict": "pass", + "evidence": "Financial Governance and Budget Enforcement Monitor (S-9) must be changed. Platform Engineer makes the change under C-5." + }, + { + "check": "SC-condition-K-1", + "verdict": "pass", + "evidence": "Condition K-1 must be shown to hold: The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. Check OC-17 on C-7 keeps it true." + }, + { + "check": "SC-condition-K-2", + "verdict": "pass", + "evidence": "Condition K-2 must be shown to hold: The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic. Check OC-2 on C-9 keeps it true." + }, + { + "check": "SC-condition-K-3", + "verdict": "pass", + "evidence": "Condition K-3 must be shown to hold: Data residency is guaranteed within the United Kingdom and European Union. Check OC-3 on C-21 keeps it true." + }, + { + "check": "SC-condition-K-4", + "verdict": "pass", + "evidence": "Condition K-4 must be shown to hold: Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks. Check OC-4 on C-16 keeps it true." + }, + { + "check": "SC-condition-K-5", + "verdict": "pass", + "evidence": "Condition K-5 must be shown to hold: Platform foundation infrastructure maintains measured monthly availability of at least 99.9%. Check OC-5 on C-15 keeps it true." + }, + { + "check": "SC-condition-K-6", + "verdict": "pass", + "evidence": "Condition K-6 must be shown to hold: Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours. Check OC-6 on C-9 keeps it true." + }, + { + "check": "SC-condition-K-7", + "verdict": "pass", + "evidence": "Condition K-7 must be shown to hold: Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour. Check OC-7 on C-8 keeps it true." + }, + { + "check": "SC-condition-K-8", + "verdict": "pass", + "evidence": "Condition K-8 must be shown to hold: Alert notifications trigger operational incident workflows for the 24/7 central platform operations team. Check OC-8 on C-2 keeps it true." + }, + { + "check": "SC-condition-K-9", + "verdict": "pass", + "evidence": "Condition K-9 must be shown to hold: Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off. Check OC-9 on C-14 keeps it true." + }, + { + "check": "SC-condition-K-10", + "verdict": "pass", + "evidence": "Condition K-10 must be shown to hold: Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing. Check OC-10 on C-10 keeps it true." + }, + { + "check": "SC-condition-K-11", + "verdict": "pass", + "evidence": "Condition K-11 must be shown to hold: Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores. Check OC-11 on C-6 keeps it true." + }, + { + "check": "SC-condition-K-12", + "verdict": "pass", + "evidence": "Condition K-12 must be shown to hold: Permissions are continuously rightsized based on IAM Recommender findings. Check OC-12 on C-11 keeps it true." + }, + { + "check": "SC-party-P-2", + "verdict": "pass", + "evidence": "Platform Provider cannot be reached. Every Capability it is answerable for has a Role in another Party to turn to." + } + ], + "proposed_additions": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/capabilities.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/capabilities.json new file mode 100644 index 0000000..d38da63 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/capabilities.json @@ -0,0 +1,191 @@ +[ + { + "id": "C-1", + "catalogue": "incident-management", + "serves": ["OR-1"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "24x7", + "effort_hours_per_month": 40, + "basis": "24x7 coverage to acknowledge priority 1 incidents within 30 minutes, managing communications and resolution." + }, + { + "id": "C-2", + "catalogue": "monitoring-and-alerting", + "serves": ["OR-11", "OR-8", "OR-13"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "24x7", + "effort_hours_per_month": 40, + "basis": "Continuous tuning of alert thresholds and immediate 24x7 response to telemetry signals across the foundation." + }, + { + "id": "C-3", + "catalogue": "problem-management", + "serves": ["OR-1", "OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Investigating the root causes of major or recurring platform faults to prevent recurrence." + }, + { + "id": "C-4", + "catalogue": "service-request-fulfilment", + "serves": ["OR-2"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Handling non-fault operational requests, access adjustments, and configuration inquiries from application teams." + }, + { + "id": "C-5", + "catalogue": "change-enablement", + "serves": ["OR-4", "OR-7"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Ensuring automated policy validations and mandatory peer reviews are correctly enforced across CI/CD pipelines." + }, + { + "id": "C-6", + "catalogue": "release-and-deployment", + "serves": ["OR-7", "OR-14"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Managing GitHub Actions infrastructure-as-code delivery pipelines and executing rollbacks if required." + }, + { + "id": "C-7", + "catalogue": "configuration-management", + "serves": ["OR-4", "OR-6"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Maintaining version-controlled desired states and auditing actual environment configurations against data residency boundaries." + }, + { + "id": "C-8", + "catalogue": "backup-and-recovery", + "serves": ["OR-10"], + "items": ["S-4", "S-7"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Ensuring compliance logs and cryptographic materials backups meet the 1-hour RPO and testing restores." + }, + { + "id": "C-9", + "catalogue": "service-continuity", + "serves": ["OR-5", "OR-9"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Maintaining runbooks for 4-hour RTO and verifying the europe-west1 secondary region warm standby readiness." + }, + { + "id": "C-10", + "catalogue": "capacity-and-performance", + "serves": ["OR-8", "OR-13", "OR-14"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Monitoring Interconnect latency and overall infrastructure capacity to maintain strict performance for payments processing and optimizing workload placements for CFE." + }, + { + "id": "C-11", + "catalogue": "access-management", + "serves": ["OR-15", "OR-3"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Executing continuous identity rightsizing based on automated findings and restricting access to approved delivery locations." + }, + { + "id": "C-12", + "catalogue": "vulnerability-and-patch-management", + "serves": ["OR-4"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Regular vulnerability scanning and patching of egress VMs and other resources to align with security standards." + }, + { + "id": "C-13", + "catalogue": "security-monitoring-and-response", + "serves": ["OR-4"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "24x7", + "effort_hours_per_month": 40, + "basis": "Continuous detection and response to security events from egress inspection and identity telemetry." + }, + { + "id": "C-14", + "catalogue": "cost-management", + "serves": ["OR-12"], + "items": ["S-9"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Monitoring foundational spend against agreed budgets and seeking sign-offs for anomalies." + }, + { + "id": "C-15", + "catalogue": "service-level-management", + "serves": ["OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Reporting monthly on the 99.9% availability requirement and other agreed service levels." + }, + { + "id": "C-16", + "catalogue": "platform-product-management", + "serves": ["OR-7"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Managing the platform's roadmap and backlog to ensure readiness within 10 weeks and driving adoption." + }, + { + "id": "C-17", + "catalogue": "self-service-and-golden-paths", + "serves": ["OR-7"], + "items": ["S-5"], + "hours": "business", + "effort_hours_per_month": 30, + "basis": "Building automated templates and CI/CD elements so teams can onboard workloads seamlessly." + }, + { + "id": "C-18", + "catalogue": "enablement", + "serves": ["OR-7"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 30, + "basis": "Actively pairing with and helping payment platform teams ingest into the foundation within the timeframe." + }, + { + "id": "C-19", + "catalogue": "delivery-and-developer-experience-measurement", + "serves": ["OR-7"], + "items": ["S-5"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Collecting delivery metrics and assessing friction to refine the deployment pipelines." + }, + { + "id": "C-20", + "catalogue": "supplier-management", + "serves": ["OR-2", "OR-13"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Managing relationships and SLAs with the cloud provider, Palo Alto Networks, and Interconnect carrier." + }, + { + "id": "C-21", + "catalogue": "regulatory-evidence", + "serves": ["OR-4", "OR-6"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 25, + "basis": "Compiling and retaining the audit evidence required for ongoing DORA, PCI-DSS v4.0, and ISO27001 assessments." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/execution.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/execution.json new file mode 100644 index 0000000..ef8f788 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/execution.json @@ -0,0 +1,151 @@ +{ + "customer_case": "case-08-landing-zone", + "execution_id": "case-02-from-architecture-r2", + "session_id": "d29687eb-8857-48e4-8c98-c2251c1c2c8f", + "intent_version": 2, + "workflow_request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.", + "baseline": null, + "overlay_skills": {}, + "options": {}, + "sources": { + "execution.json": { + "sha256": "8bf269deffc8e4bce474bd8d66959c03b04de01dc137eefaf96ead74b46a5a45", + "digested": true + }, + "scope.json": { + "sha256": "57dc28ab7397f514ef0e83b292fc2165554ac2b516775c973f4965765a9e905f", + "digested": true + }, + "requirements.json": { + "sha256": "5aecbef9eaea254fb5123bcc1d107b4a7b7d4d45d60d940464d1fde690d2973d", + "digested": true + }, + "recommendations.json": { + "sha256": "95958b1a107705280d65c9d088bba57352d1f9fa8e67e89b2b3379f78d91cd24", + "digested": true + }, + "components.json": { + "sha256": "49e47dbfa90a85d086711eb2ba66370c145f13fd3cd3595c1df1366224db392c", + "digested": true + }, + "products.json": { + "sha256": "75d51cba319f83000e6f85277b0fb21a7ec02faabac332c5d1800c8f512136db", + "digested": true + }, + "target-architecture.json": { + "sha256": "40879607aaa7041103d8f4d3ba804df8751d2159aa0ce0977d7d821d35f7608b", + "digested": true + }, + "verdicts.json": { + "sha256": "75e17da790266e94a6775d90857a9b503cacd62fd55c92be2d7a3435de301853", + "digested": true + }, + "work-breakdown.json": { + "sha256": "057fd1b0ecd6350a6f9f6101b712a61e78fc10d758a67e785a1dca7e1e674dc3", + "digested": true + } + }, + "scope": { + "status": "BASELINE_LOCKED", + "readiness": { + "supplied": 3, + "answered": 0, + "assumed": 7, + "unknown": 10, + "not_applicable": 0, + "digested": 13 + } + }, + "status": "completed", + "warnings": [], + "risks": [], + "assessment": null, + "phases": { + "scoping": "approved", + "operational_requirements": "approved", + "operating_model_design": "approved", + "validation": "approved", + "packaging": "approved" + }, + "final_document": "operating-model-guide.md", + "changes": null, + "deliverables": { + "operating_scope": { + "document": "deliverables/operating_scope.md", + "attachments": [ + "scope.json", + "scope.yaml" + ], + "status": "submitted" + }, + "operational_requirements": { + "document": "deliverables/operational_requirements.md", + "attachments": [ + "operational-requirements.json" + ], + "status": "submitted" + }, + "capability_model": { + "document": "deliverables/capability_model.md", + "attachments": [ + "capabilities.json" + ], + "status": "submitted" + }, + "target_operating_model": { + "document": "deliverables/target_operating_model.md", + "attachments": [ + "findings.json", + "organisation.mmd", + "staffing.json", + "target-operating-model.json" + ], + "status": "submitted" + }, + "validation_results": { + "document": "deliverables/validation_results.md", + "attachments": [ + "scenarios.json", + "verdicts.json" + ], + "status": "submitted" + }, + "operating_model_guide": { + "document": "deliverables/operating_model_guide.md", + "attachments": [], + "status": "submitted" + } + }, + "files": [ + "capabilities.json", + "deliverables/capability_model.md", + "deliverables/operating_model_guide.md", + "deliverables/operating_scope.md", + "deliverables/operational_requirements.md", + "deliverables/target_operating_model.md", + "deliverables/validation_results.md", + "findings.json", + "operating-model-guide.md", + "operational-requirements.json", + "organisation.mmd", + "scenarios.json", + "scope.json", + "scope.yaml", + "staffing.json", + "target-operating-model.json", + "verdicts.json" + ], + "staffing": { + "people": 10.8, + "cost_per_month": 83846, + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + } + }, + "findings": [], + "change_request": null, + "capability_sources_verified": false +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/findings.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/findings.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/findings.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/operating-model-guide.md b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/operating-model-guide.md new file mode 100644 index 0000000..4d08520 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/operating-model-guide.md @@ -0,0 +1,280 @@ +# Operating model guide: Google Cloud Payments Platform Foundation + +## Summary + +This operating model governs the enterprise-grade Google Cloud platform foundation and landing zone supporting the core payments engine, encompassing the resource hierarchy, dedicated hybrid interconnectivity, centralised egress inspection, immutable compliance audit logging, automated deployment pipelines, federated identity, customer-managed encryption keys, platform telemetry, and financial governance. Operational delivery follows a provider-operated model in which the Service Provider maintains primary operational accountability and responsibility for platform foundation health, continuous monitoring, incident resolution, infrastructure maintenance, and regulatory evidence collection, while the Customer Platform Team retains ownership of payment workload applications and consumes platform services via automated self-service delivery paths. + +Platform support and infrastructure availability are maintained continuously around the clock, with platform reliability leadership, enablement, and product management operating during standard business hours. The reader should look first at the operational assumptions and open questions—specifically the native cloud operations tooling architecture, pipeline-driven change enablement without a manual change advisory board, and interconnect circuit lead times—as well as the staffing figures, which reflect illustrative placeholder rates from a draft rate card. + +### Key figures + +- Operated: 9 Items, with 12 Conditions to keep true. +- Faults responded to: 24x7 hours. +- Capabilities: 21. +- People: 10.8. Cost a month: 83,846 GBP, from rate card version `draft-2026-10-13` (placeholder prices: an illustration). +- Scenario walk-throughs: 31 of 31 pass. +- Business outcomes: 3, 3 shown by a Measure. +- Findings for whoever owns the design: 0. + +## Business outcomes + +**BO-1: Continuous, highly available, and low-latency operation of the core payments platform foundation** Target: 99.9% foundation availability with deterministic low latency. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Platform Availability | service level | | 99.9% availability | Service level management | Platform Reliability Lead | +| Hybrid Network Latency | service level | | Deterministic low latency | Capacity and performance management | Platform Reliability Lead | + +**BO-2: Continuous compliance with DORA and PCI-DSS v4.0 standards using customer-managed encryption keys and centralised egress inspection** Target: Zero audit non-conformances. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Audit Non-conformances | service level | | Zero non-conformances | Regulatory evidence and reporting | Platform Reliability Lead | + +**BO-3: Automated, secure infrastructure release pipelines enabling workload ingestion within 10 weeks** Target: Operational readiness achieved within 10 weeks. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Deployment frequency | outcome | dora-metrics | On-demand | Delivery and developer experience measurement | Platform Product Manager | +| Developer satisfaction | guardrail | space-framework | High satisfaction | Delivery and developer experience measurement | Platform Product Manager | +| Feedback loops | leading | devex-framework | < 10 minutes | Delivery and developer experience measurement | Platform Product Manager | + +## What is operated + +| Item | What it is | Kind | Product | From | +| --- | --- | --- | --- | --- | +| S-1 | Resource Hierarchy and Environment Boundary Manager | governance | Resource Manager | C-1 | +| S-2 | Dedicated Private Hybrid Connectivity Gateway | network | Cloud Interconnect | C-2 | +| S-3 | Centralised Outbound Traffic Inspection and Egress Boundary | service | Palo Alto Networks VM-Series | C-3 | +| S-4 | Centralised Immutable Audit Event and Compliance Log Repository | storage | Cloud Storage | C-4 | +| S-5 | Automated Infrastructure Orchestration and Deployment Engine | service | GitHub Actions | C-5 | +| S-6 | Federated Identity and Role Governance Authority | identity | IAM Workload Identity Federation | C-6 | +| S-7 | Cryptographic Key Lifecycle and Encryption Controller | keys | Cloud KMS | C-7 | +| S-8 | Platform Telemetry and Operational Alerting Engine | observability | Cloud Monitoring | C-8 | +| S-9 | Financial Governance and Budget Enforcement Monitor | billing | Cloud Billing | C-9 | + +What must be kept true while it is run: + +| Condition | On | What must hold | From | +| --- | --- | --- | --- | +| K-1 | S-1, S-4, S-7 | The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. | NFR-8 | +| K-2 | S-1 | The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic. | NFR-9 | +| K-3 | S-1 | Data residency is guaranteed within the United Kingdom and European Union. | NFR-9 | +| K-4 | S-5 | Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks. | NFR-10 | +| K-5 | S-8 | Platform foundation infrastructure maintains measured monthly availability of at least 99.9%. | NFR-11 | +| K-6 | S-8 | Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours. | NFR-11 | +| K-7 | S-8 | Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour. | NFR-11 | +| K-8 | S-8 | Alert notifications trigger operational incident workflows for the 24/7 central platform operations team. | NFR-12 | +| K-9 | S-9 | Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off. | NFR-13 | +| K-10 | S-2 | Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing. | NFR-14 | +| K-11 | S-8 | Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores. | NFR-15 | +| K-12 | S-6 | Permissions are continuously rightsized based on IAM Recommender findings. | DR-OP-3 | + +## Operational requirements + +| Requirement | Kind | What must be achieved | From | +| --- | --- | --- | --- | +| OR-1 | service_level | Priority 1 critical incidents on platform foundation components are acknowledged and responded to within 30 minutes (as assumed in service_levels.priority_1_response_minutes), 24 hours a day, 7 days a week. | operations.support_hours, service_levels.priority_1_response_minutes | +| OR-2 | organisation | Platform foundation operations are delivered under a provider-operated model where external operational management is provided by the service provider. | sourcing.model | +| OR-3 | organisation | Platform operational support and administrative access are conducted exclusively from approved delivery countries: United Kingdom (GB), Poland (PL), and India (IN). | sourcing.delivery_countries | +| OR-4 | compliance | Landing zone configuration and operational governance maintain demonstrable alignment with DORA operational resiliency guidelines, PCI-DSS v4.0 perimeter isolation standards, and ISO27001 security standards during external audit. | K-1, compliance.frameworks | +| OR-5 | service_level | The secondary cloud region in europe-west1 is maintained in warm standby operational readiness to receive failover traffic at all times. | K-2 | +| OR-6 | compliance | Data storage, cryptographic processing, and compliance log retention remain strictly resident within the United Kingdom and European Union. | K-3 | +| OR-7 | service_level | Platform foundation provisioning, configuration, and operational validation are completed to support core payments engine workload ingestion within 10 weeks. | K-4 | +| OR-8 | service_level | Platform foundation infrastructure maintains measured monthly availability of at least 99.9%, available 24 hours a day, 7 days a week. | K-5, operations.availability_hours | +| OR-9 | service_level | Platform foundation disaster recovery activation achieves a recovery time objective (RTO) of no more than 4 hours (as assumed in service_levels.recovery_time_hours). | K-6, service_levels.recovery_time_hours | +| OR-10 | service_level | Disaster recovery processes achieve a recovery point objective (RPO) of no more than 1 hour of data loss. | K-7 | +| OR-11 | process | Platform telemetry alert notifications trigger operational incident workflows immediately upon detection, 24 hours a day, 7 days a week. | K-8 | +| OR-12 | process | Monthly cloud foundation expenditure does not exceed the agreed target budget allocation without prior operational review and formal sign-off. | K-9 | +| OR-13 | service_level | Hybrid connectivity between cloud and on-premises environments maintains deterministic low latency suitable for real-time payment transaction processing. | K-10 | +| OR-14 | process | Deployments and operational workload executions optimize carbon-free energy utilization based on regional carbon-free energy (CFE) scores. | K-11 | +| OR-15 | process | Access permissions across platform identities are continuously rightsized to least privilege based on automated access findings. | K-12 | + +## Capabilities + +| Capability | What it is | Serves | Covers | Hours | Effort a month | +| --- | --- | --- | --- | --- | --- | +| C-1 | Incident management | OR-1 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | 24x7 | 40 h | +| C-2 | Monitoring and alerting | OR-11, OR-8, OR-13 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | 24x7 | 40 h | +| C-3 | Problem management | OR-1, OR-8 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 20 h | +| C-4 | Service request fulfilment | OR-2 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 20 h | +| C-5 | Change enablement | OR-4, OR-7 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 15 h | +| C-6 | Release and deployment | OR-7, OR-14 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 20 h | +| C-7 | Configuration management | OR-4, OR-6 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 15 h | +| C-8 | Backup and recovery | OR-10 | S-4, S-7 | business | 10 h | +| C-9 | Service continuity | OR-5, OR-9 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 15 h | +| C-10 | Capacity and performance management | OR-8, OR-13, OR-14 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 15 h | +| C-11 | Access management | OR-15, OR-3 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 15 h | +| C-12 | Vulnerability and patch management | OR-4 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 20 h | +| C-13 | Security monitoring and response | OR-4 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | 24x7 | 40 h | +| C-14 | Cost management | OR-12 | S-9 | business | 10 h | +| C-15 | Service level management | OR-8 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 10 h | +| C-16 | Platform product management | OR-7 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 20 h | +| C-17 | Self-service and golden paths | OR-7 | S-5 | business | 30 h | +| C-18 | Enablement | OR-7 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 30 h | +| C-19 | Delivery and developer experience measurement | OR-7 | S-5 | business | 10 h | +| C-20 | Supplier management | OR-2, OR-13 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 10 h | +| C-21 | Regulatory evidence and reporting | OR-4, OR-6 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 25 h | + +## Who does what + +| Role | Party | Country | Level | Hours | +| --- | --- | --- | --- | --- | +| Platform Operations Engineer | Service Provider (provider) | PL | engineer | 24x7 | +| Platform Security & Network Engineer | Service Provider (provider) | PL | senior | 24x7 | +| Platform Reliability Lead | Service Provider (provider) | GB | lead | business | +| Platform Product Manager | Service Provider (provider) | GB | lead | business | +| Platform Enablement Engineer | Service Provider (provider) | GB | senior | business | +| Customer Stakeholder | Customer Platform Team (customer) | GB | lead | business | + +The teams, as Team Topologies names their kinds: + +| Team | Kind | Party | Roles | Purpose | +| --- | --- | --- | --- | --- | +| Central Platform Operations Team | platform | Service Provider | Platform Operations Engineer, Platform Security & Network Engineer, Platform Reliability Lead | 24/7 central platform operations and foundation management. | +| Platform Product & Enablement Team | platform | Service Provider | Platform Product Manager, Platform Enablement Engineer | Platform product management, enablement, and measurement. | +| Customer Application Teams | stream-aligned | Customer Platform Team | Customer Stakeholder | Core payments engine development and workload deployment. | + +How the teams work together: + +| From | To | Mode | Until | Purpose | +| --- | --- | --- | --- | --- | +| Customer Application Teams | Central Platform Operations Team | x-as-a-service | | Customer teams consume platform foundation services and infrastructure via automated pipelines. | +| Platform Product & Enablement Team | Customer Application Teams | facilitating | | Enablement team facilitates customer application teams in adopting pipelines and ingesting workloads. | + +| Capability | Platform Operations Engineer | Platform Security & Network Engineer | Platform Reliability Lead | Platform Product Manager | Platform Enablement Engineer | Customer Stakeholder | +| --- | --- | --- | --- | --- | --- | --- | +| Incident management (C-1) | R | | A | | | C | +| Monitoring and alerting (C-2) | R | | A | | | C | +| Problem management (C-3) | | R | A | | | C | +| Service request fulfilment (C-4) | R | | A | | | C | +| Change enablement (C-5) | | R | A | | | C | +| Release and deployment (C-6) | | R | A | | | C | +| Configuration management (C-7) | | R | A | | | C | +| Backup and recovery (C-8) | R | | A | | | C | +| Service continuity (C-9) | | R | A | | | C | +| Capacity and performance management (C-10) | | R | A | | | C | +| Access management (C-11) | R | | A | | | C | +| Vulnerability and patch management (C-12) | | R | A | | | C | +| Security monitoring and response (C-13) | | R | A | | | C | +| Cost management (C-14) | | | | A/R | | C | +| Service level management (C-15) | | | A/R | | | C | +| Platform product management (C-16) | | | | A/R | | C | +| Self-service and golden paths (C-17) | | | | A | R | C | +| Enablement (C-18) | | | | A | R | C | +| Delivery and developer experience measurement (C-19) | | | | A | R | C | +| Supplier management (C-20) | | | A/R | | | C | +| Regulatory evidence and reporting (C-21) | | R | A | | | C | + +A accountable, R responsible, C consulted, I informed. + +## Staffing and cost + +| Role | Party | Country | Level | Hours | Effort a month | People | Cost a month | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Platform Operations Engineer | Service Provider | PL | engineer | 24x7 | 125.0 h | 5.0 | 30,777 GBP | +| Platform Security & Network Engineer | Service Provider | PL | senior | 24x7 | 185.0 h | 5.0 | 41,640 GBP | +| Platform Reliability Lead | Service Provider | GB | lead | business | 20.0 h | 0.1 | 2,212 GBP | +| Platform Product Manager | Service Provider | GB | lead | business | 30.0 h | 0.2 | 3,318 GBP | +| Platform Enablement Engineer | Service Provider | GB | senior | business | 70.0 h | 0.5 | 5,899 GBP | +| Customer Stakeholder | Customer Platform Team | GB | lead | business | 0.0 h | 0.0 | not priced | + +10.8 people in all. Cost a month: 83,846 GBP. + +Worked from rate card `rate-card`, version `draft-2026-10-13`. **Its figures are placeholders: no price has been set, and this cost is an illustration.** +Converted from USD at 0.79, a rate set on 2026-10-13. +The customer's own staff are counted and not priced. A Role that covers hours wider than a working day is never smaller than the team that can cover them. + +## Validation + +| Check | What must hold | Verdict | Evidence | +| --- | --- | --- | --- | +| OC-K1 | Regulatory alignment with DORA and PCI-DSS v4.0 is maintained and verified during external audit with zero non-conformances. | pass | Capability C-21 (regulatory-evidence) exists to compile and retain audit evidence for DORA and PCI-DSS v4.0 assessments. Role R-2 is responsible for it. | +| OC-K2 | The europe-west1 secondary region is verified as a warm standby ready to receive failover traffic. | pass | Capability C-9 (service-continuity) is tasked with maintaining runbooks and verifying the europe-west1 secondary region warm standby readiness. Role R-2 is responsible. | +| OC-K3 | Data residency remains strictly within the United Kingdom and European Union boundaries. | pass | Capability C-7 (configuration-management) audits actual environment configurations against data residency boundaries (UK and EU). Role R-2 is responsible. | +| OC-K4 | Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks. | pass | Capability C-16 (platform-product-management) is responsible for ensuring the platform foundation is provisioned and validated within 10 weeks. Role R-4 is responsible. | +| OC-K5 | Platform foundation infrastructure maintains measured monthly availability of at least 99.9%. | pass | Capability C-15 (service-level-management) is set up to track and report on the 99.9% availability requirement. Role R-3 is responsible. | +| OC-K6 | Disaster recovery activation achieves a recovery time objective (RTO) of no more than 4 hours. | pass | Capability C-1 (incident-management) operates 24x7 and takes responsibility for executing disaster recovery activation within the 4-hour RTO at any time of day. Role R-1 provides the 24x7 coverage. | +| OC-K7 | Disaster recovery achieves a recovery point objective (RPO) of no more than 1 hour of data loss. | pass | Capability C-8 (backup-and-recovery) ensures backups meet the 1-hour RPO and handles testing restores. Role R-1 is responsible. | +| OC-K8 | Alert notifications trigger operational incident workflows for the 24/7 central platform operations team immediately. | pass | Capability C-2 (monitoring-and-alerting) operates 24x7 for immediate response to telemetry signals. Role R-1 provides 24x7 coverage. | +| OC-K9 | Monthly cloud foundation expenditure does not exceed the agreed target allocation without prior operational review and formal sign-off. | pass | Capability C-14 (cost-management) monitors foundation spend against agreed budgets and manages sign-offs for anomalies. Role R-4 is responsible. | +| OC-K10 | Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing. | pass | Capability C-10 (capacity-and-performance) monitors Dedicated Interconnect latency and overall infrastructure capacity for payment processing performance. Role R-2 is responsible. | +| OC-K11 | Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores. | pass | Capability C-10 (capacity-and-performance) oversees optimization of workload placements for carbon-free energy (CFE). Role R-2 is responsible. | +| OC-K12 | Permissions are continuously rightsized based on IAM Recommender findings. | pass | Capability C-11 (access-management) executes continuous identity rightsizing based on automated findings. Role R-1 is responsible. | +| OC-OR1 | Priority 1 critical incidents on platform foundation components are acknowledged and responded to within 30 minutes, 24 hours a day, 7 days a week. | pass | Capability C-1 (incident-management) explicitly offers 24x7 coverage to acknowledge priority 1 incidents within 30 minutes, supported by the 24x7 R-1 role. | +| OC-OR2 | External operational management is consistently provided by the service provider. | pass | Capability C-4 (service-request-fulfilment) represents non-fault operational management by the Service Provider. Sourcing model ensures external operational management. | +| OC-OR8 | Platform foundation infrastructure maintains measured monthly availability of at least 99.9%, available 24 hours a day, 7 days a week. | pass | Capability C-15 (service-level-management) tracks and reports on the 99.9% availability requirement on a monthly basis. Role R-3 is responsible. | +| OC-OR3-R1 | Role operates from an approved delivery country. | pass | Role R-1 operates from Poland (PL), which is an approved delivery country. | +| OC-OR3-R2 | Role operates from an approved delivery country. | pass | Role R-2 operates from Poland (PL), which is an approved delivery country. | +| OC-OR3-R3 | Role operates from an approved delivery country. | pass | Role R-3 operates from the United Kingdom (GB), which is an approved delivery country. | +| OC-OR3-R4 | Role operates from an approved delivery country. | pass | Role R-4 operates from the United Kingdom (GB), which is an approved delivery country. | +| OC-OR3-R5 | Role operates from an approved delivery country. | pass | Role R-5 operates from the United Kingdom (GB), which is an approved delivery country. | + +Each scenario is generated from the operating model and traced through it by code. + +| Scenario | Result | Path | +| --- | --- | --- | +| Resource Hierarchy and Environment Boundary Manager (S-1) fails at an hour faults are responded to (24x7). | pass | Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable. | +| Resource Hierarchy and Environment Boundary Manager (S-1) must be changed. | pass | Platform Security & Network Engineer makes the change under C-5. | +| Dedicated Private Hybrid Connectivity Gateway (S-2) fails at an hour faults are responded to (24x7). | pass | Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable. | +| Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed. | pass | Platform Security & Network Engineer makes the change under C-5. | +| Centralised Outbound Traffic Inspection and Egress Boundary (S-3) fails at an hour faults are responded to (24x7). | pass | Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable. | +| Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed. | pass | Platform Security & Network Engineer makes the change under C-5. | +| Centralised Immutable Audit Event and Compliance Log Repository (S-4) fails at an hour faults are responded to (24x7). | pass | Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable. | +| Centralised Immutable Audit Event and Compliance Log Repository (S-4) must be changed. | pass | Platform Security & Network Engineer makes the change under C-5. | +| Automated Infrastructure Orchestration and Deployment Engine (S-5) fails at an hour faults are responded to (24x7). | pass | Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable. | +| Automated Infrastructure Orchestration and Deployment Engine (S-5) must be changed. | pass | Platform Security & Network Engineer makes the change under C-5. | +| Federated Identity and Role Governance Authority (S-6) fails at an hour faults are responded to (24x7). | pass | Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable. | +| Federated Identity and Role Governance Authority (S-6) must be changed. | pass | Platform Security & Network Engineer makes the change under C-5. | +| Cryptographic Key Lifecycle and Encryption Controller (S-7) fails at an hour faults are responded to (24x7). | pass | Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable. | +| Cryptographic Key Lifecycle and Encryption Controller (S-7) must be changed. | pass | Platform Security & Network Engineer makes the change under C-5. | +| Platform Telemetry and Operational Alerting Engine (S-8) fails at an hour faults are responded to (24x7). | pass | Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable. | +| Platform Telemetry and Operational Alerting Engine (S-8) must be changed. | pass | Platform Security & Network Engineer makes the change under C-5. | +| Financial Governance and Budget Enforcement Monitor (S-9) fails at an hour faults are responded to (24x7). | pass | Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable. | +| Financial Governance and Budget Enforcement Monitor (S-9) must be changed. | pass | Platform Security & Network Engineer makes the change under C-5. | +| Condition K-1 must be shown to hold: The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. | pass | Check OC-K1 on C-21 keeps it true. | +| Condition K-2 must be shown to hold: The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic. | pass | Check OC-K2 on C-9 keeps it true. | +| Condition K-3 must be shown to hold: Data residency is guaranteed within the United Kingdom and European Union. | pass | Check OC-K3 on C-7 keeps it true. | +| Condition K-4 must be shown to hold: Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks. | pass | Check OC-K4 on C-16 keeps it true. | +| Condition K-5 must be shown to hold: Platform foundation infrastructure maintains measured monthly availability of at least 99.9%. | pass | Check OC-K5 on C-15 keeps it true. | +| Condition K-6 must be shown to hold: Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours. | pass | Check OC-K6 on C-1 keeps it true. | +| Condition K-7 must be shown to hold: Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour. | pass | Check OC-K7 on C-8 keeps it true. | +| Condition K-8 must be shown to hold: Alert notifications trigger operational incident workflows for the 24/7 central platform operations team. | pass | Check OC-K8 on C-2 keeps it true. | +| Condition K-9 must be shown to hold: Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off. | pass | Check OC-K9 on C-14 keeps it true. | +| Condition K-10 must be shown to hold: Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing. | pass | Check OC-K10 on C-10 keeps it true. | +| Condition K-11 must be shown to hold: Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores. | pass | Check OC-K11 on C-10 keeps it true. | +| Condition K-12 must be shown to hold: Permissions are continuously rightsized based on IAM Recommender findings. | pass | Check OC-K12 on C-11 keeps it true. | +| Service Provider cannot be reached. | pass | Every Capability it is answerable for has a Role in another Party to turn to. | + +## Findings and open questions + +No findings: the solution provides what operating it needs. + +Assumed for the customer, and open to change when they answer: + +- `intent.business_outcomes`: {'outcome': 'Continuous, highly available, and low-latency operation of the core payments platform foundation', 'target': '99.9% foundation availability with deterministic low latency', 'id': 'BO-1'}, {'id': 'BO-2', 'target': 'Zero audit non-conformances', 'outcome': 'Continuous compliance with DORA and PCI-DSS v4.0 standards using customer-managed encryption keys and centralised egress inspection'}, {'outcome': 'Automated, secure infrastructure release pipelines enabling workload ingestion within 10 weeks', 'target': 'Operational readiness achieved within 10 weeks', 'id': 'BO-3'}. Derived from workflow request objectives: running core payment processing reliably in Google Cloud, meeting DORA and PCI-DSS v4.0 regulatory standards, and delivering landing zone readiness within 10 weeks. +- `intent.exclusions`: Core payment application workload code maintenance and runtime application management, On-premises infrastructure hardware maintenance beyond Cloud Interconnect physical termination. Platform foundation scope excludes application-level core payment workloads and on-premises data centre hardware beyond the Interconnect boundary. +- `operations.change_process`: Changes to the platform foundation are deployed through automated GitHub Actions pipelines with mandatory peer reviews and automated policy validation, without a separate manual change advisory board.. Catalogue default aligned with request stating GitHub Actions delivery pipelines: automated pipeline deployment with peer review and automated checks. +- `organisation.existing_teams`: {'name': 'Central Platform Operations Team', 'hours': '24x7', 'party': 'provider'}. Catalogue default: all operational roles are newly provided under provider_operated model, with reference to the 24/7 central platform operations team in condition K-8. +- `service_management.tooling`: Google Cloud operations suite (Cloud Monitoring, Cloud Logging, and Cloud Alerting) without a separate external ITSM system.. Catalogue default: Cloud provider native operations tooling (Cloud Monitoring, Cloud Logging) integrated with alerting. +- `service_levels.priority_1_response_minutes`: 30. Catalogue default response time for Priority 1 critical incidents under 24x7 operational support. +- `service_levels.recovery_time_hours`: 4. Catalogue default aligned with condition K-6 specifying a 4-hour recovery time objective (RTO). + +## Sources + +The Capabilities are from our own capability catalogue, which draws on these frameworks. It cites them and certifies nothing against them. + +- ITIL 4: Incident management practice; Monitoring and event management practice; Problem management practice; Service request management practice; Change enablement practice; Release management and Deployment management practices; Service configuration management practice; Service continuity management practice; Capacity and performance management practice; Service level management practice; Supplier management practice +- Site Reliability Engineering: How Google Runs Production Systems: Managing Incidents; Monitoring Distributed Systems; Postmortem Culture: Learning from Failure; Release Engineering; Service Level Objectives +- ISO/IEC 20000-1: Incident management; Service request management; Change management; Service continuity management; Capacity management; Service level management; Supplier management +- Regulation (EU) 2022/2554, the Digital Operational Resilience Act: Article 17, ICT-related incident management process; Article 10, Detection; Article 12, Backup policies and procedures, restoration and recovery procedures and methods; Article 11, Response and recovery; Article 9, Protection and prevention; Article 28, General principles on ICT third-party risk; Article 19, Reporting of major ICT-related incidents +- NIST Cybersecurity Framework: Detect: Continuous Monitoring; Identify: Asset Management; Recover: Incident Recovery Plan Execution; Protect: Identity Management, Authentication, and Access Control; Protect: Platform Security; Detect: Adverse Event Analysis; Respond: Incident Management +- DevOps Research and Assessment (DORA) software delivery metrics: Change failure rate; Deployment frequency and lead time for changes; Deployment frequency; Lead time for changes; Change failure rate; Time to restore service +- ISO/IEC 27001: Access control; Management of technical vulnerabilities; Information security incident management; Compliance with legal and contractual requirements +- FinOps Framework: Allocation; Budgeting; Forecasting; Anomaly Management +- CNCF Platforms White Paper (platform as a product): Platform as a product; Measuring platform success; Platform capabilities; Self-service; Platform as a product +- Team Topologies: Organizing Business and Technology Teams for Fast Flow: Platform team; Thinnest viable platform; X-as-a-Service interaction mode; Enabling team; Facilitating interaction mode +- The SPACE of Developer Productivity: Satisfaction and well-being; Performance; Activity; Communication and collaboration; Efficiency and flow +- DevEx: What Actually Drives Productivity: Feedback loops; Cognitive load; Flow state + +**These references have not been checked against the frameworks by a person.** Treat each as a pointer to where to look. diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/operational-requirements.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/operational-requirements.json new file mode 100644 index 0000000..fcaec56 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/operational-requirements.json @@ -0,0 +1,126 @@ +[ + { + "id": "OR-1", + "kind": "service_level", + "statement": "Priority 1 critical incidents on platform foundation components are acknowledged and responded to within 30 minutes (as assumed in service_levels.priority_1_response_minutes), 24 hours a day, 7 days a week.", + "source": [ + "operations.support_hours", + "service_levels.priority_1_response_minutes" + ] + }, + { + "id": "OR-2", + "kind": "organisation", + "statement": "Platform foundation operations are delivered under a provider-operated model where external operational management is provided by the service provider.", + "source": [ + "sourcing.model" + ] + }, + { + "id": "OR-3", + "kind": "organisation", + "statement": "Platform operational support and administrative access are conducted exclusively from approved delivery countries: United Kingdom (GB), Poland (PL), and India (IN).", + "source": [ + "sourcing.delivery_countries" + ] + }, + { + "id": "OR-4", + "kind": "compliance", + "statement": "Landing zone configuration and operational governance maintain demonstrable alignment with DORA operational resiliency guidelines, PCI-DSS v4.0 perimeter isolation standards, and ISO27001 security standards during external audit.", + "source": [ + "K-1", + "compliance.frameworks" + ] + }, + { + "id": "OR-5", + "kind": "service_level", + "statement": "The secondary cloud region in europe-west1 is maintained in warm standby operational readiness to receive failover traffic at all times.", + "source": [ + "K-2" + ] + }, + { + "id": "OR-6", + "kind": "compliance", + "statement": "Data storage, cryptographic processing, and compliance log retention remain strictly resident within the United Kingdom and European Union.", + "source": [ + "K-3" + ] + }, + { + "id": "OR-7", + "kind": "service_level", + "statement": "Platform foundation provisioning, configuration, and operational validation are completed to support core payments engine workload ingestion within 10 weeks.", + "source": [ + "K-4" + ] + }, + { + "id": "OR-8", + "kind": "service_level", + "statement": "Platform foundation infrastructure maintains measured monthly availability of at least 99.9%, available 24 hours a day, 7 days a week.", + "source": [ + "K-5", + "operations.availability_hours" + ] + }, + { + "id": "OR-9", + "kind": "service_level", + "statement": "Platform foundation disaster recovery activation achieves a recovery time objective (RTO) of no more than 4 hours (as assumed in service_levels.recovery_time_hours).", + "source": [ + "K-6", + "service_levels.recovery_time_hours" + ] + }, + { + "id": "OR-10", + "kind": "service_level", + "statement": "Disaster recovery processes achieve a recovery point objective (RPO) of no more than 1 hour of data loss.", + "source": [ + "K-7" + ] + }, + { + "id": "OR-11", + "kind": "process", + "statement": "Platform telemetry alert notifications trigger operational incident workflows immediately upon detection, 24 hours a day, 7 days a week.", + "source": [ + "K-8" + ] + }, + { + "id": "OR-12", + "kind": "process", + "statement": "Monthly cloud foundation expenditure does not exceed the agreed target budget allocation without prior operational review and formal sign-off.", + "source": [ + "K-9" + ] + }, + { + "id": "OR-13", + "kind": "service_level", + "statement": "Hybrid connectivity between cloud and on-premises environments maintains deterministic low latency suitable for real-time payment transaction processing.", + "source": [ + "K-10" + ] + }, + { + "id": "OR-14", + "kind": "process", + "statement": "Deployments and operational workload executions optimize carbon-free energy utilization based on regional carbon-free energy (CFE) scores.", + "source": [ + "K-11" + ] + }, + { + "id": "OR-15", + "kind": "process", + "statement": "Access permissions across platform identities are continuously rightsized to least privilege based on automated access findings.", + "source": [ + "K-12" + ] + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/organisation.mmd b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/organisation.mmd new file mode 100644 index 0000000..a2cf225 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/organisation.mmd @@ -0,0 +1,19 @@ +flowchart TB + subgraph P_1["Customer Platform Team (customer)"] + subgraph T_3["Customer Application Teams (stream-aligned team)"] + R_6["Customer Stakeholder
GB, lead, business"] + end + end + subgraph P_2["Service Provider (provider)"] + subgraph T_1["Central Platform Operations Team (platform team)"] + R_1["Platform Operations Engineer
PL, engineer, 24x7"] + R_2["Platform Security & Network Engineer
PL, senior, 24x7"] + R_3["Platform Reliability Lead
GB, lead, business"] + end + subgraph T_2["Platform Product & Enablement Team (platform team)"] + R_4["Platform Product Manager
GB, lead, business"] + R_5["Platform Enablement Engineer
GB, senior, business"] + end + end + T_3 -- "x-as-a-service" --> T_1 + T_2 -- "facilitating" --> T_3 diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scenarios.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scenarios.json new file mode 100644 index 0000000..d041e9f --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scenarios.json @@ -0,0 +1,219 @@ +[ + { + "id": "SC-fault-S-1", + "scenario": "Resource Hierarchy and Environment Boundary Manager (S-1) fails at an hour faults are responded to (24x7).", + "about": "S-1", + "passes": true, + "path": "Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-1", + "scenario": "Resource Hierarchy and Environment Boundary Manager (S-1) must be changed.", + "about": "S-1", + "passes": true, + "path": "Platform Security & Network Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-2", + "scenario": "Dedicated Private Hybrid Connectivity Gateway (S-2) fails at an hour faults are responded to (24x7).", + "about": "S-2", + "passes": true, + "path": "Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-2", + "scenario": "Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed.", + "about": "S-2", + "passes": true, + "path": "Platform Security & Network Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-3", + "scenario": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) fails at an hour faults are responded to (24x7).", + "about": "S-3", + "passes": true, + "path": "Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-3", + "scenario": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed.", + "about": "S-3", + "passes": true, + "path": "Platform Security & Network Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-4", + "scenario": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) fails at an hour faults are responded to (24x7).", + "about": "S-4", + "passes": true, + "path": "Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-4", + "scenario": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) must be changed.", + "about": "S-4", + "passes": true, + "path": "Platform Security & Network Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-5", + "scenario": "Automated Infrastructure Orchestration and Deployment Engine (S-5) fails at an hour faults are responded to (24x7).", + "about": "S-5", + "passes": true, + "path": "Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-5", + "scenario": "Automated Infrastructure Orchestration and Deployment Engine (S-5) must be changed.", + "about": "S-5", + "passes": true, + "path": "Platform Security & Network Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-6", + "scenario": "Federated Identity and Role Governance Authority (S-6) fails at an hour faults are responded to (24x7).", + "about": "S-6", + "passes": true, + "path": "Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-6", + "scenario": "Federated Identity and Role Governance Authority (S-6) must be changed.", + "about": "S-6", + "passes": true, + "path": "Platform Security & Network Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-7", + "scenario": "Cryptographic Key Lifecycle and Encryption Controller (S-7) fails at an hour faults are responded to (24x7).", + "about": "S-7", + "passes": true, + "path": "Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-7", + "scenario": "Cryptographic Key Lifecycle and Encryption Controller (S-7) must be changed.", + "about": "S-7", + "passes": true, + "path": "Platform Security & Network Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-8", + "scenario": "Platform Telemetry and Operational Alerting Engine (S-8) fails at an hour faults are responded to (24x7).", + "about": "S-8", + "passes": true, + "path": "Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-8", + "scenario": "Platform Telemetry and Operational Alerting Engine (S-8) must be changed.", + "about": "S-8", + "passes": true, + "path": "Platform Security & Network Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-9", + "scenario": "Financial Governance and Budget Enforcement Monitor (S-9) fails at an hour faults are responded to (24x7).", + "about": "S-9", + "passes": true, + "path": "Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "id": "SC-change-S-9", + "scenario": "Financial Governance and Budget Enforcement Monitor (S-9) must be changed.", + "about": "S-9", + "passes": true, + "path": "Platform Security & Network Engineer makes the change under C-5." + }, + { + "id": "SC-condition-K-1", + "scenario": "Condition K-1 must be shown to hold: The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit.", + "about": "K-1", + "passes": true, + "path": "Check OC-K1 on C-21 keeps it true." + }, + { + "id": "SC-condition-K-2", + "scenario": "Condition K-2 must be shown to hold: The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic.", + "about": "K-2", + "passes": true, + "path": "Check OC-K2 on C-9 keeps it true." + }, + { + "id": "SC-condition-K-3", + "scenario": "Condition K-3 must be shown to hold: Data residency is guaranteed within the United Kingdom and European Union.", + "about": "K-3", + "passes": true, + "path": "Check OC-K3 on C-7 keeps it true." + }, + { + "id": "SC-condition-K-4", + "scenario": "Condition K-4 must be shown to hold: Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks.", + "about": "K-4", + "passes": true, + "path": "Check OC-K4 on C-16 keeps it true." + }, + { + "id": "SC-condition-K-5", + "scenario": "Condition K-5 must be shown to hold: Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.", + "about": "K-5", + "passes": true, + "path": "Check OC-K5 on C-15 keeps it true." + }, + { + "id": "SC-condition-K-6", + "scenario": "Condition K-6 must be shown to hold: Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.", + "about": "K-6", + "passes": true, + "path": "Check OC-K6 on C-1 keeps it true." + }, + { + "id": "SC-condition-K-7", + "scenario": "Condition K-7 must be shown to hold: Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour.", + "about": "K-7", + "passes": true, + "path": "Check OC-K7 on C-8 keeps it true." + }, + { + "id": "SC-condition-K-8", + "scenario": "Condition K-8 must be shown to hold: Alert notifications trigger operational incident workflows for the 24/7 central platform operations team.", + "about": "K-8", + "passes": true, + "path": "Check OC-K8 on C-2 keeps it true." + }, + { + "id": "SC-condition-K-9", + "scenario": "Condition K-9 must be shown to hold: Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off.", + "about": "K-9", + "passes": true, + "path": "Check OC-K9 on C-14 keeps it true." + }, + { + "id": "SC-condition-K-10", + "scenario": "Condition K-10 must be shown to hold: Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing.", + "about": "K-10", + "passes": true, + "path": "Check OC-K10 on C-10 keeps it true." + }, + { + "id": "SC-condition-K-11", + "scenario": "Condition K-11 must be shown to hold: Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores.", + "about": "K-11", + "passes": true, + "path": "Check OC-K11 on C-10 keeps it true." + }, + { + "id": "SC-condition-K-12", + "scenario": "Condition K-12 must be shown to hold: Permissions are continuously rightsized based on IAM Recommender findings.", + "about": "K-12", + "passes": true, + "path": "Check OC-K12 on C-11 keeps it true." + }, + { + "id": "SC-party-P-2", + "scenario": "Service Provider cannot be reached.", + "about": "P-2", + "passes": true, + "path": "Every Capability it is answerable for has a Role in another Party to turn to." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scope.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scope.json new file mode 100644 index 0000000..fe5fca9 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scope.json @@ -0,0 +1,401 @@ +{ + "kind": "platform_foundation", + "stated": { + "engagement": { + "kind": "platform_foundation" + }, + "intent": { + "statement": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control." + }, + "solution": { + "cloud_provider": "gcp", + "items": [ + { + "id": "S-1", + "name": "Resource Hierarchy and Environment Boundary Manager", + "kind": "governance", + "product": "Resource Manager", + "source_ref": "C-1", + "elements": [ + "E-1", + "E-2", + "E-3", + "E-4", + "E-5", + "E-6", + "E-7", + "E-8", + "E-9", + "E-10", + "E-34", + "E-35", + "E-36", + "E-37", + "E-38", + "E-50", + "E-51", + "E-55", + "E-60" + ] + }, + { + "id": "S-2", + "name": "Dedicated Private Hybrid Connectivity Gateway", + "kind": "network", + "product": "Cloud Interconnect", + "source_ref": "C-2", + "elements": [ + "E-11", + "E-12", + "E-13", + "E-14", + "E-56", + "E-57" + ] + }, + { + "id": "S-3", + "name": "Centralised Outbound Traffic Inspection and Egress Boundary", + "kind": "service", + "product": "Palo Alto Networks VM-Series", + "source_ref": "C-3", + "elements": [ + "E-15", + "E-16", + "E-17", + "E-18", + "E-19", + "E-20", + "E-47", + "E-48", + "E-53" + ] + }, + { + "id": "S-4", + "name": "Centralised Immutable Audit Event and Compliance Log Repository", + "kind": "storage", + "product": "Cloud Storage", + "source_ref": "C-4", + "elements": [ + "E-21", + "E-22", + "E-39" + ] + }, + { + "id": "S-5", + "name": "Automated Infrastructure Orchestration and Deployment Engine", + "kind": "service", + "product": "GitHub Actions", + "source_ref": "C-5", + "elements": [ + "E-42", + "E-43" + ] + }, + { + "id": "S-6", + "name": "Federated Identity and Role Governance Authority", + "kind": "identity", + "product": "IAM Workload Identity Federation", + "source_ref": "C-6", + "elements": [ + "E-23", + "E-24", + "E-25", + "E-26", + "E-27", + "E-45" + ] + }, + { + "id": "S-7", + "name": "Cryptographic Key Lifecycle and Encryption Controller", + "kind": "keys", + "product": "Cloud KMS", + "source_ref": "C-7", + "elements": [ + "E-28", + "E-29", + "E-30", + "E-31", + "E-49", + "E-58", + "E-59" + ] + }, + { + "id": "S-8", + "name": "Platform Telemetry and Operational Alerting Engine", + "kind": "observability", + "product": "Cloud Monitoring", + "source_ref": "C-8", + "elements": [ + "E-32", + "E-40", + "E-41", + "E-44", + "E-52" + ] + }, + { + "id": "S-9", + "name": "Financial Governance and Budget Enforcement Monitor", + "kind": "billing", + "product": "Cloud Billing", + "source_ref": "C-9", + "elements": [ + "E-33", + "E-46", + "E-54" + ] + } + ], + "conditions": [ + { + "id": "K-1", + "on": [ + "S-1", + "S-4", + "S-7" + ], + "condition": "The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit.", + "source_ref": "NFR-8" + }, + { + "id": "K-2", + "on": [ + "S-1" + ], + "condition": "The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic.", + "source_ref": "NFR-9" + }, + { + "id": "K-3", + "on": [ + "S-1" + ], + "condition": "Data residency is guaranteed within the United Kingdom and European Union.", + "source_ref": "NFR-9" + }, + { + "id": "K-4", + "on": [ + "S-5" + ], + "condition": "Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks.", + "source_ref": "NFR-10" + }, + { + "id": "K-5", + "on": [ + "S-8" + ], + "condition": "Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.", + "source_ref": "NFR-11" + }, + { + "id": "K-6", + "on": [ + "S-8" + ], + "condition": "Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.", + "source_ref": "NFR-11" + }, + { + "id": "K-7", + "on": [ + "S-8" + ], + "condition": "Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour.", + "source_ref": "NFR-11" + }, + { + "id": "K-8", + "on": [ + "S-8" + ], + "condition": "Alert notifications trigger operational incident workflows for the 24/7 central platform operations team.", + "source_ref": "NFR-12" + }, + { + "id": "K-9", + "on": [ + "S-9" + ], + "condition": "Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off.", + "source_ref": "NFR-13" + }, + { + "id": "K-10", + "on": [ + "S-2" + ], + "condition": "Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing.", + "source_ref": "NFR-14" + }, + { + "id": "K-11", + "on": [ + "S-8" + ], + "condition": "Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores.", + "source_ref": "NFR-15" + }, + { + "id": "K-12", + "on": [ + "S-6" + ], + "condition": "Permissions are continuously rightsized based on IAM Recommender findings.", + "source_ref": "DR-OP-3" + } + ], + "build": { + "duration_days": 36, + "customer_owned": [ + "Order and provision the Interconnect carrier circuits" + ] + } + }, + "operations": { + "support_hours": "24x7", + "availability_hours": "24x7" + }, + "sourcing": { + "model": "provider_operated", + "delivery_countries": [ + "GB", + "PL", + "IN" + ] + }, + "compliance": { + "frameworks": [ + "DORA", + "ISO27001", + "PCI_DSS_v4" + ] + }, + "commercial": { + "currency": "GBP" + } + }, + "binding": [ + "operations.support_hours", + "sourcing.model", + "sourcing.delivery_countries" + ], + "assumed": [ + { + "field": "intent.business_outcomes", + "value": [ + { + "outcome": "Continuous, highly available, and low-latency operation of the core payments platform foundation", + "target": "99.9% foundation availability with deterministic low latency", + "id": "BO-1" + }, + { + "id": "BO-2", + "target": "Zero audit non-conformances", + "outcome": "Continuous compliance with DORA and PCI-DSS v4.0 standards using customer-managed encryption keys and centralised egress inspection" + }, + { + "outcome": "Automated, secure infrastructure release pipelines enabling workload ingestion within 10 weeks", + "target": "Operational readiness achieved within 10 weeks", + "id": "BO-3" + } + ], + "why": "Derived from workflow request objectives: running core payment processing reliably in Google Cloud, meeting DORA and PCI-DSS v4.0 regulatory standards, and delivering landing zone readiness within 10 weeks." + }, + { + "field": "intent.exclusions", + "value": [ + "Core payment application workload code maintenance and runtime application management", + "On-premises infrastructure hardware maintenance beyond Cloud Interconnect physical termination" + ], + "why": "Platform foundation scope excludes application-level core payment workloads and on-premises data centre hardware beyond the Interconnect boundary." + }, + { + "field": "operations.change_process", + "value": "Changes to the platform foundation are deployed through automated GitHub Actions pipelines with mandatory peer reviews and automated policy validation, without a separate manual change advisory board.", + "why": "Catalogue default aligned with request stating GitHub Actions delivery pipelines: automated pipeline deployment with peer review and automated checks." + }, + { + "field": "organisation.existing_teams", + "value": [ + { + "name": "Central Platform Operations Team", + "hours": "24x7", + "party": "provider" + } + ], + "why": "Catalogue default: all operational roles are newly provided under provider_operated model, with reference to the 24/7 central platform operations team in condition K-8." + }, + { + "field": "service_management.tooling", + "value": "Google Cloud operations suite (Cloud Monitoring, Cloud Logging, and Cloud Alerting) without a separate external ITSM system.", + "why": "Catalogue default: Cloud provider native operations tooling (Cloud Monitoring, Cloud Logging) integrated with alerting." + }, + { + "field": "service_levels.priority_1_response_minutes", + "value": 30, + "why": "Catalogue default response time for Priority 1 critical incidents under 24x7 operational support." + }, + { + "field": "service_levels.recovery_time_hours", + "value": 4, + "why": "Catalogue default aligned with condition K-6 specifying a 4-hour recovery time objective (RTO)." + } + ], + "to_decide": [], + "to_ask": [ + { + "field": "metadata.client_name", + "prompt": "What is metadata.client_name?", + "severity": "LOW" + }, + { + "field": "metadata.author", + "prompt": "What is metadata.author?", + "severity": "LOW" + }, + { + "field": "metadata.revision", + "prompt": "What is metadata.revision?", + "severity": "LOW" + }, + { + "field": "intent.requirements", + "prompt": "Which requirements for operations has the customer stated outright?", + "severity": "LOW" + }, + { + "field": "solution.inherited_risks", + "prompt": "What is already known to be unverified or at risk in the solution?", + "severity": "LOW" + }, + { + "field": "operations.monitoring", + "prompt": "Which monitoring and alerting is already in place or required?", + "severity": "LOW" + }, + { + "field": "organisation.headcount_limit", + "prompt": "Is there a limit on the number of people operating this?", + "severity": "LOW" + }, + { + "field": "commercial.monthly_run_budget", + "prompt": "Is there a monthly budget for operating this, in that currency?", + "severity": "LOW" + }, + { + "field": "transition.go_live_date", + "prompt": "When must the solution be in operation?", + "severity": "LOW" + } + ], + "not_applicable": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scope.yaml new file mode 100644 index 0000000..f3aeeb3 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/scope.yaml @@ -0,0 +1,407 @@ +schema_version: 1 +metadata: + customer_case: case-08-landing-zone + client_name: ask + author: ask + revision: ask + source: architecture:case-08-landing-zone-185426 + status: BASELINE_LOCKED +engagement: + kind: platform_foundation +intent: + statement: 'We are a UK-regulated payment platform expanding into Google Cloud. + We need an + + enterprise-grade landing zone ready in under 10 weeks to support our core payments + + engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have + + existing on-premises systems that require private, low-latency connectivity over + a + + 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and + + GitHub Actions for our delivery pipelines. Security requires that all egress + + traffic flows through centralised inspection appliances before touching the public + + internet, and customer data must be encrypted with keys we control.' + requirements: ask + business_outcomes: + - outcome: Continuous, highly available, and low-latency operation of the core payments + platform foundation + target: 99.9% foundation availability with deterministic low latency + id: BO-1 + - id: BO-2 + target: Zero audit non-conformances + outcome: Continuous compliance with DORA and PCI-DSS v4.0 standards using customer-managed + encryption keys and centralised egress inspection + - outcome: Automated, secure infrastructure release pipelines enabling workload + ingestion within 10 weeks + target: Operational readiness achieved within 10 weeks + id: BO-3 + exclusions: + - Core payment application workload code maintenance and runtime application management + - On-premises infrastructure hardware maintenance beyond Cloud Interconnect physical + termination +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Resource Hierarchy and Environment Boundary Manager + kind: governance + product: Resource Manager + source_ref: C-1 + elements: + - E-1 + - E-2 + - E-3 + - E-4 + - E-5 + - E-6 + - E-7 + - E-8 + - E-9 + - E-10 + - E-34 + - E-35 + - E-36 + - E-37 + - E-38 + - E-50 + - E-51 + - E-55 + - E-60 + - id: S-2 + name: Dedicated Private Hybrid Connectivity Gateway + kind: network + product: Cloud Interconnect + source_ref: C-2 + elements: + - E-11 + - E-12 + - E-13 + - E-14 + - E-56 + - E-57 + - id: S-3 + name: Centralised Outbound Traffic Inspection and Egress Boundary + kind: service + product: Palo Alto Networks VM-Series + source_ref: C-3 + elements: + - E-15 + - E-16 + - E-17 + - E-18 + - E-19 + - E-20 + - E-47 + - E-48 + - E-53 + - id: S-4 + name: Centralised Immutable Audit Event and Compliance Log Repository + kind: storage + product: Cloud Storage + source_ref: C-4 + elements: + - E-21 + - E-22 + - E-39 + - id: S-5 + name: Automated Infrastructure Orchestration and Deployment Engine + kind: service + product: GitHub Actions + source_ref: C-5 + elements: + - E-42 + - E-43 + - id: S-6 + name: Federated Identity and Role Governance Authority + kind: identity + product: IAM Workload Identity Federation + source_ref: C-6 + elements: + - E-23 + - E-24 + - E-25 + - E-26 + - E-27 + - E-45 + - id: S-7 + name: Cryptographic Key Lifecycle and Encryption Controller + kind: keys + product: Cloud KMS + source_ref: C-7 + elements: + - E-28 + - E-29 + - E-30 + - E-31 + - E-49 + - E-58 + - E-59 + - id: S-8 + name: Platform Telemetry and Operational Alerting Engine + kind: observability + product: Cloud Monitoring + source_ref: C-8 + elements: + - E-32 + - E-40 + - E-41 + - E-44 + - E-52 + - id: S-9 + name: Financial Governance and Budget Enforcement Monitor + kind: billing + product: Cloud Billing + source_ref: C-9 + elements: + - E-33 + - E-46 + - E-54 + conditions: + - id: K-1 + 'on': + - S-1 + - S-4 + - S-7 + condition: The landing zone configuration aligns with DORA operational resiliency + guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. + source_ref: NFR-8 + - id: K-2 + 'on': + - S-1 + condition: The secondary region in europe-west1 operates as a warm standby ready + to receive failover traffic. + source_ref: NFR-9 + - id: K-3 + 'on': + - S-1 + condition: Data residency is guaranteed within the United Kingdom and European + Union. + source_ref: NFR-9 + - id: K-4 + 'on': + - S-5 + condition: Platform foundation is fully provisioned and validated for core payments + engine workload ingestion within 10 weeks. + source_ref: NFR-10 + - id: K-5 + 'on': + - S-8 + condition: Platform foundation infrastructure maintains measured monthly availability + of at least 99.9%. + source_ref: NFR-11 + - id: K-6 + 'on': + - S-8 + condition: Disaster recovery activation achieves a recovery time objective (RTO) + of 4 hours. + source_ref: NFR-11 + - id: K-7 + 'on': + - S-8 + condition: Disaster recovery processes achieve a recovery point objective (RPO) + of 1 hour. + source_ref: NFR-11 + - id: K-8 + 'on': + - S-8 + condition: Alert notifications trigger operational incident workflows for the + 24/7 central platform operations team. + source_ref: NFR-12 + - id: K-9 + 'on': + - S-9 + condition: Monthly cloud foundation expenditure does not exceed the target allocation + without operational sign-off. + source_ref: NFR-13 + - id: K-10 + 'on': + - S-2 + condition: Hybrid connection provides deterministic low latency suitable for real-time + payment transaction processing. + source_ref: NFR-14 + - id: K-11 + 'on': + - S-8 + condition: Deployments optimize carbon-free energy utilization based on Google + Cloud regional CFE scores. + source_ref: NFR-15 + - id: K-12 + 'on': + - S-6 + condition: Permissions are continuously rightsized based on IAM Recommender findings. + source_ref: DR-OP-3 + build: + duration_days: 36 + customer_owned: + - Order and provision the Interconnect carrier circuits + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Changes to the platform foundation are deployed through automated + GitHub Actions pipelines with mandatory peer reviews and automated policy validation, + without a separate manual change advisory board. + monitoring: ask +sourcing: + model: provider_operated + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: + - name: Central Platform Operations Team + hours: 24x7 + party: provider + headcount_limit: ask +service_management: + tooling: Google Cloud operations suite (Cloud Monitoring, Cloud Logging, and Cloud + Alerting) without a separate external ITSM system. +service_levels: + priority_1_response_minutes: 30 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA + - ISO27001 + - PCI_DSS_v4 +commercial: + currency: GBP + monthly_run_budget: ask +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.business_outcomes + statement: 'Assumed intent.business_outcomes is [{''outcome'': ''Continuous, highly + available, and low-latency operation of the core payments platform foundation'', + ''target'': ''99.9% foundation availability with deterministic low latency'', + ''id'': ''BO-1''}, {''id'': ''BO-2'', ''target'': ''Zero audit non-conformances'', + ''outcome'': ''Continuous compliance with DORA and PCI-DSS v4.0 standards using + customer-managed encryption keys and centralised egress inspection''}, {''outcome'': + ''Automated, secure infrastructure release pipelines enabling workload ingestion + within 10 weeks'', ''target'': ''Operational readiness achieved within 10 weeks'', + ''id'': ''BO-3''}]. Derived from workflow request objectives: running core payment + processing reliably in Google Cloud, meeting DORA and PCI-DSS v4.0 regulatory + standards, and delivering landing zone readiness within 10 weeks.' + impact_if_invalid: Changes what is measured, and what each measure is for. +- id: ASSUMP-02 + field: intent.exclusions + statement: Assumed intent.exclusions is ['Core payment application workload code + maintenance and runtime application management', 'On-premises infrastructure hardware + maintenance beyond Cloud Interconnect physical termination']. Platform foundation + scope excludes application-level core payment workloads and on-premises data centre + hardware beyond the Interconnect boundary. + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-03 + field: operations.change_process + statement: 'Assumed operations.change_process is Changes to the platform foundation + are deployed through automated GitHub Actions pipelines with mandatory peer reviews + and automated policy validation, without a separate manual change advisory board.. + Catalogue default aligned with request stating GitHub Actions delivery pipelines: + automated pipeline deployment with peer review and automated checks.' + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-04 + field: organisation.existing_teams + statement: 'Assumed organisation.existing_teams is [{''name'': ''Central Platform + Operations Team'', ''hours'': ''24x7'', ''party'': ''provider''}]. Catalogue default: + all operational roles are newly provided under provider_operated model, with reference + to the 24/7 central platform operations team in condition K-8.' + impact_if_invalid: Changes which Roles are new and which are filled. +- id: ASSUMP-05 + field: service_management.tooling + statement: 'Assumed service_management.tooling is Google Cloud operations suite + (Cloud Monitoring, Cloud Logging, and Cloud Alerting) without a separate external + ITSM system.. Catalogue default: Cloud provider native operations tooling (Cloud + Monitoring, Cloud Logging) integrated with alerting.' + impact_if_invalid: Changes tooling and the processes written for it. +- id: ASSUMP-06 + field: service_levels.priority_1_response_minutes + statement: Assumed service_levels.priority_1_response_minutes is 30. Catalogue default + response time for Priority 1 critical incidents under 24x7 operational support. + impact_if_invalid: Changes on-call arrangements and staffing. +- id: ASSUMP-07 + field: service_levels.recovery_time_hours + statement: Assumed service_levels.recovery_time_hours is 4. Catalogue default aligned + with condition K-6 specifying a 4-hour recovery time objective (RTO). + impact_if_invalid: Changes continuity arrangements. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: intent.requirements + prompt: Which requirements for operations has the customer stated outright? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: commercial.monthly_run_budget + prompt: Is there a monthly budget for operating this, in that currency? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: digested + commercial.monthly_run_budget: unknown + compliance.frameworks: digested + engagement.kind: digested + intent.business_outcomes: assumed + intent.exclusions: assumed + intent.requirements: unknown + intent.statement: digested + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: digested + metadata.revision: unknown + metadata.source: digested + operations.availability_hours: digested + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: digested + organisation.existing_teams: assumed + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: assumed + service_levels.recovery_time_hours: assumed + service_management.tooling: assumed + solution.build.customer_owned: digested + solution.build.duration_days: digested + solution.cloud_provider: digested + solution.conditions: digested + solution.inherited_risks: unknown + solution.items: digested + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 3 + answered: 0 + assumed: 7 + unknown: 10 + not_applicable: 0 + digested: 13 diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/staffing.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/staffing.json new file mode 100644 index 0000000..7c200ef --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/staffing.json @@ -0,0 +1,119 @@ +{ + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + }, + "conversion": { + "from": "USD", + "rate": 0.79, + "as_of": "2026-10-13" + }, + "roles": [ + { + "role": "R-1", + "name": "Platform Operations Engineer", + "party": "P-2", + "party_name": "Service Provider", + "country": "PL", + "level": "engineer", + "hours": "24x7", + "effort_hours_per_month": 125.0, + "people": 5.0, + "cost_per_month": 30777 + }, + { + "role": "R-2", + "name": "Platform Security & Network Engineer", + "party": "P-2", + "party_name": "Service Provider", + "country": "PL", + "level": "senior", + "hours": "24x7", + "effort_hours_per_month": 185.0, + "people": 5.0, + "cost_per_month": 41640 + }, + { + "role": "R-3", + "name": "Platform Reliability Lead", + "party": "P-2", + "party_name": "Service Provider", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 20.0, + "people": 0.1, + "cost_per_month": 2212 + }, + { + "role": "R-4", + "name": "Platform Product Manager", + "party": "P-2", + "party_name": "Service Provider", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 30.0, + "people": 0.2, + "cost_per_month": 3318 + }, + { + "role": "R-5", + "name": "Platform Enablement Engineer", + "party": "P-2", + "party_name": "Service Provider", + "country": "GB", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 70.0, + "people": 0.5, + "cost_per_month": 5899 + }, + { + "role": "R-6", + "name": "Customer Stakeholder", + "party": "P-1", + "party_name": "Customer Platform Team", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 0.0, + "people": 0.0, + "cost_per_month": null + } + ], + "people": 10.8, + "cost_per_month": 83846, + "by_party": { + "P-2": 83846 + }, + "by_country": { + "PL": 72417, + "GB": 11429 + }, + "by_capability": { + "C-1": 9849, + "C-2": 9849, + "C-4": 4924, + "C-8": 2462, + "C-11": 3693, + "C-3": 4502, + "C-5": 3376, + "C-6": 4502, + "C-7": 3376, + "C-9": 3376, + "C-10": 3376, + "C-12": 4502, + "C-13": 9003, + "C-21": 5627, + "C-15": 1106, + "C-20": 1106, + "C-14": 1106, + "C-16": 2212, + "C-17": 2528, + "C-18": 2528, + "C-19": 843 + } +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/target-operating-model.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/target-operating-model.json new file mode 100644 index 0000000..9f61612 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/target-operating-model.json @@ -0,0 +1,664 @@ +{ + "parties": [ + { + "kind": "customer", + "name": "Customer Platform Team", + "id": "P-1" + }, + { + "kind": "provider", + "name": "Service Provider", + "id": "P-2" + } + ], + "teams": [ + { + "id": "T-1", + "purpose": "24/7 central platform operations and foundation management.", + "name": "Central Platform Operations Team", + "party": "P-2", + "type": "platform" + }, + { + "purpose": "Platform product management, enablement, and measurement.", + "name": "Platform Product & Enablement Team", + "party": "P-2", + "id": "T-2", + "type": "platform" + }, + { + "name": "Customer Application Teams", + "type": "stream_aligned", + "id": "T-3", + "purpose": "Core payments engine development and workload deployment.", + "party": "P-1" + } + ], + "roles": [ + { + "party": "P-2", + "name": "Platform Operations Engineer", + "hours": "24x7", + "team": "T-1", + "level": "engineer", + "id": "R-1", + "country": "PL" + }, + { + "level": "senior", + "id": "R-2", + "country": "PL", + "hours": "24x7", + "name": "Platform Security & Network Engineer", + "party": "P-2", + "team": "T-1" + }, + { + "hours": "business", + "team": "T-1", + "id": "R-3", + "name": "Platform Reliability Lead", + "party": "P-2", + "country": "GB", + "level": "lead" + }, + { + "name": "Platform Product Manager", + "level": "lead", + "team": "T-2", + "country": "GB", + "party": "P-2", + "id": "R-4", + "hours": "business" + }, + { + "level": "senior", + "id": "R-5", + "team": "T-2", + "name": "Platform Enablement Engineer", + "hours": "business", + "party": "P-2", + "country": "GB" + }, + { + "party": "P-1", + "id": "R-6", + "country": "GB", + "hours": "business", + "team": "T-3", + "name": "Customer Stakeholder", + "level": "lead" + } + ], + "interactions": [ + { + "id": "I-1", + "purpose": "Customer teams consume platform foundation services and infrastructure via automated pipelines.", + "to": "T-1", + "mode": "x_as_a_service", + "from": "T-3" + }, + { + "mode": "facilitating", + "purpose": "Enablement team facilitates customer application teams in adopting pipelines and ingesting workloads.", + "from": "T-2", + "to": "T-3", + "id": "I-2" + } + ], + "measures": [ + { + "id": "M-1", + "framework": "dora-metrics", + "target": "On-demand", + "name": "Deployment frequency", + "outcome": "BO-3", + "measured_by": "C-19", + "layer": "outcome" + }, + { + "target": "High satisfaction", + "framework": "space-framework", + "id": "M-2", + "name": "Developer satisfaction", + "outcome": "BO-3", + "measured_by": "C-19", + "layer": "guardrail" + }, + { + "framework": "devex-framework", + "name": "Feedback loops", + "id": "M-3", + "measured_by": "C-19", + "target": "< 10 minutes", + "outcome": "BO-3", + "layer": "leading" + }, + { + "target": "99.9% availability", + "outcome": "BO-1", + "name": "Platform Availability", + "id": "M-4", + "layer": "service_level", + "measured_by": "C-15" + }, + { + "name": "Hybrid Network Latency", + "outcome": "BO-1", + "id": "M-5", + "target": "Deterministic low latency", + "layer": "service_level", + "measured_by": "C-10" + }, + { + "name": "Audit Non-conformances", + "outcome": "BO-2", + "target": "Zero non-conformances", + "measured_by": "C-21", + "layer": "service_level", + "id": "M-6" + } + ], + "responsibilities": [ + { + "role": "R-3", + "id": "RS-1-A", + "capability": "C-1", + "kind": "accountable" + }, + { + "capability": "C-1", + "id": "RS-1-R", + "kind": "responsible", + "role": "R-1" + }, + { + "role": "R-6", + "capability": "C-1", + "kind": "consulted", + "id": "RS-1-C" + }, + { + "capability": "C-2", + "kind": "accountable", + "id": "RS-2-A", + "role": "R-3" + }, + { + "id": "RS-2-R", + "capability": "C-2", + "kind": "responsible", + "role": "R-1" + }, + { + "id": "RS-2-C", + "kind": "consulted", + "capability": "C-2", + "role": "R-6" + }, + { + "capability": "C-3", + "id": "RS-3-A", + "role": "R-3", + "kind": "accountable" + }, + { + "id": "RS-3-R", + "kind": "responsible", + "capability": "C-3", + "role": "R-2" + }, + { + "id": "RS-3-C", + "role": "R-6", + "kind": "consulted", + "capability": "C-3" + }, + { + "role": "R-3", + "id": "RS-4-A", + "kind": "accountable", + "capability": "C-4" + }, + { + "id": "RS-4-R", + "role": "R-1", + "capability": "C-4", + "kind": "responsible" + }, + { + "kind": "consulted", + "id": "RS-4-C", + "role": "R-6", + "capability": "C-4" + }, + { + "id": "RS-5-A", + "capability": "C-5", + "role": "R-3", + "kind": "accountable" + }, + { + "capability": "C-5", + "id": "RS-5-R", + "role": "R-2", + "kind": "responsible" + }, + { + "kind": "consulted", + "role": "R-6", + "id": "RS-5-C", + "capability": "C-5" + }, + { + "capability": "C-6", + "id": "RS-6-A", + "role": "R-3", + "kind": "accountable" + }, + { + "kind": "responsible", + "role": "R-2", + "capability": "C-6", + "id": "RS-6-R" + }, + { + "capability": "C-6", + "role": "R-6", + "kind": "consulted", + "id": "RS-6-C" + }, + { + "role": "R-3", + "kind": "accountable", + "capability": "C-7", + "id": "RS-7-A" + }, + { + "capability": "C-7", + "id": "RS-7-R", + "role": "R-2", + "kind": "responsible" + }, + { + "kind": "consulted", + "role": "R-6", + "capability": "C-7", + "id": "RS-7-C" + }, + { + "role": "R-3", + "kind": "accountable", + "capability": "C-8", + "id": "RS-8-A" + }, + { + "kind": "responsible", + "role": "R-1", + "capability": "C-8", + "id": "RS-8-R" + }, + { + "id": "RS-8-C", + "capability": "C-8", + "role": "R-6", + "kind": "consulted" + }, + { + "capability": "C-9", + "id": "RS-9-A", + "role": "R-3", + "kind": "accountable" + }, + { + "capability": "C-9", + "id": "RS-9-R", + "kind": "responsible", + "role": "R-2" + }, + { + "capability": "C-9", + "kind": "consulted", + "role": "R-6", + "id": "RS-9-C" + }, + { + "capability": "C-10", + "role": "R-3", + "kind": "accountable", + "id": "RS-10-A" + }, + { + "role": "R-2", + "kind": "responsible", + "id": "RS-10-R", + "capability": "C-10" + }, + { + "id": "RS-10-C", + "capability": "C-10", + "role": "R-6", + "kind": "consulted" + }, + { + "kind": "accountable", + "capability": "C-11", + "id": "RS-11-A", + "role": "R-3" + }, + { + "kind": "responsible", + "role": "R-1", + "capability": "C-11", + "id": "RS-11-R" + }, + { + "role": "R-6", + "id": "RS-11-C", + "kind": "consulted", + "capability": "C-11" + }, + { + "role": "R-3", + "id": "RS-12-A", + "kind": "accountable", + "capability": "C-12" + }, + { + "capability": "C-12", + "role": "R-2", + "kind": "responsible", + "id": "RS-12-R" + }, + { + "kind": "consulted", + "capability": "C-12", + "role": "R-6", + "id": "RS-12-C" + }, + { + "kind": "accountable", + "capability": "C-13", + "id": "RS-13-A", + "role": "R-3" + }, + { + "kind": "responsible", + "role": "R-2", + "id": "RS-13-R", + "capability": "C-13" + }, + { + "role": "R-6", + "capability": "C-13", + "id": "RS-13-C", + "kind": "consulted" + }, + { + "role": "R-4", + "capability": "C-14", + "kind": "accountable", + "id": "RS-14-A" + }, + { + "role": "R-4", + "capability": "C-14", + "id": "RS-14-R", + "kind": "responsible" + }, + { + "id": "RS-14-C", + "capability": "C-14", + "kind": "consulted", + "role": "R-6" + }, + { + "id": "RS-15-A", + "role": "R-3", + "kind": "accountable", + "capability": "C-15" + }, + { + "kind": "responsible", + "id": "RS-15-R", + "capability": "C-15", + "role": "R-3" + }, + { + "id": "RS-15-C", + "kind": "consulted", + "capability": "C-15", + "role": "R-6" + }, + { + "kind": "accountable", + "id": "RS-16-A", + "capability": "C-16", + "role": "R-4" + }, + { + "role": "R-4", + "capability": "C-16", + "id": "RS-16-R", + "kind": "responsible" + }, + { + "capability": "C-16", + "id": "RS-16-C", + "role": "R-6", + "kind": "consulted" + }, + { + "id": "RS-17-A", + "kind": "accountable", + "role": "R-4", + "capability": "C-17" + }, + { + "capability": "C-17", + "kind": "responsible", + "role": "R-5", + "id": "RS-17-R" + }, + { + "kind": "consulted", + "role": "R-6", + "capability": "C-17", + "id": "RS-17-C" + }, + { + "id": "RS-18-A", + "capability": "C-18", + "kind": "accountable", + "role": "R-4" + }, + { + "id": "RS-18-R", + "capability": "C-18", + "kind": "responsible", + "role": "R-5" + }, + { + "capability": "C-18", + "id": "RS-18-C", + "role": "R-6", + "kind": "consulted" + }, + { + "id": "RS-19-A", + "role": "R-4", + "kind": "accountable", + "capability": "C-19" + }, + { + "role": "R-5", + "kind": "responsible", + "id": "RS-19-R", + "capability": "C-19" + }, + { + "capability": "C-19", + "id": "RS-19-C", + "role": "R-6", + "kind": "consulted" + }, + { + "id": "RS-20-A", + "role": "R-3", + "kind": "accountable", + "capability": "C-20" + }, + { + "id": "RS-20-R", + "role": "R-3", + "kind": "responsible", + "capability": "C-20" + }, + { + "capability": "C-20", + "kind": "consulted", + "role": "R-6", + "id": "RS-20-C" + }, + { + "role": "R-3", + "kind": "accountable", + "capability": "C-21", + "id": "RS-21-A" + }, + { + "id": "RS-21-R", + "kind": "responsible", + "role": "R-2", + "capability": "C-21" + }, + { + "kind": "consulted", + "role": "R-6", + "id": "RS-21-C", + "capability": "C-21" + } + ], + "checks": [ + { + "source": "K-1", + "condition": "Regulatory alignment with DORA and PCI-DSS v4.0 is maintained and verified during external audit with zero non-conformances.", + "id": "OC-K1", + "on": "C-21" + }, + { + "condition": "The europe-west1 secondary region is verified as a warm standby ready to receive failover traffic.", + "on": "C-9", + "source": "K-2", + "id": "OC-K2" + }, + { + "source": "K-3", + "condition": "Data residency remains strictly within the United Kingdom and European Union boundaries.", + "id": "OC-K3", + "on": "C-7" + }, + { + "on": "C-16", + "condition": "Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks.", + "id": "OC-K4", + "source": "K-4" + }, + { + "id": "OC-K5", + "condition": "Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.", + "on": "C-15", + "source": "K-5" + }, + { + "on": "C-1", + "id": "OC-K6", + "condition": "Disaster recovery activation achieves a recovery time objective (RTO) of no more than 4 hours.", + "source": "K-6" + }, + { + "id": "OC-K7", + "on": "C-8", + "source": "K-7", + "condition": "Disaster recovery achieves a recovery point objective (RPO) of no more than 1 hour of data loss." + }, + { + "condition": "Alert notifications trigger operational incident workflows for the 24/7 central platform operations team immediately.", + "source": "K-8", + "id": "OC-K8", + "on": "C-2" + }, + { + "condition": "Monthly cloud foundation expenditure does not exceed the agreed target allocation without prior operational review and formal sign-off.", + "on": "C-14", + "id": "OC-K9", + "source": "K-9" + }, + { + "id": "OC-K10", + "on": "C-10", + "condition": "Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing.", + "source": "K-10" + }, + { + "on": "C-10", + "condition": "Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores.", + "id": "OC-K11", + "source": "K-11" + }, + { + "condition": "Permissions are continuously rightsized based on IAM Recommender findings.", + "source": "K-12", + "on": "C-11", + "id": "OC-K12" + }, + { + "source": "OR-1", + "on": "C-1", + "condition": "Priority 1 critical incidents on platform foundation components are acknowledged and responded to within 30 minutes, 24 hours a day, 7 days a week.", + "id": "OC-OR1" + }, + { + "id": "OC-OR2", + "condition": "External operational management is consistently provided by the service provider.", + "source": "OR-2", + "on": "C-4" + }, + { + "on": "C-15", + "id": "OC-OR8", + "source": "OR-8", + "condition": "Platform foundation infrastructure maintains measured monthly availability of at least 99.9%, available 24 hours a day, 7 days a week." + }, + { + "source": "OR-3", + "condition": "Role operates from an approved delivery country.", + "id": "OC-OR3-R1", + "on": "R-1" + }, + { + "on": "R-2", + "condition": "Role operates from an approved delivery country.", + "id": "OC-OR3-R2", + "source": "OR-3" + }, + { + "id": "OC-OR3-R3", + "source": "OR-3", + "on": "R-3", + "condition": "Role operates from an approved delivery country." + }, + { + "condition": "Role operates from an approved delivery country.", + "source": "OR-3", + "on": "R-4", + "id": "OC-OR3-R4" + }, + { + "condition": "Role operates from an approved delivery country.", + "id": "OC-OR3-R5", + "on": "R-5", + "source": "OR-3" + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/verdicts.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/verdicts.json new file mode 100644 index 0000000..1263fa1 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2/verdicts.json @@ -0,0 +1,264 @@ +{ + "dry_run": { + "verdict": "pass", + "evidence": "31 of 31 scenario walk-throughs pass." + }, + "checks": [ + { + "check": "OC-K1", + "verdict": "pass", + "evidence": "Capability C-21 (regulatory-evidence) exists to compile and retain audit evidence for DORA and PCI-DSS v4.0 assessments. Role R-2 is responsible for it." + }, + { + "check": "OC-K2", + "verdict": "pass", + "evidence": "Capability C-9 (service-continuity) is tasked with maintaining runbooks and verifying the europe-west1 secondary region warm standby readiness. Role R-2 is responsible." + }, + { + "evidence": "Capability C-7 (configuration-management) audits actual environment configurations against data residency boundaries (UK and EU). Role R-2 is responsible.", + "check": "OC-K3", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "OC-K4", + "evidence": "Capability C-16 (platform-product-management) is responsible for ensuring the platform foundation is provisioned and validated within 10 weeks. Role R-4 is responsible." + }, + { + "check": "OC-K5", + "verdict": "pass", + "evidence": "Capability C-15 (service-level-management) is set up to track and report on the 99.9% availability requirement. Role R-3 is responsible." + }, + { + "check": "OC-K6", + "verdict": "pass", + "evidence": "Capability C-1 (incident-management) operates 24x7 and takes responsibility for executing disaster recovery activation within the 4-hour RTO at any time of day. Role R-1 provides the 24x7 coverage." + }, + { + "evidence": "Capability C-8 (backup-and-recovery) ensures backups meet the 1-hour RPO and handles testing restores. Role R-1 is responsible.", + "check": "OC-K7", + "verdict": "pass" + }, + { + "check": "OC-K8", + "verdict": "pass", + "evidence": "Capability C-2 (monitoring-and-alerting) operates 24x7 for immediate response to telemetry signals. Role R-1 provides 24x7 coverage." + }, + { + "check": "OC-K9", + "evidence": "Capability C-14 (cost-management) monitors foundation spend against agreed budgets and manages sign-offs for anomalies. Role R-4 is responsible.", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "OC-K10", + "evidence": "Capability C-10 (capacity-and-performance) monitors Dedicated Interconnect latency and overall infrastructure capacity for payment processing performance. Role R-2 is responsible." + }, + { + "verdict": "pass", + "check": "OC-K11", + "evidence": "Capability C-10 (capacity-and-performance) oversees optimization of workload placements for carbon-free energy (CFE). Role R-2 is responsible." + }, + { + "verdict": "pass", + "check": "OC-K12", + "evidence": "Capability C-11 (access-management) executes continuous identity rightsizing based on automated findings. Role R-1 is responsible." + }, + { + "evidence": "Capability C-1 (incident-management) explicitly offers 24x7 coverage to acknowledge priority 1 incidents within 30 minutes, supported by the 24x7 R-1 role.", + "check": "OC-OR1", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "OC-OR2", + "evidence": "Capability C-4 (service-request-fulfilment) represents non-fault operational management by the Service Provider. Sourcing model ensures external operational management." + }, + { + "check": "OC-OR8", + "verdict": "pass", + "evidence": "Capability C-15 (service-level-management) tracks and reports on the 99.9% availability requirement on a monthly basis. Role R-3 is responsible." + }, + { + "check": "OC-OR3-R1", + "evidence": "Role R-1 operates from Poland (PL), which is an approved delivery country.", + "verdict": "pass" + }, + { + "check": "OC-OR3-R2", + "evidence": "Role R-2 operates from Poland (PL), which is an approved delivery country.", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "Role R-3 operates from the United Kingdom (GB), which is an approved delivery country.", + "check": "OC-OR3-R3" + }, + { + "evidence": "Role R-4 operates from the United Kingdom (GB), which is an approved delivery country.", + "check": "OC-OR3-R4", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "OC-OR3-R5", + "evidence": "Role R-5 operates from the United Kingdom (GB), which is an approved delivery country." + }, + { + "check": "SC-fault-S-1", + "verdict": "pass", + "evidence": "Resource Hierarchy and Environment Boundary Manager (S-1) fails at an hour faults are responded to (24x7). Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-1", + "verdict": "pass", + "evidence": "Resource Hierarchy and Environment Boundary Manager (S-1) must be changed. Platform Security & Network Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-2", + "verdict": "pass", + "evidence": "Dedicated Private Hybrid Connectivity Gateway (S-2) fails at an hour faults are responded to (24x7). Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-2", + "verdict": "pass", + "evidence": "Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed. Platform Security & Network Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-3", + "verdict": "pass", + "evidence": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) fails at an hour faults are responded to (24x7). Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-3", + "verdict": "pass", + "evidence": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed. Platform Security & Network Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-4", + "verdict": "pass", + "evidence": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) fails at an hour faults are responded to (24x7). Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-4", + "verdict": "pass", + "evidence": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) must be changed. Platform Security & Network Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-5", + "verdict": "pass", + "evidence": "Automated Infrastructure Orchestration and Deployment Engine (S-5) fails at an hour faults are responded to (24x7). Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-5", + "verdict": "pass", + "evidence": "Automated Infrastructure Orchestration and Deployment Engine (S-5) must be changed. Platform Security & Network Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-6", + "verdict": "pass", + "evidence": "Federated Identity and Role Governance Authority (S-6) fails at an hour faults are responded to (24x7). Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-6", + "verdict": "pass", + "evidence": "Federated Identity and Role Governance Authority (S-6) must be changed. Platform Security & Network Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-7", + "verdict": "pass", + "evidence": "Cryptographic Key Lifecycle and Encryption Controller (S-7) fails at an hour faults are responded to (24x7). Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-7", + "verdict": "pass", + "evidence": "Cryptographic Key Lifecycle and Encryption Controller (S-7) must be changed. Platform Security & Network Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-8", + "verdict": "pass", + "evidence": "Platform Telemetry and Operational Alerting Engine (S-8) fails at an hour faults are responded to (24x7). Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-8", + "verdict": "pass", + "evidence": "Platform Telemetry and Operational Alerting Engine (S-8) must be changed. Platform Security & Network Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-9", + "verdict": "pass", + "evidence": "Financial Governance and Budget Enforcement Monitor (S-9) fails at an hour faults are responded to (24x7). Platform Operations Engineer responds under C-1, and Platform Reliability Lead is answerable." + }, + { + "check": "SC-change-S-9", + "verdict": "pass", + "evidence": "Financial Governance and Budget Enforcement Monitor (S-9) must be changed. Platform Security & Network Engineer makes the change under C-5." + }, + { + "check": "SC-condition-K-1", + "verdict": "pass", + "evidence": "Condition K-1 must be shown to hold: The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. Check OC-K1 on C-21 keeps it true." + }, + { + "check": "SC-condition-K-2", + "verdict": "pass", + "evidence": "Condition K-2 must be shown to hold: The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic. Check OC-K2 on C-9 keeps it true." + }, + { + "check": "SC-condition-K-3", + "verdict": "pass", + "evidence": "Condition K-3 must be shown to hold: Data residency is guaranteed within the United Kingdom and European Union. Check OC-K3 on C-7 keeps it true." + }, + { + "check": "SC-condition-K-4", + "verdict": "pass", + "evidence": "Condition K-4 must be shown to hold: Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks. Check OC-K4 on C-16 keeps it true." + }, + { + "check": "SC-condition-K-5", + "verdict": "pass", + "evidence": "Condition K-5 must be shown to hold: Platform foundation infrastructure maintains measured monthly availability of at least 99.9%. Check OC-K5 on C-15 keeps it true." + }, + { + "check": "SC-condition-K-6", + "verdict": "pass", + "evidence": "Condition K-6 must be shown to hold: Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours. Check OC-K6 on C-1 keeps it true." + }, + { + "check": "SC-condition-K-7", + "verdict": "pass", + "evidence": "Condition K-7 must be shown to hold: Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour. Check OC-K7 on C-8 keeps it true." + }, + { + "check": "SC-condition-K-8", + "verdict": "pass", + "evidence": "Condition K-8 must be shown to hold: Alert notifications trigger operational incident workflows for the 24/7 central platform operations team. Check OC-K8 on C-2 keeps it true." + }, + { + "check": "SC-condition-K-9", + "verdict": "pass", + "evidence": "Condition K-9 must be shown to hold: Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off. Check OC-K9 on C-14 keeps it true." + }, + { + "check": "SC-condition-K-10", + "verdict": "pass", + "evidence": "Condition K-10 must be shown to hold: Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing. Check OC-K10 on C-10 keeps it true." + }, + { + "check": "SC-condition-K-11", + "verdict": "pass", + "evidence": "Condition K-11 must be shown to hold: Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores. Check OC-K11 on C-10 keeps it true." + }, + { + "check": "SC-condition-K-12", + "verdict": "pass", + "evidence": "Condition K-12 must be shown to hold: Permissions are continuously rightsized based on IAM Recommender findings. Check OC-K12 on C-11 keeps it true." + }, + { + "check": "SC-party-P-2", + "verdict": "pass", + "evidence": "Service Provider cannot be reached. Every Capability it is answerable for has a Role in another Party to turn to." + } + ], + "proposed_additions": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/capabilities.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/capabilities.json new file mode 100644 index 0000000..511453d --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/capabilities.json @@ -0,0 +1,187 @@ +[ + { + "id": "C-1", + "catalogue": "incident-management", + "serves": ["OR-1", "OR-2", "OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "24x7", + "effort_hours_per_month": 40, + "basis": "Handling service disruptions across all platform items. Provides 30-minute response capability for priority 1 issues 24x7." + }, + { + "id": "C-2", + "catalogue": "monitoring-and-alerting", + "serves": ["OR-1", "OR-2", "OR-3", "OR-8"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "24x7", + "effort_hours_per_month": 30, + "basis": "Configuring, tuning, and maintaining telemetry pipelines and automated alerting rules on a continuous basis." + }, + { + "id": "C-3", + "catalogue": "problem-management", + "serves": ["OR-2"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Post-incident analysis and root cause remediation to protect the 99.9% availability target." + }, + { + "id": "C-4", + "catalogue": "service-request-fulfilment", + "serves": ["OR-12"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Managing non-incident requests from customer teams regarding the foundation environment." + }, + { + "id": "C-5", + "catalogue": "change-enablement", + "serves": ["OR-7", "OR-14"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Reviewing infrastructure changes to ensure DORA, PCI-DSS v4.0, and ISO27001 regulatory compliance." + }, + { + "id": "C-6", + "catalogue": "release-and-deployment", + "serves": ["OR-7", "OR-10"], + "items": ["S-5"], + "hours": "business", + "effort_hours_per_month": 25, + "basis": "Operating GitHub Actions CI/CD pipelines to ensure timely deployments and carbon-optimised workload placement." + }, + { + "id": "C-7", + "catalogue": "configuration-management", + "serves": ["OR-6", "OR-14"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Enforcing state definition parity across the active and warm standby regions, and auditing configurations." + }, + { + "id": "C-8", + "catalogue": "backup-and-recovery", + "serves": ["OR-4", "OR-5"], + "items": ["S-1", "S-4", "S-7"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Configuring backup policies and regularly rehearsing restoration processes for Cloud Storage, KMS, and foundation state to meet RPO/RTO targets." + }, + { + "id": "C-9", + "catalogue": "service-continuity", + "serves": ["OR-4", "OR-5", "OR-6"], + "items": ["S-1", "S-2", "S-3", "S-4", "S-5", "S-6", "S-7", "S-8", "S-9"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Planning and periodic testing of regional failovers to europe-west1 to validate readiness and timing." + }, + { + "id": "C-10", + "catalogue": "capacity-and-performance", + "serves": ["OR-2", "OR-3"], + "items": ["S-1", "S-2", "S-3"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Monitoring and optimising Interconnect bandwidth and firewall VM capacity to guarantee deterministic latency and scaling." + }, + { + "id": "C-11", + "catalogue": "access-management", + "serves": ["OR-11", "OR-13"], + "items": ["S-1", "S-6"], + "hours": "business", "effort_hours_per_month": 20, "basis": "Continuously applying automated IAM rightsizing recommendations and managing operational access from approved countries." + }, + { + "id": "C-12", + "catalogue": "vulnerability-and-patch-management", + "serves": ["OR-14"], + "items": ["S-1", "S-3", "S-6"], + "hours": "business", "effort_hours_per_month": 25, "basis": "Identifying and remediating CVEs in the cloud environment and Palo Alto VM-Series firewalls to meet compliance standards." + }, + { + "id": "C-13", + "catalogue": "security-monitoring-and-response", + "serves": ["OR-14", "OR-15"], + "items": ["S-1", "S-3", "S-4", "S-6", "S-7"], + "hours": "24x7", + "effort_hours_per_month": 30, + "basis": "Watching for and responding to security misconfigurations, anomalies, and attempts to breach residency or perimeter isolation boundaries around the clock." + }, + { + "id": "C-14", + "catalogue": "cost-management", + "serves": ["OR-9"], + "items": ["S-1", "S-9"], + "hours": "business", + "effort_hours_per_month": 15, + "basis": "Monitoring cloud expenditure against allocated budgets, and designing and operating automation to enforce hard financial limits and execution stops." + }, + { + "id": "C-15", + "catalogue": "service-level-management", + "serves": ["OR-1", "OR-2", "OR-3"], + "items": ["S-1", "S-2", "S-8"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Aggregating and publishing reporting on 99.9% uptime, 30-minute response times, and network latency commitments." + }, + { + "id": "C-16", + "catalogue": "platform-product-management", + "serves": ["OR-7", "OR-12"], + "items": ["S-1", "S-5"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Prioritising features and technical debt to ensure the platform foundation supports the core payments engine developers effectively." + }, + { + "id": "C-17", + "catalogue": "self-service-and-golden-paths", + "serves": ["OR-7", "OR-12"], + "items": ["S-1", "S-5"], + "hours": "business", + "effort_hours_per_month": 30, + "basis": "Building reusable Terraform modules and GitHub Actions templates to enable developers to deploy securely by default." + }, + { + "id": "C-18", + "catalogue": "enablement", + "serves": ["OR-7", "OR-12"], + "items": ["S-1", "S-5"], + "hours": "business", + "effort_hours_per_month": 25, + "basis": "Working directly with the payments engine team to coach them on the new GCP landing zone and CI/CD pipelines." + }, + { + "id": "C-19", + "catalogue": "delivery-and-developer-experience-measurement", + "serves": ["OR-7", "OR-12"], + "items": ["S-1", "S-5", "S-8"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Surveying cognitive load and measuring flow metrics across platform deployment activities." + }, + { + "id": "C-20", + "catalogue": "supplier-management", + "serves": ["OR-12", "OR-13"], + "items": ["S-2", "S-3"], + "hours": "business", + "effort_hours_per_month": 10, + "basis": "Managing ongoing relationship and escalations with Interconnect carriers and Palo Alto Networks." + }, + { + "id": "C-21", + "catalogue": "regulatory-evidence", + "serves": ["OR-14", "OR-15"], + "items": ["S-1", "S-4", "S-7"], + "hours": "business", + "effort_hours_per_month": 20, + "basis": "Continually capturing, preserving, and providing the documented audit evidence required to prove compliance with DORA, PCI-DSS v4.0, and ISO27001 to regulators." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/execution.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/execution.json new file mode 100644 index 0000000..0a32ce8 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/execution.json @@ -0,0 +1,151 @@ +{ + "customer_case": "case-08-landing-zone", + "execution_id": "case-02-from-architecture-r3", + "session_id": "90a6f5db-fafb-4808-99f3-8c9adba4cbf6", + "intent_version": 3, + "workflow_request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.", + "baseline": null, + "overlay_skills": {}, + "options": {}, + "sources": { + "execution.json": { + "sha256": "8bf269deffc8e4bce474bd8d66959c03b04de01dc137eefaf96ead74b46a5a45", + "digested": true + }, + "scope.json": { + "sha256": "57dc28ab7397f514ef0e83b292fc2165554ac2b516775c973f4965765a9e905f", + "digested": true + }, + "requirements.json": { + "sha256": "5aecbef9eaea254fb5123bcc1d107b4a7b7d4d45d60d940464d1fde690d2973d", + "digested": true + }, + "recommendations.json": { + "sha256": "95958b1a107705280d65c9d088bba57352d1f9fa8e67e89b2b3379f78d91cd24", + "digested": true + }, + "components.json": { + "sha256": "49e47dbfa90a85d086711eb2ba66370c145f13fd3cd3595c1df1366224db392c", + "digested": true + }, + "products.json": { + "sha256": "75d51cba319f83000e6f85277b0fb21a7ec02faabac332c5d1800c8f512136db", + "digested": true + }, + "target-architecture.json": { + "sha256": "40879607aaa7041103d8f4d3ba804df8751d2159aa0ce0977d7d821d35f7608b", + "digested": true + }, + "verdicts.json": { + "sha256": "75e17da790266e94a6775d90857a9b503cacd62fd55c92be2d7a3435de301853", + "digested": true + }, + "work-breakdown.json": { + "sha256": "057fd1b0ecd6350a6f9f6101b712a61e78fc10d758a67e785a1dca7e1e674dc3", + "digested": true + } + }, + "scope": { + "status": "BASELINE_LOCKED", + "readiness": { + "supplied": 3, + "answered": 0, + "assumed": 7, + "unknown": 10, + "not_applicable": 0, + "digested": 13 + } + }, + "status": "completed", + "warnings": [], + "risks": [], + "assessment": null, + "phases": { + "scoping": "approved", + "operational_requirements": "approved", + "operating_model_design": "approved", + "validation": "approved", + "packaging": "approved" + }, + "final_document": "operating-model-guide.md", + "changes": null, + "deliverables": { + "operating_scope": { + "document": "deliverables/operating_scope.md", + "attachments": [ + "scope.json", + "scope.yaml" + ], + "status": "submitted" + }, + "operational_requirements": { + "document": "deliverables/operational_requirements.md", + "attachments": [ + "operational-requirements.json" + ], + "status": "submitted" + }, + "capability_model": { + "document": "deliverables/capability_model.md", + "attachments": [ + "capabilities.json" + ], + "status": "submitted" + }, + "target_operating_model": { + "document": "deliverables/target_operating_model.md", + "attachments": [ + "findings.json", + "organisation.mmd", + "staffing.json", + "target-operating-model.json" + ], + "status": "submitted" + }, + "validation_results": { + "document": "deliverables/validation_results.md", + "attachments": [ + "scenarios.json", + "verdicts.json" + ], + "status": "submitted" + }, + "operating_model_guide": { + "document": "deliverables/operating_model_guide.md", + "attachments": [], + "status": "submitted" + } + }, + "files": [ + "capabilities.json", + "deliverables/capability_model.md", + "deliverables/operating_model_guide.md", + "deliverables/operating_scope.md", + "deliverables/operational_requirements.md", + "deliverables/target_operating_model.md", + "deliverables/validation_results.md", + "findings.json", + "operating-model-guide.md", + "operational-requirements.json", + "organisation.mmd", + "scenarios.json", + "scope.json", + "scope.yaml", + "staffing.json", + "target-operating-model.json", + "verdicts.json" + ], + "staffing": { + "people": 16.9, + "cost_per_month": 116409, + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + } + }, + "findings": [], + "change_request": null, + "capability_sources_verified": false +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/findings.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/findings.json new file mode 100644 index 0000000..0637a08 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/findings.json @@ -0,0 +1 @@ +[] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/operating-model-guide.md b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/operating-model-guide.md new file mode 100644 index 0000000..b983467 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/operating-model-guide.md @@ -0,0 +1,288 @@ +# Operating model guide: Google Cloud Foundation for Core Payments Engine + +## Summary + +This operating model guide governs the operation of the Google Cloud platform foundation supporting the core payments engine, encompassing the landing zone resource hierarchy, dedicated hybrid Cloud Interconnect connectivity, centralised egress traffic inspection appliances, immutable compliance audit logging, automated deployment pipelines, federated identity authority, customer-managed encryption keys, operational telemetry, and financial budget monitors. The platform operates under a provider-operated sourcing model. The Platform Service Provider delivers platform engineering, telemetry, compliance governance, and security operations from delivery locations in the United Kingdom, Poland, and India. The Customer directs platform strategy and product direction through its platform owner, maintaining internal operations liaison and escalation pathways. External suppliers provide the underlying cloud infrastructure, telecommunications carrier circuits, and inspection appliances under provider supplier management. + +Operational support and infrastructure availability are maintained continuously twenty-four hours a day, seven days a week (24x7) for incident triage, telemetry alerting, and perimeter security response, alongside standard business-hours coverage for release deployment, configuration management, and platform enablement. With all validation checks and scenario walk-throughs passing successfully, the reader should look first at the operational assumptions and regulatory baselines documented in this guide—specifically the critical incident acknowledgement target, disaster recovery recovery objectives, and the rate card pricing status—to ensure formal alignment before operational onboarding. + +### Key figures + +- Operated: 9 Items, with 12 Conditions to keep true. +- Faults responded to: 24x7 hours. +- Capabilities: 21. +- People: 16.9. Cost a month: 116,409 GBP, from rate card version `draft-2026-10-13` (placeholder prices: an illustration). +- Scenario walk-throughs: 34 of 34 pass. +- Business outcomes: 3, 3 shown by a Measure. +- Findings for whoever owns the design: 0. + +## Business outcomes + +**BO-1: Deliver an enterprise-grade Google Cloud landing zone within 10 weeks to enable ingestion and operation of the core payments engine.** Target: Foundation ready in under 10 weeks. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Deployment Lead Time | outcome | dora-metrics | Under 1 hour | Delivery and developer experience measurement | Platform Engineering Lead | +| Developer Friction | leading | devex-framework | Low friction reported in monthly surveys | Delivery and developer experience measurement | Platform Engineering Lead | +| Team Satisfaction | guardrail | space-framework | Positive quarterly trend | Delivery and developer experience measurement | Platform Engineering Lead | + +**BO-2: Maintain continuous operational resilience and regulatory compliance with DORA and PCI-DSS v4.0 with platform availability of at least 99.9%.** Target: >=99.9% monthly availability and full regulatory audit alignment. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Platform Availability | service level | | >=99.9% | Service level management | Platform Engineering Lead | +| Time to Restore Service | outcome | dora-metrics | Under 4 hours | Service level management | Platform Engineering Lead | + +**BO-3: Ensure secure, low-latency hybrid transaction transport and zero-trust data protection using customer-managed encryption keys.** Target: Deterministic low-latency 10Gbps interconnect and 100% CMEK lifecycle coverage. + +| Measure | Layer | Framework | Target | Measured by | Answerable | +| --- | --- | --- | --- | --- | --- | +| Interconnect Latency | service level | | Deterministic low latency (<5ms) | Capacity and performance management | Platform Engineering Lead | +| CMEK Lifecycle Coverage | guardrail | | 100% coverage | Regulatory evidence and reporting | Compliance & Governance Analyst | + +## What is operated + +| Item | What it is | Kind | Product | From | +| --- | --- | --- | --- | --- | +| S-1 | Resource Hierarchy and Environment Boundary Manager | governance | Resource Manager | C-1 | +| S-2 | Dedicated Private Hybrid Connectivity Gateway | network | Cloud Interconnect | C-2 | +| S-3 | Centralised Outbound Traffic Inspection and Egress Boundary | service | Palo Alto Networks VM-Series | C-3 | +| S-4 | Centralised Immutable Audit Event and Compliance Log Repository | storage | Cloud Storage | C-4 | +| S-5 | Automated Infrastructure Orchestration and Deployment Engine | service | GitHub Actions | C-5 | +| S-6 | Federated Identity and Role Governance Authority | identity | IAM Workload Identity Federation | C-6 | +| S-7 | Cryptographic Key Lifecycle and Encryption Controller | keys | Cloud KMS | C-7 | +| S-8 | Platform Telemetry and Operational Alerting Engine | observability | Cloud Monitoring | C-8 | +| S-9 | Financial Governance and Budget Enforcement Monitor | billing | Cloud Billing | C-9 | + +What must be kept true while it is run: + +| Condition | On | What must hold | From | +| --- | --- | --- | --- | +| K-1 | S-1, S-4, S-7 | The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. | NFR-8 | +| K-2 | S-1 | The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic. | NFR-9 | +| K-3 | S-1 | Data residency is guaranteed within the United Kingdom and European Union. | NFR-9 | +| K-4 | S-5 | Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks. | NFR-10 | +| K-5 | S-8 | Platform foundation infrastructure maintains measured monthly availability of at least 99.9%. | NFR-11 | +| K-6 | S-8 | Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours. | NFR-11 | +| K-7 | S-8 | Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour. | NFR-11 | +| K-8 | S-8 | Alert notifications trigger operational incident workflows for the 24/7 central platform operations team. | NFR-12 | +| K-9 | S-9 | Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off. | NFR-13 | +| K-10 | S-2 | Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing. | NFR-14 | +| K-11 | S-8 | Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores. | NFR-15 | +| K-12 | S-6 | Permissions are continuously rightsized based on IAM Recommender findings. | DR-OP-3 | + +## Operational requirements + +| Requirement | Kind | What must be achieved | From | +| --- | --- | --- | --- | +| OR-1 | service_level | A Priority 1 fault on any platform foundation Item is acknowledged within 30 minutes (as assumed in service_levels.priority_1_response_minutes), 24 hours a day, 7 days a week. | operations.support_hours, service_levels.priority_1_response_minutes | +| OR-2 | service_level | The platform foundation infrastructure maintains measured monthly service availability of at least 99.9%, evaluated over continuous 24x7 operating hours. | K-5, operations.availability_hours | +| OR-3 | service_level | The dedicated hybrid interconnect maintains deterministic low network latency suitable for real-time payment transaction processing. | K-10 | +| OR-4 | service_level | Disaster recovery activation achieves full operational restoration within a recovery time objective (RTO) of 4 hours (as assumed in service_levels.recovery_time_hours). | K-6, service_levels.recovery_time_hours | +| OR-5 | service_level | Disaster recovery failover achieves a recovery point objective (RPO) of no more than 1 hour of operational state or configuration loss. | K-7 | +| OR-6 | process | The secondary cloud region in europe-west1 is continuously maintained as an active warm standby ready to receive failover traffic immediately upon activation. | K-2 | +| OR-7 | process | The platform foundation is fully provisioned, validated, and ready for core payments engine workload ingestion within 10 weeks of initiation. | K-4 | +| OR-8 | process | Platform telemetry alert notifications automatically trigger operational incident response workflows without human delay, maintaining active 24x7 coverage. | K-8 | +| OR-9 | process | Monthly cloud foundation consumption expenditure is tracked against allocated budgets and prevented from exceeding limits without documented operational sign-off. | K-9 | +| OR-10 | process | Platform foundation deployments and workload placements optimize carbon-free energy utilization based on regional carbon-free energy scores. | K-11 | +| OR-11 | process | Access permissions across all foundation environments are continuously audited and rightsized based on automated access recommendations. | K-12 | +| OR-12 | organisation | Platform foundation operations and infrastructure maintenance are conducted under a provider-operated delivery model. | sourcing.model | +| OR-13 | organisation | All operational management and administrative access to the platform foundation are executed exclusively from delivery locations within the United Kingdom, Poland, and India. | sourcing.delivery_countries | +| OR-14 | compliance | Platform foundation configurations and audit mechanisms maintain continuous alignment with DORA operational resiliency guidelines, PCI-DSS v4.0 perimeter isolation standards, and ISO27001 information security controls during external audit. | K-1, compliance.frameworks | +| OR-15 | compliance | Data residency for all platform foundation storage, audit logs, and cryptographic keys is guaranteed within the United Kingdom and European Union. | K-3 | + +## Capabilities + +| Capability | What it is | Serves | Covers | Hours | Effort a month | +| --- | --- | --- | --- | --- | --- | +| C-1 | Incident management | OR-1, OR-2, OR-8 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | 24x7 | 40 h | +| C-2 | Monitoring and alerting | OR-1, OR-2, OR-3, OR-8 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | 24x7 | 30 h | +| C-3 | Problem management | OR-2 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 20 h | +| C-4 | Service request fulfilment | OR-12 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 20 h | +| C-5 | Change enablement | OR-7, OR-14 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 15 h | +| C-6 | Release and deployment | OR-7, OR-10 | S-5 | business | 25 h | +| C-7 | Configuration management | OR-6, OR-14 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 15 h | +| C-8 | Backup and recovery | OR-4, OR-5 | S-1, S-4, S-7 | business | 15 h | +| C-9 | Service continuity | OR-4, OR-5, OR-6 | S-1, S-2, S-3, S-4, S-5, S-6, S-7, S-8, S-9 | business | 10 h | +| C-10 | Capacity and performance management | OR-2, OR-3 | S-1, S-2, S-3 | business | 15 h | +| C-11 | Access management | OR-11, OR-13 | S-1, S-6 | business | 20 h | +| C-12 | Vulnerability and patch management | OR-14 | S-1, S-3, S-6 | business | 25 h | +| C-13 | Security monitoring and response | OR-14, OR-15 | S-1, S-3, S-4, S-6, S-7 | 24x7 | 30 h | +| C-14 | Cost management | OR-9 | S-1, S-9 | business | 15 h | +| C-15 | Service level management | OR-1, OR-2, OR-3 | S-1, S-2, S-8 | business | 10 h | +| C-16 | Platform product management | OR-7, OR-12 | S-1, S-5 | business | 20 h | +| C-17 | Self-service and golden paths | OR-7, OR-12 | S-1, S-5 | business | 30 h | +| C-18 | Enablement | OR-7, OR-12 | S-1, S-5 | business | 25 h | +| C-19 | Delivery and developer experience measurement | OR-7, OR-12 | S-1, S-5, S-8 | business | 10 h | +| C-20 | Supplier management | OR-12, OR-13 | S-2, S-3 | business | 10 h | +| C-21 | Regulatory evidence and reporting | OR-14, OR-15 | S-1, S-4, S-7 | business | 20 h | + +## Who does what + +| Role | Party | Country | Level | Hours | +| --- | --- | --- | --- | --- | +| Platform Operations Engineer | Platform Service Provider (provider) | IN | engineer | 24x7 | +| Platform Reliability Engineer | Platform Service Provider (provider) | PL | senior | 24x7 | +| Security Operations Analyst | Platform Service Provider (provider) | PL | senior | 24x7 | +| Platform Automation Engineer | Platform Service Provider (provider) | PL | senior | business | +| Platform Engineering Lead | Platform Service Provider (provider) | GB | lead | business | +| Compliance & Governance Analyst | Platform Service Provider (provider) | GB | senior | business | +| Customer Platform Owner | Customer (customer) | GB | lead | business | +| Customer Operations Liaison | Customer (customer) | GB | senior | 24x7 | + +The teams, as Team Topologies names their kinds: + +| Team | Kind | Party | Roles | Purpose | +| --- | --- | --- | --- | --- | +| Cloud Platform Foundation Team | platform | Platform Service Provider | Platform Operations Engineer, Platform Reliability Engineer, Platform Automation Engineer, Platform Engineering Lead | Builds, maintains, and reliably operates the enterprise Google Cloud landing zone. | +| Platform Security Team | complicated-subsystem | Platform Service Provider | Security Operations Analyst | Provides 24x7 security monitoring, egress inspection analysis, and vulnerability management. | +| Customer Central Platform Operations Team | stream-aligned | Customer | Customer Operations Liaison | Receives incident workflows and serves as the internal operations escalation point. | +| Customer Platform Owner Team | stream-aligned | Customer | Customer Platform Owner | Provides business context and sets the strategic direction for the platform. | +| Platform Governance Team | enabling | Platform Service Provider | Compliance & Governance Analyst | Ensures continuous regulatory compliance with DORA, PCI-DSS v4.0, and ISO27001. | + +How the teams work together: + +| From | To | Mode | Until | Purpose | +| --- | --- | --- | --- | --- | +| Customer Central Platform Operations Team | Cloud Platform Foundation Team | x-as-a-service | | Customer Central Platform Operations consumes incident workflows and alert notifications from the Provider Cloud Platform Foundation. | +| Cloud Platform Foundation Team | Platform Security Team | collaboration | PCI-DSS baseline controls are fully automated and verified by external audit, after which they revert to x_as_a_service. | Jointly establish automated security response runbooks and egress inspection rules for PCI-DSS compliance. | +| Platform Governance Team | Cloud Platform Foundation Team | facilitating | | Assisting the platform team in integrating regulatory evidence collection into the automated deployment engine. | + +| Capability | Platform Operations Engineer | Platform Reliability Engineer | Security Operations Analyst | Platform Automation Engineer | Platform Engineering Lead | Compliance & Governance Analyst | Customer Platform Owner | Customer Operations Liaison | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Incident management (C-1) | | R | | | A | | | C | +| Monitoring and alerting (C-2) | R | | | | A | | | C | +| Problem management (C-3) | | | | R | A | | | C | +| Service request fulfilment (C-4) | | | | R | A | | | C | +| Change enablement (C-5) | | | | R | A | R | | C | +| Release and deployment (C-6) | | | | R | A | | | C | +| Configuration management (C-7) | | | | R | A | | | C | +| Backup and recovery (C-8) | | | | R | A | | | C | +| Service continuity (C-9) | | R | | | A | | | C | +| Capacity and performance management (C-10) | | | | R | A | | | C | +| Access management (C-11) | | | A/R | | | | | C | +| Vulnerability and patch management (C-12) | | | A/R | | | | | C | +| Security monitoring and response (C-13) | | | A/R | | | | | C | +| Cost management (C-14) | | | | | A | R | | C | +| Service level management (C-15) | | | | | A | R | | C | +| Platform product management (C-16) | | | | | A/R | | C | | +| Self-service and golden paths (C-17) | | | | R | A | | | C | +| Enablement (C-18) | | | | R | A | | | C | +| Delivery and developer experience measurement (C-19) | | | | R | A/R | | | C | +| Supplier management (C-20) | | | | | A | R | | C | +| Regulatory evidence and reporting (C-21) | | | | | | A/R | C | | + +A accountable, R responsible, C consulted, I informed. + +## Staffing and cost + +| Role | Party | Country | Level | Hours | Effort a month | People | Cost a month | +| --- | --- | --- | --- | --- | --- | --- | --- | +| Platform Operations Engineer | Platform Service Provider | IN | engineer | 24x7 | 30.0 h | 5.0 | 17,034 GBP | +| Platform Reliability Engineer | Platform Service Provider | PL | senior | 24x7 | 50.0 h | 5.0 | 41,640 GBP | +| Security Operations Analyst | Platform Service Provider | PL | senior | 24x7 | 75.0 h | 5.0 | 41,640 GBP | +| Platform Automation Engineer | Platform Service Provider | PL | senior | business | 177.5 h | 1.2 | 8,063 GBP | +| Platform Engineering Lead | Platform Service Provider | GB | lead | business | 25.0 h | 0.2 | 2,765 GBP | +| Compliance & Governance Analyst | Platform Service Provider | GB | senior | business | 62.5 h | 0.5 | 5,267 GBP | +| Customer Platform Owner | Customer | GB | lead | business | 0.0 h | 0.0 | not priced | +| Customer Operations Liaison | Customer | GB | senior | business | 0.0 h | 0.0 | not priced | + +16.9 people in all. Cost a month: 116,409 GBP. + +Worked from rate card `rate-card`, version `draft-2026-10-13`. **Its figures are placeholders: no price has been set, and this cost is an illustration.** +Converted from USD at 0.79, a rate set on 2026-10-13. +The customer's own staff are counted and not priced. A Role that covers hours wider than a working day is never smaller than the team that can cover them. + +## Validation + +| Check | What must hold | Verdict | Evidence | +| --- | --- | --- | --- | +| OC-1 | The landing zone configurations are continuously enforced to align with DORA, PCI-DSS v4.0, and ISO27001 standard baselines. | pass | Capability C-7 provides configuration management, enforcing state definition parity and auditing configurations, and supports ensuring compliance standards like DORA, PCI-DSS v4.0, and ISO27001 are continuously enforced. Role R-4 is responsible for it. | +| OC-2 | The europe-west1 secondary region is continuously maintained as an active warm standby. | pass | Capability C-9 handles service continuity, which includes planning and periodic testing of regional failovers, ensuring the secondary region is maintained as an active warm standby. Role R-2 is responsible for it. | +| OC-3 | Configuration enforces that all data residency, audit logs, and keys remain strictly within the UK and EU. | pass | Capability C-7 enforces configuration state parity and audits configurations, satisfying the condition that data residency, logs, and keys remain in the UK and EU. | +| OC-4 | The foundation platform is verified as fully provisioned and ready for core payments engine workload ingestion within 10 weeks. | pass | Capability C-16 prioritises features and technical debt to ensure the platform foundation supports the payments engine developers effectively, meeting the 10-week provisioning timeframe. R-5 is responsible. | +| OC-5 | Continuous telemetry verifies that the platform foundation infrastructure maintains at least 99.9% monthly availability. | pass | Capability C-2 provides continuous monitoring and alerting 24x7, maintaining telemetry pipelines to verify the infrastructure maintains 99.9% availability. | +| OC-6 | Disaster recovery failover tests verify that the 4-hour RTO can be met. | pass | Capability C-9 handles disaster recovery failover tests to validate readiness and timing to ensure the 4-hour RTO can be met. | +| OC-7 | Backup and restore tests verify that the 1-hour RPO for compliance logs and KMS keys is met. | pass | Capability C-8 rehearses restoration processes for Cloud Storage and KMS to verify the 1-hour RPO is met. | +| OC-8 | Alert notifications from platform telemetry automatically trigger incident workflows for the 24x7 central platform operations team without human delay. | pass | Capability C-2 configures and maintains automated alerting rules that trigger workflows to the 24x7 customer central platform operations team (R-8 is consulted). | +| OC-9 | Cloud consumption expenditure is tracked against budgets, and automation enforces hard limits that prevent overages without operational sign-off. | pass | Capability C-14 designs and operates automation to enforce hard financial limits and execution stops, tracking cloud consumption against budgets. | +| OC-10 | The dedicated hybrid interconnect bandwidth and latency are monitored to ensure deterministic low latency for real-time payments. | pass | Capability C-10 monitors and optimises the dedicated hybrid interconnect bandwidth to guarantee deterministic low latency for real-time payments. | +| OC-11 | Workload placements in the deployment pipeline are optimized for carbon-free energy utilization. | pass | Capability C-6 operates the deployment pipelines to ensure timely deployments and carbon-optimised workload placement. | +| OC-12 | Access permissions are continuously audited and rightsized based on IAM recommender alerts. | pass | Capability C-11 continuously applies automated IAM rightsizing recommendations and manages access from approved countries. | +| OC-13 | A Priority 1 fault on the platform foundation is acknowledged within 30 minutes, at any time of day or night. | pass | Capability C-1 provides 24x7 incident management and 30-minute response capability for Priority 1 faults. | +| OC-14 | The platform foundation is led and managed by a provider-operated delivery model. | pass | Role R-5 (Platform Engineering Lead) sits within the Provider (P-2) party, managing and directing the platform strategy and operations. | +| OC-15 | Operational management is performed exclusively by staff located in the UK, Poland, or India. | pass | All provider operational staff (R-1, R-2, R-3, R-4, R-5, R-6) are based exclusively in the UK, Poland, or India, fulfilling the sourcing requirement. | +| OC-16 | Evidence of continuous alignment with DORA, PCI-DSS v4.0, and ISO27001 is gathered for external audit. | pass | Capability C-21 gathers and preserves documented audit evidence to prove compliance with DORA, PCI-DSS v4.0, and ISO27001 to external regulators. | +| OC-17 | The UK and EU data residency guarantee is evidenced for all storage, audit logs, and keys. | pass | Capability C-21 captures, preserves, and provides documented audit evidence to external regulators, which includes proving data residency constraints. | + +Each scenario is generated from the operating model and traced through it by code. + +| Scenario | Result | Path | +| --- | --- | --- | +| Resource Hierarchy and Environment Boundary Manager (S-1) fails at an hour faults are responded to (24x7). | pass | Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable. | +| Resource Hierarchy and Environment Boundary Manager (S-1) must be changed. | pass | Platform Automation Engineer makes the change under C-5. | +| Dedicated Private Hybrid Connectivity Gateway (S-2) fails at an hour faults are responded to (24x7). | pass | Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable. | +| Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed. | pass | Platform Automation Engineer makes the change under C-5. | +| Centralised Outbound Traffic Inspection and Egress Boundary (S-3) fails at an hour faults are responded to (24x7). | pass | Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable. | +| Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed. | pass | Platform Automation Engineer makes the change under C-5. | +| Centralised Immutable Audit Event and Compliance Log Repository (S-4) fails at an hour faults are responded to (24x7). | pass | Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable. | +| Centralised Immutable Audit Event and Compliance Log Repository (S-4) must be changed. | pass | Platform Automation Engineer makes the change under C-5. | +| Automated Infrastructure Orchestration and Deployment Engine (S-5) fails at an hour faults are responded to (24x7). | pass | Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable. | +| Automated Infrastructure Orchestration and Deployment Engine (S-5) must be changed. | pass | Platform Automation Engineer makes the change under C-5. | +| Federated Identity and Role Governance Authority (S-6) fails at an hour faults are responded to (24x7). | pass | Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable. | +| Federated Identity and Role Governance Authority (S-6) must be changed. | pass | Platform Automation Engineer makes the change under C-5. | +| Cryptographic Key Lifecycle and Encryption Controller (S-7) fails at an hour faults are responded to (24x7). | pass | Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable. | +| Cryptographic Key Lifecycle and Encryption Controller (S-7) must be changed. | pass | Platform Automation Engineer makes the change under C-5. | +| Platform Telemetry and Operational Alerting Engine (S-8) fails at an hour faults are responded to (24x7). | pass | Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable. | +| Platform Telemetry and Operational Alerting Engine (S-8) must be changed. | pass | Platform Automation Engineer makes the change under C-5. | +| Financial Governance and Budget Enforcement Monitor (S-9) fails at an hour faults are responded to (24x7). | pass | Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable. | +| Financial Governance and Budget Enforcement Monitor (S-9) must be changed. | pass | Platform Automation Engineer makes the change under C-5. | +| Condition K-1 must be shown to hold: The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. | pass | Check OC-1 on C-7 keeps it true. | +| Condition K-2 must be shown to hold: The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic. | pass | Check OC-2 on C-9 keeps it true. | +| Condition K-3 must be shown to hold: Data residency is guaranteed within the United Kingdom and European Union. | pass | Check OC-3 on C-7 keeps it true. | +| Condition K-4 must be shown to hold: Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks. | pass | Check OC-4 on C-16 keeps it true. | +| Condition K-5 must be shown to hold: Platform foundation infrastructure maintains measured monthly availability of at least 99.9%. | pass | Check OC-5 on C-2 keeps it true. | +| Condition K-6 must be shown to hold: Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours. | pass | Check OC-6 on C-9 keeps it true. | +| Condition K-7 must be shown to hold: Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour. | pass | Check OC-7 on C-8 keeps it true. | +| Condition K-8 must be shown to hold: Alert notifications trigger operational incident workflows for the 24/7 central platform operations team. | pass | Check OC-8 on C-2 keeps it true. | +| Condition K-9 must be shown to hold: Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off. | pass | Check OC-9 on C-14 keeps it true. | +| Condition K-10 must be shown to hold: Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing. | pass | Check OC-10 on C-10 keeps it true. | +| Condition K-11 must be shown to hold: Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores. | pass | Check OC-11 on C-6 keeps it true. | +| Condition K-12 must be shown to hold: Permissions are continuously rightsized based on IAM Recommender findings. | pass | Check OC-12 on C-11 keeps it true. | +| Platform Service Provider cannot be reached. | pass | Every Capability it is answerable for has a Role in another Party to turn to. | +| Google Cloud cannot be reached. | pass | Every Capability it is answerable for has a Role in another Party to turn to. | +| Network Carrier cannot be reached. | pass | Every Capability it is answerable for has a Role in another Party to turn to. | +| External Firewall Vendor cannot be reached. | pass | Every Capability it is answerable for has a Role in another Party to turn to. | + +## Findings and open questions + +No findings: the solution provides what operating it needs. + +Assumed for the customer, and open to change when they answer: + +- `intent.business_outcomes`: {'target': 'Foundation ready in under 10 weeks', 'id': 'BO-1', 'outcome': 'Deliver an enterprise-grade Google Cloud landing zone within 10 weeks to enable ingestion and operation of the core payments engine.'}, {'outcome': 'Maintain continuous operational resilience and regulatory compliance with DORA and PCI-DSS v4.0 with platform availability of at least 99.9%.', 'target': '>=99.9% monthly availability and full regulatory audit alignment', 'id': 'BO-2'}, {'target': 'Deterministic low-latency 10Gbps interconnect and 100% CMEK lifecycle coverage', 'id': 'BO-3', 'outcome': 'Ensure secure, low-latency hybrid transaction transport and zero-trust data protection using customer-managed encryption keys.'}. Derived from the workflow request stating the need for an enterprise-grade landing zone ready in under 10 weeks to support the core payments engine, complying with DORA and PCI-DSS v4.0, with secure hybrid connectivity and customer-controlled encryption. +- `intent.exclusions`: Core payments engine application-level development, maintenance, and transaction processing logic, On-premises data centre infrastructure and systems beyond the Cloud Interconnect demarcation, Physical telecom carrier circuit ordering and provisioning (customer-owned build item). Derived from engagement kind (platform foundation) and workflow request: application-level workloads, on-prem systems, and physical circuit provisioning are excluded from platform operations. +- `operations.change_process`: Automated CI/CD pipelines via GitHub Actions with peer review and automated policy checks; standard operational changes deployed without a separate manual CAB.. Derived from request stating GitHub Actions delivery pipelines and catalogue default guidance. +- `organisation.existing_teams`: {'party': 'customer', 'name': 'Central Platform Operations Team', 'hours': '24x7'}. Derived from condition K-8 noting alerting to the 24/7 central platform operations team, alongside provider-operated landing zone model. +- `service_management.tooling`: Google Cloud operations suite (Cloud Monitoring and Cloud Logging) integrated with native alerting workflows and GitHub Actions.. Catalogue default guidance leveraging Google Cloud operations suite and GitHub Actions specified in request. +- `service_levels.priority_1_response_minutes`: 30. Standard catalogue default for 24x7 critical infrastructure response. +- `service_levels.recovery_time_hours`: 4. Matches condition K-6 disaster recovery RTO requirement of 4 hours and catalogue default. + +## Sources + +The Capabilities are from our own capability catalogue, which draws on these frameworks. It cites them and certifies nothing against them. + +- ITIL 4: Incident management practice; Monitoring and event management practice; Problem management practice; Service request management practice; Change enablement practice; Release management and Deployment management practices; Service configuration management practice; Service continuity management practice; Capacity and performance management practice; Service level management practice; Supplier management practice +- Site Reliability Engineering: How Google Runs Production Systems: Managing Incidents; Monitoring Distributed Systems; Postmortem Culture: Learning from Failure; Release Engineering; Service Level Objectives +- ISO/IEC 20000-1: Incident management; Service request management; Change management; Service continuity management; Capacity management; Service level management; Supplier management +- Regulation (EU) 2022/2554, the Digital Operational Resilience Act: Article 17, ICT-related incident management process; Article 10, Detection; Article 12, Backup policies and procedures, restoration and recovery procedures and methods; Article 11, Response and recovery; Article 9, Protection and prevention; Article 28, General principles on ICT third-party risk; Article 19, Reporting of major ICT-related incidents +- NIST Cybersecurity Framework: Detect: Continuous Monitoring; Identify: Asset Management; Recover: Incident Recovery Plan Execution; Protect: Identity Management, Authentication, and Access Control; Protect: Platform Security; Detect: Adverse Event Analysis; Respond: Incident Management +- DevOps Research and Assessment (DORA) software delivery metrics: Change failure rate; Deployment frequency and lead time for changes; Deployment frequency; Lead time for changes; Change failure rate; Time to restore service +- ISO/IEC 27001: Access control; Management of technical vulnerabilities; Information security incident management; Compliance with legal and contractual requirements +- FinOps Framework: Allocation; Budgeting; Forecasting; Anomaly Management +- CNCF Platforms White Paper (platform as a product): Platform as a product; Measuring platform success; Platform capabilities; Self-service; Platform as a product +- Team Topologies: Organizing Business and Technology Teams for Fast Flow: Platform team; Thinnest viable platform; X-as-a-Service interaction mode; Enabling team; Facilitating interaction mode +- The SPACE of Developer Productivity: Satisfaction and well-being; Performance; Activity; Communication and collaboration; Efficiency and flow +- DevEx: What Actually Drives Productivity: Feedback loops; Cognitive load; Flow state + +**These references have not been checked against the frameworks by a person.** Treat each as a pointer to where to look. diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/operational-requirements.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/operational-requirements.json new file mode 100644 index 0000000..2d8c48c --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/operational-requirements.json @@ -0,0 +1,126 @@ +[ + { + "id": "OR-1", + "kind": "service_level", + "statement": "A Priority 1 fault on any platform foundation Item is acknowledged within 30 minutes (as assumed in service_levels.priority_1_response_minutes), 24 hours a day, 7 days a week.", + "source": [ + "operations.support_hours", + "service_levels.priority_1_response_minutes" + ] + }, + { + "id": "OR-2", + "kind": "service_level", + "statement": "The platform foundation infrastructure maintains measured monthly service availability of at least 99.9%, evaluated over continuous 24x7 operating hours.", + "source": [ + "K-5", + "operations.availability_hours" + ] + }, + { + "id": "OR-3", + "kind": "service_level", + "statement": "The dedicated hybrid interconnect maintains deterministic low network latency suitable for real-time payment transaction processing.", + "source": [ + "K-10" + ] + }, + { + "id": "OR-4", + "kind": "service_level", + "statement": "Disaster recovery activation achieves full operational restoration within a recovery time objective (RTO) of 4 hours (as assumed in service_levels.recovery_time_hours).", + "source": [ + "K-6", + "service_levels.recovery_time_hours" + ] + }, + { + "id": "OR-5", + "kind": "service_level", + "statement": "Disaster recovery failover achieves a recovery point objective (RPO) of no more than 1 hour of operational state or configuration loss.", + "source": [ + "K-7" + ] + }, + { + "id": "OR-6", + "kind": "process", + "statement": "The secondary cloud region in europe-west1 is continuously maintained as an active warm standby ready to receive failover traffic immediately upon activation.", + "source": [ + "K-2" + ] + }, + { + "id": "OR-7", + "kind": "process", + "statement": "The platform foundation is fully provisioned, validated, and ready for core payments engine workload ingestion within 10 weeks of initiation.", + "source": [ + "K-4" + ] + }, + { + "id": "OR-8", + "kind": "process", + "statement": "Platform telemetry alert notifications automatically trigger operational incident response workflows without human delay, maintaining active 24x7 coverage.", + "source": [ + "K-8" + ] + }, + { + "id": "OR-9", + "kind": "process", + "statement": "Monthly cloud foundation consumption expenditure is tracked against allocated budgets and prevented from exceeding limits without documented operational sign-off.", + "source": [ + "K-9" + ] + }, + { + "id": "OR-10", + "kind": "process", + "statement": "Platform foundation deployments and workload placements optimize carbon-free energy utilization based on regional carbon-free energy scores.", + "source": [ + "K-11" + ] + }, + { + "id": "OR-11", + "kind": "process", + "statement": "Access permissions across all foundation environments are continuously audited and rightsized based on automated access recommendations.", + "source": [ + "K-12" + ] + }, + { + "id": "OR-12", + "kind": "organisation", + "statement": "Platform foundation operations and infrastructure maintenance are conducted under a provider-operated delivery model.", + "source": [ + "sourcing.model" + ] + }, + { + "id": "OR-13", + "kind": "organisation", + "statement": "All operational management and administrative access to the platform foundation are executed exclusively from delivery locations within the United Kingdom, Poland, and India.", + "source": [ + "sourcing.delivery_countries" + ] + }, + { + "id": "OR-14", + "kind": "compliance", + "statement": "Platform foundation configurations and audit mechanisms maintain continuous alignment with DORA operational resiliency guidelines, PCI-DSS v4.0 perimeter isolation standards, and ISO27001 information security controls during external audit.", + "source": [ + "K-1", + "compliance.frameworks" + ] + }, + { + "id": "OR-15", + "kind": "compliance", + "statement": "Data residency for all platform foundation storage, audit logs, and cryptographic keys is guaranteed within the United Kingdom and European Union.", + "source": [ + "K-3" + ] + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/organisation.mmd b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/organisation.mmd new file mode 100644 index 0000000..b363277 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/organisation.mmd @@ -0,0 +1,32 @@ +flowchart TB + subgraph P_1["Customer (customer)"] + subgraph T_3["Customer Central Platform Operations Team (stream-aligned team)"] + R_8["Customer Operations Liaison
GB, senior, 24x7"] + end + subgraph T_4["Customer Platform Owner Team (stream-aligned team)"] + R_7["Customer Platform Owner
GB, lead, business"] + end + end + subgraph P_2["Platform Service Provider (provider)"] + subgraph T_1["Cloud Platform Foundation Team (platform team)"] + R_1["Platform Operations Engineer
IN, engineer, 24x7"] + R_2["Platform Reliability Engineer
PL, senior, 24x7"] + R_4["Platform Automation Engineer
PL, senior, business"] + R_5["Platform Engineering Lead
GB, lead, business"] + end + subgraph T_2["Platform Security Team (complicated-subsystem team)"] + R_3["Security Operations Analyst
PL, senior, 24x7"] + end + subgraph T_5["Platform Governance Team (enabling team)"] + R_6["Compliance & Governance Analyst
GB, senior, business"] + end + end + subgraph P_3["Google Cloud (cloud_provider)"] + end + subgraph P_4["Network Carrier (supplier)"] + end + subgraph P_5["External Firewall Vendor (supplier)"] + end + T_3 -- "x-as-a-service" --> T_1 + T_1 -- "collaboration until PCI-DSS baseline controls are fully automated and verified by external audit, after which they revert to x_as_a_service." --> T_2 + T_5 -- "facilitating" --> T_1 diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scenarios.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scenarios.json new file mode 100644 index 0000000..250af7e --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scenarios.json @@ -0,0 +1,240 @@ +[ + { + "id": "SC-fault-S-1", + "scenario": "Resource Hierarchy and Environment Boundary Manager (S-1) fails at an hour faults are responded to (24x7).", + "about": "S-1", + "passes": true, + "path": "Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "id": "SC-change-S-1", + "scenario": "Resource Hierarchy and Environment Boundary Manager (S-1) must be changed.", + "about": "S-1", + "passes": true, + "path": "Platform Automation Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-2", + "scenario": "Dedicated Private Hybrid Connectivity Gateway (S-2) fails at an hour faults are responded to (24x7).", + "about": "S-2", + "passes": true, + "path": "Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "id": "SC-change-S-2", + "scenario": "Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed.", + "about": "S-2", + "passes": true, + "path": "Platform Automation Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-3", + "scenario": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) fails at an hour faults are responded to (24x7).", + "about": "S-3", + "passes": true, + "path": "Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "id": "SC-change-S-3", + "scenario": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed.", + "about": "S-3", + "passes": true, + "path": "Platform Automation Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-4", + "scenario": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) fails at an hour faults are responded to (24x7).", + "about": "S-4", + "passes": true, + "path": "Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "id": "SC-change-S-4", + "scenario": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) must be changed.", + "about": "S-4", + "passes": true, + "path": "Platform Automation Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-5", + "scenario": "Automated Infrastructure Orchestration and Deployment Engine (S-5) fails at an hour faults are responded to (24x7).", + "about": "S-5", + "passes": true, + "path": "Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "id": "SC-change-S-5", + "scenario": "Automated Infrastructure Orchestration and Deployment Engine (S-5) must be changed.", + "about": "S-5", + "passes": true, + "path": "Platform Automation Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-6", + "scenario": "Federated Identity and Role Governance Authority (S-6) fails at an hour faults are responded to (24x7).", + "about": "S-6", + "passes": true, + "path": "Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "id": "SC-change-S-6", + "scenario": "Federated Identity and Role Governance Authority (S-6) must be changed.", + "about": "S-6", + "passes": true, + "path": "Platform Automation Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-7", + "scenario": "Cryptographic Key Lifecycle and Encryption Controller (S-7) fails at an hour faults are responded to (24x7).", + "about": "S-7", + "passes": true, + "path": "Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "id": "SC-change-S-7", + "scenario": "Cryptographic Key Lifecycle and Encryption Controller (S-7) must be changed.", + "about": "S-7", + "passes": true, + "path": "Platform Automation Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-8", + "scenario": "Platform Telemetry and Operational Alerting Engine (S-8) fails at an hour faults are responded to (24x7).", + "about": "S-8", + "passes": true, + "path": "Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "id": "SC-change-S-8", + "scenario": "Platform Telemetry and Operational Alerting Engine (S-8) must be changed.", + "about": "S-8", + "passes": true, + "path": "Platform Automation Engineer makes the change under C-5." + }, + { + "id": "SC-fault-S-9", + "scenario": "Financial Governance and Budget Enforcement Monitor (S-9) fails at an hour faults are responded to (24x7).", + "about": "S-9", + "passes": true, + "path": "Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "id": "SC-change-S-9", + "scenario": "Financial Governance and Budget Enforcement Monitor (S-9) must be changed.", + "about": "S-9", + "passes": true, + "path": "Platform Automation Engineer makes the change under C-5." + }, + { + "id": "SC-condition-K-1", + "scenario": "Condition K-1 must be shown to hold: The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit.", + "about": "K-1", + "passes": true, + "path": "Check OC-1 on C-7 keeps it true." + }, + { + "id": "SC-condition-K-2", + "scenario": "Condition K-2 must be shown to hold: The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic.", + "about": "K-2", + "passes": true, + "path": "Check OC-2 on C-9 keeps it true." + }, + { + "id": "SC-condition-K-3", + "scenario": "Condition K-3 must be shown to hold: Data residency is guaranteed within the United Kingdom and European Union.", + "about": "K-3", + "passes": true, + "path": "Check OC-3 on C-7 keeps it true." + }, + { + "id": "SC-condition-K-4", + "scenario": "Condition K-4 must be shown to hold: Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks.", + "about": "K-4", + "passes": true, + "path": "Check OC-4 on C-16 keeps it true." + }, + { + "id": "SC-condition-K-5", + "scenario": "Condition K-5 must be shown to hold: Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.", + "about": "K-5", + "passes": true, + "path": "Check OC-5 on C-2 keeps it true." + }, + { + "id": "SC-condition-K-6", + "scenario": "Condition K-6 must be shown to hold: Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.", + "about": "K-6", + "passes": true, + "path": "Check OC-6 on C-9 keeps it true." + }, + { + "id": "SC-condition-K-7", + "scenario": "Condition K-7 must be shown to hold: Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour.", + "about": "K-7", + "passes": true, + "path": "Check OC-7 on C-8 keeps it true." + }, + { + "id": "SC-condition-K-8", + "scenario": "Condition K-8 must be shown to hold: Alert notifications trigger operational incident workflows for the 24/7 central platform operations team.", + "about": "K-8", + "passes": true, + "path": "Check OC-8 on C-2 keeps it true." + }, + { + "id": "SC-condition-K-9", + "scenario": "Condition K-9 must be shown to hold: Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off.", + "about": "K-9", + "passes": true, + "path": "Check OC-9 on C-14 keeps it true." + }, + { + "id": "SC-condition-K-10", + "scenario": "Condition K-10 must be shown to hold: Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing.", + "about": "K-10", + "passes": true, + "path": "Check OC-10 on C-10 keeps it true." + }, + { + "id": "SC-condition-K-11", + "scenario": "Condition K-11 must be shown to hold: Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores.", + "about": "K-11", + "passes": true, + "path": "Check OC-11 on C-6 keeps it true." + }, + { + "id": "SC-condition-K-12", + "scenario": "Condition K-12 must be shown to hold: Permissions are continuously rightsized based on IAM Recommender findings.", + "about": "K-12", + "passes": true, + "path": "Check OC-12 on C-11 keeps it true." + }, + { + "id": "SC-party-P-2", + "scenario": "Platform Service Provider cannot be reached.", + "about": "P-2", + "passes": true, + "path": "Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "id": "SC-party-P-3", + "scenario": "Google Cloud cannot be reached.", + "about": "P-3", + "passes": true, + "path": "Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "id": "SC-party-P-4", + "scenario": "Network Carrier cannot be reached.", + "about": "P-4", + "passes": true, + "path": "Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "id": "SC-party-P-5", + "scenario": "External Firewall Vendor cannot be reached.", + "about": "P-5", + "passes": true, + "path": "Every Capability it is answerable for has a Role in another Party to turn to." + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scope.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scope.json new file mode 100644 index 0000000..29cc7f1 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scope.json @@ -0,0 +1,402 @@ +{ + "kind": "platform_foundation", + "stated": { + "engagement": { + "kind": "platform_foundation" + }, + "intent": { + "statement": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control." + }, + "solution": { + "cloud_provider": "gcp", + "items": [ + { + "id": "S-1", + "name": "Resource Hierarchy and Environment Boundary Manager", + "kind": "governance", + "product": "Resource Manager", + "source_ref": "C-1", + "elements": [ + "E-1", + "E-2", + "E-3", + "E-4", + "E-5", + "E-6", + "E-7", + "E-8", + "E-9", + "E-10", + "E-34", + "E-35", + "E-36", + "E-37", + "E-38", + "E-50", + "E-51", + "E-55", + "E-60" + ] + }, + { + "id": "S-2", + "name": "Dedicated Private Hybrid Connectivity Gateway", + "kind": "network", + "product": "Cloud Interconnect", + "source_ref": "C-2", + "elements": [ + "E-11", + "E-12", + "E-13", + "E-14", + "E-56", + "E-57" + ] + }, + { + "id": "S-3", + "name": "Centralised Outbound Traffic Inspection and Egress Boundary", + "kind": "service", + "product": "Palo Alto Networks VM-Series", + "source_ref": "C-3", + "elements": [ + "E-15", + "E-16", + "E-17", + "E-18", + "E-19", + "E-20", + "E-47", + "E-48", + "E-53" + ] + }, + { + "id": "S-4", + "name": "Centralised Immutable Audit Event and Compliance Log Repository", + "kind": "storage", + "product": "Cloud Storage", + "source_ref": "C-4", + "elements": [ + "E-21", + "E-22", + "E-39" + ] + }, + { + "id": "S-5", + "name": "Automated Infrastructure Orchestration and Deployment Engine", + "kind": "service", + "product": "GitHub Actions", + "source_ref": "C-5", + "elements": [ + "E-42", + "E-43" + ] + }, + { + "id": "S-6", + "name": "Federated Identity and Role Governance Authority", + "kind": "identity", + "product": "IAM Workload Identity Federation", + "source_ref": "C-6", + "elements": [ + "E-23", + "E-24", + "E-25", + "E-26", + "E-27", + "E-45" + ] + }, + { + "id": "S-7", + "name": "Cryptographic Key Lifecycle and Encryption Controller", + "kind": "keys", + "product": "Cloud KMS", + "source_ref": "C-7", + "elements": [ + "E-28", + "E-29", + "E-30", + "E-31", + "E-49", + "E-58", + "E-59" + ] + }, + { + "id": "S-8", + "name": "Platform Telemetry and Operational Alerting Engine", + "kind": "observability", + "product": "Cloud Monitoring", + "source_ref": "C-8", + "elements": [ + "E-32", + "E-40", + "E-41", + "E-44", + "E-52" + ] + }, + { + "id": "S-9", + "name": "Financial Governance and Budget Enforcement Monitor", + "kind": "billing", + "product": "Cloud Billing", + "source_ref": "C-9", + "elements": [ + "E-33", + "E-46", + "E-54" + ] + } + ], + "conditions": [ + { + "id": "K-1", + "on": [ + "S-1", + "S-4", + "S-7" + ], + "condition": "The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit.", + "source_ref": "NFR-8" + }, + { + "id": "K-2", + "on": [ + "S-1" + ], + "condition": "The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic.", + "source_ref": "NFR-9" + }, + { + "id": "K-3", + "on": [ + "S-1" + ], + "condition": "Data residency is guaranteed within the United Kingdom and European Union.", + "source_ref": "NFR-9" + }, + { + "id": "K-4", + "on": [ + "S-5" + ], + "condition": "Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks.", + "source_ref": "NFR-10" + }, + { + "id": "K-5", + "on": [ + "S-8" + ], + "condition": "Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.", + "source_ref": "NFR-11" + }, + { + "id": "K-6", + "on": [ + "S-8" + ], + "condition": "Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.", + "source_ref": "NFR-11" + }, + { + "id": "K-7", + "on": [ + "S-8" + ], + "condition": "Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour.", + "source_ref": "NFR-11" + }, + { + "id": "K-8", + "on": [ + "S-8" + ], + "condition": "Alert notifications trigger operational incident workflows for the 24/7 central platform operations team.", + "source_ref": "NFR-12" + }, + { + "id": "K-9", + "on": [ + "S-9" + ], + "condition": "Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off.", + "source_ref": "NFR-13" + }, + { + "id": "K-10", + "on": [ + "S-2" + ], + "condition": "Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing.", + "source_ref": "NFR-14" + }, + { + "id": "K-11", + "on": [ + "S-8" + ], + "condition": "Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores.", + "source_ref": "NFR-15" + }, + { + "id": "K-12", + "on": [ + "S-6" + ], + "condition": "Permissions are continuously rightsized based on IAM Recommender findings.", + "source_ref": "DR-OP-3" + } + ], + "build": { + "duration_days": 36, + "customer_owned": [ + "Order and provision the Interconnect carrier circuits" + ] + } + }, + "operations": { + "support_hours": "24x7", + "availability_hours": "24x7" + }, + "sourcing": { + "model": "provider_operated", + "delivery_countries": [ + "GB", + "PL", + "IN" + ] + }, + "compliance": { + "frameworks": [ + "DORA", + "ISO27001", + "PCI_DSS_v4" + ] + }, + "commercial": { + "currency": "GBP" + } + }, + "binding": [ + "operations.support_hours", + "sourcing.model", + "sourcing.delivery_countries" + ], + "assumed": [ + { + "field": "intent.business_outcomes", + "value": [ + { + "target": "Foundation ready in under 10 weeks", + "id": "BO-1", + "outcome": "Deliver an enterprise-grade Google Cloud landing zone within 10 weeks to enable ingestion and operation of the core payments engine." + }, + { + "outcome": "Maintain continuous operational resilience and regulatory compliance with DORA and PCI-DSS v4.0 with platform availability of at least 99.9%.", + "target": ">=99.9% monthly availability and full regulatory audit alignment", + "id": "BO-2" + }, + { + "target": "Deterministic low-latency 10Gbps interconnect and 100% CMEK lifecycle coverage", + "id": "BO-3", + "outcome": "Ensure secure, low-latency hybrid transaction transport and zero-trust data protection using customer-managed encryption keys." + } + ], + "why": "Derived from the workflow request stating the need for an enterprise-grade landing zone ready in under 10 weeks to support the core payments engine, complying with DORA and PCI-DSS v4.0, with secure hybrid connectivity and customer-controlled encryption." + }, + { + "field": "intent.exclusions", + "value": [ + "Core payments engine application-level development, maintenance, and transaction processing logic", + "On-premises data centre infrastructure and systems beyond the Cloud Interconnect demarcation", + "Physical telecom carrier circuit ordering and provisioning (customer-owned build item)" + ], + "why": "Derived from engagement kind (platform foundation) and workflow request: application-level workloads, on-prem systems, and physical circuit provisioning are excluded from platform operations." + }, + { + "field": "operations.change_process", + "value": "Automated CI/CD pipelines via GitHub Actions with peer review and automated policy checks; standard operational changes deployed without a separate manual CAB.", + "why": "Derived from request stating GitHub Actions delivery pipelines and catalogue default guidance." + }, + { + "field": "organisation.existing_teams", + "value": [ + { + "party": "customer", + "name": "Central Platform Operations Team", + "hours": "24x7" + } + ], + "why": "Derived from condition K-8 noting alerting to the 24/7 central platform operations team, alongside provider-operated landing zone model." + }, + { + "field": "service_management.tooling", + "value": "Google Cloud operations suite (Cloud Monitoring and Cloud Logging) integrated with native alerting workflows and GitHub Actions.", + "why": "Catalogue default guidance leveraging Google Cloud operations suite and GitHub Actions specified in request." + }, + { + "field": "service_levels.priority_1_response_minutes", + "value": 30, + "why": "Standard catalogue default for 24x7 critical infrastructure response." + }, + { + "field": "service_levels.recovery_time_hours", + "value": 4, + "why": "Matches condition K-6 disaster recovery RTO requirement of 4 hours and catalogue default." + } + ], + "to_decide": [], + "to_ask": [ + { + "field": "metadata.client_name", + "prompt": "What is metadata.client_name?", + "severity": "LOW" + }, + { + "field": "metadata.author", + "prompt": "What is metadata.author?", + "severity": "LOW" + }, + { + "field": "metadata.revision", + "prompt": "What is metadata.revision?", + "severity": "LOW" + }, + { + "field": "intent.requirements", + "prompt": "Which requirements for operations has the customer stated outright?", + "severity": "LOW" + }, + { + "field": "solution.inherited_risks", + "prompt": "What is already known to be unverified or at risk in the solution?", + "severity": "LOW" + }, + { + "field": "operations.monitoring", + "prompt": "Which monitoring and alerting is already in place or required?", + "severity": "LOW" + }, + { + "field": "organisation.headcount_limit", + "prompt": "Is there a limit on the number of people operating this?", + "severity": "LOW" + }, + { + "field": "commercial.monthly_run_budget", + "prompt": "Is there a monthly budget for operating this, in that currency?", + "severity": "LOW" + }, + { + "field": "transition.go_live_date", + "prompt": "When must the solution be in operation?", + "severity": "LOW" + } + ], + "not_applicable": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scope.yaml new file mode 100644 index 0000000..8b76f3a --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/scope.yaml @@ -0,0 +1,414 @@ +schema_version: 1 +metadata: + customer_case: case-08-landing-zone + client_name: ask + author: ask + revision: ask + source: architecture:case-08-landing-zone-185426 + status: BASELINE_LOCKED +engagement: + kind: platform_foundation +intent: + statement: 'We are a UK-regulated payment platform expanding into Google Cloud. + We need an + + enterprise-grade landing zone ready in under 10 weeks to support our core payments + + engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have + + existing on-premises systems that require private, low-latency connectivity over + a + + 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and + + GitHub Actions for our delivery pipelines. Security requires that all egress + + traffic flows through centralised inspection appliances before touching the public + + internet, and customer data must be encrypted with keys we control.' + requirements: ask + business_outcomes: + - target: Foundation ready in under 10 weeks + id: BO-1 + outcome: Deliver an enterprise-grade Google Cloud landing zone within 10 weeks + to enable ingestion and operation of the core payments engine. + - outcome: Maintain continuous operational resilience and regulatory compliance + with DORA and PCI-DSS v4.0 with platform availability of at least 99.9%. + target: '>=99.9% monthly availability and full regulatory audit alignment' + id: BO-2 + - target: Deterministic low-latency 10Gbps interconnect and 100% CMEK lifecycle + coverage + id: BO-3 + outcome: Ensure secure, low-latency hybrid transaction transport and zero-trust + data protection using customer-managed encryption keys. + exclusions: + - Core payments engine application-level development, maintenance, and transaction + processing logic + - On-premises data centre infrastructure and systems beyond the Cloud Interconnect + demarcation + - Physical telecom carrier circuit ordering and provisioning (customer-owned build + item) +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Resource Hierarchy and Environment Boundary Manager + kind: governance + product: Resource Manager + source_ref: C-1 + elements: + - E-1 + - E-2 + - E-3 + - E-4 + - E-5 + - E-6 + - E-7 + - E-8 + - E-9 + - E-10 + - E-34 + - E-35 + - E-36 + - E-37 + - E-38 + - E-50 + - E-51 + - E-55 + - E-60 + - id: S-2 + name: Dedicated Private Hybrid Connectivity Gateway + kind: network + product: Cloud Interconnect + source_ref: C-2 + elements: + - E-11 + - E-12 + - E-13 + - E-14 + - E-56 + - E-57 + - id: S-3 + name: Centralised Outbound Traffic Inspection and Egress Boundary + kind: service + product: Palo Alto Networks VM-Series + source_ref: C-3 + elements: + - E-15 + - E-16 + - E-17 + - E-18 + - E-19 + - E-20 + - E-47 + - E-48 + - E-53 + - id: S-4 + name: Centralised Immutable Audit Event and Compliance Log Repository + kind: storage + product: Cloud Storage + source_ref: C-4 + elements: + - E-21 + - E-22 + - E-39 + - id: S-5 + name: Automated Infrastructure Orchestration and Deployment Engine + kind: service + product: GitHub Actions + source_ref: C-5 + elements: + - E-42 + - E-43 + - id: S-6 + name: Federated Identity and Role Governance Authority + kind: identity + product: IAM Workload Identity Federation + source_ref: C-6 + elements: + - E-23 + - E-24 + - E-25 + - E-26 + - E-27 + - E-45 + - id: S-7 + name: Cryptographic Key Lifecycle and Encryption Controller + kind: keys + product: Cloud KMS + source_ref: C-7 + elements: + - E-28 + - E-29 + - E-30 + - E-31 + - E-49 + - E-58 + - E-59 + - id: S-8 + name: Platform Telemetry and Operational Alerting Engine + kind: observability + product: Cloud Monitoring + source_ref: C-8 + elements: + - E-32 + - E-40 + - E-41 + - E-44 + - E-52 + - id: S-9 + name: Financial Governance and Budget Enforcement Monitor + kind: billing + product: Cloud Billing + source_ref: C-9 + elements: + - E-33 + - E-46 + - E-54 + conditions: + - id: K-1 + 'on': + - S-1 + - S-4 + - S-7 + condition: The landing zone configuration aligns with DORA operational resiliency + guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. + source_ref: NFR-8 + - id: K-2 + 'on': + - S-1 + condition: The secondary region in europe-west1 operates as a warm standby ready + to receive failover traffic. + source_ref: NFR-9 + - id: K-3 + 'on': + - S-1 + condition: Data residency is guaranteed within the United Kingdom and European + Union. + source_ref: NFR-9 + - id: K-4 + 'on': + - S-5 + condition: Platform foundation is fully provisioned and validated for core payments + engine workload ingestion within 10 weeks. + source_ref: NFR-10 + - id: K-5 + 'on': + - S-8 + condition: Platform foundation infrastructure maintains measured monthly availability + of at least 99.9%. + source_ref: NFR-11 + - id: K-6 + 'on': + - S-8 + condition: Disaster recovery activation achieves a recovery time objective (RTO) + of 4 hours. + source_ref: NFR-11 + - id: K-7 + 'on': + - S-8 + condition: Disaster recovery processes achieve a recovery point objective (RPO) + of 1 hour. + source_ref: NFR-11 + - id: K-8 + 'on': + - S-8 + condition: Alert notifications trigger operational incident workflows for the + 24/7 central platform operations team. + source_ref: NFR-12 + - id: K-9 + 'on': + - S-9 + condition: Monthly cloud foundation expenditure does not exceed the target allocation + without operational sign-off. + source_ref: NFR-13 + - id: K-10 + 'on': + - S-2 + condition: Hybrid connection provides deterministic low latency suitable for real-time + payment transaction processing. + source_ref: NFR-14 + - id: K-11 + 'on': + - S-8 + condition: Deployments optimize carbon-free energy utilization based on Google + Cloud regional CFE scores. + source_ref: NFR-15 + - id: K-12 + 'on': + - S-6 + condition: Permissions are continuously rightsized based on IAM Recommender findings. + source_ref: DR-OP-3 + build: + duration_days: 36 + customer_owned: + - Order and provision the Interconnect carrier circuits + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Automated CI/CD pipelines via GitHub Actions with peer review and + automated policy checks; standard operational changes deployed without a separate + manual CAB. + monitoring: ask +sourcing: + model: provider_operated + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: + - party: customer + name: Central Platform Operations Team + hours: 24x7 + headcount_limit: ask +service_management: + tooling: Google Cloud operations suite (Cloud Monitoring and Cloud Logging) integrated + with native alerting workflows and GitHub Actions. +service_levels: + priority_1_response_minutes: 30 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA + - ISO27001 + - PCI_DSS_v4 +commercial: + currency: GBP + monthly_run_budget: ask +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.business_outcomes + statement: 'Assumed intent.business_outcomes is [{''target'': ''Foundation ready + in under 10 weeks'', ''id'': ''BO-1'', ''outcome'': ''Deliver an enterprise-grade + Google Cloud landing zone within 10 weeks to enable ingestion and operation of + the core payments engine.''}, {''outcome'': ''Maintain continuous operational + resilience and regulatory compliance with DORA and PCI-DSS v4.0 with platform + availability of at least 99.9%.'', ''target'': ''>=99.9% monthly availability + and full regulatory audit alignment'', ''id'': ''BO-2''}, {''target'': ''Deterministic + low-latency 10Gbps interconnect and 100% CMEK lifecycle coverage'', ''id'': ''BO-3'', + ''outcome'': ''Ensure secure, low-latency hybrid transaction transport and zero-trust + data protection using customer-managed encryption keys.''}]. Derived from the + workflow request stating the need for an enterprise-grade landing zone ready in + under 10 weeks to support the core payments engine, complying with DORA and PCI-DSS + v4.0, with secure hybrid connectivity and customer-controlled encryption.' + impact_if_invalid: Changes what is measured, and what each measure is for. +- id: ASSUMP-02 + field: intent.exclusions + statement: 'Assumed intent.exclusions is [''Core payments engine application-level + development, maintenance, and transaction processing logic'', ''On-premises data + centre infrastructure and systems beyond the Cloud Interconnect demarcation'', + ''Physical telecom carrier circuit ordering and provisioning (customer-owned build + item)'']. Derived from engagement kind (platform foundation) and workflow request: + application-level workloads, on-prem systems, and physical circuit provisioning + are excluded from platform operations.' + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-03 + field: operations.change_process + statement: Assumed operations.change_process is Automated CI/CD pipelines via GitHub + Actions with peer review and automated policy checks; standard operational changes + deployed without a separate manual CAB.. Derived from request stating GitHub Actions + delivery pipelines and catalogue default guidance. + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-04 + field: organisation.existing_teams + statement: 'Assumed organisation.existing_teams is [{''party'': ''customer'', ''name'': + ''Central Platform Operations Team'', ''hours'': ''24x7''}]. Derived from condition + K-8 noting alerting to the 24/7 central platform operations team, alongside provider-operated + landing zone model.' + impact_if_invalid: Changes which Roles are new and which are filled. +- id: ASSUMP-05 + field: service_management.tooling + statement: Assumed service_management.tooling is Google Cloud operations suite (Cloud + Monitoring and Cloud Logging) integrated with native alerting workflows and GitHub + Actions.. Catalogue default guidance leveraging Google Cloud operations suite + and GitHub Actions specified in request. + impact_if_invalid: Changes tooling and the processes written for it. +- id: ASSUMP-06 + field: service_levels.priority_1_response_minutes + statement: Assumed service_levels.priority_1_response_minutes is 30. Standard catalogue + default for 24x7 critical infrastructure response. + impact_if_invalid: Changes on-call arrangements and staffing. +- id: ASSUMP-07 + field: service_levels.recovery_time_hours + statement: Assumed service_levels.recovery_time_hours is 4. Matches condition K-6 + disaster recovery RTO requirement of 4 hours and catalogue default. + impact_if_invalid: Changes continuity arrangements. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: intent.requirements + prompt: Which requirements for operations has the customer stated outright? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: commercial.monthly_run_budget + prompt: Is there a monthly budget for operating this, in that currency? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: digested + commercial.monthly_run_budget: unknown + compliance.frameworks: digested + engagement.kind: digested + intent.business_outcomes: assumed + intent.exclusions: assumed + intent.requirements: unknown + intent.statement: digested + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: digested + metadata.revision: unknown + metadata.source: digested + operations.availability_hours: digested + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: digested + organisation.existing_teams: assumed + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: assumed + service_levels.recovery_time_hours: assumed + service_management.tooling: assumed + solution.build.customer_owned: digested + solution.build.duration_days: digested + solution.cloud_provider: digested + solution.conditions: digested + solution.inherited_risks: unknown + solution.items: digested + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 3 + answered: 0 + assumed: 7 + unknown: 10 + not_applicable: 0 + digested: 13 diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/staffing.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/staffing.json new file mode 100644 index 0000000..3d59087 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/staffing.json @@ -0,0 +1,144 @@ +{ + "currency": "GBP", + "rate_card": { + "name": "rate-card", + "version": "draft-2026-10-13", + "draft": true + }, + "conversion": { + "from": "USD", + "rate": 0.79, + "as_of": "2026-10-13" + }, + "roles": [ + { + "role": "R-1", + "name": "Platform Operations Engineer", + "party": "P-2", + "party_name": "Platform Service Provider", + "country": "IN", + "level": "engineer", + "hours": "24x7", + "effort_hours_per_month": 30.0, + "people": 5.0, + "cost_per_month": 17034 + }, + { + "role": "R-2", + "name": "Platform Reliability Engineer", + "party": "P-2", + "party_name": "Platform Service Provider", + "country": "PL", + "level": "senior", + "hours": "24x7", + "effort_hours_per_month": 50.0, + "people": 5.0, + "cost_per_month": 41640 + }, + { + "role": "R-3", + "name": "Security Operations Analyst", + "party": "P-2", + "party_name": "Platform Service Provider", + "country": "PL", + "level": "senior", + "hours": "24x7", + "effort_hours_per_month": 75.0, + "people": 5.0, + "cost_per_month": 41640 + }, + { + "role": "R-4", + "name": "Platform Automation Engineer", + "party": "P-2", + "party_name": "Platform Service Provider", + "country": "PL", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 177.5, + "people": 1.2, + "cost_per_month": 8063 + }, + { + "role": "R-5", + "name": "Platform Engineering Lead", + "party": "P-2", + "party_name": "Platform Service Provider", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 25.0, + "people": 0.2, + "cost_per_month": 2765 + }, + { + "role": "R-6", + "name": "Compliance & Governance Analyst", + "party": "P-2", + "party_name": "Platform Service Provider", + "country": "GB", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 62.5, + "people": 0.5, + "cost_per_month": 5267 + }, + { + "role": "R-7", + "name": "Customer Platform Owner", + "party": "P-1", + "party_name": "Customer", + "country": "GB", + "level": "lead", + "hours": "business", + "effort_hours_per_month": 0.0, + "people": 0.0, + "cost_per_month": null + }, + { + "role": "R-8", + "name": "Customer Operations Liaison", + "party": "P-1", + "party_name": "Customer", + "country": "GB", + "level": "senior", + "hours": "business", + "effort_hours_per_month": 0.0, + "people": 0.0, + "cost_per_month": null + } + ], + "people": 16.9, + "cost_per_month": 116409, + "by_party": { + "P-2": 116409 + }, + "by_country": { + "IN": 17034, + "PL": 91343, + "GB": 8032 + }, + "by_capability": { + "C-2": 17034, + "C-1": 33312, + "C-9": 8328, + "C-11": 11104, + "C-12": 13880, + "C-13": 16656, + "C-3": 909, + "C-4": 909, + "C-5": 973, + "C-6": 1136, + "C-7": 681, + "C-8": 681, + "C-10": 681, + "C-17": 1363, + "C-18": 1136, + "C-19": 780, + "C-16": 2212, + "C-14": 1264, + "C-15": 843, + "C-20": 843, + "C-21": 1685 + } +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/target-operating-model.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/target-operating-model.json new file mode 100644 index 0000000..2270f53 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/target-operating-model.json @@ -0,0 +1,722 @@ +{ + "parties": [ + { + "id": "P-1", + "kind": "customer", + "name": "Customer" + }, + { + "id": "P-2", + "kind": "provider", + "name": "Platform Service Provider" + }, + { + "id": "P-3", + "kind": "cloud_provider", + "name": "Google Cloud" + }, + { + "id": "P-4", + "kind": "supplier", + "name": "Network Carrier" + }, + { + "name": "External Firewall Vendor", + "kind": "supplier", + "id": "P-5" + } + ], + "teams": [ + { + "type": "platform", + "purpose": "Builds, maintains, and reliably operates the enterprise Google Cloud landing zone.", + "party": "P-2", + "name": "Cloud Platform Foundation Team", + "id": "T-1" + }, + { + "id": "T-2", + "purpose": "Provides 24x7 security monitoring, egress inspection analysis, and vulnerability management.", + "name": "Platform Security Team", + "party": "P-2", + "type": "complicated_subsystem" + }, + { + "purpose": "Receives incident workflows and serves as the internal operations escalation point.", + "type": "stream_aligned", + "party": "P-1", + "name": "Customer Central Platform Operations Team", + "id": "T-3" + }, + { + "purpose": "Provides business context and sets the strategic direction for the platform.", + "name": "Customer Platform Owner Team", + "id": "T-4", + "type": "stream_aligned", + "party": "P-1" + }, + { + "type": "enabling", + "id": "T-5", + "purpose": "Ensures continuous regulatory compliance with DORA, PCI-DSS v4.0, and ISO27001.", + "party": "P-2", + "name": "Platform Governance Team" + } + ], + "roles": [ + { + "level": "engineer", + "hours": "24x7", + "party": "P-2", + "name": "Platform Operations Engineer", + "team": "T-1", + "id": "R-1", + "country": "IN" + }, + { + "name": "Platform Reliability Engineer", + "team": "T-1", + "country": "PL", + "party": "P-2", + "hours": "24x7", + "id": "R-2", + "level": "senior" + }, + { + "name": "Security Operations Analyst", + "id": "R-3", + "level": "senior", + "team": "T-2", + "hours": "24x7", + "country": "PL", + "party": "P-2" + }, + { + "id": "R-4", + "name": "Platform Automation Engineer", + "team": "T-1", + "hours": "business", + "party": "P-2", + "country": "PL", + "level": "senior" + }, + { + "name": "Platform Engineering Lead", + "id": "R-5", + "team": "T-1", + "level": "lead", + "hours": "business", + "party": "P-2", + "country": "GB" + }, + { + "team": "T-5", + "hours": "business", + "party": "P-2", + "country": "GB", + "id": "R-6", + "name": "Compliance & Governance Analyst", + "level": "senior" + }, + { + "hours": "business", + "party": "P-1", + "level": "lead", + "name": "Customer Platform Owner", + "id": "R-7", + "team": "T-4", + "country": "GB" + }, + { + "name": "Customer Operations Liaison", + "team": "T-3", + "hours": "24x7", + "country": "GB", + "party": "P-1", + "level": "senior", + "id": "R-8" + } + ], + "interactions": [ + { + "mode": "x_as_a_service", + "to": "T-1", + "from": "T-3", + "purpose": "Customer Central Platform Operations consumes incident workflows and alert notifications from the Provider Cloud Platform Foundation.", + "id": "I-1" + }, + { + "id": "I-2", + "mode": "collaboration", + "purpose": "Jointly establish automated security response runbooks and egress inspection rules for PCI-DSS compliance.", + "from": "T-1", + "until": "PCI-DSS baseline controls are fully automated and verified by external audit, after which they revert to x_as_a_service.", + "to": "T-2" + }, + { + "from": "T-5", + "id": "I-3", + "mode": "facilitating", + "purpose": "Assisting the platform team in integrating regulatory evidence collection into the automated deployment engine.", + "to": "T-1" + } + ], + "measures": [ + { + "id": "M-1", + "target": "Under 1 hour", + "measured_by": "C-19", + "name": "Deployment Lead Time", + "framework": "dora-metrics", + "layer": "outcome", + "outcome": "BO-1" + }, + { + "id": "M-2", + "framework": "devex-framework", + "target": "Low friction reported in monthly surveys", + "name": "Developer Friction", + "layer": "leading", + "outcome": "BO-1", + "measured_by": "C-19" + }, + { + "measured_by": "C-19", + "framework": "space-framework", + "id": "M-3", + "name": "Team Satisfaction", + "layer": "guardrail", + "outcome": "BO-1", + "target": "Positive quarterly trend" + }, + { + "measured_by": "C-15", + "layer": "service_level", + "outcome": "BO-2", + "id": "M-4", + "target": ">=99.9%", + "name": "Platform Availability" + }, + { + "framework": "dora-metrics", + "outcome": "BO-2", + "measured_by": "C-15", + "name": "Time to Restore Service", + "id": "M-5", + "layer": "outcome", + "target": "Under 4 hours" + }, + { + "id": "M-6", + "layer": "service_level", + "target": "Deterministic low latency (<5ms)", + "name": "Interconnect Latency", + "outcome": "BO-3", + "measured_by": "C-10" + }, + { + "outcome": "BO-3", + "layer": "guardrail", + "target": "100% coverage", + "id": "M-7", + "name": "CMEK Lifecycle Coverage", + "measured_by": "C-21" + } + ], + "responsibilities": [ + { + "id": "RS-1", + "kind": "accountable", + "role": "R-5", + "capability": "C-1" + }, + { + "capability": "C-1", + "role": "R-2", + "id": "RS-2", + "kind": "responsible" + }, + { + "role": "R-8", + "capability": "C-1", + "kind": "consulted", + "id": "RS-3" + }, + { + "kind": "accountable", + "id": "RS-4", + "role": "R-5", + "capability": "C-2" + }, + { + "kind": "responsible", + "id": "RS-5", + "capability": "C-2", + "role": "R-1" + }, + { + "kind": "accountable", + "id": "RS-6", + "capability": "C-3", + "role": "R-5" + }, + { + "capability": "C-3", + "role": "R-4", + "kind": "responsible", + "id": "RS-7" + }, + { + "kind": "accountable", + "id": "RS-8", + "capability": "C-4", + "role": "R-5" + }, + { + "role": "R-4", + "kind": "responsible", + "id": "RS-9", + "capability": "C-4" + }, + { + "role": "R-5", + "capability": "C-5", + "kind": "accountable", + "id": "RS-10" + }, + { + "role": "R-4", + "kind": "responsible", + "capability": "C-5", + "id": "RS-11" + }, + { + "kind": "responsible", + "role": "R-6", + "id": "RS-12", + "capability": "C-5" + }, + { + "capability": "C-6", + "kind": "accountable", + "id": "RS-13", + "role": "R-5" + }, + { + "id": "RS-14", + "kind": "responsible", + "role": "R-4", + "capability": "C-6" + }, + { + "role": "R-5", + "id": "RS-15", + "capability": "C-7", + "kind": "accountable" + }, + { + "capability": "C-7", + "role": "R-4", + "id": "RS-16", + "kind": "responsible" + }, + { + "id": "RS-17", + "capability": "C-8", + "role": "R-5", + "kind": "accountable" + }, + { + "kind": "responsible", + "id": "RS-18", + "role": "R-4", + "capability": "C-8" + }, + { + "kind": "accountable", + "capability": "C-9", + "id": "RS-19", + "role": "R-5" + }, + { + "role": "R-2", + "id": "RS-20", + "kind": "responsible", + "capability": "C-9" + }, + { + "role": "R-5", + "capability": "C-10", + "kind": "accountable", + "id": "RS-21" + }, + { + "capability": "C-10", + "role": "R-4", + "kind": "responsible", + "id": "RS-22" + }, + { + "capability": "C-11", + "kind": "accountable", + "role": "R-3", + "id": "RS-23" + }, + { + "capability": "C-11", + "kind": "responsible", + "id": "RS-24", + "role": "R-3" + }, + { + "kind": "accountable", + "id": "RS-25", + "role": "R-3", + "capability": "C-12" + }, + { + "id": "RS-26", + "role": "R-3", + "capability": "C-12", + "kind": "responsible" + }, + { + "id": "RS-27", + "role": "R-3", + "kind": "accountable", + "capability": "C-13" + }, + { + "kind": "responsible", + "role": "R-3", + "capability": "C-13", + "id": "RS-28" + }, + { + "kind": "accountable", + "capability": "C-14", + "role": "R-5", + "id": "RS-29" + }, + { + "id": "RS-30", + "kind": "responsible", + "capability": "C-14", + "role": "R-6" + }, + { + "capability": "C-15", + "role": "R-5", + "kind": "accountable", + "id": "RS-31" + }, + { + "role": "R-6", + "id": "RS-32", + "capability": "C-15", + "kind": "responsible" + }, + { + "id": "RS-33", + "capability": "C-16", + "role": "R-5", + "kind": "accountable" + }, + { + "capability": "C-16", + "id": "RS-34", + "kind": "responsible", + "role": "R-5" + }, + { + "role": "R-7", + "capability": "C-16", + "kind": "consulted", + "id": "RS-35" + }, + { + "id": "RS-36", + "kind": "accountable", + "capability": "C-17", + "role": "R-5" + }, + { + "capability": "C-17", + "role": "R-4", + "id": "RS-37", + "kind": "responsible" + }, + { + "capability": "C-18", + "role": "R-5", + "id": "RS-38", + "kind": "accountable" + }, + { + "id": "RS-39", + "capability": "C-18", + "kind": "responsible", + "role": "R-4" + }, + { + "kind": "accountable", + "capability": "C-19", + "role": "R-5", + "id": "RS-40" + }, + { + "kind": "responsible", + "capability": "C-19", + "id": "RS-41", + "role": "R-4" + }, + { + "id": "RS-42", + "role": "R-5", + "capability": "C-19", + "kind": "responsible" + }, + { + "id": "RS-43", + "kind": "accountable", + "capability": "C-20", + "role": "R-5" + }, + { + "id": "RS-44", + "role": "R-6", + "capability": "C-20", + "kind": "responsible" + }, + { + "capability": "C-21", + "role": "R-6", + "id": "RS-45", + "kind": "accountable" + }, + { + "capability": "C-21", + "id": "RS-46", + "role": "R-6", + "kind": "responsible" + }, + { + "kind": "consulted", + "id": "RS-47", + "capability": "C-21", + "role": "R-7" + }, + { + "id": "RS-48", + "role": "R-8", + "kind": "consulted", + "capability": "C-2" + }, + { + "id": "RS-49", + "role": "R-8", + "kind": "consulted", + "capability": "C-3" + }, + { + "capability": "C-4", + "role": "R-8", + "kind": "consulted", + "id": "RS-50" + }, + { + "id": "RS-51", + "capability": "C-5", + "role": "R-8", + "kind": "consulted" + }, + { + "role": "R-8", + "kind": "consulted", + "capability": "C-6", + "id": "RS-52" + }, + { + "id": "RS-53", + "role": "R-8", + "kind": "consulted", + "capability": "C-7" + }, + { + "capability": "C-8", + "kind": "consulted", + "role": "R-8", + "id": "RS-54" + }, + { + "id": "RS-55", + "capability": "C-9", + "kind": "consulted", + "role": "R-8" + }, + { + "role": "R-8", + "kind": "consulted", + "id": "RS-56", + "capability": "C-10" + }, + { + "role": "R-8", + "kind": "consulted", + "id": "RS-57", + "capability": "C-11" + }, + { + "role": "R-8", + "kind": "consulted", + "capability": "C-12", + "id": "RS-58" + }, + { + "id": "RS-59", + "role": "R-8", + "kind": "consulted", + "capability": "C-13" + }, + { + "id": "RS-60", + "role": "R-8", + "kind": "consulted", + "capability": "C-14" + }, + { + "role": "R-8", + "kind": "consulted", + "id": "RS-61", + "capability": "C-15" + }, + { + "id": "RS-62", + "capability": "C-17", + "kind": "consulted", + "role": "R-8" + }, + { + "capability": "C-18", + "kind": "consulted", + "role": "R-8", + "id": "RS-63" + }, + { + "capability": "C-19", + "kind": "consulted", + "role": "R-8", + "id": "RS-64" + }, + { + "kind": "consulted", + "role": "R-8", + "capability": "C-20", + "id": "RS-65" + } + ], + "checks": [ + { + "on": "C-7", + "condition": "The landing zone configurations are continuously enforced to align with DORA, PCI-DSS v4.0, and ISO27001 standard baselines.", + "source": "K-1", + "id": "OC-1" + }, + { + "on": "C-9", + "id": "OC-2", + "source": "K-2", + "condition": "The europe-west1 secondary region is continuously maintained as an active warm standby." + }, + { + "source": "K-3", + "id": "OC-3", + "condition": "Configuration enforces that all data residency, audit logs, and keys remain strictly within the UK and EU.", + "on": "C-7" + }, + { + "on": "C-16", + "id": "OC-4", + "source": "K-4", + "condition": "The foundation platform is verified as fully provisioned and ready for core payments engine workload ingestion within 10 weeks." + }, + { + "on": "C-2", + "condition": "Continuous telemetry verifies that the platform foundation infrastructure maintains at least 99.9% monthly availability.", + "source": "K-5", + "id": "OC-5" + }, + { + "source": "K-6", + "on": "C-9", + "id": "OC-6", + "condition": "Disaster recovery failover tests verify that the 4-hour RTO can be met." + }, + { + "condition": "Backup and restore tests verify that the 1-hour RPO for compliance logs and KMS keys is met.", + "on": "C-8", + "source": "K-7", + "id": "OC-7" + }, + { + "id": "OC-8", + "source": "K-8", + "condition": "Alert notifications from platform telemetry automatically trigger incident workflows for the 24x7 central platform operations team without human delay.", + "on": "C-2" + }, + { + "source": "K-9", + "id": "OC-9", + "condition": "Cloud consumption expenditure is tracked against budgets, and automation enforces hard limits that prevent overages without operational sign-off.", + "on": "C-14" + }, + { + "id": "OC-10", + "on": "C-10", + "source": "K-10", + "condition": "The dedicated hybrid interconnect bandwidth and latency are monitored to ensure deterministic low latency for real-time payments." + }, + { + "on": "C-6", + "condition": "Workload placements in the deployment pipeline are optimized for carbon-free energy utilization.", + "id": "OC-11", + "source": "K-11" + }, + { + "condition": "Access permissions are continuously audited and rightsized based on IAM recommender alerts.", + "on": "C-11", + "source": "K-12", + "id": "OC-12" + }, + { + "id": "OC-13", + "on": "C-1", + "source": "OR-1", + "condition": "A Priority 1 fault on the platform foundation is acknowledged within 30 minutes, at any time of day or night." + }, + { + "on": "C-21", + "source": "OR-14", + "id": "OC-16", + "condition": "Evidence of continuous alignment with DORA, PCI-DSS v4.0, and ISO27001 is gathered for external audit." + }, + { + "id": "OC-17", + "source": "OR-15", + "condition": "The UK and EU data residency guarantee is evidenced for all storage, audit logs, and keys.", + "on": "C-21" + }, + { + "on": "R-5", + "condition": "The platform foundation is led and managed by a provider-operated delivery model.", + "source": "OR-12", + "id": "OC-14" + }, + { + "id": "OC-15", + "source": "OR-13", + "condition": "Operational management is performed exclusively by staff located in the UK, Poland, or India.", + "on": "R-2" + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/verdicts.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/verdicts.json new file mode 100644 index 0000000..2d8bc87 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3/verdicts.json @@ -0,0 +1,264 @@ +{ + "dry_run": { + "verdict": "pass", + "evidence": "34 of 34 scenario walk-throughs pass." + }, + "checks": [ + { + "verdict": "pass", + "evidence": "Capability C-7 provides configuration management, enforcing state definition parity and auditing configurations, and supports ensuring compliance standards like DORA, PCI-DSS v4.0, and ISO27001 are continuously enforced. Role R-4 is responsible for it.", + "check": "OC-1" + }, + { + "check": "OC-2", + "evidence": "Capability C-9 handles service continuity, which includes planning and periodic testing of regional failovers, ensuring the secondary region is maintained as an active warm standby. Role R-2 is responsible for it.", + "verdict": "pass" + }, + { + "evidence": "Capability C-7 enforces configuration state parity and audits configurations, satisfying the condition that data residency, logs, and keys remain in the UK and EU.", + "verdict": "pass", + "check": "OC-3" + }, + { + "verdict": "pass", + "check": "OC-4", + "evidence": "Capability C-16 prioritises features and technical debt to ensure the platform foundation supports the payments engine developers effectively, meeting the 10-week provisioning timeframe. R-5 is responsible." + }, + { + "verdict": "pass", + "evidence": "Capability C-2 provides continuous monitoring and alerting 24x7, maintaining telemetry pipelines to verify the infrastructure maintains 99.9% availability.", + "check": "OC-5" + }, + { + "verdict": "pass", + "check": "OC-6", + "evidence": "Capability C-9 handles disaster recovery failover tests to validate readiness and timing to ensure the 4-hour RTO can be met." + }, + { + "verdict": "pass", + "check": "OC-7", + "evidence": "Capability C-8 rehearses restoration processes for Cloud Storage and KMS to verify the 1-hour RPO is met." + }, + { + "verdict": "pass", + "evidence": "Capability C-2 configures and maintains automated alerting rules that trigger workflows to the 24x7 customer central platform operations team (R-8 is consulted).", + "check": "OC-8" + }, + { + "evidence": "Capability C-14 designs and operates automation to enforce hard financial limits and execution stops, tracking cloud consumption against budgets.", + "check": "OC-9", + "verdict": "pass" + }, + { + "check": "OC-10", + "verdict": "pass", + "evidence": "Capability C-10 monitors and optimises the dedicated hybrid interconnect bandwidth to guarantee deterministic low latency for real-time payments." + }, + { + "verdict": "pass", + "evidence": "Capability C-6 operates the deployment pipelines to ensure timely deployments and carbon-optimised workload placement.", + "check": "OC-11" + }, + { + "check": "OC-12", + "evidence": "Capability C-11 continuously applies automated IAM rightsizing recommendations and manages access from approved countries.", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "Capability C-1 provides 24x7 incident management and 30-minute response capability for Priority 1 faults.", + "check": "OC-13" + }, + { + "evidence": "Role R-5 (Platform Engineering Lead) sits within the Provider (P-2) party, managing and directing the platform strategy and operations.", + "verdict": "pass", + "check": "OC-14" + }, + { + "verdict": "pass", + "check": "OC-15", + "evidence": "All provider operational staff (R-1, R-2, R-3, R-4, R-5, R-6) are based exclusively in the UK, Poland, or India, fulfilling the sourcing requirement." + }, + { + "evidence": "Capability C-21 gathers and preserves documented audit evidence to prove compliance with DORA, PCI-DSS v4.0, and ISO27001 to external regulators.", + "check": "OC-16", + "verdict": "pass" + }, + { + "check": "OC-17", + "evidence": "Capability C-21 captures, preserves, and provides documented audit evidence to external regulators, which includes proving data residency constraints.", + "verdict": "pass" + }, + { + "check": "SC-fault-S-1", + "verdict": "pass", + "evidence": "Resource Hierarchy and Environment Boundary Manager (S-1) fails at an hour faults are responded to (24x7). Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "check": "SC-change-S-1", + "verdict": "pass", + "evidence": "Resource Hierarchy and Environment Boundary Manager (S-1) must be changed. Platform Automation Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-2", + "verdict": "pass", + "evidence": "Dedicated Private Hybrid Connectivity Gateway (S-2) fails at an hour faults are responded to (24x7). Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "check": "SC-change-S-2", + "verdict": "pass", + "evidence": "Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed. Platform Automation Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-3", + "verdict": "pass", + "evidence": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) fails at an hour faults are responded to (24x7). Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "check": "SC-change-S-3", + "verdict": "pass", + "evidence": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed. Platform Automation Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-4", + "verdict": "pass", + "evidence": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) fails at an hour faults are responded to (24x7). Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "check": "SC-change-S-4", + "verdict": "pass", + "evidence": "Centralised Immutable Audit Event and Compliance Log Repository (S-4) must be changed. Platform Automation Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-5", + "verdict": "pass", + "evidence": "Automated Infrastructure Orchestration and Deployment Engine (S-5) fails at an hour faults are responded to (24x7). Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "check": "SC-change-S-5", + "verdict": "pass", + "evidence": "Automated Infrastructure Orchestration and Deployment Engine (S-5) must be changed. Platform Automation Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-6", + "verdict": "pass", + "evidence": "Federated Identity and Role Governance Authority (S-6) fails at an hour faults are responded to (24x7). Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "check": "SC-change-S-6", + "verdict": "pass", + "evidence": "Federated Identity and Role Governance Authority (S-6) must be changed. Platform Automation Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-7", + "verdict": "pass", + "evidence": "Cryptographic Key Lifecycle and Encryption Controller (S-7) fails at an hour faults are responded to (24x7). Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "check": "SC-change-S-7", + "verdict": "pass", + "evidence": "Cryptographic Key Lifecycle and Encryption Controller (S-7) must be changed. Platform Automation Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-8", + "verdict": "pass", + "evidence": "Platform Telemetry and Operational Alerting Engine (S-8) fails at an hour faults are responded to (24x7). Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "check": "SC-change-S-8", + "verdict": "pass", + "evidence": "Platform Telemetry and Operational Alerting Engine (S-8) must be changed. Platform Automation Engineer makes the change under C-5." + }, + { + "check": "SC-fault-S-9", + "verdict": "pass", + "evidence": "Financial Governance and Budget Enforcement Monitor (S-9) fails at an hour faults are responded to (24x7). Platform Reliability Engineer responds under C-1, and Platform Engineering Lead is answerable." + }, + { + "check": "SC-change-S-9", + "verdict": "pass", + "evidence": "Financial Governance and Budget Enforcement Monitor (S-9) must be changed. Platform Automation Engineer makes the change under C-5." + }, + { + "check": "SC-condition-K-1", + "verdict": "pass", + "evidence": "Condition K-1 must be shown to hold: The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. Check OC-1 on C-7 keeps it true." + }, + { + "check": "SC-condition-K-2", + "verdict": "pass", + "evidence": "Condition K-2 must be shown to hold: The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic. Check OC-2 on C-9 keeps it true." + }, + { + "check": "SC-condition-K-3", + "verdict": "pass", + "evidence": "Condition K-3 must be shown to hold: Data residency is guaranteed within the United Kingdom and European Union. Check OC-3 on C-7 keeps it true." + }, + { + "check": "SC-condition-K-4", + "verdict": "pass", + "evidence": "Condition K-4 must be shown to hold: Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks. Check OC-4 on C-16 keeps it true." + }, + { + "check": "SC-condition-K-5", + "verdict": "pass", + "evidence": "Condition K-5 must be shown to hold: Platform foundation infrastructure maintains measured monthly availability of at least 99.9%. Check OC-5 on C-2 keeps it true." + }, + { + "check": "SC-condition-K-6", + "verdict": "pass", + "evidence": "Condition K-6 must be shown to hold: Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours. Check OC-6 on C-9 keeps it true." + }, + { + "check": "SC-condition-K-7", + "verdict": "pass", + "evidence": "Condition K-7 must be shown to hold: Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour. Check OC-7 on C-8 keeps it true." + }, + { + "check": "SC-condition-K-8", + "verdict": "pass", + "evidence": "Condition K-8 must be shown to hold: Alert notifications trigger operational incident workflows for the 24/7 central platform operations team. Check OC-8 on C-2 keeps it true." + }, + { + "check": "SC-condition-K-9", + "verdict": "pass", + "evidence": "Condition K-9 must be shown to hold: Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off. Check OC-9 on C-14 keeps it true." + }, + { + "check": "SC-condition-K-10", + "verdict": "pass", + "evidence": "Condition K-10 must be shown to hold: Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing. Check OC-10 on C-10 keeps it true." + }, + { + "check": "SC-condition-K-11", + "verdict": "pass", + "evidence": "Condition K-11 must be shown to hold: Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores. Check OC-11 on C-6 keeps it true." + }, + { + "check": "SC-condition-K-12", + "verdict": "pass", + "evidence": "Condition K-12 must be shown to hold: Permissions are continuously rightsized based on IAM Recommender findings. Check OC-12 on C-11 keeps it true." + }, + { + "check": "SC-party-P-2", + "verdict": "pass", + "evidence": "Platform Service Provider cannot be reached. Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "check": "SC-party-P-3", + "verdict": "pass", + "evidence": "Google Cloud cannot be reached. Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "check": "SC-party-P-4", + "verdict": "pass", + "evidence": "Network Carrier cannot be reached. Every Capability it is answerable for has a Role in another Party to turn to." + }, + { + "check": "SC-party-P-5", + "verdict": "pass", + "evidence": "External Firewall Vendor cannot be reached. Every Capability it is answerable for has a Role in another Party to turn to." + } + ], + "proposed_additions": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.md b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.md new file mode 100644 index 0000000..1750c44 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.md @@ -0,0 +1,15 @@ +# Intent 1: case-08-landing-zone + +- Recorded: 2026-10-21T21:30:39+00:00 +- Execution: `case-02-from-architecture-r1` + +## What the customer wants + +We are a UK-regulated payment platform expanding into Google Cloud. We need an +enterprise-grade landing zone ready in under 10 weeks to support our core payments +engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have +existing on-premises systems that require private, low-latency connectivity over a +10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and +GitHub Actions for our delivery pipelines. Security requires that all egress +traffic flows through centralised inspection appliances before touching the public +internet, and customer data must be encrypted with keys we control. diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.scope.completed.yaml b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.scope.completed.yaml new file mode 100644 index 0000000..aa6d2e4 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.scope.completed.yaml @@ -0,0 +1,406 @@ +schema_version: 1 +metadata: + customer_case: case-08-landing-zone + client_name: ask + author: ask + revision: ask + source: architecture:case-08-landing-zone-185426 + status: APPROVED +engagement: + kind: platform_foundation +intent: + statement: 'We are a UK-regulated payment platform expanding into Google Cloud. + We need an + + enterprise-grade landing zone ready in under 10 weeks to support our core payments + + engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have + + existing on-premises systems that require private, low-latency connectivity over + a + + 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and + + GitHub Actions for our delivery pipelines. Security requires that all egress + + traffic flows through centralised inspection appliances before touching the public + + internet, and customer data must be encrypted with keys we control.' + requirements: ask + business_outcomes: + - outcome: Enterprise landing zone ready to ingest core payments engine workloads + within 10 weeks + id: BO-1 + target: Workload ready within 10 weeks + - outcome: Continuous operational resilience and regulatory compliance under DORA + and PCI-DSS v4.0 + target: Zero audit non-conformances and >= 99.9% platform availability + id: BO-2 + - id: BO-3 + target: Continuous 10Gbps dedicated interconnect availability with centralized + inspection + outcome: Secure, deterministic low-latency hybrid connectivity supporting real-time + payment transactions + exclusions: + - Application workload code, payment engine business logic, and database-level transaction + data management + - Physical data centre hosting, hardware maintenance, and on-premises network infrastructure + - Corporate Entra ID user identity lifecycle management outside Google Cloud federation +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Resource Hierarchy and Environment Boundary Manager + kind: governance + product: Resource Manager + source_ref: C-1 + elements: + - E-1 + - E-2 + - E-3 + - E-4 + - E-5 + - E-6 + - E-7 + - E-8 + - E-9 + - E-10 + - E-34 + - E-35 + - E-36 + - E-37 + - E-38 + - E-50 + - E-51 + - E-55 + - E-60 + - id: S-2 + name: Dedicated Private Hybrid Connectivity Gateway + kind: network + product: Cloud Interconnect + source_ref: C-2 + elements: + - E-11 + - E-12 + - E-13 + - E-14 + - E-56 + - E-57 + - id: S-3 + name: Centralised Outbound Traffic Inspection and Egress Boundary + kind: service + product: Palo Alto Networks VM-Series + source_ref: C-3 + elements: + - E-15 + - E-16 + - E-17 + - E-18 + - E-19 + - E-20 + - E-47 + - E-48 + - E-53 + - id: S-4 + name: Centralised Immutable Audit Event and Compliance Log Repository + kind: storage + product: Cloud Storage + source_ref: C-4 + elements: + - E-21 + - E-22 + - E-39 + - id: S-5 + name: Automated Infrastructure Orchestration and Deployment Engine + kind: service + product: GitHub Actions + source_ref: C-5 + elements: + - E-42 + - E-43 + - id: S-6 + name: Federated Identity and Role Governance Authority + kind: identity + product: IAM Workload Identity Federation + source_ref: C-6 + elements: + - E-23 + - E-24 + - E-25 + - E-26 + - E-27 + - E-45 + - id: S-7 + name: Cryptographic Key Lifecycle and Encryption Controller + kind: keys + product: Cloud KMS + source_ref: C-7 + elements: + - E-28 + - E-29 + - E-30 + - E-31 + - E-49 + - E-58 + - E-59 + - id: S-8 + name: Platform Telemetry and Operational Alerting Engine + kind: observability + product: Cloud Monitoring + source_ref: C-8 + elements: + - E-32 + - E-40 + - E-41 + - E-44 + - E-52 + - id: S-9 + name: Financial Governance and Budget Enforcement Monitor + kind: billing + product: Cloud Billing + source_ref: C-9 + elements: + - E-33 + - E-46 + - E-54 + conditions: + - id: K-1 + 'on': + - S-1 + - S-4 + - S-7 + condition: The landing zone configuration aligns with DORA operational resiliency + guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. + source_ref: NFR-8 + - id: K-2 + 'on': + - S-1 + condition: The secondary region in europe-west1 operates as a warm standby ready + to receive failover traffic. + source_ref: NFR-9 + - id: K-3 + 'on': + - S-1 + condition: Data residency is guaranteed within the United Kingdom and European + Union. + source_ref: NFR-9 + - id: K-4 + 'on': + - S-5 + condition: Platform foundation is fully provisioned and validated for core payments + engine workload ingestion within 10 weeks. + source_ref: NFR-10 + - id: K-5 + 'on': + - S-8 + condition: Platform foundation infrastructure maintains measured monthly availability + of at least 99.9%. + source_ref: NFR-11 + - id: K-6 + 'on': + - S-8 + condition: Disaster recovery activation achieves a recovery time objective (RTO) + of 4 hours. + source_ref: NFR-11 + - id: K-7 + 'on': + - S-8 + condition: Disaster recovery processes achieve a recovery point objective (RPO) + of 1 hour. + source_ref: NFR-11 + - id: K-8 + 'on': + - S-8 + condition: Alert notifications trigger operational incident workflows for the + 24/7 central platform operations team. + source_ref: NFR-12 + - id: K-9 + 'on': + - S-9 + condition: Monthly cloud foundation expenditure does not exceed the target allocation + without operational sign-off. + source_ref: NFR-13 + - id: K-10 + 'on': + - S-2 + condition: Hybrid connection provides deterministic low latency suitable for real-time + payment transaction processing. + source_ref: NFR-14 + - id: K-11 + 'on': + - S-8 + condition: Deployments optimize carbon-free energy utilization based on Google + Cloud regional CFE scores. + source_ref: NFR-15 + - id: K-12 + 'on': + - S-6 + condition: Permissions are continuously rightsized based on IAM Recommender findings. + source_ref: DR-OP-3 + build: + duration_days: 36 + customer_owned: + - Order and provision the Interconnect carrier circuits + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Automated infrastructure-as-code deployment via GitHub Actions pipelines + with automated linting, security scans, and peer review approvals without a manual + CAB. + monitoring: ask +sourcing: + model: provider_operated + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: [] + headcount_limit: ask +service_management: + tooling: Google Cloud operations suite (Cloud Monitoring, Cloud Logging) integrated + with provider operational incident and alerting management systems. +service_levels: + priority_1_response_minutes: 30 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA + - ISO27001 + - PCI_DSS_v4 +commercial: + currency: GBP + monthly_run_budget: ask +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.business_outcomes + statement: 'Assumed intent.business_outcomes is [{''outcome'': ''Enterprise landing + zone ready to ingest core payments engine workloads within 10 weeks'', ''id'': + ''BO-1'', ''target'': ''Workload ready within 10 weeks''}, {''outcome'': ''Continuous + operational resilience and regulatory compliance under DORA and PCI-DSS v4.0'', + ''target'': ''Zero audit non-conformances and >= 99.9% platform availability'', + ''id'': ''BO-2''}, {''id'': ''BO-3'', ''target'': ''Continuous 10Gbps dedicated + interconnect availability with centralized inspection'', ''outcome'': ''Secure, + deterministic low-latency hybrid connectivity supporting real-time payment transactions''}]. + Derived from the workflow request specifying rapid 10-week landing zone delivery, + DORA and PCI-DSS v4.0 compliance, and deterministic hybrid connectivity.' + impact_if_invalid: Changes what is measured, and what each measure is for. +- id: ASSUMP-02 + field: intent.exclusions + statement: 'Assumed intent.exclusions is [''Application workload code, payment engine + business logic, and database-level transaction data management'', ''Physical data + centre hosting, hardware maintenance, and on-premises network infrastructure'', + ''Corporate Entra ID user identity lifecycle management outside Google Cloud federation'']. + Assumed from platform foundation scope: excludes application-level payment processing + logic, physical on-premises hardware maintenance, and end-user identity administration + beyond cloud federation.' + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-03 + field: operations.change_process + statement: Assumed operations.change_process is Automated infrastructure-as-code + deployment via GitHub Actions pipelines with automated linting, security scans, + and peer review approvals without a manual CAB.. Assumed based on catalogue default + and stated use of GitHub Actions delivery pipelines. + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-04 + field: organisation.existing_teams + statement: 'Assumed organisation.existing_teams is []. Assumed catalogue default: + no existing customer operational roles carry forward into the platform foundation; + operations are delivered under a provider-operated model.' + impact_if_invalid: Changes which Roles are new and which are filled. +- id: ASSUMP-05 + field: service_management.tooling + statement: Assumed service_management.tooling is Google Cloud operations suite (Cloud + Monitoring, Cloud Logging) integrated with provider operational incident and alerting + management systems.. Assumed based on catalogue default and solution item S-8 + (Cloud Monitoring). + impact_if_invalid: Changes tooling and the processes written for it. +- id: ASSUMP-06 + field: service_levels.priority_1_response_minutes + statement: Assumed service_levels.priority_1_response_minutes is 30. Assumed catalogue + default of 30 minutes for critical incident response. + impact_if_invalid: Changes on-call arrangements and staffing. +- id: ASSUMP-07 + field: service_levels.recovery_time_hours + statement: Assumed service_levels.recovery_time_hours is 4. Derived from solution + condition K-6 specifying a disaster recovery RTO of 4 hours, matching catalogue + default. + impact_if_invalid: Changes continuity arrangements. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: intent.requirements + prompt: Which requirements for operations has the customer stated outright? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: commercial.monthly_run_budget + prompt: Is there a monthly budget for operating this, in that currency? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: digested + commercial.monthly_run_budget: unknown + compliance.frameworks: digested + engagement.kind: digested + intent.business_outcomes: assumed + intent.exclusions: assumed + intent.requirements: unknown + intent.statement: digested + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: digested + metadata.revision: unknown + metadata.source: digested + operations.availability_hours: digested + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: digested + organisation.existing_teams: assumed + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: assumed + service_levels.recovery_time_hours: assumed + service_management.tooling: assumed + solution.build.customer_owned: digested + solution.build.duration_days: digested + solution.cloud_provider: digested + solution.conditions: digested + solution.inherited_risks: unknown + solution.items: digested + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 3 + answered: 0 + assumed: 7 + unknown: 10 + not_applicable: 0 + digested: 13 diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.scope.yaml new file mode 100644 index 0000000..9533549 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.scope.yaml @@ -0,0 +1,10 @@ +intent: + business_outcomes: decide +run: + pre_approved: true +sourcing: + delivery_countries: + - GB + - PL + - IN + model: provider_operated diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/components.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/components.json new file mode 100644 index 0000000..61eddc2 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/components.json @@ -0,0 +1,79 @@ +[ + { + "id": "C-1", + "name": "Resource Hierarchy and Environment Boundary Manager", + "requirements": [ + "FR-1", + "NFR-1", + "NFR-8", + "NFR-9" + ] + }, + { + "id": "C-2", + "name": "Dedicated Private Hybrid Connectivity Gateway", + "requirements": [ + "FR-2", + "NFR-3", + "NFR-14" + ] + }, + { + "id": "C-3", + "name": "Centralised Outbound Traffic Inspection and Egress Boundary", + "requirements": [ + "FR-3", + "NFR-2", + "NFR-4" + ] + }, + { + "id": "C-4", + "name": "Centralised Immutable Audit Event and Compliance Log Repository", + "requirements": [ + "FR-4", + "NFR-7", + "NFR-8" + ] + }, + { + "id": "C-5", + "name": "Automated Infrastructure Orchestration and Deployment Engine", + "requirements": [ + "FR-5", + "NFR-10" + ] + }, + { + "id": "C-6", + "name": "Federated Identity and Role Governance Authority", + "requirements": [ + "FR-6", + "NFR-5" + ] + }, + { + "id": "C-7", + "name": "Cryptographic Key Lifecycle and Encryption Controller", + "requirements": [ + "NFR-6", + "NFR-8" + ] + }, + { + "id": "C-8", + "name": "Platform Telemetry and Operational Alerting Engine", + "requirements": [ + "NFR-11", + "NFR-12", + "NFR-15" + ] + }, + { + "id": "C-9", + "name": "Financial Governance and Budget Enforcement Monitor", + "requirements": [ + "NFR-13" + ] + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/execution.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/execution.json new file mode 100644 index 0000000..126ac0c --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/execution.json @@ -0,0 +1,140 @@ +{ + "customer_case": "case-08-landing-zone", + "execution_id": "case-08-landing-zone-185426", + "session_id": "cdd6bfe7-2bd8-44af-8e15-9dcfa869eb84", + "intent_version": 1, + "workflow_request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.", + "baseline": null, + "overlay_skills": {}, + "terraform_modules": false, + "scope": { + "status": "BASELINE_LOCKED", + "readiness": { + "supplied": 44, + "answered": 0, + "assumed": 12, + "unknown": 4, + "not_applicable": 3 + } + }, + "delivery_plan": { + "work_packages": 9, + "activities": 15, + "working_days": 36.0, + "weeks": 7.2, + "team_size": 3, + "window_weeks": 10, + "within_window": true, + "critical_path": [ + "A-6", + "A-10", + "A-13", + "A-14", + "A-15" + ] + }, + "status": "completed", + "warnings": [], + "risks": [], + "assessment": { + "recommendations_quality": { + "specific": 0.91, + "actionable": 0.79, + "trade_offs": 0.95, + "traceable": 0.73 + }, + "mean": 0.84, + "meets_bar": true + }, + "phases": { + "scoping": "approved", + "requirements_discovery": "approved", + "solution_architecture": "approved", + "solution_implementation": "approved", + "solution_validation": "approved", + "packaging": "approved" + }, + "solution_guide": "solution-architecture-guide.md", + "customer_questions": "customer-questions.md", + "changes": null, + "deliverables": { + "scope": { + "document": "deliverables/scope.md", + "attachments": [ + "scope.json", + "scope.yaml" + ], + "status": "submitted" + }, + "requirements": { + "document": "deliverables/requirements.md", + "attachments": [ + "requirements.json" + ], + "status": "submitted" + }, + "technical_decomposition": { + "document": "deliverables/technical_decomposition.md", + "attachments": [ + "components.json" + ], + "status": "submitted" + }, + "product_selection": { + "document": "deliverables/product_selection.md", + "attachments": [ + "products.json" + ], + "status": "submitted" + }, + "design_recommendations": { + "document": "deliverables/design_recommendations.md", + "attachments": [ + "recommendations.json" + ], + "status": "submitted" + }, + "target_architecture": { + "document": "deliverables/target_architecture.md", + "attachments": [ + "architecture.mmd", + "target-architecture.json" + ], + "status": "submitted" + }, + "architecture_description": { + "document": "deliverables/architecture_description.md", + "attachments": [], + "status": "submitted" + }, + "deployment_guidance": { + "document": "deliverables/deployment_guidance.md", + "attachments": [ + "implementation.json", + "terraform/main.tf", + "terraform/variables.tf", + "terraform/versions.tf" + ], + "status": "submitted" + }, + "validation_results": { + "document": "deliverables/validation_results.md", + "attachments": [ + "verdicts.json" + ], + "status": "submitted" + }, + "work_breakdown": { + "document": "deliverables/work_breakdown.md", + "attachments": [ + "work-breakdown.json" + ], + "status": "submitted" + }, + "solution_guide": { + "document": "deliverables/solution_guide.md", + "attachments": [], + "status": "submitted" + } + } +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/products.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/products.json new file mode 100644 index 0000000..ca9e033 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/products.json @@ -0,0 +1,38 @@ +[ + { + "component": "C-1", + "product": "Resource Manager" + }, + { + "component": "C-2", + "product": "Cloud Interconnect" + }, + { + "component": "C-3", + "product": "Palo Alto Networks VM-Series" + }, + { + "component": "C-4", + "product": "Cloud Storage" + }, + { + "component": "C-5", + "product": "GitHub Actions" + }, + { + "component": "C-6", + "product": "IAM Workload Identity Federation" + }, + { + "component": "C-7", + "product": "Cloud KMS" + }, + { + "component": "C-8", + "product": "Cloud Monitoring" + }, + { + "component": "C-9", + "product": "Cloud Billing" + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/recommendations.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/recommendations.json new file mode 100644 index 0000000..3721e66 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/recommendations.json @@ -0,0 +1,248 @@ +[ + { + "id": "DR-SEC-1", + "pillar": "security", + "text": "Implement a VPC Service Controls perimeter around all Cardholder Data Environment (CDE) projects and the centralized audit log buckets.", + "components": [ + "C-1", + "C-4" + ], + "conditions": [ + "A VPC Service Controls perimeter encloses the CDE and audit log projects.", + "The VPC Service Controls perimeter restricts ingress and egress to authorized traffic." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-2", + "pillar": "security", + "text": "Apply Organization Policy constraints to disable default network creation, deny external IPs, prohibit service account key creation, and restrict resource locations.", + "components": [ + "C-1" + ], + "conditions": [ + "The compute.skipDefaultNetworkCreation constraint is enforced.", + "The compute.vmExternalIpAccess constraint is enforced to deny all.", + "The iam.disableServiceAccountKeyCreation constraint is enforced.", + "The gcp.resourceLocations constraint restricts resources to europe-west2 and europe-west1." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-3", + "pillar": "security", + "text": "Enable Confidential Computing on the payment engine Compute Engine instances to encrypt data in-use.", + "components": [], + "conditions": [ + "Confidential Computing is enabled on the core payment engine Compute Engine instances." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-4", + "pillar": "security", + "text": "Deploy Sensitive Data Protection (Cloud DLP) inspection templates to detect and mask PAN data.", + "components": [ + "C-4" + ], + "conditions": [ + "Sensitive Data Protection (Cloud DLP) inspection templates are configured to detect and mask PAN data." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-5", + "pillar": "security", + "text": "Enable Security Command Center at the organization level with Event and Container Threat Detection.", + "components": [ + "C-1", + "C-8" + ], + "conditions": [ + "Security Command Center is activated at the organization level.", + "Event Threat Detection and Container Threat Detection are enabled in SCC." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-6", + "pillar": "security", + "text": "Implement event-driven automated responses using Eventarc and Cloud Run functions for SCC findings.", + "components": [ + "C-8" + ], + "conditions": [ + "Eventarc is configured to route high-severity SCC findings.", + "Cloud Run functions are configured to execute automated remediation on Eventarc triggers." + ], + "outside_code": [] + }, + { + "id": "DR-REL-1", + "pillar": "reliability", + "text": "Provision redundant Dedicated Cloud Interconnect circuits across distinct physical edge colocation facilities, terminating on redundant Cloud Routers with BFD.", + "components": [ + "C-2" + ], + "conditions": [ + "Dedicated Cloud Interconnect circuits are provisioned across redundant edge colocation facilities.", + "Cloud Interconnects terminate on redundant Cloud Routers in each region.", + "Bidirectional Forwarding Detection (BFD) is enabled on the Cloud Router BGP sessions." + ], + "outside_code": [] + }, + { + "id": "DR-REL-2", + "pillar": "reliability", + "text": "Deploy Palo Alto VM-Series firewalls in a regional Managed Instance Group across at least three zones, fronted by an Internal Passthrough Network Load Balancer.", + "components": [ + "C-3" + ], + "conditions": [ + "Palo Alto VM-Series firewalls are deployed in a regional Managed Instance Group (MIG) across at least three zones.", + "An Internal Passthrough Network Load Balancer routes traffic to the VM-Series MIG.", + "Health checks are configured on the internal load balancer to probe firewall health." + ], + "outside_code": [] + }, + { + "id": "DR-REL-3", + "pillar": "reliability", + "text": "Provision dedicated regional Cloud KMS key rings in both europe-west2 and europe-west1, granting workload identity service accounts Decrypter roles on both.", + "components": [ + "C-7", + "C-6" + ], + "conditions": [ + "Regional Cloud KMS key rings are provisioned in both europe-west2 and europe-west1.", + "Workload Identity Federation service accounts are granted Decrypter roles on both regional key sets." + ], + "outside_code": [] + }, + { + "id": "DR-REL-4", + "pillar": "reliability", + "text": "Configure the Cloud Storage buckets for audit logging as Dual-Region spanning europe-west2 and europe-west1.", + "components": [ + "C-4" + ], + "conditions": [ + "Audit log Cloud Storage buckets are configured as Dual-Region spanning europe-west2 and europe-west1." + ], + "outside_code": [] + }, + { + "id": "DR-REL-5", + "pillar": "reliability", + "text": "Isolate the Terraform state of the primary region from the standby region using multi-region Cloud Storage buckets with Object Versioning.", + "components": [ + "C-5" + ], + "conditions": [ + "Terraform state for primary and standby regions is decoupled into separate failure domains.", + "Terraform state is stored in multi-region Cloud Storage buckets with Object Versioning enabled." + ], + "outside_code": [] + }, + { + "id": "DR-OP-1", + "pillar": "operational-excellence", + "text": "Integrate Policy-as-Code validation into the GitHub Actions pull request workflows.", + "components": [ + "C-5" + ], + "conditions": [ + "GitHub Actions workflows include automated Policy-as-Code validation steps prior to deployment execution." + ], + "outside_code": [] + }, + { + "id": "DR-OP-2", + "pillar": "operational-excellence", + "text": "Establish explicit Service Level Indicators (SLIs) and configure multi-burn-rate alerting policies in Cloud Monitoring.", + "components": [ + "C-8" + ], + "conditions": [ + "Service Level Objectives (SLOs) and SLIs are defined in Cloud Monitoring Service Monitoring.", + "Multi-burn-rate alerting policies are configured based on the SLO error budget." + ], + "outside_code": [] + }, + { + "id": "DR-OP-3", + "pillar": "operational-excellence", + "text": "Activate the IAM Recommender to continuously evaluate role bindings.", + "components": [ + "C-6" + ], + "conditions": [ + "IAM Recommender is activated on the foundation projects." + ], + "outside_code": [ + "Permissions are continuously rightsized based on IAM Recommender findings." + ] + }, + { + "id": "DR-COST-1", + "pillar": "cost-optimization", + "text": "Enforce a mandatory labeling taxonomy through Organization Policies and export granular billing data to BigQuery.", + "components": [ + "C-9", + "C-1" + ], + "conditions": [ + "Cloud Billing export to BigQuery is enabled.", + "A mandatory labeling taxonomy is enforced via Organization Policy Service tags." + ], + "outside_code": [] + }, + { + "id": "DR-COST-2", + "pillar": "cost-optimization", + "text": "Implement Cloud Storage Object Lifecycle Management to transition logs to Nearline storage after 30 days and to Coldline or Archive storage after 90 days.", + "components": [ + "C-4" + ], + "conditions": [ + "Object Lifecycle Management rules transition Cloud Storage objects to Nearline storage after 30 days.", + "Object Lifecycle Management rules transition Cloud Storage objects to Coldline or Archive storage after 90 days." + ], + "outside_code": [] + }, + { + "id": "DR-PERF-1", + "pillar": "performance-optimization", + "text": "Configure Cloud Routers with regional dynamic routing to localize egress traffic within the warm standby region.", + "components": [ + "C-2" + ], + "conditions": [ + "Cloud Routers are configured with regional dynamic routing mode." + ], + "outside_code": [] + }, + { + "id": "DR-PERF-2", + "pillar": "performance-optimization", + "text": "Instrument the deployment with OpenTelemetry exporting to Cloud Trace, and use Cloud Profiler on the compute instances.", + "components": [ + "C-8" + ], + "conditions": [ + "OpenTelemetry tracing is configured to export to Cloud Trace.", + "Cloud Profiler agents are enabled on the core payments engine instances." + ], + "outside_code": [] + }, + { + "id": "DR-SUST-1", + "pillar": "sustainability", + "text": "Implement a Pilot Light warm standby by maintaining active data replication while scaling the standby compute instances to a minimal baseline.", + "components": [], + "conditions": [ + "Secondary region compute instances are scaled to a minimal baseline (Pilot Light) configuration during normal operations." + ], + "outside_code": [] + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/requirements.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/requirements.json new file mode 100644 index 0000000..307613b --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/requirements.json @@ -0,0 +1,280 @@ +[ + { + "id": "FR-1", + "kind": "functional", + "text": "The platform foundation isolates development, staging, and production environments across distinct Google Cloud projects and resource folders.", + "from": [ + "REQ-02" + ] + }, + { + "id": "FR-2", + "kind": "functional", + "text": "The platform foundation establishes private bidirectional network connectivity between Google Cloud and on-premises core payment systems over Cloud Interconnect.", + "from": [ + "REQ-03" + ] + }, + { + "id": "FR-3", + "kind": "functional", + "text": "The platform foundation routes all outbound internet-bound traffic from workload spoke networks through centralised network security inspection appliances.", + "from": [ + "REQ-03" + ] + }, + { + "id": "FR-4", + "kind": "functional", + "text": "The platform foundation captures and exports all administrative and data access audit logs across environments into a centralised, tamper-resistant log repository.", + "from": [ + "REQ-02" + ] + }, + { + "id": "FR-5", + "kind": "functional", + "text": "The platform foundation provisions and manages all cloud infrastructure as code via automated CI/CD pipelines executing validated pull request workflows.", + "from": [ + "REQ-01" + ] + }, + { + "id": "FR-6", + "kind": "functional", + "text": "The platform foundation federates corporate identity for single sign-on and enforces least-privilege role-based access control for platform administrators and application squads.", + "from": [ + "REQ-04" + ] + }, + { + "id": "NFR-1", + "kind": "non_functional", + "text": "Environments are partitioned into development, staging, and production tiers, enforcing corp_workload controls on lower environments and pci_regulated_workload controls on production.", + "conditions": [ + "Separate projects or folders exist for development, staging, and production environment tiers.", + "Production environments enforce pci_regulated_workload archetype security controls.", + "Development and staging environments enforce corp_workload archetype security controls." + ], + "outside_code": [], + "from": [ + "REQ-02", + "governance.environment_tiers" + ] + }, + { + "id": "NFR-2", + "kind": "non_functional", + "text": "Network connectivity is structured as a Shared VPC hub-and-spoke topology with zero direct internet ingress to workload subnets.", + "conditions": [ + "A Shared VPC host project is connected to service projects across environment tiers.", + "Workload subnets have private Google access enabled and no external public IP addresses assigned.", + "Ingress firewall rules deny uninspected public internet traffic to workload subnets." + ], + "outside_code": [], + "from": [ + "REQ-02", + "networking.topology", + "networking.exposure" + ] + }, + { + "id": "NFR-3", + "kind": "non_functional", + "text": "Hybrid connectivity to on-premises core payment systems is enabled via a 10Gbps Dedicated Cloud Interconnect.", + "conditions": [ + "Dedicated Cloud Interconnect connections are configured for hybrid connectivity.", + "Cloud Interconnect capacity is provisioned at 10Gbps bandwidth.", + "Interconnect VLAN attachments are associated with Cloud Routers in the Shared VPC hub." + ], + "outside_code": [], + "from": [ + "REQ-03", + "networking.hybrid_connectivity.enabled", + "networking.hybrid_connectivity.type", + "networking.hybrid_connectivity.bandwidth_gbps" + ] + }, + { + "id": "NFR-4", + "kind": "non_functional", + "text": "All outbound internet traffic from workload spokes is forced through centralised inspection appliances deployed in the hub network.", + "conditions": [ + "Inspection appliance instances are deployed in a centralised hub network.", + "Custom VPC route tables direct 0.0.0.0/0 egress traffic from spokes to the centralised inspection appliance instances or internal load balancer." + ], + "outside_code": [], + "from": [ + "REQ-03", + "networking.egress_control.model" + ] + }, + { + "id": "NFR-5", + "kind": "non_functional", + "text": "Administrative access is authenticated via Entra ID single sign-on, and delivery pipelines authenticate via Workload Identity Federation without stored service account keys.", + "conditions": [ + "Workload Identity Federation pools and providers are configured for GitHub Actions authentication.", + "IAM bindings grant roles to federated GitHub Actions identities without downloadable service account keys.", + "Corporate single sign-on is configured with Entra ID as the identity source of truth." + ], + "outside_code": [], + "from": [ + "REQ-04", + "governance.identity_source_of_truth", + "governance.pipeline_authentication" + ] + }, + { + "id": "NFR-6", + "kind": "non_functional", + "text": "Customer confidential data is encrypted at rest using customer-managed encryption keys with a 90-day rotation schedule.", + "conditions": [ + "Cloud KMS customer-managed encryption keys are configured for infrastructure resources storing confidential data.", + "Cloud KMS keys have an automated rotation period set to 90 days (7776000s)." + ], + "outside_code": [], + "from": [ + "REQ-04", + "compliance.encryption.type", + "compliance.encryption.key_rotation_days" + ] + }, + { + "id": "NFR-7", + "kind": "non_functional", + "text": "Audit logs across all environments are retained for 365 days in a storage bucket protected by an immutable retention policy.", + "conditions": [ + "An organization-level or project-level log sink exports audit logs to a centralised Cloud Storage bucket.", + "The centralised audit log storage bucket specifies a retention period of 365 days (31536000 seconds).", + "The centralised audit log storage bucket has retention policy locking enabled to prevent modification or early deletion." + ], + "outside_code": [], + "from": [ + "REQ-02", + "compliance.audit_logging.retention_days", + "compliance.audit_logging.immutable_retention_policy" + ] + }, + { + "id": "NFR-8", + "kind": "non_functional", + "text": "The platform foundation enforces DORA, ISO27001, and PCI-DSS v4.0 regulatory baselines and protects confidential data classification.", + "conditions": [ + "Organization policy constraints enforce perimeter isolation and restrict unauthenticated resource sharing.", + "IAM policies restrict access to confidential data resources to authorized service identities and roles." + ], + "outside_code": [ + "The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit." + ], + "from": [ + "REQ-02", + "compliance.frameworks", + "compliance.data_classification" + ] + }, + { + "id": "NFR-9", + "kind": "non_functional", + "text": "Platform foundation resources are provisioned across europe-west2 (primary) and europe-west1 (secondary) in a warm standby model, restricting data residency to the UK and EU.", + "conditions": [ + "Primary landing zone network and foundation resources are provisioned in europe-west2.", + "Secondary landing zone network and foundation standby resources are provisioned in europe-west1.", + "Resource location organization policies restrict resource and data storage locations to europe-west2 and europe-west1." + ], + "outside_code": [ + "The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic.", + "Data residency is guaranteed within the United Kingdom and European Union." + ], + "from": [ + "REQ-02", + "estate.regions.primary", + "estate.regions.secondary", + "estate.regions.resiliency_tier", + "estate.data_residency" + ] + }, + { + "id": "NFR-10", + "kind": "non_functional", + "text": "The platform foundation landing zone is delivered and operational for workload ingestion within 10 weeks.", + "conditions": [ + "All landing zone foundation resources are codified in modular Terraform configurations." + ], + "outside_code": [ + "Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks." + ], + "from": [ + "REQ-01" + ] + }, + { + "id": "NFR-11", + "kind": "non_functional", + "text": "The platform foundation maintains 99.9% availability with a 4-hour recovery time objective (RTO) and a 1-hour recovery point objective (RPO).", + "conditions": [ + "Regional networking and core management services are deployed across at least three zones in the primary region." + ], + "outside_code": [ + "Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.", + "Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.", + "Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour." + ], + "from": [ + "REQ-02" + ] + }, + { + "id": "NFR-12", + "kind": "non_functional", + "text": "Platform operations monitoring and alerting are implemented using Google Cloud Operations Suite with 24/7 alerting capabilities.", + "conditions": [ + "Cloud Monitoring workspaces and notification channels are configured for the platform foundation.", + "Alert policies are configured for Interconnect link degradation, appliance health, and egress routing failures." + ], + "outside_code": [ + "Alert notifications trigger operational incident workflows for the 24/7 central platform operations team." + ], + "from": [] + }, + { + "id": "NFR-13", + "kind": "non_functional", + "text": "Infrastructure costs are tracked against a monthly budget of 25,000 GBP with automated alerting.", + "conditions": [ + "A Cloud Billing budget is configured for the foundation projects with currency set to GBP.", + "Budget alert thresholds are configured at 50%, 80%, 90%, and 100% of the 25,000 GBP monthly budget." + ], + "outside_code": [ + "Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off." + ], + "from": [] + }, + { + "id": "NFR-14", + "kind": "non_functional", + "text": "Hybrid networking provides dedicated 10Gbps unthrottled throughput with minimal latency between Google Cloud and on-premises core systems.", + "conditions": [ + "Cloud Routers configured for Interconnect advertise non-overlapping RFC 1918 BGP routes.", + "Dedicated Interconnect VLAN attachments are configured with dedicated 10Gbps bandwidth allocation." + ], + "outside_code": [ + "Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing." + ], + "from": [ + "REQ-03" + ] + }, + { + "id": "NFR-15", + "kind": "non_functional", + "text": "Landing zone resources are deployed in Google Cloud regions with strong low-carbon energy ratings.", + "conditions": [ + "Foundation resources are constrained to europe-west2 (London) and europe-west1 (Belgium) regions." + ], + "outside_code": [ + "Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores." + ], + "from": [] + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/scope.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/scope.json new file mode 100644 index 0000000..16e782d --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/scope.json @@ -0,0 +1,233 @@ +{ + "kind": "platform_foundation", + "stated": { + "engagement": { + "kind": "platform_foundation" + }, + "intent": { + "statement": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.\n", + "requirements": [ + { + "id": "REQ-01", + "category": "Timeline", + "statement": "Platform foundation must be ready for workload ingestion within 10 weeks." + }, + { + "id": "REQ-02", + "category": "Compliance", + "statement": "Must achieve DORA resiliency alignment and PCI-DSS v4.0 perimeter isolation." + }, + { + "id": "REQ-03", + "category": "Networking", + "statement": "10Gbps on-premises hybrid connectivity; centralised inspection appliances for egress." + }, + { + "id": "REQ-04", + "category": "Identity and security", + "statement": "Workload identity federation with GitHub Actions; Entra ID single sign-on; customer-managed encryption keys." + } + ], + "exclusions": [ + "Application container deployment; workloads are managed by application squads.", + "An active-active secondary region in phase 1; warm standby is acceptable." + ] + }, + "estate": { + "cloud_provider": "gcp", + "mode": "greenfield", + "regions": { + "primary": "europe-west2", + "secondary": "europe-west1", + "resiliency_tier": "warm_standby" + }, + "data_residency": "United Kingdom and European Union", + "existing_systems": "On-premises payment systems that stay in place and connect over the interconnect." + }, + "workload": { + "availability": { + "hours": "always" + } + }, + "compliance": { + "industry": "financial_services", + "frameworks": [ + "DORA", + "ISO27001", + "PCI_DSS_v4" + ], + "data_classification": "confidential", + "encryption": { + "type": "cmek", + "key_rotation_days": 90 + }, + "audit_logging": { + "retention_days": 365, + "immutable_retention_policy": true + } + }, + "networking": { + "exposure": "internal_only", + "topology": "shared_vpc_hub_and_spoke", + "hybrid_connectivity": { + "enabled": true, + "type": "cloud_interconnect", + "bandwidth_gbps": 10 + }, + "egress_control": { + "model": "centralized_appliance" + } + }, + "governance": { + "identity_source_of_truth": "Entra ID", + "pipeline_authentication": "workload identity federation", + "environment_tiers": [ + { + "name": "development", + "archetype": "corp_workload" + }, + { + "name": "staging", + "archetype": "corp_workload" + }, + { + "name": "production", + "archetype": "pci_regulated_workload" + } + ] + }, + "operations": { + "operating_model": "central platform team brokering access for application squads", + "support_hours": "always" + }, + "delivery": { + "window_weeks": 10, + "commercial_model": "fixed_price_milestone", + "iac_tooling": "terraform", + "ci_cd": "GitHub Actions", + "team_size": 3, + "working_days_per_week": 5 + }, + "cost": { + "currency": "GBP" + } + }, + "binding": [ + "estate.regions.primary", + "estate.regions.secondary", + "estate.regions.resiliency_tier", + "estate.data_residency", + "compliance.frameworks", + "compliance.data_classification", + "compliance.encryption.type", + "compliance.encryption.key_rotation_days", + "compliance.audit_logging.retention_days", + "compliance.audit_logging.immutable_retention_policy", + "networking.exposure", + "networking.topology", + "networking.hybrid_connectivity.enabled", + "networking.hybrid_connectivity.type", + "networking.hybrid_connectivity.bandwidth_gbps", + "networking.egress_control.model", + "governance.identity_source_of_truth", + "governance.pipeline_authentication", + "governance.environment_tiers" + ], + "assumed": [ + { + "field": "intent.success_criteria", + "value": [ + "Platform foundation ready for workload ingestion within 10 weeks", + "Compliance achieved with DORA and PCI-DSS v4.0 requirements", + "Hybrid connectivity established via 10Gbps Dedicated Interconnect", + "Centralised egress inspection operational", + "Identity federation and CMEK security controls implemented" + ], + "why": "Assumed standard success criteria based on delivery within 10 weeks and compliance with DORA and PCI-DSS v4.0." + }, + { + "field": "workload.availability.target", + "value": "99.9%", + "why": "Assumed 99.9% availability target for platform foundation infrastructure aligned with warm standby secondary region." + }, + { + "field": "workload.availability.recovery_time", + "value": "4 hours", + "why": "Assumed 4 hours recovery time objective (RTO) for platform foundation services in warm standby." + }, + { + "field": "workload.availability.recovery_point", + "value": "1 hour", + "why": "Assumed 1 hour recovery point objective (RPO) for platform foundation configuration and state." + }, + { + "field": "networking.egress_control.appliance_vendor", + "value": "Palo Alto Networks VM-Series", + "why": "Assumed Palo Alto Networks VM-Series as enterprise inspection appliance for centralized egress control." + }, + { + "field": "networking.cidr_allocations", + "value": "RFC 1918 non-overlapping IP allocation (10.0.0.0/16 partitioned across environments) sized for growth", + "why": "Assumed RFC 1918 non-overlapping IP allocation (10.0.0.0/16 partitioned across environments) sized for growth." + }, + { + "field": "operations.monitoring", + "value": "Google Cloud Operations Suite with availability and error alerting", + "why": "Assumed native Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging) with availability and error alerting." + }, + { + "field": "operations.change_process", + "value": "GitHub Actions pull request workflow with automated Terraform plan validation and peer review", + "why": "Assumed GitHub Actions pull request workflow with automated Terraform plan validation and peer approval." + }, + { + "field": "integrations", + "value": [ + { + "criticality": "critical", + "direction": "bidirectional", + "name": "On-premises core payment systems" + } + ], + "why": "Assumed integration with existing on-premises core payment systems via Dedicated Interconnect." + }, + { + "field": "cost.monthly_budget", + "value": 25000, + "why": "Assumed indicative monthly foundation infrastructure budget of 25000 GBP for 10Gbps interconnect, inspection appliances, and core services." + }, + { + "field": "constraints.products_required", + "value": [], + "why": "No mandated third-party products beyond stated tooling (GCP, Terraform, GitHub Actions, Entra ID)." + }, + { + "field": "constraints.products_forbidden", + "value": [], + "why": "No forbidden products specified." + } + ], + "to_decide": [], + "to_ask": [ + { + "field": "metadata.engagement_reference", + "prompt": "What is metadata.engagement_reference?", + "severity": "LOW" + }, + { + "field": "delivery.target_start_date", + "prompt": "When should work start?", + "severity": "LOW" + }, + { + "field": "constraints.other", + "prompt": "What is constraints.other?", + "severity": "LOW" + } + ], + "not_applicable": [ + "workload.users.count", + "workload.users.peak_concurrent", + "workload.users.who" + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/target-architecture.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/target-architecture.json new file mode 100644 index 0000000..3a35acc --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/target-architecture.json @@ -0,0 +1,1210 @@ +{ + "elements": [ + { + "resource_type": "google_folder", + "product": "Resource Manager", + "component": "C-1", + "name": "Dev Folder", + "id": "E-1" + }, + { + "resource_type": "google_folder", + "id": "E-2", + "product": "Resource Manager", + "component": "C-1", + "name": "Staging Folder" + }, + { + "component": "C-1", + "name": "Production Folder", + "id": "E-3", + "resource_type": "google_folder", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_shared_vpc_host_project", + "name": "Shared VPC Host Project", + "product": "Resource Manager", + "id": "E-4", + "component": "C-1" + }, + { + "product": "Resource Manager", + "component": "C-1", + "resource_type": "google_compute_shared_vpc_service_project", + "name": "Dev Spoke Project", + "id": "E-5" + }, + { + "name": "Stg Spoke Project", + "product": "Resource Manager", + "component": "C-1", + "resource_type": "google_compute_shared_vpc_service_project", + "id": "E-6" + }, + { + "product": "Resource Manager", + "id": "E-7", + "name": "Prod Spoke Project", + "component": "C-1", + "resource_type": "google_compute_shared_vpc_service_project" + }, + { + "product": "Resource Manager", + "name": "Hub VPC Network", + "component": "C-1", + "id": "E-8", + "resource_type": "google_compute_network" + }, + { + "id": "E-9", + "resource_type": "google_compute_subnetwork", + "name": "Dev Subnetwork", + "component": "C-1", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_subnetwork", + "component": "C-1", + "id": "E-10", + "name": "Prod Subnetwork", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_router", + "id": "E-11", + "name": "Primary Cloud Router", + "component": "C-2", + "product": "Cloud Interconnect" + }, + { + "resource_type": "google_compute_router", + "name": "Secondary Cloud Router", + "id": "E-12", + "component": "C-2", + "product": "Cloud Interconnect" + }, + { + "product": "Cloud Interconnect", + "id": "E-13", + "name": "Primary VLAN Attachment 1", + "resource_type": "google_compute_interconnect_attachment", + "component": "C-2" + }, + { + "id": "E-14", + "component": "C-2", + "resource_type": "google_compute_interconnect_attachment", + "name": "Primary VLAN Attachment 2", + "product": "Cloud Interconnect" + }, + { + "name": "Palo Alto Instance Template", + "component": "C-3", + "id": "E-15", + "product": "Palo Alto Networks VM-Series", + "resource_type": "google_compute_instance_template" + }, + { + "id": "E-16", + "name": "Palo Alto MIG Primary", + "resource_type": "google_compute_region_instance_group_manager", + "component": "C-3", + "product": "Palo Alto Networks VM-Series" + }, + { + "id": "E-17", + "component": "C-3", + "resource_type": "google_compute_forwarding_rule", + "name": "Palo Alto ILB", + "product": "Palo Alto Networks VM-Series" + }, + { + "name": "Palo Alto ILB Backend", + "resource_type": "google_compute_region_backend_service", + "component": "C-3", + "id": "E-18", + "product": "Palo Alto Networks VM-Series" + }, + { + "resource_type": "google_compute_health_check", + "component": "C-3", + "name": "Palo Alto ILB Health Check", + "product": "Palo Alto Networks VM-Series", + "id": "E-19" + }, + { + "component": "C-3", + "id": "E-20", + "name": "Egress Route to ILB", + "resource_type": "google_compute_route", + "product": "Palo Alto Networks VM-Series" + }, + { + "id": "E-21", + "component": "C-4", + "name": "Audit Logs Storage Bucket", + "product": "Cloud Storage", + "resource_type": "google_storage_bucket" + }, + { + "component": "C-4", + "product": "Cloud Storage", + "id": "E-22", + "name": "Audit Log Sink", + "resource_type": "google_logging_project_sink" + }, + { + "product": "IAM Workload Identity Federation", + "id": "E-23", + "component": "C-6", + "name": "WIF Pool", + "resource_type": "google_iam_workload_identity_pool" + }, + { + "name": "WIF Provider GitHub", + "product": "IAM Workload Identity Federation", + "id": "E-24", + "resource_type": "google_iam_workload_identity_pool_provider", + "component": "C-6" + }, + { + "component": "C-6", + "resource_type": "google_iam_workforce_pool_provider", + "name": "WIF Provider Entra ID", + "product": "IAM Workload Identity Federation", + "id": "E-25" + }, + { + "product": "IAM Workload Identity Federation", + "name": "Pipeline Service Account", + "component": "C-6", + "id": "E-26", + "resource_type": "google_service_account" + }, + { + "name": "Pipeline SA WIF Binding", + "resource_type": "google_service_account_iam_member", + "id": "E-27", + "product": "IAM Workload Identity Federation", + "component": "C-6" + }, + { + "product": "Cloud KMS", + "resource_type": "google_kms_key_ring", + "name": "Primary KMS Key Ring", + "id": "E-28", + "component": "C-7" + }, + { + "product": "Cloud KMS", + "resource_type": "google_kms_crypto_key", + "component": "C-7", + "name": "Primary KMS Crypto Key", + "id": "E-29" + }, + { + "name": "Secondary KMS Key Ring", + "id": "E-30", + "product": "Cloud KMS", + "component": "C-7", + "resource_type": "google_kms_key_ring" + }, + { + "product": "Cloud KMS", + "name": "Secondary KMS Crypto Key", + "component": "C-7", + "resource_type": "google_kms_crypto_key", + "id": "E-31" + }, + { + "resource_type": "google_monitoring_alert_policy", + "name": "Platform Alert Policy", + "product": "Cloud Monitoring", + "component": "C-8", + "id": "E-32" + }, + { + "product": "Cloud Billing", + "name": "Foundation Cloud Budget", + "component": "C-9", + "resource_type": "google_billing_budget", + "id": "E-33" + }, + { + "id": "E-34", + "name": "Org Policy skipDefaultNetworkCreation", + "resource_type": "google_project_organization_policy", + "component": "C-1", + "product": "Resource Manager" + }, + { + "component": "C-1", + "id": "E-35", + "name": "Org Policy vmExternalIpAccess", + "resource_type": "google_project_organization_policy", + "product": "Resource Manager" + }, + { + "resource_type": "google_project_organization_policy", + "name": "Org Policy disableServiceAccountKeyCreation", + "id": "E-36", + "product": "Resource Manager", + "component": "C-1" + }, + { + "resource_type": "google_project_organization_policy", + "product": "Resource Manager", + "component": "C-1", + "name": "Org Policy resourceLocations", + "id": "E-37" + }, + { + "name": "VPC SC Perimeter", + "component": "C-1", + "product": "Resource Manager", + "resource_type": "google_access_context_manager_service_perimeter", + "id": "E-38" + }, + { + "name": "DLP Inspect Template", + "component": "C-4", + "resource_type": "google_data_loss_prevention_inspect_template", + "product": "Cloud Storage", + "id": "E-39" + }, + { + "id": "E-40", + "product": "Cloud Monitoring", + "name": "SCC Eventarc Trigger", + "resource_type": "google_eventarc_trigger", + "component": "C-8" + }, + { + "product": "Cloud Monitoring", + "resource_type": "google_cloudfunctions2_function", + "component": "C-8", + "name": "SCC Remediation Function", + "id": "E-41" + }, + { + "component": "C-5", + "id": "E-42", + "name": "Primary TF State Bucket", + "resource_type": "google_storage_bucket", + "product": "GitHub Actions" + }, + { + "resource_type": "google_storage_bucket", + "id": "E-43", + "product": "GitHub Actions", + "component": "C-5", + "name": "Secondary TF State Bucket" + }, + { + "product": "Cloud Monitoring", + "component": "C-8", + "resource_type": "google_monitoring_slo", + "name": "Platform SLO", + "id": "E-44" + }, + { + "id": "E-45", + "component": "C-6", + "resource_type": "google_project_service", + "name": "IAM Recommender Service", + "product": "IAM Workload Identity Federation" + }, + { + "resource_type": "google_tags_tag_key", + "name": "Taxonomy Tag Key", + "component": "C-9", + "product": "Cloud Billing", + "id": "E-46" + }, + { + "product": "Palo Alto Networks VM-Series", + "component": "C-3", + "resource_type": "google_compute_region_autoscaler", + "name": "Standby MIG Autoscaler", + "id": "E-47" + }, + { + "name": "Deny Ingress Firewall", + "resource_type": "google_compute_firewall", + "component": "C-3", + "product": "Palo Alto Networks VM-Series", + "id": "E-48" + }, + { + "component": "C-7", + "id": "E-49", + "name": "KMS IAM Binding", + "resource_type": "google_kms_crypto_key_iam_member", + "product": "Cloud KMS" + }, + { + "id": "E-50", + "component": "C-1", + "product": "Resource Manager", + "resource_type": "google_tags_tag_binding", + "name": "Environment Tag Binding" + }, + { + "name": "Payment Engine Template", + "product": "Resource Manager", + "id": "E-51", + "resource_type": "google_compute_instance_template", + "component": "C-1" + }, + { + "name": "Platform Custom Service", + "component": "C-8", + "product": "Cloud Monitoring", + "id": "E-52", + "resource_type": "google_monitoring_custom_service" + }, + { + "name": "Palo Alto MIG Secondary", + "component": "C-3", + "resource_type": "google_compute_region_instance_group_manager", + "product": "Palo Alto Networks VM-Series", + "id": "E-53" + }, + { + "product": "Cloud Billing", + "resource_type": "google_logging_project_sink", + "name": "Billing BigQuery Sink", + "component": "C-9", + "id": "E-54" + }, + { + "name": "Prod Tag Binding", + "id": "E-55", + "resource_type": "google_tags_tag_binding", + "component": "C-1", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_router_peer", + "id": "E-56", + "component": "C-2", + "name": "Primary BGP Peer 1", + "product": "Cloud Interconnect" + }, + { + "resource_type": "google_compute_router_peer", + "id": "E-57", + "component": "C-2", + "name": "Primary BGP Peer 2", + "product": "Cloud Interconnect" + }, + { + "resource_type": "google_kms_crypto_key_iam_member", + "component": "C-7", + "name": "Secondary KMS Decrypter Binding", + "id": "E-58", + "product": "Cloud KMS" + }, + { + "component": "C-7", + "name": "Primary KMS Decrypter Binding", + "product": "Cloud KMS", + "resource_type": "google_kms_crypto_key_iam_member", + "id": "E-59" + }, + { + "id": "E-60", + "resource_type": "google_tags_tag_binding", + "name": "Staging Tag Binding", + "component": "C-1", + "product": "Resource Manager" + } + ], + "relationships": [ + { + "from": "E-5", + "id": "R-1", + "kind": "uses", + "to": "E-4" + }, + { + "id": "R-2", + "kind": "uses", + "to": "E-4", + "from": "E-6" + }, + { + "kind": "uses", + "to": "E-4", + "id": "R-3", + "from": "E-7" + }, + { + "kind": "attached to", + "to": "E-11", + "from": "E-13", + "id": "R-4" + }, + { + "from": "E-14", + "to": "E-11", + "id": "R-5", + "kind": "attached to" + }, + { + "kind": "instantiates", + "from": "E-16", + "id": "R-6", + "to": "E-15" + }, + { + "to": "E-15", + "from": "E-53", + "id": "R-6b", + "kind": "instantiates" + }, + { + "to": "E-17", + "from": "E-20", + "id": "R-7", + "kind": "routes to" + }, + { + "from": "E-17", + "id": "R-8", + "kind": "routes to", + "to": "E-18" + }, + { + "id": "R-9", + "kind": "checked by", + "to": "E-19", + "from": "E-18" + }, + { + "from": "E-22", + "id": "R-10", + "kind": "exports to", + "to": "E-21" + }, + { + "kind": "authenticates via", + "id": "R-11", + "from": "E-24", + "to": "E-23" + }, + { + "from": "E-27", + "to": "E-26", + "id": "R-12", + "kind": "granted to" + }, + { + "from": "E-29", + "to": "E-28", + "id": "R-13", + "kind": "contained in" + }, + { + "from": "E-31", + "kind": "contained in", + "id": "R-14", + "to": "E-30" + }, + { + "kind": "granted on", + "id": "R-15", + "from": "E-49", + "to": "E-29" + }, + { + "id": "R-16", + "kind": "monitors", + "to": "E-52", + "from": "E-44" + }, + { + "from": "E-47", + "kind": "scales", + "id": "R-17", + "to": "E-53" + }, + { + "id": "R-18", + "to": "E-46", + "from": "E-50", + "kind": "binds" + }, + { + "to": "E-11", + "from": "E-56", + "id": "R-19", + "kind": "contained in" + }, + { + "from": "E-57", + "kind": "contained in", + "to": "E-11", + "id": "R-20" + }, + { + "to": "E-31", + "kind": "granted on", + "from": "E-58", + "id": "R-21" + }, + { + "to": "E-29", + "from": "E-59", + "kind": "granted on", + "id": "R-22" + }, + { + "to": "E-46", + "kind": "binds", + "id": "R-23", + "from": "E-60" + } + ], + "checks": [ + { + "id": "K-1", + "source": "NFR-1", + "applies_to": "E-1", + "statement": "`display_name` is set" + }, + { + "applies_to": "E-2", + "source": "NFR-1", + "statement": "`display_name` is set", + "id": "K-2" + }, + { + "applies_to": "E-3", + "source": "NFR-1", + "statement": "`display_name` is set", + "id": "K-3" + }, + { + "statement": "`tag_value` is set to the environment archetype", + "applies_to": "E-50", + "source": "NFR-1", + "id": "K-4" + }, + { + "applies_to": "E-5", + "id": "K-5", + "statement": "`host_project` is set to the hub project", + "source": "NFR-2" + }, + { + "source": "NFR-2", + "applies_to": "E-6", + "statement": "`host_project` is set to the hub project", + "id": "K-6" + }, + { + "id": "K-7", + "source": "NFR-2", + "applies_to": "E-7", + "statement": "`host_project` is set to the hub project" + }, + { + "id": "K-8", + "applies_to": "E-9", + "statement": "`private_ip_google_access` is true", + "source": "NFR-2" + }, + { + "statement": "`private_ip_google_access` is true", + "applies_to": "E-10", + "source": "NFR-2", + "id": "K-9" + }, + { + "statement": "`direction` is \"INGRESS\"", + "source": "NFR-2", + "applies_to": "E-48", + "id": "K-10" + }, + { + "id": "K-11", + "source": "NFR-2", + "statement": "`deny.protocol` is \"all\"", + "applies_to": "E-48" + }, + { + "statement": "`source_ranges` includes \"0.0.0.0/0\"", + "id": "K-12", + "applies_to": "E-48", + "source": "NFR-2" + }, + { + "applies_to": "E-13", + "id": "K-13", + "statement": "`type` is \"DEDICATED\"", + "source": "NFR-3" + }, + { + "statement": "`type` is \"DEDICATED\"", + "id": "K-14", + "source": "NFR-3", + "applies_to": "E-14" + }, + { + "source": "NFR-3", + "applies_to": "E-13", + "id": "K-15", + "statement": "`bandwidth` is \"10G\"" + }, + { + "applies_to": "E-14", + "statement": "`bandwidth` is \"10G\"", + "id": "K-16", + "source": "NFR-3" + }, + { + "id": "K-17", + "source": "NFR-3", + "statement": "`router` is set to the Primary Router", + "applies_to": "E-13" + }, + { + "id": "K-18", + "source": "NFR-4", + "statement": "`network_interface.network` is set to the hub VPC", + "applies_to": "E-15" + }, + { + "source": "NFR-4", + "applies_to": "E-20", + "id": "K-19", + "statement": "`dest_range` is \"0.0.0.0/0\"" + }, + { + "statement": "`next_hop_ilb` is set to the ILB forwarding rule", + "applies_to": "E-20", + "source": "NFR-4", + "id": "K-20" + }, + { + "applies_to": "E-24", + "statement": "`oidc.issuer_uri` is \"https://token.actions.githubusercontent.com\"", + "source": "NFR-5", + "id": "K-21" + }, + { + "id": "K-22", + "statement": "`member` includes \"principalSet://iam.googleapis.com/\"", + "applies_to": "E-27", + "source": "NFR-5" + }, + { + "id": "K-23", + "applies_to": "E-25", + "statement": "`saml.idp_metadata_xml` is populated", + "source": "NFR-5" + }, + { + "applies_to": "E-29", + "id": "K-24", + "source": "NFR-6", + "statement": "`purpose` is \"ENCRYPT_DECRYPT\"" + }, + { + "applies_to": "E-31", + "source": "NFR-6", + "id": "K-25", + "statement": "`purpose` is \"ENCRYPT_DECRYPT\"" + }, + { + "applies_to": "E-29", + "source": "NFR-6", + "id": "K-26", + "statement": "`rotation_period` is \"7776000s\"" + }, + { + "applies_to": "E-31", + "source": "NFR-6", + "id": "K-27", + "statement": "`rotation_period` is \"7776000s\"" + }, + { + "statement": "`destination` is set to the storage bucket", + "applies_to": "E-22", + "id": "K-28", + "source": "NFR-7" + }, + { + "applies_to": "E-21", + "source": "NFR-7", + "statement": "`retention_policy.retention_period` is 31536000", + "id": "K-29" + }, + { + "statement": "`retention_policy.is_locked` is true", + "source": "NFR-7", + "applies_to": "E-21", + "id": "K-30" + }, + { + "id": "K-31", + "applies_to": "E-38", + "source": "NFR-8", + "statement": "`status.restricted_services` includes \"compute.googleapis.com\"" + }, + { + "id": "K-32", + "applies_to": "E-49", + "source": "NFR-8", + "statement": "`role` is \"roles/cloudkms.cryptoKeyEncrypterDecrypter\"" + }, + { + "statement": "`region` is \"europe-west2\"", + "source": "NFR-9", + "id": "K-33", + "applies_to": "E-11" + }, + { + "statement": "`region` is \"europe-west2\"", + "applies_to": "E-9", + "id": "K-34", + "source": "NFR-9" + }, + { + "statement": "`region` is \"europe-west1\"", + "applies_to": "E-12", + "id": "K-35", + "source": "NFR-9" + }, + { + "statement": "`constraint` is \"gcp.resourceLocations\"", + "id": "K-36", + "source": "NFR-9", + "applies_to": "E-37" + }, + { + "statement": "`list_policy.allow.values` includes \"in:europe-west2\" and \"in:europe-west1\"", + "applies_to": "E-37", + "id": "K-37", + "source": "NFR-9" + }, + { + "statement": "`project` is set", + "source": "NFR-10", + "id": "K-38", + "applies_to": "E-4" + }, + { + "source": "NFR-11", + "applies_to": "E-16", + "id": "K-39", + "statement": "`distribution_policy_zones` includes at least three zones" + }, + { + "statement": "`notification_channels` is populated", + "id": "K-40", + "applies_to": "E-32", + "source": "NFR-12" + }, + { + "statement": "`conditions.condition_threshold` or `conditions.condition_absent` is defined", + "applies_to": "E-32", + "id": "K-41", + "source": "NFR-12" + }, + { + "source": "NFR-13", + "id": "K-42", + "statement": "`amount.specified_amount.currency_code` is \"GBP\"", + "applies_to": "E-33" + }, + { + "id": "K-43", + "source": "NFR-13", + "applies_to": "E-33", + "statement": "`threshold_rules.threshold_percent` includes 0.5, 0.8, 0.9, and 1.0" + }, + { + "source": "NFR-13", + "id": "K-44", + "statement": "`amount.specified_amount.units` is 25000", + "applies_to": "E-33" + }, + { + "statement": "`bgp.advertise_mode` is \"CUSTOM\"", + "id": "K-45", + "applies_to": "E-11", + "source": "NFR-14" + }, + { + "statement": "`bgp.advertise_mode` is \"CUSTOM\"", + "id": "K-46", + "applies_to": "E-12", + "source": "NFR-14" + }, + { + "id": "K-47", + "statement": "`bandwidth` is \"10G\"", + "source": "NFR-14", + "applies_to": "E-13" + }, + { + "id": "K-48", + "source": "NFR-14", + "statement": "`bandwidth` is \"10G\"", + "applies_to": "E-14" + }, + { + "statement": "`list_policy.allow.values` enforces europe-west2 and europe-west1", + "source": "NFR-15", + "id": "K-49", + "applies_to": "E-37" + }, + { + "id": "K-50", + "applies_to": "E-38", + "statement": "`status.resources` includes the CDE and audit log projects", + "source": "DR-SEC-1" + }, + { + "source": "DR-SEC-1", + "applies_to": "E-38", + "statement": "`status.ingress_policies` and `status.egress_policies` are configured", + "id": "K-51" + }, + { + "statement": "`constraint` is \"compute.skipDefaultNetworkCreation\"", + "id": "K-52", + "applies_to": "E-34", + "source": "DR-SEC-2" + }, + { + "source": "DR-SEC-2", + "statement": "`boolean_policy.enforced` is true", + "id": "K-53", + "applies_to": "E-34" + }, + { + "applies_to": "E-35", + "id": "K-54", + "statement": "`constraint` is \"compute.vmExternalIpAccess\"", + "source": "DR-SEC-2" + }, + { + "statement": "`list_policy.deny.all` is true", + "applies_to": "E-35", + "id": "K-55", + "source": "DR-SEC-2" + }, + { + "id": "K-56", + "statement": "`constraint` is \"iam.disableServiceAccountKeyCreation\"", + "source": "DR-SEC-2", + "applies_to": "E-36" + }, + { + "statement": "`boolean_policy.enforced` is true", + "source": "DR-SEC-2", + "id": "K-57", + "applies_to": "E-36" + }, + { + "id": "K-58", + "statement": "`constraint` is \"gcp.resourceLocations\"", + "applies_to": "E-37", + "source": "DR-SEC-2" + }, + { + "id": "K-59", + "statement": "`list_policy.allow.values` restricts locations", + "source": "DR-SEC-2", + "applies_to": "E-37" + }, + { + "statement": "`confidential_instance_config.enable_confidential_compute` is true", + "source": "DR-SEC-3", + "id": "K-60", + "applies_to": "E-51" + }, + { + "id": "K-61", + "source": "DR-SEC-4", + "applies_to": "E-39", + "statement": "`inspect_config.info_types.name` includes \"CREDIT_CARD_NUMBER\"" + }, + { + "statement": "`event_data_content_type` is \"application/json\"", + "id": "K-62", + "source": "DR-SEC-6", + "applies_to": "E-40" + }, + { + "statement": "`matching_criteria.attribute` includes \"type\" matching SCC findings", + "id": "K-63", + "applies_to": "E-40", + "source": "DR-SEC-6" + }, + { + "source": "DR-SEC-6", + "id": "K-64", + "applies_to": "E-41", + "statement": "`event_trigger.event_type` matches Eventarc trigger" + }, + { + "applies_to": "E-13", + "statement": "`interconnect` is set to the primary colocation interconnect", + "id": "K-65", + "source": "DR-REL-1" + }, + { + "statement": "`interconnect` is set to the secondary colocation interconnect", + "applies_to": "E-14", + "id": "K-66", + "source": "DR-REL-1" + }, + { + "source": "DR-REL-1", + "statement": "`router` is set to the Primary Router", + "id": "K-67", + "applies_to": "E-13" + }, + { + "applies_to": "E-56", + "id": "K-68", + "source": "DR-REL-1", + "statement": "`bfd.session_initialization_mode` is \"ACTIVE\"" + }, + { + "source": "DR-REL-2", + "id": "K-69", + "statement": "`distribution_policy_zones` includes three zones", + "applies_to": "E-16" + }, + { + "id": "K-70", + "applies_to": "E-17", + "source": "DR-REL-2", + "statement": "`load_balancing_scheme` is \"INTERNAL\"" + }, + { + "applies_to": "E-18", + "statement": "`health_checks` includes the ILB Health Check", + "source": "DR-REL-2", + "id": "K-71" + }, + { + "source": "DR-REL-3", + "applies_to": "E-28", + "id": "K-72", + "statement": "`location` is \"europe-west2\"" + }, + { + "applies_to": "E-30", + "id": "K-73", + "statement": "`location` is \"europe-west1\"", + "source": "DR-REL-3" + }, + { + "source": "DR-REL-3", + "applies_to": "E-58", + "statement": "`role` is \"roles/cloudkms.cryptoKeyDecrypter\"", + "id": "K-74" + }, + { + "id": "K-75", + "statement": "`custom_placement_config.data_locations` includes \"EUROPE-WEST1\" and \"EUROPE-WEST2\"", + "applies_to": "E-21", + "source": "DR-REL-4" + }, + { + "statement": "`name` differs for primary and standby", + "id": "K-76", + "source": "DR-REL-5", + "applies_to": "E-42" + }, + { + "statement": "`name` differs for primary and standby", + "source": "DR-REL-5", + "id": "K-77", + "applies_to": "E-43" + }, + { + "statement": "`versioning.enabled` is true", + "id": "K-78", + "source": "DR-REL-5", + "applies_to": "E-42" + }, + { + "applies_to": "E-43", + "source": "DR-REL-5", + "id": "K-79", + "statement": "`versioning.enabled` is true" + }, + { + "source": "DR-OP-2", + "statement": "`goal` is 0.999", + "applies_to": "E-44", + "id": "K-80" + }, + { + "applies_to": "E-32", + "id": "K-81", + "source": "DR-OP-2", + "statement": "`conditions.condition_threshold` filters on SLO burn rate" + }, + { + "statement": "`service` is \"recommender.googleapis.com\"", + "applies_to": "E-45", + "id": "K-82", + "source": "DR-OP-3" + }, + { + "applies_to": "E-21", + "statement": "`lifecycle_rule.condition.age` is 30", + "id": "K-83", + "source": "DR-COST-2" + }, + { + "id": "K-84", + "applies_to": "E-21", + "source": "DR-COST-2", + "statement": "`lifecycle_rule.action.type` is \"SetStorageClass\" and `lifecycle_rule.action.storage_class` is \"NEARLINE\"" + }, + { + "source": "DR-COST-2", + "id": "K-85", + "applies_to": "E-21", + "statement": "`lifecycle_rule.condition.age` is 90" + }, + { + "applies_to": "E-21", + "statement": "`lifecycle_rule.action.type` is \"SetStorageClass\" and `lifecycle_rule.action.storage_class` is \"ARCHIVE\"", + "id": "K-86", + "source": "DR-COST-2" + }, + { + "applies_to": "E-8", + "statement": "`routing_mode` is \"REGIONAL\"", + "id": "K-87", + "source": "DR-PERF-1" + }, + { + "statement": "`autoscaling_policy.min_replicas` is 1", + "applies_to": "E-47", + "id": "K-88", + "source": "DR-SUST-1" + }, + { + "source": "DR-COST-1", + "statement": "`short_name` defines the taxonomy keys", + "id": "K-89", + "applies_to": "E-46" + }, + { + "applies_to": "E-54", + "source": "DR-COST-1", + "statement": "`destination` points to BigQuery", + "id": "K-90" + }, + { + "id": "K-91", + "applies_to": "E-55", + "source": "NFR-1", + "statement": "`tag_value` is set to the PCI regulated archetype" + }, + { + "id": "K-92", + "source": "DR-REL-2", + "statement": "`check_interval_sec` is set", + "applies_to": "E-19" + }, + { + "source": "NFR-5", + "applies_to": "E-23", + "id": "K-93", + "statement": "`disabled` is false" + }, + { + "applies_to": "E-26", + "source": "NFR-5", + "statement": "`disabled` is false", + "id": "K-94" + }, + { + "applies_to": "E-52", + "id": "K-95", + "statement": "`display_name` is set", + "source": "DR-OP-2" + }, + { + "applies_to": "E-53", + "id": "K-96", + "source": "DR-SUST-1", + "statement": "`base_instance_name` is set" + }, + { + "source": "DR-REL-1", + "applies_to": "E-57", + "statement": "`bfd.session_initialization_mode` is \"ACTIVE\"", + "id": "K-97" + }, + { + "source": "DR-REL-1", + "id": "K-99", + "applies_to": "E-56", + "statement": "`bfd.min_receive_interval` is 100" + }, + { + "id": "K-100", + "source": "DR-REL-1", + "applies_to": "E-57", + "statement": "`bfd.min_receive_interval` is 100" + }, + { + "source": "DR-REL-3", + "id": "K-101", + "statement": "`role` is \"roles/cloudkms.cryptoKeyDecrypter\"", + "applies_to": "E-59" + }, + { + "statement": "`tag_value` is set to the corp_workload archetype", + "applies_to": "E-60", + "source": "NFR-1", + "id": "K-102" + }, + { + "applies_to": "E-14", + "statement": "`router` is set to the Primary Router", + "id": "K-103", + "source": "NFR-3" + }, + { + "applies_to": "E-56", + "id": "K-104", + "statement": "`bfd.min_transmit_interval` is 100", + "source": "DR-REL-1" + }, + { + "source": "DR-REL-1", + "id": "K-105", + "applies_to": "E-57", + "statement": "`bfd.min_transmit_interval` is 100" + } + ], + "not_applicable": [ + { + "reason": "SCC activation and threat detection configuration are performed at the organization level outside of the foundational landing zone workspace.", + "recommendation": "DR-SEC-5" + }, + { + "reason": "Policy-as-code validation is defined in GitHub Actions CI/CD workflow YAML, which is outside the infrastructure terraform configuration.", + "recommendation": "DR-OP-1" + }, + { + "reason": "OpenTelemetry and Cloud Profiler are application-level agents installed within instance images or workloads, which are excluded from the foundational infrastructure deployment.", + "recommendation": "DR-PERF-2" + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/verdicts.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/verdicts.json new file mode 100644 index 0000000..0bef68d --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/verdicts.json @@ -0,0 +1,529 @@ +{ + "dry_run": { + "evidence": "Success! The configuration is valid.", + "verdict": "pass" + }, + "checks": [ + { + "check": "K-1", + "evidence": "display_name = \"Dev Folder\"", + "verdict": "pass" + }, + { + "check": "K-2", + "evidence": "display_name = \"Staging Folder\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-3", + "evidence": "display_name = \"Production Folder\"" + }, + { + "verdict": "pass", + "evidence": "tag_value = google_tags_tag_value.corp_workload.id", + "check": "K-4" + }, + { + "check": "K-5", + "verdict": "pass", + "evidence": "host_project = google_compute_shared_vpc_host_project.hub.project" + }, + { + "verdict": "pass", + "evidence": "host_project = google_compute_shared_vpc_host_project.hub.project", + "check": "K-6" + }, + { + "check": "K-7", + "evidence": "host_project = google_compute_shared_vpc_host_project.hub.project", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "private_ip_google_access = true", + "check": "K-8" + }, + { + "check": "K-9", + "evidence": "private_ip_google_access = true", + "verdict": "pass" + }, + { + "check": "K-10", + "evidence": "direction = \"INGRESS\"", + "verdict": "pass" + }, + { + "check": "K-11", + "verdict": "pass", + "evidence": "protocol = \"all\"" + }, + { + "verdict": "pass", + "evidence": "source_ranges = [\"0.0.0.0/0\"]", + "check": "K-12" + }, + { + "check": "K-13", + "evidence": "type = \"DEDICATED\"", + "verdict": "pass" + }, + { + "evidence": "type = \"DEDICATED\"", + "verdict": "pass", + "check": "K-14" + }, + { + "check": "K-15", + "verdict": "pass", + "evidence": "bandwidth = \"BPS_10G\"" + }, + { + "evidence": "bandwidth = \"BPS_10G\"", + "verdict": "pass", + "check": "K-16" + }, + { + "verdict": "pass", + "evidence": "router = google_compute_router.primary.id", + "check": "K-17" + }, + { + "evidence": "network = google_compute_network.hub.id", + "verdict": "pass", + "check": "K-18" + }, + { + "check": "K-19", + "evidence": "dest_range = \"0.0.0.0/0\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-20", + "evidence": "next_hop_ilb = google_compute_forwarding_rule.palo_alto_ilb.id" + }, + { + "verdict": "pass", + "check": "K-21", + "evidence": "issuer_uri = \"https://token.actions.githubusercontent.com\"" + }, + { + "verdict": "pass", + "check": "K-22", + "evidence": "member = \"principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.wif_pool.name}/attribute.repository/my-org/my-repo\"" + }, + { + "evidence": "idp_metadata_xml = \"\"", + "verdict": "pass", + "check": "K-23" + }, + { + "evidence": "purpose = \"ENCRYPT_DECRYPT\"", + "verdict": "pass", + "check": "K-24" + }, + { + "check": "K-25", + "evidence": "purpose = \"ENCRYPT_DECRYPT\"", + "verdict": "pass" + }, + { + "check": "K-26", + "verdict": "pass", + "evidence": "rotation_period = \"7776000s\"" + }, + { + "check": "K-27", + "evidence": "rotation_period = \"7776000s\"", + "verdict": "pass" + }, + { + "evidence": "destination = \"storage.googleapis.com/${google_storage_bucket.audit_logs.name}\"", + "verdict": "pass", + "check": "K-28" + }, + { + "check": "K-29", + "verdict": "pass", + "evidence": "retention_period = 31536000" + }, + { + "check": "K-30", + "verdict": "pass", + "evidence": "is_locked = true" + }, + { + "evidence": "restricted_services = [\"compute.googleapis.com\"]", + "verdict": "pass", + "check": "K-31" + }, + { + "check": "K-32", + "evidence": "role = \"roles/cloudkms.cryptoKeyEncrypterDecrypter\"", + "verdict": "pass" + }, + { + "check": "K-33", + "evidence": "region = \"europe-west2\"", + "verdict": "pass" + }, + { + "check": "K-34", + "verdict": "pass", + "evidence": "region = \"europe-west2\"" + }, + { + "verdict": "pass", + "check": "K-35", + "evidence": "region = \"europe-west1\"" + }, + { + "verdict": "pass", + "evidence": "constraint = \"gcp.resourceLocations\"", + "check": "K-36" + }, + { + "check": "K-37", + "verdict": "pass", + "evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]" + }, + { + "check": "K-38", + "evidence": "project = var.project_id", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "distribution_policy_zones = [\"europe-west2-a\", \"europe-west2-b\", \"europe-west2-c\"]", + "check": "K-39" + }, + { + "verdict": "pass", + "check": "K-40", + "evidence": "notification_channels = [google_monitoring_notification_channel.email.name]" + }, + { + "verdict": "pass", + "evidence": "condition_threshold is defined", + "check": "K-41" + }, + { + "evidence": "currency_code = \"GBP\"", + "check": "K-42", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "threshold_percent includes 0.5, 0.8, 0.9, 1.0", + "check": "K-43" + }, + { + "check": "K-44", + "verdict": "pass", + "evidence": "units = \"25000\"" + }, + { + "evidence": "advertise_mode = \"CUSTOM\"", + "verdict": "pass", + "check": "K-45" + }, + { + "verdict": "pass", + "evidence": "advertise_mode = \"CUSTOM\"", + "check": "K-46" + }, + { + "verdict": "pass", + "evidence": "bandwidth = \"BPS_10G\"", + "check": "K-47" + }, + { + "evidence": "bandwidth = \"BPS_10G\"", + "verdict": "pass", + "check": "K-48" + }, + { + "check": "K-49", + "verdict": "pass", + "evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]" + }, + { + "evidence": "resources = [\"projects/${var.prod_project_id}\", \"projects/${var.project_id}\"]", + "check": "K-50", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-51", + "evidence": "ingress_policies and egress_policies are configured" + }, + { + "verdict": "pass", + "evidence": "constraint = \"compute.skipDefaultNetworkCreation\"", + "check": "K-52" + }, + { + "evidence": "enforced = true", + "verdict": "pass", + "check": "K-53" + }, + { + "check": "K-54", + "evidence": "constraint = \"compute.vmExternalIpAccess\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "all = true", + "check": "K-55" + }, + { + "evidence": "constraint = \"iam.disableServiceAccountKeyCreation\"", + "check": "K-56", + "verdict": "pass" + }, + { + "check": "K-57", + "evidence": "enforced = true", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-58", + "evidence": "constraint = \"gcp.resourceLocations\"" + }, + { + "evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]", + "verdict": "pass", + "check": "K-59" + }, + { + "verdict": "pass", + "check": "K-60", + "evidence": "enable_confidential_compute = true" + }, + { + "check": "K-61", + "evidence": "name = \"CREDIT_CARD_NUMBER\"", + "verdict": "pass" + }, + { + "check": "K-62", + "evidence": "event_data_content_type = \"application/json\"", + "verdict": "pass" + }, + { + "evidence": "attribute = \"type\", value = \"google.cloud.securitycenter.finding.v1.saved\"", + "check": "K-63", + "verdict": "pass" + }, + { + "evidence": "event_type = \"google.cloud.securitycenter.finding.v1.saved\"", + "verdict": "pass", + "check": "K-64" + }, + { + "verdict": "pass", + "evidence": "interconnect = var.interconnect_1_url", + "check": "K-65" + }, + { + "evidence": "interconnect = var.interconnect_2_url", + "verdict": "pass", + "check": "K-66" + }, + { + "check": "K-67", + "evidence": "router = google_compute_router.primary.id", + "verdict": "pass" + }, + { + "check": "K-68", + "evidence": "session_initialization_mode = \"ACTIVE\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-69", + "evidence": "distribution_policy_zones = [\"europe-west2-a\", \"europe-west2-b\", \"europe-west2-c\"]" + }, + { + "evidence": "load_balancing_scheme = \"INTERNAL\"", + "check": "K-70", + "verdict": "pass" + }, + { + "evidence": "health_checks = [google_compute_health_check.palo_alto_ilb_hc.id]", + "verdict": "pass", + "check": "K-71" + }, + { + "verdict": "pass", + "check": "K-72", + "evidence": "location = \"europe-west2\"" + }, + { + "evidence": "location = \"europe-west1\"", + "check": "K-73", + "verdict": "pass" + }, + { + "evidence": "role = \"roles/cloudkms.cryptoKeyDecrypter\"", + "verdict": "pass", + "check": "K-74" + }, + { + "verdict": "pass", + "evidence": "data_locations = [\"EUROPE-WEST1\", \"EUROPE-WEST2\"]", + "check": "K-75" + }, + { + "check": "K-76", + "verdict": "pass", + "evidence": "name = \"${var.project_id}-tfstate-primary\"" + }, + { + "evidence": "name = \"${var.project_id}-tfstate-secondary\"", + "check": "K-77", + "verdict": "pass" + }, + { + "evidence": "enabled = true", + "check": "K-78", + "verdict": "pass" + }, + { + "check": "K-79", + "evidence": "enabled = true", + "verdict": "pass" + }, + { + "evidence": "goal = 0.999", + "verdict": "pass", + "check": "K-80" + }, + { + "evidence": "filter = \"select_slo_burn_rate(\\\"${google_monitoring_slo.platform_slo.name}\\\", 3600s)\"", + "verdict": "pass", + "check": "K-81" + }, + { + "verdict": "pass", + "check": "K-82", + "evidence": "service = \"recommender.googleapis.com\"" + }, + { + "verdict": "pass", + "evidence": "age = 30", + "check": "K-83" + }, + { + "verdict": "pass", + "check": "K-84", + "evidence": "type = \"SetStorageClass\", storage_class = \"NEARLINE\"" + }, + { + "check": "K-85", + "verdict": "pass", + "evidence": "age = 90" + }, + { + "check": "K-86", + "evidence": "type = \"SetStorageClass\", storage_class = \"ARCHIVE\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-87", + "evidence": "routing_mode = \"REGIONAL\"" + }, + { + "evidence": "min_replicas = 1", + "check": "K-88", + "verdict": "pass" + }, + { + "evidence": "short_name = \"environment\"", + "check": "K-89", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-90", + "evidence": "destination = \"bigquery.googleapis.com/...\"" + }, + { + "verdict": "pass", + "check": "K-91", + "evidence": "tag_value = google_tags_tag_value.pci_regulated.id" + }, + { + "check": "K-92", + "evidence": "check_interval_sec = 5", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-93", + "evidence": "disabled = false" + }, + { + "verdict": "pass", + "check": "K-94", + "evidence": "disabled = false" + }, + { + "evidence": "display_name = \"Platform Custom Service\"", + "check": "K-95", + "verdict": "pass" + }, + { + "check": "K-96", + "verdict": "pass", + "evidence": "base_instance_name = \"palo-alto-secondary\"" + }, + { + "verdict": "pass", + "evidence": "session_initialization_mode = \"ACTIVE\"", + "check": "K-97" + }, + { + "evidence": "min_receive_interval = 100", + "check": "K-99", + "verdict": "pass" + }, + { + "check": "K-100", + "verdict": "pass", + "evidence": "min_receive_interval = 100" + }, + { + "check": "K-101", + "evidence": "role = \"roles/cloudkms.cryptoKeyDecrypter\"", + "verdict": "pass" + }, + { + "evidence": "tag_value = google_tags_tag_value.corp_workload.id", + "verdict": "pass", + "check": "K-102" + }, + { + "verdict": "pass", + "evidence": "router = google_compute_router.primary.id", + "check": "K-103" + }, + { + "verdict": "pass", + "evidence": "min_transmit_interval = 100", + "check": "K-104" + }, + { + "check": "K-105", + "evidence": "min_transmit_interval = 100", + "verdict": "pass" + } + ], + "proposed_additions": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/work-breakdown.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/work-breakdown.json new file mode 100644 index 0000000..83069d7 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/001.sources/work-breakdown.json @@ -0,0 +1,332 @@ +{ + "work_packages": [ + { + "id": "WP-0", + "name": "Platform Foundation", + "parent": null, + "elements": [], + "scope": "The complete enterprise-grade landing zone.", + "produces": [ + "an operational landing zone" + ], + "skills": [ + "cloud architecture" + ] + }, + { + "id": "WP-1", + "name": "Automated Infrastructure Orchestration", + "parent": "WP-0", + "elements": [ + "E-42", + "E-43" + ], + "scope": "CI/CD orchestration, state management, and API enablement.", + "produces": [ + "Terraform state buckets for primary and secondary regions" + ], + "skills": [ + "cloud engineering" + ] + }, + { + "id": "WP-2", + "name": "Federated Identity and Role Governance", + "parent": "WP-0", + "elements": [ + "E-23", + "E-24", + "E-25", + "E-26", + "E-27", + "E-45" + ], + "scope": "Authentication for delivery pipelines and workforce SSO.", + "produces": [ + "Workload Identity Federation for GitHub Actions and Entra ID SSO" + ], + "skills": [ + "identity and access management" + ] + }, + { + "id": "WP-3", + "name": "Resource Hierarchy and Environment Boundary", + "parent": "WP-0", + "elements": [ + "E-1", + "E-2", + "E-3", + "E-4", + "E-5", + "E-6", + "E-7", + "E-8", + "E-9", + "E-10", + "E-34", + "E-35", + "E-36", + "E-37", + "E-38", + "E-50", + "E-51", + "E-55", + "E-60" + ], + "scope": "Google Cloud resource hierarchy, shared VPC projects, networking, and organization policies.", + "produces": [ + "Folders, projects, networking, and organization policies ensuring isolation" + ], + "skills": [ + "cloud engineering", + "network engineering" + ] + }, + { + "id": "WP-4", + "name": "Cryptographic Key Lifecycle", + "parent": "WP-0", + "elements": [ + "E-28", + "E-29", + "E-30", + "E-31", + "E-49", + "E-58", + "E-59" + ], + "scope": "Customer-managed encryption keys (CMEK) and cryptographic isolation.", + "produces": [ + "Primary and secondary KMS key rings and crypto keys with IAM bindings" + ], + "skills": [ + "security engineering" + ] + }, + { + "id": "WP-5", + "name": "Audit, Compliance, and Financial Governance", + "parent": "WP-0", + "elements": [ + "E-21", + "E-22", + "E-33", + "E-39", + "E-46", + "E-54" + ], + "scope": "Immutable compliance logs, DLP templates, and budget enforcement.", + "produces": [ + "Immutable audit log storage, billing budgets, and DLP inspection templates" + ], + "skills": [ + "cloud engineering", + "finops" + ] + }, + { + "id": "WP-6", + "name": "Centralised Outbound Traffic Inspection", + "parent": "WP-0", + "elements": [ + "E-15", + "E-16", + "E-17", + "E-18", + "E-19", + "E-20", + "E-47", + "E-48", + "E-53" + ], + "scope": "Egress inspection and threat prevention using Palo Alto VM-Series.", + "produces": [ + "Palo Alto VM-Series firewalls with internal load balancing and egress routing" + ], + "skills": [ + "network security engineering" + ] + }, + { + "id": "WP-7", + "name": "Dedicated Private Hybrid Connectivity", + "parent": "WP-0", + "elements": [ + "E-11", + "E-12", + "E-13", + "E-14", + "E-56", + "E-57" + ], + "scope": "Dedicated Interconnect termination and dynamic BGP routing.", + "produces": [ + "Primary and secondary Cloud Routers with Dedicated Interconnect VLAN attachments" + ], + "skills": [ + "network engineering" + ] + }, + { + "id": "WP-8", + "name": "Platform Telemetry and Operational Alerting", + "parent": "WP-0", + "elements": [ + "E-32", + "E-40", + "E-41", + "E-44", + "E-52" + ], + "scope": "SLO tracking, multi-burn-rate alerting, and SCC automated remediation.", + "produces": [ + "SLOs, alerting policies, and SCC automated remediation functions" + ], + "skills": [ + "site reliability engineering" + ] + } + ], + "activities": [ + { + "id": "A-1", + "work_package": "WP-1", + "name": "Enable Google Cloud APIs", + "days": 0.5, + "after": [] + }, + { + "id": "A-2", + "work_package": "WP-1", + "name": "Setup CI/CD Pipeline and State Buckets", + "days": 1, + "after": [ + "A-1" + ] + }, + { + "id": "A-3", + "work_package": "WP-2", + "name": "Configure Workload Identity and Entra ID SSO", + "days": 1.5, + "after": [ + "A-2" + ] + }, + { + "id": "A-4", + "work_package": "WP-3", + "name": "Provision Resource Hierarchy and Enforce Org Policies", + "days": 2.5, + "after": [ + "A-3" + ] + }, + { + "id": "A-5", + "work_package": "WP-3", + "name": "Provision Hub and Spoke VPC Networks", + "days": 2, + "after": [ + "A-4" + ] + }, + { + "id": "A-6", + "work_package": "WP-7", + "name": "Order and provision the Interconnect carrier circuits", + "days": 30, + "owner": "customer", + "assumption": "The customer manages the carrier's lead time and physical cross-connects.", + "after": [] + }, + { + "id": "A-7", + "work_package": "WP-4", + "name": "Configure Cloud KMS and Key Rings", + "days": 1.5, + "after": [ + "A-4" + ] + }, + { + "id": "A-8", + "work_package": "WP-5", + "name": "Set up Audit Logging, DLP, and Billing Budgets", + "days": 2, + "after": [ + "A-4" + ] + }, + { + "id": "A-9", + "work_package": "WP-6", + "name": "Deploy Palo Alto Egress Firewall Appliances", + "days": 5, + "after": [ + "A-5" + ] + }, + { + "id": "A-10", + "work_package": "WP-7", + "name": "Configure Cloud Routers and Terminate BGP over Interconnect", + "days": 3, + "after": [ + "A-5", + "A-6" + ] + }, + { + "id": "A-11", + "work_package": "WP-3", + "name": "Implement VPC Service Controls", + "days": 2, + "after": [ + "A-5", + "A-7", + "A-8" + ] + }, + { + "id": "A-12", + "work_package": "WP-8", + "name": "Deploy Platform Telemetry, SLOs, and SCC Auto-remediation", + "days": 2.5, + "after": [ + "A-8", + "A-11" + ] + }, + { + "id": "A-13", + "work_package": "WP-1", + "name": "Run validation in development environment", + "days": 1, + "after": [ + "A-9", + "A-10", + "A-12" + ] + }, + { + "id": "A-14", + "work_package": "WP-1", + "name": "Run validation in staging environment", + "days": 1, + "after": [ + "A-13" + ] + }, + { + "id": "A-15", + "work_package": "WP-1", + "name": "Run validation in production environment", + "days": 1, + "after": [ + "A-14" + ] + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.md b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.md new file mode 100644 index 0000000..93ad8ab --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.md @@ -0,0 +1,15 @@ +# Intent 2: case-08-landing-zone + +- Recorded: 2026-10-21T22:19:12+00:00 +- Execution: `case-02-from-architecture-r2` + +## What the customer wants + +We are a UK-regulated payment platform expanding into Google Cloud. We need an +enterprise-grade landing zone ready in under 10 weeks to support our core payments +engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have +existing on-premises systems that require private, low-latency connectivity over a +10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and +GitHub Actions for our delivery pipelines. Security requires that all egress +traffic flows through centralised inspection appliances before touching the public +internet, and customer data must be encrypted with keys we control. diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.scope.completed.yaml b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.scope.completed.yaml new file mode 100644 index 0000000..f3aeeb3 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.scope.completed.yaml @@ -0,0 +1,407 @@ +schema_version: 1 +metadata: + customer_case: case-08-landing-zone + client_name: ask + author: ask + revision: ask + source: architecture:case-08-landing-zone-185426 + status: BASELINE_LOCKED +engagement: + kind: platform_foundation +intent: + statement: 'We are a UK-regulated payment platform expanding into Google Cloud. + We need an + + enterprise-grade landing zone ready in under 10 weeks to support our core payments + + engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have + + existing on-premises systems that require private, low-latency connectivity over + a + + 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and + + GitHub Actions for our delivery pipelines. Security requires that all egress + + traffic flows through centralised inspection appliances before touching the public + + internet, and customer data must be encrypted with keys we control.' + requirements: ask + business_outcomes: + - outcome: Continuous, highly available, and low-latency operation of the core payments + platform foundation + target: 99.9% foundation availability with deterministic low latency + id: BO-1 + - id: BO-2 + target: Zero audit non-conformances + outcome: Continuous compliance with DORA and PCI-DSS v4.0 standards using customer-managed + encryption keys and centralised egress inspection + - outcome: Automated, secure infrastructure release pipelines enabling workload + ingestion within 10 weeks + target: Operational readiness achieved within 10 weeks + id: BO-3 + exclusions: + - Core payment application workload code maintenance and runtime application management + - On-premises infrastructure hardware maintenance beyond Cloud Interconnect physical + termination +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Resource Hierarchy and Environment Boundary Manager + kind: governance + product: Resource Manager + source_ref: C-1 + elements: + - E-1 + - E-2 + - E-3 + - E-4 + - E-5 + - E-6 + - E-7 + - E-8 + - E-9 + - E-10 + - E-34 + - E-35 + - E-36 + - E-37 + - E-38 + - E-50 + - E-51 + - E-55 + - E-60 + - id: S-2 + name: Dedicated Private Hybrid Connectivity Gateway + kind: network + product: Cloud Interconnect + source_ref: C-2 + elements: + - E-11 + - E-12 + - E-13 + - E-14 + - E-56 + - E-57 + - id: S-3 + name: Centralised Outbound Traffic Inspection and Egress Boundary + kind: service + product: Palo Alto Networks VM-Series + source_ref: C-3 + elements: + - E-15 + - E-16 + - E-17 + - E-18 + - E-19 + - E-20 + - E-47 + - E-48 + - E-53 + - id: S-4 + name: Centralised Immutable Audit Event and Compliance Log Repository + kind: storage + product: Cloud Storage + source_ref: C-4 + elements: + - E-21 + - E-22 + - E-39 + - id: S-5 + name: Automated Infrastructure Orchestration and Deployment Engine + kind: service + product: GitHub Actions + source_ref: C-5 + elements: + - E-42 + - E-43 + - id: S-6 + name: Federated Identity and Role Governance Authority + kind: identity + product: IAM Workload Identity Federation + source_ref: C-6 + elements: + - E-23 + - E-24 + - E-25 + - E-26 + - E-27 + - E-45 + - id: S-7 + name: Cryptographic Key Lifecycle and Encryption Controller + kind: keys + product: Cloud KMS + source_ref: C-7 + elements: + - E-28 + - E-29 + - E-30 + - E-31 + - E-49 + - E-58 + - E-59 + - id: S-8 + name: Platform Telemetry and Operational Alerting Engine + kind: observability + product: Cloud Monitoring + source_ref: C-8 + elements: + - E-32 + - E-40 + - E-41 + - E-44 + - E-52 + - id: S-9 + name: Financial Governance and Budget Enforcement Monitor + kind: billing + product: Cloud Billing + source_ref: C-9 + elements: + - E-33 + - E-46 + - E-54 + conditions: + - id: K-1 + 'on': + - S-1 + - S-4 + - S-7 + condition: The landing zone configuration aligns with DORA operational resiliency + guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. + source_ref: NFR-8 + - id: K-2 + 'on': + - S-1 + condition: The secondary region in europe-west1 operates as a warm standby ready + to receive failover traffic. + source_ref: NFR-9 + - id: K-3 + 'on': + - S-1 + condition: Data residency is guaranteed within the United Kingdom and European + Union. + source_ref: NFR-9 + - id: K-4 + 'on': + - S-5 + condition: Platform foundation is fully provisioned and validated for core payments + engine workload ingestion within 10 weeks. + source_ref: NFR-10 + - id: K-5 + 'on': + - S-8 + condition: Platform foundation infrastructure maintains measured monthly availability + of at least 99.9%. + source_ref: NFR-11 + - id: K-6 + 'on': + - S-8 + condition: Disaster recovery activation achieves a recovery time objective (RTO) + of 4 hours. + source_ref: NFR-11 + - id: K-7 + 'on': + - S-8 + condition: Disaster recovery processes achieve a recovery point objective (RPO) + of 1 hour. + source_ref: NFR-11 + - id: K-8 + 'on': + - S-8 + condition: Alert notifications trigger operational incident workflows for the + 24/7 central platform operations team. + source_ref: NFR-12 + - id: K-9 + 'on': + - S-9 + condition: Monthly cloud foundation expenditure does not exceed the target allocation + without operational sign-off. + source_ref: NFR-13 + - id: K-10 + 'on': + - S-2 + condition: Hybrid connection provides deterministic low latency suitable for real-time + payment transaction processing. + source_ref: NFR-14 + - id: K-11 + 'on': + - S-8 + condition: Deployments optimize carbon-free energy utilization based on Google + Cloud regional CFE scores. + source_ref: NFR-15 + - id: K-12 + 'on': + - S-6 + condition: Permissions are continuously rightsized based on IAM Recommender findings. + source_ref: DR-OP-3 + build: + duration_days: 36 + customer_owned: + - Order and provision the Interconnect carrier circuits + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Changes to the platform foundation are deployed through automated + GitHub Actions pipelines with mandatory peer reviews and automated policy validation, + without a separate manual change advisory board. + monitoring: ask +sourcing: + model: provider_operated + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: + - name: Central Platform Operations Team + hours: 24x7 + party: provider + headcount_limit: ask +service_management: + tooling: Google Cloud operations suite (Cloud Monitoring, Cloud Logging, and Cloud + Alerting) without a separate external ITSM system. +service_levels: + priority_1_response_minutes: 30 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA + - ISO27001 + - PCI_DSS_v4 +commercial: + currency: GBP + monthly_run_budget: ask +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.business_outcomes + statement: 'Assumed intent.business_outcomes is [{''outcome'': ''Continuous, highly + available, and low-latency operation of the core payments platform foundation'', + ''target'': ''99.9% foundation availability with deterministic low latency'', + ''id'': ''BO-1''}, {''id'': ''BO-2'', ''target'': ''Zero audit non-conformances'', + ''outcome'': ''Continuous compliance with DORA and PCI-DSS v4.0 standards using + customer-managed encryption keys and centralised egress inspection''}, {''outcome'': + ''Automated, secure infrastructure release pipelines enabling workload ingestion + within 10 weeks'', ''target'': ''Operational readiness achieved within 10 weeks'', + ''id'': ''BO-3''}]. Derived from workflow request objectives: running core payment + processing reliably in Google Cloud, meeting DORA and PCI-DSS v4.0 regulatory + standards, and delivering landing zone readiness within 10 weeks.' + impact_if_invalid: Changes what is measured, and what each measure is for. +- id: ASSUMP-02 + field: intent.exclusions + statement: Assumed intent.exclusions is ['Core payment application workload code + maintenance and runtime application management', 'On-premises infrastructure hardware + maintenance beyond Cloud Interconnect physical termination']. Platform foundation + scope excludes application-level core payment workloads and on-premises data centre + hardware beyond the Interconnect boundary. + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-03 + field: operations.change_process + statement: 'Assumed operations.change_process is Changes to the platform foundation + are deployed through automated GitHub Actions pipelines with mandatory peer reviews + and automated policy validation, without a separate manual change advisory board.. + Catalogue default aligned with request stating GitHub Actions delivery pipelines: + automated pipeline deployment with peer review and automated checks.' + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-04 + field: organisation.existing_teams + statement: 'Assumed organisation.existing_teams is [{''name'': ''Central Platform + Operations Team'', ''hours'': ''24x7'', ''party'': ''provider''}]. Catalogue default: + all operational roles are newly provided under provider_operated model, with reference + to the 24/7 central platform operations team in condition K-8.' + impact_if_invalid: Changes which Roles are new and which are filled. +- id: ASSUMP-05 + field: service_management.tooling + statement: 'Assumed service_management.tooling is Google Cloud operations suite + (Cloud Monitoring, Cloud Logging, and Cloud Alerting) without a separate external + ITSM system.. Catalogue default: Cloud provider native operations tooling (Cloud + Monitoring, Cloud Logging) integrated with alerting.' + impact_if_invalid: Changes tooling and the processes written for it. +- id: ASSUMP-06 + field: service_levels.priority_1_response_minutes + statement: Assumed service_levels.priority_1_response_minutes is 30. Catalogue default + response time for Priority 1 critical incidents under 24x7 operational support. + impact_if_invalid: Changes on-call arrangements and staffing. +- id: ASSUMP-07 + field: service_levels.recovery_time_hours + statement: Assumed service_levels.recovery_time_hours is 4. Catalogue default aligned + with condition K-6 specifying a 4-hour recovery time objective (RTO). + impact_if_invalid: Changes continuity arrangements. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: intent.requirements + prompt: Which requirements for operations has the customer stated outright? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: commercial.monthly_run_budget + prompt: Is there a monthly budget for operating this, in that currency? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: digested + commercial.monthly_run_budget: unknown + compliance.frameworks: digested + engagement.kind: digested + intent.business_outcomes: assumed + intent.exclusions: assumed + intent.requirements: unknown + intent.statement: digested + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: digested + metadata.revision: unknown + metadata.source: digested + operations.availability_hours: digested + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: digested + organisation.existing_teams: assumed + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: assumed + service_levels.recovery_time_hours: assumed + service_management.tooling: assumed + solution.build.customer_owned: digested + solution.build.duration_days: digested + solution.cloud_provider: digested + solution.conditions: digested + solution.inherited_risks: unknown + solution.items: digested + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 3 + answered: 0 + assumed: 7 + unknown: 10 + not_applicable: 0 + digested: 13 diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.scope.yaml new file mode 100644 index 0000000..9533549 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.scope.yaml @@ -0,0 +1,10 @@ +intent: + business_outcomes: decide +run: + pre_approved: true +sourcing: + delivery_countries: + - GB + - PL + - IN + model: provider_operated diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/components.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/components.json new file mode 100644 index 0000000..61eddc2 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/components.json @@ -0,0 +1,79 @@ +[ + { + "id": "C-1", + "name": "Resource Hierarchy and Environment Boundary Manager", + "requirements": [ + "FR-1", + "NFR-1", + "NFR-8", + "NFR-9" + ] + }, + { + "id": "C-2", + "name": "Dedicated Private Hybrid Connectivity Gateway", + "requirements": [ + "FR-2", + "NFR-3", + "NFR-14" + ] + }, + { + "id": "C-3", + "name": "Centralised Outbound Traffic Inspection and Egress Boundary", + "requirements": [ + "FR-3", + "NFR-2", + "NFR-4" + ] + }, + { + "id": "C-4", + "name": "Centralised Immutable Audit Event and Compliance Log Repository", + "requirements": [ + "FR-4", + "NFR-7", + "NFR-8" + ] + }, + { + "id": "C-5", + "name": "Automated Infrastructure Orchestration and Deployment Engine", + "requirements": [ + "FR-5", + "NFR-10" + ] + }, + { + "id": "C-6", + "name": "Federated Identity and Role Governance Authority", + "requirements": [ + "FR-6", + "NFR-5" + ] + }, + { + "id": "C-7", + "name": "Cryptographic Key Lifecycle and Encryption Controller", + "requirements": [ + "NFR-6", + "NFR-8" + ] + }, + { + "id": "C-8", + "name": "Platform Telemetry and Operational Alerting Engine", + "requirements": [ + "NFR-11", + "NFR-12", + "NFR-15" + ] + }, + { + "id": "C-9", + "name": "Financial Governance and Budget Enforcement Monitor", + "requirements": [ + "NFR-13" + ] + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/execution.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/execution.json new file mode 100644 index 0000000..126ac0c --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/execution.json @@ -0,0 +1,140 @@ +{ + "customer_case": "case-08-landing-zone", + "execution_id": "case-08-landing-zone-185426", + "session_id": "cdd6bfe7-2bd8-44af-8e15-9dcfa869eb84", + "intent_version": 1, + "workflow_request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.", + "baseline": null, + "overlay_skills": {}, + "terraform_modules": false, + "scope": { + "status": "BASELINE_LOCKED", + "readiness": { + "supplied": 44, + "answered": 0, + "assumed": 12, + "unknown": 4, + "not_applicable": 3 + } + }, + "delivery_plan": { + "work_packages": 9, + "activities": 15, + "working_days": 36.0, + "weeks": 7.2, + "team_size": 3, + "window_weeks": 10, + "within_window": true, + "critical_path": [ + "A-6", + "A-10", + "A-13", + "A-14", + "A-15" + ] + }, + "status": "completed", + "warnings": [], + "risks": [], + "assessment": { + "recommendations_quality": { + "specific": 0.91, + "actionable": 0.79, + "trade_offs": 0.95, + "traceable": 0.73 + }, + "mean": 0.84, + "meets_bar": true + }, + "phases": { + "scoping": "approved", + "requirements_discovery": "approved", + "solution_architecture": "approved", + "solution_implementation": "approved", + "solution_validation": "approved", + "packaging": "approved" + }, + "solution_guide": "solution-architecture-guide.md", + "customer_questions": "customer-questions.md", + "changes": null, + "deliverables": { + "scope": { + "document": "deliverables/scope.md", + "attachments": [ + "scope.json", + "scope.yaml" + ], + "status": "submitted" + }, + "requirements": { + "document": "deliverables/requirements.md", + "attachments": [ + "requirements.json" + ], + "status": "submitted" + }, + "technical_decomposition": { + "document": "deliverables/technical_decomposition.md", + "attachments": [ + "components.json" + ], + "status": "submitted" + }, + "product_selection": { + "document": "deliverables/product_selection.md", + "attachments": [ + "products.json" + ], + "status": "submitted" + }, + "design_recommendations": { + "document": "deliverables/design_recommendations.md", + "attachments": [ + "recommendations.json" + ], + "status": "submitted" + }, + "target_architecture": { + "document": "deliverables/target_architecture.md", + "attachments": [ + "architecture.mmd", + "target-architecture.json" + ], + "status": "submitted" + }, + "architecture_description": { + "document": "deliverables/architecture_description.md", + "attachments": [], + "status": "submitted" + }, + "deployment_guidance": { + "document": "deliverables/deployment_guidance.md", + "attachments": [ + "implementation.json", + "terraform/main.tf", + "terraform/variables.tf", + "terraform/versions.tf" + ], + "status": "submitted" + }, + "validation_results": { + "document": "deliverables/validation_results.md", + "attachments": [ + "verdicts.json" + ], + "status": "submitted" + }, + "work_breakdown": { + "document": "deliverables/work_breakdown.md", + "attachments": [ + "work-breakdown.json" + ], + "status": "submitted" + }, + "solution_guide": { + "document": "deliverables/solution_guide.md", + "attachments": [], + "status": "submitted" + } + } +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/products.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/products.json new file mode 100644 index 0000000..ca9e033 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/products.json @@ -0,0 +1,38 @@ +[ + { + "component": "C-1", + "product": "Resource Manager" + }, + { + "component": "C-2", + "product": "Cloud Interconnect" + }, + { + "component": "C-3", + "product": "Palo Alto Networks VM-Series" + }, + { + "component": "C-4", + "product": "Cloud Storage" + }, + { + "component": "C-5", + "product": "GitHub Actions" + }, + { + "component": "C-6", + "product": "IAM Workload Identity Federation" + }, + { + "component": "C-7", + "product": "Cloud KMS" + }, + { + "component": "C-8", + "product": "Cloud Monitoring" + }, + { + "component": "C-9", + "product": "Cloud Billing" + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/recommendations.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/recommendations.json new file mode 100644 index 0000000..3721e66 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/recommendations.json @@ -0,0 +1,248 @@ +[ + { + "id": "DR-SEC-1", + "pillar": "security", + "text": "Implement a VPC Service Controls perimeter around all Cardholder Data Environment (CDE) projects and the centralized audit log buckets.", + "components": [ + "C-1", + "C-4" + ], + "conditions": [ + "A VPC Service Controls perimeter encloses the CDE and audit log projects.", + "The VPC Service Controls perimeter restricts ingress and egress to authorized traffic." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-2", + "pillar": "security", + "text": "Apply Organization Policy constraints to disable default network creation, deny external IPs, prohibit service account key creation, and restrict resource locations.", + "components": [ + "C-1" + ], + "conditions": [ + "The compute.skipDefaultNetworkCreation constraint is enforced.", + "The compute.vmExternalIpAccess constraint is enforced to deny all.", + "The iam.disableServiceAccountKeyCreation constraint is enforced.", + "The gcp.resourceLocations constraint restricts resources to europe-west2 and europe-west1." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-3", + "pillar": "security", + "text": "Enable Confidential Computing on the payment engine Compute Engine instances to encrypt data in-use.", + "components": [], + "conditions": [ + "Confidential Computing is enabled on the core payment engine Compute Engine instances." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-4", + "pillar": "security", + "text": "Deploy Sensitive Data Protection (Cloud DLP) inspection templates to detect and mask PAN data.", + "components": [ + "C-4" + ], + "conditions": [ + "Sensitive Data Protection (Cloud DLP) inspection templates are configured to detect and mask PAN data." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-5", + "pillar": "security", + "text": "Enable Security Command Center at the organization level with Event and Container Threat Detection.", + "components": [ + "C-1", + "C-8" + ], + "conditions": [ + "Security Command Center is activated at the organization level.", + "Event Threat Detection and Container Threat Detection are enabled in SCC." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-6", + "pillar": "security", + "text": "Implement event-driven automated responses using Eventarc and Cloud Run functions for SCC findings.", + "components": [ + "C-8" + ], + "conditions": [ + "Eventarc is configured to route high-severity SCC findings.", + "Cloud Run functions are configured to execute automated remediation on Eventarc triggers." + ], + "outside_code": [] + }, + { + "id": "DR-REL-1", + "pillar": "reliability", + "text": "Provision redundant Dedicated Cloud Interconnect circuits across distinct physical edge colocation facilities, terminating on redundant Cloud Routers with BFD.", + "components": [ + "C-2" + ], + "conditions": [ + "Dedicated Cloud Interconnect circuits are provisioned across redundant edge colocation facilities.", + "Cloud Interconnects terminate on redundant Cloud Routers in each region.", + "Bidirectional Forwarding Detection (BFD) is enabled on the Cloud Router BGP sessions." + ], + "outside_code": [] + }, + { + "id": "DR-REL-2", + "pillar": "reliability", + "text": "Deploy Palo Alto VM-Series firewalls in a regional Managed Instance Group across at least three zones, fronted by an Internal Passthrough Network Load Balancer.", + "components": [ + "C-3" + ], + "conditions": [ + "Palo Alto VM-Series firewalls are deployed in a regional Managed Instance Group (MIG) across at least three zones.", + "An Internal Passthrough Network Load Balancer routes traffic to the VM-Series MIG.", + "Health checks are configured on the internal load balancer to probe firewall health." + ], + "outside_code": [] + }, + { + "id": "DR-REL-3", + "pillar": "reliability", + "text": "Provision dedicated regional Cloud KMS key rings in both europe-west2 and europe-west1, granting workload identity service accounts Decrypter roles on both.", + "components": [ + "C-7", + "C-6" + ], + "conditions": [ + "Regional Cloud KMS key rings are provisioned in both europe-west2 and europe-west1.", + "Workload Identity Federation service accounts are granted Decrypter roles on both regional key sets." + ], + "outside_code": [] + }, + { + "id": "DR-REL-4", + "pillar": "reliability", + "text": "Configure the Cloud Storage buckets for audit logging as Dual-Region spanning europe-west2 and europe-west1.", + "components": [ + "C-4" + ], + "conditions": [ + "Audit log Cloud Storage buckets are configured as Dual-Region spanning europe-west2 and europe-west1." + ], + "outside_code": [] + }, + { + "id": "DR-REL-5", + "pillar": "reliability", + "text": "Isolate the Terraform state of the primary region from the standby region using multi-region Cloud Storage buckets with Object Versioning.", + "components": [ + "C-5" + ], + "conditions": [ + "Terraform state for primary and standby regions is decoupled into separate failure domains.", + "Terraform state is stored in multi-region Cloud Storage buckets with Object Versioning enabled." + ], + "outside_code": [] + }, + { + "id": "DR-OP-1", + "pillar": "operational-excellence", + "text": "Integrate Policy-as-Code validation into the GitHub Actions pull request workflows.", + "components": [ + "C-5" + ], + "conditions": [ + "GitHub Actions workflows include automated Policy-as-Code validation steps prior to deployment execution." + ], + "outside_code": [] + }, + { + "id": "DR-OP-2", + "pillar": "operational-excellence", + "text": "Establish explicit Service Level Indicators (SLIs) and configure multi-burn-rate alerting policies in Cloud Monitoring.", + "components": [ + "C-8" + ], + "conditions": [ + "Service Level Objectives (SLOs) and SLIs are defined in Cloud Monitoring Service Monitoring.", + "Multi-burn-rate alerting policies are configured based on the SLO error budget." + ], + "outside_code": [] + }, + { + "id": "DR-OP-3", + "pillar": "operational-excellence", + "text": "Activate the IAM Recommender to continuously evaluate role bindings.", + "components": [ + "C-6" + ], + "conditions": [ + "IAM Recommender is activated on the foundation projects." + ], + "outside_code": [ + "Permissions are continuously rightsized based on IAM Recommender findings." + ] + }, + { + "id": "DR-COST-1", + "pillar": "cost-optimization", + "text": "Enforce a mandatory labeling taxonomy through Organization Policies and export granular billing data to BigQuery.", + "components": [ + "C-9", + "C-1" + ], + "conditions": [ + "Cloud Billing export to BigQuery is enabled.", + "A mandatory labeling taxonomy is enforced via Organization Policy Service tags." + ], + "outside_code": [] + }, + { + "id": "DR-COST-2", + "pillar": "cost-optimization", + "text": "Implement Cloud Storage Object Lifecycle Management to transition logs to Nearline storage after 30 days and to Coldline or Archive storage after 90 days.", + "components": [ + "C-4" + ], + "conditions": [ + "Object Lifecycle Management rules transition Cloud Storage objects to Nearline storage after 30 days.", + "Object Lifecycle Management rules transition Cloud Storage objects to Coldline or Archive storage after 90 days." + ], + "outside_code": [] + }, + { + "id": "DR-PERF-1", + "pillar": "performance-optimization", + "text": "Configure Cloud Routers with regional dynamic routing to localize egress traffic within the warm standby region.", + "components": [ + "C-2" + ], + "conditions": [ + "Cloud Routers are configured with regional dynamic routing mode." + ], + "outside_code": [] + }, + { + "id": "DR-PERF-2", + "pillar": "performance-optimization", + "text": "Instrument the deployment with OpenTelemetry exporting to Cloud Trace, and use Cloud Profiler on the compute instances.", + "components": [ + "C-8" + ], + "conditions": [ + "OpenTelemetry tracing is configured to export to Cloud Trace.", + "Cloud Profiler agents are enabled on the core payments engine instances." + ], + "outside_code": [] + }, + { + "id": "DR-SUST-1", + "pillar": "sustainability", + "text": "Implement a Pilot Light warm standby by maintaining active data replication while scaling the standby compute instances to a minimal baseline.", + "components": [], + "conditions": [ + "Secondary region compute instances are scaled to a minimal baseline (Pilot Light) configuration during normal operations." + ], + "outside_code": [] + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/requirements.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/requirements.json new file mode 100644 index 0000000..307613b --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/requirements.json @@ -0,0 +1,280 @@ +[ + { + "id": "FR-1", + "kind": "functional", + "text": "The platform foundation isolates development, staging, and production environments across distinct Google Cloud projects and resource folders.", + "from": [ + "REQ-02" + ] + }, + { + "id": "FR-2", + "kind": "functional", + "text": "The platform foundation establishes private bidirectional network connectivity between Google Cloud and on-premises core payment systems over Cloud Interconnect.", + "from": [ + "REQ-03" + ] + }, + { + "id": "FR-3", + "kind": "functional", + "text": "The platform foundation routes all outbound internet-bound traffic from workload spoke networks through centralised network security inspection appliances.", + "from": [ + "REQ-03" + ] + }, + { + "id": "FR-4", + "kind": "functional", + "text": "The platform foundation captures and exports all administrative and data access audit logs across environments into a centralised, tamper-resistant log repository.", + "from": [ + "REQ-02" + ] + }, + { + "id": "FR-5", + "kind": "functional", + "text": "The platform foundation provisions and manages all cloud infrastructure as code via automated CI/CD pipelines executing validated pull request workflows.", + "from": [ + "REQ-01" + ] + }, + { + "id": "FR-6", + "kind": "functional", + "text": "The platform foundation federates corporate identity for single sign-on and enforces least-privilege role-based access control for platform administrators and application squads.", + "from": [ + "REQ-04" + ] + }, + { + "id": "NFR-1", + "kind": "non_functional", + "text": "Environments are partitioned into development, staging, and production tiers, enforcing corp_workload controls on lower environments and pci_regulated_workload controls on production.", + "conditions": [ + "Separate projects or folders exist for development, staging, and production environment tiers.", + "Production environments enforce pci_regulated_workload archetype security controls.", + "Development and staging environments enforce corp_workload archetype security controls." + ], + "outside_code": [], + "from": [ + "REQ-02", + "governance.environment_tiers" + ] + }, + { + "id": "NFR-2", + "kind": "non_functional", + "text": "Network connectivity is structured as a Shared VPC hub-and-spoke topology with zero direct internet ingress to workload subnets.", + "conditions": [ + "A Shared VPC host project is connected to service projects across environment tiers.", + "Workload subnets have private Google access enabled and no external public IP addresses assigned.", + "Ingress firewall rules deny uninspected public internet traffic to workload subnets." + ], + "outside_code": [], + "from": [ + "REQ-02", + "networking.topology", + "networking.exposure" + ] + }, + { + "id": "NFR-3", + "kind": "non_functional", + "text": "Hybrid connectivity to on-premises core payment systems is enabled via a 10Gbps Dedicated Cloud Interconnect.", + "conditions": [ + "Dedicated Cloud Interconnect connections are configured for hybrid connectivity.", + "Cloud Interconnect capacity is provisioned at 10Gbps bandwidth.", + "Interconnect VLAN attachments are associated with Cloud Routers in the Shared VPC hub." + ], + "outside_code": [], + "from": [ + "REQ-03", + "networking.hybrid_connectivity.enabled", + "networking.hybrid_connectivity.type", + "networking.hybrid_connectivity.bandwidth_gbps" + ] + }, + { + "id": "NFR-4", + "kind": "non_functional", + "text": "All outbound internet traffic from workload spokes is forced through centralised inspection appliances deployed in the hub network.", + "conditions": [ + "Inspection appliance instances are deployed in a centralised hub network.", + "Custom VPC route tables direct 0.0.0.0/0 egress traffic from spokes to the centralised inspection appliance instances or internal load balancer." + ], + "outside_code": [], + "from": [ + "REQ-03", + "networking.egress_control.model" + ] + }, + { + "id": "NFR-5", + "kind": "non_functional", + "text": "Administrative access is authenticated via Entra ID single sign-on, and delivery pipelines authenticate via Workload Identity Federation without stored service account keys.", + "conditions": [ + "Workload Identity Federation pools and providers are configured for GitHub Actions authentication.", + "IAM bindings grant roles to federated GitHub Actions identities without downloadable service account keys.", + "Corporate single sign-on is configured with Entra ID as the identity source of truth." + ], + "outside_code": [], + "from": [ + "REQ-04", + "governance.identity_source_of_truth", + "governance.pipeline_authentication" + ] + }, + { + "id": "NFR-6", + "kind": "non_functional", + "text": "Customer confidential data is encrypted at rest using customer-managed encryption keys with a 90-day rotation schedule.", + "conditions": [ + "Cloud KMS customer-managed encryption keys are configured for infrastructure resources storing confidential data.", + "Cloud KMS keys have an automated rotation period set to 90 days (7776000s)." + ], + "outside_code": [], + "from": [ + "REQ-04", + "compliance.encryption.type", + "compliance.encryption.key_rotation_days" + ] + }, + { + "id": "NFR-7", + "kind": "non_functional", + "text": "Audit logs across all environments are retained for 365 days in a storage bucket protected by an immutable retention policy.", + "conditions": [ + "An organization-level or project-level log sink exports audit logs to a centralised Cloud Storage bucket.", + "The centralised audit log storage bucket specifies a retention period of 365 days (31536000 seconds).", + "The centralised audit log storage bucket has retention policy locking enabled to prevent modification or early deletion." + ], + "outside_code": [], + "from": [ + "REQ-02", + "compliance.audit_logging.retention_days", + "compliance.audit_logging.immutable_retention_policy" + ] + }, + { + "id": "NFR-8", + "kind": "non_functional", + "text": "The platform foundation enforces DORA, ISO27001, and PCI-DSS v4.0 regulatory baselines and protects confidential data classification.", + "conditions": [ + "Organization policy constraints enforce perimeter isolation and restrict unauthenticated resource sharing.", + "IAM policies restrict access to confidential data resources to authorized service identities and roles." + ], + "outside_code": [ + "The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit." + ], + "from": [ + "REQ-02", + "compliance.frameworks", + "compliance.data_classification" + ] + }, + { + "id": "NFR-9", + "kind": "non_functional", + "text": "Platform foundation resources are provisioned across europe-west2 (primary) and europe-west1 (secondary) in a warm standby model, restricting data residency to the UK and EU.", + "conditions": [ + "Primary landing zone network and foundation resources are provisioned in europe-west2.", + "Secondary landing zone network and foundation standby resources are provisioned in europe-west1.", + "Resource location organization policies restrict resource and data storage locations to europe-west2 and europe-west1." + ], + "outside_code": [ + "The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic.", + "Data residency is guaranteed within the United Kingdom and European Union." + ], + "from": [ + "REQ-02", + "estate.regions.primary", + "estate.regions.secondary", + "estate.regions.resiliency_tier", + "estate.data_residency" + ] + }, + { + "id": "NFR-10", + "kind": "non_functional", + "text": "The platform foundation landing zone is delivered and operational for workload ingestion within 10 weeks.", + "conditions": [ + "All landing zone foundation resources are codified in modular Terraform configurations." + ], + "outside_code": [ + "Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks." + ], + "from": [ + "REQ-01" + ] + }, + { + "id": "NFR-11", + "kind": "non_functional", + "text": "The platform foundation maintains 99.9% availability with a 4-hour recovery time objective (RTO) and a 1-hour recovery point objective (RPO).", + "conditions": [ + "Regional networking and core management services are deployed across at least three zones in the primary region." + ], + "outside_code": [ + "Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.", + "Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.", + "Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour." + ], + "from": [ + "REQ-02" + ] + }, + { + "id": "NFR-12", + "kind": "non_functional", + "text": "Platform operations monitoring and alerting are implemented using Google Cloud Operations Suite with 24/7 alerting capabilities.", + "conditions": [ + "Cloud Monitoring workspaces and notification channels are configured for the platform foundation.", + "Alert policies are configured for Interconnect link degradation, appliance health, and egress routing failures." + ], + "outside_code": [ + "Alert notifications trigger operational incident workflows for the 24/7 central platform operations team." + ], + "from": [] + }, + { + "id": "NFR-13", + "kind": "non_functional", + "text": "Infrastructure costs are tracked against a monthly budget of 25,000 GBP with automated alerting.", + "conditions": [ + "A Cloud Billing budget is configured for the foundation projects with currency set to GBP.", + "Budget alert thresholds are configured at 50%, 80%, 90%, and 100% of the 25,000 GBP monthly budget." + ], + "outside_code": [ + "Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off." + ], + "from": [] + }, + { + "id": "NFR-14", + "kind": "non_functional", + "text": "Hybrid networking provides dedicated 10Gbps unthrottled throughput with minimal latency between Google Cloud and on-premises core systems.", + "conditions": [ + "Cloud Routers configured for Interconnect advertise non-overlapping RFC 1918 BGP routes.", + "Dedicated Interconnect VLAN attachments are configured with dedicated 10Gbps bandwidth allocation." + ], + "outside_code": [ + "Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing." + ], + "from": [ + "REQ-03" + ] + }, + { + "id": "NFR-15", + "kind": "non_functional", + "text": "Landing zone resources are deployed in Google Cloud regions with strong low-carbon energy ratings.", + "conditions": [ + "Foundation resources are constrained to europe-west2 (London) and europe-west1 (Belgium) regions." + ], + "outside_code": [ + "Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores." + ], + "from": [] + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/scope.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/scope.json new file mode 100644 index 0000000..16e782d --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/scope.json @@ -0,0 +1,233 @@ +{ + "kind": "platform_foundation", + "stated": { + "engagement": { + "kind": "platform_foundation" + }, + "intent": { + "statement": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.\n", + "requirements": [ + { + "id": "REQ-01", + "category": "Timeline", + "statement": "Platform foundation must be ready for workload ingestion within 10 weeks." + }, + { + "id": "REQ-02", + "category": "Compliance", + "statement": "Must achieve DORA resiliency alignment and PCI-DSS v4.0 perimeter isolation." + }, + { + "id": "REQ-03", + "category": "Networking", + "statement": "10Gbps on-premises hybrid connectivity; centralised inspection appliances for egress." + }, + { + "id": "REQ-04", + "category": "Identity and security", + "statement": "Workload identity federation with GitHub Actions; Entra ID single sign-on; customer-managed encryption keys." + } + ], + "exclusions": [ + "Application container deployment; workloads are managed by application squads.", + "An active-active secondary region in phase 1; warm standby is acceptable." + ] + }, + "estate": { + "cloud_provider": "gcp", + "mode": "greenfield", + "regions": { + "primary": "europe-west2", + "secondary": "europe-west1", + "resiliency_tier": "warm_standby" + }, + "data_residency": "United Kingdom and European Union", + "existing_systems": "On-premises payment systems that stay in place and connect over the interconnect." + }, + "workload": { + "availability": { + "hours": "always" + } + }, + "compliance": { + "industry": "financial_services", + "frameworks": [ + "DORA", + "ISO27001", + "PCI_DSS_v4" + ], + "data_classification": "confidential", + "encryption": { + "type": "cmek", + "key_rotation_days": 90 + }, + "audit_logging": { + "retention_days": 365, + "immutable_retention_policy": true + } + }, + "networking": { + "exposure": "internal_only", + "topology": "shared_vpc_hub_and_spoke", + "hybrid_connectivity": { + "enabled": true, + "type": "cloud_interconnect", + "bandwidth_gbps": 10 + }, + "egress_control": { + "model": "centralized_appliance" + } + }, + "governance": { + "identity_source_of_truth": "Entra ID", + "pipeline_authentication": "workload identity federation", + "environment_tiers": [ + { + "name": "development", + "archetype": "corp_workload" + }, + { + "name": "staging", + "archetype": "corp_workload" + }, + { + "name": "production", + "archetype": "pci_regulated_workload" + } + ] + }, + "operations": { + "operating_model": "central platform team brokering access for application squads", + "support_hours": "always" + }, + "delivery": { + "window_weeks": 10, + "commercial_model": "fixed_price_milestone", + "iac_tooling": "terraform", + "ci_cd": "GitHub Actions", + "team_size": 3, + "working_days_per_week": 5 + }, + "cost": { + "currency": "GBP" + } + }, + "binding": [ + "estate.regions.primary", + "estate.regions.secondary", + "estate.regions.resiliency_tier", + "estate.data_residency", + "compliance.frameworks", + "compliance.data_classification", + "compliance.encryption.type", + "compliance.encryption.key_rotation_days", + "compliance.audit_logging.retention_days", + "compliance.audit_logging.immutable_retention_policy", + "networking.exposure", + "networking.topology", + "networking.hybrid_connectivity.enabled", + "networking.hybrid_connectivity.type", + "networking.hybrid_connectivity.bandwidth_gbps", + "networking.egress_control.model", + "governance.identity_source_of_truth", + "governance.pipeline_authentication", + "governance.environment_tiers" + ], + "assumed": [ + { + "field": "intent.success_criteria", + "value": [ + "Platform foundation ready for workload ingestion within 10 weeks", + "Compliance achieved with DORA and PCI-DSS v4.0 requirements", + "Hybrid connectivity established via 10Gbps Dedicated Interconnect", + "Centralised egress inspection operational", + "Identity federation and CMEK security controls implemented" + ], + "why": "Assumed standard success criteria based on delivery within 10 weeks and compliance with DORA and PCI-DSS v4.0." + }, + { + "field": "workload.availability.target", + "value": "99.9%", + "why": "Assumed 99.9% availability target for platform foundation infrastructure aligned with warm standby secondary region." + }, + { + "field": "workload.availability.recovery_time", + "value": "4 hours", + "why": "Assumed 4 hours recovery time objective (RTO) for platform foundation services in warm standby." + }, + { + "field": "workload.availability.recovery_point", + "value": "1 hour", + "why": "Assumed 1 hour recovery point objective (RPO) for platform foundation configuration and state." + }, + { + "field": "networking.egress_control.appliance_vendor", + "value": "Palo Alto Networks VM-Series", + "why": "Assumed Palo Alto Networks VM-Series as enterprise inspection appliance for centralized egress control." + }, + { + "field": "networking.cidr_allocations", + "value": "RFC 1918 non-overlapping IP allocation (10.0.0.0/16 partitioned across environments) sized for growth", + "why": "Assumed RFC 1918 non-overlapping IP allocation (10.0.0.0/16 partitioned across environments) sized for growth." + }, + { + "field": "operations.monitoring", + "value": "Google Cloud Operations Suite with availability and error alerting", + "why": "Assumed native Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging) with availability and error alerting." + }, + { + "field": "operations.change_process", + "value": "GitHub Actions pull request workflow with automated Terraform plan validation and peer review", + "why": "Assumed GitHub Actions pull request workflow with automated Terraform plan validation and peer approval." + }, + { + "field": "integrations", + "value": [ + { + "criticality": "critical", + "direction": "bidirectional", + "name": "On-premises core payment systems" + } + ], + "why": "Assumed integration with existing on-premises core payment systems via Dedicated Interconnect." + }, + { + "field": "cost.monthly_budget", + "value": 25000, + "why": "Assumed indicative monthly foundation infrastructure budget of 25000 GBP for 10Gbps interconnect, inspection appliances, and core services." + }, + { + "field": "constraints.products_required", + "value": [], + "why": "No mandated third-party products beyond stated tooling (GCP, Terraform, GitHub Actions, Entra ID)." + }, + { + "field": "constraints.products_forbidden", + "value": [], + "why": "No forbidden products specified." + } + ], + "to_decide": [], + "to_ask": [ + { + "field": "metadata.engagement_reference", + "prompt": "What is metadata.engagement_reference?", + "severity": "LOW" + }, + { + "field": "delivery.target_start_date", + "prompt": "When should work start?", + "severity": "LOW" + }, + { + "field": "constraints.other", + "prompt": "What is constraints.other?", + "severity": "LOW" + } + ], + "not_applicable": [ + "workload.users.count", + "workload.users.peak_concurrent", + "workload.users.who" + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/target-architecture.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/target-architecture.json new file mode 100644 index 0000000..3a35acc --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/target-architecture.json @@ -0,0 +1,1210 @@ +{ + "elements": [ + { + "resource_type": "google_folder", + "product": "Resource Manager", + "component": "C-1", + "name": "Dev Folder", + "id": "E-1" + }, + { + "resource_type": "google_folder", + "id": "E-2", + "product": "Resource Manager", + "component": "C-1", + "name": "Staging Folder" + }, + { + "component": "C-1", + "name": "Production Folder", + "id": "E-3", + "resource_type": "google_folder", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_shared_vpc_host_project", + "name": "Shared VPC Host Project", + "product": "Resource Manager", + "id": "E-4", + "component": "C-1" + }, + { + "product": "Resource Manager", + "component": "C-1", + "resource_type": "google_compute_shared_vpc_service_project", + "name": "Dev Spoke Project", + "id": "E-5" + }, + { + "name": "Stg Spoke Project", + "product": "Resource Manager", + "component": "C-1", + "resource_type": "google_compute_shared_vpc_service_project", + "id": "E-6" + }, + { + "product": "Resource Manager", + "id": "E-7", + "name": "Prod Spoke Project", + "component": "C-1", + "resource_type": "google_compute_shared_vpc_service_project" + }, + { + "product": "Resource Manager", + "name": "Hub VPC Network", + "component": "C-1", + "id": "E-8", + "resource_type": "google_compute_network" + }, + { + "id": "E-9", + "resource_type": "google_compute_subnetwork", + "name": "Dev Subnetwork", + "component": "C-1", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_subnetwork", + "component": "C-1", + "id": "E-10", + "name": "Prod Subnetwork", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_router", + "id": "E-11", + "name": "Primary Cloud Router", + "component": "C-2", + "product": "Cloud Interconnect" + }, + { + "resource_type": "google_compute_router", + "name": "Secondary Cloud Router", + "id": "E-12", + "component": "C-2", + "product": "Cloud Interconnect" + }, + { + "product": "Cloud Interconnect", + "id": "E-13", + "name": "Primary VLAN Attachment 1", + "resource_type": "google_compute_interconnect_attachment", + "component": "C-2" + }, + { + "id": "E-14", + "component": "C-2", + "resource_type": "google_compute_interconnect_attachment", + "name": "Primary VLAN Attachment 2", + "product": "Cloud Interconnect" + }, + { + "name": "Palo Alto Instance Template", + "component": "C-3", + "id": "E-15", + "product": "Palo Alto Networks VM-Series", + "resource_type": "google_compute_instance_template" + }, + { + "id": "E-16", + "name": "Palo Alto MIG Primary", + "resource_type": "google_compute_region_instance_group_manager", + "component": "C-3", + "product": "Palo Alto Networks VM-Series" + }, + { + "id": "E-17", + "component": "C-3", + "resource_type": "google_compute_forwarding_rule", + "name": "Palo Alto ILB", + "product": "Palo Alto Networks VM-Series" + }, + { + "name": "Palo Alto ILB Backend", + "resource_type": "google_compute_region_backend_service", + "component": "C-3", + "id": "E-18", + "product": "Palo Alto Networks VM-Series" + }, + { + "resource_type": "google_compute_health_check", + "component": "C-3", + "name": "Palo Alto ILB Health Check", + "product": "Palo Alto Networks VM-Series", + "id": "E-19" + }, + { + "component": "C-3", + "id": "E-20", + "name": "Egress Route to ILB", + "resource_type": "google_compute_route", + "product": "Palo Alto Networks VM-Series" + }, + { + "id": "E-21", + "component": "C-4", + "name": "Audit Logs Storage Bucket", + "product": "Cloud Storage", + "resource_type": "google_storage_bucket" + }, + { + "component": "C-4", + "product": "Cloud Storage", + "id": "E-22", + "name": "Audit Log Sink", + "resource_type": "google_logging_project_sink" + }, + { + "product": "IAM Workload Identity Federation", + "id": "E-23", + "component": "C-6", + "name": "WIF Pool", + "resource_type": "google_iam_workload_identity_pool" + }, + { + "name": "WIF Provider GitHub", + "product": "IAM Workload Identity Federation", + "id": "E-24", + "resource_type": "google_iam_workload_identity_pool_provider", + "component": "C-6" + }, + { + "component": "C-6", + "resource_type": "google_iam_workforce_pool_provider", + "name": "WIF Provider Entra ID", + "product": "IAM Workload Identity Federation", + "id": "E-25" + }, + { + "product": "IAM Workload Identity Federation", + "name": "Pipeline Service Account", + "component": "C-6", + "id": "E-26", + "resource_type": "google_service_account" + }, + { + "name": "Pipeline SA WIF Binding", + "resource_type": "google_service_account_iam_member", + "id": "E-27", + "product": "IAM Workload Identity Federation", + "component": "C-6" + }, + { + "product": "Cloud KMS", + "resource_type": "google_kms_key_ring", + "name": "Primary KMS Key Ring", + "id": "E-28", + "component": "C-7" + }, + { + "product": "Cloud KMS", + "resource_type": "google_kms_crypto_key", + "component": "C-7", + "name": "Primary KMS Crypto Key", + "id": "E-29" + }, + { + "name": "Secondary KMS Key Ring", + "id": "E-30", + "product": "Cloud KMS", + "component": "C-7", + "resource_type": "google_kms_key_ring" + }, + { + "product": "Cloud KMS", + "name": "Secondary KMS Crypto Key", + "component": "C-7", + "resource_type": "google_kms_crypto_key", + "id": "E-31" + }, + { + "resource_type": "google_monitoring_alert_policy", + "name": "Platform Alert Policy", + "product": "Cloud Monitoring", + "component": "C-8", + "id": "E-32" + }, + { + "product": "Cloud Billing", + "name": "Foundation Cloud Budget", + "component": "C-9", + "resource_type": "google_billing_budget", + "id": "E-33" + }, + { + "id": "E-34", + "name": "Org Policy skipDefaultNetworkCreation", + "resource_type": "google_project_organization_policy", + "component": "C-1", + "product": "Resource Manager" + }, + { + "component": "C-1", + "id": "E-35", + "name": "Org Policy vmExternalIpAccess", + "resource_type": "google_project_organization_policy", + "product": "Resource Manager" + }, + { + "resource_type": "google_project_organization_policy", + "name": "Org Policy disableServiceAccountKeyCreation", + "id": "E-36", + "product": "Resource Manager", + "component": "C-1" + }, + { + "resource_type": "google_project_organization_policy", + "product": "Resource Manager", + "component": "C-1", + "name": "Org Policy resourceLocations", + "id": "E-37" + }, + { + "name": "VPC SC Perimeter", + "component": "C-1", + "product": "Resource Manager", + "resource_type": "google_access_context_manager_service_perimeter", + "id": "E-38" + }, + { + "name": "DLP Inspect Template", + "component": "C-4", + "resource_type": "google_data_loss_prevention_inspect_template", + "product": "Cloud Storage", + "id": "E-39" + }, + { + "id": "E-40", + "product": "Cloud Monitoring", + "name": "SCC Eventarc Trigger", + "resource_type": "google_eventarc_trigger", + "component": "C-8" + }, + { + "product": "Cloud Monitoring", + "resource_type": "google_cloudfunctions2_function", + "component": "C-8", + "name": "SCC Remediation Function", + "id": "E-41" + }, + { + "component": "C-5", + "id": "E-42", + "name": "Primary TF State Bucket", + "resource_type": "google_storage_bucket", + "product": "GitHub Actions" + }, + { + "resource_type": "google_storage_bucket", + "id": "E-43", + "product": "GitHub Actions", + "component": "C-5", + "name": "Secondary TF State Bucket" + }, + { + "product": "Cloud Monitoring", + "component": "C-8", + "resource_type": "google_monitoring_slo", + "name": "Platform SLO", + "id": "E-44" + }, + { + "id": "E-45", + "component": "C-6", + "resource_type": "google_project_service", + "name": "IAM Recommender Service", + "product": "IAM Workload Identity Federation" + }, + { + "resource_type": "google_tags_tag_key", + "name": "Taxonomy Tag Key", + "component": "C-9", + "product": "Cloud Billing", + "id": "E-46" + }, + { + "product": "Palo Alto Networks VM-Series", + "component": "C-3", + "resource_type": "google_compute_region_autoscaler", + "name": "Standby MIG Autoscaler", + "id": "E-47" + }, + { + "name": "Deny Ingress Firewall", + "resource_type": "google_compute_firewall", + "component": "C-3", + "product": "Palo Alto Networks VM-Series", + "id": "E-48" + }, + { + "component": "C-7", + "id": "E-49", + "name": "KMS IAM Binding", + "resource_type": "google_kms_crypto_key_iam_member", + "product": "Cloud KMS" + }, + { + "id": "E-50", + "component": "C-1", + "product": "Resource Manager", + "resource_type": "google_tags_tag_binding", + "name": "Environment Tag Binding" + }, + { + "name": "Payment Engine Template", + "product": "Resource Manager", + "id": "E-51", + "resource_type": "google_compute_instance_template", + "component": "C-1" + }, + { + "name": "Platform Custom Service", + "component": "C-8", + "product": "Cloud Monitoring", + "id": "E-52", + "resource_type": "google_monitoring_custom_service" + }, + { + "name": "Palo Alto MIG Secondary", + "component": "C-3", + "resource_type": "google_compute_region_instance_group_manager", + "product": "Palo Alto Networks VM-Series", + "id": "E-53" + }, + { + "product": "Cloud Billing", + "resource_type": "google_logging_project_sink", + "name": "Billing BigQuery Sink", + "component": "C-9", + "id": "E-54" + }, + { + "name": "Prod Tag Binding", + "id": "E-55", + "resource_type": "google_tags_tag_binding", + "component": "C-1", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_router_peer", + "id": "E-56", + "component": "C-2", + "name": "Primary BGP Peer 1", + "product": "Cloud Interconnect" + }, + { + "resource_type": "google_compute_router_peer", + "id": "E-57", + "component": "C-2", + "name": "Primary BGP Peer 2", + "product": "Cloud Interconnect" + }, + { + "resource_type": "google_kms_crypto_key_iam_member", + "component": "C-7", + "name": "Secondary KMS Decrypter Binding", + "id": "E-58", + "product": "Cloud KMS" + }, + { + "component": "C-7", + "name": "Primary KMS Decrypter Binding", + "product": "Cloud KMS", + "resource_type": "google_kms_crypto_key_iam_member", + "id": "E-59" + }, + { + "id": "E-60", + "resource_type": "google_tags_tag_binding", + "name": "Staging Tag Binding", + "component": "C-1", + "product": "Resource Manager" + } + ], + "relationships": [ + { + "from": "E-5", + "id": "R-1", + "kind": "uses", + "to": "E-4" + }, + { + "id": "R-2", + "kind": "uses", + "to": "E-4", + "from": "E-6" + }, + { + "kind": "uses", + "to": "E-4", + "id": "R-3", + "from": "E-7" + }, + { + "kind": "attached to", + "to": "E-11", + "from": "E-13", + "id": "R-4" + }, + { + "from": "E-14", + "to": "E-11", + "id": "R-5", + "kind": "attached to" + }, + { + "kind": "instantiates", + "from": "E-16", + "id": "R-6", + "to": "E-15" + }, + { + "to": "E-15", + "from": "E-53", + "id": "R-6b", + "kind": "instantiates" + }, + { + "to": "E-17", + "from": "E-20", + "id": "R-7", + "kind": "routes to" + }, + { + "from": "E-17", + "id": "R-8", + "kind": "routes to", + "to": "E-18" + }, + { + "id": "R-9", + "kind": "checked by", + "to": "E-19", + "from": "E-18" + }, + { + "from": "E-22", + "id": "R-10", + "kind": "exports to", + "to": "E-21" + }, + { + "kind": "authenticates via", + "id": "R-11", + "from": "E-24", + "to": "E-23" + }, + { + "from": "E-27", + "to": "E-26", + "id": "R-12", + "kind": "granted to" + }, + { + "from": "E-29", + "to": "E-28", + "id": "R-13", + "kind": "contained in" + }, + { + "from": "E-31", + "kind": "contained in", + "id": "R-14", + "to": "E-30" + }, + { + "kind": "granted on", + "id": "R-15", + "from": "E-49", + "to": "E-29" + }, + { + "id": "R-16", + "kind": "monitors", + "to": "E-52", + "from": "E-44" + }, + { + "from": "E-47", + "kind": "scales", + "id": "R-17", + "to": "E-53" + }, + { + "id": "R-18", + "to": "E-46", + "from": "E-50", + "kind": "binds" + }, + { + "to": "E-11", + "from": "E-56", + "id": "R-19", + "kind": "contained in" + }, + { + "from": "E-57", + "kind": "contained in", + "to": "E-11", + "id": "R-20" + }, + { + "to": "E-31", + "kind": "granted on", + "from": "E-58", + "id": "R-21" + }, + { + "to": "E-29", + "from": "E-59", + "kind": "granted on", + "id": "R-22" + }, + { + "to": "E-46", + "kind": "binds", + "id": "R-23", + "from": "E-60" + } + ], + "checks": [ + { + "id": "K-1", + "source": "NFR-1", + "applies_to": "E-1", + "statement": "`display_name` is set" + }, + { + "applies_to": "E-2", + "source": "NFR-1", + "statement": "`display_name` is set", + "id": "K-2" + }, + { + "applies_to": "E-3", + "source": "NFR-1", + "statement": "`display_name` is set", + "id": "K-3" + }, + { + "statement": "`tag_value` is set to the environment archetype", + "applies_to": "E-50", + "source": "NFR-1", + "id": "K-4" + }, + { + "applies_to": "E-5", + "id": "K-5", + "statement": "`host_project` is set to the hub project", + "source": "NFR-2" + }, + { + "source": "NFR-2", + "applies_to": "E-6", + "statement": "`host_project` is set to the hub project", + "id": "K-6" + }, + { + "id": "K-7", + "source": "NFR-2", + "applies_to": "E-7", + "statement": "`host_project` is set to the hub project" + }, + { + "id": "K-8", + "applies_to": "E-9", + "statement": "`private_ip_google_access` is true", + "source": "NFR-2" + }, + { + "statement": "`private_ip_google_access` is true", + "applies_to": "E-10", + "source": "NFR-2", + "id": "K-9" + }, + { + "statement": "`direction` is \"INGRESS\"", + "source": "NFR-2", + "applies_to": "E-48", + "id": "K-10" + }, + { + "id": "K-11", + "source": "NFR-2", + "statement": "`deny.protocol` is \"all\"", + "applies_to": "E-48" + }, + { + "statement": "`source_ranges` includes \"0.0.0.0/0\"", + "id": "K-12", + "applies_to": "E-48", + "source": "NFR-2" + }, + { + "applies_to": "E-13", + "id": "K-13", + "statement": "`type` is \"DEDICATED\"", + "source": "NFR-3" + }, + { + "statement": "`type` is \"DEDICATED\"", + "id": "K-14", + "source": "NFR-3", + "applies_to": "E-14" + }, + { + "source": "NFR-3", + "applies_to": "E-13", + "id": "K-15", + "statement": "`bandwidth` is \"10G\"" + }, + { + "applies_to": "E-14", + "statement": "`bandwidth` is \"10G\"", + "id": "K-16", + "source": "NFR-3" + }, + { + "id": "K-17", + "source": "NFR-3", + "statement": "`router` is set to the Primary Router", + "applies_to": "E-13" + }, + { + "id": "K-18", + "source": "NFR-4", + "statement": "`network_interface.network` is set to the hub VPC", + "applies_to": "E-15" + }, + { + "source": "NFR-4", + "applies_to": "E-20", + "id": "K-19", + "statement": "`dest_range` is \"0.0.0.0/0\"" + }, + { + "statement": "`next_hop_ilb` is set to the ILB forwarding rule", + "applies_to": "E-20", + "source": "NFR-4", + "id": "K-20" + }, + { + "applies_to": "E-24", + "statement": "`oidc.issuer_uri` is \"https://token.actions.githubusercontent.com\"", + "source": "NFR-5", + "id": "K-21" + }, + { + "id": "K-22", + "statement": "`member` includes \"principalSet://iam.googleapis.com/\"", + "applies_to": "E-27", + "source": "NFR-5" + }, + { + "id": "K-23", + "applies_to": "E-25", + "statement": "`saml.idp_metadata_xml` is populated", + "source": "NFR-5" + }, + { + "applies_to": "E-29", + "id": "K-24", + "source": "NFR-6", + "statement": "`purpose` is \"ENCRYPT_DECRYPT\"" + }, + { + "applies_to": "E-31", + "source": "NFR-6", + "id": "K-25", + "statement": "`purpose` is \"ENCRYPT_DECRYPT\"" + }, + { + "applies_to": "E-29", + "source": "NFR-6", + "id": "K-26", + "statement": "`rotation_period` is \"7776000s\"" + }, + { + "applies_to": "E-31", + "source": "NFR-6", + "id": "K-27", + "statement": "`rotation_period` is \"7776000s\"" + }, + { + "statement": "`destination` is set to the storage bucket", + "applies_to": "E-22", + "id": "K-28", + "source": "NFR-7" + }, + { + "applies_to": "E-21", + "source": "NFR-7", + "statement": "`retention_policy.retention_period` is 31536000", + "id": "K-29" + }, + { + "statement": "`retention_policy.is_locked` is true", + "source": "NFR-7", + "applies_to": "E-21", + "id": "K-30" + }, + { + "id": "K-31", + "applies_to": "E-38", + "source": "NFR-8", + "statement": "`status.restricted_services` includes \"compute.googleapis.com\"" + }, + { + "id": "K-32", + "applies_to": "E-49", + "source": "NFR-8", + "statement": "`role` is \"roles/cloudkms.cryptoKeyEncrypterDecrypter\"" + }, + { + "statement": "`region` is \"europe-west2\"", + "source": "NFR-9", + "id": "K-33", + "applies_to": "E-11" + }, + { + "statement": "`region` is \"europe-west2\"", + "applies_to": "E-9", + "id": "K-34", + "source": "NFR-9" + }, + { + "statement": "`region` is \"europe-west1\"", + "applies_to": "E-12", + "id": "K-35", + "source": "NFR-9" + }, + { + "statement": "`constraint` is \"gcp.resourceLocations\"", + "id": "K-36", + "source": "NFR-9", + "applies_to": "E-37" + }, + { + "statement": "`list_policy.allow.values` includes \"in:europe-west2\" and \"in:europe-west1\"", + "applies_to": "E-37", + "id": "K-37", + "source": "NFR-9" + }, + { + "statement": "`project` is set", + "source": "NFR-10", + "id": "K-38", + "applies_to": "E-4" + }, + { + "source": "NFR-11", + "applies_to": "E-16", + "id": "K-39", + "statement": "`distribution_policy_zones` includes at least three zones" + }, + { + "statement": "`notification_channels` is populated", + "id": "K-40", + "applies_to": "E-32", + "source": "NFR-12" + }, + { + "statement": "`conditions.condition_threshold` or `conditions.condition_absent` is defined", + "applies_to": "E-32", + "id": "K-41", + "source": "NFR-12" + }, + { + "source": "NFR-13", + "id": "K-42", + "statement": "`amount.specified_amount.currency_code` is \"GBP\"", + "applies_to": "E-33" + }, + { + "id": "K-43", + "source": "NFR-13", + "applies_to": "E-33", + "statement": "`threshold_rules.threshold_percent` includes 0.5, 0.8, 0.9, and 1.0" + }, + { + "source": "NFR-13", + "id": "K-44", + "statement": "`amount.specified_amount.units` is 25000", + "applies_to": "E-33" + }, + { + "statement": "`bgp.advertise_mode` is \"CUSTOM\"", + "id": "K-45", + "applies_to": "E-11", + "source": "NFR-14" + }, + { + "statement": "`bgp.advertise_mode` is \"CUSTOM\"", + "id": "K-46", + "applies_to": "E-12", + "source": "NFR-14" + }, + { + "id": "K-47", + "statement": "`bandwidth` is \"10G\"", + "source": "NFR-14", + "applies_to": "E-13" + }, + { + "id": "K-48", + "source": "NFR-14", + "statement": "`bandwidth` is \"10G\"", + "applies_to": "E-14" + }, + { + "statement": "`list_policy.allow.values` enforces europe-west2 and europe-west1", + "source": "NFR-15", + "id": "K-49", + "applies_to": "E-37" + }, + { + "id": "K-50", + "applies_to": "E-38", + "statement": "`status.resources` includes the CDE and audit log projects", + "source": "DR-SEC-1" + }, + { + "source": "DR-SEC-1", + "applies_to": "E-38", + "statement": "`status.ingress_policies` and `status.egress_policies` are configured", + "id": "K-51" + }, + { + "statement": "`constraint` is \"compute.skipDefaultNetworkCreation\"", + "id": "K-52", + "applies_to": "E-34", + "source": "DR-SEC-2" + }, + { + "source": "DR-SEC-2", + "statement": "`boolean_policy.enforced` is true", + "id": "K-53", + "applies_to": "E-34" + }, + { + "applies_to": "E-35", + "id": "K-54", + "statement": "`constraint` is \"compute.vmExternalIpAccess\"", + "source": "DR-SEC-2" + }, + { + "statement": "`list_policy.deny.all` is true", + "applies_to": "E-35", + "id": "K-55", + "source": "DR-SEC-2" + }, + { + "id": "K-56", + "statement": "`constraint` is \"iam.disableServiceAccountKeyCreation\"", + "source": "DR-SEC-2", + "applies_to": "E-36" + }, + { + "statement": "`boolean_policy.enforced` is true", + "source": "DR-SEC-2", + "id": "K-57", + "applies_to": "E-36" + }, + { + "id": "K-58", + "statement": "`constraint` is \"gcp.resourceLocations\"", + "applies_to": "E-37", + "source": "DR-SEC-2" + }, + { + "id": "K-59", + "statement": "`list_policy.allow.values` restricts locations", + "source": "DR-SEC-2", + "applies_to": "E-37" + }, + { + "statement": "`confidential_instance_config.enable_confidential_compute` is true", + "source": "DR-SEC-3", + "id": "K-60", + "applies_to": "E-51" + }, + { + "id": "K-61", + "source": "DR-SEC-4", + "applies_to": "E-39", + "statement": "`inspect_config.info_types.name` includes \"CREDIT_CARD_NUMBER\"" + }, + { + "statement": "`event_data_content_type` is \"application/json\"", + "id": "K-62", + "source": "DR-SEC-6", + "applies_to": "E-40" + }, + { + "statement": "`matching_criteria.attribute` includes \"type\" matching SCC findings", + "id": "K-63", + "applies_to": "E-40", + "source": "DR-SEC-6" + }, + { + "source": "DR-SEC-6", + "id": "K-64", + "applies_to": "E-41", + "statement": "`event_trigger.event_type` matches Eventarc trigger" + }, + { + "applies_to": "E-13", + "statement": "`interconnect` is set to the primary colocation interconnect", + "id": "K-65", + "source": "DR-REL-1" + }, + { + "statement": "`interconnect` is set to the secondary colocation interconnect", + "applies_to": "E-14", + "id": "K-66", + "source": "DR-REL-1" + }, + { + "source": "DR-REL-1", + "statement": "`router` is set to the Primary Router", + "id": "K-67", + "applies_to": "E-13" + }, + { + "applies_to": "E-56", + "id": "K-68", + "source": "DR-REL-1", + "statement": "`bfd.session_initialization_mode` is \"ACTIVE\"" + }, + { + "source": "DR-REL-2", + "id": "K-69", + "statement": "`distribution_policy_zones` includes three zones", + "applies_to": "E-16" + }, + { + "id": "K-70", + "applies_to": "E-17", + "source": "DR-REL-2", + "statement": "`load_balancing_scheme` is \"INTERNAL\"" + }, + { + "applies_to": "E-18", + "statement": "`health_checks` includes the ILB Health Check", + "source": "DR-REL-2", + "id": "K-71" + }, + { + "source": "DR-REL-3", + "applies_to": "E-28", + "id": "K-72", + "statement": "`location` is \"europe-west2\"" + }, + { + "applies_to": "E-30", + "id": "K-73", + "statement": "`location` is \"europe-west1\"", + "source": "DR-REL-3" + }, + { + "source": "DR-REL-3", + "applies_to": "E-58", + "statement": "`role` is \"roles/cloudkms.cryptoKeyDecrypter\"", + "id": "K-74" + }, + { + "id": "K-75", + "statement": "`custom_placement_config.data_locations` includes \"EUROPE-WEST1\" and \"EUROPE-WEST2\"", + "applies_to": "E-21", + "source": "DR-REL-4" + }, + { + "statement": "`name` differs for primary and standby", + "id": "K-76", + "source": "DR-REL-5", + "applies_to": "E-42" + }, + { + "statement": "`name` differs for primary and standby", + "source": "DR-REL-5", + "id": "K-77", + "applies_to": "E-43" + }, + { + "statement": "`versioning.enabled` is true", + "id": "K-78", + "source": "DR-REL-5", + "applies_to": "E-42" + }, + { + "applies_to": "E-43", + "source": "DR-REL-5", + "id": "K-79", + "statement": "`versioning.enabled` is true" + }, + { + "source": "DR-OP-2", + "statement": "`goal` is 0.999", + "applies_to": "E-44", + "id": "K-80" + }, + { + "applies_to": "E-32", + "id": "K-81", + "source": "DR-OP-2", + "statement": "`conditions.condition_threshold` filters on SLO burn rate" + }, + { + "statement": "`service` is \"recommender.googleapis.com\"", + "applies_to": "E-45", + "id": "K-82", + "source": "DR-OP-3" + }, + { + "applies_to": "E-21", + "statement": "`lifecycle_rule.condition.age` is 30", + "id": "K-83", + "source": "DR-COST-2" + }, + { + "id": "K-84", + "applies_to": "E-21", + "source": "DR-COST-2", + "statement": "`lifecycle_rule.action.type` is \"SetStorageClass\" and `lifecycle_rule.action.storage_class` is \"NEARLINE\"" + }, + { + "source": "DR-COST-2", + "id": "K-85", + "applies_to": "E-21", + "statement": "`lifecycle_rule.condition.age` is 90" + }, + { + "applies_to": "E-21", + "statement": "`lifecycle_rule.action.type` is \"SetStorageClass\" and `lifecycle_rule.action.storage_class` is \"ARCHIVE\"", + "id": "K-86", + "source": "DR-COST-2" + }, + { + "applies_to": "E-8", + "statement": "`routing_mode` is \"REGIONAL\"", + "id": "K-87", + "source": "DR-PERF-1" + }, + { + "statement": "`autoscaling_policy.min_replicas` is 1", + "applies_to": "E-47", + "id": "K-88", + "source": "DR-SUST-1" + }, + { + "source": "DR-COST-1", + "statement": "`short_name` defines the taxonomy keys", + "id": "K-89", + "applies_to": "E-46" + }, + { + "applies_to": "E-54", + "source": "DR-COST-1", + "statement": "`destination` points to BigQuery", + "id": "K-90" + }, + { + "id": "K-91", + "applies_to": "E-55", + "source": "NFR-1", + "statement": "`tag_value` is set to the PCI regulated archetype" + }, + { + "id": "K-92", + "source": "DR-REL-2", + "statement": "`check_interval_sec` is set", + "applies_to": "E-19" + }, + { + "source": "NFR-5", + "applies_to": "E-23", + "id": "K-93", + "statement": "`disabled` is false" + }, + { + "applies_to": "E-26", + "source": "NFR-5", + "statement": "`disabled` is false", + "id": "K-94" + }, + { + "applies_to": "E-52", + "id": "K-95", + "statement": "`display_name` is set", + "source": "DR-OP-2" + }, + { + "applies_to": "E-53", + "id": "K-96", + "source": "DR-SUST-1", + "statement": "`base_instance_name` is set" + }, + { + "source": "DR-REL-1", + "applies_to": "E-57", + "statement": "`bfd.session_initialization_mode` is \"ACTIVE\"", + "id": "K-97" + }, + { + "source": "DR-REL-1", + "id": "K-99", + "applies_to": "E-56", + "statement": "`bfd.min_receive_interval` is 100" + }, + { + "id": "K-100", + "source": "DR-REL-1", + "applies_to": "E-57", + "statement": "`bfd.min_receive_interval` is 100" + }, + { + "source": "DR-REL-3", + "id": "K-101", + "statement": "`role` is \"roles/cloudkms.cryptoKeyDecrypter\"", + "applies_to": "E-59" + }, + { + "statement": "`tag_value` is set to the corp_workload archetype", + "applies_to": "E-60", + "source": "NFR-1", + "id": "K-102" + }, + { + "applies_to": "E-14", + "statement": "`router` is set to the Primary Router", + "id": "K-103", + "source": "NFR-3" + }, + { + "applies_to": "E-56", + "id": "K-104", + "statement": "`bfd.min_transmit_interval` is 100", + "source": "DR-REL-1" + }, + { + "source": "DR-REL-1", + "id": "K-105", + "applies_to": "E-57", + "statement": "`bfd.min_transmit_interval` is 100" + } + ], + "not_applicable": [ + { + "reason": "SCC activation and threat detection configuration are performed at the organization level outside of the foundational landing zone workspace.", + "recommendation": "DR-SEC-5" + }, + { + "reason": "Policy-as-code validation is defined in GitHub Actions CI/CD workflow YAML, which is outside the infrastructure terraform configuration.", + "recommendation": "DR-OP-1" + }, + { + "reason": "OpenTelemetry and Cloud Profiler are application-level agents installed within instance images or workloads, which are excluded from the foundational infrastructure deployment.", + "recommendation": "DR-PERF-2" + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/verdicts.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/verdicts.json new file mode 100644 index 0000000..0bef68d --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/verdicts.json @@ -0,0 +1,529 @@ +{ + "dry_run": { + "evidence": "Success! The configuration is valid.", + "verdict": "pass" + }, + "checks": [ + { + "check": "K-1", + "evidence": "display_name = \"Dev Folder\"", + "verdict": "pass" + }, + { + "check": "K-2", + "evidence": "display_name = \"Staging Folder\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-3", + "evidence": "display_name = \"Production Folder\"" + }, + { + "verdict": "pass", + "evidence": "tag_value = google_tags_tag_value.corp_workload.id", + "check": "K-4" + }, + { + "check": "K-5", + "verdict": "pass", + "evidence": "host_project = google_compute_shared_vpc_host_project.hub.project" + }, + { + "verdict": "pass", + "evidence": "host_project = google_compute_shared_vpc_host_project.hub.project", + "check": "K-6" + }, + { + "check": "K-7", + "evidence": "host_project = google_compute_shared_vpc_host_project.hub.project", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "private_ip_google_access = true", + "check": "K-8" + }, + { + "check": "K-9", + "evidence": "private_ip_google_access = true", + "verdict": "pass" + }, + { + "check": "K-10", + "evidence": "direction = \"INGRESS\"", + "verdict": "pass" + }, + { + "check": "K-11", + "verdict": "pass", + "evidence": "protocol = \"all\"" + }, + { + "verdict": "pass", + "evidence": "source_ranges = [\"0.0.0.0/0\"]", + "check": "K-12" + }, + { + "check": "K-13", + "evidence": "type = \"DEDICATED\"", + "verdict": "pass" + }, + { + "evidence": "type = \"DEDICATED\"", + "verdict": "pass", + "check": "K-14" + }, + { + "check": "K-15", + "verdict": "pass", + "evidence": "bandwidth = \"BPS_10G\"" + }, + { + "evidence": "bandwidth = \"BPS_10G\"", + "verdict": "pass", + "check": "K-16" + }, + { + "verdict": "pass", + "evidence": "router = google_compute_router.primary.id", + "check": "K-17" + }, + { + "evidence": "network = google_compute_network.hub.id", + "verdict": "pass", + "check": "K-18" + }, + { + "check": "K-19", + "evidence": "dest_range = \"0.0.0.0/0\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-20", + "evidence": "next_hop_ilb = google_compute_forwarding_rule.palo_alto_ilb.id" + }, + { + "verdict": "pass", + "check": "K-21", + "evidence": "issuer_uri = \"https://token.actions.githubusercontent.com\"" + }, + { + "verdict": "pass", + "check": "K-22", + "evidence": "member = \"principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.wif_pool.name}/attribute.repository/my-org/my-repo\"" + }, + { + "evidence": "idp_metadata_xml = \"\"", + "verdict": "pass", + "check": "K-23" + }, + { + "evidence": "purpose = \"ENCRYPT_DECRYPT\"", + "verdict": "pass", + "check": "K-24" + }, + { + "check": "K-25", + "evidence": "purpose = \"ENCRYPT_DECRYPT\"", + "verdict": "pass" + }, + { + "check": "K-26", + "verdict": "pass", + "evidence": "rotation_period = \"7776000s\"" + }, + { + "check": "K-27", + "evidence": "rotation_period = \"7776000s\"", + "verdict": "pass" + }, + { + "evidence": "destination = \"storage.googleapis.com/${google_storage_bucket.audit_logs.name}\"", + "verdict": "pass", + "check": "K-28" + }, + { + "check": "K-29", + "verdict": "pass", + "evidence": "retention_period = 31536000" + }, + { + "check": "K-30", + "verdict": "pass", + "evidence": "is_locked = true" + }, + { + "evidence": "restricted_services = [\"compute.googleapis.com\"]", + "verdict": "pass", + "check": "K-31" + }, + { + "check": "K-32", + "evidence": "role = \"roles/cloudkms.cryptoKeyEncrypterDecrypter\"", + "verdict": "pass" + }, + { + "check": "K-33", + "evidence": "region = \"europe-west2\"", + "verdict": "pass" + }, + { + "check": "K-34", + "verdict": "pass", + "evidence": "region = \"europe-west2\"" + }, + { + "verdict": "pass", + "check": "K-35", + "evidence": "region = \"europe-west1\"" + }, + { + "verdict": "pass", + "evidence": "constraint = \"gcp.resourceLocations\"", + "check": "K-36" + }, + { + "check": "K-37", + "verdict": "pass", + "evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]" + }, + { + "check": "K-38", + "evidence": "project = var.project_id", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "distribution_policy_zones = [\"europe-west2-a\", \"europe-west2-b\", \"europe-west2-c\"]", + "check": "K-39" + }, + { + "verdict": "pass", + "check": "K-40", + "evidence": "notification_channels = [google_monitoring_notification_channel.email.name]" + }, + { + "verdict": "pass", + "evidence": "condition_threshold is defined", + "check": "K-41" + }, + { + "evidence": "currency_code = \"GBP\"", + "check": "K-42", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "threshold_percent includes 0.5, 0.8, 0.9, 1.0", + "check": "K-43" + }, + { + "check": "K-44", + "verdict": "pass", + "evidence": "units = \"25000\"" + }, + { + "evidence": "advertise_mode = \"CUSTOM\"", + "verdict": "pass", + "check": "K-45" + }, + { + "verdict": "pass", + "evidence": "advertise_mode = \"CUSTOM\"", + "check": "K-46" + }, + { + "verdict": "pass", + "evidence": "bandwidth = \"BPS_10G\"", + "check": "K-47" + }, + { + "evidence": "bandwidth = \"BPS_10G\"", + "verdict": "pass", + "check": "K-48" + }, + { + "check": "K-49", + "verdict": "pass", + "evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]" + }, + { + "evidence": "resources = [\"projects/${var.prod_project_id}\", \"projects/${var.project_id}\"]", + "check": "K-50", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-51", + "evidence": "ingress_policies and egress_policies are configured" + }, + { + "verdict": "pass", + "evidence": "constraint = \"compute.skipDefaultNetworkCreation\"", + "check": "K-52" + }, + { + "evidence": "enforced = true", + "verdict": "pass", + "check": "K-53" + }, + { + "check": "K-54", + "evidence": "constraint = \"compute.vmExternalIpAccess\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "all = true", + "check": "K-55" + }, + { + "evidence": "constraint = \"iam.disableServiceAccountKeyCreation\"", + "check": "K-56", + "verdict": "pass" + }, + { + "check": "K-57", + "evidence": "enforced = true", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-58", + "evidence": "constraint = \"gcp.resourceLocations\"" + }, + { + "evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]", + "verdict": "pass", + "check": "K-59" + }, + { + "verdict": "pass", + "check": "K-60", + "evidence": "enable_confidential_compute = true" + }, + { + "check": "K-61", + "evidence": "name = \"CREDIT_CARD_NUMBER\"", + "verdict": "pass" + }, + { + "check": "K-62", + "evidence": "event_data_content_type = \"application/json\"", + "verdict": "pass" + }, + { + "evidence": "attribute = \"type\", value = \"google.cloud.securitycenter.finding.v1.saved\"", + "check": "K-63", + "verdict": "pass" + }, + { + "evidence": "event_type = \"google.cloud.securitycenter.finding.v1.saved\"", + "verdict": "pass", + "check": "K-64" + }, + { + "verdict": "pass", + "evidence": "interconnect = var.interconnect_1_url", + "check": "K-65" + }, + { + "evidence": "interconnect = var.interconnect_2_url", + "verdict": "pass", + "check": "K-66" + }, + { + "check": "K-67", + "evidence": "router = google_compute_router.primary.id", + "verdict": "pass" + }, + { + "check": "K-68", + "evidence": "session_initialization_mode = \"ACTIVE\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-69", + "evidence": "distribution_policy_zones = [\"europe-west2-a\", \"europe-west2-b\", \"europe-west2-c\"]" + }, + { + "evidence": "load_balancing_scheme = \"INTERNAL\"", + "check": "K-70", + "verdict": "pass" + }, + { + "evidence": "health_checks = [google_compute_health_check.palo_alto_ilb_hc.id]", + "verdict": "pass", + "check": "K-71" + }, + { + "verdict": "pass", + "check": "K-72", + "evidence": "location = \"europe-west2\"" + }, + { + "evidence": "location = \"europe-west1\"", + "check": "K-73", + "verdict": "pass" + }, + { + "evidence": "role = \"roles/cloudkms.cryptoKeyDecrypter\"", + "verdict": "pass", + "check": "K-74" + }, + { + "verdict": "pass", + "evidence": "data_locations = [\"EUROPE-WEST1\", \"EUROPE-WEST2\"]", + "check": "K-75" + }, + { + "check": "K-76", + "verdict": "pass", + "evidence": "name = \"${var.project_id}-tfstate-primary\"" + }, + { + "evidence": "name = \"${var.project_id}-tfstate-secondary\"", + "check": "K-77", + "verdict": "pass" + }, + { + "evidence": "enabled = true", + "check": "K-78", + "verdict": "pass" + }, + { + "check": "K-79", + "evidence": "enabled = true", + "verdict": "pass" + }, + { + "evidence": "goal = 0.999", + "verdict": "pass", + "check": "K-80" + }, + { + "evidence": "filter = \"select_slo_burn_rate(\\\"${google_monitoring_slo.platform_slo.name}\\\", 3600s)\"", + "verdict": "pass", + "check": "K-81" + }, + { + "verdict": "pass", + "check": "K-82", + "evidence": "service = \"recommender.googleapis.com\"" + }, + { + "verdict": "pass", + "evidence": "age = 30", + "check": "K-83" + }, + { + "verdict": "pass", + "check": "K-84", + "evidence": "type = \"SetStorageClass\", storage_class = \"NEARLINE\"" + }, + { + "check": "K-85", + "verdict": "pass", + "evidence": "age = 90" + }, + { + "check": "K-86", + "evidence": "type = \"SetStorageClass\", storage_class = \"ARCHIVE\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-87", + "evidence": "routing_mode = \"REGIONAL\"" + }, + { + "evidence": "min_replicas = 1", + "check": "K-88", + "verdict": "pass" + }, + { + "evidence": "short_name = \"environment\"", + "check": "K-89", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-90", + "evidence": "destination = \"bigquery.googleapis.com/...\"" + }, + { + "verdict": "pass", + "check": "K-91", + "evidence": "tag_value = google_tags_tag_value.pci_regulated.id" + }, + { + "check": "K-92", + "evidence": "check_interval_sec = 5", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-93", + "evidence": "disabled = false" + }, + { + "verdict": "pass", + "check": "K-94", + "evidence": "disabled = false" + }, + { + "evidence": "display_name = \"Platform Custom Service\"", + "check": "K-95", + "verdict": "pass" + }, + { + "check": "K-96", + "verdict": "pass", + "evidence": "base_instance_name = \"palo-alto-secondary\"" + }, + { + "verdict": "pass", + "evidence": "session_initialization_mode = \"ACTIVE\"", + "check": "K-97" + }, + { + "evidence": "min_receive_interval = 100", + "check": "K-99", + "verdict": "pass" + }, + { + "check": "K-100", + "verdict": "pass", + "evidence": "min_receive_interval = 100" + }, + { + "check": "K-101", + "evidence": "role = \"roles/cloudkms.cryptoKeyDecrypter\"", + "verdict": "pass" + }, + { + "evidence": "tag_value = google_tags_tag_value.corp_workload.id", + "verdict": "pass", + "check": "K-102" + }, + { + "verdict": "pass", + "evidence": "router = google_compute_router.primary.id", + "check": "K-103" + }, + { + "verdict": "pass", + "evidence": "min_transmit_interval = 100", + "check": "K-104" + }, + { + "check": "K-105", + "evidence": "min_transmit_interval = 100", + "verdict": "pass" + } + ], + "proposed_additions": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/work-breakdown.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/work-breakdown.json new file mode 100644 index 0000000..83069d7 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/002.sources/work-breakdown.json @@ -0,0 +1,332 @@ +{ + "work_packages": [ + { + "id": "WP-0", + "name": "Platform Foundation", + "parent": null, + "elements": [], + "scope": "The complete enterprise-grade landing zone.", + "produces": [ + "an operational landing zone" + ], + "skills": [ + "cloud architecture" + ] + }, + { + "id": "WP-1", + "name": "Automated Infrastructure Orchestration", + "parent": "WP-0", + "elements": [ + "E-42", + "E-43" + ], + "scope": "CI/CD orchestration, state management, and API enablement.", + "produces": [ + "Terraform state buckets for primary and secondary regions" + ], + "skills": [ + "cloud engineering" + ] + }, + { + "id": "WP-2", + "name": "Federated Identity and Role Governance", + "parent": "WP-0", + "elements": [ + "E-23", + "E-24", + "E-25", + "E-26", + "E-27", + "E-45" + ], + "scope": "Authentication for delivery pipelines and workforce SSO.", + "produces": [ + "Workload Identity Federation for GitHub Actions and Entra ID SSO" + ], + "skills": [ + "identity and access management" + ] + }, + { + "id": "WP-3", + "name": "Resource Hierarchy and Environment Boundary", + "parent": "WP-0", + "elements": [ + "E-1", + "E-2", + "E-3", + "E-4", + "E-5", + "E-6", + "E-7", + "E-8", + "E-9", + "E-10", + "E-34", + "E-35", + "E-36", + "E-37", + "E-38", + "E-50", + "E-51", + "E-55", + "E-60" + ], + "scope": "Google Cloud resource hierarchy, shared VPC projects, networking, and organization policies.", + "produces": [ + "Folders, projects, networking, and organization policies ensuring isolation" + ], + "skills": [ + "cloud engineering", + "network engineering" + ] + }, + { + "id": "WP-4", + "name": "Cryptographic Key Lifecycle", + "parent": "WP-0", + "elements": [ + "E-28", + "E-29", + "E-30", + "E-31", + "E-49", + "E-58", + "E-59" + ], + "scope": "Customer-managed encryption keys (CMEK) and cryptographic isolation.", + "produces": [ + "Primary and secondary KMS key rings and crypto keys with IAM bindings" + ], + "skills": [ + "security engineering" + ] + }, + { + "id": "WP-5", + "name": "Audit, Compliance, and Financial Governance", + "parent": "WP-0", + "elements": [ + "E-21", + "E-22", + "E-33", + "E-39", + "E-46", + "E-54" + ], + "scope": "Immutable compliance logs, DLP templates, and budget enforcement.", + "produces": [ + "Immutable audit log storage, billing budgets, and DLP inspection templates" + ], + "skills": [ + "cloud engineering", + "finops" + ] + }, + { + "id": "WP-6", + "name": "Centralised Outbound Traffic Inspection", + "parent": "WP-0", + "elements": [ + "E-15", + "E-16", + "E-17", + "E-18", + "E-19", + "E-20", + "E-47", + "E-48", + "E-53" + ], + "scope": "Egress inspection and threat prevention using Palo Alto VM-Series.", + "produces": [ + "Palo Alto VM-Series firewalls with internal load balancing and egress routing" + ], + "skills": [ + "network security engineering" + ] + }, + { + "id": "WP-7", + "name": "Dedicated Private Hybrid Connectivity", + "parent": "WP-0", + "elements": [ + "E-11", + "E-12", + "E-13", + "E-14", + "E-56", + "E-57" + ], + "scope": "Dedicated Interconnect termination and dynamic BGP routing.", + "produces": [ + "Primary and secondary Cloud Routers with Dedicated Interconnect VLAN attachments" + ], + "skills": [ + "network engineering" + ] + }, + { + "id": "WP-8", + "name": "Platform Telemetry and Operational Alerting", + "parent": "WP-0", + "elements": [ + "E-32", + "E-40", + "E-41", + "E-44", + "E-52" + ], + "scope": "SLO tracking, multi-burn-rate alerting, and SCC automated remediation.", + "produces": [ + "SLOs, alerting policies, and SCC automated remediation functions" + ], + "skills": [ + "site reliability engineering" + ] + } + ], + "activities": [ + { + "id": "A-1", + "work_package": "WP-1", + "name": "Enable Google Cloud APIs", + "days": 0.5, + "after": [] + }, + { + "id": "A-2", + "work_package": "WP-1", + "name": "Setup CI/CD Pipeline and State Buckets", + "days": 1, + "after": [ + "A-1" + ] + }, + { + "id": "A-3", + "work_package": "WP-2", + "name": "Configure Workload Identity and Entra ID SSO", + "days": 1.5, + "after": [ + "A-2" + ] + }, + { + "id": "A-4", + "work_package": "WP-3", + "name": "Provision Resource Hierarchy and Enforce Org Policies", + "days": 2.5, + "after": [ + "A-3" + ] + }, + { + "id": "A-5", + "work_package": "WP-3", + "name": "Provision Hub and Spoke VPC Networks", + "days": 2, + "after": [ + "A-4" + ] + }, + { + "id": "A-6", + "work_package": "WP-7", + "name": "Order and provision the Interconnect carrier circuits", + "days": 30, + "owner": "customer", + "assumption": "The customer manages the carrier's lead time and physical cross-connects.", + "after": [] + }, + { + "id": "A-7", + "work_package": "WP-4", + "name": "Configure Cloud KMS and Key Rings", + "days": 1.5, + "after": [ + "A-4" + ] + }, + { + "id": "A-8", + "work_package": "WP-5", + "name": "Set up Audit Logging, DLP, and Billing Budgets", + "days": 2, + "after": [ + "A-4" + ] + }, + { + "id": "A-9", + "work_package": "WP-6", + "name": "Deploy Palo Alto Egress Firewall Appliances", + "days": 5, + "after": [ + "A-5" + ] + }, + { + "id": "A-10", + "work_package": "WP-7", + "name": "Configure Cloud Routers and Terminate BGP over Interconnect", + "days": 3, + "after": [ + "A-5", + "A-6" + ] + }, + { + "id": "A-11", + "work_package": "WP-3", + "name": "Implement VPC Service Controls", + "days": 2, + "after": [ + "A-5", + "A-7", + "A-8" + ] + }, + { + "id": "A-12", + "work_package": "WP-8", + "name": "Deploy Platform Telemetry, SLOs, and SCC Auto-remediation", + "days": 2.5, + "after": [ + "A-8", + "A-11" + ] + }, + { + "id": "A-13", + "work_package": "WP-1", + "name": "Run validation in development environment", + "days": 1, + "after": [ + "A-9", + "A-10", + "A-12" + ] + }, + { + "id": "A-14", + "work_package": "WP-1", + "name": "Run validation in staging environment", + "days": 1, + "after": [ + "A-13" + ] + }, + { + "id": "A-15", + "work_package": "WP-1", + "name": "Run validation in production environment", + "days": 1, + "after": [ + "A-14" + ] + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.md b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.md new file mode 100644 index 0000000..4e72e5d --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.md @@ -0,0 +1,15 @@ +# Intent 3: case-08-landing-zone + +- Recorded: 2026-10-21T22:39:26+00:00 +- Execution: `case-02-from-architecture-r3` + +## What the customer wants + +We are a UK-regulated payment platform expanding into Google Cloud. We need an +enterprise-grade landing zone ready in under 10 weeks to support our core payments +engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have +existing on-premises systems that require private, low-latency connectivity over a +10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and +GitHub Actions for our delivery pipelines. Security requires that all egress +traffic flows through centralised inspection appliances before touching the public +internet, and customer data must be encrypted with keys we control. diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.scope.completed.yaml b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.scope.completed.yaml new file mode 100644 index 0000000..8b76f3a --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.scope.completed.yaml @@ -0,0 +1,414 @@ +schema_version: 1 +metadata: + customer_case: case-08-landing-zone + client_name: ask + author: ask + revision: ask + source: architecture:case-08-landing-zone-185426 + status: BASELINE_LOCKED +engagement: + kind: platform_foundation +intent: + statement: 'We are a UK-regulated payment platform expanding into Google Cloud. + We need an + + enterprise-grade landing zone ready in under 10 weeks to support our core payments + + engine. All payment processing must comply with DORA and PCI-DSS v4.0. We have + + existing on-premises systems that require private, low-latency connectivity over + a + + 10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and + + GitHub Actions for our delivery pipelines. Security requires that all egress + + traffic flows through centralised inspection appliances before touching the public + + internet, and customer data must be encrypted with keys we control.' + requirements: ask + business_outcomes: + - target: Foundation ready in under 10 weeks + id: BO-1 + outcome: Deliver an enterprise-grade Google Cloud landing zone within 10 weeks + to enable ingestion and operation of the core payments engine. + - outcome: Maintain continuous operational resilience and regulatory compliance + with DORA and PCI-DSS v4.0 with platform availability of at least 99.9%. + target: '>=99.9% monthly availability and full regulatory audit alignment' + id: BO-2 + - target: Deterministic low-latency 10Gbps interconnect and 100% CMEK lifecycle + coverage + id: BO-3 + outcome: Ensure secure, low-latency hybrid transaction transport and zero-trust + data protection using customer-managed encryption keys. + exclusions: + - Core payments engine application-level development, maintenance, and transaction + processing logic + - On-premises data centre infrastructure and systems beyond the Cloud Interconnect + demarcation + - Physical telecom carrier circuit ordering and provisioning (customer-owned build + item) +solution: + cloud_provider: gcp + items: + - id: S-1 + name: Resource Hierarchy and Environment Boundary Manager + kind: governance + product: Resource Manager + source_ref: C-1 + elements: + - E-1 + - E-2 + - E-3 + - E-4 + - E-5 + - E-6 + - E-7 + - E-8 + - E-9 + - E-10 + - E-34 + - E-35 + - E-36 + - E-37 + - E-38 + - E-50 + - E-51 + - E-55 + - E-60 + - id: S-2 + name: Dedicated Private Hybrid Connectivity Gateway + kind: network + product: Cloud Interconnect + source_ref: C-2 + elements: + - E-11 + - E-12 + - E-13 + - E-14 + - E-56 + - E-57 + - id: S-3 + name: Centralised Outbound Traffic Inspection and Egress Boundary + kind: service + product: Palo Alto Networks VM-Series + source_ref: C-3 + elements: + - E-15 + - E-16 + - E-17 + - E-18 + - E-19 + - E-20 + - E-47 + - E-48 + - E-53 + - id: S-4 + name: Centralised Immutable Audit Event and Compliance Log Repository + kind: storage + product: Cloud Storage + source_ref: C-4 + elements: + - E-21 + - E-22 + - E-39 + - id: S-5 + name: Automated Infrastructure Orchestration and Deployment Engine + kind: service + product: GitHub Actions + source_ref: C-5 + elements: + - E-42 + - E-43 + - id: S-6 + name: Federated Identity and Role Governance Authority + kind: identity + product: IAM Workload Identity Federation + source_ref: C-6 + elements: + - E-23 + - E-24 + - E-25 + - E-26 + - E-27 + - E-45 + - id: S-7 + name: Cryptographic Key Lifecycle and Encryption Controller + kind: keys + product: Cloud KMS + source_ref: C-7 + elements: + - E-28 + - E-29 + - E-30 + - E-31 + - E-49 + - E-58 + - E-59 + - id: S-8 + name: Platform Telemetry and Operational Alerting Engine + kind: observability + product: Cloud Monitoring + source_ref: C-8 + elements: + - E-32 + - E-40 + - E-41 + - E-44 + - E-52 + - id: S-9 + name: Financial Governance and Budget Enforcement Monitor + kind: billing + product: Cloud Billing + source_ref: C-9 + elements: + - E-33 + - E-46 + - E-54 + conditions: + - id: K-1 + 'on': + - S-1 + - S-4 + - S-7 + condition: The landing zone configuration aligns with DORA operational resiliency + guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit. + source_ref: NFR-8 + - id: K-2 + 'on': + - S-1 + condition: The secondary region in europe-west1 operates as a warm standby ready + to receive failover traffic. + source_ref: NFR-9 + - id: K-3 + 'on': + - S-1 + condition: Data residency is guaranteed within the United Kingdom and European + Union. + source_ref: NFR-9 + - id: K-4 + 'on': + - S-5 + condition: Platform foundation is fully provisioned and validated for core payments + engine workload ingestion within 10 weeks. + source_ref: NFR-10 + - id: K-5 + 'on': + - S-8 + condition: Platform foundation infrastructure maintains measured monthly availability + of at least 99.9%. + source_ref: NFR-11 + - id: K-6 + 'on': + - S-8 + condition: Disaster recovery activation achieves a recovery time objective (RTO) + of 4 hours. + source_ref: NFR-11 + - id: K-7 + 'on': + - S-8 + condition: Disaster recovery processes achieve a recovery point objective (RPO) + of 1 hour. + source_ref: NFR-11 + - id: K-8 + 'on': + - S-8 + condition: Alert notifications trigger operational incident workflows for the + 24/7 central platform operations team. + source_ref: NFR-12 + - id: K-9 + 'on': + - S-9 + condition: Monthly cloud foundation expenditure does not exceed the target allocation + without operational sign-off. + source_ref: NFR-13 + - id: K-10 + 'on': + - S-2 + condition: Hybrid connection provides deterministic low latency suitable for real-time + payment transaction processing. + source_ref: NFR-14 + - id: K-11 + 'on': + - S-8 + condition: Deployments optimize carbon-free energy utilization based on Google + Cloud regional CFE scores. + source_ref: NFR-15 + - id: K-12 + 'on': + - S-6 + condition: Permissions are continuously rightsized based on IAM Recommender findings. + source_ref: DR-OP-3 + build: + duration_days: 36 + customer_owned: + - Order and provision the Interconnect carrier circuits + inherited_risks: ask +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: Automated CI/CD pipelines via GitHub Actions with peer review and + automated policy checks; standard operational changes deployed without a separate + manual CAB. + monitoring: ask +sourcing: + model: provider_operated + delivery_countries: + - GB + - PL + - IN +organisation: + existing_teams: + - party: customer + name: Central Platform Operations Team + hours: 24x7 + headcount_limit: ask +service_management: + tooling: Google Cloud operations suite (Cloud Monitoring and Cloud Logging) integrated + with native alerting workflows and GitHub Actions. +service_levels: + priority_1_response_minutes: 30 + recovery_time_hours: 4 +compliance: + frameworks: + - DORA + - ISO27001 + - PCI_DSS_v4 +commercial: + currency: GBP + monthly_run_budget: ask +transition: + go_live_date: ask +run: + pre_approved: true + overlay_skills: ask +inferred_assumptions: +- id: ASSUMP-01 + field: intent.business_outcomes + statement: 'Assumed intent.business_outcomes is [{''target'': ''Foundation ready + in under 10 weeks'', ''id'': ''BO-1'', ''outcome'': ''Deliver an enterprise-grade + Google Cloud landing zone within 10 weeks to enable ingestion and operation of + the core payments engine.''}, {''outcome'': ''Maintain continuous operational + resilience and regulatory compliance with DORA and PCI-DSS v4.0 with platform + availability of at least 99.9%.'', ''target'': ''>=99.9% monthly availability + and full regulatory audit alignment'', ''id'': ''BO-2''}, {''target'': ''Deterministic + low-latency 10Gbps interconnect and 100% CMEK lifecycle coverage'', ''id'': ''BO-3'', + ''outcome'': ''Ensure secure, low-latency hybrid transaction transport and zero-trust + data protection using customer-managed encryption keys.''}]. Derived from the + workflow request stating the need for an enterprise-grade landing zone ready in + under 10 weeks to support the core payments engine, complying with DORA and PCI-DSS + v4.0, with secure hybrid connectivity and customer-controlled encryption.' + impact_if_invalid: Changes what is measured, and what each measure is for. +- id: ASSUMP-02 + field: intent.exclusions + statement: 'Assumed intent.exclusions is [''Core payments engine application-level + development, maintenance, and transaction processing logic'', ''On-premises data + centre infrastructure and systems beyond the Cloud Interconnect demarcation'', + ''Physical telecom carrier circuit ordering and provisioning (customer-owned build + item)'']. Derived from engagement kind (platform foundation) and workflow request: + application-level workloads, on-prem systems, and physical circuit provisioning + are excluded from platform operations.' + impact_if_invalid: Changes which Capabilities are designed. +- id: ASSUMP-03 + field: operations.change_process + statement: Assumed operations.change_process is Automated CI/CD pipelines via GitHub + Actions with peer review and automated policy checks; standard operational changes + deployed without a separate manual CAB.. Derived from request stating GitHub Actions + delivery pipelines and catalogue default guidance. + impact_if_invalid: Changes change enablement and release. +- id: ASSUMP-04 + field: organisation.existing_teams + statement: 'Assumed organisation.existing_teams is [{''party'': ''customer'', ''name'': + ''Central Platform Operations Team'', ''hours'': ''24x7''}]. Derived from condition + K-8 noting alerting to the 24/7 central platform operations team, alongside provider-operated + landing zone model.' + impact_if_invalid: Changes which Roles are new and which are filled. +- id: ASSUMP-05 + field: service_management.tooling + statement: Assumed service_management.tooling is Google Cloud operations suite (Cloud + Monitoring and Cloud Logging) integrated with native alerting workflows and GitHub + Actions.. Catalogue default guidance leveraging Google Cloud operations suite + and GitHub Actions specified in request. + impact_if_invalid: Changes tooling and the processes written for it. +- id: ASSUMP-06 + field: service_levels.priority_1_response_minutes + statement: Assumed service_levels.priority_1_response_minutes is 30. Standard catalogue + default for 24x7 critical infrastructure response. + impact_if_invalid: Changes on-call arrangements and staffing. +- id: ASSUMP-07 + field: service_levels.recovery_time_hours + statement: Assumed service_levels.recovery_time_hours is 4. Matches condition K-6 + disaster recovery RTO requirement of 4 hours and catalogue default. + impact_if_invalid: Changes continuity arrangements. +discovery_gaps: +- field: metadata.client_name + prompt: What is metadata.client_name? + severity: LOW +- field: metadata.author + prompt: What is metadata.author? + severity: LOW +- field: metadata.revision + prompt: What is metadata.revision? + severity: LOW +- field: intent.requirements + prompt: Which requirements for operations has the customer stated outright? + severity: LOW +- field: solution.inherited_risks + prompt: What is already known to be unverified or at risk in the solution? + severity: LOW +- field: operations.monitoring + prompt: Which monitoring and alerting is already in place or required? + severity: LOW +- field: organisation.headcount_limit + prompt: Is there a limit on the number of people operating this? + severity: LOW +- field: commercial.monthly_run_budget + prompt: Is there a monthly budget for operating this, in that currency? + severity: LOW +- field: transition.go_live_date + prompt: When must the solution be in operation? + severity: LOW +- field: run.overlay_skills + prompt: Which overlay Skills should this Execution load? + severity: LOW +provenance: + commercial.currency: digested + commercial.monthly_run_budget: unknown + compliance.frameworks: digested + engagement.kind: digested + intent.business_outcomes: assumed + intent.exclusions: assumed + intent.requirements: unknown + intent.statement: digested + metadata.author: unknown + metadata.client_name: unknown + metadata.customer_case: digested + metadata.revision: unknown + metadata.source: digested + operations.availability_hours: digested + operations.change_process: assumed + operations.monitoring: unknown + operations.support_hours: digested + organisation.existing_teams: assumed + organisation.headcount_limit: unknown + run.overlay_skills: unknown + run.pre_approved: supplied + service_levels.priority_1_response_minutes: assumed + service_levels.recovery_time_hours: assumed + service_management.tooling: assumed + solution.build.customer_owned: digested + solution.build.duration_days: digested + solution.cloud_provider: digested + solution.conditions: digested + solution.inherited_risks: unknown + solution.items: digested + sourcing.delivery_countries: supplied + sourcing.model: supplied + transition.go_live_date: unknown +readiness: + supplied: 3 + answered: 0 + assumed: 7 + unknown: 10 + not_applicable: 0 + digested: 13 diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.scope.yaml b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.scope.yaml new file mode 100644 index 0000000..9533549 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.scope.yaml @@ -0,0 +1,10 @@ +intent: + business_outcomes: decide +run: + pre_approved: true +sourcing: + delivery_countries: + - GB + - PL + - IN + model: provider_operated diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/components.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/components.json new file mode 100644 index 0000000..61eddc2 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/components.json @@ -0,0 +1,79 @@ +[ + { + "id": "C-1", + "name": "Resource Hierarchy and Environment Boundary Manager", + "requirements": [ + "FR-1", + "NFR-1", + "NFR-8", + "NFR-9" + ] + }, + { + "id": "C-2", + "name": "Dedicated Private Hybrid Connectivity Gateway", + "requirements": [ + "FR-2", + "NFR-3", + "NFR-14" + ] + }, + { + "id": "C-3", + "name": "Centralised Outbound Traffic Inspection and Egress Boundary", + "requirements": [ + "FR-3", + "NFR-2", + "NFR-4" + ] + }, + { + "id": "C-4", + "name": "Centralised Immutable Audit Event and Compliance Log Repository", + "requirements": [ + "FR-4", + "NFR-7", + "NFR-8" + ] + }, + { + "id": "C-5", + "name": "Automated Infrastructure Orchestration and Deployment Engine", + "requirements": [ + "FR-5", + "NFR-10" + ] + }, + { + "id": "C-6", + "name": "Federated Identity and Role Governance Authority", + "requirements": [ + "FR-6", + "NFR-5" + ] + }, + { + "id": "C-7", + "name": "Cryptographic Key Lifecycle and Encryption Controller", + "requirements": [ + "NFR-6", + "NFR-8" + ] + }, + { + "id": "C-8", + "name": "Platform Telemetry and Operational Alerting Engine", + "requirements": [ + "NFR-11", + "NFR-12", + "NFR-15" + ] + }, + { + "id": "C-9", + "name": "Financial Governance and Budget Enforcement Monitor", + "requirements": [ + "NFR-13" + ] + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/execution.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/execution.json new file mode 100644 index 0000000..126ac0c --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/execution.json @@ -0,0 +1,140 @@ +{ + "customer_case": "case-08-landing-zone", + "execution_id": "case-08-landing-zone-185426", + "session_id": "cdd6bfe7-2bd8-44af-8e15-9dcfa869eb84", + "intent_version": 1, + "workflow_request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.", + "baseline": null, + "overlay_skills": {}, + "terraform_modules": false, + "scope": { + "status": "BASELINE_LOCKED", + "readiness": { + "supplied": 44, + "answered": 0, + "assumed": 12, + "unknown": 4, + "not_applicable": 3 + } + }, + "delivery_plan": { + "work_packages": 9, + "activities": 15, + "working_days": 36.0, + "weeks": 7.2, + "team_size": 3, + "window_weeks": 10, + "within_window": true, + "critical_path": [ + "A-6", + "A-10", + "A-13", + "A-14", + "A-15" + ] + }, + "status": "completed", + "warnings": [], + "risks": [], + "assessment": { + "recommendations_quality": { + "specific": 0.91, + "actionable": 0.79, + "trade_offs": 0.95, + "traceable": 0.73 + }, + "mean": 0.84, + "meets_bar": true + }, + "phases": { + "scoping": "approved", + "requirements_discovery": "approved", + "solution_architecture": "approved", + "solution_implementation": "approved", + "solution_validation": "approved", + "packaging": "approved" + }, + "solution_guide": "solution-architecture-guide.md", + "customer_questions": "customer-questions.md", + "changes": null, + "deliverables": { + "scope": { + "document": "deliverables/scope.md", + "attachments": [ + "scope.json", + "scope.yaml" + ], + "status": "submitted" + }, + "requirements": { + "document": "deliverables/requirements.md", + "attachments": [ + "requirements.json" + ], + "status": "submitted" + }, + "technical_decomposition": { + "document": "deliverables/technical_decomposition.md", + "attachments": [ + "components.json" + ], + "status": "submitted" + }, + "product_selection": { + "document": "deliverables/product_selection.md", + "attachments": [ + "products.json" + ], + "status": "submitted" + }, + "design_recommendations": { + "document": "deliverables/design_recommendations.md", + "attachments": [ + "recommendations.json" + ], + "status": "submitted" + }, + "target_architecture": { + "document": "deliverables/target_architecture.md", + "attachments": [ + "architecture.mmd", + "target-architecture.json" + ], + "status": "submitted" + }, + "architecture_description": { + "document": "deliverables/architecture_description.md", + "attachments": [], + "status": "submitted" + }, + "deployment_guidance": { + "document": "deliverables/deployment_guidance.md", + "attachments": [ + "implementation.json", + "terraform/main.tf", + "terraform/variables.tf", + "terraform/versions.tf" + ], + "status": "submitted" + }, + "validation_results": { + "document": "deliverables/validation_results.md", + "attachments": [ + "verdicts.json" + ], + "status": "submitted" + }, + "work_breakdown": { + "document": "deliverables/work_breakdown.md", + "attachments": [ + "work-breakdown.json" + ], + "status": "submitted" + }, + "solution_guide": { + "document": "deliverables/solution_guide.md", + "attachments": [], + "status": "submitted" + } + } +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/products.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/products.json new file mode 100644 index 0000000..ca9e033 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/products.json @@ -0,0 +1,38 @@ +[ + { + "component": "C-1", + "product": "Resource Manager" + }, + { + "component": "C-2", + "product": "Cloud Interconnect" + }, + { + "component": "C-3", + "product": "Palo Alto Networks VM-Series" + }, + { + "component": "C-4", + "product": "Cloud Storage" + }, + { + "component": "C-5", + "product": "GitHub Actions" + }, + { + "component": "C-6", + "product": "IAM Workload Identity Federation" + }, + { + "component": "C-7", + "product": "Cloud KMS" + }, + { + "component": "C-8", + "product": "Cloud Monitoring" + }, + { + "component": "C-9", + "product": "Cloud Billing" + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/recommendations.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/recommendations.json new file mode 100644 index 0000000..3721e66 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/recommendations.json @@ -0,0 +1,248 @@ +[ + { + "id": "DR-SEC-1", + "pillar": "security", + "text": "Implement a VPC Service Controls perimeter around all Cardholder Data Environment (CDE) projects and the centralized audit log buckets.", + "components": [ + "C-1", + "C-4" + ], + "conditions": [ + "A VPC Service Controls perimeter encloses the CDE and audit log projects.", + "The VPC Service Controls perimeter restricts ingress and egress to authorized traffic." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-2", + "pillar": "security", + "text": "Apply Organization Policy constraints to disable default network creation, deny external IPs, prohibit service account key creation, and restrict resource locations.", + "components": [ + "C-1" + ], + "conditions": [ + "The compute.skipDefaultNetworkCreation constraint is enforced.", + "The compute.vmExternalIpAccess constraint is enforced to deny all.", + "The iam.disableServiceAccountKeyCreation constraint is enforced.", + "The gcp.resourceLocations constraint restricts resources to europe-west2 and europe-west1." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-3", + "pillar": "security", + "text": "Enable Confidential Computing on the payment engine Compute Engine instances to encrypt data in-use.", + "components": [], + "conditions": [ + "Confidential Computing is enabled on the core payment engine Compute Engine instances." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-4", + "pillar": "security", + "text": "Deploy Sensitive Data Protection (Cloud DLP) inspection templates to detect and mask PAN data.", + "components": [ + "C-4" + ], + "conditions": [ + "Sensitive Data Protection (Cloud DLP) inspection templates are configured to detect and mask PAN data." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-5", + "pillar": "security", + "text": "Enable Security Command Center at the organization level with Event and Container Threat Detection.", + "components": [ + "C-1", + "C-8" + ], + "conditions": [ + "Security Command Center is activated at the organization level.", + "Event Threat Detection and Container Threat Detection are enabled in SCC." + ], + "outside_code": [] + }, + { + "id": "DR-SEC-6", + "pillar": "security", + "text": "Implement event-driven automated responses using Eventarc and Cloud Run functions for SCC findings.", + "components": [ + "C-8" + ], + "conditions": [ + "Eventarc is configured to route high-severity SCC findings.", + "Cloud Run functions are configured to execute automated remediation on Eventarc triggers." + ], + "outside_code": [] + }, + { + "id": "DR-REL-1", + "pillar": "reliability", + "text": "Provision redundant Dedicated Cloud Interconnect circuits across distinct physical edge colocation facilities, terminating on redundant Cloud Routers with BFD.", + "components": [ + "C-2" + ], + "conditions": [ + "Dedicated Cloud Interconnect circuits are provisioned across redundant edge colocation facilities.", + "Cloud Interconnects terminate on redundant Cloud Routers in each region.", + "Bidirectional Forwarding Detection (BFD) is enabled on the Cloud Router BGP sessions." + ], + "outside_code": [] + }, + { + "id": "DR-REL-2", + "pillar": "reliability", + "text": "Deploy Palo Alto VM-Series firewalls in a regional Managed Instance Group across at least three zones, fronted by an Internal Passthrough Network Load Balancer.", + "components": [ + "C-3" + ], + "conditions": [ + "Palo Alto VM-Series firewalls are deployed in a regional Managed Instance Group (MIG) across at least three zones.", + "An Internal Passthrough Network Load Balancer routes traffic to the VM-Series MIG.", + "Health checks are configured on the internal load balancer to probe firewall health." + ], + "outside_code": [] + }, + { + "id": "DR-REL-3", + "pillar": "reliability", + "text": "Provision dedicated regional Cloud KMS key rings in both europe-west2 and europe-west1, granting workload identity service accounts Decrypter roles on both.", + "components": [ + "C-7", + "C-6" + ], + "conditions": [ + "Regional Cloud KMS key rings are provisioned in both europe-west2 and europe-west1.", + "Workload Identity Federation service accounts are granted Decrypter roles on both regional key sets." + ], + "outside_code": [] + }, + { + "id": "DR-REL-4", + "pillar": "reliability", + "text": "Configure the Cloud Storage buckets for audit logging as Dual-Region spanning europe-west2 and europe-west1.", + "components": [ + "C-4" + ], + "conditions": [ + "Audit log Cloud Storage buckets are configured as Dual-Region spanning europe-west2 and europe-west1." + ], + "outside_code": [] + }, + { + "id": "DR-REL-5", + "pillar": "reliability", + "text": "Isolate the Terraform state of the primary region from the standby region using multi-region Cloud Storage buckets with Object Versioning.", + "components": [ + "C-5" + ], + "conditions": [ + "Terraform state for primary and standby regions is decoupled into separate failure domains.", + "Terraform state is stored in multi-region Cloud Storage buckets with Object Versioning enabled." + ], + "outside_code": [] + }, + { + "id": "DR-OP-1", + "pillar": "operational-excellence", + "text": "Integrate Policy-as-Code validation into the GitHub Actions pull request workflows.", + "components": [ + "C-5" + ], + "conditions": [ + "GitHub Actions workflows include automated Policy-as-Code validation steps prior to deployment execution." + ], + "outside_code": [] + }, + { + "id": "DR-OP-2", + "pillar": "operational-excellence", + "text": "Establish explicit Service Level Indicators (SLIs) and configure multi-burn-rate alerting policies in Cloud Monitoring.", + "components": [ + "C-8" + ], + "conditions": [ + "Service Level Objectives (SLOs) and SLIs are defined in Cloud Monitoring Service Monitoring.", + "Multi-burn-rate alerting policies are configured based on the SLO error budget." + ], + "outside_code": [] + }, + { + "id": "DR-OP-3", + "pillar": "operational-excellence", + "text": "Activate the IAM Recommender to continuously evaluate role bindings.", + "components": [ + "C-6" + ], + "conditions": [ + "IAM Recommender is activated on the foundation projects." + ], + "outside_code": [ + "Permissions are continuously rightsized based on IAM Recommender findings." + ] + }, + { + "id": "DR-COST-1", + "pillar": "cost-optimization", + "text": "Enforce a mandatory labeling taxonomy through Organization Policies and export granular billing data to BigQuery.", + "components": [ + "C-9", + "C-1" + ], + "conditions": [ + "Cloud Billing export to BigQuery is enabled.", + "A mandatory labeling taxonomy is enforced via Organization Policy Service tags." + ], + "outside_code": [] + }, + { + "id": "DR-COST-2", + "pillar": "cost-optimization", + "text": "Implement Cloud Storage Object Lifecycle Management to transition logs to Nearline storage after 30 days and to Coldline or Archive storage after 90 days.", + "components": [ + "C-4" + ], + "conditions": [ + "Object Lifecycle Management rules transition Cloud Storage objects to Nearline storage after 30 days.", + "Object Lifecycle Management rules transition Cloud Storage objects to Coldline or Archive storage after 90 days." + ], + "outside_code": [] + }, + { + "id": "DR-PERF-1", + "pillar": "performance-optimization", + "text": "Configure Cloud Routers with regional dynamic routing to localize egress traffic within the warm standby region.", + "components": [ + "C-2" + ], + "conditions": [ + "Cloud Routers are configured with regional dynamic routing mode." + ], + "outside_code": [] + }, + { + "id": "DR-PERF-2", + "pillar": "performance-optimization", + "text": "Instrument the deployment with OpenTelemetry exporting to Cloud Trace, and use Cloud Profiler on the compute instances.", + "components": [ + "C-8" + ], + "conditions": [ + "OpenTelemetry tracing is configured to export to Cloud Trace.", + "Cloud Profiler agents are enabled on the core payments engine instances." + ], + "outside_code": [] + }, + { + "id": "DR-SUST-1", + "pillar": "sustainability", + "text": "Implement a Pilot Light warm standby by maintaining active data replication while scaling the standby compute instances to a minimal baseline.", + "components": [], + "conditions": [ + "Secondary region compute instances are scaled to a minimal baseline (Pilot Light) configuration during normal operations." + ], + "outside_code": [] + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/requirements.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/requirements.json new file mode 100644 index 0000000..307613b --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/requirements.json @@ -0,0 +1,280 @@ +[ + { + "id": "FR-1", + "kind": "functional", + "text": "The platform foundation isolates development, staging, and production environments across distinct Google Cloud projects and resource folders.", + "from": [ + "REQ-02" + ] + }, + { + "id": "FR-2", + "kind": "functional", + "text": "The platform foundation establishes private bidirectional network connectivity between Google Cloud and on-premises core payment systems over Cloud Interconnect.", + "from": [ + "REQ-03" + ] + }, + { + "id": "FR-3", + "kind": "functional", + "text": "The platform foundation routes all outbound internet-bound traffic from workload spoke networks through centralised network security inspection appliances.", + "from": [ + "REQ-03" + ] + }, + { + "id": "FR-4", + "kind": "functional", + "text": "The platform foundation captures and exports all administrative and data access audit logs across environments into a centralised, tamper-resistant log repository.", + "from": [ + "REQ-02" + ] + }, + { + "id": "FR-5", + "kind": "functional", + "text": "The platform foundation provisions and manages all cloud infrastructure as code via automated CI/CD pipelines executing validated pull request workflows.", + "from": [ + "REQ-01" + ] + }, + { + "id": "FR-6", + "kind": "functional", + "text": "The platform foundation federates corporate identity for single sign-on and enforces least-privilege role-based access control for platform administrators and application squads.", + "from": [ + "REQ-04" + ] + }, + { + "id": "NFR-1", + "kind": "non_functional", + "text": "Environments are partitioned into development, staging, and production tiers, enforcing corp_workload controls on lower environments and pci_regulated_workload controls on production.", + "conditions": [ + "Separate projects or folders exist for development, staging, and production environment tiers.", + "Production environments enforce pci_regulated_workload archetype security controls.", + "Development and staging environments enforce corp_workload archetype security controls." + ], + "outside_code": [], + "from": [ + "REQ-02", + "governance.environment_tiers" + ] + }, + { + "id": "NFR-2", + "kind": "non_functional", + "text": "Network connectivity is structured as a Shared VPC hub-and-spoke topology with zero direct internet ingress to workload subnets.", + "conditions": [ + "A Shared VPC host project is connected to service projects across environment tiers.", + "Workload subnets have private Google access enabled and no external public IP addresses assigned.", + "Ingress firewall rules deny uninspected public internet traffic to workload subnets." + ], + "outside_code": [], + "from": [ + "REQ-02", + "networking.topology", + "networking.exposure" + ] + }, + { + "id": "NFR-3", + "kind": "non_functional", + "text": "Hybrid connectivity to on-premises core payment systems is enabled via a 10Gbps Dedicated Cloud Interconnect.", + "conditions": [ + "Dedicated Cloud Interconnect connections are configured for hybrid connectivity.", + "Cloud Interconnect capacity is provisioned at 10Gbps bandwidth.", + "Interconnect VLAN attachments are associated with Cloud Routers in the Shared VPC hub." + ], + "outside_code": [], + "from": [ + "REQ-03", + "networking.hybrid_connectivity.enabled", + "networking.hybrid_connectivity.type", + "networking.hybrid_connectivity.bandwidth_gbps" + ] + }, + { + "id": "NFR-4", + "kind": "non_functional", + "text": "All outbound internet traffic from workload spokes is forced through centralised inspection appliances deployed in the hub network.", + "conditions": [ + "Inspection appliance instances are deployed in a centralised hub network.", + "Custom VPC route tables direct 0.0.0.0/0 egress traffic from spokes to the centralised inspection appliance instances or internal load balancer." + ], + "outside_code": [], + "from": [ + "REQ-03", + "networking.egress_control.model" + ] + }, + { + "id": "NFR-5", + "kind": "non_functional", + "text": "Administrative access is authenticated via Entra ID single sign-on, and delivery pipelines authenticate via Workload Identity Federation without stored service account keys.", + "conditions": [ + "Workload Identity Federation pools and providers are configured for GitHub Actions authentication.", + "IAM bindings grant roles to federated GitHub Actions identities without downloadable service account keys.", + "Corporate single sign-on is configured with Entra ID as the identity source of truth." + ], + "outside_code": [], + "from": [ + "REQ-04", + "governance.identity_source_of_truth", + "governance.pipeline_authentication" + ] + }, + { + "id": "NFR-6", + "kind": "non_functional", + "text": "Customer confidential data is encrypted at rest using customer-managed encryption keys with a 90-day rotation schedule.", + "conditions": [ + "Cloud KMS customer-managed encryption keys are configured for infrastructure resources storing confidential data.", + "Cloud KMS keys have an automated rotation period set to 90 days (7776000s)." + ], + "outside_code": [], + "from": [ + "REQ-04", + "compliance.encryption.type", + "compliance.encryption.key_rotation_days" + ] + }, + { + "id": "NFR-7", + "kind": "non_functional", + "text": "Audit logs across all environments are retained for 365 days in a storage bucket protected by an immutable retention policy.", + "conditions": [ + "An organization-level or project-level log sink exports audit logs to a centralised Cloud Storage bucket.", + "The centralised audit log storage bucket specifies a retention period of 365 days (31536000 seconds).", + "The centralised audit log storage bucket has retention policy locking enabled to prevent modification or early deletion." + ], + "outside_code": [], + "from": [ + "REQ-02", + "compliance.audit_logging.retention_days", + "compliance.audit_logging.immutable_retention_policy" + ] + }, + { + "id": "NFR-8", + "kind": "non_functional", + "text": "The platform foundation enforces DORA, ISO27001, and PCI-DSS v4.0 regulatory baselines and protects confidential data classification.", + "conditions": [ + "Organization policy constraints enforce perimeter isolation and restrict unauthenticated resource sharing.", + "IAM policies restrict access to confidential data resources to authorized service identities and roles." + ], + "outside_code": [ + "The landing zone configuration aligns with DORA operational resiliency guidelines and PCI-DSS v4.0 perimeter isolation standards during external audit." + ], + "from": [ + "REQ-02", + "compliance.frameworks", + "compliance.data_classification" + ] + }, + { + "id": "NFR-9", + "kind": "non_functional", + "text": "Platform foundation resources are provisioned across europe-west2 (primary) and europe-west1 (secondary) in a warm standby model, restricting data residency to the UK and EU.", + "conditions": [ + "Primary landing zone network and foundation resources are provisioned in europe-west2.", + "Secondary landing zone network and foundation standby resources are provisioned in europe-west1.", + "Resource location organization policies restrict resource and data storage locations to europe-west2 and europe-west1." + ], + "outside_code": [ + "The secondary region in europe-west1 operates as a warm standby ready to receive failover traffic.", + "Data residency is guaranteed within the United Kingdom and European Union." + ], + "from": [ + "REQ-02", + "estate.regions.primary", + "estate.regions.secondary", + "estate.regions.resiliency_tier", + "estate.data_residency" + ] + }, + { + "id": "NFR-10", + "kind": "non_functional", + "text": "The platform foundation landing zone is delivered and operational for workload ingestion within 10 weeks.", + "conditions": [ + "All landing zone foundation resources are codified in modular Terraform configurations." + ], + "outside_code": [ + "Platform foundation is fully provisioned and validated for core payments engine workload ingestion within 10 weeks." + ], + "from": [ + "REQ-01" + ] + }, + { + "id": "NFR-11", + "kind": "non_functional", + "text": "The platform foundation maintains 99.9% availability with a 4-hour recovery time objective (RTO) and a 1-hour recovery point objective (RPO).", + "conditions": [ + "Regional networking and core management services are deployed across at least three zones in the primary region." + ], + "outside_code": [ + "Platform foundation infrastructure maintains measured monthly availability of at least 99.9%.", + "Disaster recovery activation achieves a recovery time objective (RTO) of 4 hours.", + "Disaster recovery processes achieve a recovery point objective (RPO) of 1 hour." + ], + "from": [ + "REQ-02" + ] + }, + { + "id": "NFR-12", + "kind": "non_functional", + "text": "Platform operations monitoring and alerting are implemented using Google Cloud Operations Suite with 24/7 alerting capabilities.", + "conditions": [ + "Cloud Monitoring workspaces and notification channels are configured for the platform foundation.", + "Alert policies are configured for Interconnect link degradation, appliance health, and egress routing failures." + ], + "outside_code": [ + "Alert notifications trigger operational incident workflows for the 24/7 central platform operations team." + ], + "from": [] + }, + { + "id": "NFR-13", + "kind": "non_functional", + "text": "Infrastructure costs are tracked against a monthly budget of 25,000 GBP with automated alerting.", + "conditions": [ + "A Cloud Billing budget is configured for the foundation projects with currency set to GBP.", + "Budget alert thresholds are configured at 50%, 80%, 90%, and 100% of the 25,000 GBP monthly budget." + ], + "outside_code": [ + "Monthly cloud foundation expenditure does not exceed the target allocation without operational sign-off." + ], + "from": [] + }, + { + "id": "NFR-14", + "kind": "non_functional", + "text": "Hybrid networking provides dedicated 10Gbps unthrottled throughput with minimal latency between Google Cloud and on-premises core systems.", + "conditions": [ + "Cloud Routers configured for Interconnect advertise non-overlapping RFC 1918 BGP routes.", + "Dedicated Interconnect VLAN attachments are configured with dedicated 10Gbps bandwidth allocation." + ], + "outside_code": [ + "Hybrid connection provides deterministic low latency suitable for real-time payment transaction processing." + ], + "from": [ + "REQ-03" + ] + }, + { + "id": "NFR-15", + "kind": "non_functional", + "text": "Landing zone resources are deployed in Google Cloud regions with strong low-carbon energy ratings.", + "conditions": [ + "Foundation resources are constrained to europe-west2 (London) and europe-west1 (Belgium) regions." + ], + "outside_code": [ + "Deployments optimize carbon-free energy utilization based on Google Cloud regional CFE scores." + ], + "from": [] + } +] diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/scope.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/scope.json new file mode 100644 index 0000000..16e782d --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/scope.json @@ -0,0 +1,233 @@ +{ + "kind": "platform_foundation", + "stated": { + "engagement": { + "kind": "platform_foundation" + }, + "intent": { + "statement": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.\n", + "requirements": [ + { + "id": "REQ-01", + "category": "Timeline", + "statement": "Platform foundation must be ready for workload ingestion within 10 weeks." + }, + { + "id": "REQ-02", + "category": "Compliance", + "statement": "Must achieve DORA resiliency alignment and PCI-DSS v4.0 perimeter isolation." + }, + { + "id": "REQ-03", + "category": "Networking", + "statement": "10Gbps on-premises hybrid connectivity; centralised inspection appliances for egress." + }, + { + "id": "REQ-04", + "category": "Identity and security", + "statement": "Workload identity federation with GitHub Actions; Entra ID single sign-on; customer-managed encryption keys." + } + ], + "exclusions": [ + "Application container deployment; workloads are managed by application squads.", + "An active-active secondary region in phase 1; warm standby is acceptable." + ] + }, + "estate": { + "cloud_provider": "gcp", + "mode": "greenfield", + "regions": { + "primary": "europe-west2", + "secondary": "europe-west1", + "resiliency_tier": "warm_standby" + }, + "data_residency": "United Kingdom and European Union", + "existing_systems": "On-premises payment systems that stay in place and connect over the interconnect." + }, + "workload": { + "availability": { + "hours": "always" + } + }, + "compliance": { + "industry": "financial_services", + "frameworks": [ + "DORA", + "ISO27001", + "PCI_DSS_v4" + ], + "data_classification": "confidential", + "encryption": { + "type": "cmek", + "key_rotation_days": 90 + }, + "audit_logging": { + "retention_days": 365, + "immutable_retention_policy": true + } + }, + "networking": { + "exposure": "internal_only", + "topology": "shared_vpc_hub_and_spoke", + "hybrid_connectivity": { + "enabled": true, + "type": "cloud_interconnect", + "bandwidth_gbps": 10 + }, + "egress_control": { + "model": "centralized_appliance" + } + }, + "governance": { + "identity_source_of_truth": "Entra ID", + "pipeline_authentication": "workload identity federation", + "environment_tiers": [ + { + "name": "development", + "archetype": "corp_workload" + }, + { + "name": "staging", + "archetype": "corp_workload" + }, + { + "name": "production", + "archetype": "pci_regulated_workload" + } + ] + }, + "operations": { + "operating_model": "central platform team brokering access for application squads", + "support_hours": "always" + }, + "delivery": { + "window_weeks": 10, + "commercial_model": "fixed_price_milestone", + "iac_tooling": "terraform", + "ci_cd": "GitHub Actions", + "team_size": 3, + "working_days_per_week": 5 + }, + "cost": { + "currency": "GBP" + } + }, + "binding": [ + "estate.regions.primary", + "estate.regions.secondary", + "estate.regions.resiliency_tier", + "estate.data_residency", + "compliance.frameworks", + "compliance.data_classification", + "compliance.encryption.type", + "compliance.encryption.key_rotation_days", + "compliance.audit_logging.retention_days", + "compliance.audit_logging.immutable_retention_policy", + "networking.exposure", + "networking.topology", + "networking.hybrid_connectivity.enabled", + "networking.hybrid_connectivity.type", + "networking.hybrid_connectivity.bandwidth_gbps", + "networking.egress_control.model", + "governance.identity_source_of_truth", + "governance.pipeline_authentication", + "governance.environment_tiers" + ], + "assumed": [ + { + "field": "intent.success_criteria", + "value": [ + "Platform foundation ready for workload ingestion within 10 weeks", + "Compliance achieved with DORA and PCI-DSS v4.0 requirements", + "Hybrid connectivity established via 10Gbps Dedicated Interconnect", + "Centralised egress inspection operational", + "Identity federation and CMEK security controls implemented" + ], + "why": "Assumed standard success criteria based on delivery within 10 weeks and compliance with DORA and PCI-DSS v4.0." + }, + { + "field": "workload.availability.target", + "value": "99.9%", + "why": "Assumed 99.9% availability target for platform foundation infrastructure aligned with warm standby secondary region." + }, + { + "field": "workload.availability.recovery_time", + "value": "4 hours", + "why": "Assumed 4 hours recovery time objective (RTO) for platform foundation services in warm standby." + }, + { + "field": "workload.availability.recovery_point", + "value": "1 hour", + "why": "Assumed 1 hour recovery point objective (RPO) for platform foundation configuration and state." + }, + { + "field": "networking.egress_control.appliance_vendor", + "value": "Palo Alto Networks VM-Series", + "why": "Assumed Palo Alto Networks VM-Series as enterprise inspection appliance for centralized egress control." + }, + { + "field": "networking.cidr_allocations", + "value": "RFC 1918 non-overlapping IP allocation (10.0.0.0/16 partitioned across environments) sized for growth", + "why": "Assumed RFC 1918 non-overlapping IP allocation (10.0.0.0/16 partitioned across environments) sized for growth." + }, + { + "field": "operations.monitoring", + "value": "Google Cloud Operations Suite with availability and error alerting", + "why": "Assumed native Google Cloud Operations Suite (Cloud Monitoring and Cloud Logging) with availability and error alerting." + }, + { + "field": "operations.change_process", + "value": "GitHub Actions pull request workflow with automated Terraform plan validation and peer review", + "why": "Assumed GitHub Actions pull request workflow with automated Terraform plan validation and peer approval." + }, + { + "field": "integrations", + "value": [ + { + "criticality": "critical", + "direction": "bidirectional", + "name": "On-premises core payment systems" + } + ], + "why": "Assumed integration with existing on-premises core payment systems via Dedicated Interconnect." + }, + { + "field": "cost.monthly_budget", + "value": 25000, + "why": "Assumed indicative monthly foundation infrastructure budget of 25000 GBP for 10Gbps interconnect, inspection appliances, and core services." + }, + { + "field": "constraints.products_required", + "value": [], + "why": "No mandated third-party products beyond stated tooling (GCP, Terraform, GitHub Actions, Entra ID)." + }, + { + "field": "constraints.products_forbidden", + "value": [], + "why": "No forbidden products specified." + } + ], + "to_decide": [], + "to_ask": [ + { + "field": "metadata.engagement_reference", + "prompt": "What is metadata.engagement_reference?", + "severity": "LOW" + }, + { + "field": "delivery.target_start_date", + "prompt": "When should work start?", + "severity": "LOW" + }, + { + "field": "constraints.other", + "prompt": "What is constraints.other?", + "severity": "LOW" + } + ], + "not_applicable": [ + "workload.users.count", + "workload.users.peak_concurrent", + "workload.users.who" + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/target-architecture.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/target-architecture.json new file mode 100644 index 0000000..3a35acc --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/target-architecture.json @@ -0,0 +1,1210 @@ +{ + "elements": [ + { + "resource_type": "google_folder", + "product": "Resource Manager", + "component": "C-1", + "name": "Dev Folder", + "id": "E-1" + }, + { + "resource_type": "google_folder", + "id": "E-2", + "product": "Resource Manager", + "component": "C-1", + "name": "Staging Folder" + }, + { + "component": "C-1", + "name": "Production Folder", + "id": "E-3", + "resource_type": "google_folder", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_shared_vpc_host_project", + "name": "Shared VPC Host Project", + "product": "Resource Manager", + "id": "E-4", + "component": "C-1" + }, + { + "product": "Resource Manager", + "component": "C-1", + "resource_type": "google_compute_shared_vpc_service_project", + "name": "Dev Spoke Project", + "id": "E-5" + }, + { + "name": "Stg Spoke Project", + "product": "Resource Manager", + "component": "C-1", + "resource_type": "google_compute_shared_vpc_service_project", + "id": "E-6" + }, + { + "product": "Resource Manager", + "id": "E-7", + "name": "Prod Spoke Project", + "component": "C-1", + "resource_type": "google_compute_shared_vpc_service_project" + }, + { + "product": "Resource Manager", + "name": "Hub VPC Network", + "component": "C-1", + "id": "E-8", + "resource_type": "google_compute_network" + }, + { + "id": "E-9", + "resource_type": "google_compute_subnetwork", + "name": "Dev Subnetwork", + "component": "C-1", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_subnetwork", + "component": "C-1", + "id": "E-10", + "name": "Prod Subnetwork", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_router", + "id": "E-11", + "name": "Primary Cloud Router", + "component": "C-2", + "product": "Cloud Interconnect" + }, + { + "resource_type": "google_compute_router", + "name": "Secondary Cloud Router", + "id": "E-12", + "component": "C-2", + "product": "Cloud Interconnect" + }, + { + "product": "Cloud Interconnect", + "id": "E-13", + "name": "Primary VLAN Attachment 1", + "resource_type": "google_compute_interconnect_attachment", + "component": "C-2" + }, + { + "id": "E-14", + "component": "C-2", + "resource_type": "google_compute_interconnect_attachment", + "name": "Primary VLAN Attachment 2", + "product": "Cloud Interconnect" + }, + { + "name": "Palo Alto Instance Template", + "component": "C-3", + "id": "E-15", + "product": "Palo Alto Networks VM-Series", + "resource_type": "google_compute_instance_template" + }, + { + "id": "E-16", + "name": "Palo Alto MIG Primary", + "resource_type": "google_compute_region_instance_group_manager", + "component": "C-3", + "product": "Palo Alto Networks VM-Series" + }, + { + "id": "E-17", + "component": "C-3", + "resource_type": "google_compute_forwarding_rule", + "name": "Palo Alto ILB", + "product": "Palo Alto Networks VM-Series" + }, + { + "name": "Palo Alto ILB Backend", + "resource_type": "google_compute_region_backend_service", + "component": "C-3", + "id": "E-18", + "product": "Palo Alto Networks VM-Series" + }, + { + "resource_type": "google_compute_health_check", + "component": "C-3", + "name": "Palo Alto ILB Health Check", + "product": "Palo Alto Networks VM-Series", + "id": "E-19" + }, + { + "component": "C-3", + "id": "E-20", + "name": "Egress Route to ILB", + "resource_type": "google_compute_route", + "product": "Palo Alto Networks VM-Series" + }, + { + "id": "E-21", + "component": "C-4", + "name": "Audit Logs Storage Bucket", + "product": "Cloud Storage", + "resource_type": "google_storage_bucket" + }, + { + "component": "C-4", + "product": "Cloud Storage", + "id": "E-22", + "name": "Audit Log Sink", + "resource_type": "google_logging_project_sink" + }, + { + "product": "IAM Workload Identity Federation", + "id": "E-23", + "component": "C-6", + "name": "WIF Pool", + "resource_type": "google_iam_workload_identity_pool" + }, + { + "name": "WIF Provider GitHub", + "product": "IAM Workload Identity Federation", + "id": "E-24", + "resource_type": "google_iam_workload_identity_pool_provider", + "component": "C-6" + }, + { + "component": "C-6", + "resource_type": "google_iam_workforce_pool_provider", + "name": "WIF Provider Entra ID", + "product": "IAM Workload Identity Federation", + "id": "E-25" + }, + { + "product": "IAM Workload Identity Federation", + "name": "Pipeline Service Account", + "component": "C-6", + "id": "E-26", + "resource_type": "google_service_account" + }, + { + "name": "Pipeline SA WIF Binding", + "resource_type": "google_service_account_iam_member", + "id": "E-27", + "product": "IAM Workload Identity Federation", + "component": "C-6" + }, + { + "product": "Cloud KMS", + "resource_type": "google_kms_key_ring", + "name": "Primary KMS Key Ring", + "id": "E-28", + "component": "C-7" + }, + { + "product": "Cloud KMS", + "resource_type": "google_kms_crypto_key", + "component": "C-7", + "name": "Primary KMS Crypto Key", + "id": "E-29" + }, + { + "name": "Secondary KMS Key Ring", + "id": "E-30", + "product": "Cloud KMS", + "component": "C-7", + "resource_type": "google_kms_key_ring" + }, + { + "product": "Cloud KMS", + "name": "Secondary KMS Crypto Key", + "component": "C-7", + "resource_type": "google_kms_crypto_key", + "id": "E-31" + }, + { + "resource_type": "google_monitoring_alert_policy", + "name": "Platform Alert Policy", + "product": "Cloud Monitoring", + "component": "C-8", + "id": "E-32" + }, + { + "product": "Cloud Billing", + "name": "Foundation Cloud Budget", + "component": "C-9", + "resource_type": "google_billing_budget", + "id": "E-33" + }, + { + "id": "E-34", + "name": "Org Policy skipDefaultNetworkCreation", + "resource_type": "google_project_organization_policy", + "component": "C-1", + "product": "Resource Manager" + }, + { + "component": "C-1", + "id": "E-35", + "name": "Org Policy vmExternalIpAccess", + "resource_type": "google_project_organization_policy", + "product": "Resource Manager" + }, + { + "resource_type": "google_project_organization_policy", + "name": "Org Policy disableServiceAccountKeyCreation", + "id": "E-36", + "product": "Resource Manager", + "component": "C-1" + }, + { + "resource_type": "google_project_organization_policy", + "product": "Resource Manager", + "component": "C-1", + "name": "Org Policy resourceLocations", + "id": "E-37" + }, + { + "name": "VPC SC Perimeter", + "component": "C-1", + "product": "Resource Manager", + "resource_type": "google_access_context_manager_service_perimeter", + "id": "E-38" + }, + { + "name": "DLP Inspect Template", + "component": "C-4", + "resource_type": "google_data_loss_prevention_inspect_template", + "product": "Cloud Storage", + "id": "E-39" + }, + { + "id": "E-40", + "product": "Cloud Monitoring", + "name": "SCC Eventarc Trigger", + "resource_type": "google_eventarc_trigger", + "component": "C-8" + }, + { + "product": "Cloud Monitoring", + "resource_type": "google_cloudfunctions2_function", + "component": "C-8", + "name": "SCC Remediation Function", + "id": "E-41" + }, + { + "component": "C-5", + "id": "E-42", + "name": "Primary TF State Bucket", + "resource_type": "google_storage_bucket", + "product": "GitHub Actions" + }, + { + "resource_type": "google_storage_bucket", + "id": "E-43", + "product": "GitHub Actions", + "component": "C-5", + "name": "Secondary TF State Bucket" + }, + { + "product": "Cloud Monitoring", + "component": "C-8", + "resource_type": "google_monitoring_slo", + "name": "Platform SLO", + "id": "E-44" + }, + { + "id": "E-45", + "component": "C-6", + "resource_type": "google_project_service", + "name": "IAM Recommender Service", + "product": "IAM Workload Identity Federation" + }, + { + "resource_type": "google_tags_tag_key", + "name": "Taxonomy Tag Key", + "component": "C-9", + "product": "Cloud Billing", + "id": "E-46" + }, + { + "product": "Palo Alto Networks VM-Series", + "component": "C-3", + "resource_type": "google_compute_region_autoscaler", + "name": "Standby MIG Autoscaler", + "id": "E-47" + }, + { + "name": "Deny Ingress Firewall", + "resource_type": "google_compute_firewall", + "component": "C-3", + "product": "Palo Alto Networks VM-Series", + "id": "E-48" + }, + { + "component": "C-7", + "id": "E-49", + "name": "KMS IAM Binding", + "resource_type": "google_kms_crypto_key_iam_member", + "product": "Cloud KMS" + }, + { + "id": "E-50", + "component": "C-1", + "product": "Resource Manager", + "resource_type": "google_tags_tag_binding", + "name": "Environment Tag Binding" + }, + { + "name": "Payment Engine Template", + "product": "Resource Manager", + "id": "E-51", + "resource_type": "google_compute_instance_template", + "component": "C-1" + }, + { + "name": "Platform Custom Service", + "component": "C-8", + "product": "Cloud Monitoring", + "id": "E-52", + "resource_type": "google_monitoring_custom_service" + }, + { + "name": "Palo Alto MIG Secondary", + "component": "C-3", + "resource_type": "google_compute_region_instance_group_manager", + "product": "Palo Alto Networks VM-Series", + "id": "E-53" + }, + { + "product": "Cloud Billing", + "resource_type": "google_logging_project_sink", + "name": "Billing BigQuery Sink", + "component": "C-9", + "id": "E-54" + }, + { + "name": "Prod Tag Binding", + "id": "E-55", + "resource_type": "google_tags_tag_binding", + "component": "C-1", + "product": "Resource Manager" + }, + { + "resource_type": "google_compute_router_peer", + "id": "E-56", + "component": "C-2", + "name": "Primary BGP Peer 1", + "product": "Cloud Interconnect" + }, + { + "resource_type": "google_compute_router_peer", + "id": "E-57", + "component": "C-2", + "name": "Primary BGP Peer 2", + "product": "Cloud Interconnect" + }, + { + "resource_type": "google_kms_crypto_key_iam_member", + "component": "C-7", + "name": "Secondary KMS Decrypter Binding", + "id": "E-58", + "product": "Cloud KMS" + }, + { + "component": "C-7", + "name": "Primary KMS Decrypter Binding", + "product": "Cloud KMS", + "resource_type": "google_kms_crypto_key_iam_member", + "id": "E-59" + }, + { + "id": "E-60", + "resource_type": "google_tags_tag_binding", + "name": "Staging Tag Binding", + "component": "C-1", + "product": "Resource Manager" + } + ], + "relationships": [ + { + "from": "E-5", + "id": "R-1", + "kind": "uses", + "to": "E-4" + }, + { + "id": "R-2", + "kind": "uses", + "to": "E-4", + "from": "E-6" + }, + { + "kind": "uses", + "to": "E-4", + "id": "R-3", + "from": "E-7" + }, + { + "kind": "attached to", + "to": "E-11", + "from": "E-13", + "id": "R-4" + }, + { + "from": "E-14", + "to": "E-11", + "id": "R-5", + "kind": "attached to" + }, + { + "kind": "instantiates", + "from": "E-16", + "id": "R-6", + "to": "E-15" + }, + { + "to": "E-15", + "from": "E-53", + "id": "R-6b", + "kind": "instantiates" + }, + { + "to": "E-17", + "from": "E-20", + "id": "R-7", + "kind": "routes to" + }, + { + "from": "E-17", + "id": "R-8", + "kind": "routes to", + "to": "E-18" + }, + { + "id": "R-9", + "kind": "checked by", + "to": "E-19", + "from": "E-18" + }, + { + "from": "E-22", + "id": "R-10", + "kind": "exports to", + "to": "E-21" + }, + { + "kind": "authenticates via", + "id": "R-11", + "from": "E-24", + "to": "E-23" + }, + { + "from": "E-27", + "to": "E-26", + "id": "R-12", + "kind": "granted to" + }, + { + "from": "E-29", + "to": "E-28", + "id": "R-13", + "kind": "contained in" + }, + { + "from": "E-31", + "kind": "contained in", + "id": "R-14", + "to": "E-30" + }, + { + "kind": "granted on", + "id": "R-15", + "from": "E-49", + "to": "E-29" + }, + { + "id": "R-16", + "kind": "monitors", + "to": "E-52", + "from": "E-44" + }, + { + "from": "E-47", + "kind": "scales", + "id": "R-17", + "to": "E-53" + }, + { + "id": "R-18", + "to": "E-46", + "from": "E-50", + "kind": "binds" + }, + { + "to": "E-11", + "from": "E-56", + "id": "R-19", + "kind": "contained in" + }, + { + "from": "E-57", + "kind": "contained in", + "to": "E-11", + "id": "R-20" + }, + { + "to": "E-31", + "kind": "granted on", + "from": "E-58", + "id": "R-21" + }, + { + "to": "E-29", + "from": "E-59", + "kind": "granted on", + "id": "R-22" + }, + { + "to": "E-46", + "kind": "binds", + "id": "R-23", + "from": "E-60" + } + ], + "checks": [ + { + "id": "K-1", + "source": "NFR-1", + "applies_to": "E-1", + "statement": "`display_name` is set" + }, + { + "applies_to": "E-2", + "source": "NFR-1", + "statement": "`display_name` is set", + "id": "K-2" + }, + { + "applies_to": "E-3", + "source": "NFR-1", + "statement": "`display_name` is set", + "id": "K-3" + }, + { + "statement": "`tag_value` is set to the environment archetype", + "applies_to": "E-50", + "source": "NFR-1", + "id": "K-4" + }, + { + "applies_to": "E-5", + "id": "K-5", + "statement": "`host_project` is set to the hub project", + "source": "NFR-2" + }, + { + "source": "NFR-2", + "applies_to": "E-6", + "statement": "`host_project` is set to the hub project", + "id": "K-6" + }, + { + "id": "K-7", + "source": "NFR-2", + "applies_to": "E-7", + "statement": "`host_project` is set to the hub project" + }, + { + "id": "K-8", + "applies_to": "E-9", + "statement": "`private_ip_google_access` is true", + "source": "NFR-2" + }, + { + "statement": "`private_ip_google_access` is true", + "applies_to": "E-10", + "source": "NFR-2", + "id": "K-9" + }, + { + "statement": "`direction` is \"INGRESS\"", + "source": "NFR-2", + "applies_to": "E-48", + "id": "K-10" + }, + { + "id": "K-11", + "source": "NFR-2", + "statement": "`deny.protocol` is \"all\"", + "applies_to": "E-48" + }, + { + "statement": "`source_ranges` includes \"0.0.0.0/0\"", + "id": "K-12", + "applies_to": "E-48", + "source": "NFR-2" + }, + { + "applies_to": "E-13", + "id": "K-13", + "statement": "`type` is \"DEDICATED\"", + "source": "NFR-3" + }, + { + "statement": "`type` is \"DEDICATED\"", + "id": "K-14", + "source": "NFR-3", + "applies_to": "E-14" + }, + { + "source": "NFR-3", + "applies_to": "E-13", + "id": "K-15", + "statement": "`bandwidth` is \"10G\"" + }, + { + "applies_to": "E-14", + "statement": "`bandwidth` is \"10G\"", + "id": "K-16", + "source": "NFR-3" + }, + { + "id": "K-17", + "source": "NFR-3", + "statement": "`router` is set to the Primary Router", + "applies_to": "E-13" + }, + { + "id": "K-18", + "source": "NFR-4", + "statement": "`network_interface.network` is set to the hub VPC", + "applies_to": "E-15" + }, + { + "source": "NFR-4", + "applies_to": "E-20", + "id": "K-19", + "statement": "`dest_range` is \"0.0.0.0/0\"" + }, + { + "statement": "`next_hop_ilb` is set to the ILB forwarding rule", + "applies_to": "E-20", + "source": "NFR-4", + "id": "K-20" + }, + { + "applies_to": "E-24", + "statement": "`oidc.issuer_uri` is \"https://token.actions.githubusercontent.com\"", + "source": "NFR-5", + "id": "K-21" + }, + { + "id": "K-22", + "statement": "`member` includes \"principalSet://iam.googleapis.com/\"", + "applies_to": "E-27", + "source": "NFR-5" + }, + { + "id": "K-23", + "applies_to": "E-25", + "statement": "`saml.idp_metadata_xml` is populated", + "source": "NFR-5" + }, + { + "applies_to": "E-29", + "id": "K-24", + "source": "NFR-6", + "statement": "`purpose` is \"ENCRYPT_DECRYPT\"" + }, + { + "applies_to": "E-31", + "source": "NFR-6", + "id": "K-25", + "statement": "`purpose` is \"ENCRYPT_DECRYPT\"" + }, + { + "applies_to": "E-29", + "source": "NFR-6", + "id": "K-26", + "statement": "`rotation_period` is \"7776000s\"" + }, + { + "applies_to": "E-31", + "source": "NFR-6", + "id": "K-27", + "statement": "`rotation_period` is \"7776000s\"" + }, + { + "statement": "`destination` is set to the storage bucket", + "applies_to": "E-22", + "id": "K-28", + "source": "NFR-7" + }, + { + "applies_to": "E-21", + "source": "NFR-7", + "statement": "`retention_policy.retention_period` is 31536000", + "id": "K-29" + }, + { + "statement": "`retention_policy.is_locked` is true", + "source": "NFR-7", + "applies_to": "E-21", + "id": "K-30" + }, + { + "id": "K-31", + "applies_to": "E-38", + "source": "NFR-8", + "statement": "`status.restricted_services` includes \"compute.googleapis.com\"" + }, + { + "id": "K-32", + "applies_to": "E-49", + "source": "NFR-8", + "statement": "`role` is \"roles/cloudkms.cryptoKeyEncrypterDecrypter\"" + }, + { + "statement": "`region` is \"europe-west2\"", + "source": "NFR-9", + "id": "K-33", + "applies_to": "E-11" + }, + { + "statement": "`region` is \"europe-west2\"", + "applies_to": "E-9", + "id": "K-34", + "source": "NFR-9" + }, + { + "statement": "`region` is \"europe-west1\"", + "applies_to": "E-12", + "id": "K-35", + "source": "NFR-9" + }, + { + "statement": "`constraint` is \"gcp.resourceLocations\"", + "id": "K-36", + "source": "NFR-9", + "applies_to": "E-37" + }, + { + "statement": "`list_policy.allow.values` includes \"in:europe-west2\" and \"in:europe-west1\"", + "applies_to": "E-37", + "id": "K-37", + "source": "NFR-9" + }, + { + "statement": "`project` is set", + "source": "NFR-10", + "id": "K-38", + "applies_to": "E-4" + }, + { + "source": "NFR-11", + "applies_to": "E-16", + "id": "K-39", + "statement": "`distribution_policy_zones` includes at least three zones" + }, + { + "statement": "`notification_channels` is populated", + "id": "K-40", + "applies_to": "E-32", + "source": "NFR-12" + }, + { + "statement": "`conditions.condition_threshold` or `conditions.condition_absent` is defined", + "applies_to": "E-32", + "id": "K-41", + "source": "NFR-12" + }, + { + "source": "NFR-13", + "id": "K-42", + "statement": "`amount.specified_amount.currency_code` is \"GBP\"", + "applies_to": "E-33" + }, + { + "id": "K-43", + "source": "NFR-13", + "applies_to": "E-33", + "statement": "`threshold_rules.threshold_percent` includes 0.5, 0.8, 0.9, and 1.0" + }, + { + "source": "NFR-13", + "id": "K-44", + "statement": "`amount.specified_amount.units` is 25000", + "applies_to": "E-33" + }, + { + "statement": "`bgp.advertise_mode` is \"CUSTOM\"", + "id": "K-45", + "applies_to": "E-11", + "source": "NFR-14" + }, + { + "statement": "`bgp.advertise_mode` is \"CUSTOM\"", + "id": "K-46", + "applies_to": "E-12", + "source": "NFR-14" + }, + { + "id": "K-47", + "statement": "`bandwidth` is \"10G\"", + "source": "NFR-14", + "applies_to": "E-13" + }, + { + "id": "K-48", + "source": "NFR-14", + "statement": "`bandwidth` is \"10G\"", + "applies_to": "E-14" + }, + { + "statement": "`list_policy.allow.values` enforces europe-west2 and europe-west1", + "source": "NFR-15", + "id": "K-49", + "applies_to": "E-37" + }, + { + "id": "K-50", + "applies_to": "E-38", + "statement": "`status.resources` includes the CDE and audit log projects", + "source": "DR-SEC-1" + }, + { + "source": "DR-SEC-1", + "applies_to": "E-38", + "statement": "`status.ingress_policies` and `status.egress_policies` are configured", + "id": "K-51" + }, + { + "statement": "`constraint` is \"compute.skipDefaultNetworkCreation\"", + "id": "K-52", + "applies_to": "E-34", + "source": "DR-SEC-2" + }, + { + "source": "DR-SEC-2", + "statement": "`boolean_policy.enforced` is true", + "id": "K-53", + "applies_to": "E-34" + }, + { + "applies_to": "E-35", + "id": "K-54", + "statement": "`constraint` is \"compute.vmExternalIpAccess\"", + "source": "DR-SEC-2" + }, + { + "statement": "`list_policy.deny.all` is true", + "applies_to": "E-35", + "id": "K-55", + "source": "DR-SEC-2" + }, + { + "id": "K-56", + "statement": "`constraint` is \"iam.disableServiceAccountKeyCreation\"", + "source": "DR-SEC-2", + "applies_to": "E-36" + }, + { + "statement": "`boolean_policy.enforced` is true", + "source": "DR-SEC-2", + "id": "K-57", + "applies_to": "E-36" + }, + { + "id": "K-58", + "statement": "`constraint` is \"gcp.resourceLocations\"", + "applies_to": "E-37", + "source": "DR-SEC-2" + }, + { + "id": "K-59", + "statement": "`list_policy.allow.values` restricts locations", + "source": "DR-SEC-2", + "applies_to": "E-37" + }, + { + "statement": "`confidential_instance_config.enable_confidential_compute` is true", + "source": "DR-SEC-3", + "id": "K-60", + "applies_to": "E-51" + }, + { + "id": "K-61", + "source": "DR-SEC-4", + "applies_to": "E-39", + "statement": "`inspect_config.info_types.name` includes \"CREDIT_CARD_NUMBER\"" + }, + { + "statement": "`event_data_content_type` is \"application/json\"", + "id": "K-62", + "source": "DR-SEC-6", + "applies_to": "E-40" + }, + { + "statement": "`matching_criteria.attribute` includes \"type\" matching SCC findings", + "id": "K-63", + "applies_to": "E-40", + "source": "DR-SEC-6" + }, + { + "source": "DR-SEC-6", + "id": "K-64", + "applies_to": "E-41", + "statement": "`event_trigger.event_type` matches Eventarc trigger" + }, + { + "applies_to": "E-13", + "statement": "`interconnect` is set to the primary colocation interconnect", + "id": "K-65", + "source": "DR-REL-1" + }, + { + "statement": "`interconnect` is set to the secondary colocation interconnect", + "applies_to": "E-14", + "id": "K-66", + "source": "DR-REL-1" + }, + { + "source": "DR-REL-1", + "statement": "`router` is set to the Primary Router", + "id": "K-67", + "applies_to": "E-13" + }, + { + "applies_to": "E-56", + "id": "K-68", + "source": "DR-REL-1", + "statement": "`bfd.session_initialization_mode` is \"ACTIVE\"" + }, + { + "source": "DR-REL-2", + "id": "K-69", + "statement": "`distribution_policy_zones` includes three zones", + "applies_to": "E-16" + }, + { + "id": "K-70", + "applies_to": "E-17", + "source": "DR-REL-2", + "statement": "`load_balancing_scheme` is \"INTERNAL\"" + }, + { + "applies_to": "E-18", + "statement": "`health_checks` includes the ILB Health Check", + "source": "DR-REL-2", + "id": "K-71" + }, + { + "source": "DR-REL-3", + "applies_to": "E-28", + "id": "K-72", + "statement": "`location` is \"europe-west2\"" + }, + { + "applies_to": "E-30", + "id": "K-73", + "statement": "`location` is \"europe-west1\"", + "source": "DR-REL-3" + }, + { + "source": "DR-REL-3", + "applies_to": "E-58", + "statement": "`role` is \"roles/cloudkms.cryptoKeyDecrypter\"", + "id": "K-74" + }, + { + "id": "K-75", + "statement": "`custom_placement_config.data_locations` includes \"EUROPE-WEST1\" and \"EUROPE-WEST2\"", + "applies_to": "E-21", + "source": "DR-REL-4" + }, + { + "statement": "`name` differs for primary and standby", + "id": "K-76", + "source": "DR-REL-5", + "applies_to": "E-42" + }, + { + "statement": "`name` differs for primary and standby", + "source": "DR-REL-5", + "id": "K-77", + "applies_to": "E-43" + }, + { + "statement": "`versioning.enabled` is true", + "id": "K-78", + "source": "DR-REL-5", + "applies_to": "E-42" + }, + { + "applies_to": "E-43", + "source": "DR-REL-5", + "id": "K-79", + "statement": "`versioning.enabled` is true" + }, + { + "source": "DR-OP-2", + "statement": "`goal` is 0.999", + "applies_to": "E-44", + "id": "K-80" + }, + { + "applies_to": "E-32", + "id": "K-81", + "source": "DR-OP-2", + "statement": "`conditions.condition_threshold` filters on SLO burn rate" + }, + { + "statement": "`service` is \"recommender.googleapis.com\"", + "applies_to": "E-45", + "id": "K-82", + "source": "DR-OP-3" + }, + { + "applies_to": "E-21", + "statement": "`lifecycle_rule.condition.age` is 30", + "id": "K-83", + "source": "DR-COST-2" + }, + { + "id": "K-84", + "applies_to": "E-21", + "source": "DR-COST-2", + "statement": "`lifecycle_rule.action.type` is \"SetStorageClass\" and `lifecycle_rule.action.storage_class` is \"NEARLINE\"" + }, + { + "source": "DR-COST-2", + "id": "K-85", + "applies_to": "E-21", + "statement": "`lifecycle_rule.condition.age` is 90" + }, + { + "applies_to": "E-21", + "statement": "`lifecycle_rule.action.type` is \"SetStorageClass\" and `lifecycle_rule.action.storage_class` is \"ARCHIVE\"", + "id": "K-86", + "source": "DR-COST-2" + }, + { + "applies_to": "E-8", + "statement": "`routing_mode` is \"REGIONAL\"", + "id": "K-87", + "source": "DR-PERF-1" + }, + { + "statement": "`autoscaling_policy.min_replicas` is 1", + "applies_to": "E-47", + "id": "K-88", + "source": "DR-SUST-1" + }, + { + "source": "DR-COST-1", + "statement": "`short_name` defines the taxonomy keys", + "id": "K-89", + "applies_to": "E-46" + }, + { + "applies_to": "E-54", + "source": "DR-COST-1", + "statement": "`destination` points to BigQuery", + "id": "K-90" + }, + { + "id": "K-91", + "applies_to": "E-55", + "source": "NFR-1", + "statement": "`tag_value` is set to the PCI regulated archetype" + }, + { + "id": "K-92", + "source": "DR-REL-2", + "statement": "`check_interval_sec` is set", + "applies_to": "E-19" + }, + { + "source": "NFR-5", + "applies_to": "E-23", + "id": "K-93", + "statement": "`disabled` is false" + }, + { + "applies_to": "E-26", + "source": "NFR-5", + "statement": "`disabled` is false", + "id": "K-94" + }, + { + "applies_to": "E-52", + "id": "K-95", + "statement": "`display_name` is set", + "source": "DR-OP-2" + }, + { + "applies_to": "E-53", + "id": "K-96", + "source": "DR-SUST-1", + "statement": "`base_instance_name` is set" + }, + { + "source": "DR-REL-1", + "applies_to": "E-57", + "statement": "`bfd.session_initialization_mode` is \"ACTIVE\"", + "id": "K-97" + }, + { + "source": "DR-REL-1", + "id": "K-99", + "applies_to": "E-56", + "statement": "`bfd.min_receive_interval` is 100" + }, + { + "id": "K-100", + "source": "DR-REL-1", + "applies_to": "E-57", + "statement": "`bfd.min_receive_interval` is 100" + }, + { + "source": "DR-REL-3", + "id": "K-101", + "statement": "`role` is \"roles/cloudkms.cryptoKeyDecrypter\"", + "applies_to": "E-59" + }, + { + "statement": "`tag_value` is set to the corp_workload archetype", + "applies_to": "E-60", + "source": "NFR-1", + "id": "K-102" + }, + { + "applies_to": "E-14", + "statement": "`router` is set to the Primary Router", + "id": "K-103", + "source": "NFR-3" + }, + { + "applies_to": "E-56", + "id": "K-104", + "statement": "`bfd.min_transmit_interval` is 100", + "source": "DR-REL-1" + }, + { + "source": "DR-REL-1", + "id": "K-105", + "applies_to": "E-57", + "statement": "`bfd.min_transmit_interval` is 100" + } + ], + "not_applicable": [ + { + "reason": "SCC activation and threat detection configuration are performed at the organization level outside of the foundational landing zone workspace.", + "recommendation": "DR-SEC-5" + }, + { + "reason": "Policy-as-code validation is defined in GitHub Actions CI/CD workflow YAML, which is outside the infrastructure terraform configuration.", + "recommendation": "DR-OP-1" + }, + { + "reason": "OpenTelemetry and Cloud Profiler are application-level agents installed within instance images or workloads, which are excluded from the foundational infrastructure deployment.", + "recommendation": "DR-PERF-2" + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/verdicts.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/verdicts.json new file mode 100644 index 0000000..0bef68d --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/verdicts.json @@ -0,0 +1,529 @@ +{ + "dry_run": { + "evidence": "Success! The configuration is valid.", + "verdict": "pass" + }, + "checks": [ + { + "check": "K-1", + "evidence": "display_name = \"Dev Folder\"", + "verdict": "pass" + }, + { + "check": "K-2", + "evidence": "display_name = \"Staging Folder\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-3", + "evidence": "display_name = \"Production Folder\"" + }, + { + "verdict": "pass", + "evidence": "tag_value = google_tags_tag_value.corp_workload.id", + "check": "K-4" + }, + { + "check": "K-5", + "verdict": "pass", + "evidence": "host_project = google_compute_shared_vpc_host_project.hub.project" + }, + { + "verdict": "pass", + "evidence": "host_project = google_compute_shared_vpc_host_project.hub.project", + "check": "K-6" + }, + { + "check": "K-7", + "evidence": "host_project = google_compute_shared_vpc_host_project.hub.project", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "private_ip_google_access = true", + "check": "K-8" + }, + { + "check": "K-9", + "evidence": "private_ip_google_access = true", + "verdict": "pass" + }, + { + "check": "K-10", + "evidence": "direction = \"INGRESS\"", + "verdict": "pass" + }, + { + "check": "K-11", + "verdict": "pass", + "evidence": "protocol = \"all\"" + }, + { + "verdict": "pass", + "evidence": "source_ranges = [\"0.0.0.0/0\"]", + "check": "K-12" + }, + { + "check": "K-13", + "evidence": "type = \"DEDICATED\"", + "verdict": "pass" + }, + { + "evidence": "type = \"DEDICATED\"", + "verdict": "pass", + "check": "K-14" + }, + { + "check": "K-15", + "verdict": "pass", + "evidence": "bandwidth = \"BPS_10G\"" + }, + { + "evidence": "bandwidth = \"BPS_10G\"", + "verdict": "pass", + "check": "K-16" + }, + { + "verdict": "pass", + "evidence": "router = google_compute_router.primary.id", + "check": "K-17" + }, + { + "evidence": "network = google_compute_network.hub.id", + "verdict": "pass", + "check": "K-18" + }, + { + "check": "K-19", + "evidence": "dest_range = \"0.0.0.0/0\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-20", + "evidence": "next_hop_ilb = google_compute_forwarding_rule.palo_alto_ilb.id" + }, + { + "verdict": "pass", + "check": "K-21", + "evidence": "issuer_uri = \"https://token.actions.githubusercontent.com\"" + }, + { + "verdict": "pass", + "check": "K-22", + "evidence": "member = \"principalSet://iam.googleapis.com/${google_iam_workload_identity_pool.wif_pool.name}/attribute.repository/my-org/my-repo\"" + }, + { + "evidence": "idp_metadata_xml = \"\"", + "verdict": "pass", + "check": "K-23" + }, + { + "evidence": "purpose = \"ENCRYPT_DECRYPT\"", + "verdict": "pass", + "check": "K-24" + }, + { + "check": "K-25", + "evidence": "purpose = \"ENCRYPT_DECRYPT\"", + "verdict": "pass" + }, + { + "check": "K-26", + "verdict": "pass", + "evidence": "rotation_period = \"7776000s\"" + }, + { + "check": "K-27", + "evidence": "rotation_period = \"7776000s\"", + "verdict": "pass" + }, + { + "evidence": "destination = \"storage.googleapis.com/${google_storage_bucket.audit_logs.name}\"", + "verdict": "pass", + "check": "K-28" + }, + { + "check": "K-29", + "verdict": "pass", + "evidence": "retention_period = 31536000" + }, + { + "check": "K-30", + "verdict": "pass", + "evidence": "is_locked = true" + }, + { + "evidence": "restricted_services = [\"compute.googleapis.com\"]", + "verdict": "pass", + "check": "K-31" + }, + { + "check": "K-32", + "evidence": "role = \"roles/cloudkms.cryptoKeyEncrypterDecrypter\"", + "verdict": "pass" + }, + { + "check": "K-33", + "evidence": "region = \"europe-west2\"", + "verdict": "pass" + }, + { + "check": "K-34", + "verdict": "pass", + "evidence": "region = \"europe-west2\"" + }, + { + "verdict": "pass", + "check": "K-35", + "evidence": "region = \"europe-west1\"" + }, + { + "verdict": "pass", + "evidence": "constraint = \"gcp.resourceLocations\"", + "check": "K-36" + }, + { + "check": "K-37", + "verdict": "pass", + "evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]" + }, + { + "check": "K-38", + "evidence": "project = var.project_id", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "distribution_policy_zones = [\"europe-west2-a\", \"europe-west2-b\", \"europe-west2-c\"]", + "check": "K-39" + }, + { + "verdict": "pass", + "check": "K-40", + "evidence": "notification_channels = [google_monitoring_notification_channel.email.name]" + }, + { + "verdict": "pass", + "evidence": "condition_threshold is defined", + "check": "K-41" + }, + { + "evidence": "currency_code = \"GBP\"", + "check": "K-42", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "threshold_percent includes 0.5, 0.8, 0.9, 1.0", + "check": "K-43" + }, + { + "check": "K-44", + "verdict": "pass", + "evidence": "units = \"25000\"" + }, + { + "evidence": "advertise_mode = \"CUSTOM\"", + "verdict": "pass", + "check": "K-45" + }, + { + "verdict": "pass", + "evidence": "advertise_mode = \"CUSTOM\"", + "check": "K-46" + }, + { + "verdict": "pass", + "evidence": "bandwidth = \"BPS_10G\"", + "check": "K-47" + }, + { + "evidence": "bandwidth = \"BPS_10G\"", + "verdict": "pass", + "check": "K-48" + }, + { + "check": "K-49", + "verdict": "pass", + "evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]" + }, + { + "evidence": "resources = [\"projects/${var.prod_project_id}\", \"projects/${var.project_id}\"]", + "check": "K-50", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-51", + "evidence": "ingress_policies and egress_policies are configured" + }, + { + "verdict": "pass", + "evidence": "constraint = \"compute.skipDefaultNetworkCreation\"", + "check": "K-52" + }, + { + "evidence": "enforced = true", + "verdict": "pass", + "check": "K-53" + }, + { + "check": "K-54", + "evidence": "constraint = \"compute.vmExternalIpAccess\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "evidence": "all = true", + "check": "K-55" + }, + { + "evidence": "constraint = \"iam.disableServiceAccountKeyCreation\"", + "check": "K-56", + "verdict": "pass" + }, + { + "check": "K-57", + "evidence": "enforced = true", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-58", + "evidence": "constraint = \"gcp.resourceLocations\"" + }, + { + "evidence": "values = [\"in:europe-west2\", \"in:europe-west1\"]", + "verdict": "pass", + "check": "K-59" + }, + { + "verdict": "pass", + "check": "K-60", + "evidence": "enable_confidential_compute = true" + }, + { + "check": "K-61", + "evidence": "name = \"CREDIT_CARD_NUMBER\"", + "verdict": "pass" + }, + { + "check": "K-62", + "evidence": "event_data_content_type = \"application/json\"", + "verdict": "pass" + }, + { + "evidence": "attribute = \"type\", value = \"google.cloud.securitycenter.finding.v1.saved\"", + "check": "K-63", + "verdict": "pass" + }, + { + "evidence": "event_type = \"google.cloud.securitycenter.finding.v1.saved\"", + "verdict": "pass", + "check": "K-64" + }, + { + "verdict": "pass", + "evidence": "interconnect = var.interconnect_1_url", + "check": "K-65" + }, + { + "evidence": "interconnect = var.interconnect_2_url", + "verdict": "pass", + "check": "K-66" + }, + { + "check": "K-67", + "evidence": "router = google_compute_router.primary.id", + "verdict": "pass" + }, + { + "check": "K-68", + "evidence": "session_initialization_mode = \"ACTIVE\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-69", + "evidence": "distribution_policy_zones = [\"europe-west2-a\", \"europe-west2-b\", \"europe-west2-c\"]" + }, + { + "evidence": "load_balancing_scheme = \"INTERNAL\"", + "check": "K-70", + "verdict": "pass" + }, + { + "evidence": "health_checks = [google_compute_health_check.palo_alto_ilb_hc.id]", + "verdict": "pass", + "check": "K-71" + }, + { + "verdict": "pass", + "check": "K-72", + "evidence": "location = \"europe-west2\"" + }, + { + "evidence": "location = \"europe-west1\"", + "check": "K-73", + "verdict": "pass" + }, + { + "evidence": "role = \"roles/cloudkms.cryptoKeyDecrypter\"", + "verdict": "pass", + "check": "K-74" + }, + { + "verdict": "pass", + "evidence": "data_locations = [\"EUROPE-WEST1\", \"EUROPE-WEST2\"]", + "check": "K-75" + }, + { + "check": "K-76", + "verdict": "pass", + "evidence": "name = \"${var.project_id}-tfstate-primary\"" + }, + { + "evidence": "name = \"${var.project_id}-tfstate-secondary\"", + "check": "K-77", + "verdict": "pass" + }, + { + "evidence": "enabled = true", + "check": "K-78", + "verdict": "pass" + }, + { + "check": "K-79", + "evidence": "enabled = true", + "verdict": "pass" + }, + { + "evidence": "goal = 0.999", + "verdict": "pass", + "check": "K-80" + }, + { + "evidence": "filter = \"select_slo_burn_rate(\\\"${google_monitoring_slo.platform_slo.name}\\\", 3600s)\"", + "verdict": "pass", + "check": "K-81" + }, + { + "verdict": "pass", + "check": "K-82", + "evidence": "service = \"recommender.googleapis.com\"" + }, + { + "verdict": "pass", + "evidence": "age = 30", + "check": "K-83" + }, + { + "verdict": "pass", + "check": "K-84", + "evidence": "type = \"SetStorageClass\", storage_class = \"NEARLINE\"" + }, + { + "check": "K-85", + "verdict": "pass", + "evidence": "age = 90" + }, + { + "check": "K-86", + "evidence": "type = \"SetStorageClass\", storage_class = \"ARCHIVE\"", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-87", + "evidence": "routing_mode = \"REGIONAL\"" + }, + { + "evidence": "min_replicas = 1", + "check": "K-88", + "verdict": "pass" + }, + { + "evidence": "short_name = \"environment\"", + "check": "K-89", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-90", + "evidence": "destination = \"bigquery.googleapis.com/...\"" + }, + { + "verdict": "pass", + "check": "K-91", + "evidence": "tag_value = google_tags_tag_value.pci_regulated.id" + }, + { + "check": "K-92", + "evidence": "check_interval_sec = 5", + "verdict": "pass" + }, + { + "verdict": "pass", + "check": "K-93", + "evidence": "disabled = false" + }, + { + "verdict": "pass", + "check": "K-94", + "evidence": "disabled = false" + }, + { + "evidence": "display_name = \"Platform Custom Service\"", + "check": "K-95", + "verdict": "pass" + }, + { + "check": "K-96", + "verdict": "pass", + "evidence": "base_instance_name = \"palo-alto-secondary\"" + }, + { + "verdict": "pass", + "evidence": "session_initialization_mode = \"ACTIVE\"", + "check": "K-97" + }, + { + "evidence": "min_receive_interval = 100", + "check": "K-99", + "verdict": "pass" + }, + { + "check": "K-100", + "verdict": "pass", + "evidence": "min_receive_interval = 100" + }, + { + "check": "K-101", + "evidence": "role = \"roles/cloudkms.cryptoKeyDecrypter\"", + "verdict": "pass" + }, + { + "evidence": "tag_value = google_tags_tag_value.corp_workload.id", + "verdict": "pass", + "check": "K-102" + }, + { + "verdict": "pass", + "evidence": "router = google_compute_router.primary.id", + "check": "K-103" + }, + { + "verdict": "pass", + "evidence": "min_transmit_interval = 100", + "check": "K-104" + }, + { + "check": "K-105", + "evidence": "min_transmit_interval = 100", + "verdict": "pass" + } + ], + "proposed_additions": [] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/work-breakdown.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/work-breakdown.json new file mode 100644 index 0000000..83069d7 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/003.sources/work-breakdown.json @@ -0,0 +1,332 @@ +{ + "work_packages": [ + { + "id": "WP-0", + "name": "Platform Foundation", + "parent": null, + "elements": [], + "scope": "The complete enterprise-grade landing zone.", + "produces": [ + "an operational landing zone" + ], + "skills": [ + "cloud architecture" + ] + }, + { + "id": "WP-1", + "name": "Automated Infrastructure Orchestration", + "parent": "WP-0", + "elements": [ + "E-42", + "E-43" + ], + "scope": "CI/CD orchestration, state management, and API enablement.", + "produces": [ + "Terraform state buckets for primary and secondary regions" + ], + "skills": [ + "cloud engineering" + ] + }, + { + "id": "WP-2", + "name": "Federated Identity and Role Governance", + "parent": "WP-0", + "elements": [ + "E-23", + "E-24", + "E-25", + "E-26", + "E-27", + "E-45" + ], + "scope": "Authentication for delivery pipelines and workforce SSO.", + "produces": [ + "Workload Identity Federation for GitHub Actions and Entra ID SSO" + ], + "skills": [ + "identity and access management" + ] + }, + { + "id": "WP-3", + "name": "Resource Hierarchy and Environment Boundary", + "parent": "WP-0", + "elements": [ + "E-1", + "E-2", + "E-3", + "E-4", + "E-5", + "E-6", + "E-7", + "E-8", + "E-9", + "E-10", + "E-34", + "E-35", + "E-36", + "E-37", + "E-38", + "E-50", + "E-51", + "E-55", + "E-60" + ], + "scope": "Google Cloud resource hierarchy, shared VPC projects, networking, and organization policies.", + "produces": [ + "Folders, projects, networking, and organization policies ensuring isolation" + ], + "skills": [ + "cloud engineering", + "network engineering" + ] + }, + { + "id": "WP-4", + "name": "Cryptographic Key Lifecycle", + "parent": "WP-0", + "elements": [ + "E-28", + "E-29", + "E-30", + "E-31", + "E-49", + "E-58", + "E-59" + ], + "scope": "Customer-managed encryption keys (CMEK) and cryptographic isolation.", + "produces": [ + "Primary and secondary KMS key rings and crypto keys with IAM bindings" + ], + "skills": [ + "security engineering" + ] + }, + { + "id": "WP-5", + "name": "Audit, Compliance, and Financial Governance", + "parent": "WP-0", + "elements": [ + "E-21", + "E-22", + "E-33", + "E-39", + "E-46", + "E-54" + ], + "scope": "Immutable compliance logs, DLP templates, and budget enforcement.", + "produces": [ + "Immutable audit log storage, billing budgets, and DLP inspection templates" + ], + "skills": [ + "cloud engineering", + "finops" + ] + }, + { + "id": "WP-6", + "name": "Centralised Outbound Traffic Inspection", + "parent": "WP-0", + "elements": [ + "E-15", + "E-16", + "E-17", + "E-18", + "E-19", + "E-20", + "E-47", + "E-48", + "E-53" + ], + "scope": "Egress inspection and threat prevention using Palo Alto VM-Series.", + "produces": [ + "Palo Alto VM-Series firewalls with internal load balancing and egress routing" + ], + "skills": [ + "network security engineering" + ] + }, + { + "id": "WP-7", + "name": "Dedicated Private Hybrid Connectivity", + "parent": "WP-0", + "elements": [ + "E-11", + "E-12", + "E-13", + "E-14", + "E-56", + "E-57" + ], + "scope": "Dedicated Interconnect termination and dynamic BGP routing.", + "produces": [ + "Primary and secondary Cloud Routers with Dedicated Interconnect VLAN attachments" + ], + "skills": [ + "network engineering" + ] + }, + { + "id": "WP-8", + "name": "Platform Telemetry and Operational Alerting", + "parent": "WP-0", + "elements": [ + "E-32", + "E-40", + "E-41", + "E-44", + "E-52" + ], + "scope": "SLO tracking, multi-burn-rate alerting, and SCC automated remediation.", + "produces": [ + "SLOs, alerting policies, and SCC automated remediation functions" + ], + "skills": [ + "site reliability engineering" + ] + } + ], + "activities": [ + { + "id": "A-1", + "work_package": "WP-1", + "name": "Enable Google Cloud APIs", + "days": 0.5, + "after": [] + }, + { + "id": "A-2", + "work_package": "WP-1", + "name": "Setup CI/CD Pipeline and State Buckets", + "days": 1, + "after": [ + "A-1" + ] + }, + { + "id": "A-3", + "work_package": "WP-2", + "name": "Configure Workload Identity and Entra ID SSO", + "days": 1.5, + "after": [ + "A-2" + ] + }, + { + "id": "A-4", + "work_package": "WP-3", + "name": "Provision Resource Hierarchy and Enforce Org Policies", + "days": 2.5, + "after": [ + "A-3" + ] + }, + { + "id": "A-5", + "work_package": "WP-3", + "name": "Provision Hub and Spoke VPC Networks", + "days": 2, + "after": [ + "A-4" + ] + }, + { + "id": "A-6", + "work_package": "WP-7", + "name": "Order and provision the Interconnect carrier circuits", + "days": 30, + "owner": "customer", + "assumption": "The customer manages the carrier's lead time and physical cross-connects.", + "after": [] + }, + { + "id": "A-7", + "work_package": "WP-4", + "name": "Configure Cloud KMS and Key Rings", + "days": 1.5, + "after": [ + "A-4" + ] + }, + { + "id": "A-8", + "work_package": "WP-5", + "name": "Set up Audit Logging, DLP, and Billing Budgets", + "days": 2, + "after": [ + "A-4" + ] + }, + { + "id": "A-9", + "work_package": "WP-6", + "name": "Deploy Palo Alto Egress Firewall Appliances", + "days": 5, + "after": [ + "A-5" + ] + }, + { + "id": "A-10", + "work_package": "WP-7", + "name": "Configure Cloud Routers and Terminate BGP over Interconnect", + "days": 3, + "after": [ + "A-5", + "A-6" + ] + }, + { + "id": "A-11", + "work_package": "WP-3", + "name": "Implement VPC Service Controls", + "days": 2, + "after": [ + "A-5", + "A-7", + "A-8" + ] + }, + { + "id": "A-12", + "work_package": "WP-8", + "name": "Deploy Platform Telemetry, SLOs, and SCC Auto-remediation", + "days": 2.5, + "after": [ + "A-8", + "A-11" + ] + }, + { + "id": "A-13", + "work_package": "WP-1", + "name": "Run validation in development environment", + "days": 1, + "after": [ + "A-9", + "A-10", + "A-12" + ] + }, + { + "id": "A-14", + "work_package": "WP-1", + "name": "Run validation in staging environment", + "days": 1, + "after": [ + "A-13" + ] + }, + { + "id": "A-15", + "work_package": "WP-1", + "name": "Run validation in production environment", + "days": 1, + "after": [ + "A-14" + ] + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/history.json b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/history.json new file mode 100644 index 0000000..b135286 --- /dev/null +++ b/eval/results/20261021T203624Z/deliverables/cases/case-08-landing-zone/intent/history.json @@ -0,0 +1,83 @@ +{ + "customer_case": "case-08-landing-zone", + "intents": [ + { + "version": 1, + "recorded_at": "2026-10-21T21:30:39+00:00", + "statement": "intent/001.md", + "workflow_request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.", + "execution_id": "case-02-from-architecture-r1", + "session_id": "bc30257a-ff81-4114-86b9-83afd8a8ae5b", + "status": "completed_with_warnings", + "execution": "executions/case-02-from-architecture-r1", + "baseline": null, + "scope": { + "completed": "intent/001.scope.completed.yaml", + "supplied": "intent/001.scope.yaml" + }, + "sources": { + "execution.json": "8bf269deffc8e4bce474bd8d66959c03b04de01dc137eefaf96ead74b46a5a45", + "scope.json": "57dc28ab7397f514ef0e83b292fc2165554ac2b516775c973f4965765a9e905f", + "requirements.json": "5aecbef9eaea254fb5123bcc1d107b4a7b7d4d45d60d940464d1fde690d2973d", + "recommendations.json": "95958b1a107705280d65c9d088bba57352d1f9fa8e67e89b2b3379f78d91cd24", + "components.json": "49e47dbfa90a85d086711eb2ba66370c145f13fd3cd3595c1df1366224db392c", + "products.json": "75d51cba319f83000e6f85277b0fb21a7ec02faabac332c5d1800c8f512136db", + "target-architecture.json": "40879607aaa7041103d8f4d3ba804df8751d2159aa0ce0977d7d821d35f7608b", + "verdicts.json": "75e17da790266e94a6775d90857a9b503cacd62fd55c92be2d7a3435de301853", + "work-breakdown.json": "057fd1b0ecd6350a6f9f6101b712a61e78fc10d758a67e785a1dca7e1e674dc3" + } + }, + { + "version": 2, + "recorded_at": "2026-10-21T22:19:12+00:00", + "statement": "intent/002.md", + "workflow_request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.", + "execution_id": "case-02-from-architecture-r2", + "session_id": "d29687eb-8857-48e4-8c98-c2251c1c2c8f", + "status": "completed", + "execution": "executions/case-02-from-architecture-r2", + "baseline": null, + "scope": { + "completed": "intent/002.scope.completed.yaml", + "supplied": "intent/002.scope.yaml" + }, + "sources": { + "execution.json": "8bf269deffc8e4bce474bd8d66959c03b04de01dc137eefaf96ead74b46a5a45", + "scope.json": "57dc28ab7397f514ef0e83b292fc2165554ac2b516775c973f4965765a9e905f", + "requirements.json": "5aecbef9eaea254fb5123bcc1d107b4a7b7d4d45d60d940464d1fde690d2973d", + "recommendations.json": "95958b1a107705280d65c9d088bba57352d1f9fa8e67e89b2b3379f78d91cd24", + "components.json": "49e47dbfa90a85d086711eb2ba66370c145f13fd3cd3595c1df1366224db392c", + "products.json": "75d51cba319f83000e6f85277b0fb21a7ec02faabac332c5d1800c8f512136db", + "target-architecture.json": "40879607aaa7041103d8f4d3ba804df8751d2159aa0ce0977d7d821d35f7608b", + "verdicts.json": "75e17da790266e94a6775d90857a9b503cacd62fd55c92be2d7a3435de301853", + "work-breakdown.json": "057fd1b0ecd6350a6f9f6101b712a61e78fc10d758a67e785a1dca7e1e674dc3" + } + }, + { + "version": 3, + "recorded_at": "2026-10-21T22:39:26+00:00", + "statement": "intent/003.md", + "workflow_request": "We are a UK-regulated payment platform expanding into Google Cloud. We need an\nenterprise-grade landing zone ready in under 10 weeks to support our core payments\nengine. All payment processing must comply with DORA and PCI-DSS v4.0. We have\nexisting on-premises systems that require private, low-latency connectivity over a\n10Gbps dedicated interconnect. We use Entra ID for corporate single sign-on and\nGitHub Actions for our delivery pipelines. Security requires that all egress\ntraffic flows through centralised inspection appliances before touching the public\ninternet, and customer data must be encrypted with keys we control.", + "execution_id": "case-02-from-architecture-r3", + "session_id": "90a6f5db-fafb-4808-99f3-8c9adba4cbf6", + "status": "completed", + "execution": "executions/case-02-from-architecture-r3", + "baseline": null, + "scope": { + "completed": "intent/003.scope.completed.yaml", + "supplied": "intent/003.scope.yaml" + }, + "sources": { + "execution.json": "8bf269deffc8e4bce474bd8d66959c03b04de01dc137eefaf96ead74b46a5a45", + "scope.json": "57dc28ab7397f514ef0e83b292fc2165554ac2b516775c973f4965765a9e905f", + "requirements.json": "5aecbef9eaea254fb5123bcc1d107b4a7b7d4d45d60d940464d1fde690d2973d", + "recommendations.json": "95958b1a107705280d65c9d088bba57352d1f9fa8e67e89b2b3379f78d91cd24", + "components.json": "49e47dbfa90a85d086711eb2ba66370c145f13fd3cd3595c1df1366224db392c", + "products.json": "75d51cba319f83000e6f85277b0fb21a7ec02faabac332c5d1800c8f512136db", + "target-architecture.json": "40879607aaa7041103d8f4d3ba804df8751d2159aa0ce0977d7d821d35f7608b", + "verdicts.json": "75e17da790266e94a6775d90857a9b503cacd62fd55c92be2d7a3435de301853", + "work-breakdown.json": "057fd1b0ecd6350a6f9f6101b712a61e78fc10d758a67e785a1dca7e1e674dc3" + } + } + ] +} diff --git a/eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r1.json b/eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r1.json new file mode 100644 index 0000000..761559a --- /dev/null +++ b/eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r1.json @@ -0,0 +1,76 @@ +{ + "execution_id": "case-01-landing-zone-shared-r1", + "customer_case": "bench-landing-zone-shared", + "status": "completed", + "role_skills": { + "capability-modeller": [ + "operating-capabilities", + "operating-chain" + ], + "operating-model-architect": [ + "operating-capabilities", + "operating-chain", + "rate-card" + ], + "operations-analyst": [ + "operating-scoping", + "operating-chain" + ], + "orchestrator": [], + "packager": [ + "operating-guide" + ], + "reply-writer": [], + "scope-analyst": [ + "operating-scoping" + ], + "validator": [ + "operating-chain" + ] + }, + "skill_versions": { + "operating-capabilities": "49a7e66857a1", + "operating-chain": "ec53ad8875e6", + "operating-guide": "85b48d5998da", + "operating-scoping": "08da33f8cddf", + "rate-card": "f86d17a9bcce" + }, + "candidates": [ + { + "type": "revision", + "deliverable": "target_operating_model", + "reason": "OC-4 failed: The finding AF-1 does not stand because the operating model could provide the configuration of alerts itself as a process or using a tool.; OC-5 failed: The finding AF-2 does not stand because the operating model could provide a backup and recovery mechanism itself as a process or a tool.; OC-1 failed: Capability C-10 operates only during business hours, which is insufficient to ensure 99.9% availability as it requires 24x7 cover.; OC-8 failed: Capability C-15 covers measuring SLA achievement but does not cover operational costs, which is handled by C-14.", + "requested_by": "orchestrator" + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-4", + "evidence": "The finding AF-1 does not stand because the operating model could provide the configuration of alerts itself as a process or using a tool." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-5", + "evidence": "The finding AF-2 does not stand because the operating model could provide a backup and recovery mechanism itself as a process or a tool." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-1", + "evidence": "Capability C-10 operates only during business hours, which is insufficient to ensure 99.9% availability as it requires 24x7 cover." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-8", + "evidence": "Capability C-15 covers measuring SLA achievement but does not cover operational costs, which is handled by C-14." + }, + { + "type": "unverifiable_check", + "attempt": 1, + "check": "OC-2", + "evidence": "The condition that audit logs cannot be deleted is a technical constraint that cannot be verified by an operating model capability." + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r2.json b/eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r2.json new file mode 100644 index 0000000..9abeebf --- /dev/null +++ b/eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r2.json @@ -0,0 +1,64 @@ +{ + "execution_id": "case-01-landing-zone-shared-r2", + "customer_case": "bench-landing-zone-shared", + "status": "completed", + "role_skills": { + "capability-modeller": [ + "operating-capabilities", + "operating-chain" + ], + "operating-model-architect": [ + "operating-capabilities", + "operating-chain", + "rate-card" + ], + "operations-analyst": [ + "operating-scoping", + "operating-chain" + ], + "orchestrator": [], + "packager": [ + "operating-guide" + ], + "reply-writer": [], + "scope-analyst": [ + "operating-scoping" + ], + "validator": [ + "operating-chain" + ] + }, + "skill_versions": { + "operating-capabilities": "49a7e66857a1", + "operating-chain": "ec53ad8875e6", + "operating-guide": "85b48d5998da", + "operating-scoping": "08da33f8cddf", + "rate-card": "f86d17a9bcce" + }, + "candidates": [ + { + "type": "revision", + "deliverable": "target_operating_model", + "reason": "OC-1 failed: Finding AF-1 does not stand. The operating model should provide alert configurations itself using the existing native Google Cloud monitoring tools, rather than raising a finding against the solution.; OC-3 failed: Finding AF-1 does not stand. The operating model should provide alert configurations itself using the existing native Google Cloud monitoring tools, rather than raising a finding against the solution.; SC-budget failed: A month's cost of operating is compared with the budget of 60,000 GBP. The cost is 64,125 GBP, over the budget.", + "requested_by": "orchestrator" + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-1", + "evidence": "Finding AF-1 does not stand. The operating model should provide alert configurations itself using the existing native Google Cloud monitoring tools, rather than raising a finding against the solution." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-3", + "evidence": "Finding AF-1 does not stand. The operating model should provide alert configurations itself using the existing native Google Cloud monitoring tools, rather than raising a finding against the solution." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "SC-budget", + "evidence": "A month's cost of operating is compared with the budget of 60,000 GBP. The cost is 64,125 GBP, over the budget." + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r3.json b/eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r3.json new file mode 100644 index 0000000..7fb3c41 --- /dev/null +++ b/eval/results/20261021T203624Z/lessons/candidates/bench-landing-zone-shared/case-01-landing-zone-shared-r3.json @@ -0,0 +1,59 @@ +{ + "execution_id": "case-01-landing-zone-shared-r3", + "customer_case": "bench-landing-zone-shared", + "status": "completed", + "role_skills": { + "capability-modeller": [ + "operating-capabilities", + "operating-chain" + ], + "operating-model-architect": [ + "operating-capabilities", + "operating-chain", + "rate-card" + ], + "operations-analyst": [ + "operating-scoping", + "operating-chain" + ], + "orchestrator": [], + "packager": [ + "operating-guide" + ], + "reply-writer": [], + "scope-analyst": [ + "operating-scoping" + ], + "validator": [ + "operating-chain" + ] + }, + "skill_versions": { + "operating-capabilities": "49a7e66857a1", + "operating-chain": "ec53ad8875e6", + "operating-guide": "85b48d5998da", + "operating-scoping": "08da33f8cddf", + "rate-card": "f86d17a9bcce" + }, + "candidates": [ + { + "type": "revision", + "deliverable": "target_operating_model", + "reason": "OC-1 failed: The finding AF-1 does not stand because the operating model can provide alert configuration itself via C-2 (monitoring-and-alerting) using the Cloud provider's native operations tooling. The operating model should configure the alerts rather than expecting the solution to specify them.", + "requested_by": "orchestrator" + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-1", + "evidence": "The finding AF-1 does not stand because the operating model can provide alert configuration itself via C-2 (monitoring-and-alerting) using the Cloud provider's native operations tooling. The operating model should configure the alerts rather than expecting the solution to specify them." + }, + { + "type": "proposed_addition", + "attempt": 1, + "reason": "Plausibility of effort for continuous 24x7 analysis.", + "applies_to": "C-13", + "statement": "C-13 (security-monitoring-and-response) has an effort of 40 hours per month for continuous monitoring of security events and continuous analysis of audit logs. If the analysis is manual, 40 hours per month is insufficient for continuous 24x7 coverage. Consider clarifying if this relies on automated SIEM alerting or increasing the effort." + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r1.json b/eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r1.json new file mode 100644 index 0000000..3d68f7e --- /dev/null +++ b/eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r1.json @@ -0,0 +1,244 @@ +{ + "execution_id": "case-02-from-architecture-r1", + "customer_case": "case-08-landing-zone", + "status": "completed_with_warnings", + "role_skills": { + "capability-modeller": [ + "operating-capabilities", + "operating-chain" + ], + "operating-model-architect": [ + "operating-capabilities", + "operating-chain", + "rate-card" + ], + "operations-analyst": [ + "operating-scoping", + "operating-chain" + ], + "orchestrator": [], + "packager": [ + "operating-guide" + ], + "reply-writer": [], + "scope-analyst": [ + "operating-scoping" + ], + "validator": [ + "operating-chain" + ] + }, + "skill_versions": { + "operating-capabilities": "49a7e66857a1", + "operating-chain": "ec53ad8875e6", + "operating-guide": "85b48d5998da", + "operating-scoping": "08da33f8cddf", + "rate-card": "f86d17a9bcce" + }, + "candidates": [ + { + "type": "revision", + "deliverable": "target_operating_model", + "reason": "OC-2 failed: The check requires 'immediate failover traffic reception', but Capability C-9 is scoped for 'governing the 4-hour disaster recovery automation logic', meaning it cannot provide immediate failover as designed.; OC-12 failed: The check requires roles to be 'continuously rightsized', but Capability C-11 is designed to execute 'periodic least-privilege adjustments', which cannot meet the continuous requirement.; SC-change-S-2 failed: Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed. No Capability for making changes covers it with a responsible Role.; SC-change-S-3 failed: Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed. No Capability for making changes covers it with a responsible Role.; SC-change-S-6 failed: Federated Identity and Role Governance Authority (S-6) must be changed. No Capability for making changes covers it with a responsible Role.; SC-change-S-8 failed: Platform Telemetry and Operational Alerting Engine (S-8) must be changed. No Capability for making changes covers it with a responsible Role.; SC-change-S-9 failed: Financial Governance and Budget Enforcement Monitor (S-9) must be changed. No Capability for making changes covers it with a responsible Role.", + "requested_by": "orchestrator" + }, + { + "type": "revision", + "deliverable": "capability_model", + "reason": "Check OC-2 failed in two validations in a row and may not be implementable as written (The check requires maintaining an active warm standby ready to receive failover traffic, but Capability C-9 is scoped for 'governing the 4-hour disaster recovery automation logic', meaning it cannot provide immediate failover as designed. Capability C-9 should be revised to govern immediate failover traffic reception.). Rewrite it so it can be met, or remove it with a reason. Check SC-change-S-2 failed in two validations in a row and may not be implementable as written (Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed. No Capability for making changes covers it with a responsible Role.). Rewrite it so it can be met, or remove it with a reason. Check SC-change-S-3 failed in two validations in a row and may not be implementable as written (Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed. No Capability for making changes covers it with a responsible Role.). Rewrite it so it can be met, or remove it with a reason. Check SC-change-S-6 failed in two validations in a row and may not be implementable as written (Federated Identity and Role Governance Authority (S-6) must be changed. No Capability for making changes covers it with a responsible Role.). Rewrite it so it can be met, or remove it with a reason. Check SC-change-S-8 failed in two validations in a row and may not be implementable as written (Platform Telemetry and Operational Alerting Engine (S-8) must be changed. No Capability for making changes covers it with a responsible Role.). Rewrite it so it can be met, or remove it with a reason. Check SC-change-S-9 failed in two validations in a row and may not be implementable as written (Financial Governance and Budget Enforcement Monitor (S-9) must be changed. No Capability for making changes covers it with a responsible Role.). Rewrite it so it can be met, or remove it with a reason.", + "requested_by": "orchestrator" + }, + { + "type": "revision", + "deliverable": "target_operating_model", + "reason": "OC-11 failed: Capability C-7 focuses on configuration state and data residency alignment, but does not cover optimizing carbon-free energy utilization based on regional scores. The Check should be mapped to a deployment capability like C-5 or C-6, or C-7 should be updated.", + "requested_by": "orchestrator" + }, + { + "type": "revision", + "deliverable": "capability_model", + "reason": "Check OC-11 failed in two validations in a row and may not be implementable as written (The check requires automated deployments to optimize carbon-free energy utilization, and is mapped to C-6. However, C-6's basis only covers managing automated deployment pipelines to deliver foundational updates rapidly and reliably, without mentioning carbon-free energy optimization. C-6 should be updated to include this optimization.). Rewrite it so it can be met, or remove it with a reason.", + "requested_by": "orchestrator" + }, + { + "type": "revision", + "deliverable": "target_operating_model", + "reason": "OC-3 failed: Capability C-7 focuses on tracking infrastructure state files and ensuring PCI-DSS v4.0 baseline alignment, but does not mention auditing for UK/EU data residency. Data residency is instead covered by C-21. Capability C-7 should be updated to include auditing data residency state, or the Check should be re-mapped to C-21.; OC-18 failed: Capability C-1 covers triage and managing priority incidents, but does not explicitly mention conducting a written review for every Priority 1 incident. The capability basis should be updated to include post-incident written reviews.; OC-21 failed: Capability C-4 covers fulfilling non-standard operational requests, but does not explicitly state that each kind of foundation service request has a stated way to ask and a stated time to fulfil (e.g., maintaining a service request catalogue). The capability basis should be updated to include maintaining stated request methods and fulfilment times.; OC-23 failed: Capability C-6 covers resolving deployment failures and managing automated pipelines, but does not explicitly guarantee a rollback capability within the 4-hour RTO. The capability should be updated to cover rapid rollback procedures.", + "requested_by": "orchestrator" + }, + { + "type": "revision_limit", + "deliverable": "target_operating_model", + "reason": "OC-2 failed: The check requires the secondary region to be 'continuously maintained' as an active warm standby, but Capability C-9 is configured for 'business' hours, meaning maintenance cannot be continuous (24x7). C-9 should be 24x7 or another capability should provide continuous maintenance.; OC-4 failed: The check requires validation within 10 weeks, but Capability C-16 covers managing the roadmap without explicitly mentioning the 10-week requirement (which is instead mentioned in C-18). C-16 must mention the 10-week validation or the check mapped to C-18.; OC-9 failed: The check requires that expenditure does not exceed target allocations without 'documented operational sign-off', but Capability C-14 only mentions 'escalating anomalies' and does not explicitly guarantee documented sign-off.; OC-12 failed: The check requires identity permissions to be 'continuously reviewed', but Capability C-11 operates only during 'business' hours, meaning reviews cannot be continuous. C-11 must be 24x7.; OC-17 failed: The check requires verifiable alignment with DORA, PCI-DSS v4.0, and ISO 27001. Capability C-7 only ensures alignment with PCI-DSS v4.0 baseline configurations, completely missing DORA and ISO 27001 controls." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-2", + "evidence": "The check requires 'immediate failover traffic reception', but Capability C-9 is scoped for 'governing the 4-hour disaster recovery automation logic', meaning it cannot provide immediate failover as designed." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-12", + "evidence": "The check requires roles to be 'continuously rightsized', but Capability C-11 is designed to execute 'periodic least-privilege adjustments', which cannot meet the continuous requirement." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "SC-change-S-2", + "evidence": "Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed. No Capability for making changes covers it with a responsible Role." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "SC-change-S-3", + "evidence": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed. No Capability for making changes covers it with a responsible Role." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "SC-change-S-6", + "evidence": "Federated Identity and Role Governance Authority (S-6) must be changed. No Capability for making changes covers it with a responsible Role." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "SC-change-S-8", + "evidence": "Platform Telemetry and Operational Alerting Engine (S-8) must be changed. No Capability for making changes covers it with a responsible Role." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "SC-change-S-9", + "evidence": "Financial Governance and Budget Enforcement Monitor (S-9) must be changed. No Capability for making changes covers it with a responsible Role." + }, + { + "type": "proposed_addition", + "attempt": 1, + "applies_to": "C-13", + "reason": "The stated effort of 30 hours per month is implausible for 'Continuously analyzing egress inspection logs' on a 24x7 basis. Continuous manual analysis would require full-time dedicated shift coverage far exceeding 30 hours a month.", + "statement": "Revise the effort for C-13 to reflect the true cost of continuous manual log analysis, or change the basis to rely on automated anomaly alert response rather than continuous manual analysis." + }, + { + "type": "failed_check", + "attempt": 2, + "check": "OC-2", + "evidence": "The check requires maintaining an active warm standby ready to receive failover traffic, but Capability C-9 is scoped for 'governing the 4-hour disaster recovery automation logic', meaning it cannot provide immediate failover as designed. Capability C-9 should be revised to govern immediate failover traffic reception." + }, + { + "type": "failed_check", + "attempt": 2, + "check": "SC-change-S-2", + "evidence": "Dedicated Private Hybrid Connectivity Gateway (S-2) must be changed. No Capability for making changes covers it with a responsible Role." + }, + { + "type": "failed_check", + "attempt": 2, + "check": "SC-change-S-3", + "evidence": "Centralised Outbound Traffic Inspection and Egress Boundary (S-3) must be changed. No Capability for making changes covers it with a responsible Role." + }, + { + "type": "failed_check", + "attempt": 2, + "check": "SC-change-S-6", + "evidence": "Federated Identity and Role Governance Authority (S-6) must be changed. No Capability for making changes covers it with a responsible Role." + }, + { + "type": "failed_check", + "attempt": 2, + "check": "SC-change-S-8", + "evidence": "Platform Telemetry and Operational Alerting Engine (S-8) must be changed. No Capability for making changes covers it with a responsible Role." + }, + { + "type": "failed_check", + "attempt": 2, + "check": "SC-change-S-9", + "evidence": "Financial Governance and Budget Enforcement Monitor (S-9) must be changed. No Capability for making changes covers it with a responsible Role." + }, + { + "type": "failed_check", + "attempt": 3, + "check": "OC-11", + "evidence": "Capability C-7 focuses on configuration state and data residency alignment, but does not cover optimizing carbon-free energy utilization based on regional scores. The Check should be mapped to a deployment capability like C-5 or C-6, or C-7 should be updated." + }, + { + "type": "failed_check", + "attempt": 4, + "check": "OC-4", + "evidence": "The check requires handover for workload ingestion within 10 weeks of project start, and is mapped to Capability C-16. However, C-16's basis focuses on roadmap and backlog management and does not mention handover or the 10-week timeline (though C-18 does). The check fails because C-16 does not cover the condition as designed. C-16 should be updated or the check should be mapped to C-18." + }, + { + "type": "failed_check", + "attempt": 4, + "check": "OC-11", + "evidence": "The check requires automated deployments to optimize carbon-free energy utilization, and is mapped to C-6. However, C-6's basis only covers managing automated deployment pipelines to deliver foundational updates rapidly and reliably, without mentioning carbon-free energy optimization. C-6 should be updated to include this optimization." + }, + { + "type": "failed_check", + "attempt": 4, + "check": "OC-15", + "evidence": "The check requires operations to be delivered entirely under a provider-operated model, and is mapped to C-20 (holding the provider accountable). However, the model assigns C-20 to Provider roles (R-3, R-6) instead of Customer roles. A provider cannot hold itself accountable as the external MSP. Supplier management should be assigned to Customer roles (P-2) to hold the provider accountable." + }, + { + "type": "failed_check", + "attempt": 4, + "check": "OC-17", + "evidence": "The check requires Cloud Monitoring alerts to be fully integrated into the provider's event management system, and is mapped to C-2. However, C-2's basis only covers continuous monitoring and alert generation; it does not explicitly mention integrating those alerts into the provider's operational event management system. C-2 should be updated to include this integration." + }, + { + "type": "failed_check", + "attempt": 5, + "check": "OC-3", + "evidence": "Capability C-7 focuses on tracking infrastructure state files and ensuring PCI-DSS v4.0 baseline alignment, but does not mention auditing for UK/EU data residency. Data residency is instead covered by C-21. Capability C-7 should be updated to include auditing data residency state, or the Check should be re-mapped to C-21." + }, + { + "type": "failed_check", + "attempt": 5, + "check": "OC-18", + "evidence": "Capability C-1 covers triage and managing priority incidents, but does not explicitly mention conducting a written review for every Priority 1 incident. The capability basis should be updated to include post-incident written reviews." + }, + { + "type": "failed_check", + "attempt": 5, + "check": "OC-21", + "evidence": "Capability C-4 covers fulfilling non-standard operational requests, but does not explicitly state that each kind of foundation service request has a stated way to ask and a stated time to fulfil (e.g., maintaining a service request catalogue). The capability basis should be updated to include maintaining stated request methods and fulfilment times." + }, + { + "type": "failed_check", + "attempt": 5, + "check": "OC-23", + "evidence": "Capability C-6 covers resolving deployment failures and managing automated pipelines, but does not explicitly guarantee a rollback capability within the 4-hour RTO. The capability should be updated to cover rapid rollback procedures." + }, + { + "type": "failed_check", + "attempt": 6, + "check": "OC-2", + "evidence": "The check requires the secondary region to be 'continuously maintained' as an active warm standby, but Capability C-9 is configured for 'business' hours, meaning maintenance cannot be continuous (24x7). C-9 should be 24x7 or another capability should provide continuous maintenance." + }, + { + "type": "failed_check", + "attempt": 6, + "check": "OC-4", + "evidence": "The check requires validation within 10 weeks, but Capability C-16 covers managing the roadmap without explicitly mentioning the 10-week requirement (which is instead mentioned in C-18). C-16 must mention the 10-week validation or the check mapped to C-18." + }, + { + "type": "failed_check", + "attempt": 6, + "check": "OC-9", + "evidence": "The check requires that expenditure does not exceed target allocations without 'documented operational sign-off', but Capability C-14 only mentions 'escalating anomalies' and does not explicitly guarantee documented sign-off." + }, + { + "type": "failed_check", + "attempt": 6, + "check": "OC-12", + "evidence": "The check requires identity permissions to be 'continuously reviewed', but Capability C-11 operates only during 'business' hours, meaning reviews cannot be continuous. C-11 must be 24x7." + }, + { + "type": "failed_check", + "attempt": 6, + "check": "OC-17", + "evidence": "The check requires verifiable alignment with DORA, PCI-DSS v4.0, and ISO 27001. Capability C-7 only ensures alignment with PCI-DSS v4.0 baseline configurations, completely missing DORA and ISO 27001 controls." + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r2.json b/eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r2.json new file mode 100644 index 0000000..37975a3 --- /dev/null +++ b/eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r2.json @@ -0,0 +1,59 @@ +{ + "execution_id": "case-02-from-architecture-r2", + "customer_case": "case-08-landing-zone", + "status": "completed", + "role_skills": { + "capability-modeller": [ + "operating-capabilities", + "operating-chain" + ], + "operating-model-architect": [ + "operating-capabilities", + "operating-chain", + "rate-card" + ], + "operations-analyst": [ + "operating-scoping", + "operating-chain" + ], + "orchestrator": [], + "packager": [ + "operating-guide" + ], + "reply-writer": [], + "scope-analyst": [ + "operating-scoping" + ], + "validator": [ + "operating-chain" + ] + }, + "skill_versions": { + "operating-capabilities": "49a7e66857a1", + "operating-chain": "ec53ad8875e6", + "operating-guide": "85b48d5998da", + "operating-scoping": "08da33f8cddf", + "rate-card": "f86d17a9bcce" + }, + "candidates": [ + { + "type": "revision", + "deliverable": "target_operating_model", + "reason": "OC-K6 failed: The Check requires achieving an RTO of no more than 4 hours, meaning disaster recovery might need to be activated at any time. However, Capability C-9 (service-continuity), which is the capability designated to achieve this RTO, is configured for 'business' hours only. It cannot reliably meet a 4-hour RTO outside of business hours as currently designed. The capability should be expanded to 24x7 coverage, or execution should explicitly fall to a 24x7 capability.", + "requested_by": "orchestrator" + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-K6", + "evidence": "The Check requires achieving an RTO of no more than 4 hours, meaning disaster recovery might need to be activated at any time. However, Capability C-9 (service-continuity), which is the capability designated to achieve this RTO, is configured for 'business' hours only. It cannot reliably meet a 4-hour RTO outside of business hours as currently designed. The capability should be expanded to 24x7 coverage, or execution should explicitly fall to a 24x7 capability." + }, + { + "type": "refusal", + "role": "operating-model-architect", + "deliverable": "target_operating_model", + "tool": "submit_deliverable", + "message": "Not submitted. The Deliverable is missing: target-operating-model.json: business outcome BO-1 (Continuous, highly available, and low-latency operation of the core payments platform foundation) is shown by no Measure; target-operating-model.json: business outcome BO-2 (Continuous compliance with DORA and PCI-DSS v4.0 standards using customer-managed encryption keys and centralised egress inspection) is shown by no Measure; target-operating-model.json: business outcome BO-3 (Automated, secure infr" + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r3.json b/eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r3.json new file mode 100644 index 0000000..c7ec330 --- /dev/null +++ b/eval/results/20261021T203624Z/lessons/candidates/case-08-landing-zone/case-02-from-architecture-r3.json @@ -0,0 +1,114 @@ +{ + "execution_id": "case-02-from-architecture-r3", + "customer_case": "case-08-landing-zone", + "status": "completed", + "role_skills": { + "capability-modeller": [ + "operating-capabilities", + "operating-chain" + ], + "operating-model-architect": [ + "operating-capabilities", + "operating-chain", + "rate-card" + ], + "operations-analyst": [ + "operating-scoping", + "operating-chain" + ], + "orchestrator": [], + "packager": [ + "operating-guide" + ], + "reply-writer": [], + "scope-analyst": [ + "operating-scoping" + ], + "validator": [ + "operating-chain" + ] + }, + "skill_versions": { + "operating-capabilities": "49a7e66857a1", + "operating-chain": "ec53ad8875e6", + "operating-guide": "85b48d5998da", + "operating-scoping": "08da33f8cddf", + "rate-card": "f86d17a9bcce" + }, + "candidates": [ + { + "type": "revision", + "deliverable": "target_operating_model", + "reason": "OC-9 failed: Capability C-14 is designed only for 'monthly review of cloud consumption against allocated budgets and investigating anomalies.' It lacks any designed process, automation, or mechanism to enforce hard limits that prevent overages. The capability must be redesigned to provide hard limit enforcement.; OC-16 failed: Capability C-21's basis only includes gathering compliance evidence for PCI-DSS v4.0 and DORA auditors, explicitly omitting ISO27001 which is required by the condition. As designed, the model does not cover ISO27001 evidence gathering.; SC-party-P-2 failed: Platform Service Provider cannot be reached. No Role in another Party is responsible or consulted for C-2, C-3, C-4, C-5, C-6, C-7, C-8, C-9, C-10, C-11, C-12, C-13, C-14, C-15, C-17, C-18, C-19, C-20.", + "requested_by": "orchestrator" + }, + { + "type": "revision", + "deliverable": "capability_model", + "reason": "Check OC-9 failed in two validations in a row and may not be implementable as written (Capability C-14 is currently designed only for monthly review of cloud consumption and lacks any process or automation to enforce hard limits. The capability must be redesigned to provide hard limit enforcement; the finding AF-1 does not stand as the operating model could provide this mechanism itself.). Rewrite it so it can be met, or remove it with a reason. Check OC-16 failed in two validations in a row and may not be implementable as written (Capability C-21's basis only includes gathering compliance evidence for PCI-DSS v4.0 and DORA auditors, explicitly omitting ISO27001. The finding AF-2 does not stand as the operating model should provide the process and effort to gather this evidence itself.). Rewrite it so it can be met, or remove it with a reason.", + "requested_by": "orchestrator" + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-9", + "evidence": "Capability C-14 is designed only for 'monthly review of cloud consumption against allocated budgets and investigating anomalies.' It lacks any designed process, automation, or mechanism to enforce hard limits that prevent overages. The capability must be redesigned to provide hard limit enforcement." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "OC-16", + "evidence": "Capability C-21's basis only includes gathering compliance evidence for PCI-DSS v4.0 and DORA auditors, explicitly omitting ISO27001 which is required by the condition. As designed, the model does not cover ISO27001 evidence gathering." + }, + { + "type": "failed_check", + "attempt": 1, + "check": "SC-party-P-2", + "evidence": "Platform Service Provider cannot be reached. No Role in another Party is responsible or consulted for C-2, C-3, C-4, C-5, C-6, C-7, C-8, C-9, C-10, C-11, C-12, C-13, C-14, C-15, C-17, C-18, C-19, C-20." + }, + { + "type": "unverifiable_check", + "attempt": 1, + "check": "OC-4", + "evidence": "The condition specifies a one-time engagement milestone (verification of full provisioning within 10 weeks) which occurs during the build phase. An ongoing operating model governs steady-state operations and cannot continuously verify or guarantee a past delivery event, making this condition not verifiable as worded." + }, + { + "type": "proposed_addition", + "attempt": 1, + "reason": "The stated effort of 80 hours per month is implausible for 'continuous watching' which would require 720+ hours for true 24x7 manual monitoring.", + "applies_to": "C-2", + "statement": "The basis and effort of C-2 should be aligned to reflect automated alerting, or the effort vastly increased for manual continuous watching." + }, + { + "type": "proposed_addition", + "attempt": 1, + "applies_to": "C-13", + "reason": "80 hours per month is insufficient for manual 24x7 security event monitoring. If it relies on automated SIEM alerts, the basis should clarify this.", + "statement": "The effort for C-13 should clarify reliance on automated alerts or vastly increase if manual eyes-on-glass monitoring is expected 24x7." + }, + { + "type": "failed_check", + "attempt": 2, + "check": "OC-1", + "evidence": "Capability C-7's basis explicitly mentions ensuring compliance with DORA/PCI-DSS baselines but omits ISO27001, meaning the model as designed does not cover enforcing ISO27001 standard baselines." + }, + { + "type": "failed_check", + "attempt": 2, + "check": "OC-9", + "evidence": "Capability C-14 is currently designed only for monthly review of cloud consumption and lacks any process or automation to enforce hard limits. The capability must be redesigned to provide hard limit enforcement; the finding AF-1 does not stand as the operating model could provide this mechanism itself." + }, + { + "type": "failed_check", + "attempt": 2, + "check": "OC-16", + "evidence": "Capability C-21's basis only includes gathering compliance evidence for PCI-DSS v4.0 and DORA auditors, explicitly omitting ISO27001. The finding AF-2 does not stand as the operating model should provide the process and effort to gather this evidence itself." + }, + { + "type": "failed_check", + "attempt": 2, + "check": "OC-17", + "evidence": "Capability C-21's basis specifies gathering evidence for PCI-DSS v4.0 and DORA auditors, omitting the process for gathering evidence of UK and EU data residency guarantees." + } + ] +} \ No newline at end of file diff --git a/eval/results/20261021T203624Z/report.md b/eval/results/20261021T203624Z/report.md new file mode 100644 index 0000000..b5fc29c --- /dev/null +++ b/eval/results/20261021T203624Z/report.md @@ -0,0 +1,39 @@ +# Target operating model agent: evaluation + +Run `20261021T203624Z`. Each run is one pre-approved Execution with real models on a benchmark case. Every case is fictional; no customer's data is in it. + +## Results + +| Case | Run | Result | Status | Minutes | Revisions | Requirements verified | Roles | People | Cost a month | +|---|---|---|---|---|---|---|---|---|---| +| case-01-landing-zone-shared | 1 | PASS | completed | 18.8 | 1 | 10 of 10 | 5 | 6.8 | 46,252 GBP | +| case-01-landing-zone-shared | 2 | PASS | completed | 17.1 | 1 | 9 of 9 | 7 | 11.0 | 44,720 GBP | +| case-01-landing-zone-shared | 3 | PASS | completed | 20.2 | 1 | 10 of 10 | 6 | 11.4 | 47,603 GBP | +| case-02-from-architecture | 1 | FAIL | completed_with_warnings | 50.4 | 5 | 17 of 17 | 8 | 11.7 | 73,863 GBP | +| case-02-from-architecture | 2 | PASS | completed | 20.9 | 1 | 15 of 15 | 6 | 10.8 | 83,846 GBP | +| case-02-from-architecture | 3 | PASS | completed | 24.4 | 2 | 15 of 15 | 8 | 16.9 | 116,409 GBP | + +## Checks that failed + +- **case-02-from-architecture**: completed (completed_with_warnings) +- **case-02-from-architecture**: every Deliverable was submitted (operating_model_guide) +- **case-02-from-architecture**: no Check failed (OC-2, OC-4, OC-9, OC-12, OC-17) +- **case-02-from-architecture**: the guide has its sections +- **case-02-from-architecture**: the packager wrote a summary (0 words) + +## Spread across repeated runs + +The same case and inputs, run again: how far the operating model moved. + +| Case | Figure | Least | Most | Mean | Range of the mean | +|---|---|---|---|---|---| +| case-01-landing-zone-shared | people | 6.8 | 11.4 | 9.7 | 47.3% | +| case-01-landing-zone-shared | cost_per_month | 44720 | 47603 | 46191.7 | 6.2% | +| case-01-landing-zone-shared | roles | 5 | 7 | 6.0 | 33.3% | +| case-01-landing-zone-shared | checks | 15 | 19 | 16.3 | 24.5% | +| case-01-landing-zone-shared | capabilities | 21 | 21 | 21.0 | 0.0% | +| case-02-from-architecture | people | 10.8 | 16.9 | 13.1 | 46.4% | +| case-02-from-architecture | cost_per_month | 73863 | 116409 | 91372.7 | 46.6% | +| case-02-from-architecture | roles | 6 | 8 | 7.3 | 27.3% | +| case-02-from-architecture | checks | 17 | 20 | 18.0 | 16.7% | +| case-02-from-architecture | capabilities | 21 | 21 | 21.0 | 0.0% | diff --git a/eval/results/20261021T203624Z/results.json b/eval/results/20261021T203624Z/results.json new file mode 100644 index 0000000..6e58384 --- /dev/null +++ b/eval/results/20261021T203624Z/results.json @@ -0,0 +1,891 @@ +[ + { + "case": "case-01-landing-zone-shared", + "status": "completed", + "checks": [ + { + "name": "completed", + "passed": true, + "detail": "completed" + }, + { + "name": "every Deliverable was submitted", + "passed": true, + "detail": "all" + }, + { + "name": "the chain holds", + "passed": true, + "detail": "" + }, + { + "name": "no Check failed", + "passed": true, + "detail": "" + }, + { + "name": "every scenario has a path", + "passed": true, + "detail": "13 of 13" + }, + { + "name": "the guide has its sections", + "passed": true, + "detail": "Summary, Business outcomes, What is operated, Operational requirements, Capabilities, Who does what, Staffing and cost, Validation, Findings and open questions, Sources" + }, + { + "name": "the packager wrote a summary", + "passed": true, + "detail": "137 words" + }, + { + "name": "no Deliverable was revised more than 2 times", + "passed": true, + "detail": "target_operating_model x 1" + }, + { + "name": "every Operational requirement is verified by a Check", + "passed": true, + "detail": "10 of 10" + }, + { + "name": "finished within 40 minutes", + "passed": true, + "detail": "18.8" + } + ], + "minutes": 18.8, + "revisions": { + "target_operating_model": 1 + }, + "tokens": { + "scope-analyst": { + "requests": 8, + "prompt": 54343, + "cached": 15456, + "tool_prompt": 0, + "output": 490, + "thinking": 5329, + "total": 60162 + }, + "reply-writer": { + "requests": 6, + "prompt": 4128, + "cached": 0, + "tool_prompt": 0, + "output": 797, + "thinking": 8418, + "total": 13343 + }, + "operations-analyst": { + "requests": 4, + "prompt": 27540, + "cached": 7850, + "tool_prompt": 0, + "output": 1513, + "thinking": 15882, + "total": 44935 + }, + "capability-modeller": { + "requests": 5, + "prompt": 44844, + "cached": 15791, + "tool_prompt": 0, + "output": 4184, + "thinking": 11125, + "total": 60153 + }, + "operating-model-architect": { + "requests": 19, + "prompt": 359430, + "cached": 19545, + "tool_prompt": 0, + "output": 13883, + "thinking": 20668, + "total": 393981 + }, + "validator": { + "requests": 16, + "prompt": 254203, + "cached": 129659, + "tool_prompt": 0, + "output": 3306, + "thinking": 15573, + "total": 273082 + }, + "packager": { + "requests": 3, + "prompt": 51462, + "cached": 24155, + "tool_prompt": 0, + "output": 359, + "thinking": 7956, + "total": 59777 + } + }, + "figures": { + "items": 4, + "conditions": 3, + "operational_requirements": 10, + "capabilities": 21, + "custom_capabilities": 0, + "parties": 2, + "roles": 5, + "checks": 15, + "requirements_unverified": 0, + "checks_not_verifiable": 0, + "checks_waiting_on_findings": 0, + "proposed_additions": 0, + "scenarios": 13, + "findings": 0, + "business_outcomes": 3, + "measures": 4, + "people": 6.8, + "cost_per_month": 46252, + "currency": "GBP" + }, + "warnings": [], + "folder": "deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1", + "passed": true + }, + { + "case": "case-01-landing-zone-shared", + "status": "completed", + "checks": [ + { + "name": "completed", + "passed": true, + "detail": "completed" + }, + { + "name": "every Deliverable was submitted", + "passed": true, + "detail": "all" + }, + { + "name": "the chain holds", + "passed": true, + "detail": "" + }, + { + "name": "no Check failed", + "passed": true, + "detail": "" + }, + { + "name": "every scenario has a path", + "passed": true, + "detail": "13 of 13" + }, + { + "name": "the guide has its sections", + "passed": true, + "detail": "Summary, Business outcomes, What is operated, Operational requirements, Capabilities, Who does what, Staffing and cost, Validation, Findings and open questions, Sources" + }, + { + "name": "the packager wrote a summary", + "passed": true, + "detail": "163 words" + }, + { + "name": "no Deliverable was revised more than 2 times", + "passed": true, + "detail": "target_operating_model x 1" + }, + { + "name": "every Operational requirement is verified by a Check", + "passed": true, + "detail": "9 of 9" + }, + { + "name": "finished within 40 minutes", + "passed": true, + "detail": "17.1" + } + ], + "minutes": 17.1, + "revisions": { + "target_operating_model": 1 + }, + "tokens": { + "scope-analyst": { + "requests": 7, + "prompt": 48659, + "cached": 15492, + "tool_prompt": 0, + "output": 465, + "thinking": 6171, + "total": 55295 + }, + "reply-writer": { + "requests": 6, + "prompt": 4144, + "cached": 0, + "tool_prompt": 0, + "output": 820, + "thinking": 7499, + "total": 12463 + }, + "operations-analyst": { + "requests": 3, + "prompt": 16585, + "cached": 3873, + "tool_prompt": 0, + "output": 1991, + "thinking": 16707, + "total": 35283 + }, + "capability-modeller": { + "requests": 4, + "prompt": 30378, + "cached": 7804, + "tool_prompt": 0, + "output": 4679, + "thinking": 10442, + "total": 45499 + }, + "operating-model-architect": { + "requests": 23, + "prompt": 446402, + "cached": 38840, + "tool_prompt": 0, + "output": 11132, + "thinking": 19429, + "total": 476963 + }, + "validator": { + "requests": 9, + "prompt": 119967, + "cached": 55229, + "tool_prompt": 0, + "output": 2544, + "thinking": 8515, + "total": 131026 + }, + "packager": { + "requests": 4, + "prompt": 76526, + "cached": 32231, + "tool_prompt": 0, + "output": 414, + "thinking": 6276, + "total": 83216 + } + }, + "figures": { + "items": 4, + "conditions": 3, + "operational_requirements": 9, + "capabilities": 21, + "custom_capabilities": 0, + "parties": 2, + "roles": 7, + "checks": 15, + "requirements_unverified": 0, + "checks_not_verifiable": 0, + "checks_waiting_on_findings": 0, + "proposed_additions": 0, + "scenarios": 13, + "findings": 0, + "business_outcomes": 3, + "measures": 4, + "people": 11.0, + "cost_per_month": 44720, + "currency": "GBP" + }, + "warnings": [], + "folder": "deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2", + "passed": true + }, + { + "case": "case-01-landing-zone-shared", + "status": "completed", + "checks": [ + { + "name": "completed", + "passed": true, + "detail": "completed" + }, + { + "name": "every Deliverable was submitted", + "passed": true, + "detail": "all" + }, + { + "name": "the chain holds", + "passed": true, + "detail": "" + }, + { + "name": "no Check failed", + "passed": true, + "detail": "" + }, + { + "name": "every scenario has a path", + "passed": true, + "detail": "13 of 13" + }, + { + "name": "the guide has its sections", + "passed": true, + "detail": "Summary, Business outcomes, What is operated, Operational requirements, Capabilities, Who does what, Staffing and cost, Validation, Findings and open questions, Sources" + }, + { + "name": "the packager wrote a summary", + "passed": true, + "detail": "155 words" + }, + { + "name": "no Deliverable was revised more than 2 times", + "passed": true, + "detail": "target_operating_model x 1" + }, + { + "name": "every Operational requirement is verified by a Check", + "passed": true, + "detail": "10 of 10" + }, + { + "name": "finished within 40 minutes", + "passed": true, + "detail": "20.2" + } + ], + "minutes": 20.2, + "revisions": { + "target_operating_model": 1 + }, + "tokens": { + "scope-analyst": { + "requests": 7, + "prompt": 49123, + "cached": 19387, + "tool_prompt": 0, + "output": 459, + "thinking": 6063, + "total": 55645 + }, + "reply-writer": { + "requests": 6, + "prompt": 4204, + "cached": 0, + "tool_prompt": 0, + "output": 833, + "thinking": 7734, + "total": 12771 + }, + "operations-analyst": { + "requests": 4, + "prompt": 24044, + "cached": 3923, + "tool_prompt": 0, + "output": 2294, + "thinking": 14999, + "total": 41337 + }, + "capability-modeller": { + "requests": 3, + "prompt": 24774, + "cached": 3936, + "tool_prompt": 0, + "output": 4583, + "thinking": 14748, + "total": 44105 + }, + "operating-model-architect": { + "requests": 19, + "prompt": 356611, + "cached": 9523, + "tool_prompt": 0, + "output": 11354, + "thinking": 17771, + "total": 385736 + }, + "validator": { + "requests": 11, + "prompt": 161689, + "cached": 50998, + "tool_prompt": 0, + "output": 3729, + "thinking": 13885, + "total": 179303 + }, + "packager": { + "requests": 3, + "prompt": 59018, + "cached": 32277, + "tool_prompt": 0, + "output": 377, + "thinking": 2511, + "total": 61906 + } + }, + "figures": { + "items": 4, + "conditions": 3, + "operational_requirements": 10, + "capabilities": 21, + "custom_capabilities": 0, + "parties": 2, + "roles": 6, + "checks": 19, + "requirements_unverified": 0, + "checks_not_verifiable": 0, + "checks_waiting_on_findings": 0, + "proposed_additions": 0, + "scenarios": 13, + "findings": 0, + "business_outcomes": 3, + "measures": 4, + "people": 11.4, + "cost_per_month": 47603, + "currency": "GBP" + }, + "warnings": [], + "folder": "deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3", + "passed": true + }, + { + "case": "case-02-from-architecture", + "status": "completed_with_warnings", + "checks": [ + { + "name": "completed", + "passed": false, + "detail": "completed_with_warnings" + }, + { + "name": "every Deliverable was submitted", + "passed": false, + "detail": "operating_model_guide" + }, + { + "name": "the chain holds", + "passed": true, + "detail": "" + }, + { + "name": "no Check failed", + "passed": false, + "detail": "OC-2, OC-4, OC-9, OC-12, OC-17" + }, + { + "name": "every scenario has a path", + "passed": true, + "detail": "31 of 31" + }, + { + "name": "the guide has its sections", + "passed": false, + "detail": "" + }, + { + "name": "the packager wrote a summary", + "passed": false, + "detail": "0 words" + }, + { + "name": "no Deliverable was revised more than 3 times", + "passed": true, + "detail": "capability_model x 2, target_operating_model x 3" + }, + { + "name": "every Operational requirement is verified by a Check", + "passed": true, + "detail": "17 of 17" + }, + { + "name": "finished within 60 minutes", + "passed": true, + "detail": "50.4" + } + ], + "minutes": 50.4, + "revisions": { + "capability_model": 2, + "target_operating_model": 3 + }, + "tokens": { + "scope-analyst": { + "requests": 6, + "prompt": 48208, + "cached": 19432, + "tool_prompt": 0, + "output": 865, + "thinking": 6386, + "total": 55459 + }, + "reply-writer": { + "requests": 8, + "prompt": 5567, + "cached": 0, + "tool_prompt": 0, + "output": 1116, + "thinking": 9789, + "total": 16472 + }, + "operations-analyst": { + "requests": 5, + "prompt": 40800, + "cached": 19700, + "tool_prompt": 0, + "output": 3804, + "thinking": 17289, + "total": 61893 + }, + "capability-modeller": { + "requests": 14, + "prompt": 190167, + "cached": 47744, + "tool_prompt": 0, + "output": 14612, + "thinking": 50372, + "total": 255151 + }, + "operating-model-architect": { + "requests": 56, + "prompt": 1360422, + "cached": 688217, + "tool_prompt": 0, + "output": 48734, + "thinking": 94245, + "total": 1503401 + }, + "validator": { + "requests": 37, + "prompt": 685069, + "cached": 352339, + "tool_prompt": 0, + "output": 12846, + "thinking": 46979, + "total": 744894 + } + }, + "figures": { + "items": 9, + "conditions": 12, + "operational_requirements": 17, + "capabilities": 21, + "custom_capabilities": 0, + "parties": 2, + "roles": 8, + "checks": 17, + "requirements_unverified": 0, + "checks_not_verifiable": 0, + "checks_waiting_on_findings": 0, + "proposed_additions": 0, + "scenarios": 31, + "findings": 0, + "business_outcomes": 3, + "measures": 6, + "people": 11.7, + "cost_per_month": 73863, + "currency": "GBP" + }, + "warnings": [ + "target_operating_model has reached its revision limit (3). Requested change: OC-2 failed: The check requires the secondary region to be 'continuously maintained' as an active warm standby, but Capability C-9 is configured for 'business' hours, meaning maintenance cannot be continuous (24x7). C-9 should be 24x7 or another capability should provide continuous maintenance.; OC-4 failed: The check requires validation within 10 weeks, but Capability C-16 covers managing the roadmap without explicitly mentioning the 10-week requirement (which is instead mentioned in C-18). C-16 must mention the 10-week validation or the check mapped to C-18.; OC-9 failed: The check requires that expenditure does not exceed target allocations without 'documented operational sign-off', but Capability C-14 only mentions 'escalating anomalies' and does not explicitly guarantee documented sign-off.; OC-12 failed: The check requires identity permissions to be 'continuously reviewed', but Capability C-11 operates only during 'business' hours, meaning reviews cannot be continuous. C-11 must be 24x7.; OC-17 failed: The check requires verifiable alignment with DORA, PCI-DSS v4.0, and ISO 27001. Capability C-7 only ensures alignment with PCI-DSS v4.0 baseline configurations, completely missing DORA and ISO 27001 controls.. Ask the user whether to revise it again or accept it as it stands." + ], + "folder": "deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1", + "passed": false + }, + { + "case": "case-02-from-architecture", + "status": "completed", + "checks": [ + { + "name": "completed", + "passed": true, + "detail": "completed" + }, + { + "name": "every Deliverable was submitted", + "passed": true, + "detail": "all" + }, + { + "name": "the chain holds", + "passed": true, + "detail": "" + }, + { + "name": "no Check failed", + "passed": true, + "detail": "" + }, + { + "name": "every scenario has a path", + "passed": true, + "detail": "31 of 31" + }, + { + "name": "the guide has its sections", + "passed": true, + "detail": "Summary, Business outcomes, What is operated, Operational requirements, Capabilities, Who does what, Staffing and cost, Validation, Findings and open questions, Sources" + }, + { + "name": "the packager wrote a summary", + "passed": true, + "detail": "167 words" + }, + { + "name": "no Deliverable was revised more than 3 times", + "passed": true, + "detail": "target_operating_model x 1" + }, + { + "name": "every Operational requirement is verified by a Check", + "passed": true, + "detail": "15 of 15" + }, + { + "name": "finished within 60 minutes", + "passed": true, + "detail": "20.9" + } + ], + "minutes": 20.9, + "revisions": { + "target_operating_model": 1 + }, + "tokens": { + "scope-analyst": { + "requests": 7, + "prompt": 58699, + "cached": 23389, + "tool_prompt": 0, + "output": 941, + "thinking": 8227, + "total": 67867 + }, + "reply-writer": { + "requests": 6, + "prompt": 4117, + "cached": 0, + "tool_prompt": 0, + "output": 792, + "thinking": 7548, + "total": 12457 + }, + "operations-analyst": { + "requests": 4, + "prompt": 31104, + "cached": 11821, + "tool_prompt": 0, + "output": 3343, + "thinking": 21723, + "total": 56170 + }, + "capability-modeller": { + "requests": 4, + "prompt": 50775, + "cached": 0, + "tool_prompt": 0, + "output": 5417, + "thinking": 17513, + "total": 73705 + }, + "operating-model-architect": { + "requests": 26, + "prompt": 581893, + "cached": 233265, + "tool_prompt": 0, + "output": 23437, + "thinking": 25452, + "total": 630782 + }, + "validator": { + "requests": 10, + "prompt": 169718, + "cached": 35567, + "tool_prompt": 0, + "output": 4610, + "thinking": 13791, + "total": 188119 + }, + "packager": { + "requests": 3, + "prompt": 80302, + "cached": 48607, + "tool_prompt": 0, + "output": 401, + "thinking": 7230, + "total": 87933 + } + }, + "figures": { + "items": 9, + "conditions": 12, + "operational_requirements": 15, + "capabilities": 21, + "custom_capabilities": 0, + "parties": 2, + "roles": 6, + "checks": 20, + "requirements_unverified": 0, + "checks_not_verifiable": 0, + "checks_waiting_on_findings": 0, + "proposed_additions": 0, + "scenarios": 31, + "findings": 0, + "business_outcomes": 3, + "measures": 6, + "people": 10.8, + "cost_per_month": 83846, + "currency": "GBP" + }, + "warnings": [], + "folder": "deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2", + "passed": true + }, + { + "case": "case-02-from-architecture", + "status": "completed", + "checks": [ + { + "name": "completed", + "passed": true, + "detail": "completed" + }, + { + "name": "every Deliverable was submitted", + "passed": true, + "detail": "all" + }, + { + "name": "the chain holds", + "passed": true, + "detail": "" + }, + { + "name": "no Check failed", + "passed": true, + "detail": "" + }, + { + "name": "every scenario has a path", + "passed": true, + "detail": "34 of 34" + }, + { + "name": "the guide has its sections", + "passed": true, + "detail": "Summary, Business outcomes, What is operated, Operational requirements, Capabilities, Who does what, Staffing and cost, Validation, Findings and open questions, Sources" + }, + { + "name": "the packager wrote a summary", + "passed": true, + "detail": "204 words" + }, + { + "name": "no Deliverable was revised more than 3 times", + "passed": true, + "detail": "capability_model x 1, target_operating_model x 1" + }, + { + "name": "every Operational requirement is verified by a Check", + "passed": true, + "detail": "15 of 15" + }, + { + "name": "finished within 60 minutes", + "passed": true, + "detail": "24.4" + } + ], + "minutes": 24.4, + "revisions": { + "capability_model": 1, + "target_operating_model": 1 + }, + "tokens": { + "scope-analyst": { + "requests": 6, + "prompt": 49107, + "cached": 19446, + "tool_prompt": 0, + "output": 984, + "thinking": 6456, + "total": 56547 + }, + "reply-writer": { + "requests": 7, + "prompt": 4900, + "cached": 0, + "tool_prompt": 0, + "output": 927, + "thinking": 10194, + "total": 16021 + }, + "operations-analyst": { + "requests": 4, + "prompt": 31544, + "cached": 15782, + "tool_prompt": 0, + "output": 3213, + "thinking": 17810, + "total": 52567 + }, + "capability-modeller": { + "requests": 10, + "prompt": 113370, + "cached": 19712, + "tool_prompt": 0, + "output": 9290, + "thinking": 31311, + "total": 153971 + }, + "operating-model-architect": { + "requests": 21, + "prompt": 407275, + "cached": 204633, + "tool_prompt": 0, + "output": 14394, + "thinking": 24719, + "total": 446388 + }, + "validator": { + "requests": 18, + "prompt": 329934, + "cached": 221850, + "tool_prompt": 0, + "output": 5737, + "thinking": 23300, + "total": 358971 + }, + "packager": { + "requests": 3, + "prompt": 80501, + "cached": 48609, + "tool_prompt": 0, + "output": 457, + "thinking": 6246, + "total": 87204 + } + }, + "figures": { + "items": 9, + "conditions": 12, + "operational_requirements": 15, + "capabilities": 21, + "custom_capabilities": 0, + "parties": 5, + "roles": 8, + "checks": 17, + "requirements_unverified": 0, + "checks_not_verifiable": 0, + "checks_waiting_on_findings": 0, + "proposed_additions": 0, + "scenarios": 34, + "findings": 0, + "business_outcomes": 3, + "measures": 7, + "people": 16.9, + "cost_per_month": 116409, + "currency": "GBP" + }, + "warnings": [], + "folder": "deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3", + "passed": true + } +] \ No newline at end of file diff --git a/eval/results/20261021T203624Z/spread.json b/eval/results/20261021T203624Z/spread.json new file mode 100644 index 0000000..426ce0e --- /dev/null +++ b/eval/results/20261021T203624Z/spread.json @@ -0,0 +1,68 @@ +{ + "case-01-landing-zone-shared": { + "runs": 3, + "people": { + "min": 6.8, + "max": 11.4, + "mean": 9.7, + "range_pct": 47.3 + }, + "cost_per_month": { + "min": 44720, + "max": 47603, + "mean": 46191.7, + "range_pct": 6.2 + }, + "roles": { + "min": 5, + "max": 7, + "mean": 6.0, + "range_pct": 33.3 + }, + "checks": { + "min": 15, + "max": 19, + "mean": 16.3, + "range_pct": 24.5 + }, + "capabilities": { + "min": 21, + "max": 21, + "mean": 21.0, + "range_pct": 0.0 + } + }, + "case-02-from-architecture": { + "runs": 3, + "people": { + "min": 10.8, + "max": 16.9, + "mean": 13.1, + "range_pct": 46.4 + }, + "cost_per_month": { + "min": 73863, + "max": 116409, + "mean": 91372.7, + "range_pct": 46.6 + }, + "roles": { + "min": 6, + "max": 8, + "mean": 7.3, + "range_pct": 27.3 + }, + "checks": { + "min": 17, + "max": 20, + "mean": 18.0, + "range_pct": 16.7 + }, + "capabilities": { + "min": 21, + "max": 21, + "mean": 21.0, + "range_pct": 0.0 + } + } +} \ No newline at end of file