docs: publish the runs of 2026-10-21

The evaluation (both cases three times: report, results, spread, each
run's Deliverables and lesson candidates) and the hand-off through the
Orchestration engine (report, the document of links, both agents' kept
files and records). Benchmark cases only, all fictional; no machine paths,
and the links carry no signatures. Other runs stay ignored.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-22 09:25:03 +01:00
parent 1fed1cb538
commit 628b6ee170
159 changed files with 37817 additions and 2 deletions

View File

@@ -0,0 +1,891 @@
[
{
"case": "case-01-landing-zone-shared",
"status": "completed",
"checks": [
{
"name": "completed",
"passed": true,
"detail": "completed"
},
{
"name": "every Deliverable was submitted",
"passed": true,
"detail": "all"
},
{
"name": "the chain holds",
"passed": true,
"detail": ""
},
{
"name": "no Check failed",
"passed": true,
"detail": ""
},
{
"name": "every scenario has a path",
"passed": true,
"detail": "13 of 13"
},
{
"name": "the guide has its sections",
"passed": true,
"detail": "Summary, Business outcomes, What is operated, Operational requirements, Capabilities, Who does what, Staffing and cost, Validation, Findings and open questions, Sources"
},
{
"name": "the packager wrote a summary",
"passed": true,
"detail": "137 words"
},
{
"name": "no Deliverable was revised more than 2 times",
"passed": true,
"detail": "target_operating_model x 1"
},
{
"name": "every Operational requirement is verified by a Check",
"passed": true,
"detail": "10 of 10"
},
{
"name": "finished within 40 minutes",
"passed": true,
"detail": "18.8"
}
],
"minutes": 18.8,
"revisions": {
"target_operating_model": 1
},
"tokens": {
"scope-analyst": {
"requests": 8,
"prompt": 54343,
"cached": 15456,
"tool_prompt": 0,
"output": 490,
"thinking": 5329,
"total": 60162
},
"reply-writer": {
"requests": 6,
"prompt": 4128,
"cached": 0,
"tool_prompt": 0,
"output": 797,
"thinking": 8418,
"total": 13343
},
"operations-analyst": {
"requests": 4,
"prompt": 27540,
"cached": 7850,
"tool_prompt": 0,
"output": 1513,
"thinking": 15882,
"total": 44935
},
"capability-modeller": {
"requests": 5,
"prompt": 44844,
"cached": 15791,
"tool_prompt": 0,
"output": 4184,
"thinking": 11125,
"total": 60153
},
"operating-model-architect": {
"requests": 19,
"prompt": 359430,
"cached": 19545,
"tool_prompt": 0,
"output": 13883,
"thinking": 20668,
"total": 393981
},
"validator": {
"requests": 16,
"prompt": 254203,
"cached": 129659,
"tool_prompt": 0,
"output": 3306,
"thinking": 15573,
"total": 273082
},
"packager": {
"requests": 3,
"prompt": 51462,
"cached": 24155,
"tool_prompt": 0,
"output": 359,
"thinking": 7956,
"total": 59777
}
},
"figures": {
"items": 4,
"conditions": 3,
"operational_requirements": 10,
"capabilities": 21,
"custom_capabilities": 0,
"parties": 2,
"roles": 5,
"checks": 15,
"requirements_unverified": 0,
"checks_not_verifiable": 0,
"checks_waiting_on_findings": 0,
"proposed_additions": 0,
"scenarios": 13,
"findings": 0,
"business_outcomes": 3,
"measures": 4,
"people": 6.8,
"cost_per_month": 46252,
"currency": "GBP"
},
"warnings": [],
"folder": "deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r1",
"passed": true
},
{
"case": "case-01-landing-zone-shared",
"status": "completed",
"checks": [
{
"name": "completed",
"passed": true,
"detail": "completed"
},
{
"name": "every Deliverable was submitted",
"passed": true,
"detail": "all"
},
{
"name": "the chain holds",
"passed": true,
"detail": ""
},
{
"name": "no Check failed",
"passed": true,
"detail": ""
},
{
"name": "every scenario has a path",
"passed": true,
"detail": "13 of 13"
},
{
"name": "the guide has its sections",
"passed": true,
"detail": "Summary, Business outcomes, What is operated, Operational requirements, Capabilities, Who does what, Staffing and cost, Validation, Findings and open questions, Sources"
},
{
"name": "the packager wrote a summary",
"passed": true,
"detail": "163 words"
},
{
"name": "no Deliverable was revised more than 2 times",
"passed": true,
"detail": "target_operating_model x 1"
},
{
"name": "every Operational requirement is verified by a Check",
"passed": true,
"detail": "9 of 9"
},
{
"name": "finished within 40 minutes",
"passed": true,
"detail": "17.1"
}
],
"minutes": 17.1,
"revisions": {
"target_operating_model": 1
},
"tokens": {
"scope-analyst": {
"requests": 7,
"prompt": 48659,
"cached": 15492,
"tool_prompt": 0,
"output": 465,
"thinking": 6171,
"total": 55295
},
"reply-writer": {
"requests": 6,
"prompt": 4144,
"cached": 0,
"tool_prompt": 0,
"output": 820,
"thinking": 7499,
"total": 12463
},
"operations-analyst": {
"requests": 3,
"prompt": 16585,
"cached": 3873,
"tool_prompt": 0,
"output": 1991,
"thinking": 16707,
"total": 35283
},
"capability-modeller": {
"requests": 4,
"prompt": 30378,
"cached": 7804,
"tool_prompt": 0,
"output": 4679,
"thinking": 10442,
"total": 45499
},
"operating-model-architect": {
"requests": 23,
"prompt": 446402,
"cached": 38840,
"tool_prompt": 0,
"output": 11132,
"thinking": 19429,
"total": 476963
},
"validator": {
"requests": 9,
"prompt": 119967,
"cached": 55229,
"tool_prompt": 0,
"output": 2544,
"thinking": 8515,
"total": 131026
},
"packager": {
"requests": 4,
"prompt": 76526,
"cached": 32231,
"tool_prompt": 0,
"output": 414,
"thinking": 6276,
"total": 83216
}
},
"figures": {
"items": 4,
"conditions": 3,
"operational_requirements": 9,
"capabilities": 21,
"custom_capabilities": 0,
"parties": 2,
"roles": 7,
"checks": 15,
"requirements_unverified": 0,
"checks_not_verifiable": 0,
"checks_waiting_on_findings": 0,
"proposed_additions": 0,
"scenarios": 13,
"findings": 0,
"business_outcomes": 3,
"measures": 4,
"people": 11.0,
"cost_per_month": 44720,
"currency": "GBP"
},
"warnings": [],
"folder": "deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r2",
"passed": true
},
{
"case": "case-01-landing-zone-shared",
"status": "completed",
"checks": [
{
"name": "completed",
"passed": true,
"detail": "completed"
},
{
"name": "every Deliverable was submitted",
"passed": true,
"detail": "all"
},
{
"name": "the chain holds",
"passed": true,
"detail": ""
},
{
"name": "no Check failed",
"passed": true,
"detail": ""
},
{
"name": "every scenario has a path",
"passed": true,
"detail": "13 of 13"
},
{
"name": "the guide has its sections",
"passed": true,
"detail": "Summary, Business outcomes, What is operated, Operational requirements, Capabilities, Who does what, Staffing and cost, Validation, Findings and open questions, Sources"
},
{
"name": "the packager wrote a summary",
"passed": true,
"detail": "155 words"
},
{
"name": "no Deliverable was revised more than 2 times",
"passed": true,
"detail": "target_operating_model x 1"
},
{
"name": "every Operational requirement is verified by a Check",
"passed": true,
"detail": "10 of 10"
},
{
"name": "finished within 40 minutes",
"passed": true,
"detail": "20.2"
}
],
"minutes": 20.2,
"revisions": {
"target_operating_model": 1
},
"tokens": {
"scope-analyst": {
"requests": 7,
"prompt": 49123,
"cached": 19387,
"tool_prompt": 0,
"output": 459,
"thinking": 6063,
"total": 55645
},
"reply-writer": {
"requests": 6,
"prompt": 4204,
"cached": 0,
"tool_prompt": 0,
"output": 833,
"thinking": 7734,
"total": 12771
},
"operations-analyst": {
"requests": 4,
"prompt": 24044,
"cached": 3923,
"tool_prompt": 0,
"output": 2294,
"thinking": 14999,
"total": 41337
},
"capability-modeller": {
"requests": 3,
"prompt": 24774,
"cached": 3936,
"tool_prompt": 0,
"output": 4583,
"thinking": 14748,
"total": 44105
},
"operating-model-architect": {
"requests": 19,
"prompt": 356611,
"cached": 9523,
"tool_prompt": 0,
"output": 11354,
"thinking": 17771,
"total": 385736
},
"validator": {
"requests": 11,
"prompt": 161689,
"cached": 50998,
"tool_prompt": 0,
"output": 3729,
"thinking": 13885,
"total": 179303
},
"packager": {
"requests": 3,
"prompt": 59018,
"cached": 32277,
"tool_prompt": 0,
"output": 377,
"thinking": 2511,
"total": 61906
}
},
"figures": {
"items": 4,
"conditions": 3,
"operational_requirements": 10,
"capabilities": 21,
"custom_capabilities": 0,
"parties": 2,
"roles": 6,
"checks": 19,
"requirements_unverified": 0,
"checks_not_verifiable": 0,
"checks_waiting_on_findings": 0,
"proposed_additions": 0,
"scenarios": 13,
"findings": 0,
"business_outcomes": 3,
"measures": 4,
"people": 11.4,
"cost_per_month": 47603,
"currency": "GBP"
},
"warnings": [],
"folder": "deliverables/cases/bench-landing-zone-shared/executions/case-01-landing-zone-shared-r3",
"passed": true
},
{
"case": "case-02-from-architecture",
"status": "completed_with_warnings",
"checks": [
{
"name": "completed",
"passed": false,
"detail": "completed_with_warnings"
},
{
"name": "every Deliverable was submitted",
"passed": false,
"detail": "operating_model_guide"
},
{
"name": "the chain holds",
"passed": true,
"detail": ""
},
{
"name": "no Check failed",
"passed": false,
"detail": "OC-2, OC-4, OC-9, OC-12, OC-17"
},
{
"name": "every scenario has a path",
"passed": true,
"detail": "31 of 31"
},
{
"name": "the guide has its sections",
"passed": false,
"detail": ""
},
{
"name": "the packager wrote a summary",
"passed": false,
"detail": "0 words"
},
{
"name": "no Deliverable was revised more than 3 times",
"passed": true,
"detail": "capability_model x 2, target_operating_model x 3"
},
{
"name": "every Operational requirement is verified by a Check",
"passed": true,
"detail": "17 of 17"
},
{
"name": "finished within 60 minutes",
"passed": true,
"detail": "50.4"
}
],
"minutes": 50.4,
"revisions": {
"capability_model": 2,
"target_operating_model": 3
},
"tokens": {
"scope-analyst": {
"requests": 6,
"prompt": 48208,
"cached": 19432,
"tool_prompt": 0,
"output": 865,
"thinking": 6386,
"total": 55459
},
"reply-writer": {
"requests": 8,
"prompt": 5567,
"cached": 0,
"tool_prompt": 0,
"output": 1116,
"thinking": 9789,
"total": 16472
},
"operations-analyst": {
"requests": 5,
"prompt": 40800,
"cached": 19700,
"tool_prompt": 0,
"output": 3804,
"thinking": 17289,
"total": 61893
},
"capability-modeller": {
"requests": 14,
"prompt": 190167,
"cached": 47744,
"tool_prompt": 0,
"output": 14612,
"thinking": 50372,
"total": 255151
},
"operating-model-architect": {
"requests": 56,
"prompt": 1360422,
"cached": 688217,
"tool_prompt": 0,
"output": 48734,
"thinking": 94245,
"total": 1503401
},
"validator": {
"requests": 37,
"prompt": 685069,
"cached": 352339,
"tool_prompt": 0,
"output": 12846,
"thinking": 46979,
"total": 744894
}
},
"figures": {
"items": 9,
"conditions": 12,
"operational_requirements": 17,
"capabilities": 21,
"custom_capabilities": 0,
"parties": 2,
"roles": 8,
"checks": 17,
"requirements_unverified": 0,
"checks_not_verifiable": 0,
"checks_waiting_on_findings": 0,
"proposed_additions": 0,
"scenarios": 31,
"findings": 0,
"business_outcomes": 3,
"measures": 6,
"people": 11.7,
"cost_per_month": 73863,
"currency": "GBP"
},
"warnings": [
"target_operating_model has reached its revision limit (3). Requested change: OC-2 failed: The check requires the secondary region to be 'continuously maintained' as an active warm standby, but Capability C-9 is configured for 'business' hours, meaning maintenance cannot be continuous (24x7). C-9 should be 24x7 or another capability should provide continuous maintenance.; OC-4 failed: The check requires validation within 10 weeks, but Capability C-16 covers managing the roadmap without explicitly mentioning the 10-week requirement (which is instead mentioned in C-18). C-16 must mention the 10-week validation or the check mapped to C-18.; OC-9 failed: The check requires that expenditure does not exceed target allocations without 'documented operational sign-off', but Capability C-14 only mentions 'escalating anomalies' and does not explicitly guarantee documented sign-off.; OC-12 failed: The check requires identity permissions to be 'continuously reviewed', but Capability C-11 operates only during 'business' hours, meaning reviews cannot be continuous. C-11 must be 24x7.; OC-17 failed: The check requires verifiable alignment with DORA, PCI-DSS v4.0, and ISO 27001. Capability C-7 only ensures alignment with PCI-DSS v4.0 baseline configurations, completely missing DORA and ISO 27001 controls.. Ask the user whether to revise it again or accept it as it stands."
],
"folder": "deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r1",
"passed": false
},
{
"case": "case-02-from-architecture",
"status": "completed",
"checks": [
{
"name": "completed",
"passed": true,
"detail": "completed"
},
{
"name": "every Deliverable was submitted",
"passed": true,
"detail": "all"
},
{
"name": "the chain holds",
"passed": true,
"detail": ""
},
{
"name": "no Check failed",
"passed": true,
"detail": ""
},
{
"name": "every scenario has a path",
"passed": true,
"detail": "31 of 31"
},
{
"name": "the guide has its sections",
"passed": true,
"detail": "Summary, Business outcomes, What is operated, Operational requirements, Capabilities, Who does what, Staffing and cost, Validation, Findings and open questions, Sources"
},
{
"name": "the packager wrote a summary",
"passed": true,
"detail": "167 words"
},
{
"name": "no Deliverable was revised more than 3 times",
"passed": true,
"detail": "target_operating_model x 1"
},
{
"name": "every Operational requirement is verified by a Check",
"passed": true,
"detail": "15 of 15"
},
{
"name": "finished within 60 minutes",
"passed": true,
"detail": "20.9"
}
],
"minutes": 20.9,
"revisions": {
"target_operating_model": 1
},
"tokens": {
"scope-analyst": {
"requests": 7,
"prompt": 58699,
"cached": 23389,
"tool_prompt": 0,
"output": 941,
"thinking": 8227,
"total": 67867
},
"reply-writer": {
"requests": 6,
"prompt": 4117,
"cached": 0,
"tool_prompt": 0,
"output": 792,
"thinking": 7548,
"total": 12457
},
"operations-analyst": {
"requests": 4,
"prompt": 31104,
"cached": 11821,
"tool_prompt": 0,
"output": 3343,
"thinking": 21723,
"total": 56170
},
"capability-modeller": {
"requests": 4,
"prompt": 50775,
"cached": 0,
"tool_prompt": 0,
"output": 5417,
"thinking": 17513,
"total": 73705
},
"operating-model-architect": {
"requests": 26,
"prompt": 581893,
"cached": 233265,
"tool_prompt": 0,
"output": 23437,
"thinking": 25452,
"total": 630782
},
"validator": {
"requests": 10,
"prompt": 169718,
"cached": 35567,
"tool_prompt": 0,
"output": 4610,
"thinking": 13791,
"total": 188119
},
"packager": {
"requests": 3,
"prompt": 80302,
"cached": 48607,
"tool_prompt": 0,
"output": 401,
"thinking": 7230,
"total": 87933
}
},
"figures": {
"items": 9,
"conditions": 12,
"operational_requirements": 15,
"capabilities": 21,
"custom_capabilities": 0,
"parties": 2,
"roles": 6,
"checks": 20,
"requirements_unverified": 0,
"checks_not_verifiable": 0,
"checks_waiting_on_findings": 0,
"proposed_additions": 0,
"scenarios": 31,
"findings": 0,
"business_outcomes": 3,
"measures": 6,
"people": 10.8,
"cost_per_month": 83846,
"currency": "GBP"
},
"warnings": [],
"folder": "deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r2",
"passed": true
},
{
"case": "case-02-from-architecture",
"status": "completed",
"checks": [
{
"name": "completed",
"passed": true,
"detail": "completed"
},
{
"name": "every Deliverable was submitted",
"passed": true,
"detail": "all"
},
{
"name": "the chain holds",
"passed": true,
"detail": ""
},
{
"name": "no Check failed",
"passed": true,
"detail": ""
},
{
"name": "every scenario has a path",
"passed": true,
"detail": "34 of 34"
},
{
"name": "the guide has its sections",
"passed": true,
"detail": "Summary, Business outcomes, What is operated, Operational requirements, Capabilities, Who does what, Staffing and cost, Validation, Findings and open questions, Sources"
},
{
"name": "the packager wrote a summary",
"passed": true,
"detail": "204 words"
},
{
"name": "no Deliverable was revised more than 3 times",
"passed": true,
"detail": "capability_model x 1, target_operating_model x 1"
},
{
"name": "every Operational requirement is verified by a Check",
"passed": true,
"detail": "15 of 15"
},
{
"name": "finished within 60 minutes",
"passed": true,
"detail": "24.4"
}
],
"minutes": 24.4,
"revisions": {
"capability_model": 1,
"target_operating_model": 1
},
"tokens": {
"scope-analyst": {
"requests": 6,
"prompt": 49107,
"cached": 19446,
"tool_prompt": 0,
"output": 984,
"thinking": 6456,
"total": 56547
},
"reply-writer": {
"requests": 7,
"prompt": 4900,
"cached": 0,
"tool_prompt": 0,
"output": 927,
"thinking": 10194,
"total": 16021
},
"operations-analyst": {
"requests": 4,
"prompt": 31544,
"cached": 15782,
"tool_prompt": 0,
"output": 3213,
"thinking": 17810,
"total": 52567
},
"capability-modeller": {
"requests": 10,
"prompt": 113370,
"cached": 19712,
"tool_prompt": 0,
"output": 9290,
"thinking": 31311,
"total": 153971
},
"operating-model-architect": {
"requests": 21,
"prompt": 407275,
"cached": 204633,
"tool_prompt": 0,
"output": 14394,
"thinking": 24719,
"total": 446388
},
"validator": {
"requests": 18,
"prompt": 329934,
"cached": 221850,
"tool_prompt": 0,
"output": 5737,
"thinking": 23300,
"total": 358971
},
"packager": {
"requests": 3,
"prompt": 80501,
"cached": 48609,
"tool_prompt": 0,
"output": 457,
"thinking": 6246,
"total": 87204
}
},
"figures": {
"items": 9,
"conditions": 12,
"operational_requirements": 15,
"capabilities": 21,
"custom_capabilities": 0,
"parties": 5,
"roles": 8,
"checks": 17,
"requirements_unverified": 0,
"checks_not_verifiable": 0,
"checks_waiting_on_findings": 0,
"proposed_additions": 0,
"scenarios": 34,
"findings": 0,
"business_outcomes": 3,
"measures": 7,
"people": 16.9,
"cost_per_month": 116409,
"currency": "GBP"
},
"warnings": [],
"folder": "deliverables/cases/case-08-landing-zone/executions/case-02-from-architecture-r3",
"passed": true
}
]