docs: publish the runs of 2026-10-21
The evaluation (both cases three times: report, results, spread, each run's Deliverables and lesson candidates) and the hand-off through the Orchestration engine (report, the document of links, both agents' kept files and records). Benchmark cases only, all fictional; no machine paths, and the links carry no signatures. Other runs stay ignored. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,76 @@
|
||||
{
|
||||
"execution_id": "case-01-landing-zone-shared-r1",
|
||||
"customer_case": "bench-landing-zone-shared",
|
||||
"status": "completed",
|
||||
"role_skills": {
|
||||
"capability-modeller": [
|
||||
"operating-capabilities",
|
||||
"operating-chain"
|
||||
],
|
||||
"operating-model-architect": [
|
||||
"operating-capabilities",
|
||||
"operating-chain",
|
||||
"rate-card"
|
||||
],
|
||||
"operations-analyst": [
|
||||
"operating-scoping",
|
||||
"operating-chain"
|
||||
],
|
||||
"orchestrator": [],
|
||||
"packager": [
|
||||
"operating-guide"
|
||||
],
|
||||
"reply-writer": [],
|
||||
"scope-analyst": [
|
||||
"operating-scoping"
|
||||
],
|
||||
"validator": [
|
||||
"operating-chain"
|
||||
]
|
||||
},
|
||||
"skill_versions": {
|
||||
"operating-capabilities": "49a7e66857a1",
|
||||
"operating-chain": "ec53ad8875e6",
|
||||
"operating-guide": "85b48d5998da",
|
||||
"operating-scoping": "08da33f8cddf",
|
||||
"rate-card": "f86d17a9bcce"
|
||||
},
|
||||
"candidates": [
|
||||
{
|
||||
"type": "revision",
|
||||
"deliverable": "target_operating_model",
|
||||
"reason": "OC-4 failed: The finding AF-1 does not stand because the operating model could provide the configuration of alerts itself as a process or using a tool.; OC-5 failed: The finding AF-2 does not stand because the operating model could provide a backup and recovery mechanism itself as a process or a tool.; OC-1 failed: Capability C-10 operates only during business hours, which is insufficient to ensure 99.9% availability as it requires 24x7 cover.; OC-8 failed: Capability C-15 covers measuring SLA achievement but does not cover operational costs, which is handled by C-14.",
|
||||
"requested_by": "orchestrator"
|
||||
},
|
||||
{
|
||||
"type": "failed_check",
|
||||
"attempt": 1,
|
||||
"check": "OC-4",
|
||||
"evidence": "The finding AF-1 does not stand because the operating model could provide the configuration of alerts itself as a process or using a tool."
|
||||
},
|
||||
{
|
||||
"type": "failed_check",
|
||||
"attempt": 1,
|
||||
"check": "OC-5",
|
||||
"evidence": "The finding AF-2 does not stand because the operating model could provide a backup and recovery mechanism itself as a process or a tool."
|
||||
},
|
||||
{
|
||||
"type": "failed_check",
|
||||
"attempt": 1,
|
||||
"check": "OC-1",
|
||||
"evidence": "Capability C-10 operates only during business hours, which is insufficient to ensure 99.9% availability as it requires 24x7 cover."
|
||||
},
|
||||
{
|
||||
"type": "failed_check",
|
||||
"attempt": 1,
|
||||
"check": "OC-8",
|
||||
"evidence": "Capability C-15 covers measuring SLA achievement but does not cover operational costs, which is handled by C-14."
|
||||
},
|
||||
{
|
||||
"type": "unverifiable_check",
|
||||
"attempt": 1,
|
||||
"check": "OC-2",
|
||||
"evidence": "The condition that audit logs cannot be deleted is a technical constraint that cannot be verified by an operating model capability."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
{
|
||||
"execution_id": "case-01-landing-zone-shared-r2",
|
||||
"customer_case": "bench-landing-zone-shared",
|
||||
"status": "completed",
|
||||
"role_skills": {
|
||||
"capability-modeller": [
|
||||
"operating-capabilities",
|
||||
"operating-chain"
|
||||
],
|
||||
"operating-model-architect": [
|
||||
"operating-capabilities",
|
||||
"operating-chain",
|
||||
"rate-card"
|
||||
],
|
||||
"operations-analyst": [
|
||||
"operating-scoping",
|
||||
"operating-chain"
|
||||
],
|
||||
"orchestrator": [],
|
||||
"packager": [
|
||||
"operating-guide"
|
||||
],
|
||||
"reply-writer": [],
|
||||
"scope-analyst": [
|
||||
"operating-scoping"
|
||||
],
|
||||
"validator": [
|
||||
"operating-chain"
|
||||
]
|
||||
},
|
||||
"skill_versions": {
|
||||
"operating-capabilities": "49a7e66857a1",
|
||||
"operating-chain": "ec53ad8875e6",
|
||||
"operating-guide": "85b48d5998da",
|
||||
"operating-scoping": "08da33f8cddf",
|
||||
"rate-card": "f86d17a9bcce"
|
||||
},
|
||||
"candidates": [
|
||||
{
|
||||
"type": "revision",
|
||||
"deliverable": "target_operating_model",
|
||||
"reason": "OC-1 failed: Finding AF-1 does not stand. The operating model should provide alert configurations itself using the existing native Google Cloud monitoring tools, rather than raising a finding against the solution.; OC-3 failed: Finding AF-1 does not stand. The operating model should provide alert configurations itself using the existing native Google Cloud monitoring tools, rather than raising a finding against the solution.; SC-budget failed: A month's cost of operating is compared with the budget of 60,000 GBP. The cost is 64,125 GBP, over the budget.",
|
||||
"requested_by": "orchestrator"
|
||||
},
|
||||
{
|
||||
"type": "failed_check",
|
||||
"attempt": 1,
|
||||
"check": "OC-1",
|
||||
"evidence": "Finding AF-1 does not stand. The operating model should provide alert configurations itself using the existing native Google Cloud monitoring tools, rather than raising a finding against the solution."
|
||||
},
|
||||
{
|
||||
"type": "failed_check",
|
||||
"attempt": 1,
|
||||
"check": "OC-3",
|
||||
"evidence": "Finding AF-1 does not stand. The operating model should provide alert configurations itself using the existing native Google Cloud monitoring tools, rather than raising a finding against the solution."
|
||||
},
|
||||
{
|
||||
"type": "failed_check",
|
||||
"attempt": 1,
|
||||
"check": "SC-budget",
|
||||
"evidence": "A month's cost of operating is compared with the budget of 60,000 GBP. The cost is 64,125 GBP, over the budget."
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"execution_id": "case-01-landing-zone-shared-r3",
|
||||
"customer_case": "bench-landing-zone-shared",
|
||||
"status": "completed",
|
||||
"role_skills": {
|
||||
"capability-modeller": [
|
||||
"operating-capabilities",
|
||||
"operating-chain"
|
||||
],
|
||||
"operating-model-architect": [
|
||||
"operating-capabilities",
|
||||
"operating-chain",
|
||||
"rate-card"
|
||||
],
|
||||
"operations-analyst": [
|
||||
"operating-scoping",
|
||||
"operating-chain"
|
||||
],
|
||||
"orchestrator": [],
|
||||
"packager": [
|
||||
"operating-guide"
|
||||
],
|
||||
"reply-writer": [],
|
||||
"scope-analyst": [
|
||||
"operating-scoping"
|
||||
],
|
||||
"validator": [
|
||||
"operating-chain"
|
||||
]
|
||||
},
|
||||
"skill_versions": {
|
||||
"operating-capabilities": "49a7e66857a1",
|
||||
"operating-chain": "ec53ad8875e6",
|
||||
"operating-guide": "85b48d5998da",
|
||||
"operating-scoping": "08da33f8cddf",
|
||||
"rate-card": "f86d17a9bcce"
|
||||
},
|
||||
"candidates": [
|
||||
{
|
||||
"type": "revision",
|
||||
"deliverable": "target_operating_model",
|
||||
"reason": "OC-1 failed: The finding AF-1 does not stand because the operating model can provide alert configuration itself via C-2 (monitoring-and-alerting) using the Cloud provider's native operations tooling. The operating model should configure the alerts rather than expecting the solution to specify them.",
|
||||
"requested_by": "orchestrator"
|
||||
},
|
||||
{
|
||||
"type": "failed_check",
|
||||
"attempt": 1,
|
||||
"check": "OC-1",
|
||||
"evidence": "The finding AF-1 does not stand because the operating model can provide alert configuration itself via C-2 (monitoring-and-alerting) using the Cloud provider's native operations tooling. The operating model should configure the alerts rather than expecting the solution to specify them."
|
||||
},
|
||||
{
|
||||
"type": "proposed_addition",
|
||||
"attempt": 1,
|
||||
"reason": "Plausibility of effort for continuous 24x7 analysis.",
|
||||
"applies_to": "C-13",
|
||||
"statement": "C-13 (security-monitoring-and-response) has an effort of 40 hours per month for continuous monitoring of security events and continuous analysis of audit logs. If the analysis is manual, 40 hours per month is insufficient for continuous 24x7 coverage. Consider clarifying if this relies on automated SIEM alerting or increasing the effort."
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user