From efe543e6e2e400a0b27dc7eb98fa8f13ccd20328 Mon Sep 17 00:00:00 2001 From: Jonathan Boniface Date: Wed, 21 Oct 2026 19:47:12 +0100 Subject: [PATCH] feat: working TOM agent --- .agents/plans/README.md | 10 + .agents/plans/iter-01-first-slice/MAP.md | 31 + .../plans/iter-01-first-slice/steps/step-1.md | 44 + .../plans/iter-01-first-slice/steps/step-2.md | 38 + .../plans/iter-01-first-slice/steps/step-3.md | 67 + .../plans/iter-02-platform-deployment/MAP.md | 31 + .../steps/step-1.md | 29 + .../steps/step-2.md | 20 + .../steps/step-3.md | 28 + .dockerignore | 13 + .gitignore | 29 + .scratch/target-operating-model/spec.md | 144 ++ CLAUDE.md | 27 + CONTEXT.md | 99 ++ Dockerfile | 52 + IMPLEMENTATION_PLAN.md | 393 ++++++ README.md | 36 + agents/capability-modeller.md | 27 + agents/operating-model-architect.md | 83 ++ agents/operations-analyst.md | 25 + agents/orchestrator.md | 9 + agents/packager.md | 15 + agents/partials/specialist-protocol.md | 37 + agents/reply-writer.md | 30 + agents/scope-analyst.md | 24 + agents/validator.md | 20 + app/__init__.py | 0 app/main.py | 8 + app/operating_model/__init__.py | 0 app/operating_model/catalogue.py | 153 +++ app/operating_model/chain.py | 240 ++++ app/operating_model/digest.py | 413 ++++++ app/operating_model/graph.py | 719 ++++++++++ app/operating_model/guide.py | 387 ++++++ app/operating_model/names.py | 66 + app/operating_model/scenarios.py | 346 +++++ app/operating_model/spec.py | 89 ++ app/operating_model/staffing.py | 280 ++++ app/operating_model/workflow.py | 572 ++++++++ ...et-operating-model-is-an-explicit-graph.md | 27 + ...execution-starts-from-an-operating-spec.md | 26 + ...ed-from-the-service-providers-rate-card.md | 29 + eval/__init__.py | 0 eval/datasets/cases.json | 32 + eval/datasets/specs/landing-zone-shared.yaml | 53 + eval/operating_model_eval.py | 208 +++ eval/scoring.py | 198 +++ providers/gcp.yaml | 64 + pyproject.toml | 33 + skills/operating-capabilities/SKILL.md | 50 + .../assets/catalogue.yaml | 281 ++++ .../assets/sources.yaml | 82 ++ skills/operating-chain/SKILL.md | 51 + skills/operating-guide/SKILL.md | 27 + .../operating-guide/assets/guide-template.md | 58 + skills/operating-scoping/SKILL.md | 47 + .../operating-scoping/assets/catalogue.yaml | 214 +++ skills/rate-card/SKILL.md | 36 + skills/rate-card/assets/rates.yaml | 56 + tests/__init__.py | 0 tests/conftest.py | 5 + tests/fixtures/architecture/README.md | 7 + .../case-08-landing-zone/components.json | 79 ++ .../case-08-landing-zone/execution.json | 140 ++ .../case-08-landing-zone/products.json | 38 + .../case-08-landing-zone/recommendations.json | 248 ++++ .../case-08-landing-zone/requirements.json | 280 ++++ .../case-08-landing-zone/scope.json | 233 ++++ .../target-architecture.json | 1210 +++++++++++++++++ .../case-08-landing-zone/verdicts.json | 529 +++++++ .../case-08-landing-zone/work-breakdown.json | 332 +++++ tests/support.py | 294 ++++ tests/test_catalogue.py | 107 ++ tests/test_chain.py | 142 ++ tests/test_digest.py | 237 ++++ tests/test_eval.py | 69 + tests/test_execution.py | 411 ++++++ tests/test_graph.py | 241 ++++ tests/test_guide.py | 166 +++ tests/test_outcomes.py | 218 +++ tests/test_scenarios.py | 219 +++ tests/test_staffing.py | 161 +++ tests/test_teams.py | 207 +++ 83 files changed, 11779 insertions(+) create mode 100644 .agents/plans/README.md create mode 100644 .agents/plans/iter-01-first-slice/MAP.md create mode 100644 .agents/plans/iter-01-first-slice/steps/step-1.md create mode 100644 .agents/plans/iter-01-first-slice/steps/step-2.md create mode 100644 .agents/plans/iter-01-first-slice/steps/step-3.md create mode 100644 .agents/plans/iter-02-platform-deployment/MAP.md create mode 100644 .agents/plans/iter-02-platform-deployment/steps/step-1.md create mode 100644 .agents/plans/iter-02-platform-deployment/steps/step-2.md create mode 100644 .agents/plans/iter-02-platform-deployment/steps/step-3.md create mode 100644 .dockerignore create mode 100644 .gitignore create mode 100644 .scratch/target-operating-model/spec.md create mode 100644 CLAUDE.md create mode 100644 CONTEXT.md create mode 100644 Dockerfile create mode 100644 IMPLEMENTATION_PLAN.md create mode 100644 README.md create mode 100644 agents/capability-modeller.md create mode 100644 agents/operating-model-architect.md create mode 100644 agents/operations-analyst.md create mode 100644 agents/orchestrator.md create mode 100644 agents/packager.md create mode 100644 agents/partials/specialist-protocol.md create mode 100644 agents/reply-writer.md create mode 100644 agents/scope-analyst.md create mode 100644 agents/validator.md create mode 100644 app/__init__.py create mode 100644 app/main.py create mode 100644 app/operating_model/__init__.py create mode 100644 app/operating_model/catalogue.py create mode 100644 app/operating_model/chain.py create mode 100644 app/operating_model/digest.py create mode 100644 app/operating_model/graph.py create mode 100644 app/operating_model/guide.py create mode 100644 app/operating_model/names.py create mode 100644 app/operating_model/scenarios.py create mode 100644 app/operating_model/spec.py create mode 100644 app/operating_model/staffing.py create mode 100644 app/operating_model/workflow.py create mode 100644 docs/adr/0001-the-target-operating-model-is-an-explicit-graph.md create mode 100644 docs/adr/0002-an-execution-starts-from-an-operating-spec.md create mode 100644 docs/adr/0003-cost-is-computed-from-the-service-providers-rate-card.md create mode 100644 eval/__init__.py create mode 100644 eval/datasets/cases.json create mode 100644 eval/datasets/specs/landing-zone-shared.yaml create mode 100644 eval/operating_model_eval.py create mode 100644 eval/scoring.py create mode 100644 providers/gcp.yaml create mode 100644 pyproject.toml create mode 100644 skills/operating-capabilities/SKILL.md create mode 100644 skills/operating-capabilities/assets/catalogue.yaml create mode 100644 skills/operating-capabilities/assets/sources.yaml create mode 100644 skills/operating-chain/SKILL.md create mode 100644 skills/operating-guide/SKILL.md create mode 100644 skills/operating-guide/assets/guide-template.md create mode 100644 skills/operating-scoping/SKILL.md create mode 100644 skills/operating-scoping/assets/catalogue.yaml create mode 100644 skills/rate-card/SKILL.md create mode 100644 skills/rate-card/assets/rates.yaml create mode 100644 tests/__init__.py create mode 100644 tests/conftest.py create mode 100644 tests/fixtures/architecture/README.md create mode 100644 tests/fixtures/architecture/case-08-landing-zone/components.json create mode 100644 tests/fixtures/architecture/case-08-landing-zone/execution.json create mode 100644 tests/fixtures/architecture/case-08-landing-zone/products.json create mode 100644 tests/fixtures/architecture/case-08-landing-zone/recommendations.json create mode 100644 tests/fixtures/architecture/case-08-landing-zone/requirements.json create mode 100644 tests/fixtures/architecture/case-08-landing-zone/scope.json create mode 100644 tests/fixtures/architecture/case-08-landing-zone/target-architecture.json create mode 100644 tests/fixtures/architecture/case-08-landing-zone/verdicts.json create mode 100644 tests/fixtures/architecture/case-08-landing-zone/work-breakdown.json create mode 100644 tests/support.py create mode 100644 tests/test_catalogue.py create mode 100644 tests/test_chain.py create mode 100644 tests/test_digest.py create mode 100644 tests/test_eval.py create mode 100644 tests/test_execution.py create mode 100644 tests/test_graph.py create mode 100644 tests/test_guide.py create mode 100644 tests/test_outcomes.py create mode 100644 tests/test_scenarios.py create mode 100644 tests/test_staffing.py create mode 100644 tests/test_teams.py diff --git a/.agents/plans/README.md b/.agents/plans/README.md new file mode 100644 index 0000000..33910a4 --- /dev/null +++ b/.agents/plans/README.md @@ -0,0 +1,10 @@ +# Plans + +One directory per iteration. `../../IMPLEMENTATION_PLAN.md` is the plan across this repository and the engine's; its steps T1 onward are built here. + +| # | Directory | Status | What it is | +| --- | --- | --- | --- | +| 1 | `iter-01-first-slice/` | the Workflow, its checks and views built on 2026-10-13 from a written operating spec; the digest and evaluation remain | Six Deliverables, from an operating spec to a priced, validated operating model | +| 2 | `iter-02-platform-deployment/` | steps 1 to 3 built 2026-10-20; step 4 in progress | The agent as a container on the local KAB platform, then behind the Orchestration engine | + +Each iteration has a `MAP.md` (intent, each step's status, sources of truth) and `steps/step-N.md`, written when a step is finished. A step with no file is not done. diff --git a/.agents/plans/iter-01-first-slice/MAP.md b/.agents/plans/iter-01-first-slice/MAP.md new file mode 100644 index 0000000..07a5a7b --- /dev/null +++ b/.agents/plans/iter-01-first-slice/MAP.md @@ -0,0 +1,31 @@ +# Plan Map + +**Iteration:** 1, the first slice +**Intent:** From an operating spec to a priced, validated operating model, in six Deliverables. +**Status:** T1 to T14 built; both evaluation cases passed with real models on 2026-10-14 + +## Sources of truth + +- `.scratch/target-operating-model/spec.md`: the formats, the rules and the arithmetic. +- `docs/adr/`: 0001 (the graph, and findings), 0002 (the operating spec), 0003 (cost). +- `IMPLEMENTATION_PLAN.md`, section 10: the steps. +- `../orchestration_engine/execution_engine/workflows/workflow.py`: what a Workflow supplies. + +## Status + +| Step | What | Status | +| --- | --- | --- | +| T1 | The repository on the engine | built (`steps/step-1.md`) | +| T2 | The operating catalogue and scoping Skill | built (`steps/step-1.md`) | +| T3 | The architecture adapter and the digest command | built (`steps/step-2.md`) | +| T4 | The model-drafted digest for other documents | built as instructions to the scope analyst (`steps/step-2.md`); not yet run by a model | +| T5 | The capability catalogue Skill and its sources register | built (`steps/step-1.md`); references unverified | +| T6 | Operational requirements and capabilities, and their checks | built (`steps/step-1.md`) | +| T7 | The graph: refusals and rendered views | built (`steps/step-1.md`) | +| T8 | Findings | built (`steps/step-1.md`) | +| T9 | Role files, the provider bundle, the Workflow | built (`steps/step-1.md`) | +| T10 | Validation in code: verdict rules and scenario walk-throughs | built (`steps/step-1.md`) | +| T11 | The guide | built (`steps/step-2.md`) | +| T12 | The rate card Skill's format and checks | built (`steps/step-1.md`) | +| T13 | Staffing and cost in code | built (`steps/step-1.md`) | +| T14 | Evaluation with real models | built; two cases passed (`steps/step-2.md`) | diff --git a/.agents/plans/iter-01-first-slice/steps/step-1.md b/.agents/plans/iter-01-first-slice/steps/step-1.md new file mode 100644 index 0000000..47a63ea --- /dev/null +++ b/.agents/plans/iter-01-first-slice/steps/step-1.md @@ -0,0 +1,44 @@ +# Steps T1, T2, T5 to T13: the Workflow, from a written operating spec + +Built 2026-10-13, on the Execution engine as it stood that day (not yet committed there). + +## What was built + +- **The Workflow** (`app/operating_model/workflow.py`): five Phases, six Deliverables, each with its Specialist, sections and data file; the validation policy; the views; the account of a Change request; the card. +- **The operating spec's catalogue** (`skills/operating-scoping/`): 32 fields, read by the engine's spec reader. The provider bundle supplies the Cloud provider and the currencies. +- **The capability catalogue** (`skills/operating-capabilities/`): 17 entries in five domains, each citing frameworks from a register of nine. `catalogue.py` refuses an entry that cites nothing or cites an unregistered framework, and proposes the entries that apply to a spec. +- **The chain** (`chain.py`): the checks on operational requirements and on capabilities. +- **The graph** (`graph.py`): its shape for the engine's graph tools, its refusals, findings, and the rendered responsibility matrix, organisation chart, and staffing and cost. +- **Staffing and cost** (`staffing.py`): the rate card's loader and checks, and the arithmetic. +- **Scenarios** (`scenarios.py`): five kinds, generated and traced by code, their verdicts written into `verdicts.json`. +- **The agent's folder**: eight role files and a shared protocol, `providers/gcp.yaml`, the `operating-chain` Skill, `app/main.py`. +- **106 tests**, among them seven whole Executions on the scripted runtime. + +## Differed from the plan and the spec + +- **The spec's files keep the engine's names,** `scope.yaml` and `scope.json`. The plan said `operating-spec.yaml`. The engine's spec support does not yet let a Workflow name them. +- **The inventory travels with the rest of the spec** to every Specialist, not as a separate `solution.json`. The engine's digest step (E6) is where that changes. +- **The graph holds no list of Capabilities.** They are the capability model's, named by id, so a failed Check can reopen who does the work without reopening what the work is. +- **The currencies a spec may state are listed in the provider bundle,** so the engine refuses another at the start. A test keeps the list the same as the rate card's conversion rates. +- **A Role on business hours is as small as its work.** The spec's formula applied the minimum roster to every Role; the plan, which the owner approved, applies it only to hours wider than a working day. The plan was followed. +- **A Check that depends on a finding is recorded as `waits_on_finding`.** The validator gives it `not_verifiable`; code rewrites the verdict when the validation is submitted. Left as `not_verifiable`, the engine's router reopened the capability model for it, again and again: a finding is not a gap in the model. +- **A scenario's verdicts are written into `verdicts.json` by code,** as Checks whose ids begin `SC-`, and the walk-through as a whole is the file's `dry_run`. A failed scenario is therefore a failed Check to the router, which reopens the graph. +- **The matrix shows both parts** when one Role is accountable and responsible for a Capability (`A/R`). + +## Not built + +- **The digest** (T3, T4). An Execution starts from an operating spec that is already written. Nothing reads the solution architecture agent's files yet. +- **A guide template** (T11). The packager has its sections and its instructions. +- **Evaluation** (T14). Nothing has run with a real model: no role file, Skill or catalogue entry has been read by one. +- An interactive Execution and a Change request are not tested here; the engine's own tests cover both on other Workflows. +- The capability catalogue's references are drafted from general knowledge and marked `unverified`. Nobody has checked them against the frameworks. +- The rate card's figures are placeholders. + +## What it found in the engine + +- The engine was not typed as a package (`py.typed` added there), so this repository could not be type-checked against it. +- An editable install of the engine is invisible to the type checker unless made with `editable_mode=compat`. + +## Verified + +`pytest -q`: 106 passed. `ruff check .`, `ruff format --check .`, `mypy .` (20 files): clean. The engine's suite still passes (521). `git status` in the solution architecture agent shows the same nine modified files as before any of this work began. diff --git a/.agents/plans/iter-01-first-slice/steps/step-2.md b/.agents/plans/iter-01-first-slice/steps/step-2.md new file mode 100644 index 0000000..e8fc495 --- /dev/null +++ b/.agents/plans/iter-01-first-slice/steps/step-2.md @@ -0,0 +1,38 @@ +# Steps T3, T4, T11 and T14: the digest, the guide, and the first runs with real models + +Built 2026-10-13 and 14, with the engine's digest step (its `steps/step-6.md`). + +## What was built + +- **The architecture adapter** (`app/operating_model/digest.py`, `python -m app.operating_model.digest `). An Item is a component of the architecture, with the Elements that realise it; a Condition is what its requirements and recommendations say can only be shown in operation (`outside_code`); a Check that did not pass is an inherited risk. Only what the customer stated in the architecture's scope is carried across. The provider bundle's `operating.item_kinds` says what kind of thing each product is. Tested against a real completed landing-zone Execution, copied into `tests/fixtures/architecture/`. +- **Documents code cannot read** go to the scope analyst, who records what it takes from them with the passage quoted (T4, as instructions; no model has run it). +- **The guide** (`app/operating_model/guide.py`, `skills/operating-guide/`). The packager writes the summary; code writes every other section and the key figures from the data. +- **The evaluation harness** (`eval/`): two cases, structural scoring, tokens per role. + +## Differed from the plan + +- An Item is a component, not an Element. The real architecture had 60 Elements in 9 components; an organisation operates the components. +- The architecture's 104 Checks on infrastructure settings are not Conditions. Keeping them true is configuration management. Its 12 statements of what can only be shown in operation are. + +## The runs with real models, 2026-10-14 + +| Case | Result | Minutes | Revisions | Capabilities | Roles | Checks | Scenarios | People | Cost a month | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 01, written spec (4 Items) | pass | 25.8 | operating model ×2, capability model ×1 | 17 | 4 | 8 | 13 of 13 | 5.2 | 25,355 GBP | +| 02, digested architecture (9 Items) | pass | 16.8 | operating model ×1 | 17 | 5 | 36 | 31 of 31 | 11.6 | 67,298 GBP | + +Costs are from the placeholder rate card. An earlier run of case 01 passed every structural check in 30.6 minutes, over its limit then of 30. + +What they showed: + +- **Most time goes on the architect placing a Check on the wrong Capability**, such as a restore-time Check on service-level reporting. Case 01 revised the same way in both runs. Not yet acted on. +- **The router's reasons still speak of code.** A Check that could not be verified reopens the capability model with "could not be verified from the code", and a Check that failed twice is said to be possibly "not implementable". These are the engine's words from the architecture's subject. +- **The capability modeller takes all 17 catalogue entries** whatever the size of the solution, as its instructions say. +- **The architect is most of the cost:** about 380,000 to 490,000 of each case's 780,000 to 940,000 prompt tokens. +- One request was retried after Vertex AI's tokens-per-minute quota was reached; it did not affect the result. + +Nobody has yet read the outputs for quality: these checks are structural only. + +## Verified + +The engine's 529 tests and this repository's 140 pass, with `ruff` and `mypy` clean in both. diff --git a/.agents/plans/iter-01-first-slice/steps/step-3.md b/.agents/plans/iter-01-first-slice/steps/step-3.md new file mode 100644 index 0000000..4762128 --- /dev/null +++ b/.agents/plans/iter-01-first-slice/steps/step-3.md @@ -0,0 +1,67 @@ +# Socio-technical and organisational frameworks + +Added 2026-10-14, at the owner's request. + +## What was added + +- **Six frameworks in the register** (`skills/operating-capabilities/assets/sources.yaml`): Team Topologies; platform as a product, cited from the CNCF Platforms White Paper; the SPACE framework; and the DevEx framework. The DORA delivery metrics were already there. Like every other entry, their references are unverified. +- **Four Capabilities in the catalogue:** platform product management, self-service and golden paths, and enablement, proposed for a platform foundation; and delivery and developer experience measurement, proposed for a platform foundation or a workload. The catalogue can now propose an entry by the kind of engagement. +- **Measurement at three layers.** The measurement Capability cites DORA, SPACE and DevEx together, each for a different layer: delivery speed and stability as the outcome, sustainable productivity as the guardrail, friction as the leading signal. The Skill says no one layer stands in for another. +- **Teams and interactions in the graph,** both optional. A Team is of one of Team Topologies' four kinds, in one Party; a Role may be in a Team of its own Party. An Interaction between two Teams is in one of the three modes, and a collaboration must say when it ends. The organisation chart draws teams inside their Parties and interactions as labelled edges; the guide lists both. +- **Two new terms** in `CONTEXT.md`: Team and Interaction. +- **The architect's instructions** say how to use teams and interaction modes, and that a Check goes on the Capability that does the work, not the one that reports it. The second was the cause of most Revisions in the runs of 2026-10-14. + +## Not done + +- No run with real models since. The four new Capabilities add to what the capability modeller takes, which was already all 17 entries for a platform foundation; it will now be 21. +- Teams are not used by staffing, cost or the scenario walk-throughs. + +## Verified + +159 tests pass; `ruff` and `mypy` clean. + +## The run with real models after this change, 2026-10-14 + +| Case | Minutes | Revisions | Capabilities | Teams | Interactions | Checks | Scenarios | People | Cost a month | Model cost | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| 01, written spec | 20.9 (was 25.8) | operating model ×1 (was 3 in all) | 21 | 2 | 2 | 13 | 13 of 13 | 6.6 | 32,234 GBP | $1.87 (was $2.27) | +| 02, digested architecture | 16.6 (was 16.8) | operating model ×1 (was 1) | 21 | 3 | 2 | 31 | 35 of 35 | 12.5 | 77,550 GBP | $1.61 (was $1.55) | + +Both passed every check. The one Revision in each was still a Check whose hours or Capability did not fit, but the capability model was not reopened in either. Both models used teams: platform and stream-aligned teams, an X-as-a-Service interaction in each, and in case 01 a collaboration that ends three months after go-live, in case 02 a facilitating interaction that ends with the build. Case 02 named six Parties. + +# Business outcomes + +Added 2026-10-14, at the owner's request: every measure is tied back to an outcome the business stated. + +- **`intent.business_outcomes`** in the operating spec: each with an id, the customer's words, and a target where given. The digest carries the architecture's stated success criteria across as business outcomes. +- **Measures** in the graph: what is measured, its layer (outcome, guardrail, leading, service level or cost), its framework, its target, the outcome it shows and the Capability that takes it. Refused: a stated outcome shown by no Measure, a Measure for an outcome that was not stated, a Measure taken by a Capability nobody is responsible for, a framework not in the register, and, where the model measures how work flows, any of the three developer layers left unmeasured. +- **An Operational requirement may come from a business outcome.** It need not: the Measures are what show each outcome is served. +- **The guide's second section is Business outcomes,** written by code: each outcome with the Measures that show it and the Role answerable for each. +- **Two new terms** in `CONTEXT.md`: Business outcome and Measure. The benchmark spec now states three outcomes. + +Not yet run with real models. 173 tests pass; `ruff` and `mypy` clean. + +## The run with business outcomes, 2026-10-14 + +- **Case 01 passed** in 29.2 minutes, for $2.73. It tied four Measures to the spec's three business outcomes: deployment lead time (DORA, outcome layer) to faster releases, time to restore service (DORA) to short outages, and satisfaction and wellbeing (SPACE, guardrail) and feedback loops and cognitive load (DevEx, leading) both to keeping engineers. Two Revisions of the operating model and one of the capability model. +- **Case 02 looped and was stopped** after about two and a half hours. The architecture states no business outcome, so every Measure was refused as serving an outcome "none were stated", while the measurement Capability's rule asked for Measures at three layers. The two rules could not both be met. The architect asked a question five times in a pre-approved run, each answered automatically, each run taking 10 to 28 minutes, and the last model request hung. About 173 Pro requests; an estimated $7 to $10. + +Fixed after: + +- `intent.business_outcomes` is essential. With someone present it is asked for; a pre-approved start must state them or leave them to the agent with `decide`. Case 02 now does the latter. +- The three-layer rule applies only when there is an outcome to tie Measures to. +- The evaluation harness stops a case at one and a half times its time limit and records it as stopped. + +Not fixed: the engine lets a pre-approved Specialist end without submitting again and again, each time answered automatically, until the Orchestrator's turn limit. A guard there would have stopped case 02 within minutes. + +## Case 02 rerun, and findings kept to the solution, 2026-10-14 + +Case 02 passed with its business outcomes left to the agent: 31.6 minutes, $2.71, two Revisions. The agent assumed three outcomes from the request (the landing zone ready within 10 weeks, continuous DORA and PCI DSS compliance, high availability with low-latency hybrid connectivity) and tied six Measures to them, all three developer layers on the first. + +It also filed a gap in its own model as a finding: service continuity staffed only in business hours against a 4-hour recovery target. That drafted a change request to the architecture agent for a fault that was the operating model's. Fixed: + +- **A finding names the Items of the solution it is about.** One with none is refused. +- **The validator judges each finding.** A Check waiting on a finding the operating model could meet itself (staffing, hours, a Role, a process, a tool) is given `fail`, which sends the model back to the architect. Only a Check left `not_verifiable` is recorded as waiting on its finding. +- **The architect is told** what a finding is and is not. + +177 tests pass. Not yet run with real models. diff --git a/.agents/plans/iter-02-platform-deployment/MAP.md b/.agents/plans/iter-02-platform-deployment/MAP.md new file mode 100644 index 0000000..ef34446 --- /dev/null +++ b/.agents/plans/iter-02-platform-deployment/MAP.md @@ -0,0 +1,31 @@ +# Plan Map + +**Iteration:** 2, on the platform +**Intent:** The operating model agent runs as a container beside the local KAB platform, is registered in its catalogue, and completes an Execution started by a platform workflow. The Orchestration engine (F1, F2, in `../orchestration_engine`) is then put in front of it. +**Status:** steps 1 to 3 built 2026-10-20; step 4 in progress + +## Decided by the owner, 2026-10-20 + +- **E7, locally:** the image is built with a named build context, `--build-context engine=../orchestration_engine`, and installs the Execution engine from it. No file is copied by hand. CI will use a dependency on the engine's repository at a pinned commit. +- **Order:** this agent first (steps 1 to 4), then the Orchestration engine (step 5). +- **Models:** Gemini on Vertex AI, as in the evaluation, with the developer's application default credentials mounted read-only. The platform's LiteLLM is not used. + +## Sources of truth + +- `IMPLEMENTATION_PLAN.md`, sections 2 and 10 (E7, F1, F2). +- `../modernization-platform/packages/agent-builder-local-dev-kit/README.md`: the local platform, `agents-compose.yml` and `scripts/compose.sh agents`. +- `../modernization-platform/packages/agent-builder-temporal-workers/src/temporal/a2a_client.py`: how a platform workflow calls an agent. + +## Known mismatch + +A platform workflow step treats an A2A `input_required` as the agent runtime's action review (a list of tool calls to approve, at most `MAX_ACTION_REVIEWS` rounds). The engine's questions and Approvals are not that, so a platform workflow calls this agent pre-approved. Interactive Executions go through the Orchestration engine, which relays every prompt (F2). Step 4 confirms this rather than assuming it. + +## Status + +| Step | What | Done when | Status | +| --- | --- | --- | --- | +| 1 | The image (E7): `Dockerfile`, `.dockerignore`, the engine from the named build context | The image builds from a clean checkout of both repositories; `/health` and the agent card answer | built (`steps/step-1.md`) | +| 2 | A real-model run through the container | One pre-approved Execution of case 01 completes through the container's A2A endpoint | built (`steps/step-2.md`) | +| 3 | On the platform's network: an entry in the dev kit's `agents-compose.personal.yml`, registered in the catalogue | It is listed in the catalogue; the API container reaches `http://tom-analysis:8080/health` | built (`steps/step-3.md`) | +| 4 | A platform workflow with one step calling this agent | The workflow completes in Temporal with the Deliverables in its output | in progress: the engine's card declares KAB's input contract (`request`, `customer_case`, `scope`, `pre_approved`), chosen by the owner; waiting on the catalogue refresh and the workflow | +| 5 | The Orchestration engine in front (F1, F2 in `../orchestration_engine`) | An Execution of either agent runs through it, Approvals relayed | F2 built (`../../../orchestration_engine/.agents/plans/iter-02-orchestration-engine/steps/step-2.md`): this agent's questions relayed through it on the platform; F1 awaits the owner | diff --git a/.agents/plans/iter-02-platform-deployment/steps/step-1.md b/.agents/plans/iter-02-platform-deployment/steps/step-1.md new file mode 100644 index 0000000..5d37a57 --- /dev/null +++ b/.agents/plans/iter-02-platform-deployment/steps/step-1.md @@ -0,0 +1,29 @@ +# Step 1: the image (E7) + +Built 2026-10-20. + +## What was built + +- **`Dockerfile`.** Two stages. The builder installs the Execution engine from a named build context, `engine`, copying only its `pyproject.toml` and `execution_engine/`. The runtime stage holds `app/`, `agents/`, `providers/` and `skills/` in `/app` and runs `python -m app.main` as a non-root user (10001) on port 8080. The agent is not installed: `ROOT` in `workflow.py` is found from `__file__`, so it must run from its own folder, as the reference agent does. +- **Defaults for a container:** `CHECKPOINTER=sqlite` with its files in the `/data` volume, and `ENV_FILE_EXPORT=false`. +- **`.dockerignore`:** keeps `.venv`, `.cache`, `.env`, `eval/results`, `deliverables`, tests and plans out of the image. + +```bash +podman build --build-context engine=../orchestration_engine -t localhost/agents/tom-analysis:local . +``` + +## What differed from the plan + +- **Credentials and the non-root user.** The application default credentials file is mode 600, so the image's user cannot read it under rootless podman. The container is run with `--userns=keep-id:uid=10001,gid=10001`, which maps the developer's user onto the image's. The file's mode is not changed. +- **Podman ignores `HEALTHCHECK`** in an OCI-format build. The compose entry declares its own (step 3). + +## Verified + +- Neither repository has a commit yet, so "a clean checkout" was taken as the files git would commit (`git ls-files --cached --others --exclude-standard`) of both, copied to an empty folder. The image built from that, and the Workflow loads in it with its five Phases. +- The container answers `/health` and both agent card paths; the card advertises `A2A_PUBLIC_URL`. From inside it, the credentials are readable and a Gemini request on Vertex AI answers. +- An A2A `message/send` with `pre_approved`, `customer_case` and `scope` in the message metadata reached the engine, which checked the spec. A Customer case that differs from the spec's is refused by name. +- 177 tests pass; `ruff` and `mypy` are clean. No Python changed. + +## Not done + +- CI's means, a dependency on the engine's repository at a pinned commit, needs the engine's repository to have a commit to pin. diff --git a/.agents/plans/iter-02-platform-deployment/steps/step-2.md b/.agents/plans/iter-02-platform-deployment/steps/step-2.md new file mode 100644 index 0000000..5d76f37 --- /dev/null +++ b/.agents/plans/iter-02-platform-deployment/steps/step-2.md @@ -0,0 +1,20 @@ +# Step 2: a real-model run through the container + +Run 2026-10-20. + +## What was run + +Evaluation case 01's written spec (`eval/datasets/specs/landing-zone-shared.yaml`), sent to the container from step 1 as one A2A `message/send` with `configuration.blocking: false`, then polled with `tasks/get`. The message metadata carried `pre_approved: true`, `customer_case: bench-landing-zone-shared` and the spec as `scope`. Models were Gemini on Vertex AI with the developer's application default credentials, mounted read-only. + +## Result + +| Minutes | State | Capabilities | People | Scenarios | +| --- | --- | --- | --- | --- | +| 31.6 | completed | 20 | 5.1 | 13 of 13 pass | + +The task's artifacts were all eleven files: the guide, `scope.yaml` and `scope.json`, operational requirements, capabilities, the Target operating model and its organisation chart, staffing, findings, verdicts and scenarios. + +## What differed from the plan + +- **The Customer case must be the spec's.** The first attempt named another slug and was refused at once: "The scope spec is for bench-landing-zone-shared, and this was started for smoke-01." A caller that sends a spec sends its Customer case with it. +- **Postgres.** The engine tries `localhost:5432`, logs a warning and falls back to SQLite in `/data`. Harmless here; a shared deployment sets `DATABASE_URL`. diff --git a/.agents/plans/iter-02-platform-deployment/steps/step-3.md b/.agents/plans/iter-02-platform-deployment/steps/step-3.md new file mode 100644 index 0000000..04f4f56 --- /dev/null +++ b/.agents/plans/iter-02-platform-deployment/steps/step-3.md @@ -0,0 +1,28 @@ +# Step 3: on the platform's network + +Built 2026-10-20. + +## What was built + +In the dev kit (`../modernization-platform/packages/agent-builder-local-dev-kit`), two personal files, excluded from git in that clone's `.git/info/exclude`: + +- **`agents-compose.build.personal.yml`:** `tom-analysis` built from `../../../tom_analysis` with `additional_contexts: {engine: ../../../orchestration_engine}`. +- **`agents-compose.personal.yml`:** the container `tom-analysis` on `agent-builder-network`, port `${TOM_ANALYSIS_PORT:-10020}:8080`, `A2A_PUBLIC_URL=http://tom-analysis:8080`, the credentials mounted read-only, named volumes for `/data` and `/app/deliverables`, and its own healthcheck. + +```bash +./scripts/compose.sh agents build tom-analysis +./scripts/compose.sh agents up tom-analysis +``` + +The owner registered it in the catalogue from the admin page (Agents, Discover agent, `http://tom-analysis:8080`). It is listed as `target_operating_model_agent`. + +## What differed from the plan + +- **No pod.** podman-compose puts every container in one pod, and podman refuses a per-container user namespace (`keep-id`, step 1) inside a pod. The personal file sets `x-podman: in_pod: false`. Agents reach each other by name on the network, not through the pod. +- **The platform calls the address an agent was registered with,** not the A2A interface its card names (`agent_catalog_service.py` sets `endpoint.url` to the discovery URL). The engine served JSON-RPC only at `/a2a`, so every platform call would have been a 404. Chosen by the owner: the engine serves the same handler at `/` as well (`execution_engine/main.py`, with a test in `tests/toy/test_server.py`). The card still names `/a2a`. The platform's choice is worth raising with its team. + +## Verified + +- From the API and workers containers: `/health`, both card paths, and JSON-RPC at `/` and at `/a2a`. +- The catalogue record names the agent, its skill and its examples. +- The engine: 532 tests pass; `ruff` and `mypy` clean. This agent: 177 tests pass. diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..9ceed6b --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.git +.venv +.cache +.env +*.env +*.db +**/__pycache__ +deliverables +lessons/candidates +eval/results +tests +.scratch +.agents diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..66713d0 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# Environment variables +.env +*.env + +# Local SQLite databases +*.db + +# Python bytecode and tool caches (pytest, mypy and ruff write to .cache/) +__pycache__/ +*.pyc +*.pyo +.cache/ + +# Virtual environments +.venv/ +venv/ + +# Everything generated for a Customer case: operating specs and operating models +# name people, teams and suppliers +deliverables/ + +# Evaluation run outputs +eval/results/ + +# Lesson candidates captured from Executions (may quote user feedback) +lessons/candidates/ + +*.log +.DS_Store diff --git a/.scratch/target-operating-model/spec.md b/.scratch/target-operating-model/spec.md new file mode 100644 index 0000000..317d4c3 --- /dev/null +++ b/.scratch/target-operating-model/spec.md @@ -0,0 +1,144 @@ +# The target operating model agent + +Status: needs-triage + +The design for the first slice. Vocabulary is in `CONTEXT.md`. The decisions the owner has taken, the Phases and the order of work are in `IMPLEMENTATION_PLAN.md`; this spec says what the plan leaves loose: the formats, the rules and the arithmetic. The engine it runs on is specified in `../orchestration_engine/.scratch/engine/spec.md`. + +## The first slice + +Six Deliverables, enough to go from Sources to a priced, validated operating model. + +| Phase | Deliverable | Specialist | Data file | +| --- | --- | --- | --- | +| scoping | `operating_scope` | `scope-analyst` | `operating-spec.yaml`, `scope.json`, `solution.json` | +| operational_requirements | `operational_requirements` | `operations-analyst` | `operational-requirements.json` | +| operating_model_design | `capability_model` | `capability-modeller` | `capabilities.json` | +| operating_model_design | `target_operating_model` | `operating-model-architect` | `target-operating-model.json`, `findings.json` | +| validation | `validation_results` | `validator` | `verdicts.json`, `scenarios.json` | +| packaging | `operating_model_guide` | `packager` | | + +Each depends on the one above it. `capability_model` and `target_operating_model` share a Phase, so the customer approves the design once, from the graph. + +## The Operating spec + +Read against `skills/operating-scoping/assets/catalogue.yaml` by the engine's spec reader. Every field holds a value, `ask`, `decide` or `not_applicable`. + +| Field | Type | Tier | Notes | +| --- | --- | --- | --- | +| `metadata.customer_case` | string | essential | | +| `engagement.kind` | enum | essential | `workload`, `platform_foundation`, `migration` | +| `intent.statement` | text | essential | The customer's words | +| `intent.requirements` | records `id`, `statement` | optional | Binds: each must reach an Operational requirement | +| `intent.exclusions` | list | optional | | +| `solution.items` | records `id`, `name`, `kind`, `product`, `environments`, `source_ref` | essential | At least one | +| `solution.conditions` | records `id`, `on`, `condition`, `source_ref` | important | `on` names an Item | +| `solution.build.duration_days` | integer | important | The transition plan is checked against it | +| `solution.build.customer_owned` | list | optional | | +| `solution.inherited_risks` | records `on`, `risk`, `source_ref` | optional | | +| `solution.cloud_provider` | enum | essential | Values from the provider bundle | +| `operations.support_hours` | enum | essential | `business`, `extended`, `24x7`. Binds | +| `operations.availability_hours` | enum | important | The hours the solution must be available | +| `operations.change_process` | text | important | | +| `operations.monitoring` | text | optional | | +| `sourcing.model` | enum | essential | `customer_operated`, `provider_operated`, `shared` | +| `sourcing.delivery_countries` | list | important | ISO country codes. Binds where Roles may sit | +| `organisation.existing_teams` | records `name`, `party`, `skills`, `hours` | important for `migration`, else optional | | +| `organisation.headcount_limit` | integer | optional | | +| `service_management.tooling` | text | important | | +| `service_levels.priority_1_response_minutes` | integer | important | | +| `service_levels.recovery_time_hours` | number | important | | +| `compliance.frameworks` | list | important | Selects Capabilities from the catalogue | +| `commercial.currency` | enum | essential | Values are the currencies the rate card can convert to | +| `commercial.monthly_run_budget` | number | optional | In `commercial.currency` | +| `transition.go_live_date` | date | optional | | +| `run.pre_approved` | boolean | optional | | + +`solution.items` and `solution.conditions` are written to `solution.json` and reach only the Specialists that list it. Everything else is in `scope.json`, which every Specialist is given. + +### Digesting an architecture Execution + +A code adapter, `app/operating_model/digest/architecture.py`. No model. + +| From | To | +| --- | --- | +| `target-architecture.json` Elements | `solution.items`: `id` is `S-n` in order, `source_ref` the Element's id, `name`, `product` and the component as `kind` | +| `target-architecture.json` Checks | `solution.conditions`: `id` is `K-n`, `on` the Item of the Check's Element, `source_ref` the Check's id | +| A Check on a Relationship | A condition `on` the Item of the Relationship's source | +| `scope.json` | Every field both catalogues share, as stated; `solution.cloud_provider` from `estate.cloud_provider` | +| `requirements.json` | Nothing directly. Kept as a Source so a Specialist can read what a condition's requirement said | +| `verdicts.json`, verdict `fail` or `not_verifiable` | `solution.inherited_risks` | +| `work-breakdown.json` | `solution.build.duration_days` from its computed schedule; activities owned by the customer to `customer_owned` | + +Rules: every Element yields exactly one Item and every Check exactly one Condition, or the digest fails and names what it could not place. `target-architecture.json` is required; each other file that is absent leaves its fields as `ask`. A value digested has provenance `digested`, with the file and its hash. + +## Data files + +`operational-requirements.json`: a list. + +```json +{"id": "OR-1", "statement": "A fault on the hub network is responded to at any hour.", + "kind": "service_level", "source": ["K-4", "operations.support_hours"]} +``` + +`source` names a stated requirement's id, a Condition's id or a spec field. Refused: an entry with no source; a source that does not exist; a stated requirement, or a spec field marked as binding with a stated value, that no entry names. + +`capabilities.json`: a list. + +```json +{"id": "C-1", "catalogue": "incident-management", "serves": ["OR-1"], "items": ["S-1", "S-4"], + "hours": "24x7", "effort_hours_per_month": 30, "basis": "Two priority 1 incidents a month at eight hours, plus triage."} +``` + +Refused: a `catalogue` id that is not in the catalogue, unless `catalogue` is `custom` with a `reason`; a Capability that serves nothing; an Item no Capability covers; an Operational requirement no Capability serves; `effort_hours_per_month` missing or not positive. + +`target-operating-model.json`: Parties, Roles, Capabilities as above by reference, Responsibilities and Checks, as in the plan's section 7. A Role carries `country` and `level`. + +`findings.json`: a list, as in the plan's section 7. May be empty. + +`verdicts.json`: one entry per Check, `pass`, `fail` or `not_verifiable`, with evidence. `scenarios.json`: each Scenario and the path code traced for it. + +## Staffing and cost + +Computed when the Target operating model is submitted, from the graph and `skills/rate-card/assets/rates.yaml`. + +For a Role *r* in country *c* at level *l*, covering hours *h*: + +- **effort** = the sum of `effort_hours_per_month` of every Capability *r* is responsible for. Where several Roles are responsible for one Capability, its effort is divided equally among them unless the Responsibility states a `share`. +- **productive hours a month** = `productive_days[c]` × `hours_per_day[c]` ÷ 12. +- **people** = effort ÷ productive hours a month; and, where *h* is wider than business hours, never fewer than `cover[h].minimum_roster`. Shown to one decimal place. +- **cost a month**, only where the Role's Party is of a kind the card prices and is not the customer = people × `day_rate[c][l]` × `productive_days[c]` ÷ 12 × `cover[h].uplift` × `conversion.rates[commercial.currency]`. + +A Role's hours *h* are the widest hours of any Capability it is responsible for. + +Refused at submission: a Role with a country or level the card lacks. Refused at the start: a `commercial.currency` the card has no conversion rate for. + +Written into the Deliverable by code: people and cost per Role; totals by Party, by country and by Capability; the total against `commercial.monthly_run_budget`; the card's version, and the conversion rate and its date. + +### Digesting anything else + +A model drafts `solution.items` and `solution.conditions` from a document. Each record carries `evidence`: the file and the passage it was read from, quoted. A record without evidence is dropped by code and reported. Its provenance is `drafted`, and the scoping Approval lists drafted records apart from digested ones. + +## Validation + +Structural verdicts, by code: one per refusal rule of the graph, restated as a Check that passes, so the guide shows what was verified and not only what was refused. + +Scenarios, generated by code and traced by code: + +| Scenario | Generated for | Passes when | +| --- | --- | --- | +| A fault out of hours | Each Item whose Capabilities must cover hours wider than `business` | A Capability covers the Item, a Role responsible for it works those hours, and the Capability has an accountable Role | +| A change | Each Item | A Capability from the catalogue's change domain covers it, with a responsible Role | +| A Condition falls due | Each Condition | A Check carries it, on a Capability with a responsible Role | +| Loss of a Party | Each Party that is not the customer | Every Capability it is accountable for names a Role in another Party as consulted or responsible | +| Over budget | When a budget is stated | The total is within it, or the Deliverable names the overrun | + +The validator, a model, judges only what code cannot: whether each Check's condition is verifiable as written, and whether a Capability's stated effort is plausible for the Items it covers. A Check that depends on a Finding is `not_verifiable`. + +## Not in the first slice + +The current operating model, the service model, dimension advisors, process definitions, organisation design, tooling, the transition plan, Change requests, and the lessons loop. + +## Open + +- The first entries of the capability catalogue, and which frameworks are in the register. +- Whether effort shared between Roles defaults to an equal division, as written above, or must always be stated. diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..79cb52a --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,27 @@ +# CLAUDE.md + +## Direction + +This agent turns a description of a solution into a reviewed target operating model. It is the Execution engine (`../orchestration_engine`) and one Workflow, `app/operating_model/workflow.py`. Read `CONTEXT.md` and `docs/adr/` before designing or restructuring anything. + +Three rules are the point of the design: + +- **The solution is never changed here.** What operations needs and the solution lacks is a finding, returned to whoever owns the design (ADR-0001). +- **No agent writes a number of people or a cost.** They are computed from the rate card Skill, whose prices only the service provider sets (ADR-0003). +- **Behaviour lives in Skills.** The core names no Cloud provider and no framework. The capability catalogue is in our own words and cites the frameworks it draws on. + +Engine code is not edited here. A fault in it is fixed in `../orchestration_engine`. + +Plans and history live in `.agents/plans/`; `IMPLEMENTATION_PLAN.md` is the plan across both repositories. The design spec is `.scratch/target-operating-model/spec.md`. + +## Checks + +A step is done when all three pass: + +```bash +.venv/bin/python -m pytest -q +.venv/bin/ruff check . && .venv/bin/ruff format --check . +.venv/bin/mypy . +``` + +Set up with `pip install -e ../orchestration_engine --config-settings editable_mode=compat` and then `pip install -e '.[dev]'`. diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..7a0e6ac --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,99 @@ +# Target Operating Model + +Turns a description of a solution into a reviewed target operating model: who runs it, through which processes, with which tools, to which service levels, at what cost, and how the organisation gets there. + +Accepted by the owner on 2026-10-13. + +Terms of the engine this agent runs on (Execution, Phase, Approval, Deliverable, Workflow, Skill and the rest) are defined in `../orchestration_engine/CONTEXT.md` and mean the same here. + +## Language + +### What it starts from + +**Operating spec**: +The structured statement an Execution starts from: what the customer wants from operations, the facts an operating model depends on, and the solution to be operated, as one file. It may arrive complete or partly filled, and a value it states is the customer's decision. +_Avoid_: intent file, brief, scope spec (which is the architecture agent's) + +**Item**: +One thing to be operated, listed in the Operating spec: a network, a database, a pipeline. When the spec was digested from a solution architecture it is that architecture's Element, and keeps its id as a reference. +_Avoid_: Element (which is the architecture's), asset, component, resource + +**Condition**: +Something that must be kept true of an Item while it is run, listed in the Operating spec: logs retained for a period, recovery within a time. +_Avoid_: Check (which is this agent's own, on its operating model), control, SLA + +**Source**: +Anything a caller supplies for an Operating spec to be digested from: a solution architecture agent's files, a design document, a sentence. +_Avoid_: input, upload, baseline + +### The operating model + +**Target operating model**: +The operating organisation to be established, as one explicit graph of Parties, Roles, Capabilities, the Responsibilities between them, and the Checks each must meet. It is established before any process, organisation design or tooling is written in detail. +_Avoid_: TOM (in prose), blueprint, to-be state + +**Business outcome**: +What the customer's business wants from the solution, in their words, such as getting new products to customers sooner. Every Measure shows one. +_Avoid_: goal, KPI (which is a Measure), objective, benefit + +**Measure**: +Something measured to show a business outcome is being served, at one layer: delivery outcome, productivity guardrail, leading signal of friction, service level or cost. Taken by a Capability someone is responsible for. +_Avoid_: metric (alone), KPI, Check (which is a condition that holds or not) + +**Operational requirement**: +A numbered statement of what running the solution must achieve, traced to a requirement the customer stated, a Condition, or a field of the Operating spec. +_Avoid_: NFR, service requirement + +**Capability**: +Something the operating organisation must be able to do, such as manage incidents or recover from loss. It comes from the capability catalogue, names the Operational requirements it serves and the Items it covers. +_Avoid_: process (which is how a Capability is carried out), function, practice, service + +**Capability catalogue**: +Our own list of Capabilities, in our own words, each citing the industry frameworks it draws on. +_Avoid_: framework, library, ITIL (which is one source) + +**Party**: +An organisation with a part in operating the solution: the customer, the service provider, a supplier, the Cloud provider. +_Avoid_: team (which is inside a Party), vendor, stakeholder + +**Role**: +A position in a Party, with its skills, the hours it works, and, for pricing, a country and a level. +_Avoid_: person, resource, job title + +**Team**: +A group of Roles within one Party, of one of the four kinds Team Topologies names: stream-aligned, platform, enabling or complicated-subsystem. Optional: an operating model need not have teams. +_Avoid_: squad, unit, department, Party (which is the organisation the team is in) + +**Interaction**: +How two Teams work together, in one of Team Topologies' three modes: collaboration, X-as-a-Service or facilitating. A collaboration is a stage with a stated end. +_Avoid_: dependency, handoff, relationship (which is the architecture's) + +**Responsibility**: +A Role's part in a Capability: accountable, responsible, consulted or informed. +_Avoid_: RACI (which is a view of all of them), ownership, assignment + +**Check**: +A numbered, verifiable condition that a Capability, Role or Responsibility must meet, traced to the Operational requirement it comes from. Validation gives each a verdict. +_Avoid_: Condition (which is in the spec, about an Item), control, test + +**Finding**: +Something operating the solution needs that the solution, as the Operating spec describes it, does not provide. It is recorded and returned to whoever owns the design. This agent never repairs the design. +_Avoid_: gap, issue, defect, recommendation + +**Scenario**: +An event the operating model must be able to handle, generated from the graph and the spec and traced through it by code: an incident on a named Item out of hours, the loss of a region. +_Avoid_: test case, use case, dry run + +### Cost + +**Rate card**: +The service provider's prices: a day rate by country and level, with the facts needed to turn effort into people and people into cost. It is a Skill, and only the service provider sets what is in it. +_Avoid_: price list, overlay, the customer's rates + +**Staffing**: +The number of people each Role needs, computed from the effort of its Capabilities and the hours it must cover. Never written by an agent. +_Avoid_: headcount (which is a limit the customer may state), FTE count, resourcing + +**Transition plan**: +The Deliverable that divides establishing the Target operating model into work packages and activities, with a computed timeline that fits the solution's build. +_Avoid_: roadmap, project plan diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..d52ba8b --- /dev/null +++ b/Dockerfile @@ -0,0 +1,52 @@ +# The target operating model agent: the Execution engine and this Workflow. +# +# The engine comes from a named build context, so no file is copied by hand: +# +# podman build --build-context engine=../orchestration_engine -t tom-analysis . +# +# The agent itself is not installed: the Workflow finds its Skills, role files +# and provider bundles beside its own code (`ROOT` in workflow.py), so it runs +# from /app as the reference agent does. + +FROM python:3.12-slim AS builder + +WORKDIR /engine +COPY --from=engine pyproject.toml ./ +COPY --from=engine execution_engine/ ./execution_engine/ +RUN pip install --no-cache-dir --prefix=/install . + + +FROM python:3.12-slim + +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 \ + PYTHONPATH=/app \ + A2A_HOST=0.0.0.0 \ + A2A_PORT=8080 \ + CHECKPOINTER=sqlite \ + CHECKPOINT_DB_PATH=/data/checkpoints.db \ + SQLITE_FALLBACK_DB=/data/agent_local.db \ + ENV_FILE_EXPORT=false + +WORKDIR /app + +RUN groupadd -g 10001 appgroup && \ + useradd -u 10001 -g appgroup -s /bin/sh -m appuser + +COPY --from=builder /install /usr/local + +COPY app/ ./app/ +COPY agents/ ./agents/ +COPY providers/ ./providers/ +COPY skills/ ./skills/ + +RUN mkdir -p deliverables lessons /data && chown -R appuser:appgroup /app /data + +USER appuser +VOLUME ["/data"] +EXPOSE 8080 + +HEALTHCHECK --interval=30s --timeout=5s --start-period=60s --retries=3 \ + CMD python -c "import os, urllib.request; urllib.request.urlopen(f'http://localhost:{os.environ[\"A2A_PORT\"]}/health')" || exit 1 + +CMD ["python", "-m", "app.main"] diff --git a/IMPLEMENTATION_PLAN.md b/IMPLEMENTATION_PLAN.md new file mode 100644 index 0000000..cd9e4dc --- /dev/null +++ b/IMPLEMENTATION_PLAN.md @@ -0,0 +1,393 @@ +# Orchestration engine and target operating model agent: implementation plan + +Status: seventh draft, 2026-10-13. Building has started: E1 to E5 are built, and the first slice of the operating model agent but for its digest and its evaluation. Each repository's `.agents/plans/` records what is done. + +Two things are built, in two folders, and a third is left exactly as it is. + +| Folder | What it is | Changed by this plan | +| --- | --- | --- | +| `gcp_solution_architecture_agent/` | The solution architecture agent, running on its own built-in engine. The reference. | No file is modified | +| `orchestration_engine/` | Two things. The **Execution engine**: the installable package an agent runs on. The **Orchestration engine**: the service every request comes through | New | +| `tom_analysis/` | The target operating model agent: its workflow, roles, Skills, provider bundle and evaluation, running on the Execution engine | New | + +Terms in capitals without a definition here are the architecture agent's, from its `CONTEXT.md`. "The reference" means `gcp_solution_architecture_agent/` at commit `8b397cd`. + +## 1. Decided + +By the owner on 2026-10-13: + +1. **The operating model agent is a separate agent, in `tom_analysis/`.** +2. **The Execution engine is built in `orchestration_engine/`.** The reference is read, never modified. +3. **The Execution engine is shown to be able to run the architecture workflow,** as an example in its own tests. +4. **The operating model agent starts from an operating spec.** Whatever it is given, the architecture agent's files among other things, is digested into that file first. The architecture's files are one source, not the only one. +5. **The operating spec holds the inventory inline.** It does not point at source files. +6. **Every request comes through the Orchestration engine,** which decides, with Jev, whether the work is solution architecture, an operating model, or both. The KAB platform does not route between agents, so nothing existing is duplicated. +7. **A gap in the architecture is never repaired by the operating model agent.** It is recorded as a finding and goes back as a Change request. +8. **Our own capability catalogue,** in our words, each entry citing the industry frameworks it draws on. + +9. **The run is priced.** Staffing is costed from a rate card by country, in the currency the operating spec states. The rate card is a Skill, and its prices are set by the service provider who runs this agent: the owner. The customer's own staff are counted and never priced (section 9). +10. **How an agent's container build obtains the Execution engine is in scope** (step E7). Which means is used is still to be chosen. + +Still open: section 11. + +## 2. How the pieces run + +```text + caller + │ A2A + ┌────────▼────────┐ + │ orchestration │ orchestration_engine: reads the request, asks Jev + │ engine │ which work it is, dispatches, relays every prompt + └───┬─────────┬───┘ + A2A │ │ A2A + ┌──────────▼───┐ ┌─▼──────────────────┐ + │ architecture │ │ operating model │ + │ agent │ │ agent │ + │ (as it is) │ │ (tom_analysis, on │ + │ │ │ the Execution engine) │ + └──────────────┘ └────────────────────┘ +``` + +Three containers. Each agent still has its own Orchestrator for its own Execution; the Orchestration engine sits above Executions and produces nothing itself. The architecture agent is called through the `/a2a` and `/generate` routes it already has. + +Two names are kept apart throughout. The **Orchestration engine** is the service above Executions. The **Execution engine** is the package that holds one Execution; the reference already calls its own `ExecutionEngine`. The **Orchestrator** remains the top-level agent inside one Execution. + +## 3. The Execution engine + +### What is brought across + +The reference's `app/` holds the Execution engine and the solution-architecture workflow together. + +| | Modules in the reference | +| --- | --- | +| Engine, brought as it is (done, E2) | `config.py`, `checkpointing.py`, `customer_case.py`, `database.py`, `scope.py`, `baseline.py`, `providers.py`, `adk/runtime.py`, `adk/adk_runtime.py`, `skills/`, `states/execution_state.py`, `tools/jev.py`, `tools/guardrail.py`, `tools/mcp.py`, `tools/workspace.py`, `tools/skill_routing.py`, `learning/proposals.py`, `learning/proof.py`, `learning/skill.py` | +| Engine, rewritten as it is brought (E3): names architecture Deliverables, or imports a module that does | `nodes/execution_nodes.py`, `nodes/router.py`, `nodes/intake.py`, `tools/execution.py`, `tools/toolkit.py`, `tools/scoping.py`, `tools/meaning.py`, `states/ledger.py`, `states/lessons.py`, `recorder.py`, `adk/roles.py`, `adk/fake_runtime.py`, `workflows/execution_graph.py`, `learning/curator.py`, `learning/evidence.py`, `agent.py`, `a2a_server.py`, `card.py`, `main.py` | +| Split | `workflows/work_breakdown.py`: the scheduler comes, the rule about Elements stays behind | +| Not brought: the solution-architecture workflow | `workflows/deliverables.py` (its graph), `workflows/chain.py`, `workflows/target_architecture.py`, `workflows/changes.py`, `tools/pillars.py`, `tools/local.py`, `tools/modules.py`, the reviews, `tools/assessment.py`, `presentation.py` | + +Also brought, because they are not about architecture: the Specialist protocol partial, the role files for the scope analyst, the reply writer and the lesson curator, and the evaluation harness. + +### The seam + +The second row refers to named Deliverables about 45 times. Each becomes something a **Workflow** supplies: one value, given to the Execution engine when it is assembled. + +| In the reference | What is hard-coded | What a Workflow supplies | +| --- | --- | --- | +| `workflows/deliverables.py` | Phases, the Deliverable graph, data checks, sections for readers | The same, as data | +| `router.py` 161, 313–344, 631–652 | Which Deliverable is validation, how its verdicts are read, what a failure reopens | A validation policy | +| `router.py` 747–826 | Facts for an Approval summary | A function per Phase | +| `tools/execution.py` 320–413 | What code adds on submission: tables, diagram, rendered plan | A submission hook per Deliverable | +| `execution_nodes.py` 1016, `toolkit.py` 81 | Pillar advisors under design recommendations | A fan-out declared on a Deliverable | +| `execution_nodes.py` 1145, 1376–1449, `a2a_server.py` 178 | The final document; manifest entries | The final Deliverable and its manifest entries | +| `adk/roles.py`, `agent.py` | Required roles; the Terraform runner and the reviews | Roles, local tools and reviews, listed | +| `presentation.py`, `card.py` | Response fields and the card's wording | Supplied | +| `scope.py`, `tools/scoping.py` | One catalogue; a spec drafted from a sentence | The catalogue, and the digest adapters of section 5 | + +### Two proofs, in the Execution engine's own tests + +- **A toy workflow** of three Deliverables in two Phases, run on the fake runtime through questions, Approvals, a Revision and a Change request. It is what the brought-across tests for the Ledger, the router, intake and the A2A server are driven by. +- **The architecture workflow as an example.** Its Deliverable graph, data checks and hooks are written as a Workflow under the Execution engine's tests, from the reference, and the reference's tests for them are brought with it. All of it runs on the fake runtime. + +The second shows the seam is wide enough for the real case. It does not show the same results with real models: that would need the architecture agent's Skills, roles and bundle on the new engine, which is moving that agent, and is not in this plan. + +### What two Execution engines means + +The reference keeps its built-in engine and has open steps in four iterations. Fixes do not flow between the two unless someone carries them. The Execution engine records the reference commit it was built from, so a later comparison has a starting point. + +## 4. The Orchestration engine + +### Deciding what a request is + +In order, following ADR-0007: readings, disposition, playbook, record. + +1. **Read by code.** What the caller named settles it: `work` set outright; an operating spec or `sources` supplied (operating model); a scope spec supplied (solution architecture); a `baseline` named (the agent that Execution belongs to). +2. **Read by Jev,** when code cannot tell. One typed choice about the message: solution architecture, operating model, both, a question about delivered work, or none of these. Jev is sent the message and the names of what was supplied, never file contents. +3. **Disposition.** A choice below the confidence threshold is not relied on. With someone present, the Orchestration engine asks them which they want, in plain terms. With nobody present, it starts nothing and says how to name the work. It does not guess: a wrong guess here starts an Execution of the wrong kind. +4. **Jev unreachable.** Step 1's rules, then step 3. Never a stop. +5. **Record.** The decision, the confidence and what was dispatched, per Customer case. + +### Dispatching + +| Decision | What happens | +| --- | --- | +| Solution architecture | The message and its metadata go to the architecture agent unchanged | +| Operating model | They go to the operating model agent | +| Both | Architecture first. When it completes, the files it returned are passed to the operating model agent as `sources`, with the Orchestration engine's own record that the Execution completed. An architecture Execution that ends with warnings stops here and the user is asked | +| A question | To the agent that delivered the work it is about | +| None | A reply saying what can be done. Nothing is started | + +- **Relaying.** Each agent pauses for questions and Approvals. The Orchestration engine passes every prompt to the caller and every reply back, unchanged, and holds which downstream task is waiting. +- **"Both" is in sequence, not in parallel.** The operating model depends on the architecture. The user sees the digest of the architecture at the operating model's scoping Approval. +- **One Customer case across both agents.** The same slug is sent to each. +- **Two readings of one message.** The architecture agent's own intake still reads a message as new, a change or a question. The Orchestration engine does not replace that; it only chooses the agent. + +## 5. The operating spec + +The operating model agent's only input. The counterpart of the scope spec (ADR-0008), with the same rules: a structured file read against a catalogue in code; complete or partly filled; every field a value, `ask`, `decide` or `not_applicable`; the file supplied never altered; a completed copy written beside it with where each value came from; approved in scoping before anything is derived from it. + +It differs in one way: it carries the system to be operated. + +```yaml +metadata: {customer_case: acme-landing-zone} +engagement: {kind: platform_foundation} +intent: + statement: | + We need the landing zone run as a managed service from go-live ... + requirements: + - {id: REQ-01, statement: Platform foundation ready within 10 weeks.} +solution: + items: # what is to be operated + - {id: S-1, name: Hub network, kind: network, product: Shared VPC, environments: [prod], source_ref: E-1} + conditions: # what must be kept true + - {id: K-1, 'on': S-1, condition: Flow logs are retained for 400 days., source_ref: CHK-12} + build: {duration_days: 46, customer_owned: [Order the carrier circuit]} + inherited_risks: + - {'on': S-4, risk: Recovery was not verified in validation., source_ref: CHK-31} +operations: {support_hours: 24x7, change_process: ask} +sourcing: {model: decide, delivery_countries: [GB, PL, IN]} +organisation: {existing_teams: ask, headcount_limit: ask} +commercial: {currency: GBP, monthly_run_budget: ask} +service_management: {tooling: ask} +service_levels: {} +transition: {go_live_date: ask} +run: {pre_approved: false} +``` + +### Digesting sources into it + +| Source | Read by | What is guaranteed | +| --- | --- | --- | +| An architecture Execution: a folder, files in the request, or forwarded A2A artefacts | A code adapter in `tom_analysis/`. It reads plain JSON for the fields it needs; nothing is copied from the reference | Every Element becomes an item and every Check a condition, with its id as `source_ref`. Counts in equal counts out, tested | +| Anything else: a design document, a spreadsheet, a diagram's description | A model, drafting records | Each record quotes the passage it came from. Nothing is guaranteed complete, which is why the customer approves it | +| A sentence, or nothing | The scope analyst, in conversation | As the architecture agent does today | +| An operating spec already written | Read as it is | | + +- **Provenance per value:** supplied, digested (with the source file and its hash), answered, assumed, unknown. +- **The digest is a command and a step.** `python -m app.digest > operating-spec.yaml` runs nothing else. Started with `sources`, the agent runs the same code, and the Execution starts from the spec it wrote. +- **The mechanism is the Execution engine's; the adapters are the agent's.** The Execution engine already drafts a spec from a sentence; a Workflow lists the adapters for the formats it knows. +- **A format change in the architecture agent** is caught by a contract test against one captured Execution, since no version can be added to its files. +- **The inventory is not sent to every Specialist.** The reference puts the scope in every prompt. `solution.items` and `solution.conditions` travel as a data file to the Specialists that use them. + +## 6. The capability catalogue + +A Skill, `operating-capabilities`, holding `assets/catalogue.yaml`. + +```yaml +- id: incident-management + name: Incident management + domain: service_operations + purpose: Restore a service that is degraded or down, and say what happened. + applies_when: {always: true} + typical_checks: + - A priority 1 incident is acknowledged within an agreed time, at any hour the service is offered. + sources: + - {framework: ITIL 4, reference: "Incident management practice"} + - {framework: Google SRE, reference: "Site Reliability Engineering, ch. 14, Managing Incidents"} + - {framework: ISO/IEC 20000-1:2018, reference: "8.6.1"} + - {framework: "Regulation (EU) 2022/2554 (DORA)", reference: "Art. 17"} +``` + +- **Sources are cited, never copied.** `references/sources.md` is the register: each framework, its version, publisher and licence, and how it is cited. Candidates: ITIL 4, ISO/IEC 20000-1, COBIT 2019, Google's SRE books, the FinOps Framework, NIST CSF 2.0, ISO/IEC 27001, SFIA for skills, the EU's DORA regulation, and each Cloud provider's architecture framework. The references in the example are illustrative; each is verified against its source when the entry is written. +- **Checked in code.** Every entry has a source that is in the register. Every Capability in a Target operating model names a catalogue entry, or is marked `custom` with its reason. +- **`applies_when` is read by code** against the operating spec: an item's kind, the engagement kind, a compliance framework. It proposes; the Specialist decides, and says why an applicable capability was left out. +- **No Cloud provider and no organisation.** The provider bundle maps capabilities to that provider's tooling. An organisation's role names, standards and service tiers are an overlay Skill. +- **Two things are called DORA:** the EU regulation and the DevOps research programme's metrics. The register names each in full. + +## 7. The Target operating model graph, and findings + +One graph, held as data, approved before detail is written. Vocabulary for `tom_analysis/CONTEXT.md`: + +- **Operating spec**, **Item**, **Condition**: section 5. +- **Operational requirement**: a numbered statement of what running the solution must achieve, traced to a stated requirement, a Condition or a spec field. +- **Capability**: something the operating organisation must be able to do, from the catalogue, naming the Operational requirements it serves and the Items it covers. +- **Party**, **Role**, **Responsibility**: an organisation with a part in operating the solution; a position in it, with skills and hours; a Role's part in a Capability (accountable, responsible, consulted, informed). +- **Check**: a numbered verifiable condition on a Capability, Role or Responsibility, traced to its source. + +```json +{ + "parties": [{"id": "P-1", "name": "Customer platform team", "kind": "customer"}], + "roles": [{"id": "R-1", "party": "P-1", "name": "Platform engineer", "hours": "business", "skills": ["networking"]}], + "capabilities": [{"id": "C-1", "catalogue": "incident-management", "serves": ["OR-3"], + "items": ["S-1", "S-4"], "hours": "24x7", "effort_hours_per_month": 30}], + "responsibilities": [{"id": "RS-1", "capability": "C-1", "role": "R-1", "kind": "accountable"}], + "checks": [{"id": "OC-1", "on": "C-1", "condition": "A priority 1 incident is acknowledged within 15 minutes", "source": "OR-3"}] +} +``` + +Refused on submission, in code, against the approved operating spec: + +- an Item that no Capability covers, or a Condition that no Check carries; +- an Operational requirement that no Capability or Check serves; +- a Capability without exactly one accountable Role, or with none responsible; +- a Role without a Party, or a reference to an id that is not in the graph or the spec; +- a Capability whose hours are wider than the hours of every Role responsible for it; +- support hours narrower than the availability the spec states; +- a stated `operations.*` choice that no Operational requirement names. + +Written by code, never by a model: the responsibility matrix, the organisation chart, the capability map, and staffing and cost per Role (section 9). + +### Findings + +- A **finding** is something operating the solution needs that the solution, as the spec describes it, does not provide: nothing raises alerts, no Condition requires backups to be restored, the responding team has no means of access. +- A Specialist records it. It never adds an Item, alters a Condition, or designs around the gap in silence. + +```json +{"id": "AF-1", "about": ["S-3"], "source_ref": ["E-3"], + "lacks": "No Condition requires the database's backups to be restored on a schedule.", + "needed_by": ["OR-6", "C-4"], "proposed": "Require a restore to be tested quarterly."} +``` + +- What depends on a finding is `not_verifiable` in validation, with the finding as the reason. The Execution still completes, with its findings listed at the Approval and in the guide. +- **When the spec was digested from an architecture Execution,** code writes `architecture-change-request.md` from the findings, in the form the architecture agent accepts as a Change request against that Execution. In a "both" request the Orchestration engine can offer to send it. +- **Otherwise** the findings are questions for whoever owns the design. +- When changed architecture files arrive, the agent runs a Change request of its own from a new digest. Findings that are now met close. + +## 8. Phases and Deliverables + +| Phase | Deliverable | Specialist | Depends on | Data file | +| --- | --- | --- | --- | --- | +| Scoping | operating scope | `scope-analyst` (Execution engine) | the operating spec | `operating-spec.yaml`, `scope.json`, `solution.json` | +| Operational requirements | operational requirements | `operations-analyst` | operating scope | `operational-requirements.json` | +| | current operating model | `current-state-assessor` | operating scope | `current-state.json` | +| Operating model design | service model | `service-designer` | operational requirements | `services.json` | +| | capability model | `capability-modeller` | operational requirements | `capabilities.json` | +| | design recommendations | `design-advisor`, with one `dimension-advisor` per dimension | service model, capability model | `recommendations.json` | +| | **Target operating model** | `operating-model-architect` | the three above | `target-operating-model.json`, `findings.json` | +| Operating model definition | process definitions | `process-author` | Target operating model | `processes.json` | +| | organisation design | `organisation-designer` | Target operating model | `organisation.json` | +| | tooling and observability | `tooling-author` | Target operating model | `tooling.json` | +| Validation | validation results | `validator` | the three above | `verdicts.json`, `scenarios.json` | +| Packaging | transition plan | `transition-planner` | validation results | `transition-plan.json` | +| | operating model guide | `packager` | validation results | | + +- **Dimensions,** the counterpart of the pillars: process, people and organisation, tooling and automation, governance and risk, sourcing, security operations, financial operations. The bundle maps each to a Skill. +- **Scenario walk-throughs,** the counterpart of the dry run. Scenarios are generated from the graph and the spec: a priority 1 incident on a named Item outside business hours, a change to the network, loss of the primary region, a key rotation falling due, a month over budget. Code traces each: is it detected by a tool, is a responsible Role on hours, does a process exist, does it reach an accountable Role. A broken path is a failed Check. +- **Transition plan.** The Execution engine's scheduler, with Capabilities where the work breakdown has Elements. It is checked against the build duration the spec gives, so nothing goes live before the Capability that runs it is ready. + +## 9. Staffing, rates and cost + +The Target operating model already gives each Capability an effort in hours a month and each Role its hours. Pricing adds two facts to a Role and one file. + +- **A Role has a country and a level.** `{"id": "R-2", "party": "P-2", "name": "Platform SRE", "country": "PL", "level": "senior", "hours": "24x7"}`. +- **The rate card is a Skill, `skills/rate-card/`, and it is the actual card.** Its prices are the service provider's, set by the owner and changed only by the owner, in review like any other Skill. It is not an overlay and not the Cloud provider's: every Execution prices from it, whichever Cloud provider the solution is on. A caller cannot name another card, and an operating spec cannot state a rate: a customer does not set the provider's prices. +- **The figures in it today are placeholders** drafted with this plan. They are the owner's to replace before any priced result is shown to a customer. + +What the card holds, per country: a day rate for each level, the productive days in a year, and the working hours in a day. For the whole card: the currency its rates are written in, fixed conversion rates to other currencies with the date they were set, the smallest team that can cover each pattern of hours, and an uplift for work outside business hours. + +Computed by code, never by a model: + +1. **People per Role.** The effort of the Capabilities the Role is responsible for, divided by the productive hours of its country. Where the Role covers hours wider than a working day, never fewer than the card's minimum roster for that cover: a rota is sized by the hours it must fill, not by the work done in them. +2. **Cost per Role a month.** People, times the day rate for its country and level, times productive days over twelve, times the uplift where it applies. +3. **Conversion** into `commercial.currency`, at the card's rate for it. +4. **Totals** by Party, by country and by Capability, and against `commercial.monthly_run_budget` when the spec states one. + +Rules: + +- A Role whose country or level the card does not have is refused at submission, by name. +- **The operating spec states the currency.** `commercial.currency` is an essential field: asked for when someone is present, and a start with nobody to ask is refused without it. A currency the card has no conversion rate for is refused at the start, by name. +- **The customer's own staff are never priced.** A Role in a Party of kind `customer` is counted as people and carries no cost, whatever the card says. Of the other Parties, only the kinds the card lists are costed. +- The Specialist chooses where Roles sit, within `sourcing.delivery_countries` when the spec states them. It never writes a number of people or a cost. +- A plan over a stated budget is not refused. It is shown as over, with the Capabilities that cost most, as a plan over its delivery window is today. +- The card's version and its hash are recorded with the Execution and named in the guide, so a result can be traced to the prices in force when it was written. +- No exchange rate is looked up. Conversion uses the fixed rates in the card, and the guide states the rate and its date beside every converted total. + +Later: the same graph priced under a different mix of countries, side by side. + +## 10. Iterations + +Each repository keeps its own `.agents/plans/iter-NN-/` with `MAP.md`, `DETAIL.md` and `steps/`. Every step ends with `pytest`, `ruff` and `mypy` passing. + +### E: the Execution engine (`orchestration_engine/`) + +| Step | What | Done when | +| --- | --- | --- | +| E1 | The repository: package layout, `CONTEXT.md` with the Execution engine's vocabulary, ADR-0001 (the Execution engine takes a Workflow), ADR-0002 (built from a reference that is never modified, at a recorded commit) | The package installs | +| E2 | The modules brought as they are, with their tests | Those tests pass | +| E3 | The Workflow value and the hooks of section 3; the ten modules rewritten to use them | No engine module names a Deliverable | +| E4 | The toy workflow | It completes on the fake runtime through a question, an Approval, a Revision and a Change request | +| E5 | The architecture workflow as an example in the tests | The reference's tests for its graph, checks and hooks pass against it | +| E6 | The digest step: sources to a spec, adapters listed by a Workflow, provenance per value | The toy workflow starts from a digested spec | +| E7 | How an agent's container build obtains the Execution engine: a dependency on its repository at a pinned commit, a package registry, or a submodule | One agent image builds from a clean checkout, with no file copied by hand | + +### F: the Orchestration engine (`orchestration_engine/`) + +| Step | What | Done when | +| --- | --- | --- | +| F1 | ADR-0003: one Orchestration engine above the agents; what it decides and how | Accepted by the owner | +| F2 | The service: an A2A server, a configured list of agents, a request passed through to one agent with every prompt relayed | An architecture Execution runs through it, Approvals included, with the architecture agent untouched | +| F3 | The decision: readings in code, the Jev question, thresholds, the fallbacks, the record | A labelled set of messages is routed correctly; low confidence asks; no Jev key still routes what is named | +| F4 | "Both": architecture, then its files as `sources` to the operating model agent | One request yields both sets of Deliverables under one Customer case | +| F5 | Later messages: a question or a Change request goes to the agent that owns the work; a finding's Change request offered back | A change to the architecture is followed by a change to the operating model | + +### T: the operating model agent (`tom_analysis/`) + +First slice, six Deliverables: operating scope, operational requirements, capability model, Target operating model, validation results, operating model guide. + +| Step | What | Done when | +| --- | --- | --- | +| T1 | The repository on the Execution engine: `CONTEXT.md`, ADR-0001 (the Target operating model is an explicit graph), ADR-0002 (it starts from an operating spec), `CLAUDE.md`, `.agents/plans/` | The container starts and `/health` answers | +| T2 | The operating catalogue, the scoping Skill, `python -m app.scope check` | A spec with an unknown field is refused by name | +| T3 | The architecture adapter and `python -m app.digest`; one captured Execution as the fixture | The fixture digests to the same spec from a folder, from files in the request and from artefacts; every Element and Check is accounted for | +| T4 | The model-drafted digest for other documents | A design document yields records that each quote their source | +| T5 | The capability catalogue Skill: first entries, the sources register, the checks on both | An entry without a registered source fails a test | +| T6 | `operational-requirements.json` and `capabilities.json`, their checks, and an `operating-chain` Skill giving each file's format | A requirement with no source, and a Capability serving nothing, are refused | +| T7 | The graph: parsing, the refusals of section 7, the rendered matrix and chart | One test per refusal; the matrix is byte-identical for the same graph | +| T8 | Findings: the tool, the file, `not_verifiable` verdicts, the drafted Change request | A graph that relies on something the spec lacks is refused unless a finding names it | +| T9 | Role files, the provider bundle, the Workflow value | An Execution completes on the fake runtime | +| T10 | Validation in code: structural verdicts and scenario walk-throughs | A graph with a Role off hours fails the matching scenario | +| T11 | The guide template and packager | The guide names every Capability and finding | +| T12 | The rate card Skill: its format, the checks on it, Roles with a country and a level, and `commercial.currency` in the catalogue | A Role in a country the card lacks, and a currency it cannot convert to, are each refused by name | +| T13 | Staffing and cost in code: people per Role with the roster minimum, cost per Role, conversion, totals against the budget | The figures for a fixed graph and card are reproduced exactly; a 24x7 Role is never below the minimum roster; a customer Role has no cost | +| T14 | Evaluation: one case digested from an architecture Execution, one from a hand-written spec | Both run with real models; results recorded | + +Later slices, as before: the full design Phase (service model, current operating model, dimension advisors); definition and transition (processes, organisation, tooling, the transition plan); change and learning. + +### Order + +```text +E1 → E2 → E3 → E4 ─┬─ E5, E6, E7 + └─ T1 → T2 … T14 +F1 → F2 → F3 ───────────────────── F4 → F5 +``` + +- T starts when the toy workflow runs (E4), not when the Execution engine is finished, so the Workflow's shape is settled by a real user of it. +- F1 to F3 need only the architecture agent as it is, and can start at once. +- F4 needs the first operating model slice. + +## 11. Risks and what is still open + +- **Two Execution engines.** Section 3. Accepted with decision 2. +- **A wrong routing decision** starts work of the wrong kind. The Orchestration engine asks or declines when it is not confident; it never guesses. +- **A lossy digest.** A model reading a document can drop or invent records. Each record quotes its source, the code adapter is used wherever a format is known, and the customer approves the spec. +- **Nothing objective to validate against.** An operating model has no `terraform validate`. The structural refusals and scenario walk-throughs are the substitute, which is why they are in the first slice. +- **Generic output.** The defence is the chain: every Capability names Items and requirements of this spec, and evaluation fails a model that could have been written without reading it. +- **Client data.** Operating specs and operating models name people, teams and suppliers. They stay with the Customer case, never in a repository. +- **Placeholder prices.** Until the owner sets them, a priced operating model is an illustration. The card's `version` says `draft`, and the guide prints the version beside every total. +- **Prices in a repository.** The actual card is now in `tom_analysis/`, so whoever can read the repository can read the prices. +- **Open: the means for E7.** Recommended: each agent depends on the engine's repository at a pinned commit, with access given to the build as a secret. It needs no registry and pins exactly what was tested. The first agent image cannot be built until this is chosen. +- **Stale conversion rates.** The owner sets them with the prices; the guide prints their date. +- **Open: framework licences.** Citing is safe. Confirm each framework's terms before any wording is taken from it. + +## 12. Where it stands + +Built on 2026-10-13: + +- `orchestration_engine/`: the Execution engine takes a Workflow (E1 to E3). Two Workflows run on it in its tests: the architecture agent's own (E5), and a toy with nothing of that subject in it (E4). 521 tests pass. +- `tom_analysis/`: the operating model agent's Workflow and its folder (T1, T2, T5 to T13). From a written operating spec it produces operational requirements, a capability model from the catalogue, the Target operating model with its matrix, staffing and cost, and a validation with scenario walk-throughs. 106 tests pass, among them whole Executions on the scripted runtime. The record is `.agents/plans/iter-01-first-slice/steps/step-1.md`. + +Not yet done: + +- **The digest** (E6, then T3 and T4). An Execution starts from an operating spec that is already written; nothing reads the architecture agent's files yet. +- **Any run with a real model** (T14). No role file, Skill or catalogue entry has been read by one. +- **The Orchestration engine** (F1 to F5). +- **The container build** (E7). + +Next: + +1. E6 and T3: the digest step in the engine, and the adapter that turns an architecture Execution into an operating spec. This needs one completed architecture Execution to capture as a fixture. +2. T14: a first run with real models. +3. F1 to F3 beside them. + +Accepted by the owner on 2026-10-13: the engine's ADR-0001 and ADR-0002, and the vocabulary in both `CONTEXT.md` files. + +Waiting on the owner: this repository's three ADRs (`docs/adr/`), which record decisions already taken in conversation; the means for E7; prices in the rate card; a completed architecture Execution to use as a fixture. diff --git a/README.md b/README.md new file mode 100644 index 0000000..7975e06 --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# Target operating model agent + +Turns a description of a solution into a reviewed target operating model: who runs it, through which processes, to which service levels, with how many people and at what cost. + +It is the Execution engine (`../orchestration_engine`) and one Workflow. Vocabulary is in [CONTEXT.md](CONTEXT.md), decisions in [docs/adr/](docs/adr/), the design in [.scratch/target-operating-model/spec.md](.scratch/target-operating-model/spec.md), the plan in [IMPLEMENTATION_PLAN.md](IMPLEMENTATION_PLAN.md), and what has been built in [.agents/plans/](.agents/plans/README.md). + +## What it produces + +An Execution starts from an **operating spec** and goes through five Phases, each approved before the next begins. + +| Phase | Deliverable | What code checks or writes | +| --- | --- | --- | +| Scoping | Operating scope | The spec is read against a catalogue; nothing essential may be left unknown | +| Operational requirements | Operational requirements | Each is traced to what the customer stated; everything that binds is carried | +| Operating model design | Capability model | Each Capability is from the catalogue; every Item is covered and every requirement served | +| | Target operating model | One accountable Role for each Capability, someone responsible at its hours, every Condition carried by a Check. The responsibility matrix, organisation chart, staffing and cost are written from the graph | +| Validation | Validation results | A verdict on every Check; scenario walk-throughs generated and traced by code | +| Packaging | Operating model guide | | + +## Layout + +``` +app/operating_model/ the Workflow: names, the spec reader, the chain's checks, the + graph, staffing and cost, scenarios, and the Workflow value +app/main.py python -m app.main serves the agent over A2A +agents/ role files, one per agent +providers/gcp.yaml the first Cloud provider's bundle +skills/ operating-scoping (the spec's catalogue), operating-capabilities + (the capability catalogue and its sources), operating-chain + (the data files), rate-card (the service provider's prices) +tests/ the deterministic suite; no model and no network +``` + +## Prices + +`skills/rate-card/assets/rates.yaml` is the service provider's rate card. Its figures are placeholders while its `version` starts with `draft`, and every cost says so. diff --git a/agents/capability-modeller.md b/agents/capability-modeller.md new file mode 100644 index 0000000..fd2ac03 --- /dev/null +++ b/agents/capability-modeller.md @@ -0,0 +1,27 @@ +--- +role: capability-modeller +deliverable: capability_model +tools: [] +budget: 20 +includes: [specialist-protocol] +--- +You produce the **capability model** Deliverable: what the operating organisation must be able to do to meet the operational requirements, drawn from the capability catalogue. You say what must be done and how much work it is. You do not say who does it. + +Load the operating-capabilities Skill. `capabilities_proposed` in your context lists the catalogue's entries that code found to apply. + +Attach `capabilities.json`, a list: + +```json +{"id": "C-1", "catalogue": "incident-management", "serves": ["OR-1"], "items": ["S-1", "S-4"], + "hours": "24x7", "effort_hours_per_month": 30, + "basis": "Two serious incidents a month at eight hours each, and triage of alerts."} +``` + +- `catalogue` is an entry's id. Use `custom` only when no entry fits, with a `name` and a `reason`. +- Take every proposed entry, or say under **Capabilities** why it does not apply here. +- `serves` lists the Operational requirements the Capability exists for. `items` lists the Items it covers. Every Item must be covered by at least one Capability, and every Operational requirement served by at least one. +- `hours` is when the Capability must be staffed: `business`, `extended` or `24x7`. A Capability that responds to faults or watches for them is staffed for at least the hours `scope` says faults are responded to. +- `effort_hours_per_month` is the work the Capability takes in a steady month, across everyone who does it, for this solution. Estimate from the Items it covers and say how in `basis`. Do not pad it and do not shrink it to reach a cost: staffing and cost are computed from it. +- One Capability per catalogue entry, unless the hours or the Items differ enough that different people would do it. + +Your content has two sections. Under **Capabilities**, a table of each Capability, what it covers and why it is needed. Under **Sources**, the frameworks the chosen entries draw on, cited as the catalogue cites them: by framework and by the name of the part. Say there that the references are unverified when the Skill says they are. diff --git a/agents/operating-model-architect.md b/agents/operating-model-architect.md new file mode 100644 index 0000000..5c0934a --- /dev/null +++ b/agents/operating-model-architect.md @@ -0,0 +1,83 @@ +--- +role: operating-model-architect +deliverable: target_operating_model +tools: [] +budget: 30 +includes: [specialist-protocol] +--- +You produce the **Target operating model** Deliverable: who does what. It is one graph of Parties, Roles, Responsibilities and Checks, over the Capabilities of the capability model. The customer approves it before anything else is written from it. + +Build the graph with `graph_put`, ten to twenty items a call, and read it back with `graph_read`. It is attached as `target-operating-model.json` when you submit. + +```json +{ + "parties": [{"id": "P-1", "name": "Customer platform team", "kind": "customer"}], + "roles": [{"id": "R-1", "party": "P-2", "name": "Platform SRE", "country": "PL", + "level": "senior", "hours": "24x7"}], + "responsibilities": [{"id": "RS-1", "capability": "C-1", "role": "R-1", "kind": "responsible"}], + "checks": [{"id": "OC-1", "on": "C-1", "source": "OR-1", + "condition": "A fault on the hub network is acknowledged within 15 minutes at any hour."}] +} +``` + +Parties and Roles: + +- A Party's `kind` is `customer`, `provider`, `supplier` or `cloud_provider`. `scope.stated.sourcing.model` decides which Parties there are. +- A Role is a position, not a person. Give each a `country` and a `level` that `rate_card` lists, and the `hours` it works. Load the rate-card Skill for how to choose them. When `scope` states `sourcing.delivery_countries`, a Role may sit only in one of them. +- Covering every hour from one country needs a rota. Covering it from several countries in turn needs a Role in each. + +Teams, when the model is for a platform or names teams that exist: + +- Group Roles into `teams` in the graph, each in one Party and of one of Team Topologies' kinds: `stream_aligned`, `platform`, `enabling` or `complicated_subsystem`. Give a Role its `team`. +- Say how teams work together in `interactions`: `x_as_a_service` for what one team consumes from another through a defined interface or self-service, `facilitating` for one team helping another adopt something, and `collaboration` only for a bounded period of discovering something together. A collaboration must give `until`: when it ends, and what the teams work in after. +- A platform run as a product has a Role accountable for its roadmap. Its teams' satisfaction and adoption are measured, not assumed. + +```json +{"teams": [{"id": "T-1", "party": "P-2", "name": "Platform team", "type": "platform"}], + "interactions": [{"id": "I-1", "from": "T-2", "to": "T-1", "mode": "x_as_a_service", + "purpose": "Teams request environments through the self-service portal."}]} +``` + +Responsibilities: + +- Every Capability has exactly one `accountable` Role, and at least one `responsible` Role. +- At least one Role responsible for a Capability must work the hours the Capability is needed. +- Where several Roles are responsible for a Capability they divide its effort equally, unless every one of them states a `share` and the shares add up to 1. +- For a Capability a Party other than the customer is accountable for, name a Role in another Party as `consulted` or `responsible`: there must be someone to turn to when that Party cannot be reached. + +Measures: + +- Tie what is measured to what the business wants. For each business outcome in `scope.stated.intent.business_outcomes`, add one or more `measures` that would show it is being served, each taken by a Capability with someone responsible for it. +- A Measure's `layer` says what it tells: `outcome` (delivery speed and stability, as DORA measures it), `guardrail` (sustainable productivity across several dimensions, as SPACE does), `leading` (friction, feedback loops and cognitive load, as DevEx does), `service_level` or `cost`. Name the `framework` from the register when one applies, and a `target` when the customer gave one. +- When the model has the delivery and developer experience measurement Capability, measure all three of its layers: none stands in for another. +- Measure nothing that serves no stated outcome. When no outcome was stated, add no Measures and say in your rationale that none could be tied to the business. + +```json +{"measures": [{"id": "M-1", "name": "Deployment frequency", "layer": "outcome", + "outcome": "BO-1", "measured_by": "C-18", "framework": "dora-metrics", + "target": "Daily"}]} +``` + +Checks: + +- A Check is a condition that can be shown to hold or not, on a Capability, a Role or a Responsibility. Its `source` is the Operational requirement or the Condition it comes from. +- Every Condition in `scope.stated.solution.conditions` must be carried by at least one Check whose `source` is that Condition's id. +- Start from the catalogue's `typical_checks` and rewrite each for this solution, with the customer's own numbers. +- Put a Check on the Capability that does the work it describes. A Check that service is restored within four hours is on the Capability that restores it, not on the one that reports service levels. + +Findings: + +- A finding is something the **solution** does not provide: a part of the design that is missing, such as an alert, a backup, a second region or a way in for the responders. When operating the solution needs one, attach `findings.json` with an entry for it, name the Items it is about under `about`, and do not design around it. +- Anything the operating model can provide itself is never a finding: staffing, hours of cover, a Role, a team, a process, a tool, a rota. When a Check needs one of those, design it. A finding about your own model is refused in validation and sends the model back to you. + +```json +[{"id": "AF-1", "about": ["S-3"], "needed_by": ["C-4"], + "lacks": "Nothing requires the database's backups to be restored on a schedule.", + "proposed": "Require a restore of the database to be tested quarterly."}] +``` + +- A Check that cannot hold until a finding is met names it under `depends_on_finding`. + +Never write a number of people, a rate or a cost: the responsibility matrix, the organisation chart, and the staffing and cost are computed from your graph and added to your Deliverable when you submit. + +Your content is one section, **Design rationale**: how the Parties divide the work and why, why each Role sits where it does and at its level, and what the model depends on that the customer has not confirmed. diff --git a/agents/operations-analyst.md b/agents/operations-analyst.md new file mode 100644 index 0000000..d496e4a --- /dev/null +++ b/agents/operations-analyst.md @@ -0,0 +1,25 @@ +--- +role: operations-analyst +deliverable: operational_requirements +tools: [] +budget: 20 +includes: [specialist-protocol] +--- +You produce the **operational requirements** Deliverable: what running the solution must achieve, each requirement traced to what the customer stated. + +Attach `operational-requirements.json`, a list: + +```json +{"id": "OR-1", "kind": "service_level", + "statement": "A fault on any Item is acknowledged within 15 minutes, at any hour.", + "source": ["service_levels.priority_1_response_minutes", "operations.support_hours"]} +``` + +- `kind` is one of `service_level`, `process`, `organisation`, `tooling`, `compliance`. +- `source` names where the requirement comes from: the id of a business outcome under `scope.stated.intent.business_outcomes`, the id of a requirement under `scope.stated.intent.requirements`, the id of a Condition under `scope.stated.solution.conditions`, or the path of a settled field of the operating spec, such as `operations.support_hours`. A requirement with no source is refused. +- Every requirement the customer stated, and every field listed under `scope.binding`, must be the source of at least one Operational requirement. A submission is refused while one is not. +- Write what must be achieved, never how or by whom: no Capability, no Role, no tool. Those come next. +- Use the customer's own numbers. Where a number was assumed, say so in the statement's requirement by naming the field; do not invent a tighter one. +- One requirement says one thing that can be shown to hold or not. Group several Conditions under one requirement only when they ask the same thing of different Items. + +Your content has two sections. Under **Operational requirements**, list them by kind with a sentence on each. Under **What was not stated**, say what the spec leaves assumed or unknown that would change these requirements if answered differently. diff --git a/agents/orchestrator.md b/agents/orchestrator.md new file mode 100644 index 0000000..15a0175 --- /dev/null +++ b/agents/orchestrator.md @@ -0,0 +1,9 @@ +--- +role: orchestrator +tools: [] +--- +You coordinate an Execution that designs a target operating model. You are the only agent that talks to the user, and you produce no Deliverable yourself. + +Each turn, use your tools to move the Execution on: `ready_work` shows which Deliverables can be delegated and which Phases are ready for Approval; `delegate` starts Specialists on everything that is ready; `ask_user` relays their questions; `request_approval` asks the user to approve a Phase whose Deliverables are all submitted; `reopen` sends a Deliverable back with a reason; `finish` completes the Execution once the last Phase is approved. + +Delegate all ready work at once. Never write or change a Deliverable's content, and never approve a Phase on the user's behalf. diff --git a/agents/packager.md b/agents/packager.md new file mode 100644 index 0000000..d893bf9 --- /dev/null +++ b/agents/packager.md @@ -0,0 +1,15 @@ +--- +role: packager +deliverable: operating_model_guide +tools: [] +budget: 15 +includes: [specialist-protocol] +--- +You produce the **operating model guide**: the one document that assembles every other Deliverable for the people who will run the solution. You add no decision of your own. + +Load the operating-guide Skill and read its template. + +- Write the **Summary**: one or two short paragraphs saying what is operated, who operates it and how the work is divided between the Parties, during which hours, and what the reader should look at first. Read it from the Deliverables in `dependencies`. +- State no number of people, no cost and no count, in the summary or anywhere else. The key figures are set under your summary from the data. +- Submit every other heading of your `contract` with one line under it: `Written from the data.` Every table, figure, verdict and citation in those sections is written when you submit, from the data files of the other Deliverables. Whatever you put under them is replaced. +- Give the guide a title that names the solution. diff --git a/agents/partials/specialist-protocol.md b/agents/partials/specialist-protocol.md new file mode 100644 index 0000000..452a47c --- /dev/null +++ b/agents/partials/specialist-protocol.md @@ -0,0 +1,37 @@ +## How you work + +You are a Specialist in an Execution that designs a target operating model. You produce exactly one Deliverable, named in your task. An Orchestrator coordinates the Execution and is the only agent that talks to the user. + +Your task message carries JSON context: + +- `workflow_request`: what the user asked for. +- `scope`: the agreed operating spec. `stated` is the customer's and binds your Deliverable; `assumed` was chosen for them and may change when they answer; `to_ask` is unknown. `stated.solution.items` is what is to be operated and `stated.solution.conditions` is what must be kept true of it. Load the operating-scoping Skill for the rules. +- `dependencies`: every earlier Deliverable your work builds on, by kind. The ones you work from directly come whole, with `content` and `attachments`. The rest come as their data file alone: the ids to keep traceable. Call `read_deliverable` for the full text of one only when the ids are not enough. +- `feedback`: why earlier versions of your Deliverable were revised. Address every item. +- `answers`: the user's replies to questions you asked in earlier runs, oldest first. +- `contract`: what `submit_deliverable` checks. `sections` are headings your content must have and `attachments` are files you must attach. A data file must keep every id traceable; the operating-chain Skill gives each file's format. +- `draft`: the draft you sent the user with your last questions, or `null`. +- `working_copy`: what your own tools already hold from earlier attempts, or `null`. Start from it; do not start again. +- `baseline`: set only in a Change request. `deliverable` is the delivered version of your own Deliverable, and `changes` is what has changed so far. `null` otherwise. +- `rate_card`: which rate card the cost is worked from. While `draft` is `true` its figures are placeholders: say so wherever you mention a cost. +- `overlay_skills`: Skills that carry the organisation's standards or this engagement's constraints. Load each one and apply it. +- `pre_approved`: `true` when no user is available. + +Steps: + +1. Load the Skills available to you and read the part that covers your Deliverable. +2. Ground your work in `scope`, `dependencies`, `feedback` and `answers`. When you need something only the user has, call `needs_input` with all your questions at once, each saying why it matters. Your run ends there, and you are started again with the answers. +3. Draft the Deliverable to its `contract`. When `pre_approved` is `false`, confirm the draft with the user before submitting: call `needs_input` with a short summary, asking them to approve it or request changes, and pass the complete draft as `draft` with its `attachments`. When `draft` is set and the latest entry in `answers` approves it, submit that draft as it is. When `pre_approved` is `true`, ask nothing: record each assumption under an `Assumptions` heading and continue. +4. Call `submit_deliverable` with the complete markdown content and every attachment in `contract`. When it reports what is wrong, fix each item and submit again. + +Three rules hold for every Deliverable: + +- **Never change the solution.** Do not add an Item, alter a Condition, or design around something the solution lacks. What operations needs and the solution does not provide is a finding. +- **Never write a number of people or a cost.** They are computed from the rate card. +- **Never claim compliance with a framework.** The capability catalogue draws on frameworks and cites them; it certifies nothing. + +## When `baseline` is set + +You are revising a delivered Deliverable, not writing a new one. Start from `baseline.deliverable`. Change only what the change reaches, and leave everything else as it was: the same words, the same ids. A new entry takes an id that was never used; a removed one is left out; never renumber. + +You are done when `submit_deliverable` accepts the Deliverable. diff --git a/agents/reply-writer.md b/agents/reply-writer.md new file mode 100644 index 0000000..594c2b3 --- /dev/null +++ b/agents/reply-writer.md @@ -0,0 +1,30 @@ +--- +role: reply-writer +tools: [] +--- +You write what the user reads on behalf of the Orchestrator of an Execution that designs a target operating model. You put into words what has already been decided or delivered, and you decide nothing. Your task message says which of two things you are writing, and carries JSON context. + +## A summary to approve a Phase from + +The context has `phase`, the Phase the user is being asked to approve, and `facts`, the data about it: its Deliverables, counts, staffing and cost, findings, validation verdicts, and Revisions. + +Write a summary a service delivery lead can approve from: + +- Say what the Phase produced and what approving it commits them to. For the design Phase, say that validation and the guide are written from this model once they approve it. +- For the scoping Phase, `facts.scope` says which fields were assumed for them and which are still unknown. Say how many of each, name the ones that most shape an operating model, and say that operational requirements are derived from this scope once they approve it. +- State every number in `facts` that bears on the decision, exactly as given. Add no fact that is not there. +- When `facts.staffing.rate_card.draft` is `true`, say that the cost is worked from placeholder prices and is an illustration. +- Name anything they should look at before approving: findings, failed or unverifiable Checks, Deliverables that were revised. +- Use plain sentences and no headings. Keep it under 120 words. + +## An answer about delivered work + +The context has `question`, what the user asked, and `delivered`: the `request` the operating model was designed for and its `final_document`, the operating model guide. + +- Answer from the guide alone. Give the choice or the fact, and the reason the guide gives for it. +- When the guide does not say, say that it does not. Add nothing from general knowledge, and do not guess. +- Name the section of the guide the answer comes from. +- Propose no change. If the user seems to want one, say they can ask for it as a change to this operating model. +- Use plain sentences and no headings. Keep it under 150 words. + +Call `write_reply` with the text. You are done when it accepts it. diff --git a/agents/scope-analyst.md b/agents/scope-analyst.md new file mode 100644 index 0000000..510452f --- /dev/null +++ b/agents/scope-analyst.md @@ -0,0 +1,24 @@ +--- +role: scope-analyst +deliverable: operating_scope +tools: [] +budget: 30 +includes: [specialist-protocol] +--- +You produce the **operating scope** Deliverable: the agreed statement of what the customer wants from operations and the facts an operating model depends on, as a completed operating spec. You design nothing. Load the operating-scoping Skill: it says what each field means. + +The operating spec is kept for you. `scope_read` shows what is settled, what is not, and the question to ask for each field. `scope_set` records fields, several per call. + +1. Call `scope_read`. When a spec was supplied or digested, most of it is already there, and what was stated stays as stated. In particular, `solution.items` and `solution.conditions` that were supplied or digested are the solution as it was described: never add to them, remove from them or reword them. + - When your context has `source_documents`, the caller supplied documents that describe the solution and that code could not read. Read them. Where the spec does not yet hold the solution, record `solution.items` and `solution.conditions` from them with `source` `request`: one Item for each thing an organisation would operate as a unit, with a short id (`S-1`, `S-2`) and a `kind`; one Condition for each thing the documents say must be kept true in operation. Give every entry an `evidence`: the document's name and the passage, quoted, that it was read from. Record only what a document says. An entry you cannot quote evidence for is not one. +2. Read `workflow_request`. Record what it states and the spec does not yet hold, with `source` `request`: the statement itself, word for word, under `intent.statement`; the kind of engagement; and any fact it gives. Record only what the words say. +3. Settle what is not settled, the essential fields first. + - **When you may ask the user** (`pre_approved` is `false`): call `needs_input` with the questions for the essential fields, and the important ones that most change an operating model, at most eight in one call, each in the catalogue's words. Tell them they may answer, say "you decide", or skip one. Record each answer with `source` `answered`. For "you decide", choose a value and record it with `source` `assumed` and a `note`. + - **When you may not ask** (`pre_approved` is `true`): for each essential or important field not settled, choose a value from the catalogue's default guidance and what the request implies, and record it with `source` `assumed` and a `note` saying why. Leave optional fields alone. + - A field marked `decide` is yours: choose, and record it as assumed with a note. +4. Call `scope_read` again. Every essential field must be settled before you can submit. +5. Submit. Your content is the scope statement under its heading: three to six sentences saying what is to be operated, who is to operate it, during which hours, in which currency its cost will be stated, and what is out of scope, using only what the spec holds. The completed spec and the information register are written for you. Pass no attachments. + +`intent.business_outcomes` is essential: everything the operating model measures is tied to one. When you may not ask, record two or three from what the request says the solution is for, as `assumed`, each with an id (`BO-1`), the outcome in plain words, and a note saying what you read it from. + +Never record a number, a country, a regulation or a sourcing choice the customer did not state as though they had: that is `assumed`, with its note. Never record a rate or a price at all: prices are the service provider's. diff --git a/agents/validator.md b/agents/validator.md new file mode 100644 index 0000000..292be59 --- /dev/null +++ b/agents/validator.md @@ -0,0 +1,20 @@ +--- +role: validator +deliverable: validation_results +tools: [] +budget: 30 +includes: [specialist-protocol] +--- +You produce the **validation results** Deliverable: a verdict on every Check of the Target operating model. + +- Give each Check a verdict from the graph and the capability model: `pass`, `fail` or `not_verifiable`, each with the evidence you read. The Checks are the whole bar: judge the model against them and nothing else. Judge each Check by its own words. +- A Check passes when the model, as designed, can meet it: the Capability it is on exists, a Role is responsible for it at the hours the Check needs, and nothing in the model contradicts it. It fails when the model cannot meet it as designed. It is `not_verifiable` when its condition cannot be shown to hold or not as it is worded. +- A Check that names a finding under `depends_on_finding` waits on that finding. Judge the finding first: is it something the solution lacks, or something the operating model could provide itself, such as staffing, hours of cover, a Role, a process or a tool? + - When the solution lacks it, the finding stands: give the Check `not_verifiable`, with the finding as the evidence. + - When the operating model could provide it, the finding does not stand: give the Check `fail`, and say in the evidence what the operating model should provide instead. The model is then revised. +- Judge whether each Capability's stated effort is plausible for the Items it covers. One that is not is a proposed addition, naming the Capability: never a failed Check. +- What you think should also be true is a proposed addition, never a failure. +- Record your verdicts with `verdict_put`, a few Checks per call. `verdict_read` shows which Checks still have none. Submit with your content and no attachments when every Check has a verdict: what you recorded is attached as `verdicts.json`. +- Give no verdict on anything whose id begins `SC-`. Those are scenario walk-throughs: they are generated and traced by code when you submit, and added to your Deliverable for you. + +Your content is one section, **Check verdicts**: what passed, what failed and why, and what could not be verified, ending with a pass or fail verdict for the model as a whole. diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/main.py b/app/main.py new file mode 100644 index 0000000..e3f8e97 --- /dev/null +++ b/app/main.py @@ -0,0 +1,8 @@ +"""The target operating model agent's entrypoint: `python -m app.main`.""" + +from execution_engine.main import serve + +from app.operating_model.workflow import ROOT, build + +if __name__ == "__main__": + serve(build(ROOT), ROOT) diff --git a/app/operating_model/__init__.py b/app/operating_model/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/app/operating_model/catalogue.py b/app/operating_model/catalogue.py new file mode 100644 index 0000000..cc5d7e9 --- /dev/null +++ b/app/operating_model/catalogue.py @@ -0,0 +1,153 @@ +"""The capability catalogue and the register of frameworks it cites. + +Both are files of the `operating-capabilities` Skill. Code reads them to check that +every entry cites a registered framework, to propose the entries that apply to an +operating spec, and to check that a Capability names an entry that exists. +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from app.operating_model import spec + +SKILL = "operating-capabilities" + + +class CatalogueError(ValueError): + """A catalogue or register that cannot be used; the message lists every fault.""" + + +@dataclass(frozen=True) +class Entry: + id: str + name: str + domain: str + purpose: str + applies_when: Mapping[str, Any] + follows_support_hours: bool + typical_checks: tuple[str, ...] + sources: tuple[Mapping[str, str], ...] + + +@dataclass(frozen=True) +class Catalogue: + entries: Mapping[str, Entry] + frameworks: Mapping[str, Mapping[str, Any]] + verified: bool # whether a person has checked the references + + def applicable(self, scope: Mapping[str, Any]) -> list[str]: + """The entries that apply to an operating spec, in the catalogue's order. + + A proposal: the Specialist decides, and says why one it leaves out does + not apply. + """ + kinds = {str(item.get("kind")) for item in spec.items(scope)} + frameworks = { + str(name) for name in spec.value(scope, "compliance.frameworks") or [] + } + sourcing = spec.value(scope, "sourcing.model") + engagement = scope.get("kind") or spec.value(scope, "engagement.kind") + found = [] + for entry in self.entries.values(): + when = entry.applies_when + if ( + when.get("always") + or kinds & set(when.get("item_kinds") or ()) + or frameworks & set(when.get("compliance") or ()) + or sourcing in (when.get("sourcing") or ()) + or engagement in (when.get("engagement_kinds") or ()) + ): + found.append(entry.id) + return found + + +def _mapping(path: Path) -> dict[str, Any]: + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not isinstance(data, dict): + raise CatalogueError(f"{path.name}: must be a mapping") + return data + + +def load(skill_dir: Path) -> Catalogue: + """Read the catalogue and its register from the Skill's folder, and check them.""" + register = _mapping(skill_dir / "assets" / "sources.yaml") + frameworks = register.get("frameworks") or {} + problems = [ + f"sources.yaml: {name} needs {field}" + for name, framework in frameworks.items() + for field in ("name", "publisher", "licence") + if not (framework or {}).get(field) + ] + raw = _mapping(skill_dir / "assets" / "catalogue.yaml").get("capabilities") or [] + entries: dict[str, Entry] = {} + for index, item in enumerate(raw): + where = f"catalogue.yaml[{index}]" + if not isinstance(item, Mapping) or not item.get("id"): + problems.append(f"{where}: needs an id") + continue + identifier = str(item["id"]) + where = f"catalogue.yaml: {identifier}" + if identifier in entries: + problems.append(f"{where} is listed more than once") + problems += [ + f"{where} needs {field}" + for field in ("name", "domain", "purpose", "applies_when") + if not item.get(field) + ] + sources = [s for s in item.get("sources") or [] if isinstance(s, Mapping)] + if not sources: + problems.append(f"{where} cites no source") + for source in sources: + if source.get("framework") not in frameworks: + problems.append( + f"{where} cites {source.get('framework')!r}, which is not in " + "the register" + ) + if not source.get("reference"): + problems.append( + f"{where} cites {source.get('framework')} with no reference" + ) + entries[identifier] = Entry( + id=identifier, + name=str(item.get("name", "")), + domain=str(item.get("domain", "")), + purpose=str(item.get("purpose", "")), + applies_when=dict(item.get("applies_when") or {}), + follows_support_hours=bool(item.get("follows_support_hours")), + typical_checks=tuple(str(c) for c in item.get("typical_checks") or ()), + sources=tuple( + { + "framework": str(s.get("framework")), + "reference": str(s.get("reference")), + } + for s in sources + ), + ) + if problems: + raise CatalogueError("; ".join(problems)) + return Catalogue( + entries=entries, + frameworks=dict(frameworks), + verified=register.get("status") == "verified", + ) + + +def cited(catalogue: Catalogue, entry_ids: Sequence[str]) -> list[str]: + """How the entries' sources are cited, one line a framework, for a reader.""" + by_framework: dict[str, list[str]] = {} + for identifier in entry_ids: + entry = catalogue.entries.get(identifier) + for source in entry.sources if entry else (): + references = by_framework.setdefault(source["framework"], []) + if source["reference"] not in references: + references.append(source["reference"]) + return [ + f"{catalogue.frameworks[name]['name']}: {'; '.join(references)}" + for name, references in by_framework.items() + ] diff --git a/app/operating_model/chain.py b/app/operating_model/chain.py new file mode 100644 index 0000000..cb7e0af --- /dev/null +++ b/app/operating_model/chain.py @@ -0,0 +1,240 @@ +"""The data files along the chain, and what each must hold. + + operational-requirements.json what running the solution must achieve, each + traced to what the customer stated + capabilities.json what the operating organisation must be able to + do, each serving requirements and covering Items + target-operating-model.json who does it (see graph.py) + verdicts.json the verdict on each Check + +Each checker returns what is wrong, one entry per fault, so a Specialist can fix it +and submit again. `upstream` is the Deliverables a submission builds on, by kind. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Mapping, Sequence +from typing import Any + +from app.operating_model import spec +from app.operating_model.catalogue import Catalogue +from app.operating_model.names import ( + CAPABILITIES_FILE, + CAPABILITY_MODEL, + CUSTOM, + HOURS, + OPERATIONAL_REQUIREMENTS, + REQUIREMENTS_FILE, + wider, +) +from app.operating_model.spec import Upstream + +REQUIREMENT_KINDS = ( + "service_level", + "process", + "organisation", + "tooling", + "compliance", +) +DataCheck = Callable[[Mapping[str, str], Upstream | None], list[str]] + + +def objects(text: str, name: str) -> tuple[list[dict[str, Any]], list[str]]: + """The file's entries, and what is wrong with it as a list of objects with ids.""" + try: + data = json.loads(text) + except ValueError as exc: + return [], [f"{name}: not valid JSON ({exc})"] + if not isinstance(data, list): + return [], [f"{name}: must be a JSON list"] + problems = [ + f"{name}[{index}]: must be an object with an id" + for index, item in enumerate(data) + if not isinstance(item, dict) or not str(item.get("id") or "").strip() + ] + accepted = [ + item + for item in data + if isinstance(item, dict) and str(item.get("id") or "").strip() + ] + seen = [str(item["id"]) for item in accepted] + problems += [ + f"{name}: id {identifier} is used more than once" + for identifier in sorted({i for i in seen if seen.count(i) > 1}) + ] + return accepted, problems + + +def read(upstream: Upstream | None, kind: str, name: str) -> list[dict[str, Any]]: + """An upstream data file's entries; none when it is absent or unreadable.""" + text = ((upstream or {}).get(kind) or {}).get("attachments", {}).get(name, "") + return objects(text, name)[0] if text else [] + + +def requirements(upstream: Upstream | None) -> list[dict[str, Any]]: + return read(upstream, OPERATIONAL_REQUIREMENTS, REQUIREMENTS_FILE) + + +def capabilities(upstream: Upstream | None) -> list[dict[str, Any]]: + return read(upstream, CAPABILITY_MODEL, CAPABILITIES_FILE) + + +def _strings(item: Mapping[str, Any], field: str) -> list[str]: + found = item.get(field) + return [str(entry) for entry in found] if isinstance(found, list) else [] + + +def check_requirements( + attachments: Mapping[str, str], upstream: Upstream | None +) -> list[str]: + """Every Operational requirement is traced to what the customer stated, and + everything they stated that binds is carried by one.""" + name = REQUIREMENTS_FILE + entries, problems = objects(attachments[name], name) + if not entries and not problems: + return [f"{name}: lists no Operational requirements"] + scope = spec.agreed(upstream) + stated = set(spec.ids(spec.stated_requirements(scope))) + # A business outcome may be what a requirement comes from. It need not be: what + # shows each outcome is served is a Measure, checked on the operating model. + outcomes = set(spec.ids(spec.business_outcomes(scope))) + kept_true = set(spec.ids(spec.conditions(scope))) + named: set[str] = set() + for entry in entries: + where = f"{name}: {entry['id']}" + if not str(entry.get("statement") or "").strip(): + problems.append(f"{where} needs a statement") + if entry.get("kind") not in REQUIREMENT_KINDS: + problems.append( + f"{where}: kind must be one of {', '.join(REQUIREMENT_KINDS)}" + ) + sources = _strings(entry, "source") + if not sources: + problems.append( + f"{where} has no source: name the stated requirement, the Condition " + "or the field of the operating spec it comes from" + ) + for source in sources: + named.add(source) + known = ( + source in stated + or source in outcomes + or source in kept_true + or spec.settled(scope, source) + ) + if scope and not known: + problems.append( + f"{where}: source {source} is not a stated requirement, a " + "Condition or a settled field of the operating spec" + ) + if scope: + problems += [ + f"{name}: the customer's requirement {identifier} is carried by no " + "Operational requirement" + for identifier in sorted(stated - named) + ] + problems += [ + f"{name}: the customer stated {path}, and no Operational requirement " + "names it as a source" + for path in spec.binding(scope) + if path not in named + ] + return problems + + +def check_capabilities(catalogue: Catalogue) -> DataCheck: + """Every Capability comes from the catalogue, serves a requirement and covers + Items; every Item is covered and every requirement served.""" + + def check(attachments: Mapping[str, str], upstream: Upstream | None) -> list[str]: + name = CAPABILITIES_FILE + entries, problems = objects(attachments[name], name) + if not entries and not problems: + return [f"{name}: lists no Capabilities"] + scope = spec.agreed(upstream) + known_items = set(spec.ids(spec.items(scope))) + known_requirements = {str(r["id"]) for r in requirements(upstream)} + support = spec.value(scope, "operations.support_hours") + covered: set[str] = set() + served: set[str] = set() + for entry in entries: + where = f"{name}: {entry['id']}" + entry_id = entry.get("catalogue") + listed = catalogue.entries.get(str(entry_id)) + if entry_id == CUSTOM: + if not str(entry.get("reason") or "").strip(): + problems.append(f"{where} is custom and gives no reason") + if not str(entry.get("name") or "").strip(): + problems.append(f"{where} is custom and needs a name") + elif listed is None: + problems.append( + f"{where}: {entry_id!r} is not in the capability catalogue; " + f"use an entry's id, or {CUSTOM} with a reason" + ) + serves, on = _strings(entry, "serves"), _strings(entry, "items") + if not serves: + problems.append(f"{where} serves no Operational requirement") + served.update(serves) + covered.update(on) + if upstream is not None: + problems += [ + f"{where} serves {identifier}, which is not an Operational requirement" + for identifier in serves + if identifier not in known_requirements + ] + problems += [ + f"{where} covers {identifier}, which is not an Item of the operating spec" + for identifier in on + if identifier not in known_items + ] + hours = entry.get("hours") + if hours not in HOURS: + problems.append(f"{where}: hours must be one of {', '.join(HOURS)}") + elif ( + listed is not None + and listed.follows_support_hours + and support in HOURS + and wider(str(support), str(hours)) + ): + problems.append( + f"{where}: faults are responded to {support}, and {listed.name} " + f"is staffed only {hours}" + ) + effort = entry.get("effort_hours_per_month") + if ( + isinstance(effort, bool) + or not isinstance(effort, (int, float)) + or effort <= 0 + ): + problems.append( + f"{where}: effort_hours_per_month must be a number above zero" + ) + if not str(entry.get("basis") or "").strip(): + problems.append(f"{where} needs a basis for its effort") + if upstream is not None: + problems += [ + f"{name}: Item {identifier} is covered by no Capability" + for identifier in sorted(known_items - covered) + ] + problems += [ + f"{name}: Operational requirement {identifier} is served by no Capability" + for identifier in sorted(known_requirements - served) + ] + return problems + + return check + + +def names_of( + capabilities_list: Sequence[Mapping[str, Any]], catalogue: Catalogue +) -> dict[str, str]: + """Each Capability's name: its catalogue entry's, or its own when it is custom.""" + return { + str(entry["id"]): ( + catalogue.entries[str(entry.get("catalogue"))].name + if str(entry.get("catalogue")) in catalogue.entries + else str(entry.get("name") or entry["id"]) + ) + for entry in capabilities_list + } diff --git a/app/operating_model/digest.py b/app/operating_model/digest.py new file mode 100644 index 0000000..5a3f78f --- /dev/null +++ b/app/operating_model/digest.py @@ -0,0 +1,413 @@ +"""Digesting a solution architecture into an operating spec, in code. + +The solution architecture agent delivers its design as data files. This reads the +ones an operating model depends on and writes the operating spec's fields from +them: nothing is summarised and nothing is guessed, so what is digested can be +counted against what was delivered. + + components.json, products.json each component becomes an Item to be operated, + target-architecture.json with the Elements that realise it + requirements.json, what each says can only be shown in operation + recommendations.json (`outside_code`) becomes a Condition + verdicts.json a Check that failed or could not be verified + becomes an inherited risk + scope.json what the customer stated of operations, and of + the solution, is carried across as stated + work-breakdown.json, how long the build takes, and what the customer + execution.json owns of it + +An Item is a component, not a resource: an organisation operates a hybrid +connectivity gateway, not each of its six routers and attachments. A Condition is +what the architecture itself says infrastructure code cannot verify: an +availability measured in operation, a recovery rehearsed. The settings its Checks +verified at build are kept true by configuration management, and are not Conditions. + +Only what the customer stated in the architecture's scope is carried across as +stated. What that agent assumed for them is left to be asked again. +""" + +from __future__ import annotations + +import json +import math +import re +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +import yaml +from execution_engine.workflows.workflow import Digested + +GRAPH = "target-architecture.json" +COMPONENTS = "components.json" +PRODUCTS = "products.json" +REQUIREMENTS = "requirements.json" +RECOMMENDATIONS = "recommendations.json" +VERDICTS = "verdicts.json" +SCOPE = "scope.json" +PLAN = "work-breakdown.json" +MANIFEST = "execution.json" +# Every file the digest reads. A folder is read for these and nothing else. +FILES = ( + MANIFEST, + SCOPE, + REQUIREMENTS, + RECOMMENDATIONS, + COMPONENTS, + PRODUCTS, + GRAPH, + VERDICTS, + PLAN, +) +DEFAULT_KIND = "service" +SOURCE_PREFIX = "architecture:" +_HOURS = {"business_hours": "business", "extended": "extended", "always": "24x7"} +_NUMBER = re.compile(r"\d+(?:\.\d+)?") + + +def _load(sources: Mapping[str, str], name: str) -> Any: + if name not in sources: + return None + try: + return json.loads(sources[name]) + except ValueError as exc: + raise ValueError(f"{name}: not valid JSON ({exc})") from exc + + +def _records(data: Any) -> list[dict[str, Any]]: + return ( + [item for item in data if isinstance(item, dict)] + if isinstance(data, list) + else [] + ) + + +def _within(document: Any, path: str) -> Any: + for part in path.split("."): + if not isinstance(document, Mapping) or part not in document: + return None + document = document[part] + return document + + +def kind_of(product: str, item_kinds: Mapping[str, Sequence[str]]) -> str: + """What kind of thing a product is, by the words the provider bundle lists.""" + lowered = product.lower() + for kind, words in item_kinds.items(): + if any(str(word).lower() in lowered for word in words): + return kind + return DEFAULT_KIND + + +def item_kinds(root: Path, provider: str) -> dict[str, list[str]]: + """The provider bundle's table of what kind of thing each product is.""" + path = root / "providers" / f"{provider}.yaml" + data = yaml.safe_load(path.read_text(encoding="utf-8")) if path.is_file() else {} + kinds = ((data or {}).get("operating") or {}).get("item_kinds") or {} + return { + str(kind): [str(word) for word in words or []] for kind, words in kinds.items() + } + + +def architecture( + sources: Mapping[str, str], kinds: Mapping[str, Sequence[str]] | None = None +) -> Digested: + """The operating spec's fields, read from a solution architecture's data files. + + Returns nothing digested when the sources are not a solution architecture. + Raises ValueError, saying what is wrong, when they are one that cannot be used. + """ + if GRAPH not in sources: + return Digested({}) + graph = _load(sources, GRAPH) + if not isinstance(graph, dict) or not isinstance(graph.get("elements"), list): + # A ValueError is what the engine turns into a refused start. + raise ValueError( # noqa: TRY004 + f"{GRAPH}: must be an object with a list of elements" + ) + manifest = _load(sources, MANIFEST) or {} + if manifest and manifest.get("status", "completed") != "completed": + raise ValueError( + f"{MANIFEST}: the Execution did not complete ({manifest.get('status')}); " + "an operating model is designed for an architecture that was approved" + ) + elements = _records(graph["elements"]) + components = _records(_load(sources, COMPONENTS)) + product_of = { + str(item.get("component")): str(item.get("product")) + for item in _records(_load(sources, PRODUCTS)) + } + kinds = kinds or {} + notes: dict[str, str] = {} + + # --- Items: one a component, with the Elements that realise it ------------------- + realised: dict[str, list[dict[str, Any]]] = {} + for element in elements: + realised.setdefault(str(element.get("component") or ""), []).append(element) + if not components: + components = [ + {"id": identifier, "name": identifier, "requirements": []} + for identifier in realised + if identifier + ] + items: list[dict[str, Any]] = [] + item_of: dict[str, str] = {} + for component in components: + component_id = str(component.get("id")) + own = realised.get(component_id, []) + products = [str(e.get("product")) for e in own if e.get("product")] + product = product_of.get(component_id) or ( + max(set(products), key=products.count) if products else "" + ) + item_id = f"S-{len(items) + 1}" + item_of[component_id] = item_id + items.append( + { + "id": item_id, + "name": str(component.get("name") or component_id), + "kind": kind_of(product, kinds), + **({"product": product} if product else {}), + "source_ref": component_id, + "elements": [str(e.get("id")) for e in own], + } + ) + loose = realised.get("", []) + if loose: + # Elements the architecture filed under no component are still operated. + item_id = f"S-{len(items) + 1}" + item_of[""] = item_id + items.append( + { + "id": item_id, + "name": "Resources outside any component", + "kind": DEFAULT_KIND, + "elements": [str(e.get("id")) for e in loose], + } + ) + if not items: + raise ValueError(f"{GRAPH}: lists no Elements to operate") + placed = sum(len(item["elements"]) for item in items) + if placed != len(elements): + unplaced = sorted( + str(e.get("id")) + for e in elements + if str(e.get("component") or "") not in item_of + ) + raise ValueError( + f"{GRAPH}: these Elements name a component that {COMPONENTS} does not " + f"have: {', '.join(unplaced)}" + ) + notes["solution.items"] = ( + f"{len(items)} components of the solution architecture, standing for " + f"{len(elements)} Elements" + ) + + # --- Conditions: what can only be shown to hold in operation -------------------- + serving: dict[str, list[str]] = {} + for component in components: + for requirement in component.get("requirements") or []: + serving.setdefault(str(requirement), []).append(str(component.get("id"))) + conditions: list[dict[str, Any]] = [] + stated = 0 + for name in (REQUIREMENTS, RECOMMENDATIONS): + for entry in _records(_load(sources, name)): + if entry.get("change") == "removed": + continue + entry_id = str(entry.get("id")) + behind = [str(c) for c in entry.get("components") or []] or serving.get( + entry_id, [] + ) + on = [item_of[c] for c in behind if c in item_of] or [ + str(i["id"]) for i in items + ] + for statement in entry.get("outside_code") or []: + stated += 1 + conditions.append( + { + "id": f"K-{len(conditions) + 1}", + "on": on, + "condition": " ".join(str(statement).split()), + "source_ref": entry_id, + } + ) + if len(conditions) != stated: + raise ValueError( + "a condition the architecture leaves to operation was not placed" + ) + if conditions: + notes["solution.conditions"] = ( + f"{len(conditions)} conditions the solution architecture says cannot be " + "verified from infrastructure code" + ) + + # --- Inherited risks: Checks that did not pass ------------------------------------ + element_item = { + str(e.get("id")): item_of.get(str(e.get("component") or ""), items[0]["id"]) + for e in elements + } + check_on = { + str(check.get("id")): element_item.get( + str(check.get("applies_to")), items[0]["id"] + ) + for check in _records(graph.get("checks")) + } + statement_of = { + str(check.get("id")): str(check.get("statement") or "") + for check in _records(graph.get("checks")) + } + verdicts = _load(sources, VERDICTS) or {} + risks = [ + { + "on": check_on.get(str(item.get("check")), items[0]["id"]), + "risk": ( + f"Check {item.get('check')} " + + ( + "failed" + if item.get("verdict") == "fail" + else "could not be verified" + ) + + f" in validation: {statement_of.get(str(item.get('check')), '')} " + f"({' '.join(str(item.get('evidence') or '').split())})" + ).strip(), + "source_ref": str(item.get("check")), + } + for item in _records( + verdicts.get("checks") if isinstance(verdicts, dict) else None + ) + if item.get("verdict") in ("fail", "not_verifiable") + ] + + # --- What the customer stated, carried across as stated -------------------------- + scope = _load(sources, SCOPE) or {} + said = scope.get("stated") if isinstance(scope, dict) else {} + spec: dict[str, Any] = { + "solution": { + "items": items, + **({"conditions": conditions} if conditions else {}), + } + } + + def put(path: str, value: Any, note: str = "") -> None: + if value in (None, "", [], {}): + return + node = spec + *parents, last = path.split(".") + for part in parents: + node = node.setdefault(part, {}) + node[last] = value + if note: + notes[path] = note + + if risks: + put("solution.inherited_risks", risks, f"{VERDICTS}: Checks that did not pass") + execution_id = manifest.get("execution_id") + put("metadata.customer_case", manifest.get("customer_case")) + put( + "metadata.source", + f"{SOURCE_PREFIX}{execution_id}" + if execution_id + else f"{SOURCE_PREFIX}unverified", + "the Execution these files are of" + if execution_id + else f"no {MANIFEST} came with the files, so they are not known to be one " + "completed Execution", + ) + put("engagement.kind", scope.get("kind") if isinstance(scope, dict) else None) + put( + "intent.statement", + manifest.get("workflow_request") or _within(said, "intent.statement"), + "the request the solution architecture was designed for", + ) + criteria = _within(said, "intent.success_criteria") + put( + "intent.business_outcomes", + [ + {"id": f"BO-{n}", "outcome": " ".join(str(text).split())} + for n, text in enumerate(criteria or [], start=1) + if str(text).strip() + ] + if isinstance(criteria, list) + else None, + f"{SCOPE}: the success criteria the customer stated", + ) + put("solution.cloud_provider", _within(said, "estate.cloud_provider")) + hours = _within(said, "operations.support_hours") + put("operations.support_hours", _HOURS.get(str(hours)), f"{SCOPE}: {hours}") + available = _within(said, "workload.availability.hours") + put( + "operations.availability_hours", + _HOURS.get(str(available)), + f"{SCOPE}: {available}", + ) + put("operations.change_process", _within(said, "operations.change_process")) + put("operations.monitoring", _within(said, "operations.monitoring")) + put("compliance.frameworks", _within(said, "compliance.frameworks")) + recovery = _NUMBER.search( + str(_within(said, "workload.availability.recovery_time") or "") + ) + put( + "service_levels.recovery_time_hours", + float(recovery.group()) if recovery else None, + ) + currency = _within(said, "cost.currency") + put("commercial.currency", currency if isinstance(currency, str) else None) + + plan = manifest.get("delivery_plan") if isinstance(manifest, dict) else None + days = (plan or {}).get("working_days") + if isinstance(days, (int, float)) and not isinstance(days, bool): + put( + "solution.build.duration_days", + math.ceil(days), + "the build's computed length, in working days", + ) + activities = _records((_load(sources, PLAN) or {}).get("activities")) + put( + "solution.build.customer_owned", + [str(a.get("name")) for a in activities if a.get("owner") == "customer"], + f"{PLAN}: the activities the customer owns", + ) + return Digested( + spec=spec, used=tuple(n for n in FILES if n in sources), notes=notes + ) + + +def read_folder(folder: Path) -> dict[str, str]: + """The data files of an architecture Execution's folder that the digest reads.""" + return { + name: (folder / name).read_text(encoding="utf-8") + for name in FILES + if (folder / name).is_file() + } + + +def main(arguments: Sequence[str]) -> int: + """`python -m app.operating_model.digest [provider]`: print the operating + spec digested from an architecture Execution's folder, and what is left to settle.""" + from app.operating_model.workflow import ROOT + + if not arguments: + print(main.__doc__, file=sys.stderr) + return 2 + folder = Path(arguments[0]) + provider = arguments[1] if len(arguments) > 1 else "gcp" + sources = read_folder(folder) + try: + digested = architecture(sources, item_kinds(ROOT, provider)) + except ValueError as exc: + print(f"Cannot digest {folder}: {exc}", file=sys.stderr) + return 1 + if not digested.spec: + print( + f"{folder} holds no {GRAPH}: it is not a solution architecture.", + file=sys.stderr, + ) + return 1 + print(yaml.safe_dump(dict(digested.spec), sort_keys=False, allow_unicode=True)) + for path, note in digested.notes.items(): + print(f"# {path}: {note}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/app/operating_model/graph.py b/app/operating_model/graph.py new file mode 100644 index 0000000..464163b --- /dev/null +++ b/app/operating_model/graph.py @@ -0,0 +1,719 @@ +"""The Target operating model: one explicit graph, checked before anything is derived. + + parties the organisations with a part in operating the solution + teams optional: the teams within a Party, each of one of Team + Topologies' four kinds + roles positions in a Party, each in a country and at a level, and + in a team when there are teams + interactions optional: how two teams work together, in one of Team + Topologies' three modes; a collaboration says when it ends + measures what is measured to show each business outcome the customer + stated is being served, at which layer, and by which + Capability + responsibilities a Role's part in a Capability: accountable, responsible, + consulted or informed + checks numbered, verifiable conditions on a Capability, a Role or a + Responsibility, each traced to what it comes from + +The Capabilities are the capability model's, by id. A submission is refused while +the graph does not hold together, so what the customer approves is complete: every +Capability has one Role answerable for it and someone to do it at the hours it is +needed, and everything the customer said must be kept true is carried by a Check. + +The responsibility matrix, the organisation chart and the staffing and cost are +written from the graph by code, as views of it. +""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Collection, Mapping, Sequence +from typing import Any + +from execution_engine.workflows.deliverables import GraphShape, ListShape + +from app.operating_model import chain, spec, staffing +from app.operating_model.catalogue import Catalogue +from app.operating_model.names import ( + ACCOUNTABLE, + COLLABORATION, + DEVELOPER_LAYERS, + FINDINGS_FILE, + GRAPH_FILE, + HOURS, + INTERACTION_MODES, + MEASURE_LAYERS, + MEASUREMENT_ENTRY, + PARTY_KINDS, + RESPONSIBILITY_KINDS, + RESPONSIBLE, + TARGET_OPERATING_MODEL, + TEAM_TYPES, + wider, +) +from app.operating_model.spec import Upstream +from app.operating_model.staffing import RateCard + +LISTS = ( + "parties", + "teams", + "roles", + "responsibilities", + "interactions", + "measures", + "checks", +) +OUTCOMES_HEADING = "Business outcomes" +MATRIX_HEADING = "Responsibility matrix" +STAFFING_HEADING = "Staffing and cost" +FINDINGS_HEADING = "Findings" +_LETTER = {"accountable": "A", "responsible": "R", "consulted": "C", "informed": "I"} + +# The graph as the operating-model architect builds it, a few items at a time. +SHAPE = GraphShape( + attachment=GRAPH_FILE, + lists={ + "parties": ListShape(("id", "name", "kind"), label="Parties"), + "teams": ListShape( + ("id", "party", "name", "type"), optional=("purpose",), label="Teams" + ), + "roles": ListShape( + ("id", "party", "name", "country", "level", "hours"), + optional=("skills", "team"), + label="Roles", + ), + "interactions": ListShape( + ("id", "from", "to", "mode"), + optional=("purpose", "until"), + label="Interactions", + ), + "measures": ListShape( + ("id", "name", "layer", "outcome", "measured_by"), + optional=("framework", "target"), + label="Measures", + ), + "responsibilities": ListShape( + ("id", "capability", "role", "kind"), + optional=("share",), + label="Responsibilities", + ), + "checks": ListShape( + ("id", "on", "condition", "source"), + optional=("depends_on_finding",), + label="Checks", + ), + }, +) + + +def parse(text: str) -> tuple[dict[str, list[dict[str, Any]]], list[str]]: + """The graph's four lists, and what is wrong with the file's shape.""" + try: + data = json.loads(text) + except ValueError as exc: + return {name: [] for name in LISTS}, [f"{GRAPH_FILE}: not valid JSON ({exc})"] + if not isinstance(data, dict): + return {name: [] for name in LISTS}, [ + f"{GRAPH_FILE}: must be an object with {', '.join(LISTS)}" + ] + graph: dict[str, list[dict[str, Any]]] = {} + problems = [] + for name in LISTS: + entries = data.get(name) or [] + if not isinstance(entries, list): + problems.append(f"{GRAPH_FILE}: {name} must be a list") + entries = [] + graph[name] = [dict(item) for item in entries if isinstance(item, dict)] + for index, item in enumerate(entries): + where = f"{GRAPH_FILE}: {name}[{index}]" + if not isinstance(item, dict): + problems.append(f"{where} must be an object") + continue + missing = [ + field + for field in SHAPE.lists[name].required + if not str(item.get(field) or "").strip() + ] + if missing: + problems.append( + f"{where} ({item.get('id', 'no id')}) needs {', '.join(missing)}" + ) + seen = [str(item.get("id")) for item in graph[name] if item.get("id")] + problems += [ + f"{GRAPH_FILE}: id {identifier} is used more than once in {name}" + for identifier in sorted({i for i in seen if seen.count(i) > 1}) + ] + return graph, problems + + +def read(upstream: Upstream | None) -> dict[str, list[dict[str, Any]]] | None: + """The submitted graph from the Deliverables a submission builds on.""" + text = ( + ((upstream or {}).get(TARGET_OPERATING_MODEL) or {}) + .get("attachments", {}) + .get(GRAPH_FILE, "") + ) + return parse(text)[0] if text else None + + +def findings(attachments: Mapping[str, str]) -> list[dict[str, Any]]: + text = attachments.get(FINDINGS_FILE, "") + return chain.objects(text, FINDINGS_FILE)[0] if text.strip() else [] + + +def check(card: RateCard, frameworks: Collection[str] = ()) -> chain.DataCheck: + """What is wrong with a Target operating model, against the capability model, + the operating spec and the rate card. `frameworks` are the register's, which a + Measure may name.""" + + def checked(attachments: Mapping[str, str], upstream: Upstream | None) -> list[str]: + graph, problems = parse(attachments[GRAPH_FILE]) + if problems: + return problems + name = GRAPH_FILE + scope = spec.agreed(upstream) + capabilities = chain.capabilities(upstream) + capability_ids = {str(c["id"]) for c in capabilities} + requirement_ids = {str(r["id"]) for r in chain.requirements(upstream)} + conditions = spec.ids(spec.conditions(scope)) + + parties = {str(p["id"]): p for p in graph["parties"]} + roles = {str(r["id"]): r for r in graph["roles"]} + for party in graph["parties"]: + if party.get("kind") not in PARTY_KINDS: + problems.append( + f"{name}: Party {party['id']}: kind must be one of " + f"{', '.join(PARTY_KINDS)}" + ) + allowed = [ + str(c) for c in spec.value(scope, "sourcing.delivery_countries") or [] + ] + for role in graph["roles"]: + where = f"{name}: Role {role['id']}" + if role.get("party") not in parties: + problems.append( + f"{where} is in {role.get('party')}, which is not a Party" + ) + if role.get("hours") not in HOURS: + problems.append(f"{where}: hours must be one of {', '.join(HOURS)}") + country = str(role.get("country")) + if country not in card.countries: + problems.append( + f"{where} sits in {country}, which the rate card does not have: " + f"{', '.join(sorted(card.countries))}" + ) + elif allowed and country not in allowed: + problems.append( + f"{where} sits in {country}; the customer allows only " + f"{', '.join(allowed)}" + ) + if str(role.get("level")) not in card.levels: + problems.append( + f"{where} is at level {role.get('level')}, which the rate card " + f"does not have: {', '.join(card.levels)}" + ) + + problems += check_teams(graph, parties, name) + + by_capability: dict[str, list[dict[str, Any]]] = {} + for item in graph["responsibilities"]: + where = f"{name}: Responsibility {item['id']}" + if item.get("kind") not in RESPONSIBILITY_KINDS: + problems.append( + f"{where}: kind must be one of {', '.join(RESPONSIBILITY_KINDS)}" + ) + if item.get("role") not in roles: + problems.append( + f"{where} names {item.get('role')}, which is not a Role" + ) + if upstream is not None and item.get("capability") not in capability_ids: + problems.append( + f"{where} names {item.get('capability')}, which is not a " + "Capability of the capability model" + ) + by_capability.setdefault(str(item.get("capability")), []).append(item) + + for capability in capabilities: + identifier = str(capability["id"]) + held = by_capability.get(identifier, []) + accountable = [r for r in held if r.get("kind") == ACCOUNTABLE] + responsible = [r for r in held if r.get("kind") == RESPONSIBLE] + if len(accountable) != 1: + problems.append( + f"{name}: Capability {identifier} has {len(accountable)} " + "accountable Roles; it needs exactly one" + ) + if not responsible: + problems.append( + f"{name}: Capability {identifier} has no responsible Role" + ) + continue + needed = str(capability.get("hours")) + worked = [ + str(roles[str(r["role"])].get("hours")) + for r in responsible + if r.get("role") in roles + ] + if ( + needed in HOURS + and worked + and all(hours in HOURS and wider(needed, hours) for hours in worked) + ): + problems.append( + f"{name}: Capability {identifier} is needed {needed}, and no " + "Role responsible for it works those hours" + ) + problems += [ + f"{name}: {fault}" + for fault in staffing.shares(graph["responsibilities"])[1] + ] + + found = findings(attachments) + finding_ids = {str(item["id"]) for item in found} + on = ( + capability_ids + | set(roles) + | {str(r["id"]) for r in graph["responsibilities"]} + | {str(team["id"]) for team in graph["teams"]} + | {str(i["id"]) for i in graph["interactions"]} + | {str(m["id"]) for m in graph["measures"]} + ) + carried: set[str] = set() + for item in graph["checks"]: + where = f"{name}: Check {item['id']}" + source = str(item.get("source")) + carried.add(source) + if upstream is not None and item.get("on") not in on: + problems.append( + f"{where} is on {item.get('on')}, which is not a Capability, a " + "Role or a Responsibility" + ) + if upstream is not None and source not in requirement_ids | set(conditions): + problems.append( + f"{where}: source {source} is not an Operational requirement " + "or a Condition" + ) + depends = item.get("depends_on_finding") + if depends and depends not in finding_ids: + problems.append(f"{where} depends on {depends}, which is not a finding") + problems += [ + f"{name}: Condition {identifier} is carried by no Check" + for identifier in conditions + if identifier not in carried + ] + problems += check_measures( + graph, scope, capabilities, by_capability, roles, frameworks, name + ) + problems += check_findings( + attachments, + scope, + on | {str(c["id"]) for c in graph["checks"]} | requirement_ids, + ) + return problems + + return checked + + +def check_teams( + graph: Mapping[str, Sequence[Mapping[str, Any]]], + parties: Mapping[str, Any], + name: str, +) -> list[str]: + """What is wrong with the teams and how they work together. + + Teams are optional. When there are any, each is of one of Team Topologies' four + kinds, a Role in a team belongs to that team's Party, and two teams work + together in one of its three modes. A collaboration is a stage, not a standing + arrangement: it says when it ends. + """ + problems = [] + teams = {str(team["id"]): team for team in graph["teams"]} + for team in graph["teams"]: + where = f"{name}: Team {team['id']}" + if team.get("type") not in TEAM_TYPES: + problems.append(f"{where}: type must be one of {', '.join(TEAM_TYPES)}") + if team.get("party") not in parties: + problems.append(f"{where} is in {team.get('party')}, which is not a Party") + for role in graph["roles"]: + team_id = role.get("team") + if not team_id: + continue + found = teams.get(str(team_id)) + if found is None: + problems.append( + f"{name}: Role {role['id']} is in {team_id}, which is not a Team" + ) + elif found.get("party") != role.get("party"): + problems.append( + f"{name}: Role {role['id']} is in {role.get('party')}, and its team " + f"{team_id} is in {found.get('party')}" + ) + for item in graph["interactions"]: + where = f"{name}: Interaction {item['id']}" + if item.get("mode") not in INTERACTION_MODES: + problems.append( + f"{where}: mode must be one of {', '.join(INTERACTION_MODES)}" + ) + for end in ("from", "to"): + if item.get(end) not in teams: + problems.append(f"{where}: {end} {item.get(end)} is not a Team") + if item.get("from") == item.get("to"): + problems.append(f"{where} is between a team and itself") + if ( + item.get("mode") == COLLABORATION + and not str(item.get("until") or "").strip() + ): + problems.append( + f"{where} is a collaboration and does not say when it ends: give " + "`until`, and what the teams work in after" + ) + return problems + + +def check_measures( + graph: Mapping[str, Sequence[Mapping[str, Any]]], + scope: Mapping[str, Any], + capabilities: Sequence[Mapping[str, Any]], + by_capability: Mapping[str, Sequence[Mapping[str, Any]]], + roles: Mapping[str, Any], + frameworks: Collection[str], + name: str, +) -> list[str]: + """Every business outcome the customer stated is shown by something measured, + and everything measured shows one: nothing is measured for its own sake. + + A Measure is taken by a Capability someone is responsible for. Where the model + measures how work flows, it measures all three layers: delivery outcomes, + sustainable productivity and friction. No one of them stands in for the others. + """ + problems = [] + outcomes = {str(o["id"]): o for o in spec.business_outcomes(scope)} + capability_ids = {str(c["id"]) for c in capabilities} + measured: set[str] = set() + for item in graph["measures"]: + where = f"{name}: Measure {item['id']}" + if item.get("layer") not in MEASURE_LAYERS: + problems.append( + f"{where}: layer must be one of {', '.join(MEASURE_LAYERS)}" + ) + outcome = str(item.get("outcome")) + measured.add(outcome) + if outcome not in outcomes: + problems.append( + f"{where} shows {outcome}, which is not a business outcome the " + "customer stated" + + (f": {', '.join(outcomes)}" if outcomes else "; none were stated") + ) + taker = str(item.get("measured_by")) + if taker not in capability_ids: + problems.append( + f"{where} is measured by {taker}, which is not a Capability" + ) + elif not any( + r.get("kind") == "responsible" and r.get("role") in roles + for r in by_capability.get(taker, ()) + ): + problems.append( + f"{where} is measured by {taker}, and nobody is responsible for it" + ) + if item.get("framework") and frameworks and item["framework"] not in frameworks: + problems.append( + f"{where} names framework {item['framework']}, which is not in the register" + ) + problems += [ + f"{name}: business outcome {identifier} ({outcomes[identifier].get('outcome')}) " + "is shown by no Measure" + for identifier in outcomes + if identifier not in measured + ] + measurers = { + str(c["id"]) for c in capabilities if c.get("catalogue") == MEASUREMENT_ENTRY + } + # Only once there is an outcome to tie Measures to: with none, no Measure can + # be accepted, and asking for three of them could never be met. + if measurers and outcomes: + layers = { + str(m.get("layer")) + for m in graph["measures"] + if m.get("measured_by") in measurers + } + missing = [layer for layer in DEVELOPER_LAYERS if layer not in layers] + if missing: + problems.append( + f"{name}: {', '.join(sorted(measurers))} measures how work flows and has " + f"no Measure at the {', '.join(missing)} layer; each layer answers a " + "different question and none stands in for another" + ) + return problems + + +def render_outcomes( + graph: Mapping[str, Sequence[Mapping[str, Any]]], + scope: Mapping[str, Any], + names: Mapping[str, str], +) -> str: + """Each business outcome, with what shows it is served and who answers for it.""" + outcomes = spec.business_outcomes(scope) + if not outcomes: + return ( + "The customer stated no business outcomes, so nothing in this operating " + "model is tied to one. Ask what the business wants from the solution, " + "and how it would know." + ) + roles = {str(r["id"]): r for r in graph["roles"]} + answerable: dict[str, str] = {} + for item in graph["responsibilities"]: + if item.get("kind") == "accountable" and item.get("role") in roles: + answerable[str(item.get("capability"))] = str( + roles[str(item["role"])].get("name") + ) + lines = [] + for outcome in outcomes: + target = f" Target: {outcome['target']}." if outcome.get("target") else "" + lines += [f"**{outcome['id']}: {outcome.get('outcome')}**{target}", ""] + shown = [m for m in graph["measures"] if m.get("outcome") == outcome["id"]] + if not shown: + lines += ["Nothing measured shows this outcome.", ""] + continue + lines += [ + "| Measure | Layer | Framework | Target | Measured by | Answerable |", + "| --- | --- | --- | --- | --- | --- |", + ] + lines += [ + f"| {m.get('name')} | {str(m.get('layer', '')).replace('_', ' ')} | " + f"{m.get('framework') or ''} | {m.get('target') or ''} | " + f"{names.get(str(m.get('measured_by')), m.get('measured_by'))} | " + f"{answerable.get(str(m.get('measured_by')), '')} |" + for m in shown + ] + lines.append("") + return "\n".join(lines).rstrip() + + +def check_findings( + attachments: Mapping[str, str], scope: Mapping[str, Any], known: set[str] +) -> list[str]: + """What is wrong with findings.json. It may be an empty list.""" + text = attachments.get(FINDINGS_FILE, "") + if not text.strip(): + return [] + entries, problems = chain.objects(text, FINDINGS_FILE) + item_ids = set(spec.ids(spec.items(scope))) + for entry in entries: + where = f"{FINDINGS_FILE}: {entry['id']}" + for field in ("lacks", "proposed"): + if not str(entry.get(field) or "").strip(): + problems.append(f"{where} needs {field}") + about = [str(i) for i in entry.get("about") or []] + needed_by = [str(i) for i in entry.get("needed_by") or []] + if not about: + # A finding is about the solution: it names what in it is lacking. + problems.append( + f"{where} names no Item of the solution under about. What the " + "operating model can provide itself, such as staffing, hours, a Role, " + "a process or a tool, is not a finding: design it" + ) + if not needed_by: + problems.append(f"{where} must say what needs it, under needed_by") + problems += [ + f"{where} is about {identifier}, which is not an Item of the operating spec" + for identifier in about + if scope and identifier not in item_ids + ] + problems += [ + f"{where} is needed by {identifier}, which is not in the operating model" + for identifier in needed_by + if identifier not in known + ] + return problems + + +# --- the views code writes ----------------------------------------------------------- + + +def render_matrix( + graph: Mapping[str, Sequence[Mapping[str, Any]]], names: Mapping[str, str] +) -> str: + """Who is accountable, responsible, consulted and informed for each Capability.""" + roles = list(graph["roles"]) + # A Role may hold more than one part in a Capability: answerable for it and + # doing it. Both are shown. + held: dict[tuple[str, str], str] = {} + for item in graph["responsibilities"]: + key = (str(item.get("capability")), str(item.get("role"))) + letter = _LETTER.get(str(item.get("kind")), "?") + held[key] = f"{held[key]}/{letter}" if key in held else letter + header = "| Capability | " + " | ".join(str(r.get("name")) for r in roles) + " |" + rule = "| --- | " + " | ".join("---" for _ in roles) + " |" + rows = [ + f"| {name} ({identifier}) | " + + " | ".join(held.get((identifier, str(r["id"])), "") for r in roles) + + " |" + for identifier, name in names.items() + ] + legend = "A accountable, R responsible, C consulted, I informed." + return "\n".join([f"## {MATRIX_HEADING}", "", header, rule, *rows, "", legend, ""]) + + +def _node(identifier: Any) -> str: + return str(identifier).replace("-", "_") + + +def _role_node(role: Mapping[str, Any], indent: str) -> str: + return ( + f'{indent}{_node(role["id"])}["{role.get("name")}
{role.get("country")}, ' + f'{role.get("level")}, {role.get("hours")}"]' + ) + + +def render_organisation(graph: Mapping[str, Sequence[Mapping[str, Any]]]) -> str: + """The Parties, their teams and the Roles in each, and how the teams work + together, as a Mermaid chart.""" + teams = graph.get("teams") or [] + lines = ["flowchart TB"] + for party in graph["parties"]: + lines.append( + f' subgraph {_node(party["id"])}["{party.get("name")} ({party.get("kind")})"]' + ) + for team in teams: + if team.get("party") != party["id"]: + continue + kind = str(team.get("type", "")).replace("_", "-") + lines.append( + f' subgraph {_node(team["id"])}["{team.get("name")} ({kind} team)"]' + ) + lines += [ + _role_node(role, " ") + for role in graph["roles"] + if role.get("team") == team["id"] + ] + lines.append(" end") + lines += [ + _role_node(role, " ") + for role in graph["roles"] + if role.get("party") == party["id"] and not role.get("team") + ] + lines.append(" end") + for item in graph.get("interactions") or []: + mode = str(item.get("mode", "")).replace("_", "-") + label = mode + (f" until {item['until']}" if item.get("until") else "") + lines.append(f' {_node(item["from"])} -- "{label}" --> {_node(item["to"])}') + return "\n".join(lines) + "\n" + + +def _money(amount: Any, currency: str) -> str: + return "not priced" if amount is None else f"{amount:,} {currency}" + + +def render_staffing(result: Mapping[str, Any], budget: Any = None) -> str: + """People and cost per Role and in total, with the prices they were worked from.""" + currency = str(result["currency"]) + card = result["rate_card"] + conversion = result["conversion"] + lines = [ + f"## {STAFFING_HEADING}", + "", + "| Role | Party | Country | Level | Hours | Effort a month | People | Cost a month |", + "| --- | --- | --- | --- | --- | --- | --- | --- |", + ] + lines += [ + f"| {row['name']} | {row.get('party_name', row['party'])} | " + f"{row['country']} | {row['level']} | " + f"{row['hours']} | {row['effort_hours_per_month']} h | {row['people']} | " + f"{_money(row['cost_per_month'], currency)} |" + for row in result["roles"] + ] + lines += [ + "", + ( + f"{result['people']} people in all. Cost a month: " + f"{_money(result['cost_per_month'], currency)}." + ), + ] + if isinstance(budget, (int, float)) and not isinstance(budget, bool): + over = result["cost_per_month"] - budget + lines.append( + f"The monthly budget is {_money(round(budget), currency)}: " + + ( + f"this is over it by {_money(round(over), currency)}." + if over > 0 + else "this is within it." + ) + ) + lines += [ + "", + f"Worked from rate card `{card['name']}`, version `{card['version']}`" + + ( + ". **Its figures are placeholders: no price has been set, and this cost " + "is an illustration.**" + if card["draft"] + else "." + ), + ] + if conversion["from"] != currency: + lines.append( + f"Converted from {conversion['from']} at {conversion['rate']}, a rate " + f"set on {conversion['as_of']}." + ) + lines += [ + ( + "The customer's own staff are counted and not priced. A Role that covers " + "hours wider than a working day is never smaller than the team that can " + "cover them." + ), + "", + ] + return "\n".join(lines) + + +def render_findings(found: Sequence[Mapping[str, Any]]) -> str: + if not found: + return "" + lines = [ + f"## {FINDINGS_HEADING}", + "", + ( + "What operating this solution needs that the solution, as described, does " + "not provide. Nothing here was designed around: each is for whoever owns " + "the design." + ), + "", + ] + lines += [ + f"- **{item['id']}** ({', '.join(str(i) for i in item.get('about') or []) or 'the solution'}): " + f"{item.get('lacks')} Proposed: {item.get('proposed')}" + for item in found + ] + return "\n".join(lines) + "\n" + + +Views = Callable[[str, dict[str, str], Upstream | None], tuple[str, dict[str, str]]] + + +def views(catalogue: Catalogue, card: RateCard) -> Views: + """Add the matrix, the staffing and the findings to an accepted submission.""" + from app.operating_model.names import ORGANISATION_FILE, STAFFING_FILE + + def add( + content: str, attachments: dict[str, str], upstream: Upstream | None + ) -> tuple[str, dict[str, str]]: + graph, _ = parse(attachments[GRAPH_FILE]) + scope = spec.agreed(upstream) + capabilities = chain.capabilities(upstream) + currency = str(spec.value(scope, "commercial.currency") or card.currency) + result = staffing.compute(graph, capabilities, card, currency) + found = findings(attachments) + # A Revision replaces these sections, never stacks a second copy. + content = content.split(f"\n## {MATRIX_HEADING}")[0].rstrip() + parts = [ + content, + render_matrix(graph, chain.names_of(capabilities, catalogue)), + render_staffing(result, spec.value(scope, "commercial.monthly_run_budget")), + render_findings(found), + ] + attachments[ORGANISATION_FILE] = render_organisation(graph) + attachments[STAFFING_FILE] = json.dumps(result, indent=2, ensure_ascii=False) + attachments.setdefault(FINDINGS_FILE, "[]") + return "\n\n".join(part.rstrip() for part in parts if part) + "\n", attachments + + return add diff --git a/app/operating_model/guide.py b/app/operating_model/guide.py new file mode 100644 index 0000000..715dd07 --- /dev/null +++ b/app/operating_model/guide.py @@ -0,0 +1,387 @@ +"""The operating model guide: one document, assembled from every other Deliverable. + +The packager writes the summary. Every other section is written here, from the data +the other Deliverables carry: the Items and Conditions, the requirements, the +Capabilities, who does what, staffing and cost, the verdicts and the findings. A +table or a figure copied by a model can be copied wrongly; one written from the data +cannot disagree with it. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping, Sequence +from typing import Any + +from app.operating_model import catalogue as capability_catalogue +from app.operating_model import chain, graph, scenarios, spec +from app.operating_model.catalogue import Catalogue +from app.operating_model.names import ( + SCENARIOS_FILE, + STAFFING_FILE, + TARGET_OPERATING_MODEL, + VALIDATION_RESULTS, + VERDICTS_FILE, +) +from app.operating_model.spec import Upstream + +SUMMARY = "Summary" +KEY_FIGURES = "Key figures" +# The sections of the guide, in order. The packager writes the first; code writes +# the rest. +SECTIONS = ( + SUMMARY, + "Business outcomes", + "What is operated", + "Operational requirements", + "Capabilities", + "Who does what", + "Staffing and cost", + "Validation", + "Findings and open questions", + "Sources", +) +_HEADING = re.compile(r"^##\s+(?:\d+(?:\.\d+)*\.?\s+)?(.+?)\s*#*\s*$", re.MULTILINE) + + +def _json(deliverables: Upstream, kind: str, name: str) -> Any: + text = (deliverables.get(kind) or {}).get("attachments", {}).get(name, "") + try: + return json.loads(text) if text else None + except ValueError: + return None + + +def _cell(value: Any) -> str: + """A value as one cell of a table.""" + if isinstance(value, (list, tuple)): + value = ", ".join(str(item) for item in value) + return " ".join(str(value if value is not None else "").split()).replace("|", "\\|") + + +def _table(headers: Sequence[str], rows: Sequence[Sequence[Any]]) -> list[str]: + return [ + "| " + " | ".join(headers) + " |", + "| " + " | ".join("---" for _ in headers) + " |", + *("| " + " | ".join(_cell(value) for value in row) + " |" for row in rows), + ] + + +def _body(rendered: str) -> str: + """A rendered view without its own heading.""" + return ( + rendered.split("\n", 1)[1].strip() if rendered.startswith("## ") else rendered + ) + + +def business_outcomes(deliverables: Upstream, catalogue: Catalogue) -> str: + model = graph.read(deliverables) or { + "roles": [], + "responsibilities": [], + "measures": [], + } + names = chain.names_of(chain.capabilities(deliverables), catalogue) + return graph.render_outcomes(model, spec.agreed(deliverables), names) + + +def what_is_operated(scope: Mapping[str, Any]) -> str: + lines = _table( + ("Item", "What it is", "Kind", "Product", "From"), + [ + ( + i["id"], + i.get("name"), + i.get("kind"), + i.get("product"), + i.get("source_ref"), + ) + for i in spec.items(scope) + ], + ) + conditions = spec.conditions(scope) + if conditions: + lines += ["", "What must be kept true while it is run:", ""] + lines += _table( + ("Condition", "On", "What must hold", "From"), + [ + (c["id"], c.get("on"), c.get("condition"), c.get("source_ref")) + for c in conditions + ], + ) + risks = spec.records(scope, "solution.inherited_risks") + if risks: + lines += ["", "Known to be unverified or at risk when operations begin:", ""] + lines += [f"- {r.get('on')}: {r.get('risk')}" for r in risks] + return "\n".join(lines) + + +def requirements(deliverables: Upstream) -> str: + return "\n".join( + _table( + ("Requirement", "Kind", "What must be achieved", "From"), + [ + (r["id"], r.get("kind"), r.get("statement"), r.get("source")) + for r in chain.requirements(deliverables) + ], + ) + ) + + +def capabilities(deliverables: Upstream, catalogue: Catalogue) -> str: + listed = chain.capabilities(deliverables) + names = chain.names_of(listed, catalogue) + return "\n".join( + _table( + ("Capability", "What it is", "Serves", "Covers", "Hours", "Effort a month"), + [ + ( + c["id"], + names[str(c["id"])], + c.get("serves"), + c.get("items"), + c.get("hours"), + f"{c.get('effort_hours_per_month')} h", + ) + for c in listed + ], + ) + ) + + +def who_does_what(deliverables: Upstream, catalogue: Catalogue) -> str: + model = graph.read(deliverables) + if model is None: + return "" + party = {str(p["id"]): p for p in model["parties"]} + lines = _table( + ("Role", "Party", "Country", "Level", "Hours"), + [ + ( + r.get("name"), + ( + f"{party.get(str(r.get('party')), {}).get('name')} " + f"({party.get(str(r.get('party')), {}).get('kind')})" + ), + r.get("country"), + r.get("level"), + r.get("hours"), + ) + for r in model["roles"] + ], + ) + teams = model.get("teams") or [] + if teams: + lines += ["", "The teams, as Team Topologies names their kinds:", ""] + lines += _table( + ("Team", "Kind", "Party", "Roles", "Purpose"), + [ + ( + team.get("name"), + str(team.get("type", "")).replace("_", "-"), + party.get(str(team.get("party")), {}).get("name"), + [ + r.get("name") + for r in model["roles"] + if r.get("team") == team["id"] + ], + team.get("purpose"), + ) + for team in teams + ], + ) + interactions = model.get("interactions") or [] + if interactions: + named = {str(team["id"]): team.get("name") for team in teams} + lines += ["", "How the teams work together:", ""] + lines += _table( + ("From", "To", "Mode", "Until", "Purpose"), + [ + ( + named.get(str(i.get("from")), i.get("from")), + named.get(str(i.get("to")), i.get("to")), + str(i.get("mode", "")).replace("_", "-"), + i.get("until") or "", + i.get("purpose"), + ) + for i in interactions + ], + ) + names = chain.names_of(chain.capabilities(deliverables), catalogue) + return "\n".join([*lines, "", _body(graph.render_matrix(model, names))]) + + +def staffing_and_cost(deliverables: Upstream) -> str: + result = _json(deliverables, TARGET_OPERATING_MODEL, STAFFING_FILE) + if not isinstance(result, dict): + return "" + budget = spec.value(spec.agreed(deliverables), "commercial.monthly_run_budget") + return _body(graph.render_staffing(result, budget)) + + +def validation(deliverables: Upstream) -> str: + verdicts = _json(deliverables, VALIDATION_RESULTS, VERDICTS_FILE) or {} + model = graph.read(deliverables) or {"checks": []} + condition = {str(c["id"]): c.get("condition") for c in model["checks"]} + own = [ + item + for item in verdicts.get("checks") or [] + if isinstance(item, dict) + and not str(item.get("check", "")).startswith(scenarios.PREFIX) + ] + lines = _table( + ("Check", "What must hold", "Verdict", "Evidence"), + [ + ( + v.get("check"), + condition.get(str(v.get("check"))), + str(v.get("verdict")).replace("_", " ") + + (f" ({v['finding']})" if v.get("finding") else ""), + v.get("evidence"), + ) + for v in own + ], + ) + walked = _json(deliverables, VALIDATION_RESULTS, SCENARIOS_FILE) + if isinstance(walked, list) and walked: + lines += ["", _body(scenarios.render(walked))] + return "\n".join(lines) + + +def findings_and_questions(deliverables: Upstream) -> str: + scope = spec.agreed(deliverables) + found = graph.findings( + (deliverables.get(TARGET_OPERATING_MODEL) or {}).get("attachments", {}) + ) + lines = [] + if found: + lines += [_body(graph.render_findings(found)), ""] + else: + lines += ["No findings: the solution provides what operating it needs.", ""] + assumed = [a for a in scope.get("assumed") or [] if isinstance(a, Mapping)] + if assumed: + lines += ["Assumed for the customer, and open to change when they answer:", ""] + lines += [ + f"- `{a.get('field')}`: {_cell(a.get('value'))}. {a.get('why') or ''}".rstrip() + for a in assumed + ] + lines.append("") + undecided = [d for d in scope.get("to_decide") or [] if isinstance(d, Mapping)] + if undecided: + lines += ["Left to the agent by the customer, and not yet settled:", ""] + lines += [ + f"- `{d.get('field')}`: affects {d.get('affects') or 'the operating model'}." + for d in undecided + ] + lines.append("") + # What matters to an operating model and is not known. The optional fields + # nobody was asked for are not open questions. + unknown = [ + g + for g in scope.get("to_ask") or [] + if isinstance(g, Mapping) and g.get("severity") in ("HIGH", "MEDIUM") + ] + if unknown: + lines += ["Still unknown:", ""] + lines += [ + f"- `{g.get('field')}`: {g.get('prompt') or ''}".rstrip() for g in unknown + ] + return "\n".join(lines).rstrip() + + +def sources(deliverables: Upstream, catalogue: Catalogue) -> str: + entries = [str(c.get("catalogue")) for c in chain.capabilities(deliverables)] + lines = [ + ( + "The Capabilities are from our own capability catalogue, which draws on " + "these frameworks. It cites them and certifies nothing against them." + ), + "", + *(f"- {line}" for line in capability_catalogue.cited(catalogue, entries)), + ] + if not catalogue.verified: + lines += [ + "", + ( + "**These references have not been checked against the frameworks by a " + "person.** Treat each as a pointer to where to look." + ), + ] + return "\n".join(lines) + + +def key_figures(deliverables: Upstream) -> str: + """The figures a reader takes from the summary, written from the data.""" + scope = spec.agreed(deliverables) + cost = _json(deliverables, TARGET_OPERATING_MODEL, STAFFING_FILE) + walked = _json(deliverables, VALIDATION_RESULTS, SCENARIOS_FILE) or [] + found = graph.findings( + (deliverables.get(TARGET_OPERATING_MODEL) or {}).get("attachments", {}) + ) + lines = [ + ( + f"- Operated: {len(spec.items(scope))} Items, with " + f"{len(spec.conditions(scope))} Conditions to keep true." + ), + f"- Faults responded to: {spec.value(scope, 'operations.support_hours')} hours.", + f"- Capabilities: {len(chain.capabilities(deliverables))}.", + ] + if isinstance(cost, dict): + card = cost["rate_card"] + lines.append( + f"- People: {cost['people']}. Cost a month: {cost['cost_per_month']:,} " + f"{cost['currency']}, from rate card version `{card['version']}`" + + (" (placeholder prices: an illustration)." if card["draft"] else ".") + ) + if walked: + passed = sum(1 for item in walked if item.get("passes")) + lines.append(f"- Scenario walk-throughs: {passed} of {len(walked)} pass.") + outcomes = spec.business_outcomes(scope) + model = graph.read(deliverables) + if outcomes: + shown = {str(m.get("outcome")) for m in (model or {}).get("measures", [])} + lines.append( + f"- Business outcomes: {len(outcomes)}, " + f"{sum(1 for o in outcomes if str(o['id']) in shown)} shown by a Measure." + ) + else: + lines.append("- Business outcomes: none stated.") + lines.append(f"- Findings for whoever owns the design: {len(found)}.") + return "\n".join(lines) + + +def assemble(content: str, deliverables: Upstream, catalogue: Catalogue) -> str: + """The guide, with every section but the summary written from the data. + + The packager's summary is kept as written, and the key figures are set under it. + Whatever it wrote under another heading is replaced. + """ + written = { + "Business outcomes": business_outcomes(deliverables, catalogue), + "What is operated": what_is_operated(spec.agreed(deliverables)), + "Operational requirements": requirements(deliverables), + "Capabilities": capabilities(deliverables, catalogue), + "Who does what": who_does_what(deliverables, catalogue), + "Staffing and cost": staffing_and_cost(deliverables), + "Validation": validation(deliverables), + "Findings and open questions": findings_and_questions(deliverables), + "Sources": sources(deliverables, catalogue), + } + headings = list(_HEADING.finditer(content)) + title = content[: headings[0].start()].rstrip() if headings else content.rstrip() + summary = "" + for index, match in enumerate(headings): + if match.group(1).strip().lower() == SUMMARY.lower(): + end = ( + headings[index + 1].start() + if index + 1 < len(headings) + else len(content) + ) + summary = content[match.end() : end].strip() + summary = summary.split(f"\n### {KEY_FIGURES}")[0].rstrip() + parts = [ + title, + f"## {SUMMARY}\n\n{summary}\n\n### {KEY_FIGURES}\n\n{key_figures(deliverables)}", + *(f"## {heading}\n\n{written[heading]}" for heading in SECTIONS[1:]), + ] + return "\n\n".join(part.rstrip() for part in parts if part.strip()) + "\n" diff --git a/app/operating_model/names.py b/app/operating_model/names.py new file mode 100644 index 0000000..621d33c --- /dev/null +++ b/app/operating_model/names.py @@ -0,0 +1,66 @@ +"""The names this Workflow is written in: its Phases, its Deliverables and its files.""" + +from __future__ import annotations + +# Phases, in order. +SCOPING = "scoping" +REQUIREMENTS = "operational_requirements" +DESIGN = "operating_model_design" +VALIDATION = "validation" +PACKAGING = "packaging" +PHASES = (SCOPING, REQUIREMENTS, DESIGN, VALIDATION, PACKAGING) + +# Deliverable kinds. +OPERATING_SCOPE = "operating_scope" +OPERATIONAL_REQUIREMENTS = "operational_requirements" +CAPABILITY_MODEL = "capability_model" +TARGET_OPERATING_MODEL = "target_operating_model" +VALIDATION_RESULTS = "validation_results" +GUIDE = "operating_model_guide" + +# Data files. +SCOPE_DATA_FILE = "scope.json" # the engine's: what every Specialist reads of the spec +REQUIREMENTS_FILE = "operational-requirements.json" +CAPABILITIES_FILE = "capabilities.json" +GRAPH_FILE = "target-operating-model.json" +FINDINGS_FILE = "findings.json" +STAFFING_FILE = "staffing.json" +ORGANISATION_FILE = "organisation.mmd" +VERDICTS_FILE = "verdicts.json" +SCENARIOS_FILE = "scenarios.json" +CHANGES_FILE = "changes.json" +GUIDE_FILE = "operating-model-guide.md" +CHANGE_REQUEST_FILE = "architecture-change-request.md" + +# Hours a Capability must be staffed for and a Role works, narrowest first. +HOURS = ("business", "extended", "24x7") +PARTY_KINDS = ("customer", "provider", "supplier", "cloud_provider") +CUSTOMER = "customer" +ACCOUNTABLE, RESPONSIBLE, CONSULTED, INFORMED = ( + "accountable", + "responsible", + "consulted", + "informed", +) +RESPONSIBILITY_KINDS = (ACCOUNTABLE, RESPONSIBLE, CONSULTED, INFORMED) +CUSTOM = "custom" # a Capability the catalogue does not have +# Team Topologies: the four kinds of team, and the three ways two teams work together. +TEAM_TYPES = ("stream_aligned", "platform", "enabling", "complicated_subsystem") +COLLABORATION, X_AS_A_SERVICE, FACILITATING = ( + "collaboration", + "x_as_a_service", + "facilitating", +) +INTERACTION_MODES = (COLLABORATION, X_AS_A_SERVICE, FACILITATING) +# What a Measure tells about a business outcome. The first three are the layers of +# measuring how work flows (DORA, SPACE, DevEx); none stands in for another. +OUTCOME_LAYER, GUARDRAIL_LAYER, LEADING_LAYER = "outcome", "guardrail", "leading" +DEVELOPER_LAYERS = (OUTCOME_LAYER, GUARDRAIL_LAYER, LEADING_LAYER) +MEASURE_LAYERS = (*DEVELOPER_LAYERS, "service_level", "cost") +# The catalogue entry whose measures must cover all three developer layers. +MEASUREMENT_ENTRY = "delivery-and-developer-experience-measurement" + + +def wider(first: str, second: str) -> bool: + """Whether `first` hours are wider than `second`.""" + return HOURS.index(first) > HOURS.index(second) diff --git a/app/operating_model/scenarios.py b/app/operating_model/scenarios.py new file mode 100644 index 0000000..636b5c8 --- /dev/null +++ b/app/operating_model/scenarios.py @@ -0,0 +1,346 @@ +"""Scenario walk-throughs: events the operating model must be able to handle. + +Infrastructure code can be validated by running it. An operating model cannot, so +each scenario is generated from the graph and the operating spec and traced through +the graph by code: is there a Capability for it, is someone responsible at the hours +it happens, is one Role answerable. A broken path is a failed Check. No model writes +a scenario or judges one. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping, Sequence +from typing import Any + +from app.operating_model import spec +from app.operating_model.catalogue import Catalogue +from app.operating_model.names import ( + ACCOUNTABLE, + CONSULTED, + CUSTOMER, + HOURS, + RESPONSIBLE, + wider, +) + +PREFIX = "SC-" # a scenario's id; the validator gives no verdict under it +# The verdict on a Check that cannot hold until a finding is met. It is neither a +# failure of the model nor a gap in it: nothing is revised for it, and it is +# reported with the finding it waits on. +WAITING = "waits_on_finding" +HEADING = "Scenario walk-throughs" +CHANGE_DOMAIN = "change" + + +def _scenario( + identifier: str, scenario: str, about: str, fault: str | None, path: str +) -> dict[str, Any]: + return { + "id": f"{PREFIX}{identifier}", + "scenario": scenario, + "about": about, + "passes": fault is None, + "path": fault or path, + } + + +def walk( + graph: Mapping[str, Sequence[Mapping[str, Any]]], + capabilities: Sequence[Mapping[str, Any]], + scope: Mapping[str, Any], + catalogue: Catalogue, + staffing: Mapping[str, Any] | None = None, +) -> list[dict[str, Any]]: + """Every scenario for this operating model, traced.""" + roles = {str(r["id"]): r for r in graph["roles"]} + parties = {str(p["id"]): p for p in graph["parties"]} + by_capability: dict[str, list[Mapping[str, Any]]] = {} + for item in graph["responsibilities"]: + by_capability.setdefault(str(item.get("capability")), []).append(item) + + def holders(capability: str, kind: str) -> list[Mapping[str, Any]]: + return [ + roles[str(r["role"])] + for r in by_capability.get(capability, []) + if r.get("kind") == kind and str(r.get("role")) in roles + ] + + def entry_of(capability: Mapping[str, Any]) -> Any: + return catalogue.entries.get(str(capability.get("catalogue"))) + + def covering(item: str) -> list[Mapping[str, Any]]: + return [c for c in capabilities if item in (c.get("items") or [])] + + support = str(spec.value(scope, "operations.support_hours") or HOURS[0]) + found = [] + for item in spec.items(scope): + item_id, label = str(item["id"]), f"{item.get('name')} ({item['id']})" + + responders = [ + c + for c in covering(item_id) + if entry_of(c) and entry_of(c).follows_support_hours + ] + fault = None + if not responders: + fault = "No Capability that responds to faults covers it." + else: + on_hours = [ + (c, role) + for c in responders + for role in holders(str(c["id"]), RESPONSIBLE) + if str(role.get("hours")) in HOURS + and not wider(support, str(role.get("hours"))) + ] + answerable = [c for c, _ in on_hours if holders(str(c["id"]), ACCOUNTABLE)] + if not on_hours: + fault = f"No Role responsible for responding works {support} hours." + elif not answerable: + fault = "The Capability that responds has no accountable Role." + chosen = None if fault else on_hours[0] + found.append( + _scenario( + f"fault-{item_id}", + f"{label} fails at an hour faults are responded to ({support}).", + item_id, + fault, + "" + if chosen is None + else f"{chosen[1].get('name')} responds under {chosen[0]['id']}, and " + f"{holders(str(chosen[0]['id']), ACCOUNTABLE)[0].get('name')} is " + "answerable.", + ) + ) + + changers = [ + c + for c in covering(item_id) + if entry_of(c) + and entry_of(c).domain == CHANGE_DOMAIN + and holders(str(c["id"]), RESPONSIBLE) + ] + found.append( + _scenario( + f"change-{item_id}", + f"{label} must be changed.", + item_id, + None + if changers + else "No Capability for making changes covers it with a responsible Role.", + "" + if not changers + else f"{holders(str(changers[0]['id']), RESPONSIBLE)[0].get('name')} " + f"makes the change under {changers[0]['id']}.", + ) + ) + + capability_ids = {str(c["id"]) for c in capabilities} + for condition in spec.conditions(scope): + condition_id = str(condition["id"]) + carrying = [k for k in graph["checks"] if k.get("source") == condition_id] + kept = [ + k + for k in carrying + if str(k.get("on")) not in capability_ids + or holders(str(k.get("on")), RESPONSIBLE) + ] + found.append( + _scenario( + f"condition-{condition_id}", + f"Condition {condition_id} must be shown to hold: " + f"{condition.get('condition')}", + condition_id, + None + if kept + else ( + "No Check carries it." + if not carrying + else "The Check that carries it is on a Capability nobody is responsible for." + ), + "" + if not kept + else f"Check {kept[0]['id']} on {kept[0].get('on')} keeps it true.", + ) + ) + + for party in graph["parties"]: + if party.get("kind") == CUSTOMER: + continue + party_id = str(party["id"]) + stranded = [ + str(c["id"]) + for c in capabilities + if any( + a.get("party") == party_id for a in holders(str(c["id"]), ACCOUNTABLE) + ) + and not any( + parties.get(str(role.get("party")), {}).get("id") != party_id + for kind in (RESPONSIBLE, CONSULTED) + for role in holders(str(c["id"]), kind) + ) + ] + found.append( + _scenario( + f"party-{party_id}", + f"{party.get('name')} cannot be reached.", + party_id, + None + if not stranded + else "No Role in another Party is responsible or consulted for " + + ", ".join(stranded) + + ".", + "Every Capability it is answerable for has a Role in another Party to turn to.", + ) + ) + + budget = spec.value(scope, "commercial.monthly_run_budget") + if ( + staffing is not None + and isinstance(budget, (int, float)) + and not isinstance(budget, bool) + ): + cost, currency = staffing["cost_per_month"], staffing["currency"] + found.append( + _scenario( + "budget", + f"A month's cost of operating is compared with the budget of {round(budget):,} {currency}.", + "commercial.monthly_run_budget", + None + if cost <= budget + else f"The cost is {cost:,} {currency}, over the budget.", + f"The cost is {cost:,} {currency}, within the budget.", + ) + ) + return found + + +def as_verdicts(scenarios: Sequence[Mapping[str, Any]]) -> list[dict[str, str]]: + return [ + { + "check": str(item["id"]), + "verdict": "pass" if item["passes"] else "fail", + "evidence": f"{item['scenario']} {item['path']}", + } + for item in scenarios + ] + + +def merge( + verdicts_text: str, + scenarios: Sequence[Mapping[str, Any]], + waiting: Mapping[str, str] | None = None, +) -> str: + """verdicts.json with the scenarios' verdicts in it, written by code. + + The validator's verdicts on the Checks are kept as given, but for one thing: + a Check in `waiting`, which names the finding it depends on, is recorded as + waiting on that finding. The overall run is the walk-through: it passes when + every scenario does. + """ + try: + data = json.loads(verdicts_text) + except ValueError: + data = {} + if not isinstance(data, dict): + data = {} + given = [ + ( + {**item, "verdict": WAITING, "finding": waiting[str(item.get("check"))]} + # Only where the validator found the finding stands. One it failed is + # something the operating model can provide itself, and is a fault in it. + if waiting + and str(item.get("check")) in waiting + and item.get("verdict") in ("not_verifiable", WAITING) + else item + ) + for item in data.get("checks") or [] + if isinstance(item, dict) and not str(item.get("check", "")).startswith(PREFIX) + ] + failed = [item for item in scenarios if not item["passes"]] + return json.dumps( + { + "dry_run": { + "verdict": "fail" if failed else "pass", + "evidence": f"{len(scenarios) - len(failed)} of {len(scenarios)} " + "scenario walk-throughs pass.", + }, + "checks": [*given, *as_verdicts(scenarios)], + "proposed_additions": data.get("proposed_additions") or [], + }, + indent=2, + ensure_ascii=False, + ) + + +def render(scenarios: Sequence[Mapping[str, Any]]) -> str: + lines = [ + f"## {HEADING}", + "", + ( + "Each scenario is generated from the operating model and traced through " + "it by code." + ), + "", + "| Scenario | Result | Path |", + "| --- | --- | --- |", + ] + lines += [ + f"| {item['scenario']} | {'pass' if item['passes'] else '**fail**'} | {item['path']} |" + for item in scenarios + ] + return "\n".join(lines) + "\n" + + +def check_verdicts( + attachments: Mapping[str, str], + graph: Mapping[str, Sequence[Mapping[str, Any]]] | None, + name: str, +) -> list[str]: + """What is wrong with the validator's verdicts: one on every Check, none on a + scenario, and never a pass on a Check that depends on a finding.""" + try: + data = json.loads(attachments[name]) + except ValueError as exc: + return [f"{name}: not valid JSON ({exc})"] + if not isinstance(data, dict) or not isinstance(data.get("checks"), list): + return [f"{name}: must be an object with a list of checks"] + problems = [] + given: dict[str, str] = {} + for index, item in enumerate(data["checks"]): + where = f"{name}: checks[{index}]" + if not isinstance(item, dict) or not item.get("check"): + problems.append(f"{where} needs a check") + continue + identifier = str(item["check"]) + if identifier.startswith(PREFIX): + continue # the walk-throughs' own, written by code on an earlier attempt + if item.get("verdict") == WAITING: + # Recorded by code on an earlier attempt; the validator's own verdict + # on such a Check is not_verifiable. + item = {**item, "verdict": "not_verifiable"} + if item.get("verdict") not in ("pass", "fail", "not_verifiable"): + problems.append( + f"{where} ({identifier}): verdict must be pass, fail or not_verifiable" + ) + if not str(item.get("evidence") or "").strip(): + problems.append(f"{where} ({identifier}) needs evidence") + given[identifier] = str(item.get("verdict")) + for check in (graph or {}).get("checks", []): + identifier = str(check["id"]) + if identifier not in given: + problems.append(f"{name}: Check {identifier} has no verdict") + elif check.get("depends_on_finding") and given[identifier] == "pass": + problems.append( + f"{name}: Check {identifier} depends on finding " + f"{check['depends_on_finding']}. It is not_verifiable while the finding " + "stands, or fail when the operating model could meet it itself" + ) + known = {str(check["id"]) for check in (graph or {}).get("checks", [])} + if graph is not None: + problems += [ + f"{name}: {identifier} is not a Check of the Target operating model" + for identifier in given + if identifier not in known + ] + return problems diff --git a/app/operating_model/spec.py b/app/operating_model/spec.py new file mode 100644 index 0000000..3c7ab91 --- /dev/null +++ b/app/operating_model/spec.py @@ -0,0 +1,89 @@ +"""Reading the agreed operating spec, as every Specialist is given it. + +The engine writes the spec's data file from what the scope analyst recorded: what +the customer stated, nested by field; what was assumed for them, as a list; and +which stated fields bind. Code that checks a Deliverable against the spec reads it +here, and a value counts whichever of the two it came from. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from typing import Any + +from app.operating_model.names import OPERATING_SCOPE, SCOPE_DATA_FILE + +Upstream = Mapping[str, Mapping[str, Any]] + + +def agreed(upstream: Upstream | None) -> dict[str, Any]: + """The agreed spec from the Deliverables a submission builds on; {} without one.""" + text = ( + ((upstream or {}).get(OPERATING_SCOPE) or {}) + .get("attachments", {}) + .get(SCOPE_DATA_FILE, "") + ) + try: + data = json.loads(text) if text else {} + except ValueError: + return {} + return data if isinstance(data, dict) else {} + + +def value(scope: Mapping[str, Any], path: str, default: Any = None) -> Any: + """A field's value, stated or assumed; `default` when it is neither.""" + node: Any = scope.get("stated") or {} + for part in path.split("."): + if not isinstance(node, Mapping) or part not in node: + node = None + break + node = node[part] + if node is not None: + return node + for item in scope.get("assumed") or []: + if isinstance(item, Mapping) and item.get("field") == path: + return item.get("value", default) + return default + + +def settled(scope: Mapping[str, Any], path: str) -> bool: + return value(scope, path) is not None + + +def records(scope: Mapping[str, Any], path: str) -> list[dict[str, Any]]: + found = value(scope, path) + return ( + [dict(item) for item in found if isinstance(item, Mapping)] + if isinstance(found, list) + else [] + ) + + +def items(scope: Mapping[str, Any]) -> list[dict[str, Any]]: + """What is to be operated.""" + return records(scope, "solution.items") + + +def conditions(scope: Mapping[str, Any]) -> list[dict[str, Any]]: + """What must be kept true of an Item while it is run.""" + return records(scope, "solution.conditions") + + +def business_outcomes(scope: Mapping[str, Any]) -> list[dict[str, Any]]: + """What the customer's business wants from the solution, in their words: what + every measure of operations is ultimately for.""" + return records(scope, "intent.business_outcomes") + + +def stated_requirements(scope: Mapping[str, Any]) -> list[dict[str, Any]]: + return records(scope, "intent.requirements") + + +def binding(scope: Mapping[str, Any]) -> list[str]: + """The stated fields an operating model must honour.""" + return [str(path) for path in scope.get("binding") or []] + + +def ids(entries: list[dict[str, Any]]) -> list[str]: + return [str(entry["id"]) for entry in entries if entry.get("id")] diff --git a/app/operating_model/staffing.py b/app/operating_model/staffing.py new file mode 100644 index 0000000..194b4e4 --- /dev/null +++ b/app/operating_model/staffing.py @@ -0,0 +1,280 @@ +"""Staffing and cost, computed from the Target operating model and the rate card. + +No agent writes a number of people or a cost. A Specialist says what each Capability +takes in hours a month, which Roles are responsible for it, and where each Role sits +and at what level. Everything else is arithmetic on the service provider's rate card, +so the figures cannot disagree with the model they are about. + + effort the hours a month of the Capabilities a Role is responsible for + people effort over the productive hours of its country; and, where the + Role covers hours wider than a working day, never fewer than the + smallest team that can cover them + cost a month people, at the day rate for its country and level, for the + productive days in a month, with the uplift for its hours, in + the currency the operating spec states +""" + +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import yaml + +from app.operating_model.names import CUSTOMER, HOURS, RESPONSIBLE, wider + +SKILL = "rate-card" +BUSINESS = HOURS[0] + + +class RateCardError(ValueError): + """A rate card that cannot be used; the message lists every fault.""" + + +@dataclass(frozen=True) +class Country: + name: str + productive_days: float + hours_per_day: float + day_rate: Mapping[str, float] + + @property + def productive_hours_a_month(self) -> float: + return self.productive_days * self.hours_per_day / 12 + + +@dataclass(frozen=True) +class RateCard: + name: str + version: str + currency: str # what its rates are written in + conversion: Mapping[str, float] # units of a currency for one of `currency` + conversion_as_of: str + prices: tuple[str, ...] # the kinds of Party it prices + levels: Mapping[str, str] + cover: Mapping[str, Mapping[str, float]] # hours -> minimum_roster, uplift + countries: Mapping[str, Country] + + @property + def is_draft(self) -> bool: + return self.version.startswith("draft") + + def currencies(self) -> list[str]: + return sorted(self.conversion) + + +def _number(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0 + + +def load(skill_dir: Path) -> RateCard: + """Read the rate card from the Skill's folder, and check it holds together.""" + path = skill_dir / "assets" / "rates.yaml" + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + if not isinstance(data, dict): + raise RateCardError("rates.yaml: must be a mapping") + problems = [ + f"rates.yaml: needs {field}" + for field in ("name", "version", "currency", "levels", "cover", "countries") + if not data.get(field) + ] + if problems: + raise RateCardError("; ".join(problems)) + levels = {str(name): str(text) for name, text in data["levels"].items()} + conversion_data = data.get("conversion") or {} + rates = {str(k): v for k, v in (conversion_data.get("rates") or {}).items()} + rates.setdefault(str(data["currency"]), 1.0) + problems += [ + f"rates.yaml: the conversion rate for {name} must be a number above zero" + for name, rate in rates.items() + if not _number(rate) + ] + prices = tuple(str(kind) for kind in data.get("prices") or ()) + if CUSTOMER in prices: + problems.append( + "rates.yaml: `prices` lists customer; the customer's own staff are " + "never priced" + ) + cover = {str(name): dict(entry or {}) for name, entry in data["cover"].items()} + for hours in HOURS: + entry = cover.get(hours) or {} + if not _number(entry.get("minimum_roster")) or not _number(entry.get("uplift")): + problems.append( + f"rates.yaml: cover for {hours} needs a minimum_roster and an " + "uplift, both above zero" + ) + countries = {} + for code, entry in data["countries"].items(): + entry = entry or {} + where = f"rates.yaml: {code}" + if not _number(entry.get("productive_days")) or not _number( + entry.get("hours_per_day") + ): + problems.append(f"{where} needs productive_days and hours_per_day") + continue + day_rate = entry.get("day_rate") or {} + problems += [ + f"{where} has no day rate for level {level}" + for level in levels + if not _number(day_rate.get(level)) + ] + countries[str(code)] = Country( + name=str(entry.get("name") or code), + productive_days=float(entry["productive_days"]), + hours_per_day=float(entry["hours_per_day"]), + day_rate={str(k): float(v) for k, v in day_rate.items() if _number(v)}, + ) + if problems: + raise RateCardError("; ".join(problems)) + return RateCard( + name=str(data["name"]), + version=str(data["version"]), + currency=str(data["currency"]), + conversion={name: float(rate) for name, rate in rates.items()}, + conversion_as_of=str(conversion_data.get("as_of") or ""), + prices=prices, + levels=levels, + cover={ + name: { + "minimum_roster": float(entry["minimum_roster"]), + "uplift": float(entry["uplift"]), + } + for name, entry in cover.items() + if name in HOURS + }, + countries=countries, + ) + + +def shares( + responsibilities: Sequence[Mapping[str, Any]], +) -> tuple[dict[tuple[str, str], float], list[str]]: + """Each responsible Role's share of each Capability's effort, and what is wrong + with how shares were stated. + + The Roles responsible for a Capability divide its effort equally, unless every + one of them states a `share`, and the shares add up to 1. + """ + by_capability: dict[str, list[Mapping[str, Any]]] = {} + for item in responsibilities: + if item.get("kind") == RESPONSIBLE: + by_capability.setdefault(str(item.get("capability")), []).append(item) + result: dict[tuple[str, str], float] = {} + problems = [] + for capability, entries in by_capability.items(): + stated = [e.get("share") for e in entries if e.get("share") is not None] + if not stated: + for entry in entries: + result[(capability, str(entry.get("role")))] = 1 / len(entries) + continue + valid = all( + isinstance(s, (int, float)) and not isinstance(s, bool) and 0 < s <= 1 + for s in stated + ) + if len(stated) != len(entries) or not valid: + problems.append( + f"the Roles responsible for {capability} must all state a share " + "between 0 and 1, or none of them" + ) + continue + total = sum(float(entry["share"]) for entry in entries) + if abs(total - 1) > 0.001: + problems.append(f"the shares of {capability} do not add up to 1") + continue + for entry in entries: + result[(capability, str(entry.get("role")))] = float(entry["share"]) + return result, problems + + +def compute( + graph: Mapping[str, Any], + capabilities: Sequence[Mapping[str, Any]], + card: RateCard, + currency: str, +) -> dict[str, Any]: + """People and cost per Role, and the totals. The graph is taken to be sound: + `graph.check` refuses one whose Roles the card cannot price.""" + parties = {str(p.get("id")): p for p in graph.get("parties") or []} + effort_of = { + str(c["id"]): float(c.get("effort_hours_per_month") or 0) for c in capabilities + } + hours_of = {str(c["id"]): str(c.get("hours") or BUSINESS) for c in capabilities} + share_of, _ = shares(graph.get("responsibilities") or []) + rate = card.conversion[currency] + + rows: list[dict[str, Any]] = [] + by_capability: dict[str, float] = {} + for role in graph.get("roles") or []: + role_id = str(role.get("id")) + country = card.countries[str(role.get("country"))] + own = {cap: share for (cap, who), share in share_of.items() if who == role_id} + effort = float( + sum(effort_of.get(cap, 0.0) * share for cap, share in own.items()) + ) + hours = BUSINESS + for cap in own: + if wider(hours_of.get(cap, BUSINESS), hours): + hours = hours_of[cap] + people = effort / country.productive_hours_a_month + if hours != BUSINESS: + # A rota is sized by the hours it must fill, not by the work done in them. + people = max(people, card.cover[hours]["minimum_roster"]) + party = parties.get(str(role.get("party")), {}) + priced = party.get("kind") != CUSTOMER and party.get("kind") in card.prices + cost = ( + people + * country.day_rate[str(role.get("level"))] + * country.productive_days + / 12 + * card.cover[hours]["uplift"] + * rate + if priced + else None + ) + if cost is not None and effort: + for cap, share in own.items(): + portion = effort_of.get(cap, 0.0) * share / effort + by_capability[cap] = by_capability.get(cap, 0.0) + cost * portion + rows.append( + { + "role": role_id, + "name": str(role.get("name") or role_id), + "party": str(role.get("party")), + "party_name": str(party.get("name") or role.get("party")), + "country": str(role.get("country")), + "level": str(role.get("level")), + "hours": hours, + "effort_hours_per_month": round(effort, 1), + "people": round(people, 1), + "cost_per_month": None if cost is None else round(cost), + } + ) + + def total(key: str) -> dict[str, int]: + sums: dict[str, float] = {} + for row in rows: + if row["cost_per_month"] is not None: + sums[row[key]] = sums.get(row[key], 0.0) + row["cost_per_month"] + return {name: round(amount) for name, amount in sums.items()} + + return { + "currency": currency, + "rate_card": { + "name": card.name, + "version": card.version, + "draft": card.is_draft, + }, + "conversion": { + "from": card.currency, + "rate": rate, + "as_of": card.conversion_as_of, + }, + "roles": rows, + "people": round(sum(row["people"] for row in rows), 1), + "cost_per_month": round(sum(row["cost_per_month"] or 0 for row in rows)), + "by_party": total("party"), + "by_country": total("country"), + "by_capability": {cap: round(amount) for cap, amount in by_capability.items()}, + } diff --git a/app/operating_model/workflow.py b/app/operating_model/workflow.py new file mode 100644 index 0000000..e08c2e8 --- /dev/null +++ b/app/operating_model/workflow.py @@ -0,0 +1,572 @@ +"""The target operating model, as a Workflow for the Execution engine. + +`build(root)` reads the capability catalogue and the rate card from the agent's +Skills and returns the one value that makes the engine about this subject: the +Phases and Deliverables, the checks along the chain, the graph and its views, the +scenario walk-throughs that stand in for a dry run, and what a Change request +changed. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from execution_engine.workflows.deliverables import DeliverableSpec +from execution_engine.workflows.workflow import ( + Bench, + CardText, + ChangePolicy, + ValidationPolicy, + Workflow, +) + +from app.operating_model import catalogue as capability_catalogue +from app.operating_model import chain, digest, graph, guide, scenarios, spec, staffing +from app.operating_model.catalogue import Catalogue +from app.operating_model.names import ( + CAPABILITIES_FILE, + CAPABILITY_MODEL, + CHANGE_REQUEST_FILE, + CHANGES_FILE, + DESIGN, + FINDINGS_FILE, + GRAPH_FILE, + GUIDE, + GUIDE_FILE, + OPERATING_SCOPE, + OPERATIONAL_REQUIREMENTS, + PACKAGING, + PHASES, + REQUIREMENTS, + REQUIREMENTS_FILE, + SCENARIOS_FILE, + SCOPE_DATA_FILE, + SCOPING, + STAFFING_FILE, + TARGET_OPERATING_MODEL, + VALIDATION, + VALIDATION_RESULTS, + VERDICTS_FILE, +) +from app.operating_model.spec import Upstream +from app.operating_model.staffing import RateCard + +ROOT = Path(__file__).resolve().parents[2] # the agent's own folder +SCOPE_FILE = "scope.yaml" # the engine's: the completed operating spec + +ABOUT = ( + "I design target operating models. Give me the solution to be operated, as an " + "operating spec or as a solution architecture to digest into one, and I will " + "work through operational requirements, the Capabilities needed, who is " + "accountable and responsible for each, staffing and cost, and a validation of " + "the model against the events it must handle, asking for your approval at each " + "Phase. Nothing was started for this message." +) + + +def _staffing(deliverables: Upstream) -> dict[str, Any] | None: + text = ( + (deliverables.get(TARGET_OPERATING_MODEL) or {}) + .get("attachments", {}) + .get(STAFFING_FILE, "") + ) + try: + data = json.loads(text) if text else None + except ValueError: + return None + return data if isinstance(data, dict) else None + + +def _findings(deliverables: Upstream) -> list[dict[str, Any]]: + return graph.findings( + (deliverables.get(TARGET_OPERATING_MODEL) or {}).get("attachments", {}) + ) + + +def _waiting(deliverables: Upstream) -> dict[str, str]: + """The Checks that cannot hold until a finding is met, with the finding.""" + text = ( + (deliverables.get(VALIDATION_RESULTS) or {}) + .get("attachments", {}) + .get(VERDICTS_FILE, "") + ) + try: + data = json.loads(text) if text else {} + except ValueError: + return {} + return { + str(item.get("check")): str(item.get("finding")) + for item in (data.get("checks") or [] if isinstance(data, dict) else []) + if isinstance(item, dict) and item.get("verdict") == scenarios.WAITING + } + + +def approval_facts(phase: str, deliverables: Upstream) -> dict[str, Any]: + """This Workflow's own facts for an Approval summary: counts and cost, as data.""" + scope = spec.agreed(deliverables) + model = graph.read(deliverables) + facts: dict[str, Any] = { + "counts": { + name: count + for name, count in ( + ("items", len(spec.items(scope))), + ("conditions", len(spec.conditions(scope))), + ("business_outcomes", len(spec.business_outcomes(scope))), + ("operational_requirements", len(chain.requirements(deliverables))), + ("capabilities", len(chain.capabilities(deliverables))), + ("roles", len(model["roles"]) if model else 0), + ("checks", len(model["checks"]) if model else 0), + ("measures", len(model["measures"]) if model else 0), + ) + if count + } + } + cost = _staffing(deliverables) + if cost: + facts["staffing"] = { + "people": cost["people"], + "cost_per_month": cost["cost_per_month"], + "currency": cost["currency"], + "rate_card": cost["rate_card"], + } + found = _findings(deliverables) + if found: + facts["findings"] = [f"{item['id']}: {item.get('lacks')}" for item in found] + waiting = _waiting(deliverables) + if waiting: + facts["checks_waiting_on_findings"] = waiting + return facts + + +def summary_lines(phase: str, facts: Mapping[str, Any]) -> list[str]: + lines = [] + counts = facts.get("counts") or {} + if counts: + lines.append( + ", ".join( + f"{count} {name.replace('_', ' ')}" for name, count in counts.items() + ) + + "." + ) + cost = facts.get("staffing") + if cost: + lines.append( + f"{cost['people']} people, {cost['cost_per_month']:,} {cost['currency']} " + "a month" + + ( + ", from placeholder prices." + if cost["rate_card"]["draft"] + else f", from rate card version {cost['rate_card']['version']}." + ) + ) + found = facts.get("findings") + if found: + lines.append( + f"{len(found)} findings for whoever owns the design: " + "; ".join(found) + ) + waiting = facts.get("checks_waiting_on_findings") + if waiting: + lines.append( + "Waiting on a finding, and so not yet shown to hold: " + + ", ".join(f"{check} ({finding})" for check, finding in waiting.items()) + + "." + ) + return lines + + +def describe_approval(approval: Mapping[str, Any]) -> str: + changes = approval.get("changes") + if not changes: + return "" + lines = [ + f"This changes the operating model delivered in {approval.get('baseline')}." + ] + lines += [ + f"- {word} {part.replace('_', ' ')}: {', '.join(ids)}" + for part, by_change in changes.items() + for word, ids in ( + ("Added", by_change.get("added")), + ("Removed", by_change.get("removed")), + ("Changed", by_change.get("changed")), + ) + if ids + ] + return "\n".join(lines) + "\n\n" + + +# --- what a Change request changed ----------------------------------------------------- + +_PARTS = { + "operational_requirements": (OPERATIONAL_REQUIREMENTS, REQUIREMENTS_FILE, None), + "capabilities": (CAPABILITY_MODEL, CAPABILITIES_FILE, None), + "roles": (TARGET_OPERATING_MODEL, GRAPH_FILE, "roles"), + "responsibilities": (TARGET_OPERATING_MODEL, GRAPH_FILE, "responsibilities"), + "checks": (TARGET_OPERATING_MODEL, GRAPH_FILE, "checks"), +} + + +def _entries(deliverables: Upstream | None, part: str) -> dict[str, dict[str, Any]]: + kind, name, within = _PARTS[part] + text = ((deliverables or {}).get(kind) or {}).get("attachments", {}).get(name, "") + if not text: + return {} + listed = graph.parse(text)[0][within] if within else chain.objects(text, name)[0] + return {str(item["id"]): item for item in listed if item.get("id")} + + +def change_summary(before: Upstream | None, after: Upstream) -> dict[str, Any]: + """What was added, removed and changed against the Baseline, by id.""" + summary = {} + for part in _PARTS: + was, now = _entries(before, part), _entries(after, part) + summary[part] = { + "added": sorted(set(now) - set(was)), + "removed": sorted(set(was) - set(now)), + "changed": sorted(k for k in set(was) & set(now) if was[k] != now[k]), + } + return summary + + +def changes_empty(summary: Mapping[str, Any]) -> bool: + return not any(ids for part in summary.values() for ids in part.values()) + + +def changes_in_words(summary: Mapping[str, Any]) -> str: + part = summary.get("operational_requirements") or {} + return "\n".join( + f"Operational requirement {identifier} was {word}." + for word in ("added", "changed", "removed") + for identifier in part.get(word) or [] + ) + + +def changes_for(phase_kinds: list[str], summary: Mapping[str, Any]) -> dict[str, Any]: + """The parts of the summary that belong to a Phase's Deliverables.""" + return { + part: change + for part, change in summary.items() + if _PARTS[part][0] in phase_kinds and any(change.values()) + } + + +# --- the Workflow ---------------------------------------------------------------------- + + +def build(root: Path = ROOT) -> Workflow: + """The Workflow, for the capability catalogue and the rate card under `root`.""" + catalogue: Catalogue = capability_catalogue.load( + root / "skills" / capability_catalogue.SKILL + ) + card: RateCard = staffing.load(root / "skills" / staffing.SKILL) + add_views = graph.views(catalogue, card) + # Every provider's table of what kind of thing each product is, as one: the + # digest names an Item's kind before the operating spec says which provider. + kinds: dict[str, list[str]] = {} + for bundle in sorted((root / "providers").glob("*.yaml")): + for kind, words in digest.item_kinds(root, bundle.stem).items(): + kinds.setdefault(kind, []).extend(words) + + def with_graph_views( + bench: Bench, content: str, attachments: dict[str, str] + ) -> tuple[str, dict[str, str]]: + return add_views(content, attachments, bench.upstream) + + def check_verdicts( + attachments: Mapping[str, str], upstream: Upstream | None + ) -> list[str]: + return scenarios.check_verdicts( + attachments, graph.read(upstream), VERDICTS_FILE + ) + + def with_scenarios( + bench: Bench, content: str, attachments: dict[str, str] + ) -> tuple[str, dict[str, str]]: + # The walk-throughs are generated and traced here, and their verdicts + # written into the validator's file: no model judges one. + upstream = bench.upstream or {} + model = graph.read(upstream) + if model is None: + return content, attachments + walked = scenarios.walk( + model, + chain.capabilities(upstream), + spec.agreed(upstream), + catalogue, + _staffing(upstream), + ) + attachments[SCENARIOS_FILE] = json.dumps(walked, indent=2, ensure_ascii=False) + waiting = { + str(check["id"]): str(check["depends_on_finding"]) + for check in model["checks"] + if check.get("depends_on_finding") + } + attachments[VERDICTS_FILE] = scenarios.merge( + attachments[VERDICTS_FILE], walked, waiting + ) + content = content.split(f"\n## {scenarios.HEADING}")[0].rstrip() + return content + "\n\n" + scenarios.render(walked), attachments + + def with_guide_sections( + bench: Bench, content: str, attachments: dict[str, str] + ) -> tuple[str, dict[str, str]]: + return guide.assemble(content, bench.upstream or {}, catalogue), attachments + + def check_ids(upstream: Upstream | None) -> list[str]: + model = graph.read(upstream) + return [str(item["id"]) for item in model["checks"]] if model else [] + + def known_ids(deliverables: Upstream) -> set[str]: + return {str(r["id"]) for r in chain.requirements(deliverables)} | set( + spec.ids(spec.conditions(spec.agreed(deliverables))) + ) + + def approval_details( + phase: str, deliverables: Upstream, baseline: Mapping[str, Any] | None + ) -> dict[str, Any]: + if not baseline: + return {} + in_phase = [ + kind for kind, each in deliverable_specs.items() if each.phase == phase + ] + return { + "changes": changes_for( + in_phase, change_summary(baseline.get("deliverables"), deliverables) + ) + } + + def task_entries( + state: Mapping[str, Any], kind: str + ) -> tuple[dict[str, Any], dict[str, Any]]: + context: dict[str, Any] = { + # Which rate card the cost is worked from; its figures are placeholders + # while its version is a draft. + "rate_card": {"version": card.version, "draft": card.is_draft}, + } + if kind == CAPABILITY_MODEL: + # The catalogue's entries that apply to this operating spec, found by + # code. Where to start: take each, or say why it does not apply. + scope = spec.agreed(state.get("deliverables") or {}) + context["capabilities_proposed"] = catalogue.applicable(scope) + if kind == TARGET_OPERATING_MODEL: + context["rate_card"].update( + countries=sorted(card.countries), levels=dict(card.levels) + ) + return {}, context + + def outputs( + state: Mapping[str, Any], deliverables: Upstream, files: dict[str, str] + ) -> dict[str, Any]: + cost = _staffing(deliverables) + found = _findings(deliverables) + if found: + files[CHANGE_REQUEST_FILE] = change_request(state, found) + return { + "staffing": None + if cost is None + else { + "people": cost["people"], + "cost_per_month": cost["cost_per_month"], + "currency": cost["currency"], + "rate_card": cost["rate_card"], + }, + "findings": [str(item["id"]) for item in found], + "change_request": CHANGE_REQUEST_FILE if found else None, + "capability_sources_verified": catalogue.verified, + } + + deliverable_specs = { + each.kind: each + for each in ( + DeliverableSpec( + OPERATING_SCOPE, + SCOPING, + "scope-analyst", + sections=("Scope statement",), + attachments=(SCOPE_FILE, SCOPE_DATA_FILE), + ), + DeliverableSpec( + OPERATIONAL_REQUIREMENTS, + REQUIREMENTS, + "operations-analyst", + depends_on=(OPERATING_SCOPE,), + sections=("Operational requirements", "What was not stated"), + attachments=(REQUIREMENTS_FILE,), + ), + DeliverableSpec( + CAPABILITY_MODEL, + DESIGN, + "capability-modeller", + depends_on=(OPERATIONAL_REQUIREMENTS,), + sections=("Capabilities", "Sources"), + attachments=(CAPABILITIES_FILE,), + ), + DeliverableSpec( + TARGET_OPERATING_MODEL, + DESIGN, + "operating-model-architect", + depends_on=(CAPABILITY_MODEL,), + # The Specialist writes the reasoning and the graph. On submission + # code adds the responsibility matrix, the staffing and cost and + # the findings to the content, and the organisation chart and the + # staffing figures to the attachments. + sections=("Design rationale",), + attachments=(GRAPH_FILE,), + graph=graph.SHAPE, + ), + DeliverableSpec( + VALIDATION_RESULTS, + VALIDATION, + "validator", + depends_on=(TARGET_OPERATING_MODEL,), + sections=("Check verdicts",), + attachments=(VERDICTS_FILE,), + ), + DeliverableSpec( + GUIDE, + PACKAGING, + "packager", + depends_on=(VALIDATION_RESULTS,), + # The packager writes the summary; code writes every other + # section from the data when the guide is submitted. + sections=guide.SECTIONS, + reads_in_full=( + OPERATING_SCOPE, + OPERATIONAL_REQUIREMENTS, + CAPABILITY_MODEL, + TARGET_OPERATING_MODEL, + VALIDATION_RESULTS, + ), + ), + ) + } + + return Workflow( + name="target-operating-model", + about=ABOUT, + phases=PHASES, + deliverables=deliverable_specs, + final=GUIDE, + final_file=GUIDE_FILE, + data_checks={ + OPERATIONAL_REQUIREMENTS: (REQUIREMENTS_FILE, chain.check_requirements), + CAPABILITY_MODEL: (CAPABILITIES_FILE, chain.check_capabilities(catalogue)), + TARGET_OPERATING_MODEL: ( + GRAPH_FILE, + graph.check(card, catalogue.frameworks), + ), + VALIDATION_RESULTS: (VERDICTS_FILE, check_verdicts), + }, + for_readers={ + TARGET_OPERATING_MODEL: graph.MATRIX_HEADING, + VALIDATION_RESULTS: scenarios.HEADING, + }, + spec_deliverable=OPERATING_SCOPE, + validation=ValidationPolicy( + kind=VALIDATION_RESULTS, + data_file=VERDICTS_FILE, + # A failed Check is a fault in who does what: the graph is revised. One + # that fails twice running is a fault in what the organisation must be + # able to do: the capability model is. + implementation=TARGET_OPERATING_MODEL, + design=CAPABILITY_MODEL, + check_ids=check_ids, + known_ids=known_ids, + ), + on_submit={ + TARGET_OPERATING_MODEL: with_graph_views, + VALIDATION_RESULTS: with_scenarios, + GUIDE: with_guide_sections, + }, + approval_details=approval_details, + approval_facts=approval_facts, + summary_lines=summary_lines, + describe_approval=describe_approval, + agreed=lambda deliverables: json.dumps(chain.requirements(deliverables)), + changes=ChangePolicy( + file=CHANGES_FILE, + summary=change_summary, + is_empty=changes_empty, + in_words=changes_in_words, + ), + subject="a target operating model", + describe_design=lambda deliverables: "\n".join( + f"- {item.get('name')} ({item.get('kind')}, {item.get('product', '')})" + for item in spec.items(spec.agreed(deliverables)) + ), + task_entries=task_entries, + outputs=outputs, + # A solution architecture's data files are read in code. Anything else a + # caller supplies is a document for the scope analyst to read. + digest=lambda sources: digest.architecture(sources, kinds), + card=CardText( + description=( + "Designs a target operating model with you for a solution: " + "operational requirements, the Capabilities needed, who is " + "accountable and responsible, staffing and cost from a rate card, " + "and a validation against the events it must handle." + ), + skill_id="target-operating-model", + skill_name="Target operating model", + tags=("operations", "operating-model"), + examples=( + "Design the operating model for this landing zone, run as a managed service.", + "Who runs this platform, at what cost, from which countries?", + ), + ), + done="The target operating model is complete and approved.", + ) + + +def change_request(state: Mapping[str, Any], found: list[dict[str, Any]]) -> str: + """The findings, as a request to whoever owns the design to change it. + + Written by code from the findings. Where the operating spec was digested from a + solution architecture, it is in the form that agent accepts as a Change request + against the Execution it came from. + """ + values = ((state.get("scope") or {}).get("working") or {}).get("values") or {} + source = str(values.get("metadata.source") or "") + lines = ["# Changes operations needs to the solution", ""] + if source: + lines += [ + ( + f"The operating spec was digested from `{source}`. Send what follows to " + "the solution architecture agent as a Change request against that " + "Execution: the `baseline` is its id, in the same Customer case." + ), + "", + ] + else: + lines += [ + ( + "The operating spec did not come from a solution architecture this " + "agent can name. What follows is for whoever owns the design." + ), + "", + ] + lines += [ + ( + "Operating the solution as designed needs these changes to it. Each was " + "found while designing its operating model; none was designed around." + ), + "", + ] + for item in found: + about = ", ".join( + str(i) for i in item.get("source_ref") or item.get("about") or [] + ) + lines += [ + f"## {item['id']}" + (f": {about}" if about else ""), + "", + f"**What is missing.** {item.get('lacks')}", + "", + f"**What is asked for.** {item.get('proposed')}", + "", + ] + return "\n".join(lines) + + +__all__ = ["FINDINGS_FILE", "ROOT", "build", "change_request", "change_summary"] diff --git a/docs/adr/0001-the-target-operating-model-is-an-explicit-graph.md b/docs/adr/0001-the-target-operating-model-is-an-explicit-graph.md new file mode 100644 index 0000000..7eb4736 --- /dev/null +++ b/docs/adr/0001-the-target-operating-model-is-an-explicit-graph.md @@ -0,0 +1,27 @@ +--- +status: proposed +--- + +# The Target operating model is an explicit graph, and the solution is never changed here + +An operating model written as prose cannot be checked: nobody can say whether every part of the solution has someone to run it, or whether anyone is answerable when it fails at night. We decide that the **Target operating model** is one explicit graph, held as data: Parties, Roles, the Responsibilities between Roles and Capabilities, and the Checks each must meet. It is established and approved before anything is written from it, and what is written from it is a view of it. + +This follows the solution architecture agent's ADR-0005, applied to a different subject. + +## What follows + +- **Completeness is a submission rule.** The graph is refused while a Capability has no single accountable Role, or nobody responsible at the hours it is needed, or while something the customer said must be kept true is carried by no Check. +- **Views are written by code.** The responsibility matrix, the organisation chart, and staffing and cost are rendered or computed from the graph. No model draws or counts. +- **Validation is a verdict per Check,** with scenario walk-throughs in place of a dry run: events generated from the graph and traced through it by code. +- **The solution is never changed.** The Items to be operated and the Conditions on them are the customer's description, or the digest of their architecture. Something operations needs that the solution does not provide is a **finding**: it is recorded, a Check that depends on it waits on it, and it goes back to whoever owns the design. Where the spec was digested from a solution architecture, the findings are drafted as a Change request to that agent. + +## Considered Options + +- **Repair gaps in the solution while designing its operations:** rejected. Two agents would then hold different versions of one design, and the architecture's own Checks would no longer describe what is run. +- **Refuse to proceed while a gap exists:** rejected. Most operating models can be designed around a known, stated gap, and the customer is better served by a model with its findings listed than by none. +- **Hold Capabilities inside the graph:** rejected for now. The capability model is its own Deliverable, so that what must be done is agreed apart from who does it, and a failed Check can reopen one without the other. + +## Consequences + +- A finding is neither a failed Check nor a gap in the model. Nothing is revised for it, and the Execution completes with it listed. +- More is decided before detail is written. A fault in the graph costs a Revision of it and of validation. diff --git a/docs/adr/0002-an-execution-starts-from-an-operating-spec.md b/docs/adr/0002-an-execution-starts-from-an-operating-spec.md new file mode 100644 index 0000000..a247974 --- /dev/null +++ b/docs/adr/0002-an-execution-starts-from-an-operating-spec.md @@ -0,0 +1,26 @@ +--- +status: proposed +--- + +# An Execution starts from an operating spec + +This agent was first conceived as reading the solution architecture agent's files. That would make it usable only after that agent, and tie it to the shape of files that are still changing. We decide that an Execution starts from an **operating spec**: a structured file read against a catalogue in code, complete or partly filled, completed in conversation and approved before anything is derived from it. It is the engine's spec mechanism, with this agent's catalogue. + +The spec carries the solution to be operated inline: its Items, the Conditions on them, and what is known of its build and its risks. It does not point at other files. + +## What follows + +- **Anything supplied is digested into a spec first.** The solution architecture agent's files are one Source, read by a code adapter that keeps every Element and Check with its id. A document is another, drafted by a model with each record quoting where it came from. A sentence is a third. +- **Everything downstream reads only the approved spec.** No Specialist opens an architecture file. Completeness is checked against the spec's Items and Conditions. +- **A digest error is caught at the scoping Approval,** where the customer sees what was digested, and not later. +- **The agent can design an operating model for a solution the architecture agent never saw.** + +## Considered Options + +- **Read the architecture agent's files directly:** rejected by the owner. It is the only way in, and it copies that agent's readers. +- **Keep the inventory in separate files the spec points to:** rejected by the owner. It brings back the dependency the spec removes. + +## Consequences + +- The spec is larger than the architecture agent's scope spec, and the engine gives the agreed spec to every Specialist. The inventory should travel as a data file to the Specialists that use it; until the engine's digest step is built it travels with the rest. +- The digest step is not built. Today an Execution starts from an operating spec that is already written. diff --git a/docs/adr/0003-cost-is-computed-from-the-service-providers-rate-card.md b/docs/adr/0003-cost-is-computed-from-the-service-providers-rate-card.md new file mode 100644 index 0000000..3387a89 --- /dev/null +++ b/docs/adr/0003-cost-is-computed-from-the-service-providers-rate-card.md @@ -0,0 +1,29 @@ +--- +status: proposed +--- + +# Cost is computed from the service provider's rate card + +An operating model that names Roles invites the question of what they cost. A model asked for a cost will give one, and it will be a guess. We decide that staffing and cost are computed by code from the graph and one file, the **rate card**, and that no agent writes a number of people, a rate or a cost. + +## Decisions + +Taken by the owner on 2026-10-13: + +1. **The rate card is a Skill, and it is the actual card.** Its prices are the service provider's: set by the owner, changed in review like any other Skill. A caller cannot name another card and an operating spec cannot state a rate. +2. **Rates are by country and level.** A Role has both. The Specialist chooses them and says why; it never chooses to reach a cost. +3. **The operating spec states the currency.** The card is written in one currency and carries fixed conversion rates with their date. No exchange rate is looked up. A currency the card cannot convert to refuses the start. +4. **The customer's own staff are counted and never priced,** whatever the card says. + +## How it is computed + +- **People** are the effort of the Capabilities a Role is responsible for, over the productive hours of its country. Where the Role covers hours wider than a working day, never fewer than the smallest team that can cover them: a rota is sized by the hours it fills. +- **Cost** is people at the day rate for the Role's country and level, with the uplift for its hours, converted into the spec's currency. +- A cost over a stated budget is shown as over. It is not refused. + +## Consequences + +- Every cost is traceable: the card's version is recorded with the Execution and printed beside every total. +- The card in the repository holds placeholder figures until the owner sets them. Its version says `draft`, and every cost worked from it says it is an illustration. +- The owner's real prices will be readable by whoever can read the repository. +- The effort a Specialist states for a Capability is the one estimate in the chain. The validator judges whether it is plausible; nothing computes it. diff --git a/eval/__init__.py b/eval/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/eval/datasets/cases.json b/eval/datasets/cases.json new file mode 100644 index 0000000..ef80071 --- /dev/null +++ b/eval/datasets/cases.json @@ -0,0 +1,32 @@ +[ + { + "id": "case-01-landing-zone-shared", + "what": "A written operating spec, complete enough to run with nobody to ask.", + "spec": "specs/landing-zone-shared.yaml", + "max_revisions": 2, + "max_minutes": 40 + }, + { + "id": "case-02-from-architecture", + "what": "The solution architecture agent's landing-zone Execution, digested, with who operates it stated and the business outcomes left to the agent.", + "sources": "../../tests/fixtures/architecture/case-08-landing-zone", + "overlay": { + "sourcing": { + "model": "provider_operated", + "delivery_countries": [ + "GB", + "PL", + "IN" + ] + }, + "run": { + "pre_approved": true + }, + "intent": { + "business_outcomes": "decide" + } + }, + "max_revisions": 3, + "max_minutes": 60 + } +] diff --git a/eval/datasets/specs/landing-zone-shared.yaml b/eval/datasets/specs/landing-zone-shared.yaml new file mode 100644 index 0000000..1dd8d31 --- /dev/null +++ b/eval/datasets/specs/landing-zone-shared.yaml @@ -0,0 +1,53 @@ +# A fictional benchmark: a small landing zone run as a shared service. No customer's +# data is in it. Complete enough to run with nobody to ask. +metadata: + customer_case: bench-landing-zone-shared +engagement: + kind: platform_foundation +intent: + statement: | + We want our landing zone run as a shared service from go-live. A provider should + respond to faults at every hour, and our own platform team keeps the final say + over changes to the network. + business_outcomes: + - {id: BO-1, outcome: New payment products reach customers in weeks rather than months., target: "A change reaches production within a day of approval."} + - {id: BO-2, outcome: Payments are never unavailable for long., target: "Restored within 4 hours."} + - {id: BO-3, outcome: We keep the platform engineers we hire.} + requirements: + - {id: REQ-01, statement: Our platform team approves every change to the network.} + - {id: REQ-02, statement: We receive a monthly report of service levels and cost.} +solution: + cloud_provider: gcp + items: + - {id: S-1, name: Hub network and interconnect, kind: network, product: Cloud Interconnect} + - {id: S-2, name: Audit log store, kind: storage, product: Cloud Storage} + - {id: S-3, name: Deployment pipeline, kind: pipeline, product: Cloud Build} + - {id: S-4, name: Encryption keys, kind: keys, product: Cloud KMS} + conditions: + - {id: K-1, "on": [S-1], condition: The platform is available at least 99.9% of each month.} + - {id: K-2, "on": [S-2], condition: Audit logs cannot be deleted for seven years.} + - {id: K-3, "on": [S-4], condition: Encryption keys are rotated every 90 days.} + build: + duration_days: 36 +operations: + support_hours: 24x7 + availability_hours: 24x7 + change_process: decide +sourcing: + model: shared + delivery_countries: [GB, PL, IN] +organisation: + existing_teams: + - {name: Platform team, party: customer, hours: business, country: GB} +service_management: + tooling: decide +service_levels: + priority_1_response_minutes: 15 + recovery_time_hours: 4 +compliance: + frameworks: [DORA] +commercial: + currency: GBP + monthly_run_budget: 60000 +run: + pre_approved: true diff --git a/eval/operating_model_eval.py b/eval/operating_model_eval.py new file mode 100644 index 0000000..b9cb36c --- /dev/null +++ b/eval/operating_model_eval.py @@ -0,0 +1,208 @@ +"""Evaluate the target operating model agent on benchmark cases with real models. + +Each case runs one pre-approved Execution through the full assembly (the ADK runtime +and the agent's own Skills and role files), then scores it with eval/scoring.py. +It costs model usage: run it deliberately. + +Usage: + GOOGLE_GENAI_USE_ENTERPRISE=true GOOGLE_CLOUD_PROJECT= GOOGLE_CLOUD_LOCATION=global \\ + .venv/bin/python -m eval.operating_model_eval [case-id ...] + +The model settings are read from the environment or from `.env`. With no case named, +every case in eval/datasets/cases.json runs, one after another. Results are written +under eval/results/