diff --git a/.agents/plans/README.md b/.agents/plans/README.md index 6b4b318..85b4155 100644 --- a/.agents/plans/README.md +++ b/.agents/plans/README.md @@ -9,8 +9,8 @@ One directory per iteration of this agent. Each holds the plan for that iteratio | 1 | `wf-decompose-65ea49fc2d04/` | complete, superseded | The decomposer's scaffold: a linear four-phase pipeline of template functions. Its code was deleted in iteration 3 and survives only in git history. Generated by the platform decomposer; leave its files as written. | | 2 | `iter-02-solution-architect-agent/` | complete; step 9 was done in iteration 3 | The rebuild as a hierarchical ReAct agent: Orchestrator, one Specialist per Deliverable, Skills, provider bundle, A2A. | | 3 | `iter-03-kab-layout/` | complete (commit `7ee49ab`) | The container starts the new engine, the engine moved into the KAB workflow-agent folder layout, and the original pipeline was deleted. | -| 4 | `iter-04-learning-and-skills/` | steps 1–8, 11, 13 and 15–27 built; cases 01, 02 and 03 completed at 100% on them on 2026-10-07 and 08; ADR-0004 accepted 2026-10-08 (step 9); the lessons loop is built and its first six lessons are approved and published, not yet proved (step 10); overlay Skills built, not yet run (step 12); 14 deferred until a second Cloud provider | The enhancement backlog from the 2026-10-06 review: cheap fixes, the learning loop through Skills, overlay Skills, provider neutrality. Steps 17–20 are the speed-ups from the 2026-10-07 run: a working copy for files and the graph, a provider schema lookup, routing a Check the code cannot meet, and a model timeout per role. | -| 5 | `iter-05-jev-decisions/` | the router, intake, the reviews, pillar selection, Skill routing, the guardrail, meaning-aware change summaries and an assessment of every Execution are built (steps 2–5, 8, 10–12, 14, 15, 19), none of the newest yet run in a case; model tier, context selection, compaction, the cloud-read guard and readings as a node are planned (6, 7, 9, 13, 18) | Jev (a hosted decision model) for the agent's closed decisions: checks, routing, context selection. | +| 4 | `iter-04-learning-and-skills/` | steps 1–8, 11, 13 and 15–27 built; cases 01, 02 and 03 completed at 100% on them on 2026-10-07 and 08; ADR-0004 accepted 2026-10-08 (step 9); the lessons loop is built and its first six lessons are approved and published, with a first proof on case 03: no Revision, 22 minutes against 28 (step 10); overlay Skills built and run in case 06 (step 12); 14 deferred until a second Cloud provider | The enhancement backlog from the 2026-10-06 review: cheap fixes, the learning loop through Skills, overlay Skills, provider neutrality. Steps 17–20 are the speed-ups from the 2026-10-07 run: a working copy for files and the graph, a provider schema lookup, routing a Check the code cannot meet, and a model timeout per role. | +| 5 | `iter-05-jev-decisions/` | the router, intake, the reviews, pillar selection, Skill routing, the guardrail, meaning-aware change summaries and an assessment of every Execution are built (steps 2–5, 8, 10–12, 14, 15, 19), the newest ran in cases 05 and 06 on 2026-10-09; a lighter model for a first attempt was built, measured and left off (6); context selection and compaction were decided against on the numbers (7, 9); the cloud-read guard and readings as a node are planned (13, 18) | Jev (a hosted decision model) for the agent's closed decisions: checks, routing, context selection. | | 6 | `iter-06-target-architecture-graph/` | complete: all four cases pass with real models, each run twice on 2026-10-08; thresholds set | The Target architecture as one explicit graph of Elements, Relationships and Checks, established before infrastructure code, with validation as a verdict per Check. | | 7 | `iter-07-change-requests/` | complete (2026-10-09): the change case ran with real models and passed | A user restates a request to say how requirements changed; a new Execution starts from a Baseline Execution of the same Customer case and delivers the updated architecture with what changed. | diff --git a/.agents/plans/iter-04-learning-and-skills/MAP.md b/.agents/plans/iter-04-learning-and-skills/MAP.md index 435732b..19d7410 100644 --- a/.agents/plans/iter-04-learning-and-skills/MAP.md +++ b/.agents/plans/iter-04-learning-and-skills/MAP.md @@ -28,7 +28,7 @@ | 9 | ADR: the lessons loop and overlay Skills | B | S | complete: ADR-0004 accepted 2026-10-08 | | 10 | Lessons Skill and lesson curator | B | L | the loop is built; six lessons proposed from 21 Executions, approved by the owner and published on 2026-10-09; not yet proved by an evaluation with and without (`steps/step-10.md`) | | 11 | IaC patterns Skill | B | M | built; in case 01 the IaC author fell from 57 requests to 11 with no Revision; other cases unmeasured | -| 12 | Overlay Skills and an organisation standards Skill | C | M | built, with an example standards Skill and an evaluation case; not run with real models | +| 12 | Overlay Skills and an organisation standards Skill | C | M | built, with an example standards Skill; case 06 passed with it on 2026-10-09 and the code follows the standards | | 13 | Trim Specialist context | C | M | built; measured on case 01 only, together with steps 11 and 17 | | 14 | Provider-owned contracts | C | M | deferred by its own plan until a second Cloud provider is planned (ADR-0002) | | 15 | Group Executions by Customer case | added | S | complete | diff --git a/.agents/plans/iter-04-learning-and-skills/steps/step-10.md b/.agents/plans/iter-04-learning-and-skills/steps/step-10.md index d14c1df..3284a14 100644 --- a/.agents/plans/iter-04-learning-and-skills/steps/step-10.md +++ b/.agents/plans/iter-04-learning-and-skills/steps/step-10.md @@ -50,3 +50,26 @@ The owner approved all six on 2026-10-09 and they were published: five for the t Still open: retiring lessons when the upstream Skills change is not automated, and no lesson has been proved. +## Run on 2026-10-09 + +Case 06 ran with the six lessons loaded and still needed one Revision of the Target architecture, as case 03 did before the lessons existed. That neither proves nor disproves them: it also carried an overlay. `prove` has not been run. + +## L-04 retired on 2026-10-09 + +L-04 told the target architect to word Checks so the schema review would not mistake a value for a setting. The review was at fault: it read `cloudsql.iam_authentication` (a database flag) and `run.googleapis.com` (a service) as settings, and raised both again in case 06 with the lesson loaded. It now judges a dotted path only when the path starts at a name the resource type has (`_is_missing` in `app/tools/schema_review.py`). The lesson is removed from the Skill; five remain. + +## First proof, 2026-10-09 + +Case 03, one run each way, side by side, with the five lessons that remain (`20261009T195603Z` without, `20261009T195607Z` with): + +| | Without lessons | With lessons | +|---|---|---| +| Score | 100% | 100% | +| Revisions | 1, of the Target architecture | 0 | +| Minutes | 28 | 22 | +| Tokens | 3.00 million | 2.32 million | +| Cost, about | $3.47 | $2.44 | + +`prove` recorded this in the proposals as "kept". One pair suggests a direction and does not settle it: a third run the same evening, with lessons, did need a Revision of the Target architecture. `prove` also wrote to L-04's proposal, which is retired; it should skip retired lessons. + +Neither run raised the schema review's false alarm, which had been raised in case 06 before the fix. diff --git a/.agents/plans/iter-04-learning-and-skills/steps/step-12.md b/.agents/plans/iter-04-learning-and-skills/steps/step-12.md index 7ec4f83..2dc9a4b 100644 --- a/.agents/plans/iter-04-learning-and-skills/steps/step-12.md +++ b/.agents/plans/iter-04-learning-and-skills/steps/step-12.md @@ -17,3 +17,13 @@ Differs from the plan: `case-06-organisation-standards` in the evaluation dataset runs case 03's request under the example overlay. It has not been run. Tests: `tests/execution/test_overlay_skills.py` (8). + +## Run on 2026-10-09 + +`case-06-organisation-standards` (results `20261009T171333Z`): completed, 100%, 36 minutes, 3.66 million tokens, one Revision of the Target architecture. `execution.json` records the overlay with its hash. The code follows it: every resource is named `${var.workload}-${var.environment}-`, the only region is `europe-west1`, and the labels are set. The same request without the overlay last took 22 minutes and 2.32 million tokens (`20261009T103827Z`); one run each, so the difference is not attributed. + +No evaluation check scores whether an overlay was followed; this was read from the code by hand. + +## An evaluation check, later on 2026-10-09 + +A case may now carry `code_standards`: regular expressions the infrastructure code must contain or must not, scored as `code_follows_standards`. Case 06 has six: only approved regions, names carrying the environment from a variable, and the four labels. Run against the code case 06 produced earlier today, it passes. diff --git a/.agents/plans/iter-05-jev-decisions/MAP.md b/.agents/plans/iter-05-jev-decisions/MAP.md index ff2b8cc..620b81e 100644 --- a/.agents/plans/iter-05-jev-decisions/MAP.md +++ b/.agents/plans/iter-05-jev-decisions/MAP.md @@ -2,7 +2,7 @@ **Iteration:** 5, Jev decisions **Intent:** Use Jev, a hosted decision model, for the agent's closed questions (yes or no, pick one, rate on a scale) so that large-model calls are kept for writing and reasoning. The aims are fewer tokens, faster turns, and checks the engine cannot make with string matching. -**Status:** steps 2 to 5, 8, 10 to 12, 14, 15 and 19 are built or settled (2026-10-09); 16 awaits the owner's labels; 6, 7, 9, 13 and 18 are planned; 1 and 17 await the owner +**Status:** steps 2 to 12, 14, 15 and 19 are built or settled (2026-10-09); 16 awaits the owner's labels; 13 and 18 are planned; 1 and 17 await the owner **This directory covers:** steps 1 to 19 ## What Jev is @@ -77,19 +77,19 @@ Jev answers closed questions. Writing Deliverables, grounding product choices an |---|---|---|---| | 1 | Resolve the open questions | none | open, owner's input | | 2 | Decider interface and Jev client with limits | none | built on the TypeSafe SDK; unverified against the live service | -| 3 | Pillars a Revision touches | J3 | built, as a graded question; calibrated live; not run in an Execution | +| 3 | Pillars a Revision touches | J3 | built, as a graded question; calibrated live; ran in cases 05 and 06 on 2026-10-09 | | 4 | Semantic checks at submission | J4 | two reviews built, both off until a full case has run with them: coverage of the Target architecture, one choice per condition; and requirements name no products (`steps/step-4.md`) | | 5 | Finding-to-Deliverable routing | J2 | superseded: iteration 6 gives a verdict per Check, and the router sends a failed Check to the code and a gap to the Target architecture | -| 6 | Model tier per delegation | J5 | planned; held until the cases have run on this tranche, so its effect on quality can be told apart | -| 7 | Context selection | J6 | planned; held for the same reason as step 6 | -| 8 | Skill routing | J9 | built and on: the references a task needs are named in the Specialist's context; calibrated live; not run in an Execution (`steps/step-8.md`) | -| 9 | In-run context compaction | J10 | planned; held for the same reason as step 6 | +| 6 | Model tier per delegation | J5 | built and measured: Flash for the IaC author's first attempt saved five cents and took twice the requests, so it stays off (`steps/step-6.md`) | +| 7 | Context selection | J6 | decided against as planned: earlier Deliverables are a tenth of the describer's prompt (`steps/step-7.md`) | +| 8 | Skill routing | J9 | built and on: the references a task needs are named in the Specialist's context; calibrated live; ran in cases 05 and 06 on 2026-10-09 (`steps/step-8.md`) | +| 9 | In-run context compaction | J10 | decided against as planned: it would lose the prompt cache and cost more (`steps/step-9.md`) | | 10 | Evaluation rubric and lesson triage | J8 | built: the judged rubric (step 16) and triage of evidence a lesson already covers (iteration 4 step 10) | | 11 | ADR: Jev is the Orchestrator's decision point | none | complete: ADR-0007 accepted 2026-10-07 | | 12 | The Orchestrator routes by readings and a playbook; a model only writes replies | J1 | built, including intake of a message that starts something (`steps/step-12.md`); the three faults case 04 found are fixed; a completed interactive run is outstanding | | 13 | Cloud-read call guard | J7 | planned; cloud reads are off in every case, so it could not be exercised | -| 14 | Input guardrail | J11 | built and on: an MCP reply that instructs the agent is withheld, the user's text is flagged; calibrated live; not run in an Execution | -| 15 | Changes of meaning in a Change request | none | built: reworded items are listed apart in changes.json; calibrated live; not run in a Change request | +| 14 | Input guardrail | J11 | built and on: an MCP reply that instructs the agent is withheld, the user's text is flagged; calibrated live; ran in cases 05 and 06 on 2026-10-09 | +| 15 | Changes of meaning in a Change request | none | built: reworded items are listed apart in changes.json; calibrated live; ran in case 05, where nothing was reworded | | 16 | Jev as the evaluation judge, calibrated | none | judge and calibration built; stability and agreement between judges measured (`steps/step-16.md`); agreement with a person's labels awaits the owner's 32 labels | | 17 | Trace every decision | none | open, owner's decision on LangSmith | | 18 | Readings as their own graph step | none | planned; it splits the Orchestrator turn that case 04 only just passes, to save a repeated Jev call after a crash | diff --git a/.agents/plans/iter-05-jev-decisions/steps/step-14.md b/.agents/plans/iter-05-jev-decisions/steps/step-14.md index c1fbf2c..8e61637 100644 --- a/.agents/plans/iter-05-jev-decisions/steps/step-14.md +++ b/.agents/plans/iter-05-jev-decisions/steps/step-14.md @@ -14,3 +14,7 @@ Measured against live Jev on ten texts: product documentation, how-to steps writ The decider's budget is counted per Execution through a context variable set when a Specialist starts. Tests: `tests/tools/test_guardrail.py` (7). + +## Run on 2026-10-09 + +Ran in cases 05 and 06: nothing was withheld and no risk was flagged, in two Executions' worth of documentation lookups. No false alarm; nothing in these runs tried to instruct the agent, so the catching side is still only calibrated. diff --git a/.agents/plans/iter-05-jev-decisions/steps/step-15.md b/.agents/plans/iter-05-jev-decisions/steps/step-15.md index aa4a87f..fac376b 100644 --- a/.agents/plans/iter-05-jev-decisions/steps/step-15.md +++ b/.agents/plans/iter-05-jev-decisions/steps/step-15.md @@ -25,3 +25,7 @@ At 0.25 the three rewordings are read as such and no real change is. The dropped Differs from the plan: requirements are not asked about. They carry their own `change` and reason, and an `unchanged` one must keep its text exactly (iteration 7 step 3). The Baseline's verdict is not carried forward for a reworded Check: validation still gives a verdict on every Check. Tests: `tests/execution/test_change_requests.py`, "changed, or only reworded?" (2). + +## Run on 2026-10-09 + +Ran in case 05's Change request. The Specialists changed no existing item's text: Checks and recommendations were added and removed by id, so there was nothing to settle and nothing listed as reworded. diff --git a/.agents/plans/iter-05-jev-decisions/steps/step-19.md b/.agents/plans/iter-05-jev-decisions/steps/step-19.md index 83ffd9a..5f659c3 100644 --- a/.agents/plans/iter-05-jev-decisions/steps/step-19.md +++ b/.agents/plans/iter-05-jev-decisions/steps/step-19.md @@ -16,3 +16,7 @@ Differs from the plan: The judge behind it was measured in step 16: Jev answered the same question the same way every time (1.00) and agreed with Gemini on 0.91 of questions. Agreement with a person still awaits the owner's 32 labels. Tests: `tests/execution/test_assessment.py` (5). + +## Run on 2026-10-09 + +Both Executions carry an assessment in `execution.json`: case 06 a mean of 0.74 (specific 0.76, actionable 0.77, trade-offs 0.75, traceable 0.69); case 05's Change request 0.90. Both meet the bar of 0.67. diff --git a/.agents/plans/iter-05-jev-decisions/steps/step-3.md b/.agents/plans/iter-05-jev-decisions/steps/step-3.md index 55c7802..8bfdf93 100644 --- a/.agents/plans/iter-05-jev-decisions/steps/step-3.md +++ b/.agents/plans/iter-05-jev-decisions/steps/step-3.md @@ -21,3 +21,7 @@ Differs from the plan: the plan asked a yes or no per pillar. Measured against l Operations and cost are picked for almost every real change, so the saving is two to five advisors of six, not five of six. Tests: `tests/execution/test_pillars.py` (3), `tests/execution/test_change_requests.py` (2). + +## Run on 2026-10-09 + +`case-05-change-retention` (results `20261009T171336Z`): the Change request consulted 5 pillar advisors, not 6, which is what calibration predicted for this change. The Change request took about 18 minutes and 1.30 million tokens against the Baseline's 30 minutes and 3.09 million (42%), passed every change check, and changed nothing unrequested (0 of 89). diff --git a/.agents/plans/iter-05-jev-decisions/steps/step-6.md b/.agents/plans/iter-05-jev-decisions/steps/step-6.md new file mode 100644 index 0000000..54a5095 --- /dev/null +++ b/.agents/plans/iter-05-jev-decisions/steps/step-6.md @@ -0,0 +1,37 @@ +# Step 6 — Implementation Detail + +Built on 2026-10-09, **off by default** (`MODEL_TIERS`) until measured. + +Where case 06's cost went (`20261009T171333Z`, about $3.84 at the prices in the plan map): + +| Role | Model | Requests | Prompt tokens | Of them cached | Cost | +|---|---|---|---|---|---| +| validator | Pro | 22 | 1,025,080 | 82% | $1.04 | +| target-architect | Pro | 24 | 686,263 | 69% | $1.02 | +| iac-author | Pro | 16 | 434,234 | 60% | $0.63 | +| describer | Flash | 13 | 675,284 | 86% | $0.16 | +| packager | Flash | 4 | 233,373 | 72% | $0.17 | +| the other six | mixed | 38 | 371,088 | | $0.82 | + +The three roles on the Pro model are 70% of the cost. For each, output and thinking tokens cost about as much as the prompt. + +1. `models.first_attempt` in the provider bundle names a lighter model for a role's first attempt at a Deliverable. With `MODEL_TIERS` on, the role takes it when nothing has been asked of the Deliverable again: there is no feedback on it, and no earlier attempt ended without one. Any other attempt takes the role's own model. +2. Only the IaC author is named, on Flash. Its work is checked twice by something else: `terraform validate`, and the validator. +3. The validator is deliberately not named, though it costs the most. It is the check on the code. A weaker validator that passed everything would raise the evaluation's score, so the evaluation could not tell a saving from a loss. +4. The target architect is not named: its mistakes are what cause Revisions, which cost far more than its own tokens. +5. Usage is reported per role and model, so a run shows what each tier used. + +Differs from the plan: **there is no Jev question.** The plan asks Jev to pick a tier from the task. There is no data yet saying which tasks a lighter model handles, so any threshold would be a guess. The rule is the plan's second sentence alone: a Revision after a failed attempt goes to the stronger tier. + +Expected saving if quality holds: about $0.40 of a $3.84 run, roughly 10%. Tests: `tests/execution/test_model_tiers.py` (3). + +## Measured on 2026-10-09: no saving, so it stays off + +Case 03, one run each, side by side (`20261009T195607Z` without tiers, `20261009T195611Z` with): + +| | Pro throughout | Flash for the first attempt | +|---|---|---| +| IaC author's first attempt | 17 requests, $0.44 | 35 requests, $0.39 | +| Case result | 100% | 100% | + +Flash wrote code that validated and passed every Check, but took twice the requests to get there: 35 of its budget of 40, and 336 seconds. Five cents saved, at the edge of the tool budget, is not a saving worth the risk of an attempt that runs out. `MODEL_TIERS` stays off. The run with tiers also had a Revision of the Target architecture that the other did not, which is unrelated to the tier and makes the whole-run totals ($3.02 against $2.44) not comparable. diff --git a/.agents/plans/iter-05-jev-decisions/steps/step-7.md b/.agents/plans/iter-05-jev-decisions/steps/step-7.md new file mode 100644 index 0000000..e3dffe6 --- /dev/null +++ b/.agents/plans/iter-05-jev-decisions/steps/step-7.md @@ -0,0 +1,11 @@ +# Step 7 — Decision + +Decided on 2026-10-09: **not to be built as planned.** The numbers do not support it. + +The plan passes only the relevant sections of earlier Deliverables to the describer and the packager, to cut their tokens. + +- **The describer's prompt is not earlier Deliverables.** In case 06 it was given the Target architecture, about 7 thousand tokens, and its prompt averaged 52 thousand per request. The rest is its Skill and the results of seven documentation searches, which accumulate. Trimming the Target architecture would save about a tenth of its prompt. +- **The packager needs what it is given.** It assembles the guide from every Deliverable's prose, in four requests. +- **Both are on Flash.** Together they cost $0.33 of a $3.84 run, and 80% of their prompt is cached at a tenth of the price. + +What would cut the describer's tokens is fewer or shorter documentation searches, which is a change to its role file, not a Jev decision. diff --git a/.agents/plans/iter-05-jev-decisions/steps/step-8.md b/.agents/plans/iter-05-jev-decisions/steps/step-8.md index 164db1d..92b7134 100644 --- a/.agents/plans/iter-05-jev-decisions/steps/step-8.md +++ b/.agents/plans/iter-05-jev-decisions/steps/step-8.md @@ -25,3 +25,15 @@ Differs from the plan: - **Not measured against its acceptance criterion.** Whether `load_skill_resource` calls fall needs a case run with it and one without. Tests: `tests/execution/test_skill_routing.py` (5). + +## Run on 2026-10-09 + +Ran in cases 05 and 06. The IaC author was named 7 references for the web application (service, SQL, IAM, load balancer, networking, project services, secrets) and 9 for the event-driven service; on the Change request, 3 (KMS, monitoring, Pub/Sub). Both cases passed. Whether `load_skill_resource` calls fell was not measured. + +One to look at: for the retention change, whose subject is a bucket, the Cloud Storage reference was not among the three named. + +## Corrected and changed later on 2026-10-09 + +The "one to look at" above was misread. The three references (KMS, monitoring, Pub/Sub) were named for a Revision inside the Baseline, from that Revision's feedback. The Change request itself was named nine, Cloud Storage among them. + +What it did show is that a Change request's question listed every Element and not what changed. The task now says what has changed since delivery. Against live Jev on case 05's change, Cloud Storage rose from 1.63 to 1.95, the highest of all, and the load balancer, monitoring and Secret Manager patterns fell below the threshold. diff --git a/.agents/plans/iter-05-jev-decisions/steps/step-9.md b/.agents/plans/iter-05-jev-decisions/steps/step-9.md new file mode 100644 index 0000000..a6e95e4 --- /dev/null +++ b/.agents/plans/iter-05-jev-decisions/steps/step-9.md @@ -0,0 +1,11 @@ +# Step 9 — Decision + +Decided on 2026-10-09: **not to be built as planned.** It would be likely to raise cost, not lower it. + +The plan drops or truncates old tool results inside one agent run before each model request. + +- **The prompt is already mostly cached.** In case 06 the validator's prompt was 82% cached, the target architect's 69% and the IaC author's 60%, and a cached token costs a tenth of an uncached one. +- **Caching depends on the prompt's beginning staying the same.** Removing or shortening an earlier tool result changes everything after it, so the request that follows is paid at the full price. +- **The arithmetic.** A request of 46 thousand tokens, 82% cached, costs about $0.024 on the Pro model. Cut to 30 thousand with the cache lost, it costs $0.060. + +It could still help where a run nears the model's context limit, which no run has. The cost that remains in these roles is output and thinking tokens, and the number of requests; neither is what this step addresses. diff --git a/.env.example b/.env.example index fcbae7f..ff6f535 100644 --- a/.env.example +++ b/.env.example @@ -43,6 +43,8 @@ SPECIALIST_BUDGET=15 CONFIRM_DRAFTS=true # Give Specialists the lessons Skill. Set false to run without anything learned. LESSONS_ENABLED=true +# Give a role the lighter model its provider bundle names for a first attempt. +MODEL_TIERS=false # Overlay Skills: an organisation's standards or one engagement's constraints. The # folders they live in, and the ones every Execution loads (comma-separated names). OVERLAY_SKILLS_DIRS=skills/overlays diff --git a/.scratch/lessons/issues/01-create-specific-checks-for-all-constraints.md b/.scratch/lessons/issues/01-create-specific-checks-for-all-constraints.md index 7611d74..58c8b1e 100644 --- a/.scratch/lessons/issues/01-create-specific-checks-for-all-constraints.md +++ b/.scratch/lessons/issues/01-create-specific-checks-for-all-constraints.md @@ -20,7 +20,15 @@ Seen in 14 of 21 Executions, 58 times in all: validation proposed a Check the de ## Proof -Not yet measured. +Evaluation without lessons: `20261009T195603Z`; with: `20261009T195607Z`. + +| Case | Score | Revisions | Minutes | Tokens (M) | +| --- | --- | --- | --- | --- | +| case-03-internal-web-app | 1.00 → 1.00 | 1 → 0 | 28 → 22 | 3.00 → 2.32 | + +Quality held and revisions, seconds, tokens fell: kept. + +This compares every published lesson together, on one run each way. Runs of the same case vary, so one pair suggests a direction and does not settle it. ## Comments diff --git a/.scratch/lessons/issues/02-assign-checks-to-every-element.md b/.scratch/lessons/issues/02-assign-checks-to-every-element.md index 74360f2..10e9b79 100644 --- a/.scratch/lessons/issues/02-assign-checks-to-every-element.md +++ b/.scratch/lessons/issues/02-assign-checks-to-every-element.md @@ -20,7 +20,15 @@ Seen in 13 of 21 Executions, 13 times in all: submit_deliverable refused: Not su ## Proof -Not yet measured. +Evaluation without lessons: `20261009T195603Z`; with: `20261009T195607Z`. + +| Case | Score | Revisions | Minutes | Tokens (M) | +| --- | --- | --- | --- | --- | +| case-03-internal-web-app | 1.00 → 1.00 | 1 → 0 | 28 → 22 | 3.00 → 2.32 | + +Quality held and revisions, seconds, tokens fell: kept. + +This compares every published lesson together, on one run each way. Runs of the same case vary, so one pair suggests a direction and does not settle it. ## Comments diff --git a/.scratch/lessons/issues/03-verify-resource-arguments-for-checks.md b/.scratch/lessons/issues/03-verify-resource-arguments-for-checks.md index e8236e2..c4808a3 100644 --- a/.scratch/lessons/issues/03-verify-resource-arguments-for-checks.md +++ b/.scratch/lessons/issues/03-verify-resource-arguments-for-checks.md @@ -20,7 +20,15 @@ Seen in 3 of 21 Executions, 3 times in all: revised because a Check named someth ## Proof -Not yet measured. +Evaluation without lessons: `20261009T195603Z`; with: `20261009T195607Z`. + +| Case | Score | Revisions | Minutes | Tokens (M) | +| --- | --- | --- | --- | --- | +| case-03-internal-web-app | 1.00 → 1.00 | 1 → 0 | 28 → 22 | 3.00 → 2.32 | + +Quality held and revisions, seconds, tokens fell: kept. + +This compares every published lesson together, on one run each way. Runs of the same case vary, so one pair suggests a direction and does not settle it. ## Comments diff --git a/.scratch/lessons/issues/04-avoid-using-values-as-setting-names.md b/.scratch/lessons/issues/04-avoid-using-values-as-setting-names.md index aaf13fd..c11e0f8 100644 --- a/.scratch/lessons/issues/04-avoid-using-values-as-setting-names.md +++ b/.scratch/lessons/issues/04-avoid-using-values-as-setting-names.md @@ -20,8 +20,18 @@ Seen in 2 of 21 Executions, 4 times in all: submit_deliverable refused: Not subm ## Proof -Not yet measured. +Evaluation without lessons: `20261009T195603Z`; with: `20261009T195607Z`. + +| Case | Score | Revisions | Minutes | Tokens (M) | +| --- | --- | --- | --- | --- | +| case-03-internal-web-app | 1.00 → 1.00 | 1 → 0 | 28 → 22 | 3.00 → 2.32 | + +Quality held and revisions, seconds, tokens fell: kept. + +This compares every published lesson together, on one run each way. Runs of the same case vary, so one pair suggests a direction and does not settle it. ## Comments Approved by the owner on 2026-10-09. + +- 2026-10-09: retired. The schema review no longer reads a value written with dots (a database flag, a service name) as a setting, so the rule has nothing left to work around. diff --git a/.scratch/lessons/issues/05-include-required-arguments-in-alert-policies.md b/.scratch/lessons/issues/05-include-required-arguments-in-alert-policies.md index 937641a..399b34c 100644 --- a/.scratch/lessons/issues/05-include-required-arguments-in-alert-policies.md +++ b/.scratch/lessons/issues/05-include-required-arguments-in-alert-policies.md @@ -20,7 +20,15 @@ Seen in 2 of 21 Executions, 2 times in all: terraform_validate refused: terrafor ## Proof -Not yet measured. +Evaluation without lessons: `20261009T195603Z`; with: `20261009T195607Z`. + +| Case | Score | Revisions | Minutes | Tokens (M) | +| --- | --- | --- | --- | --- | +| case-03-internal-web-app | 1.00 → 1.00 | 1 → 0 | 28 → 22 | 3.00 → 2.32 | + +Quality held and revisions, seconds, tokens fell: kept. + +This compares every published lesson together, on one run each way. Runs of the same case vary, so one pair suggests a direction and does not settle it. ## Comments diff --git a/.scratch/lessons/issues/06-do-not-mark-requirements-as-not-applicable.md b/.scratch/lessons/issues/06-do-not-mark-requirements-as-not-applicable.md index 032d459..1c4d61d 100644 --- a/.scratch/lessons/issues/06-do-not-mark-requirements-as-not-applicable.md +++ b/.scratch/lessons/issues/06-do-not-mark-requirements-as-not-applicable.md @@ -20,7 +20,15 @@ Seen in 2 of 21 Executions, 2 times in all: submit_deliverable refused: Not subm ## Proof -Not yet measured. +Evaluation without lessons: `20261009T195603Z`; with: `20261009T195607Z`. + +| Case | Score | Revisions | Minutes | Tokens (M) | +| --- | --- | --- | --- | --- | +| case-03-internal-web-app | 1.00 → 1.00 | 1 → 0 | 28 → 22 | 3.00 → 2.32 | + +Quality held and revisions, seconds, tokens fell: kept. + +This compares every published lesson together, on one run each way. Runs of the same case vary, so one pair suggests a direction and does not settle it. ## Comments diff --git a/app/adk/roles.py b/app/adk/roles.py index f2c4bfa..ed794ed 100644 --- a/app/adk/roles.py +++ b/app/adk/roles.py @@ -43,9 +43,11 @@ def load_roles( specialist_budget: int = 15, include_preview_mcp: bool = True, lessons: bool = True, + model_tiers: bool = False, ) -> dict[str, RoleSpec]: """`lessons` off leaves the lessons Skill out of every role, so an evaluation - can run with and without what has been learned.""" + can run with and without what has been learned. `model_tiers` on gives a role + the lighter model the bundle names for its first attempt.""" specialists = {spec.specialist: kind for kind, spec in DEPENDENCY_GRAPH.items()} required = {ORCHESTRATOR, PILLAR_ADVISOR, REPLY_WRITER, *specialists} optional = {LESSON_CURATOR} @@ -74,6 +76,13 @@ def load_roles( roles[name] = dataclasses.replace( roles[name], skills=(*roles[name].skills, bundle.lessons_skill) ) + if model_tiers: + for name, model in bundle.models.first_attempt.items(): + if name not in roles: + raise RoleConfigError( + f"'models.first_attempt' names {name!r}, which is not a role" + ) + roles[name] = dataclasses.replace(roles[name], first_attempt_model=model) unexpected = sorted(set(roles) - required - optional) if unexpected: raise RoleConfigError( diff --git a/app/adk/runtime.py b/app/adk/runtime.py index 852cb8b..cfb6684 100644 --- a/app/adk/runtime.py +++ b/app/adk/runtime.py @@ -34,6 +34,9 @@ class RoleSpec: model: str | None = None # Seconds one model request may take; None takes the runtime's default. model_timeout_seconds: float | None = None + # A lighter model for the first attempt at a Deliverable, when the bundle + # names one and model tiers are on (iteration 5 step 6). + first_attempt_model: str | None = None @dataclass(frozen=True) diff --git a/app/agent.py b/app/agent.py index 0cd8ea1..968a66e 100644 --- a/app/agent.py +++ b/app/agent.py @@ -379,6 +379,7 @@ async def open_assembly( specialist_budget=settings.SPECIALIST_BUDGET, include_preview_mcp=settings.MCP_ENABLE_PREVIEW, lessons=settings.LESSONS_ENABLED, + model_tiers=settings.MODEL_TIERS, ) overlays = discover_overlays(settings, bundle) if runtime is None: diff --git a/app/config.py b/app/config.py index 7699fd8..b47a09d 100644 --- a/app/config.py +++ b/app/config.py @@ -64,6 +64,10 @@ class Settings(BaseSettings): # Give Specialists the lessons Skill. Off runs without anything learned, which # is how the evaluation shows whether a lesson helps. LESSONS_ENABLED: bool = True + # Give a role the lighter model its provider bundle names for a first attempt + # (`models.first_attempt`). An attempt that follows feedback, or an attempt + # that produced nothing, takes the role's own model. Off until measured. + MODEL_TIERS: bool = False # With a user present, a Specialist's first submission of a Deliverable is refused # until it has put its draft to them. Off: Specialists are only told to. CONFIRM_DRAFTS: bool = True diff --git a/app/nodes/execution_nodes.py b/app/nodes/execution_nodes.py index 08d03aa..c3aeead 100644 --- a/app/nodes/execution_nodes.py +++ b/app/nodes/execution_nodes.py @@ -458,6 +458,14 @@ def make_execution_nodes( role = dataclasses.replace( role, skills=(*role.skills, *(n for n in named if n not in role.skills)) ) + if role.first_attempt_model and task_input.get("first_attempt"): + logger.info( + "[%s] %s takes %s for a first attempt", + task_input["execution_id"], + role.name, + role.first_attempt_model, + ) + role = dataclasses.replace(role, model=role.first_attempt_model) kind = DeliverableKind(task_input["kind"]) working: dict[str, Any] = dict(task_input.get("working") or {}) upstream = task_input["context"].get("dependencies") or {} @@ -861,6 +869,10 @@ def _specialist_input( "role": spec.specialist, "kind": kind.value, "attempt": attempt, + # Nothing has been asked of this Deliverable again: no feedback on it, and + # no earlier attempt that ended without one. A lighter model may take it. + "first_attempt": not ledger.feedback(kind) + and (attempt == 1 or kind.value in deliverables), "allow_cloud_read": state.get("allow_cloud_read"), "working": dict(state.get("working", {}).get(kind.value, {})), # Whether the user has been shown a draft of this Deliverable. Answering some @@ -924,14 +936,9 @@ async def _pillar_hints( ) or baseline.get("working", {}).get(kind.value, {}).get(ADVICE_KEY) if not remembered: return {} # a first consultation: every pillar - change = "\n".join(ledger.feedback(kind)) - if not change and baseline: - summary = change_summary(baseline.get("deliverables"), deliverables) - change = "\n".join( - f"Requirement {item['id']} {item['change']}: {item['text']} " - f"({item['reason']})" - for item in summary["requirements"] - ) + change = "\n".join(ledger.feedback(kind)) or _requirement_changes( + baseline, deliverables + ) chosen = await pillars_touched( decider, list(remembered), @@ -943,6 +950,19 @@ async def _pillar_hints( return {} if chosen is None else {kind.value: chosen} +def _requirement_changes( + baseline: Mapping[str, Any], deliverables: Mapping[str, Mapping[str, Any]] +) -> str: + """In a Change request, the requirements added, changed and removed, in words.""" + if not baseline: + return "" + summary = change_summary(baseline.get("deliverables"), deliverables) + return "\n".join( + f"Requirement {item['id']} {item['change']}: {item['text']} ({item['reason']})" + for item in summary["requirements"] + ) + + async def _reading_hints( state: ExecutionState, ledger: Ledger, @@ -970,9 +990,11 @@ async def _reading_hints( for item in elements ) + changed = _requirement_changes(state.get("baseline") or {}, deliverables) + def task(kind: DeliverableKind) -> str: feedback = list(ledger.feedback(kind)) - doing = "Revise" if feedback else "Write" + doing = "Revise" if feedback or changed else "Write" parts = [ ( f"{doing} the {kind.value.replace('_', ' ')} Deliverable of a cloud " @@ -981,6 +1003,9 @@ async def _reading_hints( ] if feedback: parts.append("Feedback to address:\n" + "\n".join(feedback)) + if changed: + # In a Change request the task is the change, not the whole design. + parts.append(f"What has changed since it was delivered:\n{changed}") parts.append( f"Elements of the Target architecture:\n{design}" if design diff --git a/app/providers.py b/app/providers.py index 1208b2b..7835751 100644 --- a/app/providers.py +++ b/app/providers.py @@ -45,6 +45,9 @@ class ModelConfig: # Seconds one model request may take, for the roles whose replies are long. A # role not named here gets the runtime's default. timeouts: Mapping[str, float] = field(default_factory=dict) + # A lighter model for a role's first attempt at a Deliverable. Only for a role + # whose work something else checks; an attempt after feedback takes its own. + first_attempt: Mapping[str, str] = field(default_factory=dict) def for_role(self, role: str) -> str: return self.roles.get(role, self.default) @@ -131,6 +134,10 @@ def load_provider(name: str, root: Path | None = None) -> ProviderBundle: default=models["default"], roles=dict(models.get("roles") or {}), timeouts=timeouts, + first_attempt={ + str(role): str(model) + for role, model in (models.get("first_attempt") or {}).items() + }, ) terraform_providers = {} diff --git a/app/tools/schema_review.py b/app/tools/schema_review.py index 8e74f1a..699d384 100644 --- a/app/tools/schema_review.py +++ b/app/tools/schema_review.py @@ -64,6 +64,20 @@ def _names(block: Mapping[str, Any]) -> set[str]: return names +def _is_missing(setting: str, known: set[str]) -> bool: + """Whether a name a Check uses is a setting the resource type lacks. + + A path is judged by the setting at its end, and only when it starts at a name + the type has. One that does not start there is a value written with dots, such + as a database flag (`cloudsql.iam_authentication`) or a service + (`run.googleapis.com`): both were raised as missing settings on 2026-10-09. + """ + first, _, rest = setting.partition(".") + if rest and first not in known: + return False + return setting.split(".")[-1] not in known + + class SchemaReview: def __init__(self, runner: TerraformRunner) -> None: self._runner = runner @@ -111,9 +125,7 @@ class SchemaReview: missing = [ setting for setting in named_settings(check.statement) - # A resource type named in the statement is not a setting, and a - # path is judged by the setting at its end. - if setting not in schemas and setting.split(".")[-1] not in known + if setting not in schemas and _is_missing(setting, known) ] if missing: concerns.append( diff --git a/eval/datasets/solution_cases.json b/eval/datasets/solution_cases.json index 51425e5..37a25ac 100644 --- a/eval/datasets/solution_cases.json +++ b/eval/datasets/solution_cases.json @@ -441,6 +441,32 @@ ], "overlay_skills": [ "example-organisation-standards" + ], + "code_standards": [ + { + "name": "only approved regions", + "forbid": "\"(?!europe-west[14]\")(?:us|europe|asia|australia|northamerica|southamerica|me|africa)-[a-z]+[0-9]+\"" + }, + { + "name": "names carry the environment from a variable", + "require": "name\\s*=\\s*\"[^\"]*\\$\\{(?:var|local)\\.[a-z_]*env[a-z_]*\\}" + }, + { + "name": "label workload", + "require": "\\bworkload\\b\\s*=" + }, + { + "name": "label environment", + "require": "\\benvironment\\b\\s*=" + }, + { + "name": "label owner", + "require": "\\bowner\\b\\s*=" + }, + { + "name": "label cost-centre", + "require": "\"?cost[-_]centre\"?\\s*=" + } ] } ] diff --git a/eval/scoring.py b/eval/scoring.py index 9dd3921..971baf1 100644 --- a/eval/scoring.py +++ b/eval/scoring.py @@ -71,6 +71,10 @@ class CaseSpec: change: str = "" # Overlay Skills the case's Executions name (ADR-0004). overlay_skills: tuple[str, ...] = () + # What the infrastructure code must and must not contain, as regular + # expressions over all of it: how a case checks that standards from an overlay + # Skill reached the code. Each is {"name", "require"} or {"name", "forbid"}. + code_standards: tuple[Mapping[str, str], ...] = () @dataclass(frozen=True) @@ -114,6 +118,7 @@ def load_cases(path: Path) -> list[CaseSpec]: thresholds=dict(item.get("thresholds", {})), change=str(item.get("change", "")), overlay_skills=tuple(item.get("overlay_skills", ())), + code_standards=tuple(item.get("code_standards", ())), ) for item in json.loads(path.read_text(encoding="utf-8")) ] @@ -295,6 +300,8 @@ async def score_execution( else: terraform_ok, terraform_detail = False, "no Terraform files attached" checks.append(Check("terraform_validates", terraform_ok, terraform_detail)) + if case.code_standards: + checks.append(_code_standards(case.code_standards, terraform_files)) checks.append(_chain_check(deliverables)) checks.append(_verdicts_check(attachments(DeliverableKind.VALIDATION_RESULTS))) @@ -315,6 +322,22 @@ async def score_execution( return CaseResult(case.id, interaction.status, tuple(checks)) +def _code_standards( + standards: Sequence[Mapping[str, str]], files: Mapping[str, str] +) -> Check: + """Whether the code holds what the case requires of it and nothing it forbids.""" + code = "\n".join(files.values()) + broken = [] + for standard in standards: + required, forbidden = standard.get("require"), standard.get("forbid") + if required and not re.search(required, code): + broken.append(f"{standard['name']}: not found") + found = re.search(forbidden, code) if forbidden else None + if found: + broken.append(f"{standard['name']}: found {found.group(0)}") + return Check("code_follows_standards", bool(code) and not broken, "; ".join(broken)) + + def _chain_check(deliverables: Mapping[str, Mapping[str, Any]]) -> Check: """The Requirements chain holds end to end: every requirement reaches a Check, directly or through a component's Element, and nothing along it is missing.""" diff --git a/providers/gcp.yaml b/providers/gcp.yaml index 91dbbbc..c9e1a6a 100644 --- a/providers/gcp.yaml +++ b/providers/gcp.yaml @@ -22,6 +22,12 @@ models: iac-author: gemini/gemini-3.1-pro-preview validator: gemini/gemini-3.1-pro-preview lesson-curator: gemini/gemini-3.1-pro-preview + # A lighter model for a role's first attempt, used when MODEL_TIERS is on. Only a + # role whose work is checked by something else belongs here: the code is checked + # by terraform validate and by the validator. The validator is the check, and a + # weaker one that passes everything would look like an improvement. + first_attempt: + iac-author: gemini/gemini-3.8-flash # Seconds one model request may take, for roles whose replies are long. Other # roles get MODEL_TIMEOUT_SECONDS. The target architect timed out at 180 s in both # attempts of the 2026-10-07 run. diff --git a/skills/google-cloud-lessons/SKILL.md b/skills/google-cloud-lessons/SKILL.md index 7338d1f..43ffc8d 100644 --- a/skills/google-cloud-lessons/SKILL.md +++ b/skills/google-cloud-lessons/SKILL.md @@ -18,7 +18,7 @@ Each reference below holds the lessons for one Deliverable. Read the one for the | Deliverable | Reference | Lessons | | --- | --- | --- | | deployment guidance | `references/deployment_guidance.md` | 1 | -| target architecture | `references/target_architecture.md` | 5 | +| target architecture | `references/target_architecture.md` | 4 | This Skill is written by the lessons loop (`app/learning/`), from proposals a person approved. Do not edit it by hand: change the proposal and publish again. diff --git a/skills/google-cloud-lessons/references/target_architecture.md b/skills/google-cloud-lessons/references/target_architecture.md index 1717120..f699b07 100644 --- a/skills/google-cloud-lessons/references/target_architecture.md +++ b/skills/google-cloud-lessons/references/target_architecture.md @@ -26,14 +26,6 @@ Published 2026-10-09, from lesson proposal 02. Published 2026-10-09, from lesson proposal 03. -## L-04: Avoid Using Values as Setting Names - -**Rule.** When writing a Check for a configuration block that takes a name and value, explicitly state the block's argument name instead of writing the expected value as if it were the setting itself. - -**Why.** Automated reviews may mistake specific expected values for invalid setting names, causing warnings and submission delays. - -Published 2026-10-09, from lesson proposal 04. - ## L-06: Do Not Mark Requirements as Not Applicable **Rule.** Do not list Requirements in the not applicable field, as this field is strictly for Design Recommendations. Ensure every Requirement has at least one Check. diff --git a/tests/eval/test_scoring.py b/tests/eval/test_scoring.py index 62e5335..d459141 100644 --- a/tests/eval/test_scoring.py +++ b/tests/eval/test_scoring.py @@ -1,5 +1,6 @@ """Scoring a finished Execution against a benchmark case, without models or network.""" +import dataclasses import json from pathlib import Path @@ -772,3 +773,49 @@ def test_a_functional_requirement_is_reached_through_its_components_elements(): assert by_name["change_is_recorded"].detail == "FR-1 changed" assert by_name["changed_requirements_reach_checks"].passed + + +# --- standards an overlay Skill sets reach the code -------------------------------------- + +STANDARDS = ( + {"name": "approved regions", "forbid": r'"us-[a-z]+[0-9]+"'}, + {"name": "label owner", "require": r"\bowner\b\s*="}, +) + + +def code(text: str) -> dict: + return {"content": "guide", "attachments": {"terraform/main.tf": text}} + + +async def test_code_that_follows_the_cases_standards_passes(): + case = dataclasses.replace(CASE, code_standards=STANDARDS) + labelled = code('region = "europe-west1"\nlabels = { owner = var.owner }') + + result = await score_execution( + case, execution(deployment_guidance=labelled), HEADINGS, passing_terraform + ) + + assert check(result, "code_follows_standards").passed + + +async def test_a_broken_standard_is_named(): + case = dataclasses.replace(CASE, code_standards=STANDARDS) + + result = await score_execution( + case, + execution(deployment_guidance=code('region = "us-central1"')), + HEADINGS, + passing_terraform, + ) + + broken = check(result, "code_follows_standards") + assert not broken.passed + assert ( + broken.detail == 'approved regions: found "us-central1"; label owner: not found' + ) + + +async def test_a_case_with_no_standards_has_no_such_check(): + result = await score_execution(CASE, execution(), HEADINGS, passing_terraform) + + assert all(c.name != "code_follows_standards" for c in result.checks) diff --git a/tests/execution/test_model_tiers.py b/tests/execution/test_model_tiers.py new file mode 100644 index 0000000..2169a86 --- /dev/null +++ b/tests/execution/test_model_tiers.py @@ -0,0 +1,114 @@ +"""A lighter model for a first attempt (iteration 5 step 6).""" + +import dataclasses + +from langgraph.checkpoint.memory import InMemorySaver + +from app.adk.fake_runtime import Call, FakeAgentRuntime, sample_submission +from app.adk.roles import load_roles +from app.adk.runtime import NoDecider, RoleSpec +from app.agent import AGENTS_DIR, ExecutionEngine +from app.nodes.router import Router, Thresholds +from app.providers import load_provider +from app.recorder import InMemoryRecorder +from app.workflows.deliverables import DEPENDENCY_GRAPH +from app.workflows.deliverables import DeliverableKind as K + +AUTHOR = DEPENDENCY_GRAPH[K.DEPLOYMENT_GUIDANCE].specialist + + +def roles(**light: str) -> dict[str, RoleSpec]: + return { + "orchestrator": RoleSpec("orchestrator", "Unused.", budget=40), + **{ + spec.specialist: RoleSpec( + spec.specialist, + "Produce.", + budget=15, + model="strong", + first_attempt_model=light.get(spec.specialist.replace("-", "_")), + ) + for spec in DEPENDENCY_GRAPH.values() + }, + } + + +class Recording(FakeAgentRuntime): + def __init__(self, turns): + super().__init__(turns) + self.models: list[tuple[str, str | None]] = [] + + async def run(self, role, task, tools): + self.models.append((role.name, role.model)) + return await super().run(role, task, tools) + + +def engine(tmp_path, runtime, given) -> ExecutionEngine: + return ExecutionEngine( + runtime=runtime, + roles=given, + recorder=InMemoryRecorder(), + checkpointer=InMemorySaver(), + output_root=tmp_path, + orchestrator=Router(NoDecider(), None, Thresholds()), + ) + + +def once() -> dict: + return { + spec.specialist: [[sample_submission(kind)]] + for kind, spec in DEPENDENCY_GRAPH.items() + } + + +async def test_a_first_attempt_takes_the_lighter_model_and_other_roles_their_own( + tmp_path, +): + runtime = Recording(once()) + + await engine(tmp_path, runtime, roles(iac_author="light")).start( + "Ingest webhooks", pre_approved=True + ) + + assert dict(runtime.models)[AUTHOR] == "light" + assert {model for name, model in runtime.models if name != AUTHOR} == {"strong"} + + +async def test_an_attempt_that_ended_without_a_deliverable_is_retried_on_the_roles_own( + tmp_path, +): + turns = once() + turns[AUTHOR] = [ + [Call("needs_input", {"questions": ["Which region?"]})], + [sample_submission(K.DEPLOYMENT_GUIDANCE)], + ] + runtime = Recording(turns) + + interaction = await engine(tmp_path, runtime, roles(iac_author="light")).start( + "Ingest webhooks", pre_approved=True + ) + + assert interaction.status == "completed" + assert [model for name, model in runtime.models if name == AUTHOR] == [ + "light", + "strong", + ] + + +def test_the_bundles_lighter_models_reach_roles_only_when_tiers_are_on(): + bundle = load_provider("gcp") + + off = load_roles(AGENTS_DIR, bundle) + on = load_roles(AGENTS_DIR, bundle, model_tiers=True) + + assert all(role.first_attempt_model is None for role in off.values()) + assert on[AUTHOR].first_attempt_model == bundle.models.first_attempt[AUTHOR] + assert on[AUTHOR].model == bundle.models.for_role(AUTHOR) + # The validator is the check on the code; it is never given a lighter model. + assert "validator" not in bundle.models.first_attempt + assert on == { + name: dataclasses.replace( + role, first_attempt_model=bundle.models.first_attempt.get(name) + ) + for name, role in off.items() + } diff --git a/tests/execution/test_validation_in_pieces.py b/tests/execution/test_validation_in_pieces.py index 0992cfd..91fd10e 100644 --- a/tests/execution/test_validation_in_pieces.py +++ b/tests/execution/test_validation_in_pieces.py @@ -240,6 +240,23 @@ async def test_a_setting_found_anywhere_in_the_resource_type_is_accepted(): assert await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) == () +async def test_a_value_written_with_dots_is_not_read_as_a_setting(): + """A database flag or a service name is a value. A path is a setting only when + it starts at a name the resource type has.""" + design = graph( + [ + ("E-1", "the flag cloudsql.iam_authentication is on"), + ("E-1", "audit logs are on for run.googleapis.com"), + ("E-1", "template.scaling.made_up_count is at least 2"), + ("E-2", "the name is set"), + ] + ) + + raised = await SchemaReview(Runner())(K.TARGET_ARCHITECTURE, design, None) + + assert len(raised) == 1 and "template.scaling.made_up_count" in raised[0] + + async def test_a_resource_type_the_provider_lacks_is_raised_with_close_matches(): design = graph( [("E-1", "the name is set"), ("E-2", "the name is set")],