From ee1ba388676be078231e5697253cf8c1c8fa5f79 Mon Sep 17 00:00:00 2001 From: Jonathan Boniface Date: Sat, 10 Oct 2026 12:33:06 +0100 Subject: [PATCH] feat: a design-time coverage review, measured and left off A reviewer role reads the requirements and recommendations against the Checks when the Target architecture is first submitted. Three runs of case 03: it did not stop the validator sending the design back, and the designs it reviewed ended with 87 and 93 Checks against 52, so they cost more. It stays behind DESIGN_COVERAGE_REVIEW, off (iteration 6 step 13). Also: the schema review no longer reads `true` or `false` in backticks as a setting name. Co-Authored-By: Claude Opus 5.5 --- .../iter-06-target-architecture-graph/MAP.md | 3 +- .../steps/step-13.md | 45 ++++++ .env.example | 4 + agents/coverage-reviewer.md | 22 +++ app/adk/roles.py | 3 +- app/agent.py | 7 +- app/config.py | 6 + app/tools/design_review.py | 151 ++++++++++++++++++ app/tools/schema_review.py | 5 +- providers/gcp.yaml | 3 + tests/agents/test_roles.py | 5 +- tests/execution/test_design_review.py | 150 +++++++++++++++++ tests/execution/test_validation_in_pieces.py | 3 + 13 files changed, 401 insertions(+), 6 deletions(-) create mode 100644 .agents/plans/iter-06-target-architecture-graph/steps/step-13.md create mode 100644 agents/coverage-reviewer.md create mode 100644 app/tools/design_review.py create mode 100644 tests/execution/test_design_review.py diff --git a/.agents/plans/iter-06-target-architecture-graph/MAP.md b/.agents/plans/iter-06-target-architecture-graph/MAP.md index d7bcffb..3cbf0d8 100644 --- a/.agents/plans/iter-06-target-architecture-graph/MAP.md +++ b/.agents/plans/iter-06-target-architecture-graph/MAP.md @@ -57,7 +57,8 @@ flowchart TD | 9 | Validation gives a verdict per Check | M | complete | | 10 | Revision routing: the code or the graph | S | complete; routing unproven with a real model | | 11 | Evaluation: prove the loop converges | M | complete: all four cases pass, each run twice on 2026-10-08; thresholds set (`steps/step-11.md`) | -| 12 | Why the Target architecture is revised | S | investigated 2026-10-10: two causes, about half each; two instructions changed; a design-time coverage review is proposed, not built (`steps/step-12.md`) | +| 12 | Why the Target architecture is revised | S | investigated 2026-10-10: two causes, about half each; two instructions changed; the design-time coverage review it proposed is step 13 (`steps/step-12.md`) | +| 13 | A model reviews coverage at design time | M | built and measured on 2026-10-10: it did not prevent the validator's gaps and made designs larger; left off (`steps/step-13.md`) | Size: S under a day, M a few days. diff --git a/.agents/plans/iter-06-target-architecture-graph/steps/step-13.md b/.agents/plans/iter-06-target-architecture-graph/steps/step-13.md new file mode 100644 index 0000000..f680c77 --- /dev/null +++ b/.agents/plans/iter-06-target-architecture-graph/steps/step-13.md @@ -0,0 +1,45 @@ +# Step 13 — A model reviews coverage at design time: built, measured, left off + +Built and run on 2026-10-10. `DESIGN_COVERAGE_REVIEW` is off. + +## What was built + +Step 12 proposed making the validator's reading of coverage when the Target architecture is first submitted, before any code. + +- `agents/coverage-reviewer.md`: a role on the validator's model. It is given the non-functional requirements, the recommendations and the graph, and reports every condition no Check verifies, or that a Check verifies more weakly than the words. +- `DesignCoverageReview` in `app/tools/design_review.py`, on the submission-review seam: what it finds goes back to the target architect once and never blocks. A gap naming an id that does not exist is dropped. It runs once in an Execution. + +Tests: `tests/execution/test_design_review.py` (7). + +## Measured + +Case 03, three runs side by side. + +| | Review off | Review on | Review on | +|---|---|---|---| +| Results | `20261010T105532Z` | `20261010T105535Z` | `20261010T105540Z` | +| Score | 100% | 100% | 100% | +| Minutes | 23 | 28 | 31 | +| Cost, about | $2.90 | $3.33 | $3.75 | +| Revisions of the Target architecture | 1, a Check that cannot be implemented | 0 | 1, a gap the validator found | +| Gaps the reviewer raised | | 4 | 22 | +| Checks in the delivered design | 52 | 87 | 93 | + +The reviewer itself cost $0.19 a run. + +## Why it stays off + +- **It did not stop the validator sending the design back.** In one of the two runs the validator still found a gap, after the reviewer had raised 22. +- **It makes the design bigger.** The designs it reviewed ended with 87 and 93 Checks against 52. Every Check is code to write and a verdict to give, so both runs cost more than the run without it, including the one with no Revision. +- **Two readers do not agree on what a gap is.** The reviewer and the validator are the same model given the same question, and they raised different things. Asking earlier does not settle what will be asked later. + +Three runs are a small sample, but nothing in them points the other way. The code stays, off, for a later look; removing it is the owner's call. + +## What the runs did show + +Two of the faults behind Revisions are now recurring ones, which is what the lessons loop is for: + +- **A Check asking for a separate encryption key for database backups**: unimplementable, on 2026-10-09 and again today. +- **A condition that a region is "low-carbon"**: the validator's gap today, and among what the Jev coverage review kept raising on 2026-10-09. Code can show which region is set; it cannot show the region's carbon rating. + +And one fault in the schema review, fixed: a value in backticks, `true` or `false`, was read as a setting name and raised. `named_settings` now leaves those literals out. diff --git a/.env.example b/.env.example index 574743b..5b8cfce 100644 --- a/.env.example +++ b/.env.example @@ -59,6 +59,10 @@ MAX_PARALLEL_SPECIALISTS=4 CHECKPOINTER=sqlite CHECKPOINT_DB_PATH=checkpoints.db +# A model reads the requirements against the Checks when the Target architecture is +# first submitted, and returns what no Check verifies. One model run per Execution. +DESIGN_COVERAGE_REVIEW=false + # Jev decision service, through the TypeSafe AI Python SDK (typesafe-sdk). # Leave the key unset to run without it: every decision then takes its fallback. # TYPESAFE_API_KEY= diff --git a/agents/coverage-reviewer.md b/agents/coverage-reviewer.md new file mode 100644 index 0000000..2512f03 --- /dev/null +++ b/agents/coverage-reviewer.md @@ -0,0 +1,22 @@ +--- +role: coverage-reviewer +tools: [] +budget: 6 +--- +You review the Target architecture of a solution-architecture Execution before anyone approves it. You answer one question: do its Checks verify everything the requirements and design recommendations ask for? You change nothing and write no design. What you find goes back to the target architect once. + +Your task message carries JSON context: + +- `constraints`: the non-functional requirements and the design recommendations. Each has an `id`, its text, `conditions` (what infrastructure code can show, one by one) and sometimes `outside_code` (what code cannot show). +- `target_architecture`: the Elements, Relationships and Checks. Each Check has a `source`, the id of the requirement or recommendation it comes from, `applies_to`, the Element or Relationship it is about, and its `statement`. A recommendation listed under `not_applicable` asks nothing. + +Review once, completely: + +- Go through every constraint in turn. For each of its `conditions`, find the Checks whose `source` is that constraint and decide whether they verify the condition on every Element it applies to. +- A gap is a condition that no Check verifies, or that a Check verifies more weakly than the words. A Check that says a setting "is set" does not verify a value the condition gives. A condition with two parts needs both verified. A condition about every Element of a kind needs a Check on each one. +- For each gap, give the Check that is missing: the constraint's id as `source`, the Element or Relationship id as `applies_to`, and a `statement` that names the setting and the value to verify, as specific as the Checks already there. +- Report nothing for what a constraint lists under `outside_code`, and nothing for a recommendation that is `not_applicable`. +- Report nothing a Check already verifies in other words, and nothing that is your own idea of good practice: only what a condition's words ask for. +- Do not judge whether a Check can be implemented or whether the design is a good one. That is not this review. + +Call `report_gaps` once with every gap, or with an empty list when there are none. You are done when it records them. diff --git a/app/adk/roles.py b/app/adk/roles.py index 50c3dd7..aca3dd8 100644 --- a/app/adk/roles.py +++ b/app/adk/roles.py @@ -20,6 +20,7 @@ ORCHESTRATOR = "orchestrator" PILLAR_ADVISOR = "pillar-advisor" REPLY_WRITER = "reply-writer" LESSON_CURATOR = "lesson-curator" # runs outside any Execution +COVERAGE_REVIEWER = "coverage-reviewer" # reviews a design before its Approval PARTIALS_DIR = "partials" _ALLOWED_FIELDS = {"role", "deliverable", "tools", "budget", "includes"} @@ -51,7 +52,7 @@ def load_roles( the lighter model the bundle names for its first attempt.""" specialists = {spec.specialist: kind for kind, spec in DEPENDENCY_GRAPH.items()} required = {ORCHESTRATOR, PILLAR_ADVISOR, REPLY_WRITER, *specialists} - optional = {LESSON_CURATOR} + optional = {LESSON_CURATOR, COVERAGE_REVIEWER} learned = _deliverables_with_lessons(bundle) if lessons else set() roles = {} diff --git a/app/agent.py b/app/agent.py index 0f06b89..2a02aff 100644 --- a/app/agent.py +++ b/app/agent.py @@ -22,7 +22,7 @@ from langchain_core.runnables import RunnableConfig from langgraph.checkpoint.base import BaseCheckpointSaver from langgraph.types import Command -from app.adk.roles import load_roles +from app.adk.roles import COVERAGE_REVIEWER, load_roles from app.adk.runtime import ( AgentRuntime, Decider, @@ -50,6 +50,7 @@ from app.skills.agent_skills import discover_skills, skill_version from app.states.execution_state import check_customer_case from app.states.ledger import Ledger from app.tools.coverage import CoverageReview +from app.tools.design_review import DesignCoverageReview from app.tools.execution import ORCHESTRATOR from app.tools.jev import JevDecider, JevLimits from app.tools.local import TerraformRunner, local_tools @@ -481,6 +482,10 @@ async def open_assembly( reviews.append( RequirementsReview(decider, act=settings.JEV_REQUIREMENTS_THRESHOLD) ) + if settings.DESIGN_COVERAGE_REVIEW and COVERAGE_REVIEWER in roles: + # A model reads the requirements against the Checks, once, before the + # design is approved: the reading the validator would make after the code. + reviews.append(DesignCoverageReview(runtime, roles[COVERAGE_REVIEWER])) if settings.JEV_COVERAGE_REVIEW and isinstance(decider, JevDecider): # Optional, and off by default: Jev says which Check verifies each condition. reviews.append(CoverageReview(decider, none_at=settings.JEV_COVERAGE_THRESHOLD)) diff --git a/app/config.py b/app/config.py index cbc6607..830ba5c 100644 --- a/app/config.py +++ b/app/config.py @@ -104,6 +104,12 @@ class Settings(BaseSettings): JEV_ACT_THRESHOLD: float = 0.75 JEV_CLOSE_THRESHOLD: float = 0.15 JEV_CONFIDENCE_THRESHOLD: float = 0.6 + # Have a model read the requirements and recommendations against the Checks + # when the Target architecture is first submitted, and return what no Check + # verifies to its author. The validator makes the same reading after the code + # is written, when a gap costs a Revision of design, code and validation. One + # model run in every Execution. Off until measured. + DESIGN_COVERAGE_REVIEW: bool = False # Review a design's coverage at submission: for each condition a requirement # lists, which Check verifies it. Off until a full case has run with it. JEV_COVERAGE_REVIEW: bool = False diff --git a/app/tools/design_review.py b/app/tools/design_review.py new file mode 100644 index 0000000..2a36bf2 --- /dev/null +++ b/app/tools/design_review.py @@ -0,0 +1,151 @@ +"""A review of a design's coverage by a model, before anyone approves it. + +The validator reads the requirements and recommendations against the Checks and +proposes one for each condition nothing verifies. That reading needs no code, but it +used to happen only after the code was written: in the runs of 2026-10-09 it sent +the Target architecture back in four of seven, each time costing a Revision of the +design, the code and the validation (iteration 6 step 12). + +This is the same reading, made when the Target architecture is first submitted and +returned to its author. It is a review like the others: what it raises is returned +once and never blocks, and when it fails the submission stands. + +It runs once in an Execution. A Revision is asked for by the validator or the user, +who have just read the design; reviewing it again would pay for the same reading. +""" + +from __future__ import annotations + +import json +import logging +from collections.abc import Mapping, Sequence +from typing import Any + +from app.adk.runtime import ( + AgentRuntime, + AgentTask, + RoleSpec, + Submitted, + ToolResult, + ToolSpec, +) +from app.workflows.chain import ( + RECOMMENDATIONS_FILE, + REQUIREMENTS_FILE, + Upstream, + read_list, +) +from app.workflows.deliverables import DeliverableKind +from app.workflows.target_architecture import GRAPH_ATTACHMENT, GraphError, parse_graph + +logger = logging.getLogger(__name__) + +MAX_GAPS = 12 + + +class DesignCoverageReview: + def __init__(self, runtime: AgentRuntime, role: RoleSpec) -> None: + self._runtime = runtime + self._role = role + self._reviewed: set[str] = set() + + async def __call__( + self, + kind: DeliverableKind, + attachments: Mapping[str, str], + upstream: Upstream | None, + scope: str = "", + ) -> Sequence[str]: + """The conditions the reviewer finds no Check verifies, each as a line for + the target architect. Empty when it finds none or could not review.""" + if kind is not DeliverableKind.TARGET_ARCHITECTURE or not upstream: + return () + text = attachments.get(GRAPH_ATTACHMENT, "") + try: + graph = parse_graph(text) + except GraphError: + return () + if scope in self._reviewed: + return () + self._reviewed.add(scope) + + constraints = [ + item + for item in read_list(upstream, "requirements", REQUIREMENTS_FILE) + if item.get("kind") == "non_functional" + ] + read_list(upstream, "design_recommendations", RECOMMENDATIONS_FILE) + known = {str(item.get("id")) for item in constraints} + applies_to = {element.id for element in graph.elements} | { + relationship.id for relationship in graph.relationships + } + + async def report_gaps(gaps: Any = None) -> ToolResult: + if not isinstance(gaps, list): + return ToolResult("Not recorded: `gaps` is a list; empty when none.") + return ToolResult("Recorded.", Submitted(json.dumps(gaps))) + + tool = ToolSpec( + "report_gaps", + "Give every gap you found, in one call. An empty list when the Checks " + "verify every condition. Ends your turn.", + { + "type": "object", + "properties": { + "gaps": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string", + "description": "The requirement or recommendation id", + }, + "applies_to": { + "type": "string", + "description": "The Element or Relationship id", + }, + "statement": { + "type": "string", + "description": "The Check that is missing", + }, + }, + "required": ["source", "applies_to", "statement"], + }, + } + }, + "required": ["gaps"], + }, + report_gaps, + tier="execution", + ) + task = AgentTask( + role=self._role.name, + prompt="Review whether the Target architecture's Checks verify every " + "condition of the requirements and recommendations.", + context={ + "constraints": constraints, + "target_architecture": json.loads(text), + }, + ) + outcome = await self._runtime.run(self._role, task, [tool]) + if not isinstance(outcome, Submitted): + logger.info("[%s] design coverage review gave no answer", scope) + return () + try: + reported = json.loads(outcome.content) + except ValueError: + return () + concerns = [] + for gap in reported if isinstance(reported, list) else []: + if not isinstance(gap, Mapping): + continue + source, where = str(gap.get("source", "")), str(gap.get("applies_to", "")) + statement = " ".join(str(gap.get("statement", "")).split()) + # A gap must be about something that exists: an invented id is dropped. + if source not in known or where not in applies_to or not statement: + continue + concerns.append(f"{source}: no Check verifies this on {where}: {statement}") + logger.info( + "[%s] design coverage review raised %d gap(s)", scope, len(concerns) + ) + return tuple(concerns[:MAX_GAPS]) diff --git a/app/tools/schema_review.py b/app/tools/schema_review.py index 699d384..b80841d 100644 --- a/app/tools/schema_review.py +++ b/app/tools/schema_review.py @@ -36,6 +36,9 @@ _NAME = re.compile( r"(?![\w/\-])" ) _PATH = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)*$") +# Values a Check may put in backticks. `true` and `false` were raised as missing +# settings in a run of 2026-10-10. +_LITERALS = frozenset({"true", "false", "null", "none"}) def named_settings(statement: str) -> tuple[str, ...]: @@ -48,7 +51,7 @@ def named_settings(statement: str) -> tuple[str, ...]: found: list[str] = [] for span in _CODE.findall(statement): candidate = span.strip().split("=")[0].strip() - if _PATH.match(candidate): + if _PATH.match(candidate) and candidate not in _LITERALS: found.append(candidate) bare = _QUOTED.sub(" ", _CODE.sub(" ", statement)) found += [match.group(1).rstrip(".") for match in _NAME.finditer(bare)] diff --git a/providers/gcp.yaml b/providers/gcp.yaml index 96ac82e..5b76e77 100644 --- a/providers/gcp.yaml +++ b/providers/gcp.yaml @@ -22,6 +22,8 @@ models: iac-author: gemini/gemini-3.1-pro-preview validator: gemini/gemini-3.1-pro-preview lesson-curator: gemini/gemini-3.1-pro-preview + # The same reading the validator makes of a design's coverage, made earlier. + coverage-reviewer: gemini/gemini-3.1-pro-preview # A lighter model for a role's first attempt, used when MODEL_TIERS is on. Only a # role whose work is checked by something else belongs here: the code is checked # by terraform validate and by the validator. The validator is the check, and a @@ -36,6 +38,7 @@ models: iac-author: 300 validator: 300 # timed out five times at 180 s in a run of 2026-10-07 packager: 300 + coverage-reviewer: 300 # Skills each role may load. Role files stay provider-neutral; the bundle names Skills. role_skills: diff --git a/tests/agents/test_roles.py b/tests/agents/test_roles.py index 890e542..8bfb477 100644 --- a/tests/agents/test_roles.py +++ b/tests/agents/test_roles.py @@ -21,8 +21,9 @@ def test_every_engine_role_and_the_pillar_advisor_is_defined(bundle): expected = {"orchestrator", "reply-writer", PILLAR_ADVISOR} expected |= {spec.specialist for spec in DEPENDENCY_GRAPH.values()} - # The lesson curator runs outside any Execution; the engine does not need it. - assert set(roles) == expected | {"lesson-curator"} + # The lesson curator runs outside any Execution, and the coverage reviewer only + # when the deployment switches its review on; the engine needs neither. + assert set(roles) == expected | {"lesson-curator", "coverage-reviewer"} for role in roles.values(): assert role.instructions.strip() diff --git a/tests/execution/test_design_review.py b/tests/execution/test_design_review.py new file mode 100644 index 0000000..41cca99 --- /dev/null +++ b/tests/execution/test_design_review.py @@ -0,0 +1,150 @@ +"""A model reviews a design's coverage before it is approved (iteration 6 step 13).""" + +from langgraph.checkpoint.memory import InMemorySaver + +from app.adk.fake_runtime import SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission +from app.adk.roles import COVERAGE_REVIEWER, load_roles +from app.adk.runtime import NoDecider, RoleSpec +from app.agent import AGENTS_DIR, ExecutionEngine +from app.nodes.router import Router, Thresholds +from app.providers import load_provider +from app.recorder import InMemoryRecorder +from app.tools.design_review import DesignCoverageReview +from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind + +K = DeliverableKind +GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]} +UPSTREAM = { + "requirements": { + "attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]} + }, + "design_recommendations": { + "attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]} + }, +} +REVIEWER = RoleSpec(COVERAGE_REVIEWER, "Review.", budget=6) +GAP = { + "source": "NFR-1", + "applies_to": "E-1", + "statement": "min_instance_count is at least 2", +} + + +def reviewer(*gaps: dict, seen: dict | None = None) -> FakeAgentRuntime: + def turn(task): + if seen is not None: + seen.update(task.context) + return [Call("report_gaps", {"gaps": list(gaps)})] + + return FakeAgentRuntime({COVERAGE_REVIEWER: [turn, turn]}) + + +async def test_the_reviewer_is_given_the_constraints_and_the_design(): + seen: dict = {} + review = DesignCoverageReview(reviewer(GAP, seen=seen), REVIEWER) + + concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") + + assert concerns == ( + "NFR-1: no Check verifies this on E-1: min_instance_count is at least 2", + ) + # Non-functional requirements and recommendations; a functional requirement + # is reached through its components, not through a Check of its own. + assert [item["id"] for item in seen["constraints"]] == ["NFR-1", "DR-1"] + assert {e["id"] for e in seen["target_architecture"]["elements"]} >= {"E-1"} + + +async def test_a_gap_about_something_that_does_not_exist_is_dropped(): + invented = [ + {**GAP, "source": "NFR-99"}, + {**GAP, "applies_to": "E-99"}, + {**GAP, "statement": " "}, + "not a gap", + ] + review = DesignCoverageReview(reviewer(*invented, GAP), REVIEWER) + + concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") + + assert len(concerns) == 1 and concerns[0].startswith("NFR-1:") + + +async def test_a_design_is_reviewed_once_in_an_execution(): + runtime = reviewer(GAP) + review = DesignCoverageReview(runtime, REVIEWER) + + first = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") + again = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") + other = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-2") + + assert first and other and again == () + assert len(runtime.runs) == 2 + + +async def test_nothing_is_reviewed_but_a_readable_target_architecture(): + runtime = reviewer(GAP) + review = DesignCoverageReview(runtime, REVIEWER) + + assert await review(K.DEPLOYMENT_GUIDANCE, GRAPH, UPSTREAM, "a") == () + assert await review(K.TARGET_ARCHITECTURE, {}, UPSTREAM, "b") == () + assert await review(K.TARGET_ARCHITECTURE, GRAPH, None, "c") == () + assert runtime.runs == [] + + +async def test_a_complete_design_raises_nothing(): + review = DesignCoverageReview(reviewer(), REVIEWER) + + assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") == () + + +async def test_the_gaps_go_back_to_the_target_architect_before_the_design_stands( + tmp_path, +): + roles = { + "orchestrator": RoleSpec("orchestrator", "Unused.", budget=40), + **{ + spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15) + for spec in DEPENDENCY_GRAPH.values() + }, + } + architect = DEPENDENCY_GRAPH[K.TARGET_ARCHITECTURE].specialist + design = sample_submission(K.TARGET_ARCHITECTURE) + runtime = FakeAgentRuntime( + { + **{ + spec.specialist: [[sample_submission(kind)]] + for kind, spec in DEPENDENCY_GRAPH.items() + }, + # Submitted twice: once into the review, once after reading it. + architect: [[design, design]], + COVERAGE_REVIEWER: [[Call("report_gaps", {"gaps": [GAP]})]], + } + ) + engine = ExecutionEngine( + runtime=runtime, + roles=roles, + recorder=InMemoryRecorder(), + checkpointer=InMemorySaver(), + output_root=tmp_path, + orchestrator=Router(NoDecider(), None, Thresholds()), + submission_review=DesignCoverageReview(runtime, REVIEWER), + ) + + result = await engine.start("Ingest webhooks", pre_approved=True) + + assert result.status == "completed" + run = next(run for run in runtime.runs if run.role == architect) + first, second = (reply for _, reply in run.results) + assert first.startswith("Not submitted. A review found: NFR-1: no Check verifies") + assert not second.startswith("Not submitted") + # No Revision was needed: the gap was dealt with before anything was approved. + assert not [d for d in result.state["decisions"] if "revise" in d["action"]] + + +def test_the_reviewer_role_loads_with_the_bundles_model(): + bundle = load_provider("gcp") + roles = load_roles(AGENTS_DIR, bundle) + + role = roles[COVERAGE_REVIEWER] + assert role.model == bundle.models.for_role(COVERAGE_REVIEWER) + assert role.model == bundle.models.for_role("validator") + assert role.tools == () and role.model_timeout_seconds == 300 diff --git a/tests/execution/test_validation_in_pieces.py b/tests/execution/test_validation_in_pieces.py index 91fd10e..c20f15d 100644 --- a/tests/execution/test_validation_in_pieces.py +++ b/tests/execution/test_validation_in_pieces.py @@ -148,6 +148,9 @@ def test_the_settings_a_check_names_are_read_from_its_statement(): "template.scaling.min_instance_count", ) assert named_settings("startup_cpu_boost is true") == ("startup_cpu_boost",) + # A value in backticks is not a setting. + assert named_settings("`enable_cdn` is `false`") == ("enable_cdn",) + assert named_settings("`deletion_protection` is `true`") == ("deletion_protection",) # Values, roles and plain prose name no setting. assert named_settings("ingress is INGRESS_TRAFFIC_ALL") == () assert named_settings("roles/run.invoker is granted to allUsers") == ()