feat: a design-time coverage review, measured and left off

A reviewer role reads the requirements and recommendations against the Checks
when the Target architecture is first submitted. Three runs of case 03: it did
not stop the validator sending the design back, and the designs it reviewed
ended with 87 and 93 Checks against 52, so they cost more. It stays behind
DESIGN_COVERAGE_REVIEW, off (iteration 6 step 13).

Also: the schema review no longer reads `true` or `false` in backticks as a
setting name.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-10 12:33:06 +01:00
parent 1cf1234143
commit ee1ba38867
13 changed files with 401 additions and 6 deletions

View File

@@ -21,8 +21,9 @@ def test_every_engine_role_and_the_pillar_advisor_is_defined(bundle):
expected = {"orchestrator", "reply-writer", PILLAR_ADVISOR}
expected |= {spec.specialist for spec in DEPENDENCY_GRAPH.values()}
# The lesson curator runs outside any Execution; the engine does not need it.
assert set(roles) == expected | {"lesson-curator"}
# The lesson curator runs outside any Execution, and the coverage reviewer only
# when the deployment switches its review on; the engine needs neither.
assert set(roles) == expected | {"lesson-curator", "coverage-reviewer"}
for role in roles.values():
assert role.instructions.strip()

View File

@@ -0,0 +1,150 @@
"""A model reviews a design's coverage before it is approved (iteration 6 step 13)."""
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission
from app.adk.roles import COVERAGE_REVIEWER, load_roles
from app.adk.runtime import NoDecider, RoleSpec
from app.agent import AGENTS_DIR, ExecutionEngine
from app.nodes.router import Router, Thresholds
from app.providers import load_provider
from app.recorder import InMemoryRecorder
from app.tools.design_review import DesignCoverageReview
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
K = DeliverableKind
GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]}
UPSTREAM = {
"requirements": {
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
},
"design_recommendations": {
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
},
}
REVIEWER = RoleSpec(COVERAGE_REVIEWER, "Review.", budget=6)
GAP = {
"source": "NFR-1",
"applies_to": "E-1",
"statement": "min_instance_count is at least 2",
}
def reviewer(*gaps: dict, seen: dict | None = None) -> FakeAgentRuntime:
def turn(task):
if seen is not None:
seen.update(task.context)
return [Call("report_gaps", {"gaps": list(gaps)})]
return FakeAgentRuntime({COVERAGE_REVIEWER: [turn, turn]})
async def test_the_reviewer_is_given_the_constraints_and_the_design():
seen: dict = {}
review = DesignCoverageReview(reviewer(GAP, seen=seen), REVIEWER)
concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
assert concerns == (
"NFR-1: no Check verifies this on E-1: min_instance_count is at least 2",
)
# Non-functional requirements and recommendations; a functional requirement
# is reached through its components, not through a Check of its own.
assert [item["id"] for item in seen["constraints"]] == ["NFR-1", "DR-1"]
assert {e["id"] for e in seen["target_architecture"]["elements"]} >= {"E-1"}
async def test_a_gap_about_something_that_does_not_exist_is_dropped():
invented = [
{**GAP, "source": "NFR-99"},
{**GAP, "applies_to": "E-99"},
{**GAP, "statement": " "},
"not a gap",
]
review = DesignCoverageReview(reviewer(*invented, GAP), REVIEWER)
concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
assert len(concerns) == 1 and concerns[0].startswith("NFR-1:")
async def test_a_design_is_reviewed_once_in_an_execution():
runtime = reviewer(GAP)
review = DesignCoverageReview(runtime, REVIEWER)
first = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
again = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
other = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-2")
assert first and other and again == ()
assert len(runtime.runs) == 2
async def test_nothing_is_reviewed_but_a_readable_target_architecture():
runtime = reviewer(GAP)
review = DesignCoverageReview(runtime, REVIEWER)
assert await review(K.DEPLOYMENT_GUIDANCE, GRAPH, UPSTREAM, "a") == ()
assert await review(K.TARGET_ARCHITECTURE, {}, UPSTREAM, "b") == ()
assert await review(K.TARGET_ARCHITECTURE, GRAPH, None, "c") == ()
assert runtime.runs == []
async def test_a_complete_design_raises_nothing():
review = DesignCoverageReview(reviewer(), REVIEWER)
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") == ()
async def test_the_gaps_go_back_to_the_target_architect_before_the_design_stands(
tmp_path,
):
roles = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
architect = DEPENDENCY_GRAPH[K.TARGET_ARCHITECTURE].specialist
design = sample_submission(K.TARGET_ARCHITECTURE)
runtime = FakeAgentRuntime(
{
**{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
},
# Submitted twice: once into the review, once after reading it.
architect: [[design, design]],
COVERAGE_REVIEWER: [[Call("report_gaps", {"gaps": [GAP]})]],
}
)
engine = ExecutionEngine(
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
submission_review=DesignCoverageReview(runtime, REVIEWER),
)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
run = next(run for run in runtime.runs if run.role == architect)
first, second = (reply for _, reply in run.results)
assert first.startswith("Not submitted. A review found: NFR-1: no Check verifies")
assert not second.startswith("Not submitted")
# No Revision was needed: the gap was dealt with before anything was approved.
assert not [d for d in result.state["decisions"] if "revise" in d["action"]]
def test_the_reviewer_role_loads_with_the_bundles_model():
bundle = load_provider("gcp")
roles = load_roles(AGENTS_DIR, bundle)
role = roles[COVERAGE_REVIEWER]
assert role.model == bundle.models.for_role(COVERAGE_REVIEWER)
assert role.model == bundle.models.for_role("validator")
assert role.tools == () and role.model_timeout_seconds == 300

View File

@@ -148,6 +148,9 @@ def test_the_settings_a_check_names_are_read_from_its_statement():
"template.scaling.min_instance_count",
)
assert named_settings("startup_cpu_boost is true") == ("startup_cpu_boost",)
# A value in backticks is not a setting.
assert named_settings("`enable_cdn` is `false`") == ("enable_cdn",)
assert named_settings("`deletion_protection` is `true`") == ("deletion_protection",)
# Values, roles and plain prose name no setting.
assert named_settings("ingress is INGRESS_TRAFFIC_ALL") == ()
assert named_settings("roles/run.invoker is granted to allUsers") == ()