Files
gcp_solution_architecture_a…/tests/execution/test_design_review.py
Jonathan Boniface ee1ba38867 feat: a design-time coverage review, measured and left off
A reviewer role reads the requirements and recommendations against the Checks
when the Target architecture is first submitted. Three runs of case 03: it did
not stop the validator sending the design back, and the designs it reviewed
ended with 87 and 93 Checks against 52, so they cost more. It stays behind
DESIGN_COVERAGE_REVIEW, off (iteration 6 step 13).

Also: the schema review no longer reads `true` or `false` in backticks as a
setting name.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-10 12:33:06 +01:00

151 lines
5.6 KiB
Python

"""A model reviews a design's coverage before it is approved (iteration 6 step 13)."""
from langgraph.checkpoint.memory import InMemorySaver
from app.adk.fake_runtime import SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission
from app.adk.roles import COVERAGE_REVIEWER, load_roles
from app.adk.runtime import NoDecider, RoleSpec
from app.agent import AGENTS_DIR, ExecutionEngine
from app.nodes.router import Router, Thresholds
from app.providers import load_provider
from app.recorder import InMemoryRecorder
from app.tools.design_review import DesignCoverageReview
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
K = DeliverableKind
GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]}
UPSTREAM = {
"requirements": {
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
},
"design_recommendations": {
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
},
}
REVIEWER = RoleSpec(COVERAGE_REVIEWER, "Review.", budget=6)
GAP = {
"source": "NFR-1",
"applies_to": "E-1",
"statement": "min_instance_count is at least 2",
}
def reviewer(*gaps: dict, seen: dict | None = None) -> FakeAgentRuntime:
def turn(task):
if seen is not None:
seen.update(task.context)
return [Call("report_gaps", {"gaps": list(gaps)})]
return FakeAgentRuntime({COVERAGE_REVIEWER: [turn, turn]})
async def test_the_reviewer_is_given_the_constraints_and_the_design():
seen: dict = {}
review = DesignCoverageReview(reviewer(GAP, seen=seen), REVIEWER)
concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
assert concerns == (
"NFR-1: no Check verifies this on E-1: min_instance_count is at least 2",
)
# Non-functional requirements and recommendations; a functional requirement
# is reached through its components, not through a Check of its own.
assert [item["id"] for item in seen["constraints"]] == ["NFR-1", "DR-1"]
assert {e["id"] for e in seen["target_architecture"]["elements"]} >= {"E-1"}
async def test_a_gap_about_something_that_does_not_exist_is_dropped():
invented = [
{**GAP, "source": "NFR-99"},
{**GAP, "applies_to": "E-99"},
{**GAP, "statement": " "},
"not a gap",
]
review = DesignCoverageReview(reviewer(*invented, GAP), REVIEWER)
concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
assert len(concerns) == 1 and concerns[0].startswith("NFR-1:")
async def test_a_design_is_reviewed_once_in_an_execution():
runtime = reviewer(GAP)
review = DesignCoverageReview(runtime, REVIEWER)
first = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
again = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
other = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-2")
assert first and other and again == ()
assert len(runtime.runs) == 2
async def test_nothing_is_reviewed_but_a_readable_target_architecture():
runtime = reviewer(GAP)
review = DesignCoverageReview(runtime, REVIEWER)
assert await review(K.DEPLOYMENT_GUIDANCE, GRAPH, UPSTREAM, "a") == ()
assert await review(K.TARGET_ARCHITECTURE, {}, UPSTREAM, "b") == ()
assert await review(K.TARGET_ARCHITECTURE, GRAPH, None, "c") == ()
assert runtime.runs == []
async def test_a_complete_design_raises_nothing():
review = DesignCoverageReview(reviewer(), REVIEWER)
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") == ()
async def test_the_gaps_go_back_to_the_target_architect_before_the_design_stands(
tmp_path,
):
roles = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
architect = DEPENDENCY_GRAPH[K.TARGET_ARCHITECTURE].specialist
design = sample_submission(K.TARGET_ARCHITECTURE)
runtime = FakeAgentRuntime(
{
**{
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
},
# Submitted twice: once into the review, once after reading it.
architect: [[design, design]],
COVERAGE_REVIEWER: [[Call("report_gaps", {"gaps": [GAP]})]],
}
)
engine = ExecutionEngine(
runtime=runtime,
roles=roles,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
submission_review=DesignCoverageReview(runtime, REVIEWER),
)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
run = next(run for run in runtime.runs if run.role == architect)
first, second = (reply for _, reply in run.results)
assert first.startswith("Not submitted. A review found: NFR-1: no Check verifies")
assert not second.startswith("Not submitted")
# No Revision was needed: the gap was dealt with before anything was approved.
assert not [d for d in result.state["decisions"] if "revise" in d["action"]]
def test_the_reviewer_role_loads_with_the_bundles_model():
bundle = load_provider("gcp")
roles = load_roles(AGENTS_DIR, bundle)
role = roles[COVERAGE_REVIEWER]
assert role.model == bundle.models.for_role(COVERAGE_REVIEWER)
assert role.model == bundle.models.for_role("validator")
assert role.tools == () and role.model_timeout_seconds == 300