feat: a design-time coverage review, measured and left off
A reviewer role reads the requirements and recommendations against the Checks when the Target architecture is first submitted. Three runs of case 03: it did not stop the validator sending the design back, and the designs it reviewed ended with 87 and 93 Checks against 52, so they cost more. It stays behind DESIGN_COVERAGE_REVIEW, off (iteration 6 step 13). Also: the schema review no longer reads `true` or `false` in backticks as a setting name. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -21,8 +21,9 @@ def test_every_engine_role_and_the_pillar_advisor_is_defined(bundle):
|
||||
|
||||
expected = {"orchestrator", "reply-writer", PILLAR_ADVISOR}
|
||||
expected |= {spec.specialist for spec in DEPENDENCY_GRAPH.values()}
|
||||
# The lesson curator runs outside any Execution; the engine does not need it.
|
||||
assert set(roles) == expected | {"lesson-curator"}
|
||||
# The lesson curator runs outside any Execution, and the coverage reviewer only
|
||||
# when the deployment switches its review on; the engine needs neither.
|
||||
assert set(roles) == expected | {"lesson-curator", "coverage-reviewer"}
|
||||
for role in roles.values():
|
||||
assert role.instructions.strip()
|
||||
|
||||
|
||||
150
tests/execution/test_design_review.py
Normal file
150
tests/execution/test_design_review.py
Normal file
@@ -0,0 +1,150 @@
|
||||
"""A model reviews a design's coverage before it is approved (iteration 6 step 13)."""
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission
|
||||
from app.adk.roles import COVERAGE_REVIEWER, load_roles
|
||||
from app.adk.runtime import NoDecider, RoleSpec
|
||||
from app.agent import AGENTS_DIR, ExecutionEngine
|
||||
from app.nodes.router import Router, Thresholds
|
||||
from app.providers import load_provider
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.design_review import DesignCoverageReview
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
||||
|
||||
K = DeliverableKind
|
||||
GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]}
|
||||
UPSTREAM = {
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
||||
},
|
||||
"design_recommendations": {
|
||||
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
|
||||
},
|
||||
}
|
||||
REVIEWER = RoleSpec(COVERAGE_REVIEWER, "Review.", budget=6)
|
||||
GAP = {
|
||||
"source": "NFR-1",
|
||||
"applies_to": "E-1",
|
||||
"statement": "min_instance_count is at least 2",
|
||||
}
|
||||
|
||||
|
||||
def reviewer(*gaps: dict, seen: dict | None = None) -> FakeAgentRuntime:
|
||||
def turn(task):
|
||||
if seen is not None:
|
||||
seen.update(task.context)
|
||||
return [Call("report_gaps", {"gaps": list(gaps)})]
|
||||
|
||||
return FakeAgentRuntime({COVERAGE_REVIEWER: [turn, turn]})
|
||||
|
||||
|
||||
async def test_the_reviewer_is_given_the_constraints_and_the_design():
|
||||
seen: dict = {}
|
||||
review = DesignCoverageReview(reviewer(GAP, seen=seen), REVIEWER)
|
||||
|
||||
concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
|
||||
|
||||
assert concerns == (
|
||||
"NFR-1: no Check verifies this on E-1: min_instance_count is at least 2",
|
||||
)
|
||||
# Non-functional requirements and recommendations; a functional requirement
|
||||
# is reached through its components, not through a Check of its own.
|
||||
assert [item["id"] for item in seen["constraints"]] == ["NFR-1", "DR-1"]
|
||||
assert {e["id"] for e in seen["target_architecture"]["elements"]} >= {"E-1"}
|
||||
|
||||
|
||||
async def test_a_gap_about_something_that_does_not_exist_is_dropped():
|
||||
invented = [
|
||||
{**GAP, "source": "NFR-99"},
|
||||
{**GAP, "applies_to": "E-99"},
|
||||
{**GAP, "statement": " "},
|
||||
"not a gap",
|
||||
]
|
||||
review = DesignCoverageReview(reviewer(*invented, GAP), REVIEWER)
|
||||
|
||||
concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
|
||||
|
||||
assert len(concerns) == 1 and concerns[0].startswith("NFR-1:")
|
||||
|
||||
|
||||
async def test_a_design_is_reviewed_once_in_an_execution():
|
||||
runtime = reviewer(GAP)
|
||||
review = DesignCoverageReview(runtime, REVIEWER)
|
||||
|
||||
first = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
|
||||
again = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
|
||||
other = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-2")
|
||||
|
||||
assert first and other and again == ()
|
||||
assert len(runtime.runs) == 2
|
||||
|
||||
|
||||
async def test_nothing_is_reviewed_but_a_readable_target_architecture():
|
||||
runtime = reviewer(GAP)
|
||||
review = DesignCoverageReview(runtime, REVIEWER)
|
||||
|
||||
assert await review(K.DEPLOYMENT_GUIDANCE, GRAPH, UPSTREAM, "a") == ()
|
||||
assert await review(K.TARGET_ARCHITECTURE, {}, UPSTREAM, "b") == ()
|
||||
assert await review(K.TARGET_ARCHITECTURE, GRAPH, None, "c") == ()
|
||||
assert runtime.runs == []
|
||||
|
||||
|
||||
async def test_a_complete_design_raises_nothing():
|
||||
review = DesignCoverageReview(reviewer(), REVIEWER)
|
||||
|
||||
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") == ()
|
||||
|
||||
|
||||
async def test_the_gaps_go_back_to_the_target_architect_before_the_design_stands(
|
||||
tmp_path,
|
||||
):
|
||||
roles = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
architect = DEPENDENCY_GRAPH[K.TARGET_ARCHITECTURE].specialist
|
||||
design = sample_submission(K.TARGET_ARCHITECTURE)
|
||||
runtime = FakeAgentRuntime(
|
||||
{
|
||||
**{
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
},
|
||||
# Submitted twice: once into the review, once after reading it.
|
||||
architect: [[design, design]],
|
||||
COVERAGE_REVIEWER: [[Call("report_gaps", {"gaps": [GAP]})]],
|
||||
}
|
||||
)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=roles,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
submission_review=DesignCoverageReview(runtime, REVIEWER),
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
|
||||
assert result.status == "completed"
|
||||
run = next(run for run in runtime.runs if run.role == architect)
|
||||
first, second = (reply for _, reply in run.results)
|
||||
assert first.startswith("Not submitted. A review found: NFR-1: no Check verifies")
|
||||
assert not second.startswith("Not submitted")
|
||||
# No Revision was needed: the gap was dealt with before anything was approved.
|
||||
assert not [d for d in result.state["decisions"] if "revise" in d["action"]]
|
||||
|
||||
|
||||
def test_the_reviewer_role_loads_with_the_bundles_model():
|
||||
bundle = load_provider("gcp")
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
|
||||
role = roles[COVERAGE_REVIEWER]
|
||||
assert role.model == bundle.models.for_role(COVERAGE_REVIEWER)
|
||||
assert role.model == bundle.models.for_role("validator")
|
||||
assert role.tools == () and role.model_timeout_seconds == 300
|
||||
@@ -148,6 +148,9 @@ def test_the_settings_a_check_names_are_read_from_its_statement():
|
||||
"template.scaling.min_instance_count",
|
||||
)
|
||||
assert named_settings("startup_cpu_boost is true") == ("startup_cpu_boost",)
|
||||
# A value in backticks is not a setting.
|
||||
assert named_settings("`enable_cdn` is `false`") == ("enable_cdn",)
|
||||
assert named_settings("`deletion_protection` is `true`") == ("deletion_protection",)
|
||||
# Values, roles and plain prose name no setting.
|
||||
assert named_settings("ingress is INGRESS_TRAFFIC_ALL") == ()
|
||||
assert named_settings("roles/run.invoker is granted to allUsers") == ()
|
||||
|
||||
Reference in New Issue
Block a user