A reviewer role reads the requirements and recommendations against the Checks when the Target architecture is first submitted. Three runs of case 03: it did not stop the validator sending the design back, and the designs it reviewed ended with 87 and 93 Checks against 52, so they cost more. It stays behind DESIGN_COVERAGE_REVIEW, off (iteration 6 step 13). Also: the schema review no longer reads `true` or `false` in backticks as a setting name. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
151 lines
5.6 KiB
Python
151 lines
5.6 KiB
Python
"""A model reviews a design's coverage before it is approved (iteration 6 step 13)."""
|
|
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from app.adk.fake_runtime import SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission
|
|
from app.adk.roles import COVERAGE_REVIEWER, load_roles
|
|
from app.adk.runtime import NoDecider, RoleSpec
|
|
from app.agent import AGENTS_DIR, ExecutionEngine
|
|
from app.nodes.router import Router, Thresholds
|
|
from app.providers import load_provider
|
|
from app.recorder import InMemoryRecorder
|
|
from app.tools.design_review import DesignCoverageReview
|
|
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
|
|
|
K = DeliverableKind
|
|
GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]}
|
|
UPSTREAM = {
|
|
"requirements": {
|
|
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
|
},
|
|
"design_recommendations": {
|
|
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
|
|
},
|
|
}
|
|
REVIEWER = RoleSpec(COVERAGE_REVIEWER, "Review.", budget=6)
|
|
GAP = {
|
|
"source": "NFR-1",
|
|
"applies_to": "E-1",
|
|
"statement": "min_instance_count is at least 2",
|
|
}
|
|
|
|
|
|
def reviewer(*gaps: dict, seen: dict | None = None) -> FakeAgentRuntime:
|
|
def turn(task):
|
|
if seen is not None:
|
|
seen.update(task.context)
|
|
return [Call("report_gaps", {"gaps": list(gaps)})]
|
|
|
|
return FakeAgentRuntime({COVERAGE_REVIEWER: [turn, turn]})
|
|
|
|
|
|
async def test_the_reviewer_is_given_the_constraints_and_the_design():
|
|
seen: dict = {}
|
|
review = DesignCoverageReview(reviewer(GAP, seen=seen), REVIEWER)
|
|
|
|
concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
|
|
|
|
assert concerns == (
|
|
"NFR-1: no Check verifies this on E-1: min_instance_count is at least 2",
|
|
)
|
|
# Non-functional requirements and recommendations; a functional requirement
|
|
# is reached through its components, not through a Check of its own.
|
|
assert [item["id"] for item in seen["constraints"]] == ["NFR-1", "DR-1"]
|
|
assert {e["id"] for e in seen["target_architecture"]["elements"]} >= {"E-1"}
|
|
|
|
|
|
async def test_a_gap_about_something_that_does_not_exist_is_dropped():
|
|
invented = [
|
|
{**GAP, "source": "NFR-99"},
|
|
{**GAP, "applies_to": "E-99"},
|
|
{**GAP, "statement": " "},
|
|
"not a gap",
|
|
]
|
|
review = DesignCoverageReview(reviewer(*invented, GAP), REVIEWER)
|
|
|
|
concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
|
|
|
|
assert len(concerns) == 1 and concerns[0].startswith("NFR-1:")
|
|
|
|
|
|
async def test_a_design_is_reviewed_once_in_an_execution():
|
|
runtime = reviewer(GAP)
|
|
review = DesignCoverageReview(runtime, REVIEWER)
|
|
|
|
first = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
|
|
again = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1")
|
|
other = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-2")
|
|
|
|
assert first and other and again == ()
|
|
assert len(runtime.runs) == 2
|
|
|
|
|
|
async def test_nothing_is_reviewed_but_a_readable_target_architecture():
|
|
runtime = reviewer(GAP)
|
|
review = DesignCoverageReview(runtime, REVIEWER)
|
|
|
|
assert await review(K.DEPLOYMENT_GUIDANCE, GRAPH, UPSTREAM, "a") == ()
|
|
assert await review(K.TARGET_ARCHITECTURE, {}, UPSTREAM, "b") == ()
|
|
assert await review(K.TARGET_ARCHITECTURE, GRAPH, None, "c") == ()
|
|
assert runtime.runs == []
|
|
|
|
|
|
async def test_a_complete_design_raises_nothing():
|
|
review = DesignCoverageReview(reviewer(), REVIEWER)
|
|
|
|
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") == ()
|
|
|
|
|
|
async def test_the_gaps_go_back_to_the_target_architect_before_the_design_stands(
|
|
tmp_path,
|
|
):
|
|
roles = {
|
|
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
architect = DEPENDENCY_GRAPH[K.TARGET_ARCHITECTURE].specialist
|
|
design = sample_submission(K.TARGET_ARCHITECTURE)
|
|
runtime = FakeAgentRuntime(
|
|
{
|
|
**{
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
},
|
|
# Submitted twice: once into the review, once after reading it.
|
|
architect: [[design, design]],
|
|
COVERAGE_REVIEWER: [[Call("report_gaps", {"gaps": [GAP]})]],
|
|
}
|
|
)
|
|
engine = ExecutionEngine(
|
|
runtime=runtime,
|
|
roles=roles,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
submission_review=DesignCoverageReview(runtime, REVIEWER),
|
|
)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
|
|
assert result.status == "completed"
|
|
run = next(run for run in runtime.runs if run.role == architect)
|
|
first, second = (reply for _, reply in run.results)
|
|
assert first.startswith("Not submitted. A review found: NFR-1: no Check verifies")
|
|
assert not second.startswith("Not submitted")
|
|
# No Revision was needed: the gap was dealt with before anything was approved.
|
|
assert not [d for d in result.state["decisions"] if "revise" in d["action"]]
|
|
|
|
|
|
def test_the_reviewer_role_loads_with_the_bundles_model():
|
|
bundle = load_provider("gcp")
|
|
roles = load_roles(AGENTS_DIR, bundle)
|
|
|
|
role = roles[COVERAGE_REVIEWER]
|
|
assert role.model == bundle.models.for_role(COVERAGE_REVIEWER)
|
|
assert role.model == bundle.models.for_role("validator")
|
|
assert role.tools == () and role.model_timeout_seconds == 300
|