feat: keep an unchanged description, list conditions, review coverage with Jev
After case 02 (completed at 100%, but with two Revisions of the Target architecture for gaps validation found one pass at a time): - A Deliverable stands while what it was written from is unchanged. The architecture description is written from the Elements and Relationships, so a Revision that only adds or rewords Checks no longer runs the describer. - Non-functional requirements and recommendations list their conditions, one entry per separate thing that must be true; submission requires them. - A coverage review at submission (JEV_COVERAGE_REVIEW, off by default): for each condition, Jev chooses which Check verifies it, or none. Asked as one broad yes-or-no per requirement it could not tell a complete design from one with known gaps; asked as a choice per condition it raised all four known gaps of case 02 and nothing on the complete graph. An opt-in live test (LIVE_JEV=1) repeats that. - The validator is told to report every gap in its first pass. Plans record the case 02 run and both forms of the review with figures. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
401
tests/execution/test_coverage_review.py
Normal file
401
tests/execution/test_coverage_review.py
Normal file
@@ -0,0 +1,401 @@
|
||||
"""Two savings on a Revision of the design.
|
||||
|
||||
A coverage review at submission finds what validation would otherwise find later,
|
||||
at the cost of a Revision. And a Deliverable written from the shape of the design is
|
||||
not written again when a Revision only changes its Checks.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import (
|
||||
SAMPLE_DATA,
|
||||
Call,
|
||||
FakeAgentRuntime,
|
||||
FakeDecider,
|
||||
sample_submission,
|
||||
)
|
||||
from app.adk.runtime import Answer, NoDecider, RoleSpec, Submitted
|
||||
from app.agent import ExecutionEngine
|
||||
from app.nodes.router import Router, Thresholds
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.coverage import CoverageReview
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
||||
|
||||
K = DeliverableKind
|
||||
GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]}
|
||||
UPSTREAM = {
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
||||
},
|
||||
"technical_decomposition": {
|
||||
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
|
||||
},
|
||||
"product_selection": {
|
||||
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
|
||||
},
|
||||
"design_recommendations": {
|
||||
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
|
||||
},
|
||||
}
|
||||
CONTENT = "## Design rationale\nReasons.\n"
|
||||
|
||||
|
||||
# --- the review ------------------------------------------------------------------------
|
||||
|
||||
|
||||
def choice(option: str, probability: float) -> Answer:
|
||||
return Answer(option, probability, {option: probability})
|
||||
|
||||
|
||||
async def test_the_review_asks_which_check_verifies_each_condition():
|
||||
decider = FakeDecider({"NFR-1#1": choice("K-1", 0.9), "DR-1#1": choice("K-2", 0.9)})
|
||||
|
||||
concerns = await CoverageReview(decider)(
|
||||
K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1"
|
||||
)
|
||||
|
||||
state, questions, scope = decider.asked[0]
|
||||
assert concerns == ()
|
||||
assert scope == "exec-1"
|
||||
assert "E-1 Ingress service (Example Run, example_run_service)" in state
|
||||
# Constraints only: a functional requirement is carried by its component.
|
||||
assert set(questions) == {"NFR-1#1", "DR-1#1"}
|
||||
question = questions["NFR-1#1"]
|
||||
assert question.kind == "choice" and question.options == ("K-1", "none")
|
||||
assert 'Condition: "The service stays available when one instance fails."' in (
|
||||
question.instructions
|
||||
)
|
||||
assert "- K-1 on Ingress service: The service runs at least two instances." in (
|
||||
question.instructions
|
||||
)
|
||||
|
||||
|
||||
async def test_a_condition_no_check_verifies_is_raised_with_the_checks_it_has():
|
||||
decider = FakeDecider({"NFR-1#1": choice("none", 0.9), "DR-1#1": choice("K-2", 1)})
|
||||
|
||||
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM)
|
||||
|
||||
condition = "The service stays available when one instance fails."
|
||||
assert concerns == (f'NFR-1: no Check verifies "{condition}" (its Checks: K-1)',)
|
||||
|
||||
|
||||
async def test_an_uncertain_none_is_not_raised():
|
||||
decider = FakeDecider({"NFR-1#1": choice("none", 0.45), "DR-1#1": choice("K-2", 1)})
|
||||
|
||||
review = CoverageReview(decider, none_at=0.6)
|
||||
|
||||
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
||||
|
||||
|
||||
async def test_each_condition_of_a_requirement_is_asked_about_separately():
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
requirements[1]["conditions"] = [
|
||||
"Survives one instance failing.",
|
||||
"Survives a zone failing.",
|
||||
]
|
||||
upstream = {
|
||||
**UPSTREAM,
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": json.dumps(requirements)}
|
||||
},
|
||||
}
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"NFR-1#1": choice("K-1", 0.9),
|
||||
"NFR-1#2": choice("none", 0.8),
|
||||
"DR-1#1": choice("K-2", 1),
|
||||
}
|
||||
)
|
||||
|
||||
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
|
||||
|
||||
assert len(concerns) == 1 and "Survives a zone failing." in concerns[0]
|
||||
|
||||
|
||||
async def test_an_item_without_conditions_is_read_as_one_condition_its_text():
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
del requirements[1]["conditions"]
|
||||
upstream = {
|
||||
**UPSTREAM,
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": json.dumps(requirements)}
|
||||
},
|
||||
}
|
||||
decider = FakeDecider({"NFR-1#1": choice("K-1", 1), "DR-1#1": choice("K-2", 1)})
|
||||
|
||||
await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
|
||||
|
||||
assert 'Condition: "99.9% availability."' in (
|
||||
decider.asked[0][1]["NFR-1#1"].instructions
|
||||
)
|
||||
|
||||
|
||||
async def test_no_decision_and_other_deliverables_raise_nothing():
|
||||
review = CoverageReview(NoDecider())
|
||||
decider = FakeDecider({})
|
||||
|
||||
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
||||
assert await CoverageReview(decider)(K.REQUIREMENTS, {}, UPSTREAM) == ()
|
||||
assert await CoverageReview(decider)(K.TARGET_ARCHITECTURE, {}, UPSTREAM) == ()
|
||||
assert decider.asked == []
|
||||
|
||||
|
||||
async def test_a_recommendation_marked_not_applicable_is_not_reviewed():
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
graph["checks"][1]["source"] = "NFR-1"
|
||||
graph["not_applicable"] = [{"recommendation": "DR-1", "reason": "No identity."}]
|
||||
decider = FakeDecider({"NFR-1#1": choice("K-1", 1)})
|
||||
|
||||
await CoverageReview(decider)(
|
||||
K.TARGET_ARCHITECTURE,
|
||||
{"target-architecture.json": json.dumps(graph)},
|
||||
UPSTREAM,
|
||||
)
|
||||
|
||||
assert set(decider.asked[0][1]) == {"NFR-1#1"}
|
||||
|
||||
|
||||
# --- at submission ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def submit_with(review, working=None):
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(
|
||||
K.TARGET_ARCHITECTURE,
|
||||
(),
|
||||
allow_cloud_read=False,
|
||||
upstream=UPSTREAM,
|
||||
working={} if working is None else working,
|
||||
review=review,
|
||||
)
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
return submit.handler
|
||||
|
||||
|
||||
async def test_what_the_review_raises_is_returned_once_and_then_accepted():
|
||||
calls = []
|
||||
|
||||
async def review(attachments):
|
||||
calls.append(attachments)
|
||||
return ['NFR-1: no Check verifies "Two instances." (its Checks: K-1)']
|
||||
|
||||
working: dict = {}
|
||||
submit = submit_with(review, working)
|
||||
|
||||
first = await submit(content=CONTENT, attachments=dict(GRAPH))
|
||||
second = await submit(content=CONTENT)
|
||||
|
||||
assert first.outcome is None
|
||||
assert first.message.startswith("Not submitted. A review found")
|
||||
assert "NFR-1: no Check verifies" in first.message
|
||||
assert "runs once" in first.message
|
||||
# The graph it sent is kept, so it amends that and does not send it again.
|
||||
assert isinstance(second.outcome, Submitted)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_a_structural_fault_is_reported_before_any_review():
|
||||
async def review(attachments):
|
||||
raise AssertionError("not reached")
|
||||
|
||||
result = await submit_with(review)(content=CONTENT, attachments={})
|
||||
|
||||
assert "attachment: target-architecture.json" in result.message
|
||||
|
||||
|
||||
async def test_a_review_that_fails_or_raises_nothing_never_stops_a_submission():
|
||||
async def broken(attachments):
|
||||
raise RuntimeError("service down")
|
||||
|
||||
async def quiet(attachments):
|
||||
return ()
|
||||
|
||||
for review in (broken, quiet):
|
||||
result = await submit_with(review)(content=CONTENT, attachments=dict(GRAPH))
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
|
||||
|
||||
# --- a description stands while the shape of the design does ---------------------------
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def graph_with(**changes) -> Call:
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
for key, change in changes.items():
|
||||
graph[key] = change(graph[key])
|
||||
return Call(
|
||||
"submit_deliverable",
|
||||
{
|
||||
"content": CONTENT,
|
||||
"attachments": {"target-architecture.json": json.dumps(graph)},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def addition() -> Call:
|
||||
"""A validation that passes every Check and proposes one more."""
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
verdicts["proposed_additions"] = [
|
||||
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": "NFR-1"}
|
||||
]
|
||||
return Call(
|
||||
call.tool,
|
||||
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
||||
)
|
||||
|
||||
|
||||
async def run(tmp_path, second_graph: Call, **options):
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]] * 2
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
turns["target-architect"] = [
|
||||
[sample_submission(K.TARGET_ARCHITECTURE)],
|
||||
[second_graph],
|
||||
]
|
||||
turns["validator"] = [[addition()], [sample_submission(K.VALIDATION_RESULTS)]]
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
**options,
|
||||
)
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
assert result.status == "completed"
|
||||
return result, [run.role for run in runtime.runs]
|
||||
|
||||
|
||||
async def test_the_description_is_kept_when_a_revision_only_changes_checks(tmp_path):
|
||||
reworded = graph_with(
|
||||
checks=lambda checks: [
|
||||
{**checks[0], "statement": "min_instance_count is at least 2"},
|
||||
checks[1],
|
||||
]
|
||||
)
|
||||
|
||||
result, roles_run = await run(tmp_path, reworded)
|
||||
|
||||
assert roles_run.count("target-architect") == 2
|
||||
assert roles_run.count("iac-author") == 2 # the code answers the Checks
|
||||
assert roles_run.count("describer") == 1
|
||||
# The Ledger still records a second submission: the Deliverable was not skipped.
|
||||
submitted = [
|
||||
event
|
||||
for event in result.state["ledger"]["events"]
|
||||
if event["type"] == "submitted"
|
||||
and event["subject"] == "architecture_description"
|
||||
]
|
||||
assert len(submitted) == 2
|
||||
|
||||
|
||||
async def test_the_description_is_written_again_when_the_shape_changes(tmp_path):
|
||||
renamed = graph_with(
|
||||
elements=lambda elements: [{**elements[0], "name": "Edge service"}, elements[1]]
|
||||
)
|
||||
|
||||
_, roles_run = await run(tmp_path, renamed)
|
||||
|
||||
assert roles_run.count("describer") == 2
|
||||
|
||||
|
||||
async def test_the_engine_reviews_a_design_before_it_is_submitted(tmp_path):
|
||||
raised = []
|
||||
|
||||
async def review(kind, attachments, upstream, scope=""):
|
||||
if kind is K.TARGET_ARCHITECTURE and not raised:
|
||||
raised.append(scope)
|
||||
return ['NFR-1 "Two instances." (its Checks: K-1)']
|
||||
return ()
|
||||
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
turns["target-architect"] = [
|
||||
[sample_submission(K.TARGET_ARCHITECTURE)] * 2 # refused once, then accepted
|
||||
]
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
submission_review=review,
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True, execution_id="e1")
|
||||
|
||||
assert result.status == "completed"
|
||||
assert raised == ["e1"]
|
||||
architect = next(run for run in runtime.runs if run.role == "target-architect")
|
||||
assert architect.results[0][1].startswith("Not submitted. A review found")
|
||||
# What a review raises is kept as a lesson candidate, like any refusal.
|
||||
assert any(
|
||||
"A review found" in json.dumps(outcome.get("refusals"))
|
||||
for outcome in result.state["outcomes"]
|
||||
)
|
||||
|
||||
|
||||
async def test_feedback_on_the_description_itself_has_it_written_again(tmp_path):
|
||||
"""Kept only while nobody has asked for the description to change."""
|
||||
from app.nodes.execution_nodes import STANDING, make_execution_nodes
|
||||
|
||||
nodes = make_execution_nodes(
|
||||
runtime=FakeAgentRuntime(
|
||||
{"describer": [[sample_submission(K.ARCHITECTURE_DESCRIPTION, "new")]]}
|
||||
),
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
output_root=tmp_path,
|
||||
)
|
||||
upstream = {"target_architecture": {"attachments": dict(GRAPH)}}
|
||||
from app.workflows.chain import graph_structure
|
||||
|
||||
standing = {
|
||||
"basis": graph_structure(upstream),
|
||||
"feedback": [],
|
||||
"content": "old",
|
||||
"attachments": {},
|
||||
}
|
||||
task = {
|
||||
"execution_id": "e1",
|
||||
"role": "describer",
|
||||
"kind": "architecture_description",
|
||||
"attempt": 2,
|
||||
"allow_cloud_read": False,
|
||||
"working": {STANDING: standing},
|
||||
"context": {
|
||||
"dependencies": upstream,
|
||||
"feedback": ["Dependency x was revised: y"],
|
||||
},
|
||||
}
|
||||
|
||||
kept = await nodes.specialist_task(task)
|
||||
asked = await nodes.specialist_task(
|
||||
{**task, "context": {**task["context"], "feedback": ["Make the flow clearer."]}}
|
||||
)
|
||||
|
||||
assert kept["outcomes"][0]["content"] == "old"
|
||||
assert "new" in asked["outcomes"][0]["content"]
|
||||
assert asked["working"]["architecture_description"][STANDING]["feedback"] == [
|
||||
"Make the flow clearer."
|
||||
]
|
||||
Reference in New Issue
Block a user