Files
orchestration_engine/tests/execution/test_coverage_review.py

447 lines
15 KiB
Python

"""Two savings on a Revision of the design.
A coverage review at submission finds what validation would otherwise find later,
at the cost of a Revision. And a Deliverable written from the shape of the design is
not written again when a Revision only changes its Checks.
"""
import json
from langgraph.checkpoint.memory import InMemorySaver
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime, FakeDecider
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec, Submitted
from execution_engine.agent import ExecutionEngine
from execution_engine.nodes.router import Router, Thresholds
from execution_engine.recorder import InMemoryRecorder
from execution_engine.tools.execution import specialist_tools
from tests.reference_workflow.coverage import CoverageReview
from tests.reference_workflow.workflow import (
DEPENDENCY_GRAPH,
SAMPLE_DATA,
WORKFLOW,
DeliverableKind,
sample_submission,
)
K = DeliverableKind
GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]}
UPSTREAM = {
"requirements": {
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
},
"technical_decomposition": {
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
},
"product_selection": {
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
},
"design_recommendations": {
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
},
}
CONTENT = "## Design rationale\nReasons.\n"
# --- the review ------------------------------------------------------------------------
def choice(option: str, probability: float) -> Answer:
return Answer(option, probability, {option: probability})
async def test_the_review_asks_which_check_verifies_each_condition():
decider = FakeDecider({"NFR-1#1": choice("K-1", 0.9), "DR-1#1": choice("K-2", 0.9)})
concerns = await CoverageReview(decider)(
K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1"
)
state, questions, scope = decider.asked[0]
assert concerns == ()
assert scope == "exec-1"
assert "E-1 Ingress service (Example Run, example_run_service)" in state
# Constraints only: a functional requirement is carried by its component.
assert set(questions) == {"NFR-1#1", "DR-1#1"}
question = questions["NFR-1#1"]
assert question.kind == "choice" and question.options == ("K-1", "none")
assert 'Condition: "The service stays available when one instance fails."' in (
question.instructions
)
assert "- K-1 on Ingress service: The service runs at least two instances." in (
question.instructions
)
async def test_a_condition_no_check_verifies_is_raised_with_the_checks_it_has():
decider = FakeDecider({"NFR-1#1": choice("none", 0.9), "DR-1#1": choice("K-2", 1)})
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM)
condition = "The service stays available when one instance fails."
assert concerns == (f'NFR-1: no Check verifies "{condition}" (its Checks: K-1)',)
async def test_an_uncertain_none_is_not_raised():
decider = FakeDecider({"NFR-1#1": choice("none", 0.45), "DR-1#1": choice("K-2", 1)})
review = CoverageReview(decider, none_at=0.6)
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
async def test_each_condition_of_a_requirement_is_asked_about_separately():
requirements = json.loads(SAMPLE_DATA["requirements.json"])
requirements[1]["conditions"] = [
"Survives one instance failing.",
"Survives a zone failing.",
]
upstream = {
**UPSTREAM,
"requirements": {
"attachments": {"requirements.json": json.dumps(requirements)}
},
}
decider = FakeDecider(
{
"NFR-1#1": choice("K-1", 0.9),
"NFR-1#2": choice("none", 0.8),
"DR-1#1": choice("K-2", 1),
}
)
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
assert len(concerns) == 1 and "Survives a zone failing." in concerns[0]
async def test_an_item_without_conditions_is_read_as_one_condition_its_text():
requirements = json.loads(SAMPLE_DATA["requirements.json"])
del requirements[1]["conditions"]
upstream = {
**UPSTREAM,
"requirements": {
"attachments": {"requirements.json": json.dumps(requirements)}
},
}
decider = FakeDecider({"NFR-1#1": choice("K-1", 1), "DR-1#1": choice("K-2", 1)})
await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
assert 'Condition: "99.9% availability."' in (
decider.asked[0][1]["NFR-1#1"].instructions
)
async def test_no_decision_and_other_deliverables_raise_nothing():
review = CoverageReview(NoDecider())
decider = FakeDecider({})
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
assert await CoverageReview(decider)(K.REQUIREMENTS, {}, UPSTREAM) == ()
assert await CoverageReview(decider)(K.TARGET_ARCHITECTURE, {}, UPSTREAM) == ()
assert decider.asked == []
async def test_a_recommendation_marked_not_applicable_is_not_reviewed():
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
graph["checks"][1]["source"] = "NFR-1"
graph["not_applicable"] = [{"recommendation": "DR-1", "reason": "No identity."}]
decider = FakeDecider({"NFR-1#1": choice("K-1", 1)})
await CoverageReview(decider)(
K.TARGET_ARCHITECTURE,
{"target-architecture.json": json.dumps(graph)},
UPSTREAM,
)
assert set(decider.asked[0][1]) == {"NFR-1#1"}
# --- at submission ---------------------------------------------------------------------
def submit_with(review, working=None):
(submit,) = (
tool
for tool in specialist_tools(
WORKFLOW,
K.TARGET_ARCHITECTURE,
(),
allow_cloud_read=False,
upstream=UPSTREAM,
working={} if working is None else working,
review=review,
)
if tool.name == "submit_deliverable"
)
return submit.handler
async def test_what_the_review_raises_is_returned_once_and_then_accepted():
calls = []
async def review(attachments):
calls.append(attachments)
return ['NFR-1: no Check verifies "Two instances." (its Checks: K-1)']
working: dict = {}
submit = submit_with(review, working)
first = await submit(content=CONTENT, attachments=dict(GRAPH))
second = await submit(content=CONTENT)
assert first.outcome is None
assert first.message.startswith("Not submitted. A review found")
assert "NFR-1: no Check verifies" in first.message
assert "runs once" in first.message
# The graph it sent is kept, so it amends that and does not send it again.
assert isinstance(second.outcome, Submitted)
assert len(calls) == 1
async def test_a_structural_fault_is_reported_before_any_review():
async def review(attachments):
raise AssertionError("not reached")
result = await submit_with(review)(content=CONTENT, attachments={})
assert "attachment: target-architecture.json" in result.message
async def test_a_review_that_fails_or_raises_nothing_never_stops_a_submission():
async def broken(attachments):
raise RuntimeError("service down")
async def quiet(attachments):
return ()
for review in (broken, quiet):
result = await submit_with(review)(content=CONTENT, attachments=dict(GRAPH))
assert isinstance(result.outcome, Submitted)
# --- a description stands while the shape of the design does ---------------------------
ROLES = {
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
**{
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
for spec in DEPENDENCY_GRAPH.values()
},
}
def graph_with(**changes) -> Call:
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
for key, change in changes.items():
graph[key] = change(graph[key])
return Call(
"submit_deliverable",
{
"content": CONTENT,
"attachments": {"target-architecture.json": json.dumps(graph)},
},
)
def addition() -> Call:
"""A validation that passes every Check and proposes one more."""
call = sample_submission(K.VALIDATION_RESULTS)
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
verdicts["proposed_additions"] = [
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": "NFR-1"}
]
return Call(
call.tool,
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
)
async def run(tmp_path, second_graph: Call, **options):
turns = {
spec.specialist: [[sample_submission(kind)]] * 2
for kind, spec in DEPENDENCY_GRAPH.items()
}
turns["target-architect"] = [
[sample_submission(K.TARGET_ARCHITECTURE)],
[second_graph],
]
turns["validator"] = [[addition()], [sample_submission(K.VALIDATION_RESULTS)]]
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
**options,
)
result = await engine.start("Ingest webhooks", pre_approved=True)
assert result.status == "completed"
return result, [run.role for run in runtime.runs]
async def test_the_description_is_kept_when_a_revision_only_changes_checks(tmp_path):
reworded = graph_with(
checks=lambda checks: [
{**checks[0], "statement": "min_instance_count is at least 2"},
checks[1],
]
)
result, roles_run = await run(tmp_path, reworded)
assert roles_run.count("target-architect") == 2
assert roles_run.count("iac-author") == 2 # the code answers the Checks
assert roles_run.count("describer") == 1
# The Ledger still records a second submission: the Deliverable was not skipped.
submitted = [
event
for event in result.state["ledger"]["events"]
if event["type"] == "submitted"
and event["subject"] == "architecture_description"
]
assert len(submitted) == 2
async def test_the_description_is_written_again_when_the_shape_changes(tmp_path):
renamed = graph_with(
elements=lambda elements: [{**elements[0], "name": "Edge service"}, elements[1]]
)
_, roles_run = await run(tmp_path, renamed)
assert roles_run.count("describer") == 2
async def test_the_engine_reviews_a_design_before_it_is_submitted(tmp_path):
raised = []
async def review(kind, attachments, upstream, scope=""):
if kind == K.TARGET_ARCHITECTURE and not raised:
raised.append(scope)
return ['NFR-1 "Two instances." (its Checks: K-1)']
return ()
turns = {
spec.specialist: [[sample_submission(kind)]]
for kind, spec in DEPENDENCY_GRAPH.items()
}
turns["target-architect"] = [
[sample_submission(K.TARGET_ARCHITECTURE)] * 2 # refused once, then accepted
]
runtime = FakeAgentRuntime(turns)
engine = ExecutionEngine(
workflow=WORKFLOW,
runtime=runtime,
roles=ROLES,
recorder=InMemoryRecorder(),
checkpointer=InMemorySaver(),
output_root=tmp_path,
orchestrator=Router(NoDecider(), None, Thresholds()),
submission_review=review,
)
result = await engine.start("Ingest webhooks", pre_approved=True, execution_id="e1")
assert result.status == "completed"
assert raised == ["e1"]
architect = next(run for run in runtime.runs if run.role == "target-architect")
assert architect.results[0][1].startswith("Not submitted. A review found")
# What a review raises is kept as a lesson candidate, like any refusal.
assert any(
"A review found" in json.dumps(outcome.get("refusals"))
for outcome in result.state["outcomes"]
)
async def test_feedback_on_the_description_itself_has_it_written_again(tmp_path):
"""Kept only while nobody has asked for the description to change."""
from execution_engine.nodes.execution_nodes import STANDING, make_execution_nodes
nodes = make_execution_nodes(
workflow=WORKFLOW,
runtime=FakeAgentRuntime(
{"describer": [[sample_submission(K.ARCHITECTURE_DESCRIPTION, "new")]]}
),
roles=ROLES,
recorder=InMemoryRecorder(),
output_root=tmp_path,
)
upstream = {"target_architecture": {"attachments": dict(GRAPH)}}
from tests.reference_workflow.chain import graph_structure
standing = {
"basis": graph_structure(upstream),
"feedback": [],
"content": "old",
"attachments": {},
}
task = {
"execution_id": "e1",
"role": "describer",
"kind": "architecture_description",
"attempt": 2,
"allow_cloud_read": False,
"working": {STANDING: standing},
"context": {
"dependencies": upstream,
"feedback": ["Dependency x was revised: y"],
},
}
kept = await nodes.specialist_task(task)
asked = await nodes.specialist_task(
{**task, "context": {**task["context"], "feedback": ["Make the flow clearer."]}}
)
assert kept["outcomes"][0]["content"] == "old"
assert "new" in asked["outcomes"][0]["content"]
assert asked["working"]["architecture_description"][STANDING]["feedback"] == [
"Make the flow clearer."
]
# --- requirements name no products ------------------------------------------------------
async def test_a_requirement_that_names_a_product_is_raised():
from tests.reference_workflow.requirements_review import RequirementsReview
requirements = [
{"id": "FR-1", "kind": "functional", "text": "Run the API on Cloud Run."},
{"id": "FR-2", "kind": "functional", "text": "Accept signed webhooks."},
]
decider = FakeDecider({"FR-1": Answer(0.93, 0.86), "FR-2": Answer(0.05, 0.9)})
raised = await RequirementsReview(decider)(
K.REQUIREMENTS, {"requirements.json": json.dumps(requirements)}, None, "e1"
)
assert len(raised) == 1 and raised[0].startswith("FR-1 names a product")
_, questions, scope = decider.asked[0]
assert set(questions) == {"FR-1", "FR-2"} and scope == "e1"
assert "Accept signed webhooks." in questions["FR-2"].instructions
async def test_the_requirements_review_looks_only_at_requirements_and_never_blocks():
from tests.reference_workflow.requirements_review import RequirementsReview
decider = FakeDecider({})
review = RequirementsReview(decider)
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
assert await review(K.REQUIREMENTS, {"requirements.json": "not json"}, None) == ()
assert decider.asked == []
assert (
await RequirementsReview(NoDecider())(
K.REQUIREMENTS,
{"requirements.json": SAMPLE_DATA["requirements.json"]},
None,
)
== ()
)