447 lines
15 KiB
Python
447 lines
15 KiB
Python
"""Two savings on a Revision of the design.
|
|
|
|
A coverage review at submission finds what validation would otherwise find later,
|
|
at the cost of a Revision. And a Deliverable written from the shape of the design is
|
|
not written again when a Revision only changes its Checks.
|
|
"""
|
|
|
|
import json
|
|
|
|
from langgraph.checkpoint.memory import InMemorySaver
|
|
|
|
from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime, FakeDecider
|
|
from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec, Submitted
|
|
from execution_engine.agent import ExecutionEngine
|
|
from execution_engine.nodes.router import Router, Thresholds
|
|
from execution_engine.recorder import InMemoryRecorder
|
|
from execution_engine.tools.execution import specialist_tools
|
|
from tests.reference_workflow.coverage import CoverageReview
|
|
from tests.reference_workflow.workflow import (
|
|
DEPENDENCY_GRAPH,
|
|
SAMPLE_DATA,
|
|
WORKFLOW,
|
|
DeliverableKind,
|
|
sample_submission,
|
|
)
|
|
|
|
K = DeliverableKind
|
|
GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]}
|
|
UPSTREAM = {
|
|
"requirements": {
|
|
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
|
},
|
|
"technical_decomposition": {
|
|
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
|
|
},
|
|
"product_selection": {
|
|
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
|
|
},
|
|
"design_recommendations": {
|
|
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
|
|
},
|
|
}
|
|
CONTENT = "## Design rationale\nReasons.\n"
|
|
|
|
|
|
# --- the review ------------------------------------------------------------------------
|
|
|
|
|
|
def choice(option: str, probability: float) -> Answer:
|
|
return Answer(option, probability, {option: probability})
|
|
|
|
|
|
async def test_the_review_asks_which_check_verifies_each_condition():
|
|
decider = FakeDecider({"NFR-1#1": choice("K-1", 0.9), "DR-1#1": choice("K-2", 0.9)})
|
|
|
|
concerns = await CoverageReview(decider)(
|
|
K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1"
|
|
)
|
|
|
|
state, questions, scope = decider.asked[0]
|
|
assert concerns == ()
|
|
assert scope == "exec-1"
|
|
assert "E-1 Ingress service (Example Run, example_run_service)" in state
|
|
# Constraints only: a functional requirement is carried by its component.
|
|
assert set(questions) == {"NFR-1#1", "DR-1#1"}
|
|
question = questions["NFR-1#1"]
|
|
assert question.kind == "choice" and question.options == ("K-1", "none")
|
|
assert 'Condition: "The service stays available when one instance fails."' in (
|
|
question.instructions
|
|
)
|
|
assert "- K-1 on Ingress service: The service runs at least two instances." in (
|
|
question.instructions
|
|
)
|
|
|
|
|
|
async def test_a_condition_no_check_verifies_is_raised_with_the_checks_it_has():
|
|
decider = FakeDecider({"NFR-1#1": choice("none", 0.9), "DR-1#1": choice("K-2", 1)})
|
|
|
|
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM)
|
|
|
|
condition = "The service stays available when one instance fails."
|
|
assert concerns == (f'NFR-1: no Check verifies "{condition}" (its Checks: K-1)',)
|
|
|
|
|
|
async def test_an_uncertain_none_is_not_raised():
|
|
decider = FakeDecider({"NFR-1#1": choice("none", 0.45), "DR-1#1": choice("K-2", 1)})
|
|
|
|
review = CoverageReview(decider, none_at=0.6)
|
|
|
|
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
|
|
|
|
|
async def test_each_condition_of_a_requirement_is_asked_about_separately():
|
|
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
|
requirements[1]["conditions"] = [
|
|
"Survives one instance failing.",
|
|
"Survives a zone failing.",
|
|
]
|
|
upstream = {
|
|
**UPSTREAM,
|
|
"requirements": {
|
|
"attachments": {"requirements.json": json.dumps(requirements)}
|
|
},
|
|
}
|
|
decider = FakeDecider(
|
|
{
|
|
"NFR-1#1": choice("K-1", 0.9),
|
|
"NFR-1#2": choice("none", 0.8),
|
|
"DR-1#1": choice("K-2", 1),
|
|
}
|
|
)
|
|
|
|
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
|
|
|
|
assert len(concerns) == 1 and "Survives a zone failing." in concerns[0]
|
|
|
|
|
|
async def test_an_item_without_conditions_is_read_as_one_condition_its_text():
|
|
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
|
del requirements[1]["conditions"]
|
|
upstream = {
|
|
**UPSTREAM,
|
|
"requirements": {
|
|
"attachments": {"requirements.json": json.dumps(requirements)}
|
|
},
|
|
}
|
|
decider = FakeDecider({"NFR-1#1": choice("K-1", 1), "DR-1#1": choice("K-2", 1)})
|
|
|
|
await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
|
|
|
|
assert 'Condition: "99.9% availability."' in (
|
|
decider.asked[0][1]["NFR-1#1"].instructions
|
|
)
|
|
|
|
|
|
async def test_no_decision_and_other_deliverables_raise_nothing():
|
|
review = CoverageReview(NoDecider())
|
|
decider = FakeDecider({})
|
|
|
|
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
|
assert await CoverageReview(decider)(K.REQUIREMENTS, {}, UPSTREAM) == ()
|
|
assert await CoverageReview(decider)(K.TARGET_ARCHITECTURE, {}, UPSTREAM) == ()
|
|
assert decider.asked == []
|
|
|
|
|
|
async def test_a_recommendation_marked_not_applicable_is_not_reviewed():
|
|
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
|
graph["checks"][1]["source"] = "NFR-1"
|
|
graph["not_applicable"] = [{"recommendation": "DR-1", "reason": "No identity."}]
|
|
decider = FakeDecider({"NFR-1#1": choice("K-1", 1)})
|
|
|
|
await CoverageReview(decider)(
|
|
K.TARGET_ARCHITECTURE,
|
|
{"target-architecture.json": json.dumps(graph)},
|
|
UPSTREAM,
|
|
)
|
|
|
|
assert set(decider.asked[0][1]) == {"NFR-1#1"}
|
|
|
|
|
|
# --- at submission ---------------------------------------------------------------------
|
|
|
|
|
|
def submit_with(review, working=None):
|
|
(submit,) = (
|
|
tool
|
|
for tool in specialist_tools(
|
|
WORKFLOW,
|
|
K.TARGET_ARCHITECTURE,
|
|
(),
|
|
allow_cloud_read=False,
|
|
upstream=UPSTREAM,
|
|
working={} if working is None else working,
|
|
review=review,
|
|
)
|
|
if tool.name == "submit_deliverable"
|
|
)
|
|
return submit.handler
|
|
|
|
|
|
async def test_what_the_review_raises_is_returned_once_and_then_accepted():
|
|
calls = []
|
|
|
|
async def review(attachments):
|
|
calls.append(attachments)
|
|
return ['NFR-1: no Check verifies "Two instances." (its Checks: K-1)']
|
|
|
|
working: dict = {}
|
|
submit = submit_with(review, working)
|
|
|
|
first = await submit(content=CONTENT, attachments=dict(GRAPH))
|
|
second = await submit(content=CONTENT)
|
|
|
|
assert first.outcome is None
|
|
assert first.message.startswith("Not submitted. A review found")
|
|
assert "NFR-1: no Check verifies" in first.message
|
|
assert "runs once" in first.message
|
|
# The graph it sent is kept, so it amends that and does not send it again.
|
|
assert isinstance(second.outcome, Submitted)
|
|
assert len(calls) == 1
|
|
|
|
|
|
async def test_a_structural_fault_is_reported_before_any_review():
|
|
async def review(attachments):
|
|
raise AssertionError("not reached")
|
|
|
|
result = await submit_with(review)(content=CONTENT, attachments={})
|
|
|
|
assert "attachment: target-architecture.json" in result.message
|
|
|
|
|
|
async def test_a_review_that_fails_or_raises_nothing_never_stops_a_submission():
|
|
async def broken(attachments):
|
|
raise RuntimeError("service down")
|
|
|
|
async def quiet(attachments):
|
|
return ()
|
|
|
|
for review in (broken, quiet):
|
|
result = await submit_with(review)(content=CONTENT, attachments=dict(GRAPH))
|
|
assert isinstance(result.outcome, Submitted)
|
|
|
|
|
|
# --- a description stands while the shape of the design does ---------------------------
|
|
|
|
ROLES = {
|
|
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
|
**{
|
|
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
|
for spec in DEPENDENCY_GRAPH.values()
|
|
},
|
|
}
|
|
|
|
|
|
def graph_with(**changes) -> Call:
|
|
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
|
for key, change in changes.items():
|
|
graph[key] = change(graph[key])
|
|
return Call(
|
|
"submit_deliverable",
|
|
{
|
|
"content": CONTENT,
|
|
"attachments": {"target-architecture.json": json.dumps(graph)},
|
|
},
|
|
)
|
|
|
|
|
|
def addition() -> Call:
|
|
"""A validation that passes every Check and proposes one more."""
|
|
call = sample_submission(K.VALIDATION_RESULTS)
|
|
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
|
verdicts["proposed_additions"] = [
|
|
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": "NFR-1"}
|
|
]
|
|
return Call(
|
|
call.tool,
|
|
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
|
)
|
|
|
|
|
|
async def run(tmp_path, second_graph: Call, **options):
|
|
turns = {
|
|
spec.specialist: [[sample_submission(kind)]] * 2
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
turns["target-architect"] = [
|
|
[sample_submission(K.TARGET_ARCHITECTURE)],
|
|
[second_graph],
|
|
]
|
|
turns["validator"] = [[addition()], [sample_submission(K.VALIDATION_RESULTS)]]
|
|
runtime = FakeAgentRuntime(turns)
|
|
engine = ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
**options,
|
|
)
|
|
result = await engine.start("Ingest webhooks", pre_approved=True)
|
|
assert result.status == "completed"
|
|
return result, [run.role for run in runtime.runs]
|
|
|
|
|
|
async def test_the_description_is_kept_when_a_revision_only_changes_checks(tmp_path):
|
|
reworded = graph_with(
|
|
checks=lambda checks: [
|
|
{**checks[0], "statement": "min_instance_count is at least 2"},
|
|
checks[1],
|
|
]
|
|
)
|
|
|
|
result, roles_run = await run(tmp_path, reworded)
|
|
|
|
assert roles_run.count("target-architect") == 2
|
|
assert roles_run.count("iac-author") == 2 # the code answers the Checks
|
|
assert roles_run.count("describer") == 1
|
|
# The Ledger still records a second submission: the Deliverable was not skipped.
|
|
submitted = [
|
|
event
|
|
for event in result.state["ledger"]["events"]
|
|
if event["type"] == "submitted"
|
|
and event["subject"] == "architecture_description"
|
|
]
|
|
assert len(submitted) == 2
|
|
|
|
|
|
async def test_the_description_is_written_again_when_the_shape_changes(tmp_path):
|
|
renamed = graph_with(
|
|
elements=lambda elements: [{**elements[0], "name": "Edge service"}, elements[1]]
|
|
)
|
|
|
|
_, roles_run = await run(tmp_path, renamed)
|
|
|
|
assert roles_run.count("describer") == 2
|
|
|
|
|
|
async def test_the_engine_reviews_a_design_before_it_is_submitted(tmp_path):
|
|
raised = []
|
|
|
|
async def review(kind, attachments, upstream, scope=""):
|
|
if kind == K.TARGET_ARCHITECTURE and not raised:
|
|
raised.append(scope)
|
|
return ['NFR-1 "Two instances." (its Checks: K-1)']
|
|
return ()
|
|
|
|
turns = {
|
|
spec.specialist: [[sample_submission(kind)]]
|
|
for kind, spec in DEPENDENCY_GRAPH.items()
|
|
}
|
|
turns["target-architect"] = [
|
|
[sample_submission(K.TARGET_ARCHITECTURE)] * 2 # refused once, then accepted
|
|
]
|
|
runtime = FakeAgentRuntime(turns)
|
|
engine = ExecutionEngine(
|
|
workflow=WORKFLOW,
|
|
runtime=runtime,
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
checkpointer=InMemorySaver(),
|
|
output_root=tmp_path,
|
|
orchestrator=Router(NoDecider(), None, Thresholds()),
|
|
submission_review=review,
|
|
)
|
|
|
|
result = await engine.start("Ingest webhooks", pre_approved=True, execution_id="e1")
|
|
|
|
assert result.status == "completed"
|
|
assert raised == ["e1"]
|
|
architect = next(run for run in runtime.runs if run.role == "target-architect")
|
|
assert architect.results[0][1].startswith("Not submitted. A review found")
|
|
# What a review raises is kept as a lesson candidate, like any refusal.
|
|
assert any(
|
|
"A review found" in json.dumps(outcome.get("refusals"))
|
|
for outcome in result.state["outcomes"]
|
|
)
|
|
|
|
|
|
async def test_feedback_on_the_description_itself_has_it_written_again(tmp_path):
|
|
"""Kept only while nobody has asked for the description to change."""
|
|
from execution_engine.nodes.execution_nodes import STANDING, make_execution_nodes
|
|
|
|
nodes = make_execution_nodes(
|
|
workflow=WORKFLOW,
|
|
runtime=FakeAgentRuntime(
|
|
{"describer": [[sample_submission(K.ARCHITECTURE_DESCRIPTION, "new")]]}
|
|
),
|
|
roles=ROLES,
|
|
recorder=InMemoryRecorder(),
|
|
output_root=tmp_path,
|
|
)
|
|
upstream = {"target_architecture": {"attachments": dict(GRAPH)}}
|
|
from tests.reference_workflow.chain import graph_structure
|
|
|
|
standing = {
|
|
"basis": graph_structure(upstream),
|
|
"feedback": [],
|
|
"content": "old",
|
|
"attachments": {},
|
|
}
|
|
task = {
|
|
"execution_id": "e1",
|
|
"role": "describer",
|
|
"kind": "architecture_description",
|
|
"attempt": 2,
|
|
"allow_cloud_read": False,
|
|
"working": {STANDING: standing},
|
|
"context": {
|
|
"dependencies": upstream,
|
|
"feedback": ["Dependency x was revised: y"],
|
|
},
|
|
}
|
|
|
|
kept = await nodes.specialist_task(task)
|
|
asked = await nodes.specialist_task(
|
|
{**task, "context": {**task["context"], "feedback": ["Make the flow clearer."]}}
|
|
)
|
|
|
|
assert kept["outcomes"][0]["content"] == "old"
|
|
assert "new" in asked["outcomes"][0]["content"]
|
|
assert asked["working"]["architecture_description"][STANDING]["feedback"] == [
|
|
"Make the flow clearer."
|
|
]
|
|
|
|
|
|
# --- requirements name no products ------------------------------------------------------
|
|
|
|
|
|
async def test_a_requirement_that_names_a_product_is_raised():
|
|
from tests.reference_workflow.requirements_review import RequirementsReview
|
|
|
|
requirements = [
|
|
{"id": "FR-1", "kind": "functional", "text": "Run the API on Cloud Run."},
|
|
{"id": "FR-2", "kind": "functional", "text": "Accept signed webhooks."},
|
|
]
|
|
decider = FakeDecider({"FR-1": Answer(0.93, 0.86), "FR-2": Answer(0.05, 0.9)})
|
|
|
|
raised = await RequirementsReview(decider)(
|
|
K.REQUIREMENTS, {"requirements.json": json.dumps(requirements)}, None, "e1"
|
|
)
|
|
|
|
assert len(raised) == 1 and raised[0].startswith("FR-1 names a product")
|
|
_, questions, scope = decider.asked[0]
|
|
assert set(questions) == {"FR-1", "FR-2"} and scope == "e1"
|
|
assert "Accept signed webhooks." in questions["FR-2"].instructions
|
|
|
|
|
|
async def test_the_requirements_review_looks_only_at_requirements_and_never_blocks():
|
|
from tests.reference_workflow.requirements_review import RequirementsReview
|
|
|
|
decider = FakeDecider({})
|
|
review = RequirementsReview(decider)
|
|
|
|
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
|
assert await review(K.REQUIREMENTS, {"requirements.json": "not json"}, None) == ()
|
|
assert decider.asked == []
|
|
assert (
|
|
await RequirementsReview(NoDecider())(
|
|
K.REQUIREMENTS,
|
|
{"requirements.json": SAMPLE_DATA["requirements.json"]},
|
|
None,
|
|
)
|
|
== ()
|
|
)
|