"""Two savings on a Revision of the design. A coverage review at submission finds what validation would otherwise find later, at the cost of a Revision. And a Deliverable written from the shape of the design is not written again when a Revision only changes its Checks. """ import json from langgraph.checkpoint.memory import InMemorySaver from execution_engine.adk.fake_runtime import Call, FakeAgentRuntime, FakeDecider from execution_engine.adk.runtime import Answer, NoDecider, RoleSpec, Submitted from execution_engine.agent import ExecutionEngine from execution_engine.nodes.router import Router, Thresholds from execution_engine.recorder import InMemoryRecorder from execution_engine.tools.execution import specialist_tools from tests.reference_workflow.coverage import CoverageReview from tests.reference_workflow.workflow import ( DEPENDENCY_GRAPH, SAMPLE_DATA, WORKFLOW, DeliverableKind, sample_submission, ) K = DeliverableKind GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]} UPSTREAM = { "requirements": { "attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]} }, "technical_decomposition": { "attachments": {"components.json": SAMPLE_DATA["components.json"]} }, "product_selection": { "attachments": {"products.json": SAMPLE_DATA["products.json"]} }, "design_recommendations": { "attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]} }, } CONTENT = "## Design rationale\nReasons.\n" # --- the review ------------------------------------------------------------------------ def choice(option: str, probability: float) -> Answer: return Answer(option, probability, {option: probability}) async def test_the_review_asks_which_check_verifies_each_condition(): decider = FakeDecider({"NFR-1#1": choice("K-1", 0.9), "DR-1#1": choice("K-2", 0.9)}) concerns = await CoverageReview(decider)( K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1" ) state, questions, scope = decider.asked[0] assert concerns == () assert scope == "exec-1" assert "E-1 Ingress service (Example Run, example_run_service)" in state # Constraints only: a functional requirement is carried by its component. assert set(questions) == {"NFR-1#1", "DR-1#1"} question = questions["NFR-1#1"] assert question.kind == "choice" and question.options == ("K-1", "none") assert 'Condition: "The service stays available when one instance fails."' in ( question.instructions ) assert "- K-1 on Ingress service: The service runs at least two instances." in ( question.instructions ) async def test_a_condition_no_check_verifies_is_raised_with_the_checks_it_has(): decider = FakeDecider({"NFR-1#1": choice("none", 0.9), "DR-1#1": choice("K-2", 1)}) concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) condition = "The service stays available when one instance fails." assert concerns == (f'NFR-1: no Check verifies "{condition}" (its Checks: K-1)',) async def test_an_uncertain_none_is_not_raised(): decider = FakeDecider({"NFR-1#1": choice("none", 0.45), "DR-1#1": choice("K-2", 1)}) review = CoverageReview(decider, none_at=0.6) assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == () async def test_each_condition_of_a_requirement_is_asked_about_separately(): requirements = json.loads(SAMPLE_DATA["requirements.json"]) requirements[1]["conditions"] = [ "Survives one instance failing.", "Survives a zone failing.", ] upstream = { **UPSTREAM, "requirements": { "attachments": {"requirements.json": json.dumps(requirements)} }, } decider = FakeDecider( { "NFR-1#1": choice("K-1", 0.9), "NFR-1#2": choice("none", 0.8), "DR-1#1": choice("K-2", 1), } ) concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream) assert len(concerns) == 1 and "Survives a zone failing." in concerns[0] async def test_an_item_without_conditions_is_read_as_one_condition_its_text(): requirements = json.loads(SAMPLE_DATA["requirements.json"]) del requirements[1]["conditions"] upstream = { **UPSTREAM, "requirements": { "attachments": {"requirements.json": json.dumps(requirements)} }, } decider = FakeDecider({"NFR-1#1": choice("K-1", 1), "DR-1#1": choice("K-2", 1)}) await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream) assert 'Condition: "99.9% availability."' in ( decider.asked[0][1]["NFR-1#1"].instructions ) async def test_no_decision_and_other_deliverables_raise_nothing(): review = CoverageReview(NoDecider()) decider = FakeDecider({}) assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == () assert await CoverageReview(decider)(K.REQUIREMENTS, {}, UPSTREAM) == () assert await CoverageReview(decider)(K.TARGET_ARCHITECTURE, {}, UPSTREAM) == () assert decider.asked == [] async def test_a_recommendation_marked_not_applicable_is_not_reviewed(): graph = json.loads(SAMPLE_DATA["target-architecture.json"]) graph["checks"][1]["source"] = "NFR-1" graph["not_applicable"] = [{"recommendation": "DR-1", "reason": "No identity."}] decider = FakeDecider({"NFR-1#1": choice("K-1", 1)}) await CoverageReview(decider)( K.TARGET_ARCHITECTURE, {"target-architecture.json": json.dumps(graph)}, UPSTREAM, ) assert set(decider.asked[0][1]) == {"NFR-1#1"} # --- at submission --------------------------------------------------------------------- def submit_with(review, working=None): (submit,) = ( tool for tool in specialist_tools( WORKFLOW, K.TARGET_ARCHITECTURE, (), allow_cloud_read=False, upstream=UPSTREAM, working={} if working is None else working, review=review, ) if tool.name == "submit_deliverable" ) return submit.handler async def test_what_the_review_raises_is_returned_once_and_then_accepted(): calls = [] async def review(attachments): calls.append(attachments) return ['NFR-1: no Check verifies "Two instances." (its Checks: K-1)'] working: dict = {} submit = submit_with(review, working) first = await submit(content=CONTENT, attachments=dict(GRAPH)) second = await submit(content=CONTENT) assert first.outcome is None assert first.message.startswith("Not submitted. A review found") assert "NFR-1: no Check verifies" in first.message assert "runs once" in first.message # The graph it sent is kept, so it amends that and does not send it again. assert isinstance(second.outcome, Submitted) assert len(calls) == 1 async def test_a_structural_fault_is_reported_before_any_review(): async def review(attachments): raise AssertionError("not reached") result = await submit_with(review)(content=CONTENT, attachments={}) assert "attachment: target-architecture.json" in result.message async def test_a_review_that_fails_or_raises_nothing_never_stops_a_submission(): async def broken(attachments): raise RuntimeError("service down") async def quiet(attachments): return () for review in (broken, quiet): result = await submit_with(review)(content=CONTENT, attachments=dict(GRAPH)) assert isinstance(result.outcome, Submitted) # --- a description stands while the shape of the design does --------------------------- ROLES = { "orchestrator": RoleSpec("orchestrator", "Unused.", budget=40), **{ spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15) for spec in DEPENDENCY_GRAPH.values() }, } def graph_with(**changes) -> Call: graph = json.loads(SAMPLE_DATA["target-architecture.json"]) for key, change in changes.items(): graph[key] = change(graph[key]) return Call( "submit_deliverable", { "content": CONTENT, "attachments": {"target-architecture.json": json.dumps(graph)}, }, ) def addition() -> Call: """A validation that passes every Check and proposes one more.""" call = sample_submission(K.VALIDATION_RESULTS) verdicts = json.loads(SAMPLE_DATA["verdicts.json"]) verdicts["proposed_additions"] = [ {"statement": "A timeout is set.", "applies_to": "E-1", "reason": "NFR-1"} ] return Call( call.tool, {**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}}, ) async def run(tmp_path, second_graph: Call, **options): turns = { spec.specialist: [[sample_submission(kind)]] * 2 for kind, spec in DEPENDENCY_GRAPH.items() } turns["target-architect"] = [ [sample_submission(K.TARGET_ARCHITECTURE)], [second_graph], ] turns["validator"] = [[addition()], [sample_submission(K.VALIDATION_RESULTS)]] runtime = FakeAgentRuntime(turns) engine = ExecutionEngine( workflow=WORKFLOW, runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), **options, ) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" return result, [run.role for run in runtime.runs] async def test_the_description_is_kept_when_a_revision_only_changes_checks(tmp_path): reworded = graph_with( checks=lambda checks: [ {**checks[0], "statement": "min_instance_count is at least 2"}, checks[1], ] ) result, roles_run = await run(tmp_path, reworded) assert roles_run.count("target-architect") == 2 assert roles_run.count("iac-author") == 2 # the code answers the Checks assert roles_run.count("describer") == 1 # The Ledger still records a second submission: the Deliverable was not skipped. submitted = [ event for event in result.state["ledger"]["events"] if event["type"] == "submitted" and event["subject"] == "architecture_description" ] assert len(submitted) == 2 async def test_the_description_is_written_again_when_the_shape_changes(tmp_path): renamed = graph_with( elements=lambda elements: [{**elements[0], "name": "Edge service"}, elements[1]] ) _, roles_run = await run(tmp_path, renamed) assert roles_run.count("describer") == 2 async def test_the_engine_reviews_a_design_before_it_is_submitted(tmp_path): raised = [] async def review(kind, attachments, upstream, scope=""): if kind == K.TARGET_ARCHITECTURE and not raised: raised.append(scope) return ['NFR-1 "Two instances." (its Checks: K-1)'] return () turns = { spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() } turns["target-architect"] = [ [sample_submission(K.TARGET_ARCHITECTURE)] * 2 # refused once, then accepted ] runtime = FakeAgentRuntime(turns) engine = ExecutionEngine( workflow=WORKFLOW, runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), submission_review=review, ) result = await engine.start("Ingest webhooks", pre_approved=True, execution_id="e1") assert result.status == "completed" assert raised == ["e1"] architect = next(run for run in runtime.runs if run.role == "target-architect") assert architect.results[0][1].startswith("Not submitted. A review found") # What a review raises is kept as a lesson candidate, like any refusal. assert any( "A review found" in json.dumps(outcome.get("refusals")) for outcome in result.state["outcomes"] ) async def test_feedback_on_the_description_itself_has_it_written_again(tmp_path): """Kept only while nobody has asked for the description to change.""" from execution_engine.nodes.execution_nodes import STANDING, make_execution_nodes nodes = make_execution_nodes( workflow=WORKFLOW, runtime=FakeAgentRuntime( {"describer": [[sample_submission(K.ARCHITECTURE_DESCRIPTION, "new")]]} ), roles=ROLES, recorder=InMemoryRecorder(), output_root=tmp_path, ) upstream = {"target_architecture": {"attachments": dict(GRAPH)}} from tests.reference_workflow.chain import graph_structure standing = { "basis": graph_structure(upstream), "feedback": [], "content": "old", "attachments": {}, } task = { "execution_id": "e1", "role": "describer", "kind": "architecture_description", "attempt": 2, "allow_cloud_read": False, "working": {STANDING: standing}, "context": { "dependencies": upstream, "feedback": ["Dependency x was revised: y"], }, } kept = await nodes.specialist_task(task) asked = await nodes.specialist_task( {**task, "context": {**task["context"], "feedback": ["Make the flow clearer."]}} ) assert kept["outcomes"][0]["content"] == "old" assert "new" in asked["outcomes"][0]["content"] assert asked["working"]["architecture_description"][STANDING]["feedback"] == [ "Make the flow clearer." ] # --- requirements name no products ------------------------------------------------------ async def test_a_requirement_that_names_a_product_is_raised(): from tests.reference_workflow.requirements_review import RequirementsReview requirements = [ {"id": "FR-1", "kind": "functional", "text": "Run the API on Cloud Run."}, {"id": "FR-2", "kind": "functional", "text": "Accept signed webhooks."}, ] decider = FakeDecider({"FR-1": Answer(0.93, 0.86), "FR-2": Answer(0.05, 0.9)}) raised = await RequirementsReview(decider)( K.REQUIREMENTS, {"requirements.json": json.dumps(requirements)}, None, "e1" ) assert len(raised) == 1 and raised[0].startswith("FR-1 names a product") _, questions, scope = decider.asked[0] assert set(questions) == {"FR-1", "FR-2"} and scope == "e1" assert "Accept signed webhooks." in questions["FR-2"].instructions async def test_the_requirements_review_looks_only_at_requirements_and_never_blocks(): from tests.reference_workflow.requirements_review import RequirementsReview decider = FakeDecider({}) review = RequirementsReview(decider) assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == () assert await review(K.REQUIREMENTS, {"requirements.json": "not json"}, None) == () assert decider.asked == [] assert ( await RequirementsReview(NoDecider())( K.REQUIREMENTS, {"requirements.json": SAMPLE_DATA["requirements.json"]}, None, ) == () )