"""A model reviews a design's coverage before it is approved (iteration 6 step 13).""" from langgraph.checkpoint.memory import InMemorySaver from app.adk.fake_runtime import SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission from app.adk.roles import COVERAGE_REVIEWER, load_roles from app.adk.runtime import NoDecider, RoleSpec from app.agent import AGENTS_DIR, ExecutionEngine from app.nodes.router import Router, Thresholds from app.providers import load_provider from app.recorder import InMemoryRecorder from app.tools.design_review import DesignCoverageReview from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind K = DeliverableKind GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]} UPSTREAM = { "requirements": { "attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]} }, "design_recommendations": { "attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]} }, } REVIEWER = RoleSpec(COVERAGE_REVIEWER, "Review.", budget=6) GAP = { "source": "NFR-1", "applies_to": "E-1", "statement": "min_instance_count is at least 2", } def reviewer(*gaps: dict, seen: dict | None = None) -> FakeAgentRuntime: def turn(task): if seen is not None: seen.update(task.context) return [Call("report_gaps", {"gaps": list(gaps)})] return FakeAgentRuntime({COVERAGE_REVIEWER: [turn, turn]}) async def test_the_reviewer_is_given_the_constraints_and_the_design(): seen: dict = {} review = DesignCoverageReview(reviewer(GAP, seen=seen), REVIEWER) concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") assert concerns == ( "NFR-1: no Check verifies this on E-1: min_instance_count is at least 2", ) # Non-functional requirements and recommendations; a functional requirement # is reached through its components, not through a Check of its own. assert [item["id"] for item in seen["constraints"]] == ["NFR-1", "DR-1"] assert {e["id"] for e in seen["target_architecture"]["elements"]} >= {"E-1"} async def test_a_gap_about_something_that_does_not_exist_is_dropped(): invented = [ {**GAP, "source": "NFR-99"}, {**GAP, "applies_to": "E-99"}, {**GAP, "statement": " "}, "not a gap", ] review = DesignCoverageReview(reviewer(*invented, GAP), REVIEWER) concerns = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") assert len(concerns) == 1 and concerns[0].startswith("NFR-1:") async def test_a_design_is_reviewed_once_in_an_execution(): runtime = reviewer(GAP) review = DesignCoverageReview(runtime, REVIEWER) first = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") again = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") other = await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-2") assert first and other and again == () assert len(runtime.runs) == 2 async def test_nothing_is_reviewed_but_a_readable_target_architecture(): runtime = reviewer(GAP) review = DesignCoverageReview(runtime, REVIEWER) assert await review(K.DEPLOYMENT_GUIDANCE, GRAPH, UPSTREAM, "a") == () assert await review(K.TARGET_ARCHITECTURE, {}, UPSTREAM, "b") == () assert await review(K.TARGET_ARCHITECTURE, GRAPH, None, "c") == () assert runtime.runs == [] async def test_a_complete_design_raises_nothing(): review = DesignCoverageReview(reviewer(), REVIEWER) assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1") == () async def test_the_gaps_go_back_to_the_target_architect_before_the_design_stands( tmp_path, ): roles = { "orchestrator": RoleSpec("orchestrator", "Unused.", budget=40), **{ spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15) for spec in DEPENDENCY_GRAPH.values() }, } architect = DEPENDENCY_GRAPH[K.TARGET_ARCHITECTURE].specialist design = sample_submission(K.TARGET_ARCHITECTURE) runtime = FakeAgentRuntime( { **{ spec.specialist: [[sample_submission(kind)]] for kind, spec in DEPENDENCY_GRAPH.items() }, # Submitted twice: once into the review, once after reading it. architect: [[design, design]], COVERAGE_REVIEWER: [[Call("report_gaps", {"gaps": [GAP]})]], } ) engine = ExecutionEngine( runtime=runtime, roles=roles, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, orchestrator=Router(NoDecider(), None, Thresholds()), submission_review=DesignCoverageReview(runtime, REVIEWER), ) result = await engine.start("Ingest webhooks", pre_approved=True) assert result.status == "completed" run = next(run for run in runtime.runs if run.role == architect) first, second = (reply for _, reply in run.results) assert first.startswith("Not submitted. A review found: NFR-1: no Check verifies") assert not second.startswith("Not submitted") # No Revision was needed: the gap was dealt with before anything was approved. assert not [d for d in result.state["decisions"] if "revise" in d["action"]] def test_the_reviewer_role_loads_with_the_bundles_model(): bundle = load_provider("gcp") roles = load_roles(AGENTS_DIR, bundle) role = roles[COVERAGE_REVIEWER] assert role.model == bundle.models.for_role(COVERAGE_REVIEWER) assert role.model == bundle.models.for_role("validator") assert role.tools == () and role.model_timeout_seconds == 300