"""The Requirements chain: the Target architecture graph and the data files around it. ADR-0005: the outcome is understood before code is written, and validation answers the graph's Checks and nothing else. """ import json import pytest from langgraph.checkpoint.memory import InMemorySaver from app.adk.fake_runtime import SAMPLE_DATA, Call, FakeAgentRuntime, sample_submission from app.adk.runtime import RoleSpec from app.agent import ExecutionEngine from app.recorder import InMemoryRecorder from app.tools.execution import specialist_tools from app.tools.local import mermaid_check from app.workflows.chain import unimplementable_checks, validation_outcome from app.workflows.deliverables import ( DEPENDENCY_GRAPH, DeliverableKind, PhaseId, check_submission, ) from app.workflows.target_architecture import ( GraphError, chain_gaps, parse_graph, render_mermaid, ) K = DeliverableKind P = PhaseId REQUIREMENTS = json.loads(SAMPLE_DATA["requirements.json"]) COMPONENTS = json.loads(SAMPLE_DATA["components.json"]) PRODUCTS = json.loads(SAMPLE_DATA["products.json"]) RECOMMENDATIONS = json.loads(SAMPLE_DATA["recommendations.json"]) def graph_data() -> dict: return json.loads(SAMPLE_DATA["target-architecture.json"]) def problems(data: dict) -> tuple[str, ...]: with pytest.raises(GraphError) as raised: parse_graph(json.dumps(data)) return raised.value.problems def gaps(data: dict, **overrides) -> tuple[str, ...]: links = { "requirements": REQUIREMENTS, "components": COMPONENTS, "products": PRODUCTS, "recommendations": RECOMMENDATIONS, **overrides, } return chain_gaps(parse_graph(json.dumps(data)), **links) def upstream(*kinds: DeliverableKind, **replaced: str) -> dict: """The Deliverables a submission builds on, from the coherent sample.""" built = {} for kind in kinds: args = sample_submission(kind).args attachments = { name: replaced.get(name, text) for name, text in args["attachments"].items() } built[kind.value] = {"content": args["content"], "attachments": attachments} return built DESIGN = ( K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION, K.PRODUCT_SELECTION, K.DESIGN_RECOMMENDATIONS, ) # --- the graph is well-formed ---------------------------------------------------------- def test_the_sample_graph_parses_into_elements_relationships_and_checks(): graph = parse_graph(SAMPLE_DATA["target-architecture.json"]) assert [element.id for element in graph.elements] == ["E-1", "E-2"] assert graph.elements[1].component is None # a supporting Element assert [(r.source, r.target) for r in graph.relationships] == [("E-1", "E-2")] assert graph.check_ids == ("K-1", "K-2") assert graph.summary() == { "elements": 2, "relationships": 1, "checks": 2, "not_applicable": [], } def test_a_relationship_must_join_two_elements(): data = graph_data() data["relationships"][0]["to"] = "E-9" assert problems(data) == ( "target-architecture.json: Relationship R-1 to E-9, which is not an Element", ) def test_a_check_must_apply_to_something_in_the_graph(): data = graph_data() data["checks"][0]["applies_to"] = "E-9" assert problems(data) == ( ( "target-architecture.json: Check K-1 applies to E-9, " "which is not an Element or Relationship" ), "target-architecture.json: Element E-1 has no Check", ) def test_every_element_needs_a_check_and_every_id_is_used_once(): data = graph_data() data["checks"] = [data["checks"][0]] data["relationships"][0]["id"] = "E-1" assert problems(data) == ( "target-architecture.json: id E-1 is used more than once", "target-architecture.json: Element E-2 has no Check", ) def test_a_check_without_a_statement_is_named(): data = graph_data() data["checks"][1]["statement"] = " " assert ( "target-architecture.json.checks[1]: needs a non-empty statement" in problems(data) ) @pytest.mark.parametrize("text", ["not json", "[]", '{"elements": "many"}']) def test_a_graph_that_is_not_the_right_shape_is_refused(text): with pytest.raises(GraphError): parse_graph(text) # --- the Requirements chain has no gap ------------------------------------------------- def test_the_sample_chain_is_complete(): assert gaps(graph_data()) == () def test_a_non_functional_requirement_with_no_check_is_a_gap(): retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."} assert gaps(graph_data(), requirements=[*REQUIREMENTS, retention]) == ( "Requirement NFR-2 has no Check", ) def test_a_functional_requirement_is_carried_by_its_components_element(): # FR-1 is the source of no Check, but C-1 serves it and E-1 realises C-1. assert not [gap for gap in gaps(graph_data()) if "FR-1" in gap] unserved = [{**COMPONENTS[0], "requirements": ["NFR-1"]}] assert gaps(graph_data(), components=unserved) == ("Requirement FR-1 has no Check",) def test_a_component_with_no_element_is_a_gap(): replay = {"id": "C-2", "name": "Replay service", "requirements": []} assert gaps(graph_data(), components=[*COMPONENTS, replay]) == ( "Component C-2 has no Element", ) def test_a_selected_product_that_no_element_uses_is_a_gap(): armour = [{"component": "C-1", "product": "Example Armor"}] assert gaps(graph_data(), products=armour) == ( "Product Example Armor selected for C-1 appears on no Element", ) def test_a_recommendation_needs_a_check_or_a_reason_it_does_not_apply(): cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []} recommendations = [*RECOMMENDATIONS, cmek] assert gaps(graph_data(), recommendations=recommendations) == ( "Recommendation DR-2 has no Check", ) excused = graph_data() excused["not_applicable"] = [ {"recommendation": "DR-2", "reason": "No regulated data."} ] assert gaps(excused, recommendations=recommendations) == () assert parse_graph(json.dumps(excused)).summary()["not_applicable"] == [ {"recommendation": "DR-2", "reason": "No regulated data."} ] def test_a_check_must_come_from_a_requirement_or_recommendation(): data = graph_data() data["checks"][0]["source"] = "my own idea" assert gaps(data) == ( ( "Check K-1 has source my own idea, which is not a requirement or design " "recommendation" ), "Requirement NFR-1 has no Check", ) def test_a_target_architecture_with_a_gap_cannot_be_submitted(): retention = {"id": "NFR-2", "kind": "non_functional", "text": "Retain 30 days."} built_on = upstream( *DESIGN, **{"requirements.json": json.dumps([*REQUIREMENTS, retention])} ) args = sample_submission(K.TARGET_ARCHITECTURE).args assert check_submission( K.TARGET_ARCHITECTURE, args["content"], args["attachments"], built_on ) == ("Requirement NFR-2 has no Check",) assert ( check_submission( K.TARGET_ARCHITECTURE, args["content"], args["attachments"], upstream(*DESIGN), ) == () ) # --- the links before the graph -------------------------------------------------------- def test_requirements_need_ids_and_a_kind(): def missing(items) -> tuple[str, ...]: args = sample_submission(K.REQUIREMENTS).args return check_submission( K.REQUIREMENTS, args["content"], {"requirements.json": json.dumps(items)} ) assert missing(REQUIREMENTS) == () assert missing([]) == ("requirements.json: lists no requirements",) assert missing([REQUIREMENTS[0], {**REQUIREMENTS[0], "kind": "must"}]) == ( "requirements.json: id FR-1 is used more than once", "requirements.json: FR-1 kind must be one of functional, non_functional", ) assert missing([{"id": "FR-1", "kind": "functional"}]) == ( "requirements.json[0]: needs text", ) def test_every_functional_requirement_is_served_by_a_component(): args = sample_submission(K.TECHNICAL_DECOMPOSITION).args def missing(components) -> tuple[str, ...]: return check_submission( K.TECHNICAL_DECOMPOSITION, args["content"], {"components.json": json.dumps(components)}, upstream(K.REQUIREMENTS), ) assert missing(COMPONENTS) == () assert missing([{"id": "C-1", "name": "Ingress", "requirements": ["NFR-9"]}]) == ( "components.json: C-1 serves NFR-9, which is not a requirement", "components.json: functional requirement FR-1 is served by no component", ) def test_every_component_gets_a_product_and_recommendations_name_real_components(): selection = sample_submission(K.PRODUCT_SELECTION).args advice = sample_submission(K.DESIGN_RECOMMENDATIONS).args built_on = upstream(K.REQUIREMENTS, K.TECHNICAL_DECOMPOSITION) assert check_submission( K.PRODUCT_SELECTION, selection["content"], {"products.json": json.dumps([{"component": "C-7", "product": "X"}])}, built_on, ) == ( "products.json: C-7 is not a component", "products.json: component C-1 has no product selected", ) wandering = [{**RECOMMENDATIONS[0], "components": ["C-7"]}] assert check_submission( K.DESIGN_RECOMMENDATIONS, advice["content"], {"recommendations.json": json.dumps(wandering)}, built_on, ) == ("recommendations.json: DR-1 names C-7, which is not a component",) # --- code and validation answer the graph ---------------------------------------------- EVERYTHING_BEFORE_CODE = (*DESIGN, K.TARGET_ARCHITECTURE) def test_deployment_guidance_must_account_for_every_check(): args = sample_submission(K.DEPLOYMENT_GUIDANCE).args partial = json.dumps( [ {"check": "K-1", "file": "terraform/main.tf", "address": "a"}, {"check": "K-7", "file": "terraform/elsewhere.tf", "address": "b"}, ] ) missing = check_submission( K.DEPLOYMENT_GUIDANCE, args["content"], {**args["attachments"], "implementation.json": partial}, upstream(*EVERYTHING_BEFORE_CODE), ) assert missing == ( "implementation.json: K-7 is not a Check in the Target architecture", "implementation.json: Check K-2 is not accounted for", "implementation.json: K-7 names terraform/elsewhere.tf, which is not attached", ) def test_a_check_that_cannot_be_implemented_is_accounted_for_with_its_reason(): args = sample_submission(K.DEPLOYMENT_GUIDANCE).args implementation = json.dumps( [ {"check": "K-1", "file": "terraform/main.tf", "address": "a"}, { "check": "K-2", "unimplementable": "example_service_account has no such argument", }, ] ) missing = check_submission( K.DEPLOYMENT_GUIDANCE, args["content"], {**args["attachments"], "implementation.json": implementation}, upstream(*EVERYTHING_BEFORE_CODE), ) assert missing == () assert unimplementable_checks(implementation) == [ {"check": "K-2", "reason": "example_service_account has no such argument"} ] assert unimplementable_checks(SAMPLE_DATA["implementation.json"]) == [] def test_an_unimplementable_entry_needs_its_reason(): args = sample_submission(K.DEPLOYMENT_GUIDANCE).args implementation = json.dumps( [ {"check": "K-1", "file": "terraform/main.tf", "address": "a"}, {"check": "K-2", "unimplementable": ""}, ] ) missing = check_submission( K.DEPLOYMENT_GUIDANCE, args["content"], {**args["attachments"], "implementation.json": implementation}, upstream(*EVERYTHING_BEFORE_CODE), ) assert missing == ( "implementation.json[0]: needs unimplementable", "implementation.json: Check K-2 is not accounted for", ) def verdicts(**changes) -> str: data = json.loads(SAMPLE_DATA["verdicts.json"]) data.update(changes) return json.dumps(data) def validation_missing(text: str) -> tuple[str, ...]: args = sample_submission(K.VALIDATION_RESULTS).args return check_submission( K.VALIDATION_RESULTS, args["content"], {"verdicts.json": text}, upstream(*EVERYTHING_BEFORE_CODE, K.DEPLOYMENT_GUIDANCE), ) def test_validation_gives_exactly_one_verdict_per_check(): one = {"check": "K-1", "verdict": "pass", "evidence": "seen"} invented = {"check": "K-9", "verdict": "fail", "evidence": "No Cloud Armor rule."} assert validation_missing(SAMPLE_DATA["verdicts.json"]) == () assert validation_missing(verdicts(checks=[one, one, invented])) == ( "verdicts.json.checks: check K-1 is used more than once", "verdicts.json.checks: K-9 is not a Check in the Target architecture", "verdicts.json.checks: Check K-2 is not accounted for", ) def test_the_validator_cannot_fail_the_code_on_something_the_graph_lacks(): """What it notices beyond the Checks is a proposed addition, never a failure.""" addition = { "statement": "Cloud Armor rate limiting is configured.", "applies_to": "E-1", "reason": "NFR-1", } text = verdicts(proposed_additions=[addition]) assert validation_missing(text) == () outcome = validation_outcome(text) assert outcome["failed"] == [] assert outcome["proposed_additions"] == [addition] def test_a_verdict_needs_evidence_and_a_known_value_and_the_dry_run_a_result(): vague = [ {"check": "K-1", "verdict": "probably", "evidence": "x"}, {"check": "K-2", "verdict": "pass", "evidence": " "}, ] assert validation_missing(verdicts(dry_run={"verdict": "ok"}, checks=vague)) == ( "verdicts.json.dry_run: needs a verdict (pass, fail, not_run) and evidence", "verdicts.json.checks[1]: needs evidence", "verdicts.json.checks: K-1 verdict must be one of pass, fail, not_verifiable", "verdicts.json.checks: Check K-2 is not accounted for", ) def test_a_validation_outcome_separates_faults_in_code_from_gaps_in_the_graph(): checks = [ {"check": "K-1", "verdict": "fail", "evidence": "min instances is 1"}, {"check": "K-2", "verdict": "not_verifiable", "evidence": "No such argument."}, ] outcome = validation_outcome(verdicts(checks=checks)) assert outcome["failed"] == [{"check": "K-1", "evidence": "min instances is 1"}] assert outcome["not_verifiable"] == [ {"check": "K-2", "evidence": "No such argument."} ] assert outcome["passed"] == [] assert outcome["dry_run"]["verdict"] == "pass" # --- the diagram is a view of the graph ------------------------------------------------ async def test_the_diagram_shows_every_element_and_relationship_once(): graph = parse_graph(SAMPLE_DATA["target-architecture.json"]) diagram = render_mermaid(graph) assert diagram == render_mermaid(graph) # deterministic assert diagram.count("Ingress service") == 1 assert diagram.count("Ingress identity") == 1 assert diagram.count("-->") == 1 assert "runs as" in diagram assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed") async def test_names_with_characters_mermaid_reserves_still_render(): data = graph_data() data["elements"][0]["name"] = 'Ingest "edge" [v2] (public) | {x}' data["elements"][0]["id"] = "svc/ingest.v2" data["relationships"][0]["from"] = "svc/ingest.v2" data["checks"][0]["applies_to"] = "svc/ingest.v2" data["relationships"][0]["kind"] = "calls (HTTPS) | signed" diagram = render_mermaid(parse_graph(json.dumps(data))) assert (await mermaid_check(diagram)).message.startswith("Mermaid check passed") assert "svc/ingest.v2" not in diagram async def test_the_engine_draws_the_diagram_when_the_target_architecture_is_submitted(): submit = specialist_tools( K.TARGET_ARCHITECTURE, extra=[], allow_cloud_read=False, upstream=upstream(*DESIGN), )[0] args = sample_submission(K.TARGET_ARCHITECTURE).args result = await submit.handler(**args) assert result.outcome.attachments["architecture.mmd"] == render_mermaid( parse_graph(SAMPLE_DATA["target-architecture.json"]) ) async def test_a_gap_is_refused_to_the_specialist_with_the_missing_link_named(): cmek = {"id": "DR-2", "pillar": "security", "text": "Use CMEK.", "components": []} built_on = upstream( *DESIGN, **{"recommendations.json": json.dumps([*RECOMMENDATIONS, cmek])} ) submit = specialist_tools( K.TARGET_ARCHITECTURE, extra=[], allow_cloud_read=False, upstream=built_on )[0] result = await submit.handler(**sample_submission(K.TARGET_ARCHITECTURE).args) assert result.outcome is None assert result.message == ( "Not submitted. The Deliverable is missing: Recommendation DR-2 has no Check" ) # --- through the engine ---------------------------------------------------------------- ROLES = { "orchestrator": RoleSpec("orchestrator", "Coordinate.", budget=40), **{ spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15) for spec in DEPENDENCY_GRAPH.values() }, } def delegate(*kinds: DeliverableKind) -> Call: return Call("delegate", {"kinds": [kind.value for kind in kinds]}) def approve(phase: PhaseId) -> Call: return Call("request_approval", {"phase": phase.value, "summary": "done"}) def specialist(kind: DeliverableKind) -> str: return DEPENDENCY_GRAPH[kind].specialist def to_design_approval() -> list[list[Call]]: return [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [approve(P.REQUIREMENTS_DISCOVERY), delegate(K.PRODUCT_SELECTION)], [delegate(K.DESIGN_RECOMMENDATIONS)], [delegate(K.TARGET_ARCHITECTURE)], ] def engine_for(tmp_path, turns: dict) -> tuple[ExecutionEngine, FakeAgentRuntime]: runtime = FakeAgentRuntime(turns) engine = ExecutionEngine( runtime=runtime, roles=ROLES, recorder=InMemoryRecorder(), checkpointer=InMemorySaver(), output_root=tmp_path, ) return engine, runtime async def test_the_user_approves_the_graph_before_any_code_is_written(tmp_path): once = { specialist(kind): [[sample_submission(kind)]] for kind in (*DESIGN, K.TARGET_ARCHITECTURE) } engine, runtime = engine_for( tmp_path, { **once, "orchestrator": [ [delegate(K.REQUIREMENTS)], [delegate(K.TECHNICAL_DECOMPOSITION)], [approve(P.REQUIREMENTS_DISCOVERY)], [delegate(K.PRODUCT_SELECTION)], [delegate(K.DESIGN_RECOMMENDATIONS)], [delegate(K.TARGET_ARCHITECTURE)], # Tries to start the code before asking: refused, then asks. [delegate(K.DEPLOYMENT_GUIDANCE), approve(P.SOLUTION_ARCHITECTURE)], ], }, ) first = await engine.start("Ingest webhooks") paused = await engine.resume( first.execution_id, {"approval": {"phase": "requirements_discovery", "decision": "approve"}}, ) assert paused.prompt["approval"]["phase"] == "solution_architecture" assert paused.prompt["approval"]["target_architecture"] == { "elements": 2, "relationships": 1, "checks": 2, "not_applicable": [], } refused = runtime.runs[-1].results[0] assert refused[0] == "delegate" and refused[1].startswith("Refused") assert specialist(K.DEPLOYMENT_GUIDANCE) not in [run.role for run in runtime.runs] async def test_the_orchestrator_is_told_what_validation_decided(tmp_path): failing = json.loads(SAMPLE_DATA["verdicts.json"]) failing["checks"][0] = { "check": "K-1", "verdict": "fail", "evidence": "min instances is 1", } failed_validation = sample_submission(K.VALIDATION_RESULTS) failed_validation = Call( failed_validation.tool, { **failed_validation.args, "attachments": {"verdicts.json": json.dumps(failing)}, }, ) once = { specialist(kind): [[sample_submission(kind)]] for kind in (*DESIGN, K.TARGET_ARCHITECTURE, K.ARCHITECTURE_DESCRIPTION) } engine, runtime = engine_for( tmp_path, { **once, "orchestrator": [ *to_design_approval(), [ approve(P.SOLUTION_ARCHITECTURE), delegate(K.ARCHITECTURE_DESCRIPTION, K.DEPLOYMENT_GUIDANCE), ], [approve(P.SOLUTION_IMPLEMENTATION), delegate(K.VALIDATION_RESULTS)], [ Call( "reopen", { "kind": "deployment_guidance", "reason": "K-1 failed: min instances is 1", }, ), delegate(K.DEPLOYMENT_GUIDANCE), ], [], ], specialist(K.DEPLOYMENT_GUIDANCE): [ [sample_submission(K.DEPLOYMENT_GUIDANCE)], [sample_submission(K.DEPLOYMENT_GUIDANCE)], ], specialist(K.VALIDATION_RESULTS): [[failed_validation]], }, ) result = await engine.start("Ingest webhooks", pre_approved=True) turns = [run for run in runtime.runs if run.role == "orchestrator"] before, after = turns[6].task.context, turns[7].task.context assert before["validation"] is None assert after["validation"]["failed"] == [ {"check": "K-1", "evidence": "min instances is 1"} ] assert after["validation"]["proposed_additions"] == [] # Once the code is reopened those verdicts are about work that no longer exists, # so the next turn must not see them and reopen the same thing again. assert turns[8].task.context["validation"] is None # Revising the code leaves the approved design standing. phases = result.state["ledger"]["phases"] assert phases["solution_architecture"] == "approved" revised = [ run for run in runtime.runs if run.role == specialist(K.DEPLOYMENT_GUIDANCE) ][1] assert revised.task.context["feedback"] == ["K-1 failed: min instances is 1"]