"""The coverage review against the live decision service. Opt-in: it costs calls. LIVE_JEV=1 .venv/bin/python -m pytest tests/execution/test_coverage_live.py The graph is the Target architecture evaluation case 02 ended with on 2026-10-07, which validation passed with nothing proposed. Its first two validations proposed four additions; the Checks added for them are removed here to rebuild those gaps. The review must raise each gap, and nothing on the complete graph. """ import json import os from pathlib import Path import pytest from app.agent import open_decider from app.config import Settings from app.tools.coverage import CoverageReview from app.workflows.deliverables import DeliverableKind pytestmark = pytest.mark.skipif( not os.environ.get("LIVE_JEV"), reason="set LIVE_JEV=1 to call the live service" ) GRAPH = Path(__file__).parents[1] / "fixtures/coverage/case-02-target-architecture.json" # (source, condition, whether the Check for it is removed below) CONDITIONS = [ ( "NFR-1", "Ingestion endpoints enforce TLS 1.2 or higher for data in transit", True, ), ("NFR-8", "The Validation Worker's compute scales to zero when idle", True), ( "NFR-10", ( "The dashboard shows queue depth, processing latency and validation " "failure rate" ), True, ), ("DR-10", "An alert watches the Validation Worker's 5xx error rate", True), ("NFR-1", "Stored data is encrypted at rest", False), ("NFR-8", "The ingestion service's compute scales to zero when idle", False), ("NFR-9", "Quarantined records in dead-letter storage expire after 30 days", False), ( "DR-7", "The push subscription has a dead-letter topic with max delivery attempts of 5", False, ), ("DR-10", "An alert watches the age of the oldest unacknowledged message", False), ] GAPS = {condition for _, condition, removed in CONDITIONS if removed} def upstream() -> dict: by_source: dict[str, list[str]] = {} for source, condition, _ in CONDITIONS: by_source.setdefault(source, []).append(condition) requirements = [ {"id": s, "kind": "non_functional", "text": s, "conditions": c} for s, c in by_source.items() if s.startswith("NFR") ] recommendations = [ {"id": s, "pillar": "reliability", "text": s, "components": [], "conditions": c} for s, c in by_source.items() if s.startswith("DR") ] return { "requirements": { "attachments": {"requirements.json": json.dumps(requirements)} }, "design_recommendations": { "attachments": {"recommendations.json": json.dumps(recommendations)} }, } def with_the_gaps(graph: dict) -> dict: """The graph without what the two Revisions added.""" cut = json.loads(json.dumps(graph)) added = {"E-18", "E-19", "E-20"} # the TLS policy and what it attaches to cut["elements"] = [e for e in cut["elements"] if e["id"] not in added] dangling = { r["id"] for r in cut["relationships"] if r["from"] in added or r["to"] in added } cut["relationships"] = [r for r in cut["relationships"] if r["id"] not in dangling] cut["checks"] = [ c for c in cut["checks"] if c["applies_to"] not in added | dangling and c["id"] not in {"K-50", "K-51", "K-52"} ] for check in cut["checks"]: if check["id"] == "K-25": check["statement"] = "dashboard_json is set" return cut async def raised(graph: dict) -> set[str]: settings = Settings() decider = open_decider(settings) try: concerns = await CoverageReview( decider, none_at=settings.JEV_COVERAGE_THRESHOLD )( DeliverableKind.TARGET_ARCHITECTURE, {"target-architecture.json": json.dumps(graph)}, upstream(), "coverage-live", ) finally: aclose = getattr(decider, "aclose", None) if aclose: await aclose() return { condition for _, condition, _ in CONDITIONS if any(f'"{condition}"' in line for line in concerns) } async def test_nothing_is_raised_on_the_design_validation_passed(): assert await raised(json.loads(GRAPH.read_text())) == set() async def test_every_known_gap_is_raised_and_nothing_else(): assert await raised(with_the_gaps(json.loads(GRAPH.read_text()))) == GAPS