Files
gcp_solution_architecture_a…/tests/execution/test_coverage_live.py
Jonathan Boniface 5adc911a82 feat: keep an unchanged description, list conditions, review coverage with Jev
After case 02 (completed at 100%, but with two Revisions of the Target
architecture for gaps validation found one pass at a time):

- A Deliverable stands while what it was written from is unchanged. The
  architecture description is written from the Elements and Relationships,
  so a Revision that only adds or rewords Checks no longer runs the describer.
- Non-functional requirements and recommendations list their conditions,
  one entry per separate thing that must be true; submission requires them.
- A coverage review at submission (JEV_COVERAGE_REVIEW, off by default):
  for each condition, Jev chooses which Check verifies it, or none. Asked as
  one broad yes-or-no per requirement it could not tell a complete design
  from one with known gaps; asked as a choice per condition it raised all
  four known gaps of case 02 and nothing on the complete graph. An opt-in
  live test (LIVE_JEV=1) repeats that.
- The validator is told to report every gap in its first pass.

Plans record the case 02 run and both forms of the review with figures.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-07 20:58:38 +01:00

132 lines
4.4 KiB
Python

"""The coverage review against the live decision service. Opt-in: it costs calls.
LIVE_JEV=1 .venv/bin/python -m pytest tests/execution/test_coverage_live.py
The graph is the Target architecture evaluation case 02 ended with on 2026-10-07,
which validation passed with nothing proposed. Its first two validations proposed
four additions; the Checks added for them are removed here to rebuild those gaps.
The review must raise each gap, and nothing on the complete graph.
"""
import json
import os
from pathlib import Path
import pytest
from app.agent import open_decider
from app.config import Settings
from app.tools.coverage import CoverageReview
from app.workflows.deliverables import DeliverableKind
pytestmark = pytest.mark.skipif(
not os.environ.get("LIVE_JEV"), reason="set LIVE_JEV=1 to call the live service"
)
GRAPH = Path(__file__).parents[1] / "fixtures/coverage/case-02-target-architecture.json"
# (source, condition, whether the Check for it is removed below)
CONDITIONS = [
(
"NFR-1",
"Ingestion endpoints enforce TLS 1.2 or higher for data in transit",
True,
),
("NFR-8", "The Validation Worker's compute scales to zero when idle", True),
(
"NFR-10",
(
"The dashboard shows queue depth, processing latency and validation "
"failure rate"
),
True,
),
("DR-10", "An alert watches the Validation Worker's 5xx error rate", True),
("NFR-1", "Stored data is encrypted at rest", False),
("NFR-8", "The ingestion service's compute scales to zero when idle", False),
("NFR-9", "Quarantined records in dead-letter storage expire after 30 days", False),
(
"DR-7",
"The push subscription has a dead-letter topic with max delivery attempts of 5",
False,
),
("DR-10", "An alert watches the age of the oldest unacknowledged message", False),
]
GAPS = {condition for _, condition, removed in CONDITIONS if removed}
def upstream() -> dict:
by_source: dict[str, list[str]] = {}
for source, condition, _ in CONDITIONS:
by_source.setdefault(source, []).append(condition)
requirements = [
{"id": s, "kind": "non_functional", "text": s, "conditions": c}
for s, c in by_source.items()
if s.startswith("NFR")
]
recommendations = [
{"id": s, "pillar": "reliability", "text": s, "components": [], "conditions": c}
for s, c in by_source.items()
if s.startswith("DR")
]
return {
"requirements": {
"attachments": {"requirements.json": json.dumps(requirements)}
},
"design_recommendations": {
"attachments": {"recommendations.json": json.dumps(recommendations)}
},
}
def with_the_gaps(graph: dict) -> dict:
"""The graph without what the two Revisions added."""
cut = json.loads(json.dumps(graph))
added = {"E-18", "E-19", "E-20"} # the TLS policy and what it attaches to
cut["elements"] = [e for e in cut["elements"] if e["id"] not in added]
dangling = {
r["id"] for r in cut["relationships"] if r["from"] in added or r["to"] in added
}
cut["relationships"] = [r for r in cut["relationships"] if r["id"] not in dangling]
cut["checks"] = [
c
for c in cut["checks"]
if c["applies_to"] not in added | dangling
and c["id"] not in {"K-50", "K-51", "K-52"}
]
for check in cut["checks"]:
if check["id"] == "K-25":
check["statement"] = "dashboard_json is set"
return cut
async def raised(graph: dict) -> set[str]:
settings = Settings()
decider = open_decider(settings)
try:
concerns = await CoverageReview(
decider, none_at=settings.JEV_COVERAGE_THRESHOLD
)(
DeliverableKind.TARGET_ARCHITECTURE,
{"target-architecture.json": json.dumps(graph)},
upstream(),
"coverage-live",
)
finally:
aclose = getattr(decider, "aclose", None)
if aclose:
await aclose()
return {
condition
for _, condition, _ in CONDITIONS
if any(f'"{condition}"' in line for line in concerns)
}
async def test_nothing_is_raised_on_the_design_validation_passed():
assert await raised(json.loads(GRAPH.read_text())) == set()
async def test_every_known_gap_is_raised_and_nothing_else():
assert await raised(with_the_gaps(json.loads(GRAPH.read_text()))) == GAPS