After case 02 (completed at 100%, but with two Revisions of the Target architecture for gaps validation found one pass at a time): - A Deliverable stands while what it was written from is unchanged. The architecture description is written from the Elements and Relationships, so a Revision that only adds or rewords Checks no longer runs the describer. - Non-functional requirements and recommendations list their conditions, one entry per separate thing that must be true; submission requires them. - A coverage review at submission (JEV_COVERAGE_REVIEW, off by default): for each condition, Jev chooses which Check verifies it, or none. Asked as one broad yes-or-no per requirement it could not tell a complete design from one with known gaps; asked as a choice per condition it raised all four known gaps of case 02 and nothing on the complete graph. An opt-in live test (LIVE_JEV=1) repeats that. - The validator is told to report every gap in its first pass. Plans record the case 02 run and both forms of the review with figures. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
132 lines
4.4 KiB
Python
132 lines
4.4 KiB
Python
"""The coverage review against the live decision service. Opt-in: it costs calls.
|
|
|
|
LIVE_JEV=1 .venv/bin/python -m pytest tests/execution/test_coverage_live.py
|
|
|
|
The graph is the Target architecture evaluation case 02 ended with on 2026-10-07,
|
|
which validation passed with nothing proposed. Its first two validations proposed
|
|
four additions; the Checks added for them are removed here to rebuild those gaps.
|
|
The review must raise each gap, and nothing on the complete graph.
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.agent import open_decider
|
|
from app.config import Settings
|
|
from app.tools.coverage import CoverageReview
|
|
from app.workflows.deliverables import DeliverableKind
|
|
|
|
pytestmark = pytest.mark.skipif(
|
|
not os.environ.get("LIVE_JEV"), reason="set LIVE_JEV=1 to call the live service"
|
|
)
|
|
|
|
GRAPH = Path(__file__).parents[1] / "fixtures/coverage/case-02-target-architecture.json"
|
|
|
|
# (source, condition, whether the Check for it is removed below)
|
|
CONDITIONS = [
|
|
(
|
|
"NFR-1",
|
|
"Ingestion endpoints enforce TLS 1.2 or higher for data in transit",
|
|
True,
|
|
),
|
|
("NFR-8", "The Validation Worker's compute scales to zero when idle", True),
|
|
(
|
|
"NFR-10",
|
|
(
|
|
"The dashboard shows queue depth, processing latency and validation "
|
|
"failure rate"
|
|
),
|
|
True,
|
|
),
|
|
("DR-10", "An alert watches the Validation Worker's 5xx error rate", True),
|
|
("NFR-1", "Stored data is encrypted at rest", False),
|
|
("NFR-8", "The ingestion service's compute scales to zero when idle", False),
|
|
("NFR-9", "Quarantined records in dead-letter storage expire after 30 days", False),
|
|
(
|
|
"DR-7",
|
|
"The push subscription has a dead-letter topic with max delivery attempts of 5",
|
|
False,
|
|
),
|
|
("DR-10", "An alert watches the age of the oldest unacknowledged message", False),
|
|
]
|
|
GAPS = {condition for _, condition, removed in CONDITIONS if removed}
|
|
|
|
|
|
def upstream() -> dict:
|
|
by_source: dict[str, list[str]] = {}
|
|
for source, condition, _ in CONDITIONS:
|
|
by_source.setdefault(source, []).append(condition)
|
|
requirements = [
|
|
{"id": s, "kind": "non_functional", "text": s, "conditions": c}
|
|
for s, c in by_source.items()
|
|
if s.startswith("NFR")
|
|
]
|
|
recommendations = [
|
|
{"id": s, "pillar": "reliability", "text": s, "components": [], "conditions": c}
|
|
for s, c in by_source.items()
|
|
if s.startswith("DR")
|
|
]
|
|
return {
|
|
"requirements": {
|
|
"attachments": {"requirements.json": json.dumps(requirements)}
|
|
},
|
|
"design_recommendations": {
|
|
"attachments": {"recommendations.json": json.dumps(recommendations)}
|
|
},
|
|
}
|
|
|
|
|
|
def with_the_gaps(graph: dict) -> dict:
|
|
"""The graph without what the two Revisions added."""
|
|
cut = json.loads(json.dumps(graph))
|
|
added = {"E-18", "E-19", "E-20"} # the TLS policy and what it attaches to
|
|
cut["elements"] = [e for e in cut["elements"] if e["id"] not in added]
|
|
dangling = {
|
|
r["id"] for r in cut["relationships"] if r["from"] in added or r["to"] in added
|
|
}
|
|
cut["relationships"] = [r for r in cut["relationships"] if r["id"] not in dangling]
|
|
cut["checks"] = [
|
|
c
|
|
for c in cut["checks"]
|
|
if c["applies_to"] not in added | dangling
|
|
and c["id"] not in {"K-50", "K-51", "K-52"}
|
|
]
|
|
for check in cut["checks"]:
|
|
if check["id"] == "K-25":
|
|
check["statement"] = "dashboard_json is set"
|
|
return cut
|
|
|
|
|
|
async def raised(graph: dict) -> set[str]:
|
|
settings = Settings()
|
|
decider = open_decider(settings)
|
|
try:
|
|
concerns = await CoverageReview(
|
|
decider, none_at=settings.JEV_COVERAGE_THRESHOLD
|
|
)(
|
|
DeliverableKind.TARGET_ARCHITECTURE,
|
|
{"target-architecture.json": json.dumps(graph)},
|
|
upstream(),
|
|
"coverage-live",
|
|
)
|
|
finally:
|
|
aclose = getattr(decider, "aclose", None)
|
|
if aclose:
|
|
await aclose()
|
|
return {
|
|
condition
|
|
for _, condition, _ in CONDITIONS
|
|
if any(f'"{condition}"' in line for line in concerns)
|
|
}
|
|
|
|
|
|
async def test_nothing_is_raised_on_the_design_validation_passed():
|
|
assert await raised(json.loads(GRAPH.read_text())) == set()
|
|
|
|
|
|
async def test_every_known_gap_is_raised_and_nothing_else():
|
|
assert await raised(with_the_gaps(json.loads(GRAPH.read_text()))) == GAPS
|