feat: keep an unchanged description, list conditions, review coverage with Jev
After case 02 (completed at 100%, but with two Revisions of the Target architecture for gaps validation found one pass at a time): - A Deliverable stands while what it was written from is unchanged. The architecture description is written from the Elements and Relationships, so a Revision that only adds or rewords Checks no longer runs the describer. - Non-functional requirements and recommendations list their conditions, one entry per separate thing that must be true; submission requires them. - A coverage review at submission (JEV_COVERAGE_REVIEW, off by default): for each condition, Jev chooses which Check verifies it, or none. Asked as one broad yes-or-no per requirement it could not tell a complete design from one with known gaps; asked as a choice per condition it raised all four known gaps of case 02 and nothing on the complete graph. An opt-in live test (LIVE_JEV=1) repeats that. - The validator is told to report every gap in its first pass. Plans record the case 02 run and both forms of the review with figures. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
131
tests/execution/test_coverage_live.py
Normal file
131
tests/execution/test_coverage_live.py
Normal file
@@ -0,0 +1,131 @@
|
||||
"""The coverage review against the live decision service. Opt-in: it costs calls.
|
||||
|
||||
LIVE_JEV=1 .venv/bin/python -m pytest tests/execution/test_coverage_live.py
|
||||
|
||||
The graph is the Target architecture evaluation case 02 ended with on 2026-10-07,
|
||||
which validation passed with nothing proposed. Its first two validations proposed
|
||||
four additions; the Checks added for them are removed here to rebuild those gaps.
|
||||
The review must raise each gap, and nothing on the complete graph.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.agent import open_decider
|
||||
from app.config import Settings
|
||||
from app.tools.coverage import CoverageReview
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not os.environ.get("LIVE_JEV"), reason="set LIVE_JEV=1 to call the live service"
|
||||
)
|
||||
|
||||
GRAPH = Path(__file__).parents[1] / "fixtures/coverage/case-02-target-architecture.json"
|
||||
|
||||
# (source, condition, whether the Check for it is removed below)
|
||||
CONDITIONS = [
|
||||
(
|
||||
"NFR-1",
|
||||
"Ingestion endpoints enforce TLS 1.2 or higher for data in transit",
|
||||
True,
|
||||
),
|
||||
("NFR-8", "The Validation Worker's compute scales to zero when idle", True),
|
||||
(
|
||||
"NFR-10",
|
||||
(
|
||||
"The dashboard shows queue depth, processing latency and validation "
|
||||
"failure rate"
|
||||
),
|
||||
True,
|
||||
),
|
||||
("DR-10", "An alert watches the Validation Worker's 5xx error rate", True),
|
||||
("NFR-1", "Stored data is encrypted at rest", False),
|
||||
("NFR-8", "The ingestion service's compute scales to zero when idle", False),
|
||||
("NFR-9", "Quarantined records in dead-letter storage expire after 30 days", False),
|
||||
(
|
||||
"DR-7",
|
||||
"The push subscription has a dead-letter topic with max delivery attempts of 5",
|
||||
False,
|
||||
),
|
||||
("DR-10", "An alert watches the age of the oldest unacknowledged message", False),
|
||||
]
|
||||
GAPS = {condition for _, condition, removed in CONDITIONS if removed}
|
||||
|
||||
|
||||
def upstream() -> dict:
|
||||
by_source: dict[str, list[str]] = {}
|
||||
for source, condition, _ in CONDITIONS:
|
||||
by_source.setdefault(source, []).append(condition)
|
||||
requirements = [
|
||||
{"id": s, "kind": "non_functional", "text": s, "conditions": c}
|
||||
for s, c in by_source.items()
|
||||
if s.startswith("NFR")
|
||||
]
|
||||
recommendations = [
|
||||
{"id": s, "pillar": "reliability", "text": s, "components": [], "conditions": c}
|
||||
for s, c in by_source.items()
|
||||
if s.startswith("DR")
|
||||
]
|
||||
return {
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": json.dumps(requirements)}
|
||||
},
|
||||
"design_recommendations": {
|
||||
"attachments": {"recommendations.json": json.dumps(recommendations)}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def with_the_gaps(graph: dict) -> dict:
|
||||
"""The graph without what the two Revisions added."""
|
||||
cut = json.loads(json.dumps(graph))
|
||||
added = {"E-18", "E-19", "E-20"} # the TLS policy and what it attaches to
|
||||
cut["elements"] = [e for e in cut["elements"] if e["id"] not in added]
|
||||
dangling = {
|
||||
r["id"] for r in cut["relationships"] if r["from"] in added or r["to"] in added
|
||||
}
|
||||
cut["relationships"] = [r for r in cut["relationships"] if r["id"] not in dangling]
|
||||
cut["checks"] = [
|
||||
c
|
||||
for c in cut["checks"]
|
||||
if c["applies_to"] not in added | dangling
|
||||
and c["id"] not in {"K-50", "K-51", "K-52"}
|
||||
]
|
||||
for check in cut["checks"]:
|
||||
if check["id"] == "K-25":
|
||||
check["statement"] = "dashboard_json is set"
|
||||
return cut
|
||||
|
||||
|
||||
async def raised(graph: dict) -> set[str]:
|
||||
settings = Settings()
|
||||
decider = open_decider(settings)
|
||||
try:
|
||||
concerns = await CoverageReview(
|
||||
decider, none_at=settings.JEV_COVERAGE_THRESHOLD
|
||||
)(
|
||||
DeliverableKind.TARGET_ARCHITECTURE,
|
||||
{"target-architecture.json": json.dumps(graph)},
|
||||
upstream(),
|
||||
"coverage-live",
|
||||
)
|
||||
finally:
|
||||
aclose = getattr(decider, "aclose", None)
|
||||
if aclose:
|
||||
await aclose()
|
||||
return {
|
||||
condition
|
||||
for _, condition, _ in CONDITIONS
|
||||
if any(f'"{condition}"' in line for line in concerns)
|
||||
}
|
||||
|
||||
|
||||
async def test_nothing_is_raised_on_the_design_validation_passed():
|
||||
assert await raised(json.loads(GRAPH.read_text())) == set()
|
||||
|
||||
|
||||
async def test_every_known_gap_is_raised_and_nothing_else():
|
||||
assert await raised(with_the_gaps(json.loads(GRAPH.read_text()))) == GAPS
|
||||
Reference in New Issue
Block a user