feat: keep an unchanged description, list conditions, review coverage with Jev
After case 02 (completed at 100%, but with two Revisions of the Target architecture for gaps validation found one pass at a time): - A Deliverable stands while what it was written from is unchanged. The architecture description is written from the Elements and Relationships, so a Revision that only adds or rewords Checks no longer runs the describer. - Non-functional requirements and recommendations list their conditions, one entry per separate thing that must be true; submission requires them. - A coverage review at submission (JEV_COVERAGE_REVIEW, off by default): for each condition, Jev chooses which Check verifies it, or none. Asked as one broad yes-or-no per requirement it could not tell a complete design from one with known gaps; asked as a choice per condition it raised all four known gaps of case 02 and nothing on the complete graph. An opt-in live test (LIVE_JEV=1) repeats that. - The validator is told to report every gap in its first pass. Plans record the case 02 run and both forms of the review with figures. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
131
tests/execution/test_coverage_live.py
Normal file
131
tests/execution/test_coverage_live.py
Normal file
@@ -0,0 +1,131 @@
|
||||
"""The coverage review against the live decision service. Opt-in: it costs calls.
|
||||
|
||||
LIVE_JEV=1 .venv/bin/python -m pytest tests/execution/test_coverage_live.py
|
||||
|
||||
The graph is the Target architecture evaluation case 02 ended with on 2026-10-07,
|
||||
which validation passed with nothing proposed. Its first two validations proposed
|
||||
four additions; the Checks added for them are removed here to rebuild those gaps.
|
||||
The review must raise each gap, and nothing on the complete graph.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.agent import open_decider
|
||||
from app.config import Settings
|
||||
from app.tools.coverage import CoverageReview
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not os.environ.get("LIVE_JEV"), reason="set LIVE_JEV=1 to call the live service"
|
||||
)
|
||||
|
||||
GRAPH = Path(__file__).parents[1] / "fixtures/coverage/case-02-target-architecture.json"
|
||||
|
||||
# (source, condition, whether the Check for it is removed below)
|
||||
CONDITIONS = [
|
||||
(
|
||||
"NFR-1",
|
||||
"Ingestion endpoints enforce TLS 1.2 or higher for data in transit",
|
||||
True,
|
||||
),
|
||||
("NFR-8", "The Validation Worker's compute scales to zero when idle", True),
|
||||
(
|
||||
"NFR-10",
|
||||
(
|
||||
"The dashboard shows queue depth, processing latency and validation "
|
||||
"failure rate"
|
||||
),
|
||||
True,
|
||||
),
|
||||
("DR-10", "An alert watches the Validation Worker's 5xx error rate", True),
|
||||
("NFR-1", "Stored data is encrypted at rest", False),
|
||||
("NFR-8", "The ingestion service's compute scales to zero when idle", False),
|
||||
("NFR-9", "Quarantined records in dead-letter storage expire after 30 days", False),
|
||||
(
|
||||
"DR-7",
|
||||
"The push subscription has a dead-letter topic with max delivery attempts of 5",
|
||||
False,
|
||||
),
|
||||
("DR-10", "An alert watches the age of the oldest unacknowledged message", False),
|
||||
]
|
||||
GAPS = {condition for _, condition, removed in CONDITIONS if removed}
|
||||
|
||||
|
||||
def upstream() -> dict:
|
||||
by_source: dict[str, list[str]] = {}
|
||||
for source, condition, _ in CONDITIONS:
|
||||
by_source.setdefault(source, []).append(condition)
|
||||
requirements = [
|
||||
{"id": s, "kind": "non_functional", "text": s, "conditions": c}
|
||||
for s, c in by_source.items()
|
||||
if s.startswith("NFR")
|
||||
]
|
||||
recommendations = [
|
||||
{"id": s, "pillar": "reliability", "text": s, "components": [], "conditions": c}
|
||||
for s, c in by_source.items()
|
||||
if s.startswith("DR")
|
||||
]
|
||||
return {
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": json.dumps(requirements)}
|
||||
},
|
||||
"design_recommendations": {
|
||||
"attachments": {"recommendations.json": json.dumps(recommendations)}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def with_the_gaps(graph: dict) -> dict:
|
||||
"""The graph without what the two Revisions added."""
|
||||
cut = json.loads(json.dumps(graph))
|
||||
added = {"E-18", "E-19", "E-20"} # the TLS policy and what it attaches to
|
||||
cut["elements"] = [e for e in cut["elements"] if e["id"] not in added]
|
||||
dangling = {
|
||||
r["id"] for r in cut["relationships"] if r["from"] in added or r["to"] in added
|
||||
}
|
||||
cut["relationships"] = [r for r in cut["relationships"] if r["id"] not in dangling]
|
||||
cut["checks"] = [
|
||||
c
|
||||
for c in cut["checks"]
|
||||
if c["applies_to"] not in added | dangling
|
||||
and c["id"] not in {"K-50", "K-51", "K-52"}
|
||||
]
|
||||
for check in cut["checks"]:
|
||||
if check["id"] == "K-25":
|
||||
check["statement"] = "dashboard_json is set"
|
||||
return cut
|
||||
|
||||
|
||||
async def raised(graph: dict) -> set[str]:
|
||||
settings = Settings()
|
||||
decider = open_decider(settings)
|
||||
try:
|
||||
concerns = await CoverageReview(
|
||||
decider, none_at=settings.JEV_COVERAGE_THRESHOLD
|
||||
)(
|
||||
DeliverableKind.TARGET_ARCHITECTURE,
|
||||
{"target-architecture.json": json.dumps(graph)},
|
||||
upstream(),
|
||||
"coverage-live",
|
||||
)
|
||||
finally:
|
||||
aclose = getattr(decider, "aclose", None)
|
||||
if aclose:
|
||||
await aclose()
|
||||
return {
|
||||
condition
|
||||
for _, condition, _ in CONDITIONS
|
||||
if any(f'"{condition}"' in line for line in concerns)
|
||||
}
|
||||
|
||||
|
||||
async def test_nothing_is_raised_on_the_design_validation_passed():
|
||||
assert await raised(json.loads(GRAPH.read_text())) == set()
|
||||
|
||||
|
||||
async def test_every_known_gap_is_raised_and_nothing_else():
|
||||
assert await raised(with_the_gaps(json.loads(GRAPH.read_text()))) == GAPS
|
||||
401
tests/execution/test_coverage_review.py
Normal file
401
tests/execution/test_coverage_review.py
Normal file
@@ -0,0 +1,401 @@
|
||||
"""Two savings on a Revision of the design.
|
||||
|
||||
A coverage review at submission finds what validation would otherwise find later,
|
||||
at the cost of a Revision. And a Deliverable written from the shape of the design is
|
||||
not written again when a Revision only changes its Checks.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import (
|
||||
SAMPLE_DATA,
|
||||
Call,
|
||||
FakeAgentRuntime,
|
||||
FakeDecider,
|
||||
sample_submission,
|
||||
)
|
||||
from app.adk.runtime import Answer, NoDecider, RoleSpec, Submitted
|
||||
from app.agent import ExecutionEngine
|
||||
from app.nodes.router import Router, Thresholds
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.coverage import CoverageReview
|
||||
from app.tools.execution import specialist_tools
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH, DeliverableKind
|
||||
|
||||
K = DeliverableKind
|
||||
GRAPH = {"target-architecture.json": SAMPLE_DATA["target-architecture.json"]}
|
||||
UPSTREAM = {
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": SAMPLE_DATA["requirements.json"]}
|
||||
},
|
||||
"technical_decomposition": {
|
||||
"attachments": {"components.json": SAMPLE_DATA["components.json"]}
|
||||
},
|
||||
"product_selection": {
|
||||
"attachments": {"products.json": SAMPLE_DATA["products.json"]}
|
||||
},
|
||||
"design_recommendations": {
|
||||
"attachments": {"recommendations.json": SAMPLE_DATA["recommendations.json"]}
|
||||
},
|
||||
}
|
||||
CONTENT = "## Design rationale\nReasons.\n"
|
||||
|
||||
|
||||
# --- the review ------------------------------------------------------------------------
|
||||
|
||||
|
||||
def choice(option: str, probability: float) -> Answer:
|
||||
return Answer(option, probability, {option: probability})
|
||||
|
||||
|
||||
async def test_the_review_asks_which_check_verifies_each_condition():
|
||||
decider = FakeDecider({"NFR-1#1": choice("K-1", 0.9), "DR-1#1": choice("K-2", 0.9)})
|
||||
|
||||
concerns = await CoverageReview(decider)(
|
||||
K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM, "exec-1"
|
||||
)
|
||||
|
||||
state, questions, scope = decider.asked[0]
|
||||
assert concerns == ()
|
||||
assert scope == "exec-1"
|
||||
assert "E-1 Ingress service (Example Run, example_run_service)" in state
|
||||
# Constraints only: a functional requirement is carried by its component.
|
||||
assert set(questions) == {"NFR-1#1", "DR-1#1"}
|
||||
question = questions["NFR-1#1"]
|
||||
assert question.kind == "choice" and question.options == ("K-1", "none")
|
||||
assert 'Condition: "The service stays available when one instance fails."' in (
|
||||
question.instructions
|
||||
)
|
||||
assert "- K-1 on Ingress service: The service runs at least two instances." in (
|
||||
question.instructions
|
||||
)
|
||||
|
||||
|
||||
async def test_a_condition_no_check_verifies_is_raised_with_the_checks_it_has():
|
||||
decider = FakeDecider({"NFR-1#1": choice("none", 0.9), "DR-1#1": choice("K-2", 1)})
|
||||
|
||||
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM)
|
||||
|
||||
condition = "The service stays available when one instance fails."
|
||||
assert concerns == (f'NFR-1: no Check verifies "{condition}" (its Checks: K-1)',)
|
||||
|
||||
|
||||
async def test_an_uncertain_none_is_not_raised():
|
||||
decider = FakeDecider({"NFR-1#1": choice("none", 0.45), "DR-1#1": choice("K-2", 1)})
|
||||
|
||||
review = CoverageReview(decider, none_at=0.6)
|
||||
|
||||
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
||||
|
||||
|
||||
async def test_each_condition_of_a_requirement_is_asked_about_separately():
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
requirements[1]["conditions"] = [
|
||||
"Survives one instance failing.",
|
||||
"Survives a zone failing.",
|
||||
]
|
||||
upstream = {
|
||||
**UPSTREAM,
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": json.dumps(requirements)}
|
||||
},
|
||||
}
|
||||
decider = FakeDecider(
|
||||
{
|
||||
"NFR-1#1": choice("K-1", 0.9),
|
||||
"NFR-1#2": choice("none", 0.8),
|
||||
"DR-1#1": choice("K-2", 1),
|
||||
}
|
||||
)
|
||||
|
||||
concerns = await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
|
||||
|
||||
assert len(concerns) == 1 and "Survives a zone failing." in concerns[0]
|
||||
|
||||
|
||||
async def test_an_item_without_conditions_is_read_as_one_condition_its_text():
|
||||
requirements = json.loads(SAMPLE_DATA["requirements.json"])
|
||||
del requirements[1]["conditions"]
|
||||
upstream = {
|
||||
**UPSTREAM,
|
||||
"requirements": {
|
||||
"attachments": {"requirements.json": json.dumps(requirements)}
|
||||
},
|
||||
}
|
||||
decider = FakeDecider({"NFR-1#1": choice("K-1", 1), "DR-1#1": choice("K-2", 1)})
|
||||
|
||||
await CoverageReview(decider)(K.TARGET_ARCHITECTURE, GRAPH, upstream)
|
||||
|
||||
assert 'Condition: "99.9% availability."' in (
|
||||
decider.asked[0][1]["NFR-1#1"].instructions
|
||||
)
|
||||
|
||||
|
||||
async def test_no_decision_and_other_deliverables_raise_nothing():
|
||||
review = CoverageReview(NoDecider())
|
||||
decider = FakeDecider({})
|
||||
|
||||
assert await review(K.TARGET_ARCHITECTURE, GRAPH, UPSTREAM) == ()
|
||||
assert await CoverageReview(decider)(K.REQUIREMENTS, {}, UPSTREAM) == ()
|
||||
assert await CoverageReview(decider)(K.TARGET_ARCHITECTURE, {}, UPSTREAM) == ()
|
||||
assert decider.asked == []
|
||||
|
||||
|
||||
async def test_a_recommendation_marked_not_applicable_is_not_reviewed():
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
graph["checks"][1]["source"] = "NFR-1"
|
||||
graph["not_applicable"] = [{"recommendation": "DR-1", "reason": "No identity."}]
|
||||
decider = FakeDecider({"NFR-1#1": choice("K-1", 1)})
|
||||
|
||||
await CoverageReview(decider)(
|
||||
K.TARGET_ARCHITECTURE,
|
||||
{"target-architecture.json": json.dumps(graph)},
|
||||
UPSTREAM,
|
||||
)
|
||||
|
||||
assert set(decider.asked[0][1]) == {"NFR-1#1"}
|
||||
|
||||
|
||||
# --- at submission ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def submit_with(review, working=None):
|
||||
(submit,) = (
|
||||
tool
|
||||
for tool in specialist_tools(
|
||||
K.TARGET_ARCHITECTURE,
|
||||
(),
|
||||
allow_cloud_read=False,
|
||||
upstream=UPSTREAM,
|
||||
working={} if working is None else working,
|
||||
review=review,
|
||||
)
|
||||
if tool.name == "submit_deliverable"
|
||||
)
|
||||
return submit.handler
|
||||
|
||||
|
||||
async def test_what_the_review_raises_is_returned_once_and_then_accepted():
|
||||
calls = []
|
||||
|
||||
async def review(attachments):
|
||||
calls.append(attachments)
|
||||
return ['NFR-1: no Check verifies "Two instances." (its Checks: K-1)']
|
||||
|
||||
working: dict = {}
|
||||
submit = submit_with(review, working)
|
||||
|
||||
first = await submit(content=CONTENT, attachments=dict(GRAPH))
|
||||
second = await submit(content=CONTENT)
|
||||
|
||||
assert first.outcome is None
|
||||
assert first.message.startswith("Not submitted. A review found")
|
||||
assert "NFR-1: no Check verifies" in first.message
|
||||
assert "runs once" in first.message
|
||||
# The graph it sent is kept, so it amends that and does not send it again.
|
||||
assert isinstance(second.outcome, Submitted)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_a_structural_fault_is_reported_before_any_review():
|
||||
async def review(attachments):
|
||||
raise AssertionError("not reached")
|
||||
|
||||
result = await submit_with(review)(content=CONTENT, attachments={})
|
||||
|
||||
assert "attachment: target-architecture.json" in result.message
|
||||
|
||||
|
||||
async def test_a_review_that_fails_or_raises_nothing_never_stops_a_submission():
|
||||
async def broken(attachments):
|
||||
raise RuntimeError("service down")
|
||||
|
||||
async def quiet(attachments):
|
||||
return ()
|
||||
|
||||
for review in (broken, quiet):
|
||||
result = await submit_with(review)(content=CONTENT, attachments=dict(GRAPH))
|
||||
assert isinstance(result.outcome, Submitted)
|
||||
|
||||
|
||||
# --- a description stands while the shape of the design does ---------------------------
|
||||
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(spec.specialist, "Produce.", budget=15)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def graph_with(**changes) -> Call:
|
||||
graph = json.loads(SAMPLE_DATA["target-architecture.json"])
|
||||
for key, change in changes.items():
|
||||
graph[key] = change(graph[key])
|
||||
return Call(
|
||||
"submit_deliverable",
|
||||
{
|
||||
"content": CONTENT,
|
||||
"attachments": {"target-architecture.json": json.dumps(graph)},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def addition() -> Call:
|
||||
"""A validation that passes every Check and proposes one more."""
|
||||
call = sample_submission(K.VALIDATION_RESULTS)
|
||||
verdicts = json.loads(SAMPLE_DATA["verdicts.json"])
|
||||
verdicts["proposed_additions"] = [
|
||||
{"statement": "A timeout is set.", "applies_to": "E-1", "reason": "NFR-1"}
|
||||
]
|
||||
return Call(
|
||||
call.tool,
|
||||
{**call.args, "attachments": {"verdicts.json": json.dumps(verdicts)}},
|
||||
)
|
||||
|
||||
|
||||
async def run(tmp_path, second_graph: Call, **options):
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]] * 2
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
turns["target-architect"] = [
|
||||
[sample_submission(K.TARGET_ARCHITECTURE)],
|
||||
[second_graph],
|
||||
]
|
||||
turns["validator"] = [[addition()], [sample_submission(K.VALIDATION_RESULTS)]]
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
**options,
|
||||
)
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True)
|
||||
assert result.status == "completed"
|
||||
return result, [run.role for run in runtime.runs]
|
||||
|
||||
|
||||
async def test_the_description_is_kept_when_a_revision_only_changes_checks(tmp_path):
|
||||
reworded = graph_with(
|
||||
checks=lambda checks: [
|
||||
{**checks[0], "statement": "min_instance_count is at least 2"},
|
||||
checks[1],
|
||||
]
|
||||
)
|
||||
|
||||
result, roles_run = await run(tmp_path, reworded)
|
||||
|
||||
assert roles_run.count("target-architect") == 2
|
||||
assert roles_run.count("iac-author") == 2 # the code answers the Checks
|
||||
assert roles_run.count("describer") == 1
|
||||
# The Ledger still records a second submission: the Deliverable was not skipped.
|
||||
submitted = [
|
||||
event
|
||||
for event in result.state["ledger"]["events"]
|
||||
if event["type"] == "submitted"
|
||||
and event["subject"] == "architecture_description"
|
||||
]
|
||||
assert len(submitted) == 2
|
||||
|
||||
|
||||
async def test_the_description_is_written_again_when_the_shape_changes(tmp_path):
|
||||
renamed = graph_with(
|
||||
elements=lambda elements: [{**elements[0], "name": "Edge service"}, elements[1]]
|
||||
)
|
||||
|
||||
_, roles_run = await run(tmp_path, renamed)
|
||||
|
||||
assert roles_run.count("describer") == 2
|
||||
|
||||
|
||||
async def test_the_engine_reviews_a_design_before_it_is_submitted(tmp_path):
|
||||
raised = []
|
||||
|
||||
async def review(kind, attachments, upstream, scope=""):
|
||||
if kind is K.TARGET_ARCHITECTURE and not raised:
|
||||
raised.append(scope)
|
||||
return ['NFR-1 "Two instances." (its Checks: K-1)']
|
||||
return ()
|
||||
|
||||
turns = {
|
||||
spec.specialist: [[sample_submission(kind)]]
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
turns["target-architect"] = [
|
||||
[sample_submission(K.TARGET_ARCHITECTURE)] * 2 # refused once, then accepted
|
||||
]
|
||||
runtime = FakeAgentRuntime(turns)
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
submission_review=review,
|
||||
)
|
||||
|
||||
result = await engine.start("Ingest webhooks", pre_approved=True, execution_id="e1")
|
||||
|
||||
assert result.status == "completed"
|
||||
assert raised == ["e1"]
|
||||
architect = next(run for run in runtime.runs if run.role == "target-architect")
|
||||
assert architect.results[0][1].startswith("Not submitted. A review found")
|
||||
# What a review raises is kept as a lesson candidate, like any refusal.
|
||||
assert any(
|
||||
"A review found" in json.dumps(outcome.get("refusals"))
|
||||
for outcome in result.state["outcomes"]
|
||||
)
|
||||
|
||||
|
||||
async def test_feedback_on_the_description_itself_has_it_written_again(tmp_path):
|
||||
"""Kept only while nobody has asked for the description to change."""
|
||||
from app.nodes.execution_nodes import STANDING, make_execution_nodes
|
||||
|
||||
nodes = make_execution_nodes(
|
||||
runtime=FakeAgentRuntime(
|
||||
{"describer": [[sample_submission(K.ARCHITECTURE_DESCRIPTION, "new")]]}
|
||||
),
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
output_root=tmp_path,
|
||||
)
|
||||
upstream = {"target_architecture": {"attachments": dict(GRAPH)}}
|
||||
from app.workflows.chain import graph_structure
|
||||
|
||||
standing = {
|
||||
"basis": graph_structure(upstream),
|
||||
"feedback": [],
|
||||
"content": "old",
|
||||
"attachments": {},
|
||||
}
|
||||
task = {
|
||||
"execution_id": "e1",
|
||||
"role": "describer",
|
||||
"kind": "architecture_description",
|
||||
"attempt": 2,
|
||||
"allow_cloud_read": False,
|
||||
"working": {STANDING: standing},
|
||||
"context": {
|
||||
"dependencies": upstream,
|
||||
"feedback": ["Dependency x was revised: y"],
|
||||
},
|
||||
}
|
||||
|
||||
kept = await nodes.specialist_task(task)
|
||||
asked = await nodes.specialist_task(
|
||||
{**task, "context": {**task["context"], "feedback": ["Make the flow clearer."]}}
|
||||
)
|
||||
|
||||
assert kept["outcomes"][0]["content"] == "old"
|
||||
assert "new" in asked["outcomes"][0]["content"]
|
||||
assert asked["working"]["architecture_description"][STANDING]["feedback"] == [
|
||||
"Make the flow clearer."
|
||||
]
|
||||
@@ -665,3 +665,51 @@ async def test_the_orchestrator_is_told_what_validation_decided(tmp_path):
|
||||
run for run in runtime.runs if run.role == specialist(K.DEPLOYMENT_GUIDANCE)
|
||||
][1]
|
||||
assert revised.task.context["feedback"] == ["K-1 failed: min instances is 1"]
|
||||
|
||||
|
||||
# --- constraints list their conditions --------------------------------------------------
|
||||
|
||||
|
||||
def test_a_non_functional_requirement_must_list_its_conditions():
|
||||
requirements = [
|
||||
{"id": "FR-1", "kind": "functional", "text": "Accept events."},
|
||||
{"id": "NFR-1", "kind": "non_functional", "text": "Encrypted and retained."},
|
||||
{
|
||||
"id": "NFR-2",
|
||||
"kind": "non_functional",
|
||||
"text": "Available.",
|
||||
"conditions": [],
|
||||
},
|
||||
]
|
||||
args = sample_submission(K.REQUIREMENTS).args
|
||||
|
||||
missing = check_submission(
|
||||
K.REQUIREMENTS,
|
||||
args["content"],
|
||||
{"requirements.json": json.dumps(requirements)},
|
||||
)
|
||||
|
||||
needs = (
|
||||
"needs `conditions`: a list with one entry for each separate thing it asks for"
|
||||
)
|
||||
assert missing == (
|
||||
f"requirements.json: NFR-1 {needs}",
|
||||
f"requirements.json: NFR-2 {needs}",
|
||||
)
|
||||
|
||||
|
||||
def test_a_recommendation_must_list_its_conditions():
|
||||
recommendations = json.loads(SAMPLE_DATA["recommendations.json"])
|
||||
del recommendations[0]["conditions"]
|
||||
args = sample_submission(K.DESIGN_RECOMMENDATIONS).args
|
||||
|
||||
missing = check_submission(
|
||||
K.DESIGN_RECOMMENDATIONS,
|
||||
args["content"],
|
||||
{"recommendations.json": json.dumps(recommendations)},
|
||||
)
|
||||
|
||||
needs = (
|
||||
"needs `conditions`: a list with one entry for each separate thing it asks for"
|
||||
)
|
||||
assert missing == (f"recommendations.json: DR-1 {needs}",)
|
||||
|
||||
604
tests/fixtures/coverage/case-02-target-architecture.json
vendored
Normal file
604
tests/fixtures/coverage/case-02-target-architecture.json
vendored
Normal file
@@ -0,0 +1,604 @@
|
||||
{
|
||||
"elements": [
|
||||
{
|
||||
"name": "Ingestion Endpoint Service",
|
||||
"product": "Cloud Run",
|
||||
"resource_type": "google_cloud_run_v2_service",
|
||||
"id": "E-1",
|
||||
"component": "C-1"
|
||||
},
|
||||
{
|
||||
"resource_type": "google_pubsub_topic",
|
||||
"component": "C-2",
|
||||
"product": "Cloud Pub/Sub",
|
||||
"id": "E-2",
|
||||
"name": "Asynchronous Ingestion Buffer Topic"
|
||||
},
|
||||
{
|
||||
"id": "E-3",
|
||||
"component": "C-3",
|
||||
"product": "Cloud Run",
|
||||
"resource_type": "google_cloud_run_v2_service",
|
||||
"name": "Validation Worker"
|
||||
},
|
||||
{
|
||||
"component": "C-4",
|
||||
"name": "Validated Analytical Data Dataset",
|
||||
"id": "E-4",
|
||||
"resource_type": "google_bigquery_dataset",
|
||||
"product": "BigQuery"
|
||||
},
|
||||
{
|
||||
"id": "E-5",
|
||||
"resource_type": "google_storage_bucket",
|
||||
"component": "C-5",
|
||||
"name": "Quarantine Storage Bucket",
|
||||
"product": "Cloud Storage"
|
||||
},
|
||||
{
|
||||
"product": "Cloud Logging and Monitoring",
|
||||
"name": "Queue Lag Alert Policy",
|
||||
"resource_type": "google_monitoring_alert_policy",
|
||||
"id": "E-6",
|
||||
"component": "C-6"
|
||||
},
|
||||
{
|
||||
"component": "C-6",
|
||||
"name": "Error Rate Alert Policy",
|
||||
"product": "Cloud Logging and Monitoring",
|
||||
"resource_type": "google_monitoring_alert_policy",
|
||||
"id": "E-7"
|
||||
},
|
||||
{
|
||||
"resource_type": "google_pubsub_subscription",
|
||||
"component": "C-2",
|
||||
"name": "Asynchronous Ingestion Subscription",
|
||||
"id": "E-8",
|
||||
"product": "Cloud Pub/Sub"
|
||||
},
|
||||
{
|
||||
"resource_type": "google_pubsub_topic",
|
||||
"product": "Cloud Pub/Sub",
|
||||
"id": "E-9",
|
||||
"component": "C-2",
|
||||
"name": "Dead-Letter Topic"
|
||||
},
|
||||
{
|
||||
"name": "Ingestion Service Account",
|
||||
"id": "E-10",
|
||||
"resource_type": "google_service_account",
|
||||
"product": "IAM"
|
||||
},
|
||||
{
|
||||
"product": "IAM",
|
||||
"id": "E-11",
|
||||
"name": "Validation Worker Service Account",
|
||||
"resource_type": "google_service_account"
|
||||
},
|
||||
{
|
||||
"product": "IAM",
|
||||
"id": "E-12",
|
||||
"resource_type": "google_service_account",
|
||||
"name": "Pub/Sub Push Service Account"
|
||||
},
|
||||
{
|
||||
"product": "Cloud KMS",
|
||||
"name": "CMEK Crypto Key",
|
||||
"id": "E-13",
|
||||
"resource_type": "google_kms_crypto_key"
|
||||
},
|
||||
{
|
||||
"name": "Load Balancer Backend",
|
||||
"product": "Cloud Load Balancing",
|
||||
"resource_type": "google_compute_backend_service",
|
||||
"id": "E-14"
|
||||
},
|
||||
{
|
||||
"product": "Cloud Armor",
|
||||
"id": "E-15",
|
||||
"name": "Cloud Armor Security Policy",
|
||||
"resource_type": "google_compute_security_policy"
|
||||
},
|
||||
{
|
||||
"id": "E-16",
|
||||
"product": "Cloud Logging and Monitoring",
|
||||
"name": "Audit Log Bucket Config",
|
||||
"resource_type": "google_logging_project_bucket_config"
|
||||
},
|
||||
{
|
||||
"resource_type": "google_monitoring_dashboard",
|
||||
"product": "Cloud Logging and Monitoring",
|
||||
"id": "E-17",
|
||||
"component": "C-6",
|
||||
"name": "Operational Dashboard"
|
||||
},
|
||||
{
|
||||
"id": "E-18",
|
||||
"component": "C-1",
|
||||
"resource_type": "google_compute_ssl_policy",
|
||||
"product": "Cloud Load Balancing",
|
||||
"name": "Load Balancer SSL Policy"
|
||||
},
|
||||
{
|
||||
"resource_type": "google_compute_target_https_proxy",
|
||||
"product": "Cloud Load Balancing",
|
||||
"name": "Load Balancer Target HTTPS Proxy",
|
||||
"component": "C-1",
|
||||
"id": "E-19"
|
||||
},
|
||||
{
|
||||
"name": "Load Balancer URL Map",
|
||||
"resource_type": "google_compute_url_map",
|
||||
"id": "E-20",
|
||||
"product": "Cloud Load Balancing",
|
||||
"component": "C-1"
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"to": "E-2",
|
||||
"id": "R-1",
|
||||
"kind": "publishes to",
|
||||
"from": "E-1"
|
||||
},
|
||||
{
|
||||
"id": "R-2",
|
||||
"kind": "runs as",
|
||||
"from": "E-1",
|
||||
"to": "E-10"
|
||||
},
|
||||
{
|
||||
"to": "E-2",
|
||||
"kind": "is granted access to",
|
||||
"id": "R-3",
|
||||
"from": "E-10"
|
||||
},
|
||||
{
|
||||
"kind": "has subscription",
|
||||
"from": "E-2",
|
||||
"to": "E-8",
|
||||
"id": "R-4"
|
||||
},
|
||||
{
|
||||
"to": "E-3",
|
||||
"id": "R-5",
|
||||
"kind": "pushes to",
|
||||
"from": "E-8"
|
||||
},
|
||||
{
|
||||
"kind": "authenticates as",
|
||||
"from": "E-8",
|
||||
"id": "R-6",
|
||||
"to": "E-12"
|
||||
},
|
||||
{
|
||||
"kind": "is granted access to",
|
||||
"from": "E-12",
|
||||
"to": "E-3",
|
||||
"id": "R-7"
|
||||
},
|
||||
{
|
||||
"id": "R-8",
|
||||
"to": "E-11",
|
||||
"from": "E-3",
|
||||
"kind": "runs as"
|
||||
},
|
||||
{
|
||||
"to": "E-4",
|
||||
"id": "R-9",
|
||||
"from": "E-3",
|
||||
"kind": "writes to"
|
||||
},
|
||||
{
|
||||
"kind": "writes to",
|
||||
"from": "E-3",
|
||||
"to": "E-5",
|
||||
"id": "R-10"
|
||||
},
|
||||
{
|
||||
"id": "R-11",
|
||||
"kind": "is granted access to",
|
||||
"to": "E-4",
|
||||
"from": "E-11"
|
||||
},
|
||||
{
|
||||
"kind": "is granted access to",
|
||||
"from": "E-11",
|
||||
"to": "E-5",
|
||||
"id": "R-12"
|
||||
},
|
||||
{
|
||||
"kind": "routes dead letters to",
|
||||
"from": "E-8",
|
||||
"id": "R-13",
|
||||
"to": "E-9"
|
||||
},
|
||||
{
|
||||
"to": "E-13",
|
||||
"id": "R-14",
|
||||
"kind": "is encrypted with",
|
||||
"from": "E-2"
|
||||
},
|
||||
{
|
||||
"kind": "is encrypted with",
|
||||
"from": "E-5",
|
||||
"to": "E-13",
|
||||
"id": "R-15"
|
||||
},
|
||||
{
|
||||
"id": "R-16",
|
||||
"from": "E-14",
|
||||
"kind": "fronts",
|
||||
"to": "E-1"
|
||||
},
|
||||
{
|
||||
"kind": "uses policy",
|
||||
"from": "E-14",
|
||||
"to": "E-15",
|
||||
"id": "R-17"
|
||||
},
|
||||
{
|
||||
"from": "E-6",
|
||||
"kind": "monitors",
|
||||
"to": "E-8",
|
||||
"id": "R-18"
|
||||
},
|
||||
{
|
||||
"to": "E-1",
|
||||
"kind": "monitors",
|
||||
"from": "E-7",
|
||||
"id": "R-19"
|
||||
},
|
||||
{
|
||||
"id": "R-20",
|
||||
"from": "E-7",
|
||||
"kind": "monitors",
|
||||
"to": "E-3"
|
||||
},
|
||||
{
|
||||
"id": "R-21",
|
||||
"kind": "routes to",
|
||||
"from": "E-20",
|
||||
"to": "E-14"
|
||||
},
|
||||
{
|
||||
"to": "E-20",
|
||||
"from": "E-19",
|
||||
"kind": "uses map",
|
||||
"id": "R-22"
|
||||
},
|
||||
{
|
||||
"to": "E-18",
|
||||
"kind": "uses policy",
|
||||
"from": "E-19",
|
||||
"id": "R-23"
|
||||
}
|
||||
],
|
||||
"checks": [
|
||||
{
|
||||
"applies_to": "E-1",
|
||||
"source": "DR-1",
|
||||
"statement": "ingress is INGRESS_TRAFFIC_INTERNAL_LOAD_BALANCER",
|
||||
"id": "K-1"
|
||||
},
|
||||
{
|
||||
"id": "K-2",
|
||||
"applies_to": "E-1",
|
||||
"statement": "template.service_account points to E-10",
|
||||
"source": "DR-2"
|
||||
},
|
||||
{
|
||||
"source": "DR-12",
|
||||
"applies_to": "E-1",
|
||||
"statement": "template.containers.resources.startup_cpu_boost is true",
|
||||
"id": "K-3"
|
||||
},
|
||||
{
|
||||
"source": "DR-13",
|
||||
"applies_to": "E-1",
|
||||
"statement": "template.max_instance_request_concurrency is at least 80",
|
||||
"id": "K-4"
|
||||
},
|
||||
{
|
||||
"id": "K-5",
|
||||
"applies_to": "E-1",
|
||||
"statement": "location is a Low-CO2 region",
|
||||
"source": "DR-14"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-2",
|
||||
"statement": "kms_key_name points to E-13",
|
||||
"source": "DR-5",
|
||||
"id": "K-6"
|
||||
},
|
||||
{
|
||||
"source": "DR-3",
|
||||
"applies_to": "E-3",
|
||||
"id": "K-7",
|
||||
"statement": "template.service_account points to E-11"
|
||||
},
|
||||
{
|
||||
"id": "K-8",
|
||||
"statement": "template.containers.resources.startup_cpu_boost is true",
|
||||
"applies_to": "E-3",
|
||||
"source": "DR-12"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-3",
|
||||
"id": "K-9",
|
||||
"statement": "template.max_instance_request_concurrency is at least 80",
|
||||
"source": "DR-13"
|
||||
},
|
||||
{
|
||||
"id": "K-10",
|
||||
"statement": "location is a Low-CO2 region",
|
||||
"applies_to": "E-3",
|
||||
"source": "DR-14"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-4",
|
||||
"id": "K-11",
|
||||
"statement": "location is set to a multi-region",
|
||||
"source": "DR-9"
|
||||
},
|
||||
{
|
||||
"id": "K-12",
|
||||
"statement": "encryption.default_kms_key_name points to E-13",
|
||||
"applies_to": "E-5",
|
||||
"source": "DR-5"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-5",
|
||||
"id": "K-13",
|
||||
"statement": "uniform_bucket_level_access is true",
|
||||
"source": "DR-6"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-5",
|
||||
"statement": "public_access_prevention is enforced",
|
||||
"source": "DR-6",
|
||||
"id": "K-14"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-5",
|
||||
"id": "K-15",
|
||||
"statement": "location is set to a multi-region",
|
||||
"source": "DR-9"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-5",
|
||||
"source": "DR-11",
|
||||
"id": "K-16",
|
||||
"statement": "lifecycle_rule has action.type = Delete and condition.age = 30"
|
||||
},
|
||||
{
|
||||
"source": "DR-10",
|
||||
"id": "K-17",
|
||||
"applies_to": "E-6",
|
||||
"statement": "conditions block filters on oldest_unacked_message_age"
|
||||
},
|
||||
{
|
||||
"id": "K-19",
|
||||
"source": "DR-4",
|
||||
"statement": "push_config.oidc_token.service_account_email points to E-12",
|
||||
"applies_to": "E-8"
|
||||
},
|
||||
{
|
||||
"id": "K-20",
|
||||
"statement": "dead_letter_policy points to E-9",
|
||||
"applies_to": "E-8",
|
||||
"source": "DR-7"
|
||||
},
|
||||
{
|
||||
"id": "K-21",
|
||||
"source": "DR-7",
|
||||
"applies_to": "E-8",
|
||||
"statement": "dead_letter_policy.max_delivery_attempts is 5"
|
||||
},
|
||||
{
|
||||
"statement": "retry_policy block is configured with minimum_backoff and maximum_backoff",
|
||||
"id": "K-22",
|
||||
"applies_to": "E-8",
|
||||
"source": "DR-8"
|
||||
},
|
||||
{
|
||||
"statement": "retention_days is at least 365",
|
||||
"applies_to": "E-16",
|
||||
"id": "K-23",
|
||||
"source": "NFR-4"
|
||||
},
|
||||
{
|
||||
"source": "NFR-4",
|
||||
"statement": "locked is true",
|
||||
"applies_to": "E-16",
|
||||
"id": "K-24"
|
||||
},
|
||||
{
|
||||
"id": "K-25",
|
||||
"source": "NFR-10",
|
||||
"applies_to": "E-17",
|
||||
"statement": "dashboard_json contains widgets for queue depth, processing latency, and validation failure rate"
|
||||
},
|
||||
{
|
||||
"statement": "role is roles/pubsub.publisher",
|
||||
"id": "K-26",
|
||||
"source": "NFR-2",
|
||||
"applies_to": "R-3"
|
||||
},
|
||||
{
|
||||
"id": "K-27",
|
||||
"statement": "role is roles/run.invoker",
|
||||
"source": "DR-4",
|
||||
"applies_to": "R-7"
|
||||
},
|
||||
{
|
||||
"source": "DR-3",
|
||||
"id": "K-28",
|
||||
"statement": "role is roles/bigquery.dataEditor",
|
||||
"applies_to": "R-11"
|
||||
},
|
||||
{
|
||||
"id": "K-29",
|
||||
"source": "NFR-2",
|
||||
"statement": "role is roles/storage.objectCreator",
|
||||
"applies_to": "R-12"
|
||||
},
|
||||
{
|
||||
"id": "K-30",
|
||||
"source": "NFR-1",
|
||||
"statement": "default_encryption_configuration is configured",
|
||||
"applies_to": "E-4"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-1",
|
||||
"statement": "template.scaling.min_instance_count is 0 or not set",
|
||||
"source": "NFR-8",
|
||||
"id": "K-31"
|
||||
},
|
||||
{
|
||||
"source": "NFR-12",
|
||||
"id": "K-32",
|
||||
"statement": "template.scaling.max_instance_count is at least 100",
|
||||
"applies_to": "E-1"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-9",
|
||||
"id": "K-33",
|
||||
"source": "DR-7",
|
||||
"statement": "message_retention_duration is configured"
|
||||
},
|
||||
{
|
||||
"statement": "account_id is configured",
|
||||
"source": "DR-2",
|
||||
"applies_to": "E-10",
|
||||
"id": "K-34"
|
||||
},
|
||||
{
|
||||
"statement": "account_id is configured",
|
||||
"id": "K-35",
|
||||
"applies_to": "E-11",
|
||||
"source": "DR-3"
|
||||
},
|
||||
{
|
||||
"id": "K-36",
|
||||
"statement": "account_id is configured",
|
||||
"applies_to": "E-12",
|
||||
"source": "DR-4"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-13",
|
||||
"statement": "rotation_period is configured",
|
||||
"id": "K-37",
|
||||
"source": "DR-5"
|
||||
},
|
||||
{
|
||||
"source": "DR-1",
|
||||
"id": "K-38",
|
||||
"applies_to": "E-14",
|
||||
"statement": "security_policy points to E-15"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-15",
|
||||
"statement": "type is CLOUD_ARMOR",
|
||||
"id": "K-39",
|
||||
"source": "DR-1"
|
||||
},
|
||||
{
|
||||
"source": "NFR-3",
|
||||
"id": "K-40",
|
||||
"applies_to": "E-5",
|
||||
"statement": "public_access_prevention is enforced"
|
||||
},
|
||||
{
|
||||
"statement": "location is configured",
|
||||
"source": "NFR-5",
|
||||
"id": "K-41",
|
||||
"applies_to": "E-1"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-2",
|
||||
"statement": "message_retention_duration is configured",
|
||||
"source": "NFR-6",
|
||||
"id": "K-42"
|
||||
},
|
||||
{
|
||||
"applies_to": "E-4",
|
||||
"statement": "location is set to a multi-region",
|
||||
"id": "K-43",
|
||||
"source": "NFR-7"
|
||||
},
|
||||
{
|
||||
"statement": "lifecycle_rule has action.type = Delete and condition.age = 30",
|
||||
"source": "NFR-9",
|
||||
"id": "K-44",
|
||||
"applies_to": "E-5"
|
||||
},
|
||||
{
|
||||
"source": "NFR-11",
|
||||
"applies_to": "E-6",
|
||||
"id": "K-45",
|
||||
"statement": "conditions block filters on oldest_unacked_message_age"
|
||||
},
|
||||
{
|
||||
"statement": "template.containers.resources.startup_cpu_boost is true",
|
||||
"source": "NFR-13",
|
||||
"applies_to": "E-1",
|
||||
"id": "K-46"
|
||||
},
|
||||
{
|
||||
"source": "NFR-14",
|
||||
"applies_to": "E-8",
|
||||
"statement": "push_config is configured",
|
||||
"id": "K-47"
|
||||
},
|
||||
{
|
||||
"source": "NFR-8",
|
||||
"id": "K-50",
|
||||
"statement": "template.scaling.min_instance_count is 0",
|
||||
"applies_to": "E-3"
|
||||
},
|
||||
{
|
||||
"statement": "template.scaling.max_instance_count is configured to a baseline max",
|
||||
"applies_to": "E-3",
|
||||
"source": "NFR-8",
|
||||
"id": "K-51"
|
||||
},
|
||||
{
|
||||
"statement": "condition filters on validation worker Cloud Run 5xx errors",
|
||||
"source": "DR-10",
|
||||
"applies_to": "R-20",
|
||||
"id": "K-52"
|
||||
},
|
||||
{
|
||||
"source": "NFR-1",
|
||||
"statement": "min_tls_version is TLS_1_2",
|
||||
"id": "K-53",
|
||||
"applies_to": "E-18"
|
||||
},
|
||||
{
|
||||
"id": "K-54",
|
||||
"source": "NFR-1",
|
||||
"applies_to": "E-19",
|
||||
"statement": "ssl_policy points to E-18"
|
||||
},
|
||||
{
|
||||
"source": "NFR-1",
|
||||
"statement": "url_map points to E-20",
|
||||
"applies_to": "E-19",
|
||||
"id": "K-55"
|
||||
},
|
||||
{
|
||||
"id": "K-56",
|
||||
"source": "NFR-1",
|
||||
"applies_to": "E-20",
|
||||
"statement": "default_service points to E-14"
|
||||
},
|
||||
{
|
||||
"statement": "enabled is true",
|
||||
"applies_to": "E-7",
|
||||
"id": "K-57",
|
||||
"source": "NFR-11"
|
||||
}
|
||||
],
|
||||
"not_applicable": []
|
||||
}
|
||||
Reference in New Issue
Block a user