Files
gcp_solution_architecture_a…/app/tools/coverage.py
Jonathan Boniface 5adc911a82 feat: keep an unchanged description, list conditions, review coverage with Jev
After case 02 (completed at 100%, but with two Revisions of the Target
architecture for gaps validation found one pass at a time):

- A Deliverable stands while what it was written from is unchanged. The
  architecture description is written from the Elements and Relationships,
  so a Revision that only adds or rewords Checks no longer runs the describer.
- Non-functional requirements and recommendations list their conditions,
  one entry per separate thing that must be true; submission requires them.
- A coverage review at submission (JEV_COVERAGE_REVIEW, off by default):
  for each condition, Jev chooses which Check verifies it, or none. Asked as
  one broad yes-or-no per requirement it could not tell a complete design
  from one with known gaps; asked as a choice per condition it raised all
  four known gaps of case 02 and nothing on the complete graph. An opt-in
  live test (LIVE_JEV=1) repeats that.
- The validator is told to report every gap in its first pass.

Plans record the case 02 run and both forms of the review with figures.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-07 20:58:38 +01:00

171 lines
6.1 KiB
Python

"""A review of the Target architecture's coverage, before it is submitted.
The structural check knows that every non-functional requirement and every design
recommendation is the source of a Check. It cannot know whether those Checks verify
everything the text asks for: a requirement with three conditions and a Check for one
passes it. The validator finds the rest later, a few at a time, and each finding
costs a Revision of the design, the code and the validation.
Requirements and recommendations list their conditions one by one. For each
condition this asks the decider (ADR-0007) which of the source's Checks verifies it,
with "none" as an answer. A condition answered "none" is raised.
Two things make the question answerable, found by measurement on 2026-10-07 (see
.agents/plans/iter-05-jev-decisions/steps/step-4.md). It is about one explicit
condition, not a whole requirement; and it is a choice among the Checks, not a yes or
no. Asked the broad way, a complete design and an incomplete one scored alike.
It never blocks: with no decision, the submission stands on its structure.
"""
from __future__ import annotations
import logging
from collections.abc import Mapping, Sequence
from typing import Any
from app.adk.runtime import Decider, Question
from app.workflows.chain import (
RECOMMENDATIONS_FILE,
REQUIREMENTS_FILE,
Upstream,
conditions_of,
read_list,
)
from app.workflows.deliverables import DeliverableKind
from app.workflows.target_architecture import (
GRAPH_ATTACHMENT,
Check,
GraphError,
TargetArchitecture,
parse_graph,
)
logger = logging.getLogger(__name__)
NONE = "none"
def _described(graph: TargetArchitecture) -> str:
lines = ["Elements of the Target architecture:"]
lines += [
f"- {e.id} {e.name} ({e.product}, {e.resource_type})"
+ (f", realises component {e.component}" if e.component else "")
for e in graph.elements
]
lines.append("Relationships:")
lines += [f"- {r.id}: {r.source} {r.kind} {r.target}" for r in graph.relationships]
return "\n".join(lines)
def _constraints(upstream: Upstream, graph: TargetArchitecture) -> list[dict[str, Any]]:
"""What must be covered: every non-functional requirement, and every design
recommendation that applies."""
excused = {item.recommendation for item in graph.not_applicable}
return [
item
for item in read_list(upstream, "requirements", REQUIREMENTS_FILE)
if item.get("kind") == "non_functional"
] + [
item
for item in read_list(upstream, "design_recommendations", RECOMMENDATIONS_FILE)
if item.get("id") not in excused
]
def coverage_questions(
graph: TargetArchitecture, constraints: Sequence[Mapping[str, Any]]
) -> dict[str, tuple[str, str, Question]]:
"""One question per condition: (source id, the condition, the question).
A source with no Check is the structural check's to report, so it is not asked
about here.
"""
names = {element.id: element.name for element in graph.elements}
names.update(
{
r.id: f"{names[r.source]} {r.kind} {names[r.target]}"
for r in graph.relationships
}
)
by_source: dict[str, list[Check]] = {}
for check in graph.checks:
by_source.setdefault(check.source, []).append(check)
questions: dict[str, tuple[str, str, Question]] = {}
for item in constraints:
source = item.get("id")
checks = by_source.get(source, []) if isinstance(source, str) else []
if not checks:
continue
listed = "\n".join(
f"- {check.id} on {names[check.applies_to]}: {check.statement}"
for check in checks
)
for number, condition in enumerate(conditions_of(item), start=1):
questions[f"{source}#{number}"] = (
str(source),
condition,
Question(
"choice",
f'Condition: "{condition}".\nChecks:\n{listed}\n'
"Which one Check verifies this condition? Answer none when "
"no Check does.",
(*(check.id for check in checks), NONE),
),
)
return questions
class CoverageReview:
def __init__(self, decider: Decider, *, none_at: float = 0.6) -> None:
self._decider = decider
# How likely "none" must be before a condition is raised. Below it the
# answer is too uncertain to send work back on.
self._none_at = none_at
async def __call__(
self,
kind: DeliverableKind,
attachments: Mapping[str, str],
upstream: Upstream | None,
scope: str = "",
) -> Sequence[str]:
"""The conditions no Check appears to verify, each as a line for the
Specialist. Empty when coverage looks complete or was not judged."""
if kind is not DeliverableKind.TARGET_ARCHITECTURE or not upstream:
return ()
try:
graph = parse_graph(attachments.get(GRAPH_ATTACHMENT, ""))
except GraphError:
return ()
asked = coverage_questions(graph, _constraints(upstream, graph))
if not asked:
return ()
answers = await self._decider.decide(
_described(graph),
{key: question for key, (_, _, question) in asked.items()},
scope=scope,
)
if not answers:
return ()
concerns = []
for key, (source, condition, question) in asked.items():
answer = answers.get(key)
if answer is None or answer.value != NONE:
continue
if answer.probabilities.get(NONE, answer.confidence) < self._none_at:
continue
checks = ", ".join(question.options[:-1])
concerns.append(
f'{source}: no Check verifies "{condition}" (its Checks: {checks})'
)
if concerns:
logger.info(
"[%s] coverage review raised %d of %d conditions",
scope,
len(concerns),
len(asked),
)
return tuple(concerns)