After case 02 (completed at 100%, but with two Revisions of the Target architecture for gaps validation found one pass at a time): - A Deliverable stands while what it was written from is unchanged. The architecture description is written from the Elements and Relationships, so a Revision that only adds or rewords Checks no longer runs the describer. - Non-functional requirements and recommendations list their conditions, one entry per separate thing that must be true; submission requires them. - A coverage review at submission (JEV_COVERAGE_REVIEW, off by default): for each condition, Jev chooses which Check verifies it, or none. Asked as one broad yes-or-no per requirement it could not tell a complete design from one with known gaps; asked as a choice per condition it raised all four known gaps of case 02 and nothing on the complete graph. An opt-in live test (LIVE_JEV=1) repeats that. - The validator is told to report every gap in its first pass. Plans record the case 02 run and both forms of the review with figures. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
171 lines
6.1 KiB
Python
171 lines
6.1 KiB
Python
"""A review of the Target architecture's coverage, before it is submitted.
|
|
|
|
The structural check knows that every non-functional requirement and every design
|
|
recommendation is the source of a Check. It cannot know whether those Checks verify
|
|
everything the text asks for: a requirement with three conditions and a Check for one
|
|
passes it. The validator finds the rest later, a few at a time, and each finding
|
|
costs a Revision of the design, the code and the validation.
|
|
|
|
Requirements and recommendations list their conditions one by one. For each
|
|
condition this asks the decider (ADR-0007) which of the source's Checks verifies it,
|
|
with "none" as an answer. A condition answered "none" is raised.
|
|
|
|
Two things make the question answerable, found by measurement on 2026-10-07 (see
|
|
.agents/plans/iter-05-jev-decisions/steps/step-4.md). It is about one explicit
|
|
condition, not a whole requirement; and it is a choice among the Checks, not a yes or
|
|
no. Asked the broad way, a complete design and an incomplete one scored alike.
|
|
|
|
It never blocks: with no decision, the submission stands on its structure.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
from collections.abc import Mapping, Sequence
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import Decider, Question
|
|
from app.workflows.chain import (
|
|
RECOMMENDATIONS_FILE,
|
|
REQUIREMENTS_FILE,
|
|
Upstream,
|
|
conditions_of,
|
|
read_list,
|
|
)
|
|
from app.workflows.deliverables import DeliverableKind
|
|
from app.workflows.target_architecture import (
|
|
GRAPH_ATTACHMENT,
|
|
Check,
|
|
GraphError,
|
|
TargetArchitecture,
|
|
parse_graph,
|
|
)
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
NONE = "none"
|
|
|
|
|
|
def _described(graph: TargetArchitecture) -> str:
|
|
lines = ["Elements of the Target architecture:"]
|
|
lines += [
|
|
f"- {e.id} {e.name} ({e.product}, {e.resource_type})"
|
|
+ (f", realises component {e.component}" if e.component else "")
|
|
for e in graph.elements
|
|
]
|
|
lines.append("Relationships:")
|
|
lines += [f"- {r.id}: {r.source} {r.kind} {r.target}" for r in graph.relationships]
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _constraints(upstream: Upstream, graph: TargetArchitecture) -> list[dict[str, Any]]:
|
|
"""What must be covered: every non-functional requirement, and every design
|
|
recommendation that applies."""
|
|
excused = {item.recommendation for item in graph.not_applicable}
|
|
return [
|
|
item
|
|
for item in read_list(upstream, "requirements", REQUIREMENTS_FILE)
|
|
if item.get("kind") == "non_functional"
|
|
] + [
|
|
item
|
|
for item in read_list(upstream, "design_recommendations", RECOMMENDATIONS_FILE)
|
|
if item.get("id") not in excused
|
|
]
|
|
|
|
|
|
def coverage_questions(
|
|
graph: TargetArchitecture, constraints: Sequence[Mapping[str, Any]]
|
|
) -> dict[str, tuple[str, str, Question]]:
|
|
"""One question per condition: (source id, the condition, the question).
|
|
|
|
A source with no Check is the structural check's to report, so it is not asked
|
|
about here.
|
|
"""
|
|
names = {element.id: element.name for element in graph.elements}
|
|
names.update(
|
|
{
|
|
r.id: f"{names[r.source]} {r.kind} {names[r.target]}"
|
|
for r in graph.relationships
|
|
}
|
|
)
|
|
by_source: dict[str, list[Check]] = {}
|
|
for check in graph.checks:
|
|
by_source.setdefault(check.source, []).append(check)
|
|
|
|
questions: dict[str, tuple[str, str, Question]] = {}
|
|
for item in constraints:
|
|
source = item.get("id")
|
|
checks = by_source.get(source, []) if isinstance(source, str) else []
|
|
if not checks:
|
|
continue
|
|
listed = "\n".join(
|
|
f"- {check.id} on {names[check.applies_to]}: {check.statement}"
|
|
for check in checks
|
|
)
|
|
for number, condition in enumerate(conditions_of(item), start=1):
|
|
questions[f"{source}#{number}"] = (
|
|
str(source),
|
|
condition,
|
|
Question(
|
|
"choice",
|
|
f'Condition: "{condition}".\nChecks:\n{listed}\n'
|
|
"Which one Check verifies this condition? Answer none when "
|
|
"no Check does.",
|
|
(*(check.id for check in checks), NONE),
|
|
),
|
|
)
|
|
return questions
|
|
|
|
|
|
class CoverageReview:
|
|
def __init__(self, decider: Decider, *, none_at: float = 0.6) -> None:
|
|
self._decider = decider
|
|
# How likely "none" must be before a condition is raised. Below it the
|
|
# answer is too uncertain to send work back on.
|
|
self._none_at = none_at
|
|
|
|
async def __call__(
|
|
self,
|
|
kind: DeliverableKind,
|
|
attachments: Mapping[str, str],
|
|
upstream: Upstream | None,
|
|
scope: str = "",
|
|
) -> Sequence[str]:
|
|
"""The conditions no Check appears to verify, each as a line for the
|
|
Specialist. Empty when coverage looks complete or was not judged."""
|
|
if kind is not DeliverableKind.TARGET_ARCHITECTURE or not upstream:
|
|
return ()
|
|
try:
|
|
graph = parse_graph(attachments.get(GRAPH_ATTACHMENT, ""))
|
|
except GraphError:
|
|
return ()
|
|
asked = coverage_questions(graph, _constraints(upstream, graph))
|
|
if not asked:
|
|
return ()
|
|
answers = await self._decider.decide(
|
|
_described(graph),
|
|
{key: question for key, (_, _, question) in asked.items()},
|
|
scope=scope,
|
|
)
|
|
if not answers:
|
|
return ()
|
|
concerns = []
|
|
for key, (source, condition, question) in asked.items():
|
|
answer = answers.get(key)
|
|
if answer is None or answer.value != NONE:
|
|
continue
|
|
if answer.probabilities.get(NONE, answer.confidence) < self._none_at:
|
|
continue
|
|
checks = ", ".join(question.options[:-1])
|
|
concerns.append(
|
|
f'{source}: no Check verifies "{condition}" (its Checks: {checks})'
|
|
)
|
|
if concerns:
|
|
logger.info(
|
|
"[%s] coverage review raised %d of %d conditions",
|
|
scope,
|
|
len(concerns),
|
|
len(asked),
|
|
)
|
|
return tuple(concerns)
|