feat: keep an unchanged description, list conditions, review coverage with Jev

After case 02 (completed at 100%, but with two Revisions of the Target
architecture for gaps validation found one pass at a time):

- A Deliverable stands while what it was written from is unchanged. The
  architecture description is written from the Elements and Relationships,
  so a Revision that only adds or rewords Checks no longer runs the describer.
- Non-functional requirements and recommendations list their conditions,
  one entry per separate thing that must be true; submission requires them.
- A coverage review at submission (JEV_COVERAGE_REVIEW, off by default):
  for each condition, Jev chooses which Check verifies it, or none. Asked as
  one broad yes-or-no per requirement it could not tell a complete design
  from one with known gaps; asked as a choice per condition it raised all
  four known gaps of case 02 and nothing on the complete graph. An opt-in
  live test (LIVE_JEV=1) repeats that.
- The validator is told to report every gap in its first pass.

Plans record the case 02 run and both forms of the review with figures.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-07 20:58:38 +01:00
parent f1f6f1b83a
commit 5adc911a82
29 changed files with 1687 additions and 16 deletions

View File

@@ -40,7 +40,12 @@ SAMPLE_DATA: dict[str, str] = {
"requirements.json": json.dumps(
[
{"id": "FR-1", "kind": "functional", "text": "Accept events."},
{"id": "NFR-1", "kind": "non_functional", "text": "99.9% availability."},
{
"id": "NFR-1",
"kind": "non_functional",
"text": "99.9% availability.",
"conditions": ["The service stays available when one instance fails."],
},
]
),
"components.json": json.dumps(
@@ -54,6 +59,7 @@ SAMPLE_DATA: dict[str, str] = {
"pillar": "security",
"text": "Use a dedicated identity.",
"components": ["C-1"],
"conditions": ["The service runs as an identity nothing else uses."],
}
]
),

View File

@@ -255,6 +255,23 @@ class NoDecider:
return None
class SubmissionReview(Protocol):
"""A review of a submission's meaning, beyond what its structure shows.
Returns what it would have the Specialist look at again, one line each; empty
when it has nothing to raise or could not decide. `kind` is the Deliverable's
kind, `upstream` the Deliverables it builds on.
"""
async def __call__(
self,
kind: Any,
attachments: Mapping[str, str],
upstream: Mapping[str, Mapping[str, Any]] | None,
scope: str = "",
) -> Sequence[str]: ...
class AgentRuntime(Protocol):
async def run(
self, role: RoleSpec, task: AgentTask, tools: Sequence[ToolSpec]

View File

@@ -29,6 +29,7 @@ from app.adk.runtime import (
NoDecider,
RoleSpec,
RunLimiter,
SubmissionReview,
ToolFactory,
)
from app.checkpointing import open_checkpointer
@@ -46,6 +47,7 @@ from app.recorder import DatabaseRecorder, ExecutionRecorder
from app.skills.agent_skills import skill_version
from app.states.execution_state import check_customer_case
from app.states.ledger import Ledger
from app.tools.coverage import CoverageReview
from app.tools.execution import ORCHESTRATOR
from app.tools.jev import JevDecider, JevLimits
from app.tools.local import TerraformRunner, local_tools
@@ -84,6 +86,7 @@ class ExecutionEngine:
limiter: RunLimiter | None = None,
skill_versions: Mapping[str, str] | None = None,
orchestrator: OrchestratorPolicy | None = None,
submission_review: SubmissionReview | None = None,
) -> None:
required = {
ORCHESTRATOR,
@@ -104,6 +107,7 @@ class ExecutionEngine:
limiter=limiter,
skill_versions=skill_versions,
orchestrator=orchestrator,
submission_review=submission_review,
).compile(checkpointer=checkpointer)
async def start(
@@ -334,6 +338,13 @@ async def open_assembly(
name: skill_version(meta.path) for name, meta in bundle.skills.items()
},
orchestrator=orchestrator,
# Optional, and off by default: Jev reviews a design's coverage before
# it is submitted. Without it a submission stands on its structure.
submission_review=CoverageReview(
decider, none_at=settings.JEV_COVERAGE_THRESHOLD
)
if settings.JEV_COVERAGE_REVIEW and isinstance(decider, JevDecider)
else None,
)
try:
yield Assembly(engine, bundle, roles, dict(catalog.gaps), decider)

View File

@@ -82,6 +82,11 @@ class Settings(BaseSettings):
JEV_ACT_THRESHOLD: float = 0.75
JEV_CLOSE_THRESHOLD: float = 0.15
JEV_CONFIDENCE_THRESHOLD: float = 0.6
# Review a design's coverage at submission: for each condition a requirement
# lists, which Check verifies it. Off until a full case has run with it.
JEV_COVERAGE_REVIEW: bool = False
# How likely "no Check verifies this" must be before the condition is raised.
JEV_COVERAGE_THRESHOLD: float = 0.6
# Paths
BASE_DIR: Path = Path(__file__).resolve().parent.parent

View File

@@ -28,6 +28,7 @@ from app.adk.runtime import (
RoleSpec,
RunLimiter,
Stopped,
SubmissionReview,
Submitted,
ToolFactory,
ToolSpec,
@@ -61,6 +62,9 @@ from app.workflows.deliverables import (
logger = logging.getLogger(__name__)
# Working-memory key: the last submission of a Deliverable with what it was written from.
STANDING = "standing"
NO_USER_REPLY = (
"No user is available. State your assumption, record it in the Deliverable under "
"'Assumptions', and proceed."
@@ -98,6 +102,7 @@ def make_execution_nodes(
limiter: RunLimiter | None = None,
skill_versions: Mapping[str, str] | None = None,
orchestrator: OrchestratorPolicy | None = None,
submission_review: SubmissionReview | None = None,
) -> ExecutionNodes:
tool_factory = tool_factory or _no_tools
# How an Orchestrator turn is decided. The default drives the tools from the
@@ -357,6 +362,48 @@ def make_execution_nodes(
working[workspace.FILES] = dict(
upstream.get(source.value, {}).get("attachments", {})
)
spec = DEPENDENCY_GRAPH[kind]
# Feedback on this Deliverable itself. A note that a dependency was revised
# is not a reason to write it again; the basis says whether that matters.
feedback = [
item
for item in task_input["context"].get("feedback") or []
if not str(item).startswith("Dependency ")
]
basis = spec.written_from(upstream) if spec.written_from else None
standing = working.get(STANDING)
if (
basis
and isinstance(standing, Mapping)
and standing.get("basis") == basis
and standing.get("feedback") == feedback
):
# What it was written from has not changed, so the last submission
# still stands: no Specialist is run.
logger.info(
"[%s] %s kept %s (attempt %d): written from an unchanged basis",
task_input["execution_id"],
role.name,
kind,
task_input["attempt"],
)
kept = Submitted(standing["content"], dict(standing["attachments"]))
return {
"outcomes": [
{
**_outcome_record(role.name, kind, task_input["attempt"], kept),
"refusals": [],
}
],
"working": {kind.value: working},
}
async def review(attachments: Mapping[str, str]) -> Sequence[str]:
assert submission_review is not None
return await submission_review(
kind, attachments, upstream, task_input["execution_id"]
)
refusals: list[dict[str, Any]] = []
tools = _noting_refusals(
specialist_tools(
@@ -366,6 +413,7 @@ def make_execution_nodes(
upstream=upstream,
working=working,
library=task_input.get("library"),
review=review if submission_review else None,
),
refusals,
role.name,
@@ -402,6 +450,13 @@ def make_execution_nodes(
time.monotonic() - started,
_summarise_outcome(outcome),
)
if basis and isinstance(outcome, Submitted):
working[STANDING] = {
"basis": basis,
"feedback": feedback,
"content": outcome.content,
"attachments": dict(outcome.attachments),
}
return {
"outcomes": [
{

170
app/tools/coverage.py Normal file
View File

@@ -0,0 +1,170 @@
"""A review of the Target architecture's coverage, before it is submitted.
The structural check knows that every non-functional requirement and every design
recommendation is the source of a Check. It cannot know whether those Checks verify
everything the text asks for: a requirement with three conditions and a Check for one
passes it. The validator finds the rest later, a few at a time, and each finding
costs a Revision of the design, the code and the validation.
Requirements and recommendations list their conditions one by one. For each
condition this asks the decider (ADR-0007) which of the source's Checks verifies it,
with "none" as an answer. A condition answered "none" is raised.
Two things make the question answerable, found by measurement on 2026-10-07 (see
.agents/plans/iter-05-jev-decisions/steps/step-4.md). It is about one explicit
condition, not a whole requirement; and it is a choice among the Checks, not a yes or
no. Asked the broad way, a complete design and an incomplete one scored alike.
It never blocks: with no decision, the submission stands on its structure.
"""
from __future__ import annotations
import logging
from collections.abc import Mapping, Sequence
from typing import Any
from app.adk.runtime import Decider, Question
from app.workflows.chain import (
RECOMMENDATIONS_FILE,
REQUIREMENTS_FILE,
Upstream,
conditions_of,
read_list,
)
from app.workflows.deliverables import DeliverableKind
from app.workflows.target_architecture import (
GRAPH_ATTACHMENT,
Check,
GraphError,
TargetArchitecture,
parse_graph,
)
logger = logging.getLogger(__name__)
NONE = "none"
def _described(graph: TargetArchitecture) -> str:
lines = ["Elements of the Target architecture:"]
lines += [
f"- {e.id} {e.name} ({e.product}, {e.resource_type})"
+ (f", realises component {e.component}" if e.component else "")
for e in graph.elements
]
lines.append("Relationships:")
lines += [f"- {r.id}: {r.source} {r.kind} {r.target}" for r in graph.relationships]
return "\n".join(lines)
def _constraints(upstream: Upstream, graph: TargetArchitecture) -> list[dict[str, Any]]:
"""What must be covered: every non-functional requirement, and every design
recommendation that applies."""
excused = {item.recommendation for item in graph.not_applicable}
return [
item
for item in read_list(upstream, "requirements", REQUIREMENTS_FILE)
if item.get("kind") == "non_functional"
] + [
item
for item in read_list(upstream, "design_recommendations", RECOMMENDATIONS_FILE)
if item.get("id") not in excused
]
def coverage_questions(
graph: TargetArchitecture, constraints: Sequence[Mapping[str, Any]]
) -> dict[str, tuple[str, str, Question]]:
"""One question per condition: (source id, the condition, the question).
A source with no Check is the structural check's to report, so it is not asked
about here.
"""
names = {element.id: element.name for element in graph.elements}
names.update(
{
r.id: f"{names[r.source]} {r.kind} {names[r.target]}"
for r in graph.relationships
}
)
by_source: dict[str, list[Check]] = {}
for check in graph.checks:
by_source.setdefault(check.source, []).append(check)
questions: dict[str, tuple[str, str, Question]] = {}
for item in constraints:
source = item.get("id")
checks = by_source.get(source, []) if isinstance(source, str) else []
if not checks:
continue
listed = "\n".join(
f"- {check.id} on {names[check.applies_to]}: {check.statement}"
for check in checks
)
for number, condition in enumerate(conditions_of(item), start=1):
questions[f"{source}#{number}"] = (
str(source),
condition,
Question(
"choice",
f'Condition: "{condition}".\nChecks:\n{listed}\n'
"Which one Check verifies this condition? Answer none when "
"no Check does.",
(*(check.id for check in checks), NONE),
),
)
return questions
class CoverageReview:
def __init__(self, decider: Decider, *, none_at: float = 0.6) -> None:
self._decider = decider
# How likely "none" must be before a condition is raised. Below it the
# answer is too uncertain to send work back on.
self._none_at = none_at
async def __call__(
self,
kind: DeliverableKind,
attachments: Mapping[str, str],
upstream: Upstream | None,
scope: str = "",
) -> Sequence[str]:
"""The conditions no Check appears to verify, each as a line for the
Specialist. Empty when coverage looks complete or was not judged."""
if kind is not DeliverableKind.TARGET_ARCHITECTURE or not upstream:
return ()
try:
graph = parse_graph(attachments.get(GRAPH_ATTACHMENT, ""))
except GraphError:
return ()
asked = coverage_questions(graph, _constraints(upstream, graph))
if not asked:
return ()
answers = await self._decider.decide(
_described(graph),
{key: question for key, (_, _, question) in asked.items()},
scope=scope,
)
if not answers:
return ()
concerns = []
for key, (source, condition, question) in asked.items():
answer = answers.get(key)
if answer is None or answer.value != NONE:
continue
if answer.probabilities.get(NONE, answer.confidence) < self._none_at:
continue
checks = ", ".join(question.options[:-1])
concerns.append(
f'{source}: no Check verifies "{condition}" (its Checks: {checks})'
)
if concerns:
logger.info(
"[%s] coverage review raised %d of %d conditions",
scope,
len(concerns),
len(asked),
)
return tuple(concerns)

View File

@@ -7,7 +7,8 @@ outcome.
from __future__ import annotations
from collections.abc import Mapping, MutableMapping, Sequence
import logging
from collections.abc import Awaitable, Callable, Mapping, MutableMapping, Sequence
from typing import Any
from app.adk.runtime import (
@@ -46,6 +47,7 @@ from app.workflows.target_architecture import (
)
ORCHESTRATOR = "orchestrator"
logger = logging.getLogger(__name__)
_TARGET = DeliverableKind.TARGET_ARCHITECTURE
CLOUD_READ = "cloud_read"
@@ -259,6 +261,7 @@ def specialist_tools(
upstream: Mapping[str, Mapping[str, Any]] | None = None,
working: MutableMapping[str, Any] | None = None,
library: Mapping[str, Mapping[str, Any]] | None = None,
review: Callable[[Mapping[str, str]], Awaitable[Sequence[str]]] | None = None,
) -> list[ToolSpec]:
"""A Specialist's tools: submit, ask, its working copy, and its role's tools with
the cloud-read gate.
@@ -267,10 +270,13 @@ def specialist_tools(
along the Requirements chain are checked. `working` is the Deliverable's working
memory, where its working copy is kept. `library` is the earlier Deliverables
given to the Specialist as data files alone, to read whole when it needs one.
`review` looks at the meaning of a structurally complete submission; what it
raises is returned to the Specialist once per run, and never a second time.
"""
working = {} if working is None else working
is_graph = kind is DeliverableKind.TARGET_ARCHITECTURE
own_files = DEPENDENCY_GRAPH[kind].working_copy is kind
reviewed = False
async def submit_deliverable(content: str, attachments: Any = None) -> ToolResult:
try:
@@ -295,11 +301,28 @@ def specialist_tools(
)
if own_files:
working_files(working).update(attachments)
if is_graph:
keep_graph(working, attachments[GRAPH_ATTACHMENT])
nonlocal reviewed
if review and not reviewed:
reviewed = True
try:
concerns = await review(attachments)
except Exception: # a review must never stop a submission
logger.exception("The submission review failed; accepting")
concerns = ()
if concerns:
return ToolResult(
"Not submitted. A review found: "
+ "; ".join(concerns)
+ ". Look at each one: add what is missing, or leave it when "
"it is already covered. This review runs once, so your next "
"submission is accepted."
)
if is_graph:
# The tables and the diagram are views of the graph, so the engine
# writes them: they cannot disagree with it.
graph = parse_graph(attachments[GRAPH_ATTACHMENT])
keep_graph(working, attachments[GRAPH_ATTACHMENT])
content = content.rstrip() + "\n\n" + render_tables(graph)
attachments[DIAGRAM_ATTACHMENT] = render_mermaid(graph)
return ToolResult(f"Submitted {kind}.", Submitted(content, attachments))

View File

@@ -67,6 +67,29 @@ def read_graph(upstream: Upstream | None) -> TargetArchitecture | None:
return None
def graph_structure(upstream: Upstream | None) -> str | None:
"""The Target architecture's Elements and Relationships, as one comparable value.
What a Deliverable written from the shape of the design, and not from its
Checks, was written from. None when there is no readable graph.
"""
graph = read_graph(upstream)
if graph is None:
return None
return json.dumps(
{
"elements": [
[e.id, e.name, e.product, e.resource_type, e.component]
for e in graph.elements
],
"relationships": [
[r.id, r.source, r.target, r.kind] for r in graph.relationships
],
},
sort_keys=True,
)
def _objects(
text: str, name: str, strings: Sequence[str], lists: Sequence[str] = ()
) -> tuple[list[dict[str, Any]], list[str]]:
@@ -113,6 +136,40 @@ def _repeated(
return problems
def conditions_of(item: Mapping[str, Any]) -> tuple[str, ...]:
"""The separate things a requirement or recommendation asks for.
Each is one statement that a Check can verify on its own. An item written before
conditions were listed is read as one condition: its whole text.
"""
listed = item.get("conditions")
if isinstance(listed, list):
conditions = tuple(
c.strip() for c in listed if isinstance(c, str) and c.strip()
)
if conditions:
return conditions
text = item.get("text")
return (text.strip(),) if isinstance(text, str) and text.strip() else ()
def _lacking_conditions(items: Sequence[Mapping[str, Any]], name: str) -> list[str]:
"""A constraint must say, one by one, what would have to be true for it to be met."""
problems = []
for item in items:
listed = item.get("conditions")
if (
not isinstance(listed, list)
or not listed
or not all(isinstance(c, str) and c.strip() for c in listed)
):
problems.append(
f"{name}: {item['id']} needs `conditions`: a list with one entry "
"for each separate thing it asks for"
)
return problems
def check_requirements(
attachments: Mapping[str, str], upstream: Upstream | None
) -> list[str]:
@@ -126,6 +183,9 @@ def check_requirements(
]
if not items and not problems:
problems.append(f"{name}: lists no requirements")
problems += _lacking_conditions(
[item for item in items if item["kind"] == "non_functional"], name
)
return problems
@@ -189,6 +249,7 @@ def check_recommendations(
problems += _repeated(items, name)
if not items and not problems:
problems.append(f"{name}: lists no recommendations")
problems += _lacking_conditions(items, name)
components = read_list(upstream, "technical_decomposition", COMPONENTS_FILE)
if components:
known = {item["id"] for item in components}

View File

@@ -28,6 +28,7 @@ from app.workflows.chain import (
check_requirements,
check_target_architecture,
check_verdicts,
graph_structure,
)
from app.workflows.target_architecture import GRAPH_ATTACHMENT
@@ -72,6 +73,10 @@ class DeliverableSpec:
# files are its working copy, kept across attempts and attached on submission.
# Another kind: a fresh copy of that Deliverable's files on every attempt.
working_copy: DeliverableKind | None = None
# What the Deliverable is written from, as one comparable value. When a Revision
# upstream leaves that value as it was, the last submission still stands and the
# Specialist is not run again.
written_from: Callable[[Upstream | None], str | None] | None = None
def contract(self) -> dict[str, list[str]]:
"""What a submission must contain, as plain data for the Specialist's context."""
@@ -146,6 +151,9 @@ DEPENDENCY_GRAPH: Mapping[DeliverableKind, DeliverableSpec] = {
"describer",
depends_on=(_K.TARGET_ARCHITECTURE,),
sections=("Components", "Flow"),
# It describes the Elements and how they connect. A Revision that only
# adds or rewords Checks leaves it true.
written_from=graph_structure,
),
DeliverableSpec(
_K.DEPLOYMENT_GUIDANCE,

View File

@@ -18,7 +18,13 @@ from typing import Any, cast
from langgraph.graph import END, START, StateGraph
from app.adk.runtime import AgentRuntime, RoleSpec, RunLimiter, ToolFactory
from app.adk.runtime import (
AgentRuntime,
RoleSpec,
RunLimiter,
SubmissionReview,
ToolFactory,
)
from app.nodes.execution_nodes import make_execution_nodes
from app.nodes.router import OrchestratorPolicy
from app.recorder import ExecutionRecorder
@@ -36,6 +42,7 @@ def build_execution_graph(
limiter: RunLimiter | None = None,
skill_versions: Mapping[str, str] | None = None,
orchestrator: OrchestratorPolicy | None = None,
submission_review: SubmissionReview | None = None,
) -> StateGraph:
nodes = make_execution_nodes(
runtime=runtime,
@@ -47,6 +54,7 @@ def build_execution_graph(
limiter=limiter,
skill_versions=skill_versions,
orchestrator=orchestrator,
submission_review=submission_review,
)
graph = StateGraph(ExecutionState)