A reviewer role reads the requirements and recommendations against the Checks when the Target architecture is first submitted. Three runs of case 03: it did not stop the validator sending the design back, and the designs it reviewed ended with 87 and 93 Checks against 52, so they cost more. It stays behind DESIGN_COVERAGE_REVIEW, off (iteration 6 step 13). Also: the schema review no longer reads `true` or `false` in backticks as a setting name. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
152 lines
5.8 KiB
Python
152 lines
5.8 KiB
Python
"""A review of a design's coverage by a model, before anyone approves it.
|
|
|
|
The validator reads the requirements and recommendations against the Checks and
|
|
proposes one for each condition nothing verifies. That reading needs no code, but it
|
|
used to happen only after the code was written: in the runs of 2026-10-09 it sent
|
|
the Target architecture back in four of seven, each time costing a Revision of the
|
|
design, the code and the validation (iteration 6 step 12).
|
|
|
|
This is the same reading, made when the Target architecture is first submitted and
|
|
returned to its author. It is a review like the others: what it raises is returned
|
|
once and never blocks, and when it fails the submission stands.
|
|
|
|
It runs once in an Execution. A Revision is asked for by the validator or the user,
|
|
who have just read the design; reviewing it again would pay for the same reading.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
from collections.abc import Mapping, Sequence
|
|
from typing import Any
|
|
|
|
from app.adk.runtime import (
|
|
AgentRuntime,
|
|
AgentTask,
|
|
RoleSpec,
|
|
Submitted,
|
|
ToolResult,
|
|
ToolSpec,
|
|
)
|
|
from app.workflows.chain import (
|
|
RECOMMENDATIONS_FILE,
|
|
REQUIREMENTS_FILE,
|
|
Upstream,
|
|
read_list,
|
|
)
|
|
from app.workflows.deliverables import DeliverableKind
|
|
from app.workflows.target_architecture import GRAPH_ATTACHMENT, GraphError, parse_graph
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
MAX_GAPS = 12
|
|
|
|
|
|
class DesignCoverageReview:
|
|
def __init__(self, runtime: AgentRuntime, role: RoleSpec) -> None:
|
|
self._runtime = runtime
|
|
self._role = role
|
|
self._reviewed: set[str] = set()
|
|
|
|
async def __call__(
|
|
self,
|
|
kind: DeliverableKind,
|
|
attachments: Mapping[str, str],
|
|
upstream: Upstream | None,
|
|
scope: str = "",
|
|
) -> Sequence[str]:
|
|
"""The conditions the reviewer finds no Check verifies, each as a line for
|
|
the target architect. Empty when it finds none or could not review."""
|
|
if kind is not DeliverableKind.TARGET_ARCHITECTURE or not upstream:
|
|
return ()
|
|
text = attachments.get(GRAPH_ATTACHMENT, "")
|
|
try:
|
|
graph = parse_graph(text)
|
|
except GraphError:
|
|
return ()
|
|
if scope in self._reviewed:
|
|
return ()
|
|
self._reviewed.add(scope)
|
|
|
|
constraints = [
|
|
item
|
|
for item in read_list(upstream, "requirements", REQUIREMENTS_FILE)
|
|
if item.get("kind") == "non_functional"
|
|
] + read_list(upstream, "design_recommendations", RECOMMENDATIONS_FILE)
|
|
known = {str(item.get("id")) for item in constraints}
|
|
applies_to = {element.id for element in graph.elements} | {
|
|
relationship.id for relationship in graph.relationships
|
|
}
|
|
|
|
async def report_gaps(gaps: Any = None) -> ToolResult:
|
|
if not isinstance(gaps, list):
|
|
return ToolResult("Not recorded: `gaps` is a list; empty when none.")
|
|
return ToolResult("Recorded.", Submitted(json.dumps(gaps)))
|
|
|
|
tool = ToolSpec(
|
|
"report_gaps",
|
|
"Give every gap you found, in one call. An empty list when the Checks "
|
|
"verify every condition. Ends your turn.",
|
|
{
|
|
"type": "object",
|
|
"properties": {
|
|
"gaps": {
|
|
"type": "array",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"source": {
|
|
"type": "string",
|
|
"description": "The requirement or recommendation id",
|
|
},
|
|
"applies_to": {
|
|
"type": "string",
|
|
"description": "The Element or Relationship id",
|
|
},
|
|
"statement": {
|
|
"type": "string",
|
|
"description": "The Check that is missing",
|
|
},
|
|
},
|
|
"required": ["source", "applies_to", "statement"],
|
|
},
|
|
}
|
|
},
|
|
"required": ["gaps"],
|
|
},
|
|
report_gaps,
|
|
tier="execution",
|
|
)
|
|
task = AgentTask(
|
|
role=self._role.name,
|
|
prompt="Review whether the Target architecture's Checks verify every "
|
|
"condition of the requirements and recommendations.",
|
|
context={
|
|
"constraints": constraints,
|
|
"target_architecture": json.loads(text),
|
|
},
|
|
)
|
|
outcome = await self._runtime.run(self._role, task, [tool])
|
|
if not isinstance(outcome, Submitted):
|
|
logger.info("[%s] design coverage review gave no answer", scope)
|
|
return ()
|
|
try:
|
|
reported = json.loads(outcome.content)
|
|
except ValueError:
|
|
return ()
|
|
concerns = []
|
|
for gap in reported if isinstance(reported, list) else []:
|
|
if not isinstance(gap, Mapping):
|
|
continue
|
|
source, where = str(gap.get("source", "")), str(gap.get("applies_to", ""))
|
|
statement = " ".join(str(gap.get("statement", "")).split())
|
|
# A gap must be about something that exists: an invented id is dropped.
|
|
if source not in known or where not in applies_to or not statement:
|
|
continue
|
|
concerns.append(f"{source}: no Check verifies this on {where}: {statement}")
|
|
logger.info(
|
|
"[%s] design coverage review raised %d gap(s)", scope, len(concerns)
|
|
)
|
|
return tuple(concerns[:MAX_GAPS])
|