Files

156 lines
5.8 KiB
Python

"""A review of a design's coverage by a model, before anyone approves it.
The validator reads the requirements and recommendations against the Checks and
proposes one for each condition nothing verifies. That reading needs no code, but it
used to happen only after the code was written: in the runs of 2026-10-09 it sent
the Target architecture back in four of seven, each time costing a Revision of the
design, the code and the validation (iteration 6 step 12).
This is the same reading, made when the Target architecture is first submitted and
returned to its author. It is a review like the others: what it raises is returned
once and never blocks, and when it fails the submission stands.
It runs once in an Execution. A Revision is asked for by the validator or the user,
who have just read the design; reviewing it again would pay for the same reading.
"""
from __future__ import annotations
import json
import logging
from collections.abc import Mapping, Sequence
from typing import Any
from execution_engine.adk.runtime import (
AgentRuntime,
AgentTask,
RoleSpec,
Submitted,
ToolResult,
ToolSpec,
)
from tests.reference_workflow.chain import (
RECOMMENDATIONS_FILE,
REQUIREMENTS_FILE,
Upstream,
read_list,
)
from tests.reference_workflow.kinds import DeliverableKind
from tests.reference_workflow.target_architecture import (
GRAPH_ATTACHMENT,
GraphError,
parse_graph,
)
logger = logging.getLogger(__name__)
MAX_GAPS = 12
class DesignCoverageReview:
def __init__(self, runtime: AgentRuntime, role: RoleSpec) -> None:
self._runtime = runtime
self._role = role
self._reviewed: set[str] = set()
async def __call__(
self,
kind: DeliverableKind,
attachments: Mapping[str, str],
upstream: Upstream | None,
scope: str = "",
) -> Sequence[str]:
"""The conditions the reviewer finds no Check verifies, each as a line for
the target architect. Empty when it finds none or could not review."""
if kind != DeliverableKind.TARGET_ARCHITECTURE or not upstream:
return ()
text = attachments.get(GRAPH_ATTACHMENT, "")
try:
graph = parse_graph(text)
except GraphError:
return ()
if scope in self._reviewed:
return ()
self._reviewed.add(scope)
constraints = [
item
for item in read_list(upstream, "requirements", REQUIREMENTS_FILE)
if item.get("kind") == "non_functional"
] + read_list(upstream, "design_recommendations", RECOMMENDATIONS_FILE)
known = {str(item.get("id")) for item in constraints}
applies_to = {element.id for element in graph.elements} | {
relationship.id for relationship in graph.relationships
}
async def report_gaps(gaps: Any = None) -> ToolResult:
if not isinstance(gaps, list):
return ToolResult("Not recorded: `gaps` is a list; empty when none.")
return ToolResult("Recorded.", Submitted(json.dumps(gaps)))
tool = ToolSpec(
"report_gaps",
"Give every gap you found, in one call. An empty list when the Checks "
"verify every condition. Ends your turn.",
{
"type": "object",
"properties": {
"gaps": {
"type": "array",
"items": {
"type": "object",
"properties": {
"source": {
"type": "string",
"description": "The requirement or recommendation id",
},
"applies_to": {
"type": "string",
"description": "The Element or Relationship id",
},
"statement": {
"type": "string",
"description": "The Check that is missing",
},
},
"required": ["source", "applies_to", "statement"],
},
}
},
"required": ["gaps"],
},
report_gaps,
tier="execution",
)
task = AgentTask(
role=self._role.name,
prompt="Review whether the Target architecture's Checks verify every "
"condition of the requirements and recommendations.",
context={
"constraints": constraints,
"target_architecture": json.loads(text),
},
)
outcome = await self._runtime.run(self._role, task, [tool])
if not isinstance(outcome, Submitted):
logger.info("[%s] design coverage review gave no answer", scope)
return ()
try:
reported = json.loads(outcome.content)
except ValueError:
return ()
concerns = []
for gap in reported if isinstance(reported, list) else []:
if not isinstance(gap, Mapping):
continue
source, where = str(gap.get("source", "")), str(gap.get("applies_to", ""))
statement = " ".join(str(gap.get("statement", "")).split())
# A gap must be about something that exists: an invented id is dropped.
if source not in known or where not in applies_to or not statement:
continue
concerns.append(f"{source}: no Check verifies this on {where}: {statement}")
logger.info(
"[%s] design coverage review raised %d gap(s)", scope, len(concerns)
)
return tuple(concerns[:MAX_GAPS])