feat: record verdicts in pieces and check the design against the schema
From the case 02 run cancelled at 66 minutes on 2026-10-07: - The validator records verdicts a few Checks at a time (verdict_put) and they are attached on submission. It timed out five times writing them in one reply. Verdicts on the same code survive a run cut short; verdicts on changed code are dropped. Its model timeout is 300 s. - A schema review at submission, in code: every setting a Check names is looked up on its Element's resource type, and an unknown resource type is reported. Two Revisions in that run were for Checks no code could meet. On the two designs that passed validation it raises 2 of 96 Checks, both one metric name used as a value, so it is returned once and never blocks. - Reviews run together; the Jev coverage review stays off. Plans record both steps and the cancelled run. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
19
app/agent.py
19
app/agent.py
@@ -52,6 +52,7 @@ from app.tools.execution import ORCHESTRATOR
|
||||
from app.tools.jev import JevDecider, JevLimits
|
||||
from app.tools.local import TerraformRunner, local_tools
|
||||
from app.tools.mcp import McpCatalog, auth_from_config
|
||||
from app.tools.schema_review import Reviews, SchemaReview
|
||||
from app.tools.toolkit import make_tool_factory
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
||||
from app.workflows.execution_graph import build_execution_graph
|
||||
@@ -323,6 +324,16 @@ async def open_assembly(
|
||||
f"ORCHESTRATOR must be router or model, not {settings.ORCHESTRATOR!r}"
|
||||
)
|
||||
|
||||
# What is looked at in a design before it is submitted. Each raises what it finds
|
||||
# once and never blocks.
|
||||
reviews: list[SubmissionReview] = []
|
||||
if bundle.terraform_providers:
|
||||
# In code: every setting a Check names exists on its resource type.
|
||||
reviews.append(SchemaReview(runner))
|
||||
if settings.JEV_COVERAGE_REVIEW and isinstance(decider, JevDecider):
|
||||
# Optional, and off by default: Jev says which Check verifies each condition.
|
||||
reviews.append(CoverageReview(decider, none_at=settings.JEV_COVERAGE_THRESHOLD))
|
||||
|
||||
async with open_checkpointer(settings) as checkpointer:
|
||||
engine = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
@@ -338,13 +349,7 @@ async def open_assembly(
|
||||
name: skill_version(meta.path) for name, meta in bundle.skills.items()
|
||||
},
|
||||
orchestrator=orchestrator,
|
||||
# Optional, and off by default: Jev reviews a design's coverage before
|
||||
# it is submitted. Without it a submission stands on its structure.
|
||||
submission_review=CoverageReview(
|
||||
decider, none_at=settings.JEV_COVERAGE_THRESHOLD
|
||||
)
|
||||
if settings.JEV_COVERAGE_REVIEW and isinstance(decider, JevDecider)
|
||||
else None,
|
||||
submission_review=Reviews(reviews) if reviews else None,
|
||||
)
|
||||
try:
|
||||
yield Assembly(engine, bundle, roles, dict(catalog.gaps), decider)
|
||||
|
||||
@@ -22,13 +22,19 @@ from app.adk.runtime import (
|
||||
)
|
||||
from app.states.ledger import Ledger, LedgerRefusal, PhaseStatus
|
||||
from app.tools.workspace import (
|
||||
FILES,
|
||||
basis_of,
|
||||
file_tools,
|
||||
graph_text,
|
||||
graph_tools,
|
||||
keep_graph,
|
||||
safe_relative,
|
||||
verdict_tools,
|
||||
verdicts_text,
|
||||
working_files,
|
||||
working_verdicts,
|
||||
)
|
||||
from app.workflows.chain import VERDICTS_FILE, read_graph
|
||||
from app.workflows.deliverables import (
|
||||
DEPENDENCY_GRAPH,
|
||||
DeliverableKind,
|
||||
@@ -49,6 +55,7 @@ from app.workflows.target_architecture import (
|
||||
ORCHESTRATOR = "orchestrator"
|
||||
logger = logging.getLogger(__name__)
|
||||
_TARGET = DeliverableKind.TARGET_ARCHITECTURE
|
||||
_CODE = DeliverableKind.DEPLOYMENT_GUIDANCE
|
||||
CLOUD_READ = "cloud_read"
|
||||
|
||||
_NO_ARGS: Mapping[str, Any] = {"type": "object", "properties": {}}
|
||||
@@ -277,6 +284,13 @@ def specialist_tools(
|
||||
is_graph = kind is DeliverableKind.TARGET_ARCHITECTURE
|
||||
own_files = DEPENDENCY_GRAPH[kind].working_copy is kind
|
||||
reviewed = False
|
||||
is_validation = kind is DeliverableKind.VALIDATION_RESULTS
|
||||
judged = ((upstream or {}).get(_CODE.value) or {}).get("attachments") or {}
|
||||
# The validator's own copy of the code is the same files; either marks them.
|
||||
basis = basis_of(judged or working.get(FILES) or {})
|
||||
if is_validation:
|
||||
working_verdicts(working, basis) # drops verdicts given on other code
|
||||
graph_now = read_graph(upstream)
|
||||
|
||||
async def submit_deliverable(content: str, attachments: Any = None) -> ToolResult:
|
||||
try:
|
||||
@@ -288,6 +302,12 @@ def specialist_tools(
|
||||
attachments = {**working_files(working), **attachments}
|
||||
if is_graph and GRAPH_ATTACHMENT not in attachments and graph_text(working):
|
||||
attachments[GRAPH_ATTACHMENT] = graph_text(working)
|
||||
if (
|
||||
is_validation
|
||||
and VERDICTS_FILE not in attachments
|
||||
and verdicts_text(working)
|
||||
):
|
||||
attachments[VERDICTS_FILE] = verdicts_text(working)
|
||||
unsafe = [name for name in attachments if not safe_relative(name)]
|
||||
if unsafe:
|
||||
return _refused(
|
||||
@@ -400,6 +420,11 @@ def specialist_tools(
|
||||
else ()
|
||||
),
|
||||
*(graph_tools(working) if is_graph else ()),
|
||||
*(
|
||||
verdict_tools(working, basis, graph_now.check_ids if graph_now else ())
|
||||
if is_validation
|
||||
else ()
|
||||
),
|
||||
*(file_tools(working) if own_files else ()),
|
||||
*(gate_cloud_read(tool, allow_cloud_read) for tool in extra),
|
||||
]
|
||||
|
||||
149
app/tools/schema_review.py
Normal file
149
app/tools/schema_review.py
Normal file
@@ -0,0 +1,149 @@
|
||||
"""A review of the Target architecture against the provider's schema, in code.
|
||||
|
||||
A Check that names a setting its Element's resource type does not have cannot be met
|
||||
by any code. Left alone it costs a whole Revision: the code's author finds it, the
|
||||
design is reopened, and everything after it is done again. In the run of 2026-10-07
|
||||
that happened twice, with a schema lookup available to the Specialist that wrote the
|
||||
Checks.
|
||||
|
||||
This reads the settings each Check names and looks them up, so the design is told
|
||||
before it is submitted. It is a review, not a rule: a Check is prose, and a word that
|
||||
looks like a setting may be a value. What it raises is returned once.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import difflib
|
||||
import logging
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Any
|
||||
|
||||
from app.adk.runtime import ToolResult
|
||||
from app.tools.local import TerraformRunner
|
||||
from app.workflows.deliverables import DeliverableKind
|
||||
from app.workflows.target_architecture import GRAPH_ATTACHMENT, GraphError, parse_graph
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_QUOTED = re.compile(r"\"[^\"]*\"|'[^']*'")
|
||||
_CODE = re.compile(r"`([^`]+)`")
|
||||
# snake_case with at least one underscore, or a dotted path of lower-case names.
|
||||
_NAME = re.compile(
|
||||
r"(?<![\w/.\-])"
|
||||
r"([a-z][a-z0-9]*(?:_[a-z0-9]+)+(?:\.[a-z][a-z0-9_]*)*"
|
||||
r"|[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)+)"
|
||||
r"(?![\w/\-])"
|
||||
)
|
||||
_PATH = re.compile(r"^[a-z][a-z0-9_]*(?:\.[a-z][a-z0-9_]*)*$")
|
||||
|
||||
|
||||
def named_settings(statement: str) -> tuple[str, ...]:
|
||||
"""The settings a Check's statement names, as written.
|
||||
|
||||
Anything in backticks that reads as a name or a dotted path, and bare
|
||||
snake_case names and dotted paths. Quoted text is a value, so it is skipped,
|
||||
as are paths such as roles/x.y.
|
||||
"""
|
||||
found: list[str] = []
|
||||
for span in _CODE.findall(statement):
|
||||
candidate = span.strip().split("=")[0].strip()
|
||||
if _PATH.match(candidate):
|
||||
found.append(candidate)
|
||||
bare = _QUOTED.sub(" ", _CODE.sub(" ", statement))
|
||||
found += [match.group(1).rstrip(".") for match in _NAME.finditer(bare)]
|
||||
return tuple(dict.fromkeys(found))
|
||||
|
||||
|
||||
def _names(block: Mapping[str, Any]) -> set[str]:
|
||||
"""Every argument and block name anywhere in a resource type's schema."""
|
||||
names = set(block.get("attributes") or {})
|
||||
for name, nested in (block.get("block_types") or {}).items():
|
||||
names.add(name)
|
||||
names |= _names(nested.get("block") or {})
|
||||
return names
|
||||
|
||||
|
||||
class SchemaReview:
|
||||
def __init__(self, runner: TerraformRunner) -> None:
|
||||
self._runner = runner
|
||||
self._names: dict[str, set[str]] = {}
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
kind: DeliverableKind,
|
||||
attachments: Mapping[str, str],
|
||||
upstream: Mapping[str, Mapping[str, Any]] | None,
|
||||
scope: str = "",
|
||||
) -> Sequence[str]:
|
||||
"""Resource types the provider does not have, and Checks naming a setting
|
||||
their Element's resource type does not have."""
|
||||
if kind is not DeliverableKind.TARGET_ARCHITECTURE:
|
||||
return ()
|
||||
try:
|
||||
graph = parse_graph(attachments.get(GRAPH_ATTACHMENT, ""))
|
||||
except GraphError:
|
||||
return ()
|
||||
schemas = await self._runner._load_schemas()
|
||||
if isinstance(schemas, ToolResult):
|
||||
logger.info("Schema review skipped: %s", schemas.message[:120])
|
||||
return ()
|
||||
|
||||
concerns: list[str] = []
|
||||
types = {element.id: element.resource_type for element in graph.elements}
|
||||
for element in graph.elements:
|
||||
if element.resource_type not in schemas:
|
||||
close = difflib.get_close_matches(
|
||||
element.resource_type, schemas, n=3, cutoff=0.6
|
||||
)
|
||||
hint = f" Close matches: {', '.join(close)}." if close else ""
|
||||
concerns.append(
|
||||
f"Element {element.id}: {element.resource_type} is not a "
|
||||
f"resource type of the provider.{hint}"
|
||||
)
|
||||
for check in graph.checks:
|
||||
resource_type = types.get(check.applies_to)
|
||||
if resource_type not in schemas:
|
||||
continue # a Check on a Relationship, or a type already raised
|
||||
known = self._names.setdefault(
|
||||
resource_type, _names(schemas[resource_type][1])
|
||||
)
|
||||
missing = [
|
||||
setting
|
||||
for setting in named_settings(check.statement)
|
||||
# A resource type named in the statement is not a setting, and a
|
||||
# path is judged by the setting at its end.
|
||||
if setting not in schemas and setting.split(".")[-1] not in known
|
||||
]
|
||||
if missing:
|
||||
concerns.append(
|
||||
f"Check {check.id}: {resource_type} has no setting named "
|
||||
f"{', '.join(missing)}, so no code can meet it there. Put the "
|
||||
"Check on the Element whose resource type has that setting, "
|
||||
"or reword it. Leave it if that word is a value, not a setting"
|
||||
)
|
||||
if concerns:
|
||||
logger.info("[%s] schema review raised %d item(s)", scope, len(concerns))
|
||||
return tuple(concerns)
|
||||
|
||||
|
||||
class Reviews:
|
||||
"""Several reviews as one: everything any of them raises."""
|
||||
|
||||
def __init__(self, reviews: Sequence[Any]) -> None:
|
||||
self._reviews = tuple(reviews)
|
||||
|
||||
async def __call__(
|
||||
self,
|
||||
kind: Any,
|
||||
attachments: Mapping[str, str],
|
||||
upstream: Mapping[str, Mapping[str, Any]] | None,
|
||||
scope: str = "",
|
||||
) -> Sequence[str]:
|
||||
raised: list[str] = []
|
||||
for review in self._reviews:
|
||||
try:
|
||||
raised += await review(kind, attachments, upstream, scope)
|
||||
except Exception: # one review failing must not lose the others
|
||||
logger.exception("A submission review failed; continuing")
|
||||
return tuple(raised)
|
||||
@@ -8,12 +8,15 @@ last attempt instead of from nothing.
|
||||
|
||||
It lives in the Deliverable's working memory:
|
||||
|
||||
working["files"] {path: content}
|
||||
working["graph"] the Target architecture, as target-architecture.json holds it
|
||||
working["files"] {path: content}
|
||||
working["graph"] the Target architecture, as target-architecture.json holds it
|
||||
working["verdicts"] the validation's verdicts, as verdicts.json holds them, with
|
||||
a mark of the code they were given on
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
from collections.abc import Mapping, MutableMapping, Sequence
|
||||
from pathlib import PurePosixPath
|
||||
@@ -269,6 +272,150 @@ def graph_tools(working: MutableMapping[str, Any]) -> list[ToolSpec]:
|
||||
]
|
||||
|
||||
|
||||
# --- the verdicts --------------------------------------------------------------------
|
||||
|
||||
VERDICTS = "verdicts"
|
||||
|
||||
|
||||
def basis_of(files: Mapping[str, str]) -> str:
|
||||
"""A mark of the files a validation judged, to tell when they have changed."""
|
||||
digest = hashlib.sha256()
|
||||
for name in sorted(files):
|
||||
digest.update(name.encode())
|
||||
digest.update(b"\0")
|
||||
digest.update(files[name].encode())
|
||||
digest.update(b"\0")
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def working_verdicts(working: MutableMapping[str, Any], basis: str) -> dict[str, Any]:
|
||||
"""The working verdicts for the code marked `basis`.
|
||||
|
||||
Verdicts given on other code are dropped: they are about work that has changed.
|
||||
Verdicts on the same code are kept, so a run cut short carries on.
|
||||
"""
|
||||
verdicts = working.get(VERDICTS)
|
||||
if not isinstance(verdicts, dict) or verdicts.get("basis") != basis:
|
||||
verdicts = working[VERDICTS] = {"basis": basis}
|
||||
if not isinstance(verdicts.get("checks"), list):
|
||||
verdicts["checks"] = []
|
||||
if not isinstance(verdicts.get("proposed_additions"), list):
|
||||
verdicts["proposed_additions"] = []
|
||||
return verdicts
|
||||
|
||||
|
||||
def verdicts_text(working: Mapping[str, Any]) -> str:
|
||||
"""The working verdicts as verdicts.json, or "" when none have been recorded."""
|
||||
verdicts = working.get(VERDICTS)
|
||||
if not isinstance(verdicts, Mapping) or not (
|
||||
verdicts.get("checks") or verdicts.get("dry_run")
|
||||
):
|
||||
return ""
|
||||
return json.dumps(
|
||||
{
|
||||
"dry_run": verdicts.get("dry_run") or {},
|
||||
"checks": verdicts.get("checks") or [],
|
||||
"proposed_additions": verdicts.get("proposed_additions") or [],
|
||||
},
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
|
||||
|
||||
def verdict_tools(
|
||||
working: MutableMapping[str, Any], basis: str, check_ids: Sequence[str] = ()
|
||||
) -> list[ToolSpec]:
|
||||
"""Tools that record a validation a few verdicts at a time."""
|
||||
expected = tuple(check_ids)
|
||||
|
||||
def remaining(verdicts: Mapping[str, Any]) -> str:
|
||||
given = {item.get("check") for item in verdicts["checks"]}
|
||||
left = [identifier for identifier in expected if identifier not in given]
|
||||
if not expected:
|
||||
return f"{len(given)} Checks have a verdict."
|
||||
if not left:
|
||||
return f"All {len(expected)} Checks have a verdict."
|
||||
shown = ", ".join(left[:20]) + (" ..." if len(left) > 20 else "")
|
||||
return f"{len(given)} of {len(expected)} Checks have a verdict. Left: {shown}."
|
||||
|
||||
async def verdict_put(
|
||||
checks: Sequence[Any] = (),
|
||||
dry_run: Any = None,
|
||||
proposed_additions: Sequence[Any] = (),
|
||||
) -> ToolResult:
|
||||
verdicts = working_verdicts(working, basis)
|
||||
bad = [
|
||||
f"checks[{index}]"
|
||||
for index, item in enumerate(checks or ())
|
||||
if not isinstance(item, Mapping)
|
||||
or not isinstance(item.get("check"), str)
|
||||
or not item["check"].strip()
|
||||
]
|
||||
if bad:
|
||||
return ToolResult(
|
||||
f"Refused: each verdict needs a `check` id: {', '.join(bad)}"
|
||||
)
|
||||
for item in checks or ():
|
||||
for index, existing in enumerate(verdicts["checks"]):
|
||||
if existing.get("check") == item["check"]:
|
||||
verdicts["checks"][index] = dict(item)
|
||||
break
|
||||
else:
|
||||
verdicts["checks"].append(dict(item))
|
||||
if isinstance(dry_run, Mapping) and dry_run:
|
||||
verdicts["dry_run"] = dict(dry_run)
|
||||
for item in proposed_additions or ():
|
||||
if (
|
||||
isinstance(item, Mapping)
|
||||
and dict(item) not in verdicts["proposed_additions"]
|
||||
):
|
||||
verdicts["proposed_additions"].append(dict(item))
|
||||
return ToolResult(f"Recorded. {remaining(verdicts)}")
|
||||
|
||||
async def verdict_read() -> ToolResult:
|
||||
verdicts = working_verdicts(working, basis)
|
||||
return ToolResult(
|
||||
(verdicts_text(working) or "No verdicts recorded yet.")
|
||||
+ f"\n{remaining(verdicts)}"
|
||||
)
|
||||
|
||||
text = {"type": "string"}
|
||||
verdict = _schema(
|
||||
{"check": text, "verdict": text, "evidence": text},
|
||||
["check", "verdict", "evidence"],
|
||||
)
|
||||
return [
|
||||
ToolSpec(
|
||||
"verdict_put",
|
||||
"Record verdicts, a few Checks per call, replacing any already "
|
||||
"recorded for the same Check. Also takes the dry-run's result and "
|
||||
"proposed additions. What you record is attached as verdicts.json "
|
||||
"when you submit, and is kept if your run is cut short.",
|
||||
_schema(
|
||||
{
|
||||
"checks": {"type": "array", "items": verdict},
|
||||
"dry_run": _schema({"verdict": text, "evidence": text}, []),
|
||||
"proposed_additions": {
|
||||
"type": "array",
|
||||
"items": _schema(
|
||||
{"statement": text, "applies_to": text, "reason": text},
|
||||
["statement", "applies_to", "reason"],
|
||||
),
|
||||
},
|
||||
},
|
||||
[],
|
||||
),
|
||||
verdict_put,
|
||||
),
|
||||
ToolSpec(
|
||||
"verdict_read",
|
||||
"Read the verdicts recorded so far and which Checks have none.",
|
||||
{"type": "object", "properties": {}},
|
||||
verdict_read,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _counts(graph: Mapping[str, Sequence[Any]]) -> str:
|
||||
return (
|
||||
f"The graph has {len(graph['elements'])} Elements, "
|
||||
@@ -286,4 +433,8 @@ def summary(working: Mapping[str, Any]) -> dict[str, Any] | None:
|
||||
graph = working.get(GRAPH)
|
||||
if isinstance(graph, Mapping) and any(graph.get(name) for name in _GRAPH_LISTS):
|
||||
told["graph"] = {name: len(graph.get(name) or ()) for name in _GRAPH_LISTS}
|
||||
verdicts = working.get(VERDICTS)
|
||||
if isinstance(verdicts, Mapping) and verdicts.get("checks"):
|
||||
# Only useful when the code is unchanged; the tools drop them otherwise.
|
||||
told["verdicts_recorded"] = len(verdicts["checks"])
|
||||
return told or None
|
||||
|
||||
Reference in New Issue
Block a user