feat: record verdicts in pieces and check the design against the schema

From the case 02 run cancelled at 66 minutes on 2026-10-07:

- The validator records verdicts a few Checks at a time (verdict_put) and
  they are attached on submission. It timed out five times writing them in
  one reply. Verdicts on the same code survive a run cut short; verdicts on
  changed code are dropped. Its model timeout is 300 s.
- A schema review at submission, in code: every setting a Check names is
  looked up on its Element's resource type, and an unknown resource type is
  reported. Two Revisions in that run were for Checks no code could meet.
  On the two designs that passed validation it raises 2 of 96 Checks, both
  one metric name used as a value, so it is returned once and never blocks.
- Reviews run together; the Jev coverage review stays off.

Plans record both steps and the cancelled run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-08 09:39:13 +01:00
parent 5adc911a82
commit 58e778ca7a
16 changed files with 691 additions and 14 deletions

View File

@@ -8,12 +8,15 @@ last attempt instead of from nothing.
It lives in the Deliverable's working memory:
working["files"] {path: content}
working["graph"] the Target architecture, as target-architecture.json holds it
working["files"] {path: content}
working["graph"] the Target architecture, as target-architecture.json holds it
working["verdicts"] the validation's verdicts, as verdicts.json holds them, with
a mark of the code they were given on
"""
from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping, MutableMapping, Sequence
from pathlib import PurePosixPath
@@ -269,6 +272,150 @@ def graph_tools(working: MutableMapping[str, Any]) -> list[ToolSpec]:
]
# --- the verdicts --------------------------------------------------------------------
VERDICTS = "verdicts"
def basis_of(files: Mapping[str, str]) -> str:
"""A mark of the files a validation judged, to tell when they have changed."""
digest = hashlib.sha256()
for name in sorted(files):
digest.update(name.encode())
digest.update(b"\0")
digest.update(files[name].encode())
digest.update(b"\0")
return digest.hexdigest()
def working_verdicts(working: MutableMapping[str, Any], basis: str) -> dict[str, Any]:
"""The working verdicts for the code marked `basis`.
Verdicts given on other code are dropped: they are about work that has changed.
Verdicts on the same code are kept, so a run cut short carries on.
"""
verdicts = working.get(VERDICTS)
if not isinstance(verdicts, dict) or verdicts.get("basis") != basis:
verdicts = working[VERDICTS] = {"basis": basis}
if not isinstance(verdicts.get("checks"), list):
verdicts["checks"] = []
if not isinstance(verdicts.get("proposed_additions"), list):
verdicts["proposed_additions"] = []
return verdicts
def verdicts_text(working: Mapping[str, Any]) -> str:
"""The working verdicts as verdicts.json, or "" when none have been recorded."""
verdicts = working.get(VERDICTS)
if not isinstance(verdicts, Mapping) or not (
verdicts.get("checks") or verdicts.get("dry_run")
):
return ""
return json.dumps(
{
"dry_run": verdicts.get("dry_run") or {},
"checks": verdicts.get("checks") or [],
"proposed_additions": verdicts.get("proposed_additions") or [],
},
indent=2,
ensure_ascii=False,
)
def verdict_tools(
working: MutableMapping[str, Any], basis: str, check_ids: Sequence[str] = ()
) -> list[ToolSpec]:
"""Tools that record a validation a few verdicts at a time."""
expected = tuple(check_ids)
def remaining(verdicts: Mapping[str, Any]) -> str:
given = {item.get("check") for item in verdicts["checks"]}
left = [identifier for identifier in expected if identifier not in given]
if not expected:
return f"{len(given)} Checks have a verdict."
if not left:
return f"All {len(expected)} Checks have a verdict."
shown = ", ".join(left[:20]) + (" ..." if len(left) > 20 else "")
return f"{len(given)} of {len(expected)} Checks have a verdict. Left: {shown}."
async def verdict_put(
checks: Sequence[Any] = (),
dry_run: Any = None,
proposed_additions: Sequence[Any] = (),
) -> ToolResult:
verdicts = working_verdicts(working, basis)
bad = [
f"checks[{index}]"
for index, item in enumerate(checks or ())
if not isinstance(item, Mapping)
or not isinstance(item.get("check"), str)
or not item["check"].strip()
]
if bad:
return ToolResult(
f"Refused: each verdict needs a `check` id: {', '.join(bad)}"
)
for item in checks or ():
for index, existing in enumerate(verdicts["checks"]):
if existing.get("check") == item["check"]:
verdicts["checks"][index] = dict(item)
break
else:
verdicts["checks"].append(dict(item))
if isinstance(dry_run, Mapping) and dry_run:
verdicts["dry_run"] = dict(dry_run)
for item in proposed_additions or ():
if (
isinstance(item, Mapping)
and dict(item) not in verdicts["proposed_additions"]
):
verdicts["proposed_additions"].append(dict(item))
return ToolResult(f"Recorded. {remaining(verdicts)}")
async def verdict_read() -> ToolResult:
verdicts = working_verdicts(working, basis)
return ToolResult(
(verdicts_text(working) or "No verdicts recorded yet.")
+ f"\n{remaining(verdicts)}"
)
text = {"type": "string"}
verdict = _schema(
{"check": text, "verdict": text, "evidence": text},
["check", "verdict", "evidence"],
)
return [
ToolSpec(
"verdict_put",
"Record verdicts, a few Checks per call, replacing any already "
"recorded for the same Check. Also takes the dry-run's result and "
"proposed additions. What you record is attached as verdicts.json "
"when you submit, and is kept if your run is cut short.",
_schema(
{
"checks": {"type": "array", "items": verdict},
"dry_run": _schema({"verdict": text, "evidence": text}, []),
"proposed_additions": {
"type": "array",
"items": _schema(
{"statement": text, "applies_to": text, "reason": text},
["statement", "applies_to", "reason"],
),
},
},
[],
),
verdict_put,
),
ToolSpec(
"verdict_read",
"Read the verdicts recorded so far and which Checks have none.",
{"type": "object", "properties": {}},
verdict_read,
),
]
def _counts(graph: Mapping[str, Sequence[Any]]) -> str:
return (
f"The graph has {len(graph['elements'])} Elements, "
@@ -286,4 +433,8 @@ def summary(working: Mapping[str, Any]) -> dict[str, Any] | None:
graph = working.get(GRAPH)
if isinstance(graph, Mapping) and any(graph.get(name) for name in _GRAPH_LISTS):
told["graph"] = {name: len(graph.get(name) or ()) for name in _GRAPH_LISTS}
verdicts = working.get(VERDICTS)
if isinstance(verdicts, Mapping) and verdicts.get("checks"):
# Only useful when the code is unchanged; the tools drop them otherwise.
told["verdicts_recorded"] = len(verdicts["checks"])
return told or None