Files
gcp_solution_architecture_a…/app/tools/workspace.py
Jonathan Boniface 58e778ca7a feat: record verdicts in pieces and check the design against the schema
From the case 02 run cancelled at 66 minutes on 2026-10-07:

- The validator records verdicts a few Checks at a time (verdict_put) and
  they are attached on submission. It timed out five times writing them in
  one reply. Verdicts on the same code survive a run cut short; verdicts on
  changed code are dropped. Its model timeout is 300 s.
- A schema review at submission, in code: every setting a Check names is
  looked up on its Element's resource type, and an unknown resource type is
  reported. Two Revisions in that run were for Checks no code could meet.
  On the two designs that passed validation it raises 2 of 96 Checks, both
  one metric name used as a value, so it is returned once and never blocks.
- Reviews run together; the Jev coverage review stays off.

Plans record both steps and the cancelled run.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-08 09:39:13 +01:00

441 lines
16 KiB
Python

"""A Specialist's working copy: files and a graph built up over several tool calls.
A model that must return a whole Deliverable in one reply spends most of its output
repeating itself, and a long reply is what times out. With a working copy it writes a
file once, changes only the lines that need changing, and adds to a graph a few items
at a time. The working copy is kept with the Execution, so a Revision starts from the
last attempt instead of from nothing.
It lives in the Deliverable's working memory:
working["files"] {path: content}
working["graph"] the Target architecture, as target-architecture.json holds it
working["verdicts"] the validation's verdicts, as verdicts.json holds them, with
a mark of the code they were given on
"""
from __future__ import annotations
import hashlib
import json
from collections.abc import Mapping, MutableMapping, Sequence
from pathlib import PurePosixPath
from typing import Any
from app.adk.runtime import ToolResult, ToolSpec
FILES = "files"
GRAPH = "graph"
_GRAPH_LISTS = ("elements", "relationships", "checks", "not_applicable")
_MAX_READ = 40_000
def safe_relative(name: str) -> bool:
path = PurePosixPath(name)
return bool(name) and not path.is_absolute() and ".." not in path.parts
def working_files(working: MutableMapping[str, Any]) -> MutableMapping[str, str]:
files = working.get(FILES)
if not isinstance(files, MutableMapping):
files = working[FILES] = {}
return files
def _schema(properties: Mapping[str, Any], required: Sequence[str]) -> dict[str, Any]:
return {
"type": "object",
"properties": dict(properties),
"required": list(required),
}
def _lines(content: str) -> int:
return content.count("\n") + (1 if content and not content.endswith("\n") else 0)
def file_tools(working: MutableMapping[str, Any]) -> list[ToolSpec]:
"""Tools that write, change and read the files of a working copy."""
files = working_files(working)
async def write_file(path: str, content: str) -> ToolResult:
if not safe_relative(path):
return ToolResult(
f"Refused: {path!r} must be a relative path without '..'."
)
files[path] = content
return ToolResult(f"Wrote {path} ({_lines(content)} lines).")
async def edit_file(path: str, old: str, new: str) -> ToolResult:
if path not in files:
return ToolResult(f"Refused: {path} is not in your working copy.")
found = files[path].count(old) if old else 0
if found != 1:
return ToolResult(
f"Refused: `old` must match exactly one place in {path}; it matches "
f"{found}. Include more of the surrounding lines."
)
files[path] = files[path].replace(old, new)
return ToolResult(f"Edited {path}.")
async def read_file(path: str = "") -> ToolResult:
if not path:
if not files:
return ToolResult("Your working copy is empty.")
return ToolResult(
"Your working copy:\n"
+ "\n".join(
f"- {name} ({_lines(files[name])} lines)" for name in sorted(files)
)
)
if path not in files:
return ToolResult(f"Refused: {path} is not in your working copy.")
content = files[path]
if len(content) > _MAX_READ:
content = content[:_MAX_READ] + "\n[truncated]"
return ToolResult(content)
async def delete_file(path: str) -> ToolResult:
if files.pop(path, None) is None:
return ToolResult(f"Refused: {path} is not in your working copy.")
return ToolResult(f"Deleted {path}.")
text = {"type": "string"}
return [
ToolSpec(
"write_file",
"Write one file of your working copy, replacing it if it exists. Your "
"working copy is attached for you when you submit.",
_schema({"path": text, "content": text}, ["path", "content"]),
write_file,
),
ToolSpec(
"edit_file",
"Change part of one file: replace the single place `old` matches with "
"`new`. Use this for a fix instead of writing the file again.",
_schema({"path": text, "old": text, "new": text}, ["path", "old", "new"]),
edit_file,
),
ToolSpec(
"read_file",
"Read one file of your working copy, or list its files when `path` is "
"left out.",
_schema({"path": text}, []),
read_file,
),
ToolSpec(
"delete_file",
"Remove one file from your working copy.",
_schema({"path": text}, ["path"]),
delete_file,
),
]
# --- the graph -----------------------------------------------------------------------
def working_graph(working: MutableMapping[str, Any]) -> dict[str, list[dict[str, Any]]]:
graph = working.get(GRAPH)
if not isinstance(graph, dict):
graph = working[GRAPH] = {}
for name in _GRAPH_LISTS:
if not isinstance(graph.get(name), list):
graph[name] = []
return graph
def graph_text(working: MutableMapping[str, Any]) -> str:
"""The working graph as target-architecture.json, or "" when nothing is in it."""
graph = working.get(GRAPH)
if not isinstance(graph, dict) or not any(graph.get(n) for n in _GRAPH_LISTS):
return ""
return json.dumps(
{name: graph.get(name, []) for name in _GRAPH_LISTS},
indent=2,
ensure_ascii=False,
)
def keep_graph(working: MutableMapping[str, Any], text: str) -> None:
"""Make a submitted graph the working graph, so a Revision amends it."""
try:
data = json.loads(text)
except ValueError:
return
if isinstance(data, dict):
working[GRAPH] = {
name: [item for item in data.get(name, []) if isinstance(item, dict)]
for name in _GRAPH_LISTS
}
def _key(name: str) -> str:
return "recommendation" if name == "not_applicable" else "id"
def graph_tools(working: MutableMapping[str, Any]) -> list[ToolSpec]:
"""Tools that build the Target architecture a few items at a time."""
async def graph_put(
elements: Sequence[Any] = (),
relationships: Sequence[Any] = (),
checks: Sequence[Any] = (),
not_applicable: Sequence[Any] = (),
) -> ToolResult:
graph = working_graph(working)
given = dict(
zip(
_GRAPH_LISTS,
(elements, relationships, checks, not_applicable),
strict=True,
)
)
bad = [
f"{name}[{index}]"
for name, items in given.items()
for index, item in enumerate(items or ())
if not isinstance(item, Mapping)
or not isinstance(item.get(_key(name)), str)
or not item[_key(name)].strip()
]
if bad:
return ToolResult(
"Refused: each item must be an object with an `id` (a "
f"`recommendation` under not_applicable): {', '.join(bad)}"
)
added = replaced = 0
for name, items in given.items():
key = _key(name)
for item in items or ():
for index, existing in enumerate(graph[name]):
if existing.get(key) == item[key]:
graph[name][index] = dict(item)
replaced += 1
break
else:
graph[name].append(dict(item))
added += 1
return ToolResult(f"Added {added}, replaced {replaced}. {_counts(graph)}")
async def graph_remove(ids: Sequence[str]) -> ToolResult:
graph = working_graph(working)
wanted = set(ids)
removed = 0
for name in _GRAPH_LISTS:
kept = [item for item in graph[name] if item.get(_key(name)) not in wanted]
removed += len(graph[name]) - len(kept)
graph[name] = kept
return ToolResult(f"Removed {removed}. {_counts(graph)}")
async def graph_read() -> ToolResult:
return ToolResult(graph_text(working) or "Your working graph is empty.")
def items(*fields: str, optional: tuple[str, ...] = ()) -> dict[str, Any]:
return {
"type": "array",
"items": _schema(
{name: {"type": "string"} for name in (*fields, *optional)}, fields
),
}
lists = {
"elements": items(
"id", "name", "product", "resource_type", optional=("component",)
),
"relationships": items("id", "from", "to", "kind"),
"checks": items("id", "applies_to", "statement", "source"),
"not_applicable": items("recommendation", "reason"),
}
return [
ToolSpec(
"graph_put",
"Add items to your working graph, or replace the ones with the same id. "
"Call it several times with a few items each. Your working graph is "
"attached as target-architecture.json when you submit.",
_schema(lists, []),
graph_put,
),
ToolSpec(
"graph_remove",
"Remove items from your working graph by id (a not-applicable entry by "
"its recommendation id).",
_schema({"ids": {"type": "array", "items": {"type": "string"}}}, ["ids"]),
graph_remove,
),
ToolSpec(
"graph_read",
"Read your working graph.",
{"type": "object", "properties": {}},
graph_read,
),
]
# --- the verdicts --------------------------------------------------------------------
VERDICTS = "verdicts"
def basis_of(files: Mapping[str, str]) -> str:
"""A mark of the files a validation judged, to tell when they have changed."""
digest = hashlib.sha256()
for name in sorted(files):
digest.update(name.encode())
digest.update(b"\0")
digest.update(files[name].encode())
digest.update(b"\0")
return digest.hexdigest()
def working_verdicts(working: MutableMapping[str, Any], basis: str) -> dict[str, Any]:
"""The working verdicts for the code marked `basis`.
Verdicts given on other code are dropped: they are about work that has changed.
Verdicts on the same code are kept, so a run cut short carries on.
"""
verdicts = working.get(VERDICTS)
if not isinstance(verdicts, dict) or verdicts.get("basis") != basis:
verdicts = working[VERDICTS] = {"basis": basis}
if not isinstance(verdicts.get("checks"), list):
verdicts["checks"] = []
if not isinstance(verdicts.get("proposed_additions"), list):
verdicts["proposed_additions"] = []
return verdicts
def verdicts_text(working: Mapping[str, Any]) -> str:
"""The working verdicts as verdicts.json, or "" when none have been recorded."""
verdicts = working.get(VERDICTS)
if not isinstance(verdicts, Mapping) or not (
verdicts.get("checks") or verdicts.get("dry_run")
):
return ""
return json.dumps(
{
"dry_run": verdicts.get("dry_run") or {},
"checks": verdicts.get("checks") or [],
"proposed_additions": verdicts.get("proposed_additions") or [],
},
indent=2,
ensure_ascii=False,
)
def verdict_tools(
working: MutableMapping[str, Any], basis: str, check_ids: Sequence[str] = ()
) -> list[ToolSpec]:
"""Tools that record a validation a few verdicts at a time."""
expected = tuple(check_ids)
def remaining(verdicts: Mapping[str, Any]) -> str:
given = {item.get("check") for item in verdicts["checks"]}
left = [identifier for identifier in expected if identifier not in given]
if not expected:
return f"{len(given)} Checks have a verdict."
if not left:
return f"All {len(expected)} Checks have a verdict."
shown = ", ".join(left[:20]) + (" ..." if len(left) > 20 else "")
return f"{len(given)} of {len(expected)} Checks have a verdict. Left: {shown}."
async def verdict_put(
checks: Sequence[Any] = (),
dry_run: Any = None,
proposed_additions: Sequence[Any] = (),
) -> ToolResult:
verdicts = working_verdicts(working, basis)
bad = [
f"checks[{index}]"
for index, item in enumerate(checks or ())
if not isinstance(item, Mapping)
or not isinstance(item.get("check"), str)
or not item["check"].strip()
]
if bad:
return ToolResult(
f"Refused: each verdict needs a `check` id: {', '.join(bad)}"
)
for item in checks or ():
for index, existing in enumerate(verdicts["checks"]):
if existing.get("check") == item["check"]:
verdicts["checks"][index] = dict(item)
break
else:
verdicts["checks"].append(dict(item))
if isinstance(dry_run, Mapping) and dry_run:
verdicts["dry_run"] = dict(dry_run)
for item in proposed_additions or ():
if (
isinstance(item, Mapping)
and dict(item) not in verdicts["proposed_additions"]
):
verdicts["proposed_additions"].append(dict(item))
return ToolResult(f"Recorded. {remaining(verdicts)}")
async def verdict_read() -> ToolResult:
verdicts = working_verdicts(working, basis)
return ToolResult(
(verdicts_text(working) or "No verdicts recorded yet.")
+ f"\n{remaining(verdicts)}"
)
text = {"type": "string"}
verdict = _schema(
{"check": text, "verdict": text, "evidence": text},
["check", "verdict", "evidence"],
)
return [
ToolSpec(
"verdict_put",
"Record verdicts, a few Checks per call, replacing any already "
"recorded for the same Check. Also takes the dry-run's result and "
"proposed additions. What you record is attached as verdicts.json "
"when you submit, and is kept if your run is cut short.",
_schema(
{
"checks": {"type": "array", "items": verdict},
"dry_run": _schema({"verdict": text, "evidence": text}, []),
"proposed_additions": {
"type": "array",
"items": _schema(
{"statement": text, "applies_to": text, "reason": text},
["statement", "applies_to", "reason"],
),
},
},
[],
),
verdict_put,
),
ToolSpec(
"verdict_read",
"Read the verdicts recorded so far and which Checks have none.",
{"type": "object", "properties": {}},
verdict_read,
),
]
def _counts(graph: Mapping[str, Sequence[Any]]) -> str:
return (
f"The graph has {len(graph['elements'])} Elements, "
f"{len(graph['relationships'])} Relationships and "
f"{len(graph['checks'])} Checks."
)
def summary(working: Mapping[str, Any]) -> dict[str, Any] | None:
"""What a Specialist is told it already has, at the start of a run."""
told: dict[str, Any] = {}
files = working.get(FILES)
if isinstance(files, Mapping) and files:
told["files"] = sorted(files)
graph = working.get(GRAPH)
if isinstance(graph, Mapping) and any(graph.get(name) for name in _GRAPH_LISTS):
told["graph"] = {name: len(graph.get(name) or ()) for name in _GRAPH_LISTS}
verdicts = working.get(VERDICTS)
if isinstance(verdicts, Mapping) and verdicts.get("checks"):
# Only useful when the code is unchanged; the tools drop them otherwise.
told["verdicts_recorded"] = len(verdicts["checks"])
return told or None