fixed multile issues
Some checks failed
validation / verify (push) Failing after 8s

This commit is contained in:
2026-10-07 17:33:36 +01:00
parent 4c4d44ef87
commit 5bb22d984e
76 changed files with 5799 additions and 179 deletions

View File

@@ -1,16 +1,23 @@
"""Local tools: a Mermaid syntax check and Terraform checks in a scratch directory.
Terraform runs only fmt, init -backend=false, validate and plan. No command that
changes cloud resources is ever constructed.
Terraform runs only fmt, init -backend=false, validate, plan and providers schema. No
command that changes cloud resources is ever constructed.
A Specialist with a working copy (app/tools/workspace.py) runs the checks on it and
passes no files; `terraform_fmt` formats the working copy in place. That is what
keeps a fix cheap: the model changes a line and validates, without sending every file
again on each call.
"""
from __future__ import annotations
import asyncio
import difflib
import json
import os
import re
import tempfile
from collections.abc import Awaitable, Callable, Iterable, Mapping
from collections.abc import Awaitable, Callable, Iterable, Mapping, MutableMapping
from dataclasses import dataclass, field
from pathlib import Path, PurePosixPath
from typing import Any
@@ -79,10 +86,19 @@ class TerraformRunner:
# Providers are downloaded here once and reused by every later check. Without it,
# each validate or plan downloads its providers again into a scratch directory.
plugin_cache_dir: str | None = None
# The providers the schema lookup answers for, as the provider bundle pins them:
# {local name: {"source": ..., "version": ...}}.
providers: Mapping[str, Mapping[str, str]] = field(default_factory=dict)
# Terraform's plugin cache is not safe for concurrent installs.
_init_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False)
_schema_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False)
# resource type -> (provider label, the resource's schema block)
_schemas: dict[str, tuple[str, dict[str, Any]]] | None = field(
default=None, repr=False
)
async def fmt(self, files: Mapping[str, str]) -> ToolResult:
async def formatted(self, files: Mapping[str, str]) -> dict[str, str] | ToolResult:
"""The files formatting changes, with their new content; or why it failed."""
refusal = _check_files(files)
if refusal:
return refusal
@@ -93,19 +109,24 @@ class TerraformRunner:
)
if code != 0:
return ToolResult(f"terraform fmt failed:\n{output}")
changed = [
name for name in files if (root / name).read_text() != files[name]
]
if not changed:
return ToolResult("terraform fmt: all files are already formatted.")
formatted = "\n\n".join(
f"--- {name}\n{(root / name).read_text()}" for name in changed
)
return ToolResult(
_clip(
f"terraform fmt reformatted {len(changed)} file(s):\n\n{formatted}"
)
)
return {
name: (root / name).read_text()
for name in files
if (root / name).read_text() != files[name]
}
async def fmt(self, files: Mapping[str, str]) -> ToolResult:
changed = await self.formatted(files)
if isinstance(changed, ToolResult):
return changed
if not changed:
return ToolResult("terraform fmt: all files are already formatted.")
formatted = "\n\n".join(
f"--- {name}\n{content}" for name, content in changed.items()
)
return ToolResult(
_clip(f"terraform fmt reformatted {len(changed)} file(s):\n\n{formatted}")
)
async def validate(self, files: Mapping[str, str]) -> ToolResult:
refusal = _check_files(files)
@@ -150,6 +171,87 @@ class TerraformRunner:
_clip(f"terraform plan {verdict} (nothing was applied):\n{output}")
)
async def schema(
self, resource_type: str, block: str = "", find: str = ""
) -> ToolResult:
"""What a resource type accepts, from the pinned provider's own schema."""
loaded = await self._load_schemas()
if isinstance(loaded, ToolResult):
return loaded
if resource_type not in loaded:
close = difflib.get_close_matches(resource_type, loaded, n=5, cutoff=0.6)
hint = f" Close matches: {', '.join(close)}." if close else ""
return ToolResult(
f"No resource type {resource_type} in {self._provider_labels()}.{hint}"
)
label, root = loaded[resource_type]
title = f"{resource_type} ({label})"
if find:
return ToolResult(_clip(_find_in_schema(title, root, find)))
node = root
for step in [part for part in block.split(".") if part]:
nested = node.get("block_types", {}).get(step)
if nested is None:
inside = ", ".join(sorted(node.get("block_types", {}))) or "none"
return ToolResult(
f"{title} has no block {block!r}: {step!r} is not a block there. "
f"Blocks at that level: {inside}."
)
node = nested["block"]
heading = f"{title}, block {block}" if block else title
return ToolResult(_clip(_render_schema(heading, node)))
def _provider_labels(self) -> str:
return (
", ".join(
f"{pin['source']} {pin['version']}" for pin in self.providers.values()
)
or "no pinned provider"
)
async def _load_schemas(self) -> dict[str, tuple[str, dict[str, Any]]] | ToolResult:
async with self._schema_lock:
if self._schemas is not None:
return self._schemas
if not self.providers:
return ToolResult(
"Schema lookup is unavailable: the Cloud provider bundle pins no "
"Terraform provider."
)
required = "\n".join(
f' {name} = {{ source = "{pin["source"]}", '
f'version = "{pin["version"]}" }}'
for name, pin in self.providers.items()
)
with tempfile.TemporaryDirectory(prefix="tf-schema-") as tmp:
root = Path(tmp)
(root / "versions.tf").write_text(
f"terraform {{\n required_providers {{\n{required}\n }}\n}}\n"
)
failure = await self._init(root)
if failure:
return failure
code, output = await self._run(
[self.binary, "providers", "schema", "-json"], root
)
if code != 0:
return ToolResult(
_clip(f"terraform providers schema failed:\n{output}")
)
try:
data = json.loads(output)
except ValueError:
return ToolResult("terraform providers schema returned no JSON.")
labels = {pin["source"]: pin["version"] for pin in self.providers.values()}
schemas: dict[str, tuple[str, dict[str, Any]]] = {}
for address, provider in (data.get("provider_schemas") or {}).items():
source = address.split("/", 1)[-1] # drop the registry host
label = f"{source} {labels.get(source, '')}".strip()
for name, resource in (provider.get("resource_schemas") or {}).items():
schemas[name] = (label, resource.get("block") or {})
self._schemas = schemas
return schemas
async def _init(self, root: Path) -> ToolResult | None:
command = [self.binary, "init", "-backend=false", "-input=false", "-no-color"]
if self.plugin_cache_dir:
@@ -236,6 +338,100 @@ def _clip(text: str) -> str:
return text if len(text) <= _MAX_OUTPUT else text[:_MAX_OUTPUT] + "\n[truncated]"
def _type(value: Any) -> str:
if isinstance(value, str):
return value
if isinstance(value, list) and len(value) == 2 and value[0] != "object":
return f"{value[0]} of {_type(value[1])}"
return "object"
def _use(attribute: Mapping[str, Any]) -> str:
if attribute.get("required"):
return "required"
return "optional" if attribute.get("optional") else "read-only"
def _arguments(node: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]:
"""The arguments code may set: everything but the read-only ones."""
return {
name: attribute
for name, attribute in sorted((node.get("attributes") or {}).items())
if _use(attribute) != "read-only"
}
def _block_note(nested: Mapping[str, Any]) -> str:
notes = [str(nested.get("nesting_mode", "block"))]
if nested.get("min_items"):
notes.append(f"at least {nested['min_items']}")
if nested.get("max_items"):
notes.append(f"at most {nested['max_items']}")
return ", ".join(notes)
def _render_schema(heading: str, node: Mapping[str, Any]) -> str:
"""One level in full, and the blocks beneath it as an outline of names."""
lines = [heading, "", "Arguments:"]
arguments = _arguments(node)
for name, attribute in arguments.items():
described = " ".join(str(attribute.get("description") or "").split())[:160]
deprecated = " DEPRECATED" if attribute.get("deprecated") else ""
lines.append(
f" {name} ({_type(attribute.get('type'))}, {_use(attribute)})"
f"{deprecated}" + (f": {described}" if described else "")
)
if not arguments:
lines.append(" none")
def outline(parent: Mapping[str, Any], depth: int) -> None:
for name, nested in sorted((parent.get("block_types") or {}).items()):
inner = nested.get("block") or {}
names = ", ".join(
f"{argument}*" if attribute.get("required") else argument
for argument, attribute in _arguments(inner).items()
)
lines.append(
f"{' ' * depth}{name} {{ }} ({_block_note(nested)}): {names or '-'}"
)
outline(inner, depth + 1)
lines += ["", "Blocks (* marks a required argument):"]
before = len(lines)
outline(node, 1)
if len(lines) == before:
lines.append(" none")
lines += ["", "Pass `block`, such as a.b, for one block's arguments in full."]
return "\n".join(lines)
def _find_in_schema(title: str, root: Mapping[str, Any], wanted: str) -> str:
"""Every argument or block whose name contains `wanted`, with where it sits."""
needle = wanted.lower()
found: list[str] = []
def search(node: Mapping[str, Any], path: str) -> None:
for name, attribute in sorted((node.get("attributes") or {}).items()):
if needle in name.lower():
where = f"{path}.{name}" if path else name
found.append(
f" {where} ({_type(attribute.get('type'))}, {_use(attribute)})"
)
for name, nested in sorted((node.get("block_types") or {}).items()):
where = f"{path}.{name}" if path else name
if needle in name.lower():
found.append(f" {where} {{ }} ({_block_note(nested)})")
search(nested.get("block") or {}, where)
search(root, "")
if not found:
return (
f"{title} has no argument or block named like {wanted!r}. It cannot be "
"set on this resource type."
)
return f"{title}: {wanted!r} is found at\n" + "\n".join(found)
_FILES_SCHEMA: dict[str, Any] = {
**FILES_SCHEMA,
"description": "Terraform files in one directory, e.g. terraform/main.tf",
@@ -244,19 +440,53 @@ _FILES_SCHEMA: dict[str, Any] = {
def _with_files(
method: Callable[..., Awaitable[ToolResult]],
working_copy: MutableMapping[str, str] | None = None,
) -> Callable[..., Awaitable[ToolResult]]:
"""Adapt a runner method to take the FILES_SCHEMA argument."""
"""Adapt a runner method to take the FILES_SCHEMA argument.
async def handler(files: Any, **kwargs: Any) -> ToolResult:
With a working copy, the files are the Terraform files in it and the argument is
optional: files passed anyway are written to the working copy first.
"""
async def handler(files: Any = None, **kwargs: Any) -> ToolResult:
try:
converted = files_from_argument(files)
except TypeError as exc:
return ToolResult(f"Refused: files: {exc}")
return await method(converted, **kwargs)
if working_copy is None:
return await method(converted, **kwargs)
working_copy.update(converted)
return await method(_terraform_files(working_copy), **kwargs)
return handler
def _terraform_files(files: Mapping[str, str]) -> dict[str, str]:
return {
name: content
for name, content in files.items()
if name.endswith(_TERRAFORM_SUFFIXES)
}
def _fmt_in_place(
runner: TerraformRunner, working_copy: MutableMapping[str, str]
) -> Callable[..., Awaitable[ToolResult]]:
async def format_working_copy(files: Mapping[str, str]) -> ToolResult:
changed = await runner.formatted(files)
if isinstance(changed, ToolResult):
return changed
working_copy.update(changed)
if not changed:
return ToolResult("terraform fmt: all files are already formatted.")
return ToolResult(
f"terraform fmt reformatted {len(changed)} file(s) in your working "
f"copy: {', '.join(sorted(changed))}."
)
return _with_files(format_working_copy, working_copy)
LOCAL_TOOLS: dict[str, tuple[str, str]] = {
"mermaid_check": ("general", "Check a Mermaid diagram's basic structure."),
"terraform_fmt": ("general", "Format Terraform files and return any that changed."),
@@ -268,10 +498,48 @@ LOCAL_TOOLS: dict[str, tuple[str, str]] = {
"cloud_read",
"Preview Terraform against the live cloud environment with a plan. Never applies.",
),
"terraform_schema": (
"general",
(
"Look up what a Terraform resource type accepts, from the provider's own "
"schema: its arguments and blocks, one block in full with `block`, or "
"whether an argument exists anywhere in it with `find`."
),
),
}
_ON_WORKING_COPY = " Runs on your working copy; pass no files."
def local_tools(
names: Iterable[str],
runner: TerraformRunner,
working_copy: MutableMapping[str, str] | None = None,
) -> list[ToolSpec]:
"""The named tools. With a `working_copy`, the Terraform checks run on it."""
files_required = [] if working_copy is not None else ["files"]
suffix = _ON_WORKING_COPY if working_copy is not None else ""
def checks(name: str, method: Callable[..., Awaitable[ToolResult]]) -> ToolSpec:
properties: dict[str, Any] = {"files": _FILES_SCHEMA}
if name == "terraform_plan":
properties["variables"] = {
"type": "object",
"additionalProperties": {"type": "string"},
}
tier, description = LOCAL_TOOLS[name]
return ToolSpec(
name,
description + suffix,
{
"type": "object",
"properties": properties,
"required": files_required,
},
method,
tier=tier,
)
def local_tools(names: Iterable[str], runner: TerraformRunner) -> list[ToolSpec]:
builders = {
"mermaid_check": lambda: ToolSpec(
"mermaid_check",
@@ -284,44 +552,41 @@ def local_tools(names: Iterable[str], runner: TerraformRunner) -> list[ToolSpec]
mermaid_check,
tier="general",
),
"terraform_fmt": lambda: ToolSpec(
"terraform_fmt": lambda: checks(
"terraform_fmt",
LOCAL_TOOLS["terraform_fmt"][1],
{
"type": "object",
"properties": {"files": _FILES_SCHEMA},
"required": ["files"],
},
_with_files(runner.fmt),
tier="general",
_fmt_in_place(runner, working_copy)
if working_copy is not None
else _with_files(runner.fmt),
),
"terraform_validate": lambda: ToolSpec(
"terraform_validate",
LOCAL_TOOLS["terraform_validate"][1],
{
"type": "object",
"properties": {"files": _FILES_SCHEMA},
"required": ["files"],
},
_with_files(runner.validate),
tier="general",
"terraform_validate": lambda: checks(
"terraform_validate", _with_files(runner.validate, working_copy)
),
"terraform_plan": lambda: ToolSpec(
"terraform_plan",
LOCAL_TOOLS["terraform_plan"][1],
"terraform_plan": lambda: checks(
"terraform_plan", _with_files(runner.plan, working_copy)
),
"terraform_schema": lambda: ToolSpec(
"terraform_schema",
LOCAL_TOOLS["terraform_schema"][1],
{
"type": "object",
"properties": {
"files": _FILES_SCHEMA,
"variables": {
"type": "object",
"additionalProperties": {"type": "string"},
"resource_type": {
"type": "string",
"description": "For example google_cloud_run_v2_service",
},
"block": {
"type": "string",
"description": "A nested block, such as template.containers",
},
"find": {
"type": "string",
"description": "An argument or block name to look for",
},
},
"required": ["files"],
"required": ["resource_type"],
},
_with_files(runner.plan),
tier="cloud_read",
runner.schema,
tier="general",
),
}
return [builders[name]() for name in names]