Files
gcp_solution_architecture_a…/app/tools/local.py
Jonathan Boniface f1f6f1b83a feat: load the env file at start-up, trim schema lookups, remove legacy samples
Start-up
- Export the env file's variables for other libraries (the Cloud provider's
  client) into the process environment; the app's own settings and secrets
  are never exported, and a value already in the environment wins.
- Use GOOGLE_GENAI_USE_ENTERPRISE in place of the deprecated
  GOOGLE_GENAI_USE_VERTEXAI.

Target architect
- Look each distinct resource type up once instead of once per setting: in
  the run of 2026-10-07 it made 39 lookups for 8 types and exhausted its
  budget. terraform_schema's find takes several names; budget 30 to 45.
- A cut-off schema outline says so.

Removed, as leftovers of the original pipeline that nothing read
- terraform/ (no longer passed terraform validate), scripts/validate.sh,
  tests/test_package.py and the CI steps that ran on them.
- The sample outputs under docs/ and requirements.yaml.
- CI now pins Terraform 1.14.3, as the container image does.

Plans record the case 01 run: completed at 100% in 18 minutes with no
Revision, against 60 minutes and warnings before.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-07 20:06:58 +01:00

608 lines
23 KiB
Python

"""Local tools: a Mermaid syntax check and Terraform checks in a scratch directory.
Terraform runs only fmt, init -backend=false, validate, plan and providers schema. No
command that changes cloud resources is ever constructed.
A Specialist with a working copy (app/tools/workspace.py) runs the checks on it and
passes no files; `terraform_fmt` formats the working copy in place. That is what
keeps a fix cheap: the model changes a line and validates, without sending every file
again on each call.
"""
from __future__ import annotations
import asyncio
import difflib
import json
import os
import re
import tempfile
from collections.abc import Awaitable, Callable, Iterable, Mapping, MutableMapping
from dataclasses import dataclass, field
from pathlib import Path, PurePosixPath
from typing import Any
from app.adk.runtime import FILES_SCHEMA, ToolResult, ToolSpec, files_from_argument
_MAX_OUTPUT = 8_000
_TERRAFORM_SUFFIXES = (".tf", ".tfvars", ".tf.json")
_VARIABLE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
_MERMAID_TYPES = (
"flowchart",
"graph",
"sequenceDiagram",
"classDiagram",
"stateDiagram",
"stateDiagram-v2",
"erDiagram",
"C4Context",
"C4Container",
"C4Component",
"C4Deployment",
"architecture-beta",
"block-beta",
)
_BRACKETS = {"]": "[", ")": "(", "}": "{"}
async def mermaid_check(diagram: str) -> ToolResult:
"""A basic structural check; it does not render the diagram."""
problems = []
lines = [
line.strip()
for line in diagram.splitlines()
if line.strip() and not line.strip().startswith("%%")
]
if not diagram.strip():
problems.append("the diagram is empty")
elif not lines or lines[0].split()[0] not in _MERMAID_TYPES:
problems.append(
"the first line must declare a diagram type, such as 'flowchart TD'"
)
for number, line in enumerate(diagram.splitlines(), start=1):
stack: list[str] = []
in_quotes = False
for char in line:
if char == '"':
in_quotes = not in_quotes
elif in_quotes:
continue
elif char in "[({":
stack.append(char)
elif char in _BRACKETS and (not stack or stack.pop() != _BRACKETS[char]):
stack.append("!")
break
if stack:
problems.append(f"Unbalanced brackets on line {number}: {line.strip()}")
if problems:
return ToolResult("Mermaid check failed: " + "; ".join(problems))
return ToolResult("Mermaid check passed (structure only; not rendered).")
@dataclass
class TerraformRunner:
binary: str = "terraform"
timeout_seconds: float = 300
# Providers are downloaded here once and reused by every later check. Without it,
# each validate or plan downloads its providers again into a scratch directory.
plugin_cache_dir: str | None = None
# The providers the schema lookup answers for, as the provider bundle pins them:
# {local name: {"source": ..., "version": ...}}.
providers: Mapping[str, Mapping[str, str]] = field(default_factory=dict)
# Terraform's plugin cache is not safe for concurrent installs.
_init_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False)
_schema_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False)
# resource type -> (provider label, the resource's schema block)
_schemas: dict[str, tuple[str, dict[str, Any]]] | None = field(
default=None, repr=False
)
async def formatted(self, files: Mapping[str, str]) -> dict[str, str] | ToolResult:
"""The files formatting changes, with their new content; or why it failed."""
refusal = _check_files(files)
if refusal:
return refusal
with tempfile.TemporaryDirectory(prefix="tf-fmt-") as tmp:
root = _write(Path(tmp), files)
code, output = await self._run(
[self.binary, "fmt", "-recursive", "-list=true"], root
)
if code != 0:
return ToolResult(f"terraform fmt failed:\n{output}")
return {
name: (root / name).read_text()
for name in files
if (root / name).read_text() != files[name]
}
async def fmt(self, files: Mapping[str, str]) -> ToolResult:
changed = await self.formatted(files)
if isinstance(changed, ToolResult):
return changed
if not changed:
return ToolResult("terraform fmt: all files are already formatted.")
formatted = "\n\n".join(
f"--- {name}\n{content}" for name, content in changed.items()
)
return ToolResult(
_clip(f"terraform fmt reformatted {len(changed)} file(s):\n\n{formatted}")
)
async def validate(self, files: Mapping[str, str]) -> ToolResult:
refusal = _check_files(files)
if refusal:
return refusal
with tempfile.TemporaryDirectory(prefix="tf-validate-") as tmp:
root = _write(Path(tmp), files) / _module_dir(files)
failure = await self._init(root)
if failure:
return failure
code, output = await self._run([self.binary, "validate", "-no-color"], root)
verdict = "passed" if code == 0 else "failed"
return ToolResult(_clip(f"terraform validate {verdict}:\n{output}"))
async def plan(
self, files: Mapping[str, str], variables: Mapping[str, str] | None = None
) -> ToolResult:
refusal = _check_files(files)
if refusal:
return refusal
variables = dict(variables or {})
bad = [name for name in variables if not _VARIABLE_NAME.match(name)]
if bad:
return ToolResult(f"Refused: invalid variable names {', '.join(bad)}")
with tempfile.TemporaryDirectory(prefix="tf-plan-") as tmp:
root = _write(Path(tmp), files) / _module_dir(files)
failure = await self._init(root)
if failure:
return failure
args = [
self.binary,
"plan",
"-no-color",
"-input=false",
"-lock=false",
"-refresh=true",
*(f"-var={name}={value}" for name, value in variables.items()),
]
code, output = await self._run(args, root)
verdict = "completed" if code == 0 else "failed"
return ToolResult(
_clip(f"terraform plan {verdict} (nothing was applied):\n{output}")
)
async def schema(
self, resource_type: str, block: str = "", find: str = ""
) -> ToolResult:
"""What a resource type accepts, from the pinned provider's own schema."""
loaded = await self._load_schemas()
if isinstance(loaded, ToolResult):
return loaded
if resource_type not in loaded:
close = difflib.get_close_matches(resource_type, loaded, n=5, cutoff=0.6)
hint = f" Close matches: {', '.join(close)}." if close else ""
return ToolResult(
f"No resource type {resource_type} in {self._provider_labels()}.{hint}"
)
label, root = loaded[resource_type]
title = f"{resource_type} ({label})"
if find:
# Several names in one call: one answer each, and one model request.
names = [name for name in re.split(r"[,\s]+", find) if name]
return ToolResult(
_clip("\n".join(_find_in_schema(title, root, name) for name in names))
)
node = root
for step in [part for part in block.split(".") if part]:
nested = node.get("block_types", {}).get(step)
if nested is None:
inside = ", ".join(sorted(node.get("block_types", {}))) or "none"
return ToolResult(
f"{title} has no block {block!r}: {step!r} is not a block there. "
f"Blocks at that level: {inside}."
)
node = nested["block"]
heading = f"{title}, block {block}" if block else title
rendered = _render_schema(heading, node)
if len(rendered) > _MAX_OUTPUT:
rendered = (
rendered[:_MAX_OUTPUT]
+ "\n[cut short: use `block` or `find` for what is missing]"
)
return ToolResult(rendered)
def _provider_labels(self) -> str:
return (
", ".join(
f"{pin['source']} {pin['version']}" for pin in self.providers.values()
)
or "no pinned provider"
)
async def _load_schemas(self) -> dict[str, tuple[str, dict[str, Any]]] | ToolResult:
async with self._schema_lock:
if self._schemas is not None:
return self._schemas
if not self.providers:
return ToolResult(
"Schema lookup is unavailable: the Cloud provider bundle pins no "
"Terraform provider."
)
required = "\n".join(
f' {name} = {{ source = "{pin["source"]}", '
f'version = "{pin["version"]}" }}'
for name, pin in self.providers.items()
)
with tempfile.TemporaryDirectory(prefix="tf-schema-") as tmp:
root = Path(tmp)
(root / "versions.tf").write_text(
f"terraform {{\n required_providers {{\n{required}\n }}\n}}\n"
)
failure = await self._init(root)
if failure:
return failure
code, output = await self._run(
[self.binary, "providers", "schema", "-json"], root
)
if code != 0:
return ToolResult(
_clip(f"terraform providers schema failed:\n{output}")
)
try:
data = json.loads(output)
except ValueError:
return ToolResult("terraform providers schema returned no JSON.")
labels = {pin["source"]: pin["version"] for pin in self.providers.values()}
schemas: dict[str, tuple[str, dict[str, Any]]] = {}
for address, provider in (data.get("provider_schemas") or {}).items():
source = address.split("/", 1)[-1] # drop the registry host
label = f"{source} {labels.get(source, '')}".strip()
for name, resource in (provider.get("resource_schemas") or {}).items():
schemas[name] = (label, resource.get("block") or {})
self._schemas = schemas
return schemas
async def _init(self, root: Path) -> ToolResult | None:
command = [self.binary, "init", "-backend=false", "-input=false", "-no-color"]
if self.plugin_cache_dir:
async with self._init_lock:
code, output = await self._run(command, root)
else:
code, output = await self._run(command, root)
if code != 0:
return ToolResult(_clip(f"terraform init failed:\n{output}"))
return None
async def _run(
self, args: list[str], cwd: Path, env: Mapping[str, str] | None = None
) -> tuple[int, str]:
environment = {**os.environ, "TF_IN_AUTOMATION": "1", **(env or {})}
if self.plugin_cache_dir:
cache = Path(self.plugin_cache_dir).expanduser().resolve()
cache.mkdir(parents=True, exist_ok=True)
environment["TF_PLUGIN_CACHE_DIR"] = str(cache)
# Each check runs in a scratch directory with no dependency lock file.
# Without this, Terraform downloads a cached provider again to verify it.
environment["TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE"] = "true"
try:
process = await asyncio.create_subprocess_exec(
*args,
cwd=cwd,
env=environment,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.STDOUT,
)
except FileNotFoundError:
return 127, f"{self.binary} is not installed or not on PATH."
try:
stdout, _ = await asyncio.wait_for(
process.communicate(), timeout=self.timeout_seconds
)
except TimeoutError:
process.kill()
await process.wait()
return 124, f"{' '.join(args[:2])} timed out after {self.timeout_seconds}s."
return process.returncode or 0, stdout.decode(errors="replace")
def _check_files(files: Mapping[str, str]) -> ToolResult | None:
if not files:
return ToolResult("Refused: provide at least one Terraform file.")
bad = []
for name in files:
path = PurePosixPath(name)
if (
path.is_absolute()
or ".." in path.parts
or not name.endswith(_TERRAFORM_SUFFIXES)
):
bad.append(name)
if bad:
return ToolResult(
"Refused: file names must be relative .tf, .tf.json or .tfvars paths "
f"without '..': {', '.join(bad)}"
)
directories = {str(PurePosixPath(name).parent) for name in files}
if len(directories) > 1:
return ToolResult(
"Refused: put all files in one directory (one root module), for example "
f"terraform/main.tf and terraform/variables.tf; got {', '.join(sorted(directories))}"
)
return None
def _module_dir(files: Mapping[str, str]) -> str:
"""The one directory holding the files; Terraform runs there."""
return str(PurePosixPath(next(iter(files))).parent)
def _write(root: Path, files: Mapping[str, str]) -> Path:
for name, content in files.items():
target = root / name
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(content, encoding="utf-8")
return root
def _clip(text: str) -> str:
return text if len(text) <= _MAX_OUTPUT else text[:_MAX_OUTPUT] + "\n[truncated]"
def _type(value: Any) -> str:
if isinstance(value, str):
return value
if isinstance(value, list) and len(value) == 2 and value[0] != "object":
return f"{value[0]} of {_type(value[1])}"
return "object"
def _use(attribute: Mapping[str, Any]) -> str:
if attribute.get("required"):
return "required"
return "optional" if attribute.get("optional") else "read-only"
def _arguments(node: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]:
"""The arguments code may set: everything but the read-only ones."""
return {
name: attribute
for name, attribute in sorted((node.get("attributes") or {}).items())
if _use(attribute) != "read-only"
}
def _block_note(nested: Mapping[str, Any]) -> str:
notes = [str(nested.get("nesting_mode", "block"))]
if nested.get("min_items"):
notes.append(f"at least {nested['min_items']}")
if nested.get("max_items"):
notes.append(f"at most {nested['max_items']}")
return ", ".join(notes)
def _render_schema(heading: str, node: Mapping[str, Any]) -> str:
"""One level in full, and the blocks beneath it as an outline of names."""
lines = [heading, "", "Arguments:"]
arguments = _arguments(node)
for name, attribute in arguments.items():
described = " ".join(str(attribute.get("description") or "").split())[:160]
deprecated = " DEPRECATED" if attribute.get("deprecated") else ""
lines.append(
f" {name} ({_type(attribute.get('type'))}, {_use(attribute)})"
f"{deprecated}" + (f": {described}" if described else "")
)
if not arguments:
lines.append(" none")
def outline(parent: Mapping[str, Any], depth: int) -> None:
for name, nested in sorted((parent.get("block_types") or {}).items()):
inner = nested.get("block") or {}
names = ", ".join(
f"{argument}*" if attribute.get("required") else argument
for argument, attribute in _arguments(inner).items()
)
lines.append(
f"{' ' * depth}{name} {{ }} ({_block_note(nested)}): {names or '-'}"
)
outline(inner, depth + 1)
lines += ["", "Blocks (* marks a required argument):"]
before = len(lines)
outline(node, 1)
if len(lines) == before:
lines.append(" none")
lines += ["", "Pass `block`, such as a.b, for one block's arguments in full."]
return "\n".join(lines)
def _find_in_schema(title: str, root: Mapping[str, Any], wanted: str) -> str:
"""Every argument or block whose name contains `wanted`, with where it sits."""
needle = wanted.lower()
found: list[str] = []
def search(node: Mapping[str, Any], path: str) -> None:
for name, attribute in sorted((node.get("attributes") or {}).items()):
if needle in name.lower():
where = f"{path}.{name}" if path else name
found.append(
f" {where} ({_type(attribute.get('type'))}, {_use(attribute)})"
)
for name, nested in sorted((node.get("block_types") or {}).items()):
where = f"{path}.{name}" if path else name
if needle in name.lower():
found.append(f" {where} {{ }} ({_block_note(nested)})")
search(nested.get("block") or {}, where)
search(root, "")
if not found:
return (
f"{title} has no argument or block named like {wanted!r}. It cannot be "
"set on this resource type."
)
return f"{title}: {wanted!r} is found at\n" + "\n".join(found)
_FILES_SCHEMA: dict[str, Any] = {
**FILES_SCHEMA,
"description": "Terraform files in one directory, e.g. terraform/main.tf",
}
def _with_files(
method: Callable[..., Awaitable[ToolResult]],
working_copy: MutableMapping[str, str] | None = None,
) -> Callable[..., Awaitable[ToolResult]]:
"""Adapt a runner method to take the FILES_SCHEMA argument.
With a working copy, the files are the Terraform files in it and the argument is
optional: files passed anyway are written to the working copy first.
"""
async def handler(files: Any = None, **kwargs: Any) -> ToolResult:
try:
converted = files_from_argument(files)
except TypeError as exc:
return ToolResult(f"Refused: files: {exc}")
if working_copy is None:
return await method(converted, **kwargs)
working_copy.update(converted)
return await method(_terraform_files(working_copy), **kwargs)
return handler
def _terraform_files(files: Mapping[str, str]) -> dict[str, str]:
return {
name: content
for name, content in files.items()
if name.endswith(_TERRAFORM_SUFFIXES)
}
def _fmt_in_place(
runner: TerraformRunner, working_copy: MutableMapping[str, str]
) -> Callable[..., Awaitable[ToolResult]]:
async def format_working_copy(files: Mapping[str, str]) -> ToolResult:
changed = await runner.formatted(files)
if isinstance(changed, ToolResult):
return changed
working_copy.update(changed)
if not changed:
return ToolResult("terraform fmt: all files are already formatted.")
return ToolResult(
f"terraform fmt reformatted {len(changed)} file(s) in your working "
f"copy: {', '.join(sorted(changed))}."
)
return _with_files(format_working_copy, working_copy)
LOCAL_TOOLS: dict[str, tuple[str, str]] = {
"mermaid_check": ("general", "Check a Mermaid diagram's basic structure."),
"terraform_fmt": ("general", "Format Terraform files and return any that changed."),
"terraform_validate": (
"general",
"Validate Terraform files offline (init -backend=false, validate).",
),
"terraform_plan": (
"cloud_read",
"Preview Terraform against the live cloud environment with a plan. Never applies.",
),
"terraform_schema": (
"general",
(
"Look up what a Terraform resource type accepts, from the provider's own "
"schema. With `resource_type` alone: every argument and block it has, "
"which is enough to see whether a setting exists. `block` gives one "
"block in full. `find` says where named settings sit, or that the "
"resource type has none."
),
),
}
_ON_WORKING_COPY = " Runs on your working copy; pass no files."
def local_tools(
names: Iterable[str],
runner: TerraformRunner,
working_copy: MutableMapping[str, str] | None = None,
) -> list[ToolSpec]:
"""The named tools. With a `working_copy`, the Terraform checks run on it."""
files_required = [] if working_copy is not None else ["files"]
suffix = _ON_WORKING_COPY if working_copy is not None else ""
def checks(name: str, method: Callable[..., Awaitable[ToolResult]]) -> ToolSpec:
properties: dict[str, Any] = {"files": _FILES_SCHEMA}
if name == "terraform_plan":
properties["variables"] = {
"type": "object",
"additionalProperties": {"type": "string"},
}
tier, description = LOCAL_TOOLS[name]
return ToolSpec(
name,
description + suffix,
{
"type": "object",
"properties": properties,
"required": files_required,
},
method,
tier=tier,
)
builders = {
"mermaid_check": lambda: ToolSpec(
"mermaid_check",
LOCAL_TOOLS["mermaid_check"][1],
{
"type": "object",
"properties": {"diagram": {"type": "string"}},
"required": ["diagram"],
},
mermaid_check,
tier="general",
),
"terraform_fmt": lambda: checks(
"terraform_fmt",
_fmt_in_place(runner, working_copy)
if working_copy is not None
else _with_files(runner.fmt),
),
"terraform_validate": lambda: checks(
"terraform_validate", _with_files(runner.validate, working_copy)
),
"terraform_plan": lambda: checks(
"terraform_plan", _with_files(runner.plan, working_copy)
),
"terraform_schema": lambda: ToolSpec(
"terraform_schema",
LOCAL_TOOLS["terraform_schema"][1],
{
"type": "object",
"properties": {
"resource_type": {
"type": "string",
"description": "For example google_cloud_run_v2_service",
},
"block": {
"type": "string",
"description": "A nested block, such as template.containers",
},
"find": {
"type": "string",
"description": (
"Argument or block names to look for; separate several "
"with commas to check them in one call"
),
},
},
"required": ["resource_type"],
},
runner.schema,
tier="general",
),
}
return [builders[name]() for name in names]