"""Local tools: a Mermaid syntax check and Terraform checks in a scratch directory. Terraform runs only fmt, init -backend=false, validate, plan and providers schema. No command that changes cloud resources is ever constructed. A Specialist with a working copy (app/tools/workspace.py) runs the checks on it and passes no files; `terraform_fmt` formats the working copy in place. That is what keeps a fix cheap: the model changes a line and validates, without sending every file again on each call. """ from __future__ import annotations import asyncio import difflib import json import os import re import tempfile from collections.abc import ( Awaitable, Callable, Iterable, Mapping, MutableMapping, Sequence, ) from dataclasses import dataclass, field from pathlib import Path, PurePosixPath from typing import Any from execution_engine.adk.runtime import ( FILES_SCHEMA, ToolResult, ToolSpec, files_from_argument, ) from tests.reference_workflow.modules import ( ModuleLookup, allowed_namespaces, check_module_sources, ) _MAX_OUTPUT = 8_000 _MAX_OUTLINE = 4_000 # one resource type, when several are looked up together _MAX_TYPES = 20 # resource types in one lookup _TERRAFORM_SUFFIXES = (".tf", ".tfvars", ".tf.json") _VARIABLE_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$") _MERMAID_TYPES = ( "flowchart", "graph", "sequenceDiagram", "classDiagram", "stateDiagram", "stateDiagram-v2", "erDiagram", "C4Context", "C4Container", "C4Component", "C4Deployment", "architecture-beta", "block-beta", ) _BRACKETS = {"]": "[", ")": "(", "}": "{"} async def mermaid_check(diagram: str) -> ToolResult: """A basic structural check; it does not render the diagram.""" problems = [] lines = [ line.strip() for line in diagram.splitlines() if line.strip() and not line.strip().startswith("%%") ] if not diagram.strip(): problems.append("the diagram is empty") elif not lines or lines[0].split()[0] not in _MERMAID_TYPES: problems.append( "the first line must declare a diagram type, such as 'flowchart TD'" ) for number, line in enumerate(diagram.splitlines(), start=1): stack: list[str] = [] in_quotes = False for char in line: if char == '"': in_quotes = not in_quotes elif in_quotes: continue elif char in "[({": stack.append(char) elif char in _BRACKETS and (not stack or stack.pop() != _BRACKETS[char]): stack.append("!") break if stack: problems.append(f"Unbalanced brackets on line {number}: {line.strip()}") if problems: return ToolResult("Mermaid check failed: " + "; ".join(problems)) return ToolResult("Mermaid check passed (structure only; not rendered).") @dataclass class TerraformRunner: binary: str = "terraform" timeout_seconds: float = 300 # Providers are downloaded here once and reused by every later check. Without it, # each validate or plan downloads its providers again into a scratch directory. plugin_cache_dir: str | None = None # The providers the schema lookup answers for, as the provider bundle pins them: # {local name: {"source": ..., "version": ...}}. providers: Mapping[str, Mapping[str, str]] = field(default_factory=dict) # Registry namespaces a `module` may come from, as the provider bundle lists # them. `terraform init` downloads and evaluates whatever a source names, so # with none listed every module call is refused. module_namespaces: tuple[str, ...] = () # Terraform's plugin cache is not safe for concurrent installs. _init_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False) _schema_lock: asyncio.Lock = field(default_factory=asyncio.Lock, repr=False) # resource type -> (provider label, the resource's schema block) _schemas: dict[str, tuple[str, dict[str, Any]]] | None = field( default=None, repr=False ) async def formatted(self, files: Mapping[str, str]) -> dict[str, str] | ToolResult: """The files formatting changes, with their new content; or why it failed.""" refusal = _check_files(files) if refusal: return refusal with tempfile.TemporaryDirectory(prefix="tf-fmt-") as tmp: root = _write(Path(tmp), files) code, output = await self._run( [self.binary, "fmt", "-recursive", "-list=true"], root ) if code != 0: return ToolResult(f"terraform fmt failed:\n{output}") return { name: (root / name).read_text() for name in files if (root / name).read_text() != files[name] } async def fmt(self, files: Mapping[str, str]) -> ToolResult: changed = await self.formatted(files) if isinstance(changed, ToolResult): return changed if not changed: return ToolResult("terraform fmt: all files are already formatted.") formatted = "\n\n".join( f"--- {name}\n{content}" for name, content in changed.items() ) return ToolResult( _clip(f"terraform fmt reformatted {len(changed)} file(s):\n\n{formatted}") ) async def validate(self, files: Mapping[str, str]) -> ToolResult: refusal = ( _check_files(files) or check_providers(files, self.providers) or check_module_sources(files, allowed_namespaces(self.module_namespaces)) ) if refusal: return refusal with tempfile.TemporaryDirectory(prefix="tf-validate-") as tmp: root = _write(Path(tmp), files) / _module_dir(files) failure = await self._init(root) if failure: return failure code, output = await self._run([self.binary, "validate", "-no-color"], root) verdict = "passed" if code == 0 else "failed" return ToolResult(_clip(f"terraform validate {verdict}:\n{output}")) async def plan( self, files: Mapping[str, str], variables: Mapping[str, str] | None = None ) -> ToolResult: refusal = ( _check_files(files) or check_providers(files, self.providers) or check_module_sources(files, allowed_namespaces(self.module_namespaces)) ) if refusal: return refusal variables = dict(variables or {}) bad = [name for name in variables if not _VARIABLE_NAME.match(name)] if bad: return ToolResult(f"Refused: invalid variable names {', '.join(bad)}") with tempfile.TemporaryDirectory(prefix="tf-plan-") as tmp: root = _write(Path(tmp), files) / _module_dir(files) failure = await self._init(root) if failure: return failure args = [ self.binary, "plan", "-no-color", "-input=false", "-lock=false", "-refresh=true", *(f"-var={name}={value}" for name, value in variables.items()), ] code, output = await self._run(args, root) verdict = "completed" if code == 0 else "failed" return ToolResult( _clip(f"terraform plan {verdict} (nothing was applied):\n{output}") ) async def schema( self, resource_type: str = "", block: str = "", find: str = "", resource_types: Sequence[str] = (), ) -> ToolResult: """What a resource type accepts, from the pinned provider's own schema. Several resource types at once come back as outlines: every argument and block each accepts, by name. That is one call for a whole design, where one call per type or per setting is what used up a Specialist's budget. """ loaded = await self._load_schemas() if isinstance(loaded, ToolResult): return loaded wanted = list( dict.fromkeys( name for name in [*resource_types, *re.split(r"[,\s]+", resource_type)] if name ) ) if not wanted: return ToolResult("Refused: name at least one resource type.") if len(wanted) > 1: parts = [] for name in wanted[:_MAX_TYPES]: if name not in loaded: close = difflib.get_close_matches(name, loaded, n=3, cutoff=0.6) hint = f" Close matches: {', '.join(close)}." if close else "" parts.append( f"{name}: not a resource type of {self._provider_labels()}." f"{hint}" ) continue parts.append(_outline_only(name, loaded[name][1])) if len(wanted) > _MAX_TYPES: parts.append( f"[{len(wanted) - _MAX_TYPES} more not shown: ask for at most " f"{_MAX_TYPES} at a time]" ) return ToolResult( f"{self._provider_labels()}. * marks a required argument.\n\n" + "\n\n".join(parts) ) resource_type = wanted[0] if resource_type not in loaded: close = difflib.get_close_matches(resource_type, loaded, n=5, cutoff=0.6) hint = f" Close matches: {', '.join(close)}." if close else "" return ToolResult( f"No resource type {resource_type} in {self._provider_labels()}.{hint}" ) label, root = loaded[resource_type] title = f"{resource_type} ({label})" if find: # Several names in one call: one answer each, and one model request. names = [name for name in re.split(r"[,\s]+", find) if name] return ToolResult( _clip("\n".join(_find_in_schema(title, root, name) for name in names)) ) node = root for step in [part for part in block.split(".") if part]: nested = node.get("block_types", {}).get(step) if nested is None: inside = ", ".join(sorted(node.get("block_types", {}))) or "none" return ToolResult( f"{title} has no block {block!r}: {step!r} is not a block there. " f"Blocks at that level: {inside}." ) node = nested["block"] heading = f"{title}, block {block}" if block else title rendered = _render_schema(heading, node) if len(rendered) > _MAX_OUTPUT: rendered = ( rendered[:_MAX_OUTPUT] + "\n[cut short: use `block` or `find` for what is missing]" ) return ToolResult(rendered) def _provider_labels(self) -> str: return ( ", ".join( f"{pin['source']} {pin['version']}" for pin in self.providers.values() ) or "no pinned provider" ) async def _load_schemas(self) -> dict[str, tuple[str, dict[str, Any]]] | ToolResult: async with self._schema_lock: if self._schemas is not None: return self._schemas if not self.providers: return ToolResult( "Schema lookup is unavailable: the Cloud provider bundle pins no " "Terraform provider." ) required = "\n".join( f' {name} = {{ source = "{pin["source"]}", ' f'version = "{pin["version"]}" }}' for name, pin in self.providers.items() ) with tempfile.TemporaryDirectory(prefix="tf-schema-") as tmp: root = Path(tmp) (root / "versions.tf").write_text( f"terraform {{\n required_providers {{\n{required}\n }}\n}}\n" ) failure = await self._init(root) if failure: return failure code, output = await self._run( [self.binary, "providers", "schema", "-json"], root ) if code != 0: return ToolResult( _clip(f"terraform providers schema failed:\n{output}") ) try: data = json.loads(output) except ValueError: return ToolResult("terraform providers schema returned no JSON.") labels = {pin["source"]: pin["version"] for pin in self.providers.values()} schemas: dict[str, tuple[str, dict[str, Any]]] = {} for address, provider in (data.get("provider_schemas") or {}).items(): source = address.split("/", 1)[-1] # drop the registry host label = f"{source} {labels.get(source, '')}".strip() for name, resource in (provider.get("resource_schemas") or {}).items(): schemas[name] = (label, resource.get("block") or {}) self._schemas = schemas return schemas async def _init(self, root: Path) -> ToolResult | None: command = [self.binary, "init", "-backend=false", "-input=false", "-no-color"] if self.plugin_cache_dir: async with self._init_lock: code, output = await self._run(command, root) else: code, output = await self._run(command, root) if code != 0: return ToolResult(_clip(f"terraform init failed:\n{output}")) return None async def _run( self, args: list[str], cwd: Path, env: Mapping[str, str] | None = None ) -> tuple[int, str]: environment = {**os.environ, "TF_IN_AUTOMATION": "1", **(env or {})} if self.plugin_cache_dir: cache = Path(self.plugin_cache_dir).expanduser().resolve() cache.mkdir(parents=True, exist_ok=True) environment["TF_PLUGIN_CACHE_DIR"] = str(cache) # Each check runs in a scratch directory with no dependency lock file. # Without this, Terraform downloads a cached provider again to verify it. environment["TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE"] = "true" try: process = await asyncio.create_subprocess_exec( *args, cwd=cwd, env=environment, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.STDOUT, ) except FileNotFoundError: return 127, f"{self.binary} is not installed or not on PATH." try: stdout, _ = await asyncio.wait_for( process.communicate(), timeout=self.timeout_seconds ) except TimeoutError: process.kill() await process.wait() return 124, f"{' '.join(args[:2])} timed out after {self.timeout_seconds}s." return process.returncode or 0, stdout.decode(errors="replace") _BLOCK_TYPE = re.compile( r'^\s*(?:resource|data)\s+"([a-z0-9]+)_[a-z0-9_]*"', re.MULTILINE ) _PROVIDER_SOURCE = re.compile(r'^\s*source\s*=\s*"([\w.-]+/[\w.-]+)"\s*$', re.MULTILINE) # Built into Terraform itself: terraform_data and the terraform_remote_state source. _BUILT_IN = frozenset({"terraform"}) def check_providers( files: Mapping[str, str], pinned: Mapping[str, Mapping[str, str]] ) -> ToolResult | None: """Refuse code that uses a provider the bundle does not pin. `terraform init` downloads whatever provider the code names, at whatever version is current. A design with a source repository in it brought the GitHub provider into generated code that way on 2026-10-11. With nothing pinned there is nothing to hold code to, and it passes. """ if not pinned: return None names = set(pinned) | _BUILT_IN sources = {pin["source"] for pin in pinned.values()} code = "\n".join(text for name, text in files.items() if name.endswith(".tf")) foreign = sorted({m.group(1) for m in _BLOCK_TYPE.finditer(code)} - names) undeclared = sorted({m.group(1) for m in _PROVIDER_SOURCE.finditer(code)} - sources) if not foreign and not undeclared: return None found = [f"resources of the {name} provider" for name in foreign] found += [f"the provider {source}" for source in undeclared] return ToolResult( "Refused: the code uses " + ", ".join(found) + ". Only these providers may be used: " + ", ".join(sorted(sources)) + ". Write resources of those alone, and describe anything outside them in " "the deployment guidance as a step the user takes." ) def _check_files(files: Mapping[str, str]) -> ToolResult | None: if not files: return ToolResult("Refused: provide at least one Terraform file.") bad = [] for name in files: path = PurePosixPath(name) if ( path.is_absolute() or ".." in path.parts or not name.endswith(_TERRAFORM_SUFFIXES) ): bad.append(name) if bad: return ToolResult( "Refused: file names must be relative .tf, .tf.json or .tfvars paths " f"without '..': {', '.join(bad)}" ) directories = {str(PurePosixPath(name).parent) for name in files} if len(directories) > 1: return ToolResult( "Refused: put all files in one directory (one root module), for example " f"terraform/main.tf and terraform/variables.tf; got {', '.join(sorted(directories))}" ) return None def _module_dir(files: Mapping[str, str]) -> str: """The one directory holding the files; Terraform runs there.""" return str(PurePosixPath(next(iter(files))).parent) def _write(root: Path, files: Mapping[str, str]) -> Path: for name, content in files.items(): target = root / name target.parent.mkdir(parents=True, exist_ok=True) target.write_text(content, encoding="utf-8") return root def _clip(text: str) -> str: return text if len(text) <= _MAX_OUTPUT else text[:_MAX_OUTPUT] + "\n[truncated]" def _type(value: Any) -> str: if isinstance(value, str): return value if isinstance(value, list) and len(value) == 2 and value[0] != "object": return f"{value[0]} of {_type(value[1])}" return "object" def _use(attribute: Mapping[str, Any]) -> str: if attribute.get("required"): return "required" return "optional" if attribute.get("optional") else "read-only" def _arguments(node: Mapping[str, Any]) -> dict[str, Mapping[str, Any]]: """The arguments code may set: everything but the read-only ones.""" return { name: attribute for name, attribute in sorted((node.get("attributes") or {}).items()) if _use(attribute) != "read-only" } def _block_note(nested: Mapping[str, Any]) -> str: notes = [str(nested.get("nesting_mode", "block"))] if nested.get("min_items"): notes.append(f"at least {nested['min_items']}") if nested.get("max_items"): notes.append(f"at most {nested['max_items']}") return ", ".join(notes) def _render_schema(heading: str, node: Mapping[str, Any]) -> str: """One level in full, and the blocks beneath it as an outline of names.""" lines = [heading, "", "Arguments:"] arguments = _arguments(node) for name, attribute in arguments.items(): described = " ".join(str(attribute.get("description") or "").split())[:160] deprecated = " DEPRECATED" if attribute.get("deprecated") else "" lines.append( f" {name} ({_type(attribute.get('type'))}, {_use(attribute)})" f"{deprecated}" + (f": {described}" if described else "") ) if not arguments: lines.append(" none") def outline(parent: Mapping[str, Any], depth: int) -> None: for name, nested in sorted((parent.get("block_types") or {}).items()): inner = nested.get("block") or {} names = ", ".join( f"{argument}*" if attribute.get("required") else argument for argument, attribute in _arguments(inner).items() ) lines.append( f"{' ' * depth}{name} {{ }} ({_block_note(nested)}): {names or '-'}" ) outline(inner, depth + 1) lines += ["", "Blocks (* marks a required argument):"] before = len(lines) outline(node, 1) if len(lines) == before: lines.append(" none") lines += ["", "Pass `block`, such as a.b, for one block's arguments in full."] return "\n".join(lines) def _outline_only(resource_type: str, root: Mapping[str, Any]) -> str: """One resource type as names alone: its arguments on a line, then every block with its arguments. Enough to see whether a setting exists and where.""" lines = [resource_type, " " + (_names_line(root) or "-")] def outline(parent: Mapping[str, Any], depth: int) -> None: for name, nested in sorted((parent.get("block_types") or {}).items()): inner = nested.get("block") or {} lines.append(f"{' ' * depth}{name} {{ }}: {_names_line(inner) or '-'}") outline(inner, depth + 1) outline(root, 1) text = "\n".join(lines) if len(text) > _MAX_OUTLINE: text = text[:_MAX_OUTLINE] + "\n [cut short: look this type up alone]" return text def _names_line(node: Mapping[str, Any]) -> str: return ", ".join( f"{name}*" if attribute.get("required") else name for name, attribute in _arguments(node).items() ) def _find_in_schema(title: str, root: Mapping[str, Any], wanted: str) -> str: """Every argument or block whose name contains `wanted`, with where it sits.""" needle = wanted.lower() found: list[str] = [] def search(node: Mapping[str, Any], path: str) -> None: for name, attribute in sorted((node.get("attributes") or {}).items()): if needle in name.lower(): where = f"{path}.{name}" if path else name found.append( f" {where} ({_type(attribute.get('type'))}, {_use(attribute)})" ) for name, nested in sorted((node.get("block_types") or {}).items()): where = f"{path}.{name}" if path else name if needle in name.lower(): found.append(f" {where} {{ }} ({_block_note(nested)})") search(nested.get("block") or {}, where) search(root, "") if not found: return ( f"{title} has no argument or block named like {wanted!r}. It cannot be " "set on this resource type." ) return f"{title}: {wanted!r} is found at\n" + "\n".join(found) _FILES_SCHEMA: dict[str, Any] = { **FILES_SCHEMA, "description": "Terraform files in one directory, e.g. terraform/main.tf", } def _with_files( method: Callable[..., Awaitable[ToolResult]], working_copy: MutableMapping[str, str] | None = None, ) -> Callable[..., Awaitable[ToolResult]]: """Adapt a runner method to take the FILES_SCHEMA argument. With a working copy, the files are the Terraform files in it and the argument is optional: files passed anyway are written to the working copy first. """ async def handler(files: Any = None, **kwargs: Any) -> ToolResult: try: converted = files_from_argument(files) except TypeError as exc: return ToolResult(f"Refused: files: {exc}") if working_copy is None: return await method(converted, **kwargs) working_copy.update(converted) return await method(_terraform_files(working_copy), **kwargs) return handler def _terraform_files(files: Mapping[str, str]) -> dict[str, str]: return { name: content for name, content in files.items() if name.endswith(_TERRAFORM_SUFFIXES) } def _fmt_in_place( runner: TerraformRunner, working_copy: MutableMapping[str, str] ) -> Callable[..., Awaitable[ToolResult]]: async def format_working_copy(files: Mapping[str, str]) -> ToolResult: changed = await runner.formatted(files) if isinstance(changed, ToolResult): return changed working_copy.update(changed) if not changed: return ToolResult("terraform fmt: all files are already formatted.") return ToolResult( f"terraform fmt reformatted {len(changed)} file(s) in your working " f"copy: {', '.join(sorted(changed))}." ) return _with_files(format_working_copy, working_copy) LOCAL_TOOLS: dict[str, tuple[str, str]] = { "mermaid_check": ("general", "Check a Mermaid diagram's basic structure."), "terraform_fmt": ("general", "Format Terraform files and return any that changed."), "terraform_validate": ( "general", "Validate Terraform files offline (init -backend=false, validate).", ), "terraform_plan": ( "cloud_read", "Preview Terraform against the live cloud environment with a plan. Never applies.", ), "terraform_module": ( "general", ( "Look up a published Terraform module: its inputs, outputs and " "submodules, and whether it works with the pinned provider. Give " "`source` as //, with //modules/ for " "a submodule. `version` is optional and defaults to the latest. `find` " "narrows the inputs to those matching the words given." ), ), "terraform_schema": ( "general", ( "Look up what a Terraform resource type accepts, from the provider's own " "schema. With `resource_types`, a list: an outline of every argument " "and block each one accepts, in one call. With `resource_type` alone: " "the same for one type, with descriptions. `block` gives one block in " "full. `find` says where named settings sit, or that the resource type " "has none." ), ), } _ON_WORKING_COPY = " Runs on your working copy; pass no files." def local_tools( names: Iterable[str], runner: TerraformRunner, working_copy: MutableMapping[str, str] | None = None, ) -> list[ToolSpec]: """The named tools. With a `working_copy`, the Terraform checks run on it.""" files_required = [] if working_copy is not None else ["files"] suffix = _ON_WORKING_COPY if working_copy is not None else "" def checks(name: str, method: Callable[..., Awaitable[ToolResult]]) -> ToolSpec: properties: dict[str, Any] = {"files": _FILES_SCHEMA} if name == "terraform_plan": properties["variables"] = { "type": "object", "additionalProperties": {"type": "string"}, } tier, description = LOCAL_TOOLS[name] return ToolSpec( name, description + suffix, { "type": "object", "properties": properties, "required": files_required, }, method, tier=tier, ) builders = { "mermaid_check": lambda: ToolSpec( "mermaid_check", LOCAL_TOOLS["mermaid_check"][1], { "type": "object", "properties": {"diagram": {"type": "string"}}, "required": ["diagram"], }, mermaid_check, tier="general", ), "terraform_fmt": lambda: checks( "terraform_fmt", _fmt_in_place(runner, working_copy) if working_copy is not None else _with_files(runner.fmt), ), "terraform_validate": lambda: checks( "terraform_validate", _with_files(runner.validate, working_copy) ), "terraform_plan": lambda: checks( "terraform_plan", _with_files(runner.plan, working_copy) ), "terraform_module": lambda: ToolSpec( "terraform_module", LOCAL_TOOLS["terraform_module"][1], { "type": "object", "properties": { "source": { "type": "string", "description": ( "For example terraform-google-modules/network/google" ), }, "version": {"type": "string", "description": "Such as 18.3.1"}, "find": { "type": "string", "description": "Words to look for in input names, comma-separated", }, }, "required": ["source"], }, ModuleLookup( namespaces=runner.module_namespaces, pinned={name: pin["version"] for name, pin in runner.providers.items()}, ).describe, tier="general", ), "terraform_schema": lambda: ToolSpec( "terraform_schema", LOCAL_TOOLS["terraform_schema"][1], { "type": "object", "properties": { "resource_type": { "type": "string", "description": "For example google_cloud_run_v2_service", }, "resource_types": { "type": "array", "items": {"type": "string"}, "description": ( "Several resource types in one call: each comes back " "as an outline of every argument and block it accepts" ), }, "block": { "type": "string", "description": "A nested block, such as template.containers", }, "find": { "type": "string", "description": ( "Argument or block names to look for; separate several " "with commas to check them in one call" ), }, }, "required": [], }, runner.schema, tier="general", ), } return [builders[name]() for name in names]