Design is now approved before any infrastructure code is written (ADR-0005). A new target-architect Specialist produces one graph of Elements, Relationships and Checks; the diagram is rendered from it, deployment guidance must account for every Check, and validation gives a verdict per Check. Requirements, components, products and recommendations carry ids in data files, and a submission with a gap in the Requirements chain is refused. Also in this commit: - Customer cases group Executions, with an intent history per case - lesson candidates are captured when an Execution ends - evaluation gains chain, verdict, judged and interactive checks, and per-case thresholds - Terraform providers are cached between checks - the requirements analyst proposes defaults instead of asking - the legacy-baseline case and the checks that only read it are removed - ADR-0006 (Change requests) accepted; ADR-0004 and ADR-0007 proposed - plans for iterations 4 to 7 under .agents/plans/ Unproven with a real model: no evaluation run has completed on this design. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
246 lines
7.5 KiB
Python
246 lines
7.5 KiB
Python
"""Local tools: Mermaid check and Terraform fmt/validate/plan in a scratch directory."""
|
|
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.tools.local import (
|
|
LOCAL_TOOLS,
|
|
TerraformRunner,
|
|
local_tools,
|
|
mermaid_check,
|
|
)
|
|
|
|
needs_terraform = pytest.mark.skipif(
|
|
shutil.which("terraform") is None, reason="terraform CLI not installed"
|
|
)
|
|
|
|
VALID_TF = {
|
|
"main.tf": 'variable "region" {\n type = string\n}\n\noutput "region" {\n value = var.region\n}\n',
|
|
}
|
|
|
|
|
|
# --- mermaid ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_mermaid_accepts_a_flowchart():
|
|
result = await mermaid_check(
|
|
"flowchart TD\n A[Client] -->|HTTPS| B(Cloud Run)\n B --> C[(Database)]\n"
|
|
)
|
|
|
|
assert result.message.startswith("Mermaid check passed")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("diagram", "problem"),
|
|
[
|
|
("", "empty"),
|
|
("A --> B", "diagram type"),
|
|
("flowchart TD\n A[Client --> B", "Unbalanced"),
|
|
("%% comment only\n", "diagram type"),
|
|
],
|
|
)
|
|
async def test_mermaid_reports_problems(diagram, problem):
|
|
result = await mermaid_check(diagram)
|
|
|
|
assert result.message.startswith("Mermaid check failed")
|
|
assert problem in result.message
|
|
|
|
|
|
# --- terraform ---------------------------------------------------------------------------
|
|
|
|
|
|
@needs_terraform
|
|
async def test_fmt_returns_formatted_files_without_touching_the_input():
|
|
files = {"main.tf": 'variable "region" {\ntype=string\n}\n'}
|
|
|
|
result = await TerraformRunner().fmt(files)
|
|
|
|
assert "main.tf" in result.message
|
|
assert " type = string" in result.message
|
|
assert files["main.tf"] == 'variable "region" {\ntype=string\n}\n'
|
|
|
|
|
|
@needs_terraform
|
|
async def test_fmt_reports_already_formatted_files():
|
|
result = await TerraformRunner().fmt(VALID_TF)
|
|
|
|
assert result.message == "terraform fmt: all files are already formatted."
|
|
|
|
|
|
@needs_terraform
|
|
async def test_validate_passes_a_valid_configuration():
|
|
result = await TerraformRunner().validate(VALID_TF)
|
|
|
|
assert result.message.startswith("terraform validate passed")
|
|
|
|
|
|
@needs_terraform
|
|
async def test_validate_reports_errors():
|
|
result = await TerraformRunner().validate(
|
|
{"main.tf": 'output "x" {\n value = var.missing\n}\n'}
|
|
)
|
|
|
|
assert result.message.startswith("terraform validate failed")
|
|
assert "missing" in result.message
|
|
|
|
|
|
@needs_terraform
|
|
async def test_validate_runs_in_the_directory_holding_the_files():
|
|
broken = {"terraform/main.tf": 'output "x" {\n value = var.missing\n}\n'}
|
|
|
|
result = await TerraformRunner().validate(broken)
|
|
|
|
assert result.message.startswith("terraform validate failed")
|
|
|
|
|
|
async def test_files_across_directories_are_refused():
|
|
result = await TerraformRunner().validate(
|
|
{"terraform/main.tf": "", "modules/net/main.tf": ""}
|
|
)
|
|
|
|
assert result.message.startswith("Refused: put all files in one directory")
|
|
|
|
|
|
async def test_unsafe_file_names_are_refused():
|
|
runner = TerraformRunner()
|
|
|
|
for files in ({"../main.tf": ""}, {"/etc/main.tf": ""}, {"notes.txt": ""}, {}):
|
|
result = await runner.validate(files)
|
|
assert result.message.startswith("Refused"), files
|
|
|
|
|
|
async def test_missing_terraform_binary_is_reported_clearly():
|
|
result = await TerraformRunner(binary="terraform-not-installed").validate(VALID_TF)
|
|
|
|
assert "terraform-not-installed" in result.message
|
|
assert "not installed" in result.message
|
|
|
|
|
|
async def test_plan_never_runs_apply_and_refuses_unsafe_variables(monkeypatch):
|
|
commands: list[list[str]] = []
|
|
|
|
async def fake_run(self, args, cwd, env=None):
|
|
commands.append(list(args))
|
|
return 0, "ok"
|
|
|
|
monkeypatch.setattr(TerraformRunner, "_run", fake_run)
|
|
runner = TerraformRunner()
|
|
|
|
result = await runner.plan(VALID_TF, {"region": "europe-west2"})
|
|
refused = await runner.plan(VALID_TF, {"bad name": "x"})
|
|
|
|
assert result.message.startswith("terraform plan")
|
|
assert refused.message.startswith("Refused")
|
|
flat = [arg for command in commands for arg in command]
|
|
assert "apply" not in flat and "destroy" not in flat
|
|
plan = next(command for command in commands if command[1] == "plan")
|
|
assert "-lock=false" in plan and "-input=false" in plan
|
|
assert "-var=region=europe-west2" in plan
|
|
|
|
|
|
# --- registry ------------------------------------------------------------------------------
|
|
|
|
|
|
def test_local_tools_are_built_by_name_with_their_tier():
|
|
tools = local_tools(["mermaid_check", "terraform_plan"], TerraformRunner())
|
|
|
|
assert [(t.name, t.tier) for t in tools] == [
|
|
("mermaid_check", "general"),
|
|
("terraform_plan", "cloud_read"),
|
|
]
|
|
assert set(LOCAL_TOOLS) == {
|
|
"mermaid_check",
|
|
"terraform_fmt",
|
|
"terraform_validate",
|
|
"terraform_plan",
|
|
}
|
|
with pytest.raises(KeyError, match="terraform_apply"):
|
|
local_tools(["terraform_apply"], TerraformRunner())
|
|
|
|
|
|
# --- the provider cache ----------------------------------------------------------------
|
|
|
|
|
|
def fake_terraform(tmp_path: Path) -> tuple[Path, Path]:
|
|
"""A stand-in terraform that records the cache settings it was run with."""
|
|
log = tmp_path / "calls.log"
|
|
binary = tmp_path / "terraform"
|
|
binary.write_text(
|
|
"#!/bin/sh\n"
|
|
f'echo "$1 cache=$TF_PLUGIN_CACHE_DIR '
|
|
f'reuse=$TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE" >> {log}\n'
|
|
'echo "Success! The configuration is valid."\n'
|
|
)
|
|
binary.chmod(0o755)
|
|
return binary, log
|
|
|
|
|
|
async def test_providers_are_cached_in_a_directory_the_runner_creates(tmp_path):
|
|
binary, log = fake_terraform(tmp_path)
|
|
cache = tmp_path / "not" / "yet" / "there"
|
|
runner = TerraformRunner(binary=str(binary), plugin_cache_dir=str(cache))
|
|
|
|
result = await runner.validate(VALID_TF)
|
|
|
|
assert result.message.startswith("terraform validate passed")
|
|
assert cache.is_dir()
|
|
# Scratch directories have no lock file, so reuse must be allowed explicitly.
|
|
assert log.read_text().splitlines() == [
|
|
f"init cache={cache} reuse=true",
|
|
f"validate cache={cache} reuse=true",
|
|
]
|
|
|
|
|
|
async def test_without_a_cache_directory_terraform_is_left_to_its_defaults(tmp_path):
|
|
binary, log = fake_terraform(tmp_path)
|
|
|
|
await TerraformRunner(binary=str(binary)).validate(VALID_TF)
|
|
|
|
assert log.read_text().splitlines() == [
|
|
"init cache= reuse=",
|
|
"validate cache= reuse=",
|
|
]
|
|
|
|
|
|
async def test_concurrent_checks_install_into_the_cache_one_at_a_time(tmp_path):
|
|
import asyncio
|
|
|
|
marker = tmp_path / "installing"
|
|
overlaps = tmp_path / "overlaps"
|
|
binary = tmp_path / "terraform"
|
|
binary.write_text(
|
|
"#!/bin/sh\n"
|
|
'if [ "$1" = init ]; then\n'
|
|
f" [ -e {marker} ] && echo overlap >> {overlaps}\n"
|
|
f" touch {marker}; sleep 0.2; rm -f {marker}\n"
|
|
"fi\n"
|
|
"echo ok\n"
|
|
)
|
|
binary.chmod(0o755)
|
|
runner = TerraformRunner(binary=str(binary), plugin_cache_dir=str(tmp_path / "c"))
|
|
|
|
await asyncio.gather(*(runner.validate(VALID_TF) for _ in range(3)))
|
|
|
|
assert not overlaps.exists()
|
|
|
|
|
|
def test_the_cache_setting_resolves_under_the_repository_and_can_be_disabled():
|
|
from app.config import Settings
|
|
|
|
default = Settings(_env_file=None)
|
|
assert default.terraform_plugin_cache() == str(
|
|
default.BASE_DIR / ".cache" / "terraform-plugins"
|
|
)
|
|
assert (
|
|
Settings(_env_file=None, TF_PLUGIN_CACHE_DIR="").terraform_plugin_cache()
|
|
is None
|
|
)
|
|
assert (
|
|
Settings(
|
|
_env_file=None, TF_PLUGIN_CACHE_DIR="/var/cache/tf"
|
|
).terraform_plugin_cache()
|
|
== "/var/cache/tf"
|
|
)
|