Some checks failed
validation / verify (push) Failing after 11s
Case 08 named the customer's stated topology in its design and did not build it; every Check passed because none asked for it. - The catalogue marks the 26 fields that bind a design. The agreed scope lists the ones the customer stated; what was assumed for them does not bind. - A requirement says what it comes from. A submission of requirements is refused when one of the customer's own requirements, or a stated binding field, is named by none, or when the requirement carrying a choice is not a constraint with a condition. The existing chain then owes it a Check. - Generated code may use only the providers the bundle pins: a resource or a required_providers entry of any other is refused before Terraform runs. - The guide opens with how much of the design the customer stated, how much was assumed and how much is unknown. Not run with real models. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
455 lines
15 KiB
Python
455 lines
15 KiB
Python
"""Local tools: Mermaid check and Terraform fmt/validate/plan in a scratch directory."""
|
|
|
|
import json
|
|
import shutil
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from app.tools.local import (
|
|
LOCAL_TOOLS,
|
|
TerraformRunner,
|
|
local_tools,
|
|
mermaid_check,
|
|
)
|
|
|
|
needs_terraform = pytest.mark.skipif(
|
|
shutil.which("terraform") is None, reason="terraform CLI not installed"
|
|
)
|
|
|
|
VALID_TF = {
|
|
"main.tf": 'variable "region" {\n type = string\n}\n\noutput "region" {\n value = var.region\n}\n',
|
|
}
|
|
|
|
|
|
# --- mermaid ---------------------------------------------------------------------------
|
|
|
|
|
|
async def test_mermaid_accepts_a_flowchart():
|
|
result = await mermaid_check(
|
|
"flowchart TD\n A[Client] -->|HTTPS| B(Cloud Run)\n B --> C[(Database)]\n"
|
|
)
|
|
|
|
assert result.message.startswith("Mermaid check passed")
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("diagram", "problem"),
|
|
[
|
|
("", "empty"),
|
|
("A --> B", "diagram type"),
|
|
("flowchart TD\n A[Client --> B", "Unbalanced"),
|
|
("%% comment only\n", "diagram type"),
|
|
],
|
|
)
|
|
async def test_mermaid_reports_problems(diagram, problem):
|
|
result = await mermaid_check(diagram)
|
|
|
|
assert result.message.startswith("Mermaid check failed")
|
|
assert problem in result.message
|
|
|
|
|
|
# --- terraform ---------------------------------------------------------------------------
|
|
|
|
|
|
@needs_terraform
|
|
async def test_fmt_returns_formatted_files_without_touching_the_input():
|
|
files = {"main.tf": 'variable "region" {\ntype=string\n}\n'}
|
|
|
|
result = await TerraformRunner().fmt(files)
|
|
|
|
assert "main.tf" in result.message
|
|
assert " type = string" in result.message
|
|
assert files["main.tf"] == 'variable "region" {\ntype=string\n}\n'
|
|
|
|
|
|
@needs_terraform
|
|
async def test_fmt_reports_already_formatted_files():
|
|
result = await TerraformRunner().fmt(VALID_TF)
|
|
|
|
assert result.message == "terraform fmt: all files are already formatted."
|
|
|
|
|
|
@needs_terraform
|
|
async def test_validate_passes_a_valid_configuration():
|
|
result = await TerraformRunner().validate(VALID_TF)
|
|
|
|
assert result.message.startswith("terraform validate passed")
|
|
|
|
|
|
@needs_terraform
|
|
async def test_validate_reports_errors():
|
|
result = await TerraformRunner().validate(
|
|
{"main.tf": 'output "x" {\n value = var.missing\n}\n'}
|
|
)
|
|
|
|
assert result.message.startswith("terraform validate failed")
|
|
assert "missing" in result.message
|
|
|
|
|
|
@needs_terraform
|
|
async def test_validate_runs_in_the_directory_holding_the_files():
|
|
broken = {"terraform/main.tf": 'output "x" {\n value = var.missing\n}\n'}
|
|
|
|
result = await TerraformRunner().validate(broken)
|
|
|
|
assert result.message.startswith("terraform validate failed")
|
|
|
|
|
|
async def test_files_across_directories_are_refused():
|
|
result = await TerraformRunner().validate(
|
|
{"terraform/main.tf": "", "modules/net/main.tf": ""}
|
|
)
|
|
|
|
assert result.message.startswith("Refused: put all files in one directory")
|
|
|
|
|
|
async def test_unsafe_file_names_are_refused():
|
|
runner = TerraformRunner()
|
|
|
|
for files in ({"../main.tf": ""}, {"/etc/main.tf": ""}, {"notes.txt": ""}, {}):
|
|
result = await runner.validate(files)
|
|
assert result.message.startswith("Refused"), files
|
|
|
|
|
|
async def test_missing_terraform_binary_is_reported_clearly():
|
|
result = await TerraformRunner(binary="terraform-not-installed").validate(VALID_TF)
|
|
|
|
assert "terraform-not-installed" in result.message
|
|
assert "not installed" in result.message
|
|
|
|
|
|
async def test_plan_never_runs_apply_and_refuses_unsafe_variables(monkeypatch):
|
|
commands: list[list[str]] = []
|
|
|
|
async def fake_run(self, args, cwd, env=None):
|
|
commands.append(list(args))
|
|
return 0, "ok"
|
|
|
|
monkeypatch.setattr(TerraformRunner, "_run", fake_run)
|
|
runner = TerraformRunner()
|
|
|
|
result = await runner.plan(VALID_TF, {"region": "europe-west2"})
|
|
refused = await runner.plan(VALID_TF, {"bad name": "x"})
|
|
|
|
assert result.message.startswith("terraform plan")
|
|
assert refused.message.startswith("Refused")
|
|
flat = [arg for command in commands for arg in command]
|
|
assert "apply" not in flat and "destroy" not in flat
|
|
plan = next(command for command in commands if command[1] == "plan")
|
|
assert "-lock=false" in plan and "-input=false" in plan
|
|
assert "-var=region=europe-west2" in plan
|
|
|
|
|
|
# --- schema lookup -----------------------------------------------------------------------
|
|
|
|
SCHEMA = {
|
|
"provider_schemas": {
|
|
"registry.terraform.io/example/cloud": {
|
|
"resource_schemas": {
|
|
"cloud_service": {
|
|
"block": {
|
|
"attributes": {
|
|
"name": {"type": "string", "required": True},
|
|
"labels": {"type": ["map", "string"], "optional": True},
|
|
"uri": {"type": "string", "computed": True},
|
|
},
|
|
"block_types": {
|
|
"template": {
|
|
"nesting_mode": "list",
|
|
"max_items": 1,
|
|
"block": {
|
|
"attributes": {
|
|
"cpu_boost": {
|
|
"type": "bool",
|
|
"optional": True,
|
|
"description": "Boost CPU at start.",
|
|
}
|
|
}
|
|
},
|
|
}
|
|
},
|
|
}
|
|
},
|
|
"cloud_job": {"block": {"attributes": {"name": {"type": "string"}}}},
|
|
}
|
|
}
|
|
}
|
|
}
|
|
PIN = {"cloud": {"source": "example/cloud", "version": "1.2.3"}}
|
|
|
|
|
|
def schema_runner(monkeypatch) -> tuple[TerraformRunner, list[list[str]]]:
|
|
commands: list[list[str]] = []
|
|
|
|
async def fake_run(self, args, cwd, env=None):
|
|
commands.append(list(args))
|
|
if args[1] == "init":
|
|
assert (
|
|
'cloud = { source = "example/cloud", version = "1.2.3" }'
|
|
in (Path(cwd) / "versions.tf").read_text()
|
|
)
|
|
return 0, "ok"
|
|
return 0, json.dumps(SCHEMA)
|
|
|
|
monkeypatch.setattr(TerraformRunner, "_run", fake_run)
|
|
return TerraformRunner(providers=PIN), commands
|
|
|
|
|
|
async def test_schema_lists_what_code_may_set_and_outlines_the_blocks(monkeypatch):
|
|
runner, _ = schema_runner(monkeypatch)
|
|
|
|
result = await runner.schema("cloud_service")
|
|
|
|
assert result.message.startswith("cloud_service (example/cloud 1.2.3)")
|
|
assert " name (string, required)" in result.message
|
|
assert " labels (map of string, optional)" in result.message
|
|
assert "uri" not in result.message # read-only: code cannot set it
|
|
assert " template { } (list, at most 1): cpu_boost" in result.message
|
|
|
|
|
|
async def test_schema_gives_one_block_in_full(monkeypatch):
|
|
runner, _ = schema_runner(monkeypatch)
|
|
|
|
result = await runner.schema("cloud_service", block="template")
|
|
missing = await runner.schema("cloud_service", block="template.containers")
|
|
|
|
assert " cpu_boost (bool, optional): Boost CPU at start." in result.message
|
|
assert "'containers' is not a block there" in missing.message
|
|
|
|
|
|
async def test_schema_says_where_a_setting_sits_or_that_it_cannot_be_set(monkeypatch):
|
|
runner, _ = schema_runner(monkeypatch)
|
|
|
|
found = await runner.schema("cloud_service", find="cpu_boost")
|
|
absent = await runner.schema("cloud_job", find="cpu_boost")
|
|
|
|
assert "template.cpu_boost (bool, optional)" in found.message
|
|
assert "It cannot be set on this resource type." in absent.message
|
|
|
|
|
|
async def test_schema_answers_for_several_settings_in_one_call(monkeypatch):
|
|
runner, _ = schema_runner(monkeypatch)
|
|
|
|
result = await runner.schema("cloud_service", find="cpu_boost, labels,ingress")
|
|
|
|
assert "template.cpu_boost (bool, optional)" in result.message
|
|
assert " labels (map of string, optional)" in result.message
|
|
assert "no argument or block named like 'ingress'" in result.message
|
|
|
|
|
|
async def test_schema_suggests_close_resource_types_and_is_loaded_once(monkeypatch):
|
|
runner, commands = schema_runner(monkeypatch)
|
|
|
|
unknown = await runner.schema("cloud_servce")
|
|
await runner.schema("cloud_job")
|
|
|
|
assert "Close matches: cloud_service" in unknown.message
|
|
assert [command[1] for command in commands] == ["init", "providers"]
|
|
assert "apply" not in [arg for command in commands for arg in command]
|
|
|
|
|
|
async def test_several_resource_types_are_outlined_in_one_call(monkeypatch):
|
|
"""One lookup for a whole design: looking types up one by one used up the
|
|
target architect's tool calls in three of five runs."""
|
|
runner, commands = schema_runner(monkeypatch)
|
|
|
|
result = await runner.schema(resource_types=["cloud_service", "cloud_job", "nope"])
|
|
by_commas = await runner.schema(resource_type="cloud_service, cloud_job")
|
|
|
|
assert result.message.startswith("example/cloud 1.2.3. * marks a required")
|
|
assert "cloud_service\n labels, name*" in result.message
|
|
assert " template { }: cpu_boost" in result.message
|
|
assert "uri" not in result.message # read-only: code cannot set it
|
|
assert "nope: not a resource type of example/cloud 1.2.3." in result.message
|
|
assert "cloud_job\n" in by_commas.message and "template { }" in by_commas.message
|
|
assert [command[1] for command in commands] == ["init", "providers"]
|
|
|
|
|
|
async def test_a_lookup_needs_a_resource_type(monkeypatch):
|
|
runner, _ = schema_runner(monkeypatch)
|
|
|
|
assert (await runner.schema()).message.startswith("Refused: name at least one")
|
|
|
|
|
|
async def test_schema_is_unavailable_without_a_pinned_provider():
|
|
result = await TerraformRunner().schema("cloud_service")
|
|
|
|
assert "pins no Terraform provider" in result.message
|
|
|
|
|
|
# --- registry ------------------------------------------------------------------------------
|
|
|
|
|
|
def test_local_tools_are_built_by_name_with_their_tier():
|
|
tools = local_tools(["mermaid_check", "terraform_plan"], TerraformRunner())
|
|
|
|
assert [(t.name, t.tier) for t in tools] == [
|
|
("mermaid_check", "general"),
|
|
("terraform_plan", "cloud_read"),
|
|
]
|
|
assert set(LOCAL_TOOLS) == {
|
|
"mermaid_check",
|
|
"terraform_fmt",
|
|
"terraform_validate",
|
|
"terraform_plan",
|
|
"terraform_schema",
|
|
"terraform_module",
|
|
}
|
|
with pytest.raises(KeyError, match="terraform_apply"):
|
|
local_tools(["terraform_apply"], TerraformRunner())
|
|
|
|
|
|
# --- the provider cache ----------------------------------------------------------------
|
|
|
|
|
|
def fake_terraform(tmp_path: Path) -> tuple[Path, Path]:
|
|
"""A stand-in terraform that records the cache settings it was run with."""
|
|
log = tmp_path / "calls.log"
|
|
binary = tmp_path / "terraform"
|
|
binary.write_text(
|
|
"#!/bin/sh\n"
|
|
f'echo "$1 cache=$TF_PLUGIN_CACHE_DIR '
|
|
f'reuse=$TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE" >> {log}\n'
|
|
'echo "Success! The configuration is valid."\n'
|
|
)
|
|
binary.chmod(0o755)
|
|
return binary, log
|
|
|
|
|
|
async def test_providers_are_cached_in_a_directory_the_runner_creates(tmp_path):
|
|
binary, log = fake_terraform(tmp_path)
|
|
cache = tmp_path / "not" / "yet" / "there"
|
|
runner = TerraformRunner(binary=str(binary), plugin_cache_dir=str(cache))
|
|
|
|
result = await runner.validate(VALID_TF)
|
|
|
|
assert result.message.startswith("terraform validate passed")
|
|
assert cache.is_dir()
|
|
# Scratch directories have no lock file, so reuse must be allowed explicitly.
|
|
assert log.read_text().splitlines() == [
|
|
f"init cache={cache} reuse=true",
|
|
f"validate cache={cache} reuse=true",
|
|
]
|
|
|
|
|
|
async def test_without_a_cache_directory_terraform_is_left_to_its_defaults(tmp_path):
|
|
binary, log = fake_terraform(tmp_path)
|
|
|
|
await TerraformRunner(binary=str(binary)).validate(VALID_TF)
|
|
|
|
assert log.read_text().splitlines() == [
|
|
"init cache= reuse=",
|
|
"validate cache= reuse=",
|
|
]
|
|
|
|
|
|
async def test_concurrent_checks_install_into_the_cache_one_at_a_time(tmp_path):
|
|
import asyncio
|
|
|
|
marker = tmp_path / "installing"
|
|
overlaps = tmp_path / "overlaps"
|
|
binary = tmp_path / "terraform"
|
|
binary.write_text(
|
|
"#!/bin/sh\n"
|
|
'if [ "$1" = init ]; then\n'
|
|
f" [ -e {marker} ] && echo overlap >> {overlaps}\n"
|
|
f" touch {marker}; sleep 0.2; rm -f {marker}\n"
|
|
"fi\n"
|
|
"echo ok\n"
|
|
)
|
|
binary.chmod(0o755)
|
|
runner = TerraformRunner(binary=str(binary), plugin_cache_dir=str(tmp_path / "c"))
|
|
|
|
await asyncio.gather(*(runner.validate(VALID_TF) for _ in range(3)))
|
|
|
|
assert not overlaps.exists()
|
|
|
|
|
|
def test_the_cache_setting_resolves_under_the_repository_and_can_be_disabled():
|
|
from app.config import Settings
|
|
|
|
default = Settings(_env_file=None)
|
|
assert default.terraform_plugin_cache() == str(
|
|
default.BASE_DIR / ".cache" / "terraform-plugins"
|
|
)
|
|
assert (
|
|
Settings(_env_file=None, TF_PLUGIN_CACHE_DIR="").terraform_plugin_cache()
|
|
is None
|
|
)
|
|
assert (
|
|
Settings(
|
|
_env_file=None, TF_PLUGIN_CACHE_DIR="/var/cache/tf"
|
|
).terraform_plugin_cache()
|
|
== "/var/cache/tf"
|
|
)
|
|
|
|
|
|
# --- only the providers the bundle pins ---------------------------------------------------
|
|
|
|
PINNED = {"cloud": {"source": "example/cloud", "version": "1.2.3"}}
|
|
|
|
|
|
def provider_refusal(code: str, pinned=PINNED) -> str:
|
|
from app.tools.local import check_providers
|
|
|
|
result = check_providers({"terraform/main.tf": code}, pinned)
|
|
return result.message if result else ""
|
|
|
|
|
|
def test_code_of_the_pinned_provider_alone_is_accepted():
|
|
code = """
|
|
terraform {
|
|
required_providers {
|
|
cloud = {
|
|
source = "example/cloud"
|
|
version = "1.2.3"
|
|
}
|
|
}
|
|
}
|
|
resource "cloud_service" "web" {}
|
|
data "cloud_project" "this" {}
|
|
resource "terraform_data" "marker" {}
|
|
module "network" {
|
|
source = "example-modules/network/cloud"
|
|
version = "1.0.0"
|
|
}
|
|
"""
|
|
|
|
assert provider_refusal(code) == ""
|
|
|
|
|
|
def test_a_resource_of_a_provider_that_is_not_pinned_is_refused():
|
|
message = provider_refusal('resource "sourcehost_repository" "infra" {}')
|
|
|
|
assert message.startswith("Refused: the code uses resources of the sourcehost")
|
|
assert "Only these providers may be used: example/cloud" in message
|
|
|
|
|
|
def test_declaring_another_provider_is_refused():
|
|
declared = """
|
|
terraform {
|
|
required_providers {
|
|
sourcehost = {
|
|
source = "integrations/sourcehost"
|
|
}
|
|
}
|
|
}
|
|
"""
|
|
|
|
assert "the provider integrations/sourcehost" in provider_refusal(declared)
|
|
|
|
|
|
def test_with_nothing_pinned_there_is_nothing_to_hold_code_to():
|
|
assert provider_refusal('resource "anything_at_all" "x" {}', pinned={}) == ""
|
|
|
|
|
|
async def test_validation_refuses_a_foreign_provider_before_terraform_runs():
|
|
runner = TerraformRunner(binary="no-such-binary", providers=PINNED)
|
|
|
|
result = await runner.validate(
|
|
{"terraform/main.tf": 'resource "sourcehost_repository" "infra" {}'}
|
|
)
|
|
|
|
assert result.message.startswith("Refused: the code uses")
|