Files
orchestration_engine/tests/tools/test_local.py

455 lines
15 KiB
Python

"""Local tools: Mermaid check and Terraform fmt/validate/plan in a scratch directory."""
import json
import shutil
from pathlib import Path
import pytest
from tests.reference_workflow.local import (
LOCAL_TOOLS,
TerraformRunner,
local_tools,
mermaid_check,
)
needs_terraform = pytest.mark.skipif(
shutil.which("terraform") is None, reason="terraform CLI not installed"
)
VALID_TF = {
"main.tf": 'variable "region" {\n type = string\n}\n\noutput "region" {\n value = var.region\n}\n',
}
# --- mermaid ---------------------------------------------------------------------------
async def test_mermaid_accepts_a_flowchart():
result = await mermaid_check(
"flowchart TD\n A[Client] -->|HTTPS| B(Cloud Run)\n B --> C[(Database)]\n"
)
assert result.message.startswith("Mermaid check passed")
@pytest.mark.parametrize(
("diagram", "problem"),
[
("", "empty"),
("A --> B", "diagram type"),
("flowchart TD\n A[Client --> B", "Unbalanced"),
("%% comment only\n", "diagram type"),
],
)
async def test_mermaid_reports_problems(diagram, problem):
result = await mermaid_check(diagram)
assert result.message.startswith("Mermaid check failed")
assert problem in result.message
# --- terraform ---------------------------------------------------------------------------
@needs_terraform
async def test_fmt_returns_formatted_files_without_touching_the_input():
files = {"main.tf": 'variable "region" {\ntype=string\n}\n'}
result = await TerraformRunner().fmt(files)
assert "main.tf" in result.message
assert " type = string" in result.message
assert files["main.tf"] == 'variable "region" {\ntype=string\n}\n'
@needs_terraform
async def test_fmt_reports_already_formatted_files():
result = await TerraformRunner().fmt(VALID_TF)
assert result.message == "terraform fmt: all files are already formatted."
@needs_terraform
async def test_validate_passes_a_valid_configuration():
result = await TerraformRunner().validate(VALID_TF)
assert result.message.startswith("terraform validate passed")
@needs_terraform
async def test_validate_reports_errors():
result = await TerraformRunner().validate(
{"main.tf": 'output "x" {\n value = var.missing\n}\n'}
)
assert result.message.startswith("terraform validate failed")
assert "missing" in result.message
@needs_terraform
async def test_validate_runs_in_the_directory_holding_the_files():
broken = {"terraform/main.tf": 'output "x" {\n value = var.missing\n}\n'}
result = await TerraformRunner().validate(broken)
assert result.message.startswith("terraform validate failed")
async def test_files_across_directories_are_refused():
result = await TerraformRunner().validate(
{"terraform/main.tf": "", "modules/net/main.tf": ""}
)
assert result.message.startswith("Refused: put all files in one directory")
async def test_unsafe_file_names_are_refused():
runner = TerraformRunner()
for files in ({"../main.tf": ""}, {"/etc/main.tf": ""}, {"notes.txt": ""}, {}):
result = await runner.validate(files)
assert result.message.startswith("Refused"), files
async def test_missing_terraform_binary_is_reported_clearly():
result = await TerraformRunner(binary="terraform-not-installed").validate(VALID_TF)
assert "terraform-not-installed" in result.message
assert "not installed" in result.message
async def test_plan_never_runs_apply_and_refuses_unsafe_variables(monkeypatch):
commands: list[list[str]] = []
async def fake_run(self, args, cwd, env=None):
commands.append(list(args))
return 0, "ok"
monkeypatch.setattr(TerraformRunner, "_run", fake_run)
runner = TerraformRunner()
result = await runner.plan(VALID_TF, {"region": "europe-west2"})
refused = await runner.plan(VALID_TF, {"bad name": "x"})
assert result.message.startswith("terraform plan")
assert refused.message.startswith("Refused")
flat = [arg for command in commands for arg in command]
assert "apply" not in flat and "destroy" not in flat
plan = next(command for command in commands if command[1] == "plan")
assert "-lock=false" in plan and "-input=false" in plan
assert "-var=region=europe-west2" in plan
# --- schema lookup -----------------------------------------------------------------------
SCHEMA = {
"provider_schemas": {
"registry.terraform.io/example/cloud": {
"resource_schemas": {
"cloud_service": {
"block": {
"attributes": {
"name": {"type": "string", "required": True},
"labels": {"type": ["map", "string"], "optional": True},
"uri": {"type": "string", "computed": True},
},
"block_types": {
"template": {
"nesting_mode": "list",
"max_items": 1,
"block": {
"attributes": {
"cpu_boost": {
"type": "bool",
"optional": True,
"description": "Boost CPU at start.",
}
}
},
}
},
}
},
"cloud_job": {"block": {"attributes": {"name": {"type": "string"}}}},
}
}
}
}
PIN = {"cloud": {"source": "example/cloud", "version": "1.2.3"}}
def schema_runner(monkeypatch) -> tuple[TerraformRunner, list[list[str]]]:
commands: list[list[str]] = []
async def fake_run(self, args, cwd, env=None):
commands.append(list(args))
if args[1] == "init":
assert (
'cloud = { source = "example/cloud", version = "1.2.3" }'
in (Path(cwd) / "versions.tf").read_text()
)
return 0, "ok"
return 0, json.dumps(SCHEMA)
monkeypatch.setattr(TerraformRunner, "_run", fake_run)
return TerraformRunner(providers=PIN), commands
async def test_schema_lists_what_code_may_set_and_outlines_the_blocks(monkeypatch):
runner, _ = schema_runner(monkeypatch)
result = await runner.schema("cloud_service")
assert result.message.startswith("cloud_service (example/cloud 1.2.3)")
assert " name (string, required)" in result.message
assert " labels (map of string, optional)" in result.message
assert "uri" not in result.message # read-only: code cannot set it
assert " template { } (list, at most 1): cpu_boost" in result.message
async def test_schema_gives_one_block_in_full(monkeypatch):
runner, _ = schema_runner(monkeypatch)
result = await runner.schema("cloud_service", block="template")
missing = await runner.schema("cloud_service", block="template.containers")
assert " cpu_boost (bool, optional): Boost CPU at start." in result.message
assert "'containers' is not a block there" in missing.message
async def test_schema_says_where_a_setting_sits_or_that_it_cannot_be_set(monkeypatch):
runner, _ = schema_runner(monkeypatch)
found = await runner.schema("cloud_service", find="cpu_boost")
absent = await runner.schema("cloud_job", find="cpu_boost")
assert "template.cpu_boost (bool, optional)" in found.message
assert "It cannot be set on this resource type." in absent.message
async def test_schema_answers_for_several_settings_in_one_call(monkeypatch):
runner, _ = schema_runner(monkeypatch)
result = await runner.schema("cloud_service", find="cpu_boost, labels,ingress")
assert "template.cpu_boost (bool, optional)" in result.message
assert " labels (map of string, optional)" in result.message
assert "no argument or block named like 'ingress'" in result.message
async def test_schema_suggests_close_resource_types_and_is_loaded_once(monkeypatch):
runner, commands = schema_runner(monkeypatch)
unknown = await runner.schema("cloud_servce")
await runner.schema("cloud_job")
assert "Close matches: cloud_service" in unknown.message
assert [command[1] for command in commands] == ["init", "providers"]
assert "apply" not in [arg for command in commands for arg in command]
async def test_several_resource_types_are_outlined_in_one_call(monkeypatch):
"""One lookup for a whole design: looking types up one by one used up the
target architect's tool calls in three of five runs."""
runner, commands = schema_runner(monkeypatch)
result = await runner.schema(resource_types=["cloud_service", "cloud_job", "nope"])
by_commas = await runner.schema(resource_type="cloud_service, cloud_job")
assert result.message.startswith("example/cloud 1.2.3. * marks a required")
assert "cloud_service\n labels, name*" in result.message
assert " template { }: cpu_boost" in result.message
assert "uri" not in result.message # read-only: code cannot set it
assert "nope: not a resource type of example/cloud 1.2.3." in result.message
assert "cloud_job\n" in by_commas.message and "template { }" in by_commas.message
assert [command[1] for command in commands] == ["init", "providers"]
async def test_a_lookup_needs_a_resource_type(monkeypatch):
runner, _ = schema_runner(monkeypatch)
assert (await runner.schema()).message.startswith("Refused: name at least one")
async def test_schema_is_unavailable_without_a_pinned_provider():
result = await TerraformRunner().schema("cloud_service")
assert "pins no Terraform provider" in result.message
# --- registry ------------------------------------------------------------------------------
def test_local_tools_are_built_by_name_with_their_tier():
tools = local_tools(["mermaid_check", "terraform_plan"], TerraformRunner())
assert [(t.name, t.tier) for t in tools] == [
("mermaid_check", "general"),
("terraform_plan", "cloud_read"),
]
assert set(LOCAL_TOOLS) == {
"mermaid_check",
"terraform_fmt",
"terraform_validate",
"terraform_plan",
"terraform_schema",
"terraform_module",
}
with pytest.raises(KeyError, match="terraform_apply"):
local_tools(["terraform_apply"], TerraformRunner())
# --- the provider cache ----------------------------------------------------------------
def fake_terraform(tmp_path: Path) -> tuple[Path, Path]:
"""A stand-in terraform that records the cache settings it was run with."""
log = tmp_path / "calls.log"
binary = tmp_path / "terraform"
binary.write_text(
"#!/bin/sh\n"
f'echo "$1 cache=$TF_PLUGIN_CACHE_DIR '
f'reuse=$TF_PLUGIN_CACHE_MAY_BREAK_DEPENDENCY_LOCK_FILE" >> {log}\n'
'echo "Success! The configuration is valid."\n'
)
binary.chmod(0o755)
return binary, log
async def test_providers_are_cached_in_a_directory_the_runner_creates(tmp_path):
binary, log = fake_terraform(tmp_path)
cache = tmp_path / "not" / "yet" / "there"
runner = TerraformRunner(binary=str(binary), plugin_cache_dir=str(cache))
result = await runner.validate(VALID_TF)
assert result.message.startswith("terraform validate passed")
assert cache.is_dir()
# Scratch directories have no lock file, so reuse must be allowed explicitly.
assert log.read_text().splitlines() == [
f"init cache={cache} reuse=true",
f"validate cache={cache} reuse=true",
]
async def test_without_a_cache_directory_terraform_is_left_to_its_defaults(tmp_path):
binary, log = fake_terraform(tmp_path)
await TerraformRunner(binary=str(binary)).validate(VALID_TF)
assert log.read_text().splitlines() == [
"init cache= reuse=",
"validate cache= reuse=",
]
async def test_concurrent_checks_install_into_the_cache_one_at_a_time(tmp_path):
import asyncio
marker = tmp_path / "installing"
overlaps = tmp_path / "overlaps"
binary = tmp_path / "terraform"
binary.write_text(
"#!/bin/sh\n"
'if [ "$1" = init ]; then\n'
f" [ -e {marker} ] && echo overlap >> {overlaps}\n"
f" touch {marker}; sleep 0.2; rm -f {marker}\n"
"fi\n"
"echo ok\n"
)
binary.chmod(0o755)
runner = TerraformRunner(binary=str(binary), plugin_cache_dir=str(tmp_path / "c"))
await asyncio.gather(*(runner.validate(VALID_TF) for _ in range(3)))
assert not overlaps.exists()
def test_the_cache_setting_resolves_under_the_repository_and_can_be_disabled():
from execution_engine.config import Settings
default = Settings(_env_file=None)
assert default.terraform_plugin_cache() == str(
default.BASE_DIR / ".cache" / "terraform-plugins"
)
assert (
Settings(_env_file=None, TF_PLUGIN_CACHE_DIR="").terraform_plugin_cache()
is None
)
assert (
Settings(
_env_file=None, TF_PLUGIN_CACHE_DIR="/var/cache/tf"
).terraform_plugin_cache()
== "/var/cache/tf"
)
# --- only the providers the bundle pins ---------------------------------------------------
PINNED = {"cloud": {"source": "example/cloud", "version": "1.2.3"}}
def provider_refusal(code: str, pinned=PINNED) -> str:
from tests.reference_workflow.local import check_providers
result = check_providers({"terraform/main.tf": code}, pinned)
return result.message if result else ""
def test_code_of_the_pinned_provider_alone_is_accepted():
code = """
terraform {
required_providers {
cloud = {
source = "example/cloud"
version = "1.2.3"
}
}
}
resource "cloud_service" "web" {}
data "cloud_project" "this" {}
resource "terraform_data" "marker" {}
module "network" {
source = "example-modules/network/cloud"
version = "1.0.0"
}
"""
assert provider_refusal(code) == ""
def test_a_resource_of_a_provider_that_is_not_pinned_is_refused():
message = provider_refusal('resource "sourcehost_repository" "infra" {}')
assert message.startswith("Refused: the code uses resources of the sourcehost")
assert "Only these providers may be used: example/cloud" in message
def test_declaring_another_provider_is_refused():
declared = """
terraform {
required_providers {
sourcehost = {
source = "integrations/sourcehost"
}
}
}
"""
assert "the provider integrations/sourcehost" in provider_refusal(declared)
def test_with_nothing_pinned_there_is_nothing_to_hold_code_to():
assert provider_refusal('resource "anything_at_all" "x" {}', pinned={}) == ""
async def test_validation_refuses_a_foreign_provider_before_terraform_runs():
runner = TerraformRunner(binary="no-such-binary", providers=PINNED)
result = await runner.validate(
{"terraform/main.tf": 'resource "sourcehost_repository" "infra" {}'}
)
assert result.message.startswith("Refused: the code uses")