feat: opt-in published Terraform modules from approved sources (iteration 8)
Infrastructure code is plain resources by default. An Execution may opt in with terraform_modules; the deployment default is TERRAFORM_MODULES, off. - A module is accepted only from the registry namespaces the provider bundle lists (terraform-google-modules), at one exact version. Any other source is refused before Terraform runs; until now nothing stopped generated code naming any source for terraform init to download. - terraform_module looks up a module's inputs, outputs and submodules from the registry and says when it cannot be used with the pinned provider. - skills/google-cloud-iac-modules: when a module fits, and four modules with examples a test validates against hashicorp/google 8.6.0. - Opted in, the IaC author and validator get the Skill and the lookup. Not opted in, they get neither and a module block is refused in code. - A Change request follows its Baseline. execution.json records the choice. - case-07 runs case 03's request with modules opted into. Azure (aztfm) and AWS (terraform-aws-modules) sources are recorded in the plan for when those Cloud providers are added. No case has run with this. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -52,6 +52,7 @@ def test_roles_name_only_the_local_tools_they_need(bundle):
|
||||
"terraform_fmt",
|
||||
"terraform_validate",
|
||||
"terraform_schema",
|
||||
"terraform_module",
|
||||
)
|
||||
# The architect only looks settings up; it writes and runs no code.
|
||||
assert roles["target-architect"].tools == ("terraform_schema",)
|
||||
@@ -59,6 +60,7 @@ def test_roles_name_only_the_local_tools_they_need(bundle):
|
||||
"terraform_validate",
|
||||
"terraform_plan",
|
||||
"terraform_schema",
|
||||
"terraform_module",
|
||||
)
|
||||
assert roles["requirements-analyst"].tools == ()
|
||||
|
||||
|
||||
205
tests/execution/test_terraform_modules.py
Normal file
205
tests/execution/test_terraform_modules.py
Normal file
@@ -0,0 +1,205 @@
|
||||
"""Published Terraform modules are opt-in per Execution (iteration 8).
|
||||
|
||||
Infrastructure code is plain resources by default. An Execution that opts in is
|
||||
given the modules Skill and the module lookup, and only then is a module call from
|
||||
an approved source accepted.
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
from langgraph.checkpoint.memory import InMemorySaver
|
||||
|
||||
from app.adk.fake_runtime import FakeAgentRuntime, sample_submission
|
||||
from app.adk.roles import load_roles
|
||||
from app.adk.runtime import NoDecider, RoleSpec
|
||||
from app.agent import AGENTS_DIR, ExecutionEngine
|
||||
from app.nodes.router import Router, Thresholds
|
||||
from app.providers import load_provider
|
||||
from app.recorder import InMemoryRecorder
|
||||
from app.tools.local import TerraformRunner
|
||||
from app.tools.modules import MODULE_TOOL, MODULES_OPTED_IN, NOT_OPTED_IN
|
||||
from app.workflows.deliverables import DEPENDENCY_GRAPH
|
||||
from app.workflows.deliverables import DeliverableKind as K
|
||||
|
||||
AUTHOR = DEPENDENCY_GRAPH[K.DEPLOYMENT_GUIDANCE].specialist
|
||||
CASE = "acme"
|
||||
MODULE_CALL = """
|
||||
module "network" {
|
||||
source = "approved-modules/network/cloud"
|
||||
version = "1.2.3"
|
||||
}
|
||||
"""
|
||||
ROLES = {
|
||||
"orchestrator": RoleSpec("orchestrator", "Unused.", budget=40),
|
||||
**{
|
||||
spec.specialist: RoleSpec(
|
||||
spec.specialist,
|
||||
"Produce.",
|
||||
budget=15,
|
||||
skills=("base",),
|
||||
tools=(MODULE_TOOL,) if spec.specialist == AUTHOR else (),
|
||||
modules_skill="modules-skill" if spec.specialist == AUTHOR else None,
|
||||
)
|
||||
for spec in DEPENDENCY_GRAPH.values()
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class Recording(FakeAgentRuntime):
|
||||
def __init__(self, turns):
|
||||
super().__init__(turns)
|
||||
self.seen: dict[str, dict] = {}
|
||||
|
||||
async def run(self, role, task, tools):
|
||||
self.seen[role.name] = {
|
||||
"skills": role.skills,
|
||||
"role_tools": role.tools,
|
||||
"context": task.context.get("terraform_modules"),
|
||||
"opted_in": MODULES_OPTED_IN.get(),
|
||||
}
|
||||
return await super().run(role, task, tools)
|
||||
|
||||
|
||||
def engine(tmp_path, **options) -> tuple[ExecutionEngine, Recording]:
|
||||
runtime = Recording(
|
||||
{
|
||||
spec.specialist: [[sample_submission(kind)]] * 2
|
||||
for kind, spec in DEPENDENCY_GRAPH.items()
|
||||
}
|
||||
)
|
||||
built = ExecutionEngine(
|
||||
runtime=runtime,
|
||||
roles=ROLES,
|
||||
recorder=InMemoryRecorder(),
|
||||
checkpointer=InMemorySaver(),
|
||||
output_root=tmp_path,
|
||||
orchestrator=Router(NoDecider(), None, Thresholds()),
|
||||
**options,
|
||||
)
|
||||
return built, runtime
|
||||
|
||||
|
||||
def manifest(tmp_path, execution_id: str) -> dict:
|
||||
folder = tmp_path / "deliverables" / "cases" / CASE / "executions" / execution_id
|
||||
return json.loads((folder / "execution.json").read_text())
|
||||
|
||||
|
||||
async def test_by_default_code_is_plain_resources(tmp_path):
|
||||
built, runtime = engine(tmp_path, modules_available=True)
|
||||
|
||||
await built.start("Ingest", customer_case=CASE, pre_approved=True, execution_id="a")
|
||||
|
||||
author = runtime.seen[AUTHOR]
|
||||
assert author["skills"] == ("base",)
|
||||
assert MODULE_TOOL not in author["role_tools"]
|
||||
assert author["context"] is False and author["opted_in"] is False
|
||||
assert manifest(tmp_path, "a")["terraform_modules"] is False
|
||||
|
||||
|
||||
async def test_an_execution_that_opts_in_is_given_the_skill_and_the_lookup(tmp_path):
|
||||
built, runtime = engine(tmp_path, modules_available=True)
|
||||
|
||||
await built.start(
|
||||
"Ingest",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id="a",
|
||||
terraform_modules=True,
|
||||
)
|
||||
|
||||
author = runtime.seen[AUTHOR]
|
||||
assert author["skills"] == ("base", "modules-skill")
|
||||
assert MODULE_TOOL in author["role_tools"]
|
||||
assert author["context"] is True and author["opted_in"] is True
|
||||
# A role that writes no code is given neither.
|
||||
assert runtime.seen["requirements-analyst"]["skills"] == ("base",)
|
||||
assert manifest(tmp_path, "a")["terraform_modules"] is True
|
||||
|
||||
|
||||
async def test_the_deployment_may_make_modules_the_default(tmp_path):
|
||||
built, runtime = engine(tmp_path, modules_available=True, default_modules=True)
|
||||
|
||||
await built.start("Ingest", customer_case=CASE, pre_approved=True, execution_id="a")
|
||||
assert runtime.seen[AUTHOR]["opted_in"] is True
|
||||
|
||||
await built.start(
|
||||
"Ingest",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id="b",
|
||||
terraform_modules=False,
|
||||
)
|
||||
assert runtime.seen[AUTHOR]["opted_in"] is False
|
||||
|
||||
|
||||
async def test_opting_in_is_refused_when_the_provider_approves_no_sources(tmp_path):
|
||||
built, runtime = engine(tmp_path)
|
||||
|
||||
with pytest.raises(ValueError, match="no approved module sources"):
|
||||
await built.start("Ingest", pre_approved=True, terraform_modules=True)
|
||||
with pytest.raises(ValueError, match="true or false"):
|
||||
await built.start("Ingest", pre_approved=True, terraform_modules="yes")
|
||||
assert runtime.seen == {}
|
||||
|
||||
|
||||
async def test_a_change_request_does_as_its_baseline_did(tmp_path):
|
||||
built, runtime = engine(tmp_path, modules_available=True)
|
||||
await built.start(
|
||||
"Ingest",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id="v1",
|
||||
terraform_modules=True,
|
||||
)
|
||||
runtime.seen.clear()
|
||||
|
||||
# sample submissions change nothing, so the Change request ends with warnings;
|
||||
# what matters is which way it started.
|
||||
await built.start(
|
||||
"Retention changes.",
|
||||
customer_case=CASE,
|
||||
pre_approved=True,
|
||||
execution_id="v2",
|
||||
baseline="v1",
|
||||
)
|
||||
|
||||
assert built is not None
|
||||
state = (await built.interaction("v2")).state
|
||||
assert state["terraform_modules"] is True
|
||||
|
||||
|
||||
async def test_a_module_call_is_refused_unless_the_execution_opted_in():
|
||||
runner = TerraformRunner(
|
||||
binary="no-such-binary", module_namespaces=("approved-modules",)
|
||||
)
|
||||
files = {"terraform/main.tf": MODULE_CALL}
|
||||
|
||||
token = MODULES_OPTED_IN.set(False)
|
||||
try:
|
||||
assert (await runner.validate(files)).message == NOT_OPTED_IN
|
||||
# Plain resources are untouched by the switch.
|
||||
plain = await runner.validate({"terraform/main.tf": "locals {}"})
|
||||
assert not plain.message.startswith("Refused")
|
||||
finally:
|
||||
MODULES_OPTED_IN.reset(token)
|
||||
|
||||
token = MODULES_OPTED_IN.set(True)
|
||||
try:
|
||||
opted = await runner.validate(files)
|
||||
assert not opted.message.startswith("Refused")
|
||||
finally:
|
||||
MODULES_OPTED_IN.reset(token)
|
||||
|
||||
|
||||
def test_the_gcp_bundle_names_its_modules_skill_for_the_roles_that_run_terraform():
|
||||
bundle = load_provider("gcp")
|
||||
roles = load_roles(AGENTS_DIR, bundle)
|
||||
|
||||
assert bundle.terraform_module_namespaces == ("terraform-google-modules",)
|
||||
with_skill = {name for name, role in roles.items() if role.modules_skill}
|
||||
assert with_skill == {"iac-author", "validator"}
|
||||
# The Skill is not among any role's Skills until an Execution opts in.
|
||||
assert all(
|
||||
bundle.terraform_modules_skill not in role.skills for role in roles.values()
|
||||
)
|
||||
72
tests/test_iac_modules.py
Normal file
72
tests/test_iac_modules.py
Normal file
@@ -0,0 +1,72 @@
|
||||
"""The IaC modules Skill: every module call validates against the pinned provider.
|
||||
|
||||
A module is usable only with provider versions its own requirements admit, and
|
||||
several of the organisation's modules do not admit the pinned one. Each reference's
|
||||
example is run through `terraform validate`, which downloads the module, so a
|
||||
version bump that breaks one is found here. With no network the test is skipped.
|
||||
"""
|
||||
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from app.config import Settings
|
||||
from app.providers import load_provider
|
||||
from app.tools.local import TerraformRunner
|
||||
from app.tools.modules import module_calls
|
||||
|
||||
SKILL = "google-cloud-iac-modules"
|
||||
SKILLS = Path(__file__).resolve().parent.parent / "skills"
|
||||
ROOT = SKILLS / SKILL
|
||||
VERSIONS = SKILLS / "google-cloud-iac-patterns" / "assets" / "versions.tf"
|
||||
REFERENCES = sorted((ROOT / "references").glob("*.md"))
|
||||
_HCL = re.compile(r"^```hcl\n(.*?)^```", re.MULTILINE | re.DOTALL)
|
||||
|
||||
needs_terraform = pytest.mark.skipif(
|
||||
shutil.which("terraform") is None, reason="terraform CLI not installed"
|
||||
)
|
||||
|
||||
|
||||
def _code(reference: Path) -> str:
|
||||
return "\n".join(_HCL.findall(reference.read_text(encoding="utf-8")))
|
||||
|
||||
|
||||
def test_every_reference_is_listed_and_calls_only_allowed_pinned_modules():
|
||||
bundle = load_provider("gcp")
|
||||
skill = (ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||||
assert REFERENCES
|
||||
listed = set(re.findall(r"references/([\w-]+\.md)", skill))
|
||||
assert listed == {reference.name for reference in REFERENCES}
|
||||
for reference in REFERENCES:
|
||||
calls = module_calls({"main.tf": _code(reference)})
|
||||
assert calls, reference.name
|
||||
for call in calls:
|
||||
assert call.source.split("/")[0] in bundle.terraform_module_namespaces
|
||||
# The version the example uses is the one the Skill's table gives.
|
||||
assert call.version in skill, f"{reference.name}: {call.version}"
|
||||
|
||||
|
||||
def test_the_skill_names_the_provider_version_the_bundle_pins():
|
||||
pin = load_provider("gcp").terraform_providers["google"]
|
||||
skill = (ROOT / "SKILL.md").read_text(encoding="utf-8")
|
||||
assert f"{pin['source']} {pin['version']}" in skill
|
||||
|
||||
|
||||
@needs_terraform
|
||||
@pytest.mark.parametrize("reference", REFERENCES, ids=lambda path: path.stem)
|
||||
async def test_module_example_validates(reference: Path):
|
||||
bundle = load_provider("gcp")
|
||||
runner = TerraformRunner(
|
||||
plugin_cache_dir=Settings().terraform_plugin_cache(),
|
||||
module_namespaces=bundle.terraform_module_namespaces,
|
||||
)
|
||||
files = {
|
||||
"terraform/versions.tf": VERSIONS.read_text(),
|
||||
"terraform/main.tf": _code(reference),
|
||||
}
|
||||
result = await runner.validate(files)
|
||||
if result.message.startswith("terraform init failed"):
|
||||
pytest.skip("the module or the pinned provider could not be downloaded")
|
||||
assert result.message.startswith("terraform validate passed"), result.message
|
||||
@@ -294,6 +294,7 @@ def test_local_tools_are_built_by_name_with_their_tier():
|
||||
"terraform_validate",
|
||||
"terraform_plan",
|
||||
"terraform_schema",
|
||||
"terraform_module",
|
||||
}
|
||||
with pytest.raises(KeyError, match="terraform_apply"):
|
||||
local_tools(["terraform_apply"], TerraformRunner())
|
||||
|
||||
203
tests/tools/test_modules.py
Normal file
203
tests/tools/test_modules.py
Normal file
@@ -0,0 +1,203 @@
|
||||
"""Terraform modules: which may be used, and what one accepts (iteration 8)."""
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from app.tools import modules
|
||||
from app.tools.local import TerraformRunner
|
||||
from app.tools.modules import ModuleLookup, admits, check_module_sources, module_calls
|
||||
|
||||
ALLOWED = ("example-modules",)
|
||||
GOOD = """
|
||||
module "network" {
|
||||
source = "example-modules/network/cloud"
|
||||
version = "1.2.3"
|
||||
|
||||
subnets = [
|
||||
{
|
||||
source = "not the module's source"
|
||||
version = "nor its version"
|
||||
}
|
||||
]
|
||||
}
|
||||
"""
|
||||
|
||||
|
||||
def refused(code: str, allowed=ALLOWED) -> str:
|
||||
result = check_module_sources({"terraform/main.tf": code}, allowed)
|
||||
return result.message if result else ""
|
||||
|
||||
|
||||
def test_a_module_call_is_read_with_its_own_source_and_version():
|
||||
(call,) = module_calls({"main.tf": GOOD})
|
||||
|
||||
assert (call.name, call.source, call.version) == (
|
||||
"network",
|
||||
"example-modules/network/cloud",
|
||||
"1.2.3",
|
||||
)
|
||||
|
||||
|
||||
def test_a_module_from_an_allowed_namespace_at_an_exact_version_is_accepted():
|
||||
assert refused(GOOD) == ""
|
||||
assert refused(GOOD.replace("network/cloud", "network/cloud//modules/nat")) == ""
|
||||
assert refused('resource "thing" "x" {}') == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"source",
|
||||
[
|
||||
"someone-else/network/cloud",
|
||||
"git::https://example.com/network.git",
|
||||
"github.com/example-modules/network",
|
||||
"./modules/network",
|
||||
"../shared",
|
||||
"https://example.com/network.zip",
|
||||
"",
|
||||
],
|
||||
)
|
||||
def test_any_other_source_is_refused(source):
|
||||
message = refused(GOOD.replace("example-modules/network/cloud", source))
|
||||
|
||||
assert message.startswith("Refused") and "example-modules" in message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", ["~> 1.2", ">= 1.0", "1.2", "latest", ""])
|
||||
def test_a_version_that_is_not_exact_is_refused(version):
|
||||
assert "one exact version" in refused(GOOD.replace("1.2.3", version))
|
||||
|
||||
|
||||
def test_with_no_namespace_configured_every_module_is_refused():
|
||||
assert "none are configured" in refused(GOOD, allowed=())
|
||||
|
||||
|
||||
def test_a_module_in_json_syntax_is_refused():
|
||||
result = check_module_sources(
|
||||
{"terraform/main.tf.json": '{"module": {"x": {"source": "a/b/c"}}}'}, ALLOWED
|
||||
)
|
||||
|
||||
assert result and result.message.startswith("Refused")
|
||||
|
||||
|
||||
async def test_validation_refuses_a_disallowed_module_before_terraform_runs():
|
||||
runner = TerraformRunner(binary="no-such-binary", module_namespaces=ALLOWED)
|
||||
|
||||
result = await runner.validate(
|
||||
{"terraform/main.tf": GOOD.replace("example-modules", "someone-else")}
|
||||
)
|
||||
|
||||
assert result.message.startswith("Refused: module")
|
||||
|
||||
|
||||
def test_a_version_constraint_is_read():
|
||||
assert admits(">= 6.37.0, < 9", "8.6.0")
|
||||
assert not admits(">= 6.2, < 8", "8.6.0")
|
||||
assert admits("~> 3.0", "3.7.2") and not admits("~> 3.0", "4.0.0")
|
||||
assert admits("~> 3.1.2", "3.1.9") and not admits("~> 3.1.2", "3.2.0")
|
||||
assert admits("", "8.6.0")
|
||||
|
||||
|
||||
REGISTRY_ANSWER = {
|
||||
"version": "1.2.3",
|
||||
"root": {
|
||||
"inputs": [
|
||||
{"name": "labels", "type": "map(string)", "description": "Labels."},
|
||||
{
|
||||
"name": "name",
|
||||
"type": "string",
|
||||
"description": "Name.",
|
||||
"required": True,
|
||||
},
|
||||
],
|
||||
"outputs": [{"name": "id"}],
|
||||
"provider_dependencies": [{"name": "cloud", "version": ">= 6, < 8"}],
|
||||
},
|
||||
"submodules": [
|
||||
{
|
||||
"path": "modules/nat",
|
||||
"inputs": [{"name": "router", "type": "string", "required": True}],
|
||||
"provider_dependencies": [{"name": "random", "version": "~> 3.0"}],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def registry(monkeypatch):
|
||||
asked: list[str] = []
|
||||
|
||||
class Client:
|
||||
def __init__(self, **_):
|
||||
pass
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_):
|
||||
return False
|
||||
|
||||
async def get(self, url):
|
||||
asked.append(url)
|
||||
if "missing" in url:
|
||||
return httpx.Response(404, json={"errors": ["Not Found"]})
|
||||
return httpx.Response(200, json=REGISTRY_ANSWER)
|
||||
|
||||
monkeypatch.setattr(modules.httpx, "AsyncClient", Client)
|
||||
return asked
|
||||
|
||||
|
||||
async def test_a_modules_inputs_come_back_required_first_with_its_outputs(registry):
|
||||
lookup = ModuleLookup(namespaces=ALLOWED, pinned={"cloud": "7.1.0"})
|
||||
|
||||
reply = (await lookup.describe("example-modules/network/cloud", "1.2.3")).message
|
||||
|
||||
assert reply.index("- name (required") < reply.index("- labels (optional")
|
||||
assert "Outputs: id" in reply and "Submodules: modules/nat" in reply
|
||||
assert "NOT USABLE" not in reply
|
||||
assert registry == [f"{modules.REGISTRY}/example-modules/network/cloud/1.2.3"]
|
||||
# The same module is not fetched twice.
|
||||
await lookup.describe("example-modules/network/cloud", "1.2.3", find="name")
|
||||
assert len(registry) == 1
|
||||
|
||||
|
||||
async def test_a_module_the_pinned_provider_does_not_satisfy_is_called_out(registry):
|
||||
lookup = ModuleLookup(namespaces=ALLOWED, pinned={"cloud": "8.6.0"})
|
||||
|
||||
reply = (await lookup.describe("example-modules/network/cloud")).message
|
||||
|
||||
assert "NOT USABLE: it requires cloud >= 6, < 8" in reply
|
||||
assert "pinned to cloud 8.6.0" in reply
|
||||
|
||||
|
||||
async def test_a_submodule_is_described_by_its_own_inputs(registry):
|
||||
lookup = ModuleLookup(namespaces=ALLOWED, pinned={"cloud": "8.6.0"})
|
||||
|
||||
reply = (
|
||||
await lookup.describe("example-modules/network/cloud//modules/nat")
|
||||
).message
|
||||
|
||||
assert "- router (required" in reply and "- name" not in reply
|
||||
assert "It also uses the random provider" in reply
|
||||
missing = await lookup.describe("example-modules/network/cloud//modules/vpn")
|
||||
assert "has no submodule" in missing.message and "modules/nat" in missing.message
|
||||
|
||||
|
||||
async def test_find_narrows_the_inputs(registry):
|
||||
lookup = ModuleLookup(namespaces=ALLOWED)
|
||||
|
||||
reply = (
|
||||
await lookup.describe("example-modules/network/cloud", find="label")
|
||||
).message
|
||||
|
||||
assert "- labels" in reply and "- name (" not in reply
|
||||
|
||||
|
||||
async def test_a_module_outside_the_allowed_namespaces_is_not_looked_up(registry):
|
||||
lookup = ModuleLookup(namespaces=ALLOWED)
|
||||
|
||||
reply = (await lookup.describe("someone-else/network/cloud")).message
|
||||
|
||||
assert reply.startswith("Refused") and registry == []
|
||||
assert (
|
||||
"no module" in (await lookup.describe("example-modules/missing/cloud")).message
|
||||
)
|
||||
Reference in New Issue
Block a user