feat: keep delivered work in S3-compatible object storage (ADR-0004 phase 3)
The store's rules now sit on a Blobs layer: a folder, as before, or a prefix in an S3-compatible bucket (MinIO locally), chosen by STORE_BACKEND. The layout is the same in both; the record is written last. - S3_BUCKET, S3_PREFIX, S3_ENDPOINT_URL, S3_REGION; AWS credentials - an s3 extra (boto3), installed in the image; moto for tests - every store test runs on both kinds of storage Retention, the platform's sign-in and each kept file as a platform artifact are left: the last needs the platform's S3 backend, which it does not yet have. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -2,10 +2,30 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from collections.abc import Iterator
|
||||
from typing import Any
|
||||
|
||||
import boto3
|
||||
import pytest
|
||||
from moto import mock_aws
|
||||
|
||||
from orchestration.config import OrchestrationSettings
|
||||
from orchestration.links import Signer
|
||||
from orchestration.store import Store
|
||||
from orchestration.store import S3Blobs, Store, VolumeBlobs, store_from_settings
|
||||
|
||||
|
||||
@pytest.fixture(params=["volume", "s3"])
|
||||
def make_store(request: Any, tmp_path: Any) -> Iterator[Any]:
|
||||
"""A store on each kind of storage: a folder, and S3 (moto in this process)."""
|
||||
if request.param == "volume":
|
||||
yield lambda: Store(VolumeBlobs(tmp_path))
|
||||
return
|
||||
with mock_aws():
|
||||
client = boto3.client("s3", region_name="us-east-1")
|
||||
yield lambda: Store(
|
||||
S3Blobs(client, "kept", prefix="orchestration/", create=True)
|
||||
)
|
||||
|
||||
|
||||
FILES = {
|
||||
"guide.md": "# Guide",
|
||||
@@ -30,8 +50,8 @@ def kept(store: Store, execution_id: str = "a1", **overrides: object) -> dict:
|
||||
return store.keep(**values)
|
||||
|
||||
|
||||
def test_an_execution_is_kept_with_its_record(tmp_path):
|
||||
store = Store(tmp_path)
|
||||
def test_an_execution_is_kept_with_its_record(make_store):
|
||||
store = make_store()
|
||||
|
||||
record = kept(store)
|
||||
|
||||
@@ -43,8 +63,8 @@ def test_an_execution_is_kept_with_its_record(tmp_path):
|
||||
assert store.record("t1", "acme", "a1") == record
|
||||
|
||||
|
||||
def test_a_file_comes_back_with_its_media_type(tmp_path):
|
||||
store = Store(tmp_path)
|
||||
def test_a_file_comes_back_with_its_media_type(make_store):
|
||||
store = make_store()
|
||||
kept(store)
|
||||
|
||||
assert store.file("t1", "acme", "a1", "guide.md") == (
|
||||
@@ -59,8 +79,8 @@ def test_a_file_comes_back_with_its_media_type(tmp_path):
|
||||
assert store.file("t1", "acme", "a1", "missing.md") is None
|
||||
|
||||
|
||||
def test_what_is_kept_is_never_changed(tmp_path):
|
||||
store = Store(tmp_path)
|
||||
def test_what_is_kept_is_never_changed(make_store):
|
||||
store = make_store()
|
||||
kept(store)
|
||||
|
||||
with pytest.raises(ValueError, match="a1"):
|
||||
@@ -70,8 +90,8 @@ def test_what_is_kept_is_never_changed(tmp_path):
|
||||
@pytest.mark.parametrize(
|
||||
"name", ["../escape.md", "/etc/passwd", "a/../../b", "a\\b", ""]
|
||||
)
|
||||
def test_a_file_name_outside_the_execution_is_refused(tmp_path, name):
|
||||
store = Store(tmp_path)
|
||||
def test_a_file_name_outside_the_execution_is_refused(make_store, name):
|
||||
store = make_store()
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
kept(store, files={name: "x"})
|
||||
@@ -79,13 +99,13 @@ def test_a_file_name_outside_the_execution_is_refused(tmp_path, name):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("slug", ["..", "a/b", "", "a b"])
|
||||
def test_a_tenant_or_case_that_is_not_a_slug_is_refused(tmp_path, slug):
|
||||
def test_a_tenant_or_case_that_is_not_a_slug_is_refused(make_store, slug):
|
||||
with pytest.raises(ValueError):
|
||||
kept(Store(tmp_path), case=slug)
|
||||
kept(make_store(), case=slug)
|
||||
|
||||
|
||||
def test_latest_is_the_newest_completed_execution_of_the_work(tmp_path):
|
||||
store = Store(tmp_path)
|
||||
def test_latest_is_the_newest_completed_execution_of_the_work(make_store):
|
||||
store = make_store()
|
||||
kept(store, "a1")
|
||||
kept(store, "a2")
|
||||
kept(store, "a3", status="completed_with_warnings")
|
||||
@@ -101,6 +121,53 @@ def test_latest_is_the_newest_completed_execution_of_the_work(tmp_path):
|
||||
]
|
||||
|
||||
|
||||
def test_a_delivered_file_named_like_a_record_is_not_taken_for_one(make_store):
|
||||
store = make_store()
|
||||
kept(store, files={"record.json": '{"not": "a record"}'})
|
||||
|
||||
assert [r["execution_id"] for r in store.case("t1", "acme")] == ["a1"]
|
||||
|
||||
|
||||
def test_object_storage_keeps_the_same_layout_under_its_prefix():
|
||||
with mock_aws():
|
||||
client = boto3.client("s3", region_name="us-east-1")
|
||||
kept(Store(S3Blobs(client, "kept", prefix="orchestration/", create=True)))
|
||||
|
||||
keys = sorted(
|
||||
item["Key"] for item in client.list_objects_v2(Bucket="kept")["Contents"]
|
||||
)
|
||||
|
||||
base = "orchestration/tenants/t1/cases/acme/executions/a1"
|
||||
assert keys == [
|
||||
f"{base}/files/graph.json",
|
||||
f"{base}/files/guide.md",
|
||||
f"{base}/files/terraform/main.tf",
|
||||
f"{base}/record.json",
|
||||
]
|
||||
|
||||
|
||||
def test_the_settings_choose_the_storage(tmp_path, monkeypatch):
|
||||
assert store_from_settings(OrchestrationSettings(_env_file=None)) is None # type: ignore[call-arg]
|
||||
volume = store_from_settings(
|
||||
OrchestrationSettings(_env_file=None, STORE_DIR=str(tmp_path)) # type: ignore[call-arg]
|
||||
)
|
||||
assert isinstance(volume, Store) and isinstance(volume.blobs, VolumeBlobs)
|
||||
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test")
|
||||
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test")
|
||||
with mock_aws():
|
||||
s3 = store_from_settings(
|
||||
OrchestrationSettings( # type: ignore[call-arg]
|
||||
_env_file=None,
|
||||
STORE_BACKEND="s3",
|
||||
S3_BUCKET="kept",
|
||||
S3_REGION="us-east-1",
|
||||
)
|
||||
)
|
||||
assert isinstance(s3, Store) and isinstance(s3.blobs, S3Blobs)
|
||||
kept(s3)
|
||||
assert s3.record("t1", "acme", "a1") is not None
|
||||
|
||||
|
||||
def test_a_signed_path_is_accepted_and_a_changed_one_is_not():
|
||||
signer = Signer("secret", ttl_seconds=60, now=lambda: 1000)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user