feat: keep delivered work in S3-compatible object storage (ADR-0004 phase 3)

The store's rules now sit on a Blobs layer: a folder, as before, or a
prefix in an S3-compatible bucket (MinIO locally), chosen by STORE_BACKEND.
The layout is the same in both; the record is written last.

- S3_BUCKET, S3_PREFIX, S3_ENDPOINT_URL, S3_REGION; AWS credentials
- an s3 extra (boto3), installed in the image; moto for tests
- every store test runs on both kinds of storage

Retention, the platform's sign-in and each kept file as a platform
artifact are left: the last needs the platform's S3 backend, which it does
not yet have.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-21 21:08:39 +01:00
parent 1ec939f4fd
commit 83df77005a
9 changed files with 314 additions and 57 deletions

View File

@@ -2,10 +2,30 @@
from __future__ import annotations
import pytest
from collections.abc import Iterator
from typing import Any
import boto3
import pytest
from moto import mock_aws
from orchestration.config import OrchestrationSettings
from orchestration.links import Signer
from orchestration.store import Store
from orchestration.store import S3Blobs, Store, VolumeBlobs, store_from_settings
@pytest.fixture(params=["volume", "s3"])
def make_store(request: Any, tmp_path: Any) -> Iterator[Any]:
"""A store on each kind of storage: a folder, and S3 (moto in this process)."""
if request.param == "volume":
yield lambda: Store(VolumeBlobs(tmp_path))
return
with mock_aws():
client = boto3.client("s3", region_name="us-east-1")
yield lambda: Store(
S3Blobs(client, "kept", prefix="orchestration/", create=True)
)
FILES = {
"guide.md": "# Guide",
@@ -30,8 +50,8 @@ def kept(store: Store, execution_id: str = "a1", **overrides: object) -> dict:
return store.keep(**values)
def test_an_execution_is_kept_with_its_record(tmp_path):
store = Store(tmp_path)
def test_an_execution_is_kept_with_its_record(make_store):
store = make_store()
record = kept(store)
@@ -43,8 +63,8 @@ def test_an_execution_is_kept_with_its_record(tmp_path):
assert store.record("t1", "acme", "a1") == record
def test_a_file_comes_back_with_its_media_type(tmp_path):
store = Store(tmp_path)
def test_a_file_comes_back_with_its_media_type(make_store):
store = make_store()
kept(store)
assert store.file("t1", "acme", "a1", "guide.md") == (
@@ -59,8 +79,8 @@ def test_a_file_comes_back_with_its_media_type(tmp_path):
assert store.file("t1", "acme", "a1", "missing.md") is None
def test_what_is_kept_is_never_changed(tmp_path):
store = Store(tmp_path)
def test_what_is_kept_is_never_changed(make_store):
store = make_store()
kept(store)
with pytest.raises(ValueError, match="a1"):
@@ -70,8 +90,8 @@ def test_what_is_kept_is_never_changed(tmp_path):
@pytest.mark.parametrize(
"name", ["../escape.md", "/etc/passwd", "a/../../b", "a\\b", ""]
)
def test_a_file_name_outside_the_execution_is_refused(tmp_path, name):
store = Store(tmp_path)
def test_a_file_name_outside_the_execution_is_refused(make_store, name):
store = make_store()
with pytest.raises(ValueError):
kept(store, files={name: "x"})
@@ -79,13 +99,13 @@ def test_a_file_name_outside_the_execution_is_refused(tmp_path, name):
@pytest.mark.parametrize("slug", ["..", "a/b", "", "a b"])
def test_a_tenant_or_case_that_is_not_a_slug_is_refused(tmp_path, slug):
def test_a_tenant_or_case_that_is_not_a_slug_is_refused(make_store, slug):
with pytest.raises(ValueError):
kept(Store(tmp_path), case=slug)
kept(make_store(), case=slug)
def test_latest_is_the_newest_completed_execution_of_the_work(tmp_path):
store = Store(tmp_path)
def test_latest_is_the_newest_completed_execution_of_the_work(make_store):
store = make_store()
kept(store, "a1")
kept(store, "a2")
kept(store, "a3", status="completed_with_warnings")
@@ -101,6 +121,53 @@ def test_latest_is_the_newest_completed_execution_of_the_work(tmp_path):
]
def test_a_delivered_file_named_like_a_record_is_not_taken_for_one(make_store):
store = make_store()
kept(store, files={"record.json": '{"not": "a record"}'})
assert [r["execution_id"] for r in store.case("t1", "acme")] == ["a1"]
def test_object_storage_keeps_the_same_layout_under_its_prefix():
with mock_aws():
client = boto3.client("s3", region_name="us-east-1")
kept(Store(S3Blobs(client, "kept", prefix="orchestration/", create=True)))
keys = sorted(
item["Key"] for item in client.list_objects_v2(Bucket="kept")["Contents"]
)
base = "orchestration/tenants/t1/cases/acme/executions/a1"
assert keys == [
f"{base}/files/graph.json",
f"{base}/files/guide.md",
f"{base}/files/terraform/main.tf",
f"{base}/record.json",
]
def test_the_settings_choose_the_storage(tmp_path, monkeypatch):
assert store_from_settings(OrchestrationSettings(_env_file=None)) is None # type: ignore[call-arg]
volume = store_from_settings(
OrchestrationSettings(_env_file=None, STORE_DIR=str(tmp_path)) # type: ignore[call-arg]
)
assert isinstance(volume, Store) and isinstance(volume.blobs, VolumeBlobs)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test")
with mock_aws():
s3 = store_from_settings(
OrchestrationSettings( # type: ignore[call-arg]
_env_file=None,
STORE_BACKEND="s3",
S3_BUCKET="kept",
S3_REGION="us-east-1",
)
)
assert isinstance(s3, Store) and isinstance(s3.blobs, S3Blobs)
kept(s3)
assert s3.record("t1", "acme", "a1") is not None
def test_a_signed_path_is_accepted_and_a_changed_one_is_not():
signer = Signer("secret", ttl_seconds=60, now=lambda: 1000)