Files
orchestration_engine/tests/orchestration/test_store.py
Jonathan Boniface 83df77005a feat: keep delivered work in S3-compatible object storage (ADR-0004 phase 3)
The store's rules now sit on a Blobs layer: a folder, as before, or a
prefix in an S3-compatible bucket (MinIO locally), chosen by STORE_BACKEND.
The layout is the same in both; the record is written last.

- S3_BUCKET, S3_PREFIX, S3_ENDPOINT_URL, S3_REGION; AWS credentials
- an s3 extra (boto3), installed in the image; moto for tests
- every store test runs on both kinds of storage

Retention, the platform's sign-in and each kept file as a platform
artifact are left: the last needs the platform's S3 backend, which it does
not yet have.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 21:08:39 +01:00

194 lines
6.1 KiB
Python

"""The store of what agents delivered, and the signed links to it (ADR-0004)."""
from __future__ import annotations
from collections.abc import Iterator
from typing import Any
import boto3
import pytest
from moto import mock_aws
from orchestration.config import OrchestrationSettings
from orchestration.links import Signer
from orchestration.store import S3Blobs, Store, VolumeBlobs, store_from_settings
@pytest.fixture(params=["volume", "s3"])
def make_store(request: Any, tmp_path: Any) -> Iterator[Any]:
"""A store on each kind of storage: a folder, and S3 (moto in this process)."""
if request.param == "volume":
yield lambda: Store(VolumeBlobs(tmp_path))
return
with mock_aws():
client = boto3.client("s3", region_name="us-east-1")
yield lambda: Store(
S3Blobs(client, "kept", prefix="orchestration/", create=True)
)
FILES = {
"guide.md": "# Guide",
"graph.json": '{"elements": []}',
"terraform/main.tf": 'resource "x" "y" {}',
}
def kept(store: Store, execution_id: str = "a1", **overrides: object) -> dict:
values: dict = {
"tenant": "t1",
"case": "acme",
"execution_id": execution_id,
"work": "architecture",
"status": "completed",
"request": "A landing zone",
"files": FILES,
"guide": "guide.md",
"project_id": "p1",
}
values.update(overrides)
return store.keep(**values)
def test_an_execution_is_kept_with_its_record(make_store):
store = make_store()
record = kept(store)
assert record["work"] == "architecture"
assert record["status"] == "completed"
assert record["customer_case"] == "acme"
assert record["project_id"] == "p1"
assert record["files"] == sorted(FILES)
assert store.record("t1", "acme", "a1") == record
def test_a_file_comes_back_with_its_media_type(make_store):
store = make_store()
kept(store)
assert store.file("t1", "acme", "a1", "guide.md") == (
b"# Guide",
"text/markdown; charset=utf-8",
)
assert store.file("t1", "acme", "a1", "graph.json")[1] == "application/json"
assert (
store.file("t1", "acme", "a1", "terraform/main.tf")[0]
== FILES["terraform/main.tf"].encode()
)
assert store.file("t1", "acme", "a1", "missing.md") is None
def test_what_is_kept_is_never_changed(make_store):
store = make_store()
kept(store)
with pytest.raises(ValueError, match="a1"):
kept(store, files={"guide.md": "# Another"})
@pytest.mark.parametrize(
"name", ["../escape.md", "/etc/passwd", "a/../../b", "a\\b", ""]
)
def test_a_file_name_outside_the_execution_is_refused(make_store, name):
store = make_store()
with pytest.raises(ValueError):
kept(store, files={name: "x"})
assert store.file("t1", "acme", "a1", name) is None
@pytest.mark.parametrize("slug", ["..", "a/b", "", "a b"])
def test_a_tenant_or_case_that_is_not_a_slug_is_refused(make_store, slug):
with pytest.raises(ValueError):
kept(make_store(), case=slug)
def test_latest_is_the_newest_completed_execution_of_the_work(make_store):
store = make_store()
kept(store, "a1")
kept(store, "a2")
kept(store, "a3", status="completed_with_warnings")
kept(store, "o1", work="operations")
assert store.latest("t1", "acme", "architecture")["execution_id"] == "a2"
assert store.latest("t1", "acme", "nothing") is None
assert [r["execution_id"] for r in store.case("t1", "acme")] == [
"o1",
"a3",
"a2",
"a1",
]
def test_a_delivered_file_named_like_a_record_is_not_taken_for_one(make_store):
store = make_store()
kept(store, files={"record.json": '{"not": "a record"}'})
assert [r["execution_id"] for r in store.case("t1", "acme")] == ["a1"]
def test_object_storage_keeps_the_same_layout_under_its_prefix():
with mock_aws():
client = boto3.client("s3", region_name="us-east-1")
kept(Store(S3Blobs(client, "kept", prefix="orchestration/", create=True)))
keys = sorted(
item["Key"] for item in client.list_objects_v2(Bucket="kept")["Contents"]
)
base = "orchestration/tenants/t1/cases/acme/executions/a1"
assert keys == [
f"{base}/files/graph.json",
f"{base}/files/guide.md",
f"{base}/files/terraform/main.tf",
f"{base}/record.json",
]
def test_the_settings_choose_the_storage(tmp_path, monkeypatch):
assert store_from_settings(OrchestrationSettings(_env_file=None)) is None # type: ignore[call-arg]
volume = store_from_settings(
OrchestrationSettings(_env_file=None, STORE_DIR=str(tmp_path)) # type: ignore[call-arg]
)
assert isinstance(volume, Store) and isinstance(volume.blobs, VolumeBlobs)
monkeypatch.setenv("AWS_ACCESS_KEY_ID", "test")
monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "test")
with mock_aws():
s3 = store_from_settings(
OrchestrationSettings( # type: ignore[call-arg]
_env_file=None,
STORE_BACKEND="s3",
S3_BUCKET="kept",
S3_REGION="us-east-1",
)
)
assert isinstance(s3, Store) and isinstance(s3.blobs, S3Blobs)
kept(s3)
assert s3.record("t1", "acme", "a1") is not None
def test_a_signed_path_is_accepted_and_a_changed_one_is_not():
signer = Signer("secret", ttl_seconds=60, now=lambda: 1000)
query = signer.sign("/tenants/t1/cases/acme")
assert signer.verify("/tenants/t1/cases/acme", **query)
assert not signer.verify("/tenants/t1/cases/other", **query)
assert not signer.verify("/tenants/t1/cases/acme", exp=query["exp"], sig="0" * 64)
assert not Signer("other", ttl_seconds=60, now=lambda: 1000).verify(
"/tenants/t1/cases/acme", **query
)
def test_a_link_expires_unless_its_lifetime_is_unlimited():
clock = {"now": 1000}
expiring = Signer("secret", ttl_seconds=60, now=lambda: clock["now"])
forever = Signer("secret", ttl_seconds=0, now=lambda: clock["now"])
short, lasting = expiring.sign("/x"), forever.sign("/x")
clock["now"] = 1061
assert not expiring.verify("/x", **short)
assert forever.verify("/x", **lasting)