feat: keep delivered work in S3-compatible object storage (ADR-0004 phase 3)
The store's rules now sit on a Blobs layer: a folder, as before, or a prefix in an S3-compatible bucket (MinIO locally), chosen by STORE_BACKEND. The layout is the same in both; the record is written last. - S3_BUCKET, S3_PREFIX, S3_ENDPOINT_URL, S3_REGION; AWS credentials - an s3 extra (boto3), installed in the image; moto for tests - every store test runs on both kinds of storage Retention, the platform's sign-in and each kept file as a platform artifact are left: the last needs the platform's S3 backend, which it does not yet have. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -27,9 +27,16 @@ class OrchestrationSettings(BaseSettings):
|
||||
# agent's work takes.
|
||||
CONNECT_TIMEOUT_SECONDS: float = 10
|
||||
|
||||
# What agents deliver is kept here and served at signed links (ADR-0004).
|
||||
# Unset: nothing is kept, and a caller asking for links gets none.
|
||||
# What agents deliver is kept and served at signed links (ADR-0004): in a folder
|
||||
# (`volume`, STORE_DIR; unset, nothing is kept) or in an S3-compatible bucket
|
||||
# (`s3`). The bucket's credentials are the standard AWS_ACCESS_KEY_ID and
|
||||
# AWS_SECRET_ACCESS_KEY.
|
||||
STORE_BACKEND: str = "volume"
|
||||
STORE_DIR: str | None = None
|
||||
S3_BUCKET: str | None = None
|
||||
S3_PREFIX: str = ""
|
||||
S3_ENDPOINT_URL: str | None = None # MinIO, or another S3-compatible service
|
||||
S3_REGION: str = "us-east-1"
|
||||
# The base of every link: the address a browser reaches this service at, which
|
||||
# is not its address inside the platform's network. Unset: A2A_PUBLIC_URL.
|
||||
PUBLIC_URL: str | None = None
|
||||
|
||||
@@ -29,7 +29,7 @@ from orchestration.delivery import Delivery
|
||||
from orchestration.links import Signer
|
||||
from orchestration.relay import AgentClients, RelayExecutor
|
||||
from orchestration.serving import kept_routes
|
||||
from orchestration.store import Store
|
||||
from orchestration.store import Store, store_from_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -121,8 +121,8 @@ def _delivery(
|
||||
) -> Delivery | None:
|
||||
"""Where what agents deliver is kept, and how it is linked to; None when
|
||||
nothing is to be kept."""
|
||||
if store is None and settings.STORE_DIR:
|
||||
store = Store(settings.STORE_DIR)
|
||||
if store is None:
|
||||
store = store_from_settings(settings)
|
||||
if store is None:
|
||||
return None
|
||||
if signer is None:
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
"""What agents delivered, kept by tenant, Customer case and Execution (ADR-0004).
|
||||
|
||||
<root>/tenants/<tenant>/cases/<case>/executions/<id>/record.json
|
||||
/files/<name as delivered>
|
||||
tenants/<tenant>/cases/<case>/executions/<id>/record.json
|
||||
/files/<name as delivered>
|
||||
|
||||
An Execution is written once. Its files are what the agent delivered, by the names
|
||||
it gave them; its record says what it was.
|
||||
it gave them; its record says what it was. The same layout is kept in a folder or
|
||||
under a prefix in an S3-compatible bucket.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -15,7 +16,10 @@ import re
|
||||
import time
|
||||
from collections.abc import Mapping
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any, Protocol
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from orchestration.config import OrchestrationSettings
|
||||
|
||||
_SLUG = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
||||
_MEDIA_TYPES = {
|
||||
@@ -53,21 +57,128 @@ def media_type(name: str) -> str:
|
||||
return _MEDIA_TYPES.get(PurePosixPath(name).suffix.lower(), _TEXT)
|
||||
|
||||
|
||||
class Store:
|
||||
class Blobs(Protocol):
|
||||
"""Where a store's keys are kept: a folder, or a bucket."""
|
||||
|
||||
def put(self, key: str, data: bytes, content_type: str) -> None: ...
|
||||
def get(self, key: str) -> bytes | None: ...
|
||||
def exists(self, key: str) -> bool: ...
|
||||
def keys(self, prefix: str) -> list[str]: ...
|
||||
|
||||
|
||||
class VolumeBlobs:
|
||||
"""Keys as files under a folder."""
|
||||
|
||||
def __init__(self, root: str | Path) -> None:
|
||||
self.root = Path(root)
|
||||
|
||||
def _case(self, tenant: str, case: str) -> Path:
|
||||
return (
|
||||
self.root
|
||||
/ "tenants"
|
||||
/ slug(tenant, "Tenant")
|
||||
/ "cases"
|
||||
/ slug(case, "Case")
|
||||
def _path(self, key: str) -> Path:
|
||||
path = (self.root / key).resolve()
|
||||
if self.root.resolve() not in path.parents:
|
||||
raise ValueError(f"{key!r} is outside the store")
|
||||
return path
|
||||
|
||||
def put(self, key: str, data: bytes, content_type: str) -> None:
|
||||
path = self._path(key)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
|
||||
def get(self, key: str) -> bytes | None:
|
||||
path = self._path(key)
|
||||
return path.read_bytes() if path.is_file() else None
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
return self._path(key).is_file()
|
||||
|
||||
def keys(self, prefix: str) -> list[str]:
|
||||
folder = self.root / prefix
|
||||
if not folder.is_dir():
|
||||
return []
|
||||
return [
|
||||
path.relative_to(self.root).as_posix()
|
||||
for path in folder.rglob("*")
|
||||
if path.is_file()
|
||||
]
|
||||
|
||||
|
||||
class S3Blobs:
|
||||
"""Keys as objects under a prefix in an S3-compatible bucket (MinIO locally)."""
|
||||
|
||||
def __init__(
|
||||
self, client: Any, bucket: str, prefix: str = "", create: bool = False
|
||||
) -> None:
|
||||
self._client = client
|
||||
self.bucket = bucket
|
||||
self.prefix = prefix.lstrip("/")
|
||||
if create:
|
||||
self._ensure_bucket()
|
||||
|
||||
def _ensure_bucket(self) -> None:
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
try:
|
||||
self._client.head_bucket(Bucket=self.bucket)
|
||||
except ClientError:
|
||||
self._client.create_bucket(Bucket=self.bucket)
|
||||
|
||||
def put(self, key: str, data: bytes, content_type: str) -> None:
|
||||
self._client.put_object(
|
||||
Bucket=self.bucket,
|
||||
Key=self.prefix + key,
|
||||
Body=data,
|
||||
ContentType=content_type,
|
||||
)
|
||||
|
||||
def _execution(self, tenant: str, case: str, execution_id: str) -> Path:
|
||||
return self._case(tenant, case) / "executions" / slug(execution_id, "Execution")
|
||||
def get(self, key: str) -> bytes | None:
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
try:
|
||||
response = self._client.get_object(
|
||||
Bucket=self.bucket, Key=self.prefix + key
|
||||
)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
||||
return None
|
||||
raise
|
||||
data: bytes = response["Body"].read()
|
||||
return data
|
||||
|
||||
def exists(self, key: str) -> bool:
|
||||
from botocore.exceptions import ClientError
|
||||
|
||||
try:
|
||||
self._client.head_object(Bucket=self.bucket, Key=self.prefix + key)
|
||||
except ClientError as exc:
|
||||
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
||||
return False
|
||||
raise
|
||||
return True
|
||||
|
||||
def keys(self, prefix: str) -> list[str]:
|
||||
found: list[str] = []
|
||||
pages = self._client.get_paginator("list_objects_v2").paginate(
|
||||
Bucket=self.bucket, Prefix=self.prefix + prefix
|
||||
)
|
||||
for page in pages:
|
||||
for item in page.get("Contents", []):
|
||||
found.append(str(item["Key"])[len(self.prefix) :])
|
||||
return found
|
||||
|
||||
|
||||
class Store:
|
||||
def __init__(self, blobs: Blobs | str | Path) -> None:
|
||||
self.blobs: Blobs = (
|
||||
VolumeBlobs(blobs) if isinstance(blobs, (str, Path)) else blobs
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _case(tenant: str, case: str) -> str:
|
||||
return f"tenants/{slug(tenant, 'Tenant')}/cases/{slug(case, 'Case')}"
|
||||
|
||||
def _execution(self, tenant: str, case: str, execution_id: str) -> str:
|
||||
return (
|
||||
f"{self._case(tenant, case)}/executions/{slug(execution_id, 'Execution')}"
|
||||
)
|
||||
|
||||
def keep(
|
||||
self,
|
||||
@@ -86,12 +197,12 @@ class Store:
|
||||
already kept is refused."""
|
||||
folder = self._execution(tenant, case, execution_id)
|
||||
paths = {name: file_name(name) for name in files}
|
||||
if folder.exists():
|
||||
if self.blobs.exists(f"{folder}/record.json"):
|
||||
raise ValueError(f"Execution {execution_id} is already kept")
|
||||
for name, path in paths.items():
|
||||
target = folder / "files" / path
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
target.write_text(files[name], encoding="utf-8")
|
||||
self.blobs.put(
|
||||
f"{folder}/files/{path}", files[name].encode("utf-8"), media_type(name)
|
||||
)
|
||||
record = {
|
||||
"tenant": tenant,
|
||||
"customer_case": case,
|
||||
@@ -105,20 +216,26 @@ class Store:
|
||||
"kept_at": dt.datetime.now(dt.UTC).isoformat(),
|
||||
"kept_ns": time.time_ns(),
|
||||
}
|
||||
folder.mkdir(parents=True, exist_ok=True)
|
||||
(folder / "record.json").write_text(json.dumps(record, indent=2), "utf-8")
|
||||
# The record last: an Execution is kept once its record is.
|
||||
self.blobs.put(
|
||||
f"{folder}/record.json",
|
||||
json.dumps(record, indent=2).encode("utf-8"),
|
||||
"application/json",
|
||||
)
|
||||
return record
|
||||
|
||||
def record(
|
||||
self, tenant: str, case: str, execution_id: str
|
||||
) -> dict[str, Any] | None:
|
||||
try:
|
||||
path = self._execution(tenant, case, execution_id) / "record.json"
|
||||
data = self.blobs.get(
|
||||
f"{self._execution(tenant, case, execution_id)}/record.json"
|
||||
)
|
||||
except ValueError:
|
||||
return None
|
||||
if not path.is_file():
|
||||
if data is None:
|
||||
return None
|
||||
record: dict[str, Any] = json.loads(path.read_text("utf-8"))
|
||||
record: dict[str, Any] = json.loads(data)
|
||||
return record
|
||||
|
||||
def file(
|
||||
@@ -126,13 +243,13 @@ class Store:
|
||||
) -> tuple[bytes, str] | None:
|
||||
"""A kept file's content and media type, or None."""
|
||||
try:
|
||||
folder = self._execution(tenant, case, execution_id) / "files"
|
||||
target = folder / file_name(name)
|
||||
key = (
|
||||
f"{self._execution(tenant, case, execution_id)}/files/{file_name(name)}"
|
||||
)
|
||||
data = self.blobs.get(key)
|
||||
except ValueError:
|
||||
return None
|
||||
if not target.is_file() or folder.resolve() not in target.resolve().parents:
|
||||
return None
|
||||
return target.read_bytes(), media_type(name)
|
||||
return None if data is None else (data, media_type(name))
|
||||
|
||||
def files(self, tenant: str, case: str, execution_id: str) -> dict[str, str]:
|
||||
"""Every file kept for an Execution, by name."""
|
||||
@@ -147,12 +264,17 @@ class Store:
|
||||
def case(self, tenant: str, case: str) -> list[dict[str, Any]]:
|
||||
"""The Customer case's Executions, newest first."""
|
||||
try:
|
||||
folder = self._case(tenant, case) / "executions"
|
||||
prefix = f"{self._case(tenant, case)}/executions/"
|
||||
except ValueError:
|
||||
return []
|
||||
records = [
|
||||
json.loads(path.read_text("utf-8")) for path in folder.glob("*/record.json")
|
||||
]
|
||||
records = []
|
||||
for key in self.blobs.keys(prefix):
|
||||
# <id>/record.json; a delivered file of that name is under files/.
|
||||
parts = key[len(prefix) :].split("/")
|
||||
if len(parts) == 2 and parts[1] == "record.json":
|
||||
data = self.blobs.get(key)
|
||||
if data is not None:
|
||||
records.append(json.loads(data))
|
||||
return sorted(records, key=lambda r: r.get("kept_ns", 0), reverse=True)
|
||||
|
||||
def latest(self, tenant: str, case: str, work: str) -> dict[str, Any] | None:
|
||||
@@ -165,3 +287,25 @@ class Store:
|
||||
),
|
||||
None,
|
||||
)
|
||||
|
||||
|
||||
def store_from_settings(settings: OrchestrationSettings) -> Store | None:
|
||||
"""The store the settings name; None when nothing is to be kept."""
|
||||
if settings.STORE_BACKEND == "s3":
|
||||
import boto3
|
||||
|
||||
if not settings.S3_BUCKET:
|
||||
raise ValueError("STORE_BACKEND=s3 needs S3_BUCKET")
|
||||
client = boto3.client(
|
||||
"s3",
|
||||
endpoint_url=settings.S3_ENDPOINT_URL or None,
|
||||
region_name=settings.S3_REGION,
|
||||
)
|
||||
return Store(
|
||||
S3Blobs(client, settings.S3_BUCKET, settings.S3_PREFIX, create=True)
|
||||
)
|
||||
if settings.STORE_BACKEND != "volume":
|
||||
raise ValueError(
|
||||
f"STORE_BACKEND must be volume or s3, not {settings.STORE_BACKEND!r}"
|
||||
)
|
||||
return Store(VolumeBlobs(settings.STORE_DIR)) if settings.STORE_DIR else None
|
||||
|
||||
Reference in New Issue
Block a user