Files
Jonathan Boniface 83df77005a feat: keep delivered work in S3-compatible object storage (ADR-0004 phase 3)
The store's rules now sit on a Blobs layer: a folder, as before, or a
prefix in an S3-compatible bucket (MinIO locally), chosen by STORE_BACKEND.
The layout is the same in both; the record is written last.

- S3_BUCKET, S3_PREFIX, S3_ENDPOINT_URL, S3_REGION; AWS credentials
- an s3 extra (boto3), installed in the image; moto for tests
- every store test runs on both kinds of storage

Retention, the platform's sign-in and each kept file as a platform
artifact are left: the last needs the platform's S3 backend, which it does
not yet have.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-21 21:08:39 +01:00

312 lines
10 KiB
Python

"""What agents delivered, kept by tenant, Customer case and Execution (ADR-0004).
tenants/<tenant>/cases/<case>/executions/<id>/record.json
/files/<name as delivered>
An Execution is written once. Its files are what the agent delivered, by the names
it gave them; its record says what it was. The same layout is kept in a folder or
under a prefix in an S3-compatible bucket.
"""
from __future__ import annotations
import datetime as dt
import json
import re
import time
from collections.abc import Mapping
from pathlib import Path, PurePosixPath
from typing import TYPE_CHECKING, Any, Protocol
if TYPE_CHECKING:
from orchestration.config import OrchestrationSettings
_SLUG = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
_MEDIA_TYPES = {
".md": "text/markdown; charset=utf-8",
".json": "application/json",
".yaml": "application/yaml",
".yml": "application/yaml",
".html": "text/html; charset=utf-8",
".csv": "text/csv; charset=utf-8",
}
_TEXT = "text/plain; charset=utf-8"
def slug(value: str, what: str) -> str:
"""`value`, when it can name a folder: letters, digits, dot, dash, underscore."""
if not _SLUG.match(value or "") or value in {".", ".."}:
raise ValueError(f"{what} {value!r} must be letters, digits, '.', '-' or '_'")
return value
def file_name(name: str) -> PurePosixPath:
"""A delivered file's name as a path inside its Execution, or ValueError."""
path = PurePosixPath(name)
if (
not name
or "\\" in name
or path.is_absolute()
or any(part in {"", ".", ".."} for part in name.split("/"))
):
raise ValueError(f"{name!r} is not a file name inside an Execution")
return path
def media_type(name: str) -> str:
return _MEDIA_TYPES.get(PurePosixPath(name).suffix.lower(), _TEXT)
class Blobs(Protocol):
"""Where a store's keys are kept: a folder, or a bucket."""
def put(self, key: str, data: bytes, content_type: str) -> None: ...
def get(self, key: str) -> bytes | None: ...
def exists(self, key: str) -> bool: ...
def keys(self, prefix: str) -> list[str]: ...
class VolumeBlobs:
"""Keys as files under a folder."""
def __init__(self, root: str | Path) -> None:
self.root = Path(root)
def _path(self, key: str) -> Path:
path = (self.root / key).resolve()
if self.root.resolve() not in path.parents:
raise ValueError(f"{key!r} is outside the store")
return path
def put(self, key: str, data: bytes, content_type: str) -> None:
path = self._path(key)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_bytes(data)
def get(self, key: str) -> bytes | None:
path = self._path(key)
return path.read_bytes() if path.is_file() else None
def exists(self, key: str) -> bool:
return self._path(key).is_file()
def keys(self, prefix: str) -> list[str]:
folder = self.root / prefix
if not folder.is_dir():
return []
return [
path.relative_to(self.root).as_posix()
for path in folder.rglob("*")
if path.is_file()
]
class S3Blobs:
"""Keys as objects under a prefix in an S3-compatible bucket (MinIO locally)."""
def __init__(
self, client: Any, bucket: str, prefix: str = "", create: bool = False
) -> None:
self._client = client
self.bucket = bucket
self.prefix = prefix.lstrip("/")
if create:
self._ensure_bucket()
def _ensure_bucket(self) -> None:
from botocore.exceptions import ClientError
try:
self._client.head_bucket(Bucket=self.bucket)
except ClientError:
self._client.create_bucket(Bucket=self.bucket)
def put(self, key: str, data: bytes, content_type: str) -> None:
self._client.put_object(
Bucket=self.bucket,
Key=self.prefix + key,
Body=data,
ContentType=content_type,
)
def get(self, key: str) -> bytes | None:
from botocore.exceptions import ClientError
try:
response = self._client.get_object(
Bucket=self.bucket, Key=self.prefix + key
)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
return None
raise
data: bytes = response["Body"].read()
return data
def exists(self, key: str) -> bool:
from botocore.exceptions import ClientError
try:
self._client.head_object(Bucket=self.bucket, Key=self.prefix + key)
except ClientError as exc:
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
return False
raise
return True
def keys(self, prefix: str) -> list[str]:
found: list[str] = []
pages = self._client.get_paginator("list_objects_v2").paginate(
Bucket=self.bucket, Prefix=self.prefix + prefix
)
for page in pages:
for item in page.get("Contents", []):
found.append(str(item["Key"])[len(self.prefix) :])
return found
class Store:
def __init__(self, blobs: Blobs | str | Path) -> None:
self.blobs: Blobs = (
VolumeBlobs(blobs) if isinstance(blobs, (str, Path)) else blobs
)
@staticmethod
def _case(tenant: str, case: str) -> str:
return f"tenants/{slug(tenant, 'Tenant')}/cases/{slug(case, 'Case')}"
def _execution(self, tenant: str, case: str, execution_id: str) -> str:
return (
f"{self._case(tenant, case)}/executions/{slug(execution_id, 'Execution')}"
)
def keep(
self,
*,
tenant: str,
case: str,
execution_id: str,
work: str,
status: str,
request: str,
files: Mapping[str, str],
guide: str | None = None,
project_id: str | None = None,
) -> dict[str, Any]:
"""Keep one Execution's files and record it. Written once: an Execution
already kept is refused."""
folder = self._execution(tenant, case, execution_id)
paths = {name: file_name(name) for name in files}
if self.blobs.exists(f"{folder}/record.json"):
raise ValueError(f"Execution {execution_id} is already kept")
for name, path in paths.items():
self.blobs.put(
f"{folder}/files/{path}", files[name].encode("utf-8"), media_type(name)
)
record = {
"tenant": tenant,
"customer_case": case,
"execution_id": execution_id,
"work": work,
"status": status,
"request": request,
"project_id": project_id,
"guide": guide if guide in files else None,
"files": sorted(files),
"kept_at": dt.datetime.now(dt.UTC).isoformat(),
"kept_ns": time.time_ns(),
}
# The record last: an Execution is kept once its record is.
self.blobs.put(
f"{folder}/record.json",
json.dumps(record, indent=2).encode("utf-8"),
"application/json",
)
return record
def record(
self, tenant: str, case: str, execution_id: str
) -> dict[str, Any] | None:
try:
data = self.blobs.get(
f"{self._execution(tenant, case, execution_id)}/record.json"
)
except ValueError:
return None
if data is None:
return None
record: dict[str, Any] = json.loads(data)
return record
def file(
self, tenant: str, case: str, execution_id: str, name: str
) -> tuple[bytes, str] | None:
"""A kept file's content and media type, or None."""
try:
key = (
f"{self._execution(tenant, case, execution_id)}/files/{file_name(name)}"
)
data = self.blobs.get(key)
except ValueError:
return None
return None if data is None else (data, media_type(name))
def files(self, tenant: str, case: str, execution_id: str) -> dict[str, str]:
"""Every file kept for an Execution, by name."""
record = self.record(tenant, case, execution_id) or {}
found: dict[str, str] = {}
for name in record.get("files", []):
kept = self.file(tenant, case, execution_id, name)
if kept is not None:
found[name] = kept[0].decode("utf-8")
return found
def case(self, tenant: str, case: str) -> list[dict[str, Any]]:
"""The Customer case's Executions, newest first."""
try:
prefix = f"{self._case(tenant, case)}/executions/"
except ValueError:
return []
records = []
for key in self.blobs.keys(prefix):
# <id>/record.json; a delivered file of that name is under files/.
parts = key[len(prefix) :].split("/")
if len(parts) == 2 and parts[1] == "record.json":
data = self.blobs.get(key)
if data is not None:
records.append(json.loads(data))
return sorted(records, key=lambda r: r.get("kept_ns", 0), reverse=True)
def latest(self, tenant: str, case: str, work: str) -> dict[str, Any] | None:
"""The newest Execution of `work` in the Customer case that completed."""
return next(
(
record
for record in self.case(tenant, case)
if record["work"] == work and record["status"] == "completed"
),
None,
)
def store_from_settings(settings: OrchestrationSettings) -> Store | None:
"""The store the settings name; None when nothing is to be kept."""
if settings.STORE_BACKEND == "s3":
import boto3
if not settings.S3_BUCKET:
raise ValueError("STORE_BACKEND=s3 needs S3_BUCKET")
client = boto3.client(
"s3",
endpoint_url=settings.S3_ENDPOINT_URL or None,
region_name=settings.S3_REGION,
)
return Store(
S3Blobs(client, settings.S3_BUCKET, settings.S3_PREFIX, create=True)
)
if settings.STORE_BACKEND != "volume":
raise ValueError(
f"STORE_BACKEND must be volume or s3, not {settings.STORE_BACKEND!r}"
)
return Store(VolumeBlobs(settings.STORE_DIR)) if settings.STORE_DIR else None