The store's rules now sit on a Blobs layer: a folder, as before, or a prefix in an S3-compatible bucket (MinIO locally), chosen by STORE_BACKEND. The layout is the same in both; the record is written last. - S3_BUCKET, S3_PREFIX, S3_ENDPOINT_URL, S3_REGION; AWS credentials - an s3 extra (boto3), installed in the image; moto for tests - every store test runs on both kinds of storage Retention, the platform's sign-in and each kept file as a platform artifact are left: the last needs the platform's S3 backend, which it does not yet have. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
312 lines
10 KiB
Python
312 lines
10 KiB
Python
"""What agents delivered, kept by tenant, Customer case and Execution (ADR-0004).
|
|
|
|
tenants/<tenant>/cases/<case>/executions/<id>/record.json
|
|
/files/<name as delivered>
|
|
|
|
An Execution is written once. Its files are what the agent delivered, by the names
|
|
it gave them; its record says what it was. The same layout is kept in a folder or
|
|
under a prefix in an S3-compatible bucket.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import datetime as dt
|
|
import json
|
|
import re
|
|
import time
|
|
from collections.abc import Mapping
|
|
from pathlib import Path, PurePosixPath
|
|
from typing import TYPE_CHECKING, Any, Protocol
|
|
|
|
if TYPE_CHECKING:
|
|
from orchestration.config import OrchestrationSettings
|
|
|
|
_SLUG = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
|
|
_MEDIA_TYPES = {
|
|
".md": "text/markdown; charset=utf-8",
|
|
".json": "application/json",
|
|
".yaml": "application/yaml",
|
|
".yml": "application/yaml",
|
|
".html": "text/html; charset=utf-8",
|
|
".csv": "text/csv; charset=utf-8",
|
|
}
|
|
_TEXT = "text/plain; charset=utf-8"
|
|
|
|
|
|
def slug(value: str, what: str) -> str:
|
|
"""`value`, when it can name a folder: letters, digits, dot, dash, underscore."""
|
|
if not _SLUG.match(value or "") or value in {".", ".."}:
|
|
raise ValueError(f"{what} {value!r} must be letters, digits, '.', '-' or '_'")
|
|
return value
|
|
|
|
|
|
def file_name(name: str) -> PurePosixPath:
|
|
"""A delivered file's name as a path inside its Execution, or ValueError."""
|
|
path = PurePosixPath(name)
|
|
if (
|
|
not name
|
|
or "\\" in name
|
|
or path.is_absolute()
|
|
or any(part in {"", ".", ".."} for part in name.split("/"))
|
|
):
|
|
raise ValueError(f"{name!r} is not a file name inside an Execution")
|
|
return path
|
|
|
|
|
|
def media_type(name: str) -> str:
|
|
return _MEDIA_TYPES.get(PurePosixPath(name).suffix.lower(), _TEXT)
|
|
|
|
|
|
class Blobs(Protocol):
|
|
"""Where a store's keys are kept: a folder, or a bucket."""
|
|
|
|
def put(self, key: str, data: bytes, content_type: str) -> None: ...
|
|
def get(self, key: str) -> bytes | None: ...
|
|
def exists(self, key: str) -> bool: ...
|
|
def keys(self, prefix: str) -> list[str]: ...
|
|
|
|
|
|
class VolumeBlobs:
|
|
"""Keys as files under a folder."""
|
|
|
|
def __init__(self, root: str | Path) -> None:
|
|
self.root = Path(root)
|
|
|
|
def _path(self, key: str) -> Path:
|
|
path = (self.root / key).resolve()
|
|
if self.root.resolve() not in path.parents:
|
|
raise ValueError(f"{key!r} is outside the store")
|
|
return path
|
|
|
|
def put(self, key: str, data: bytes, content_type: str) -> None:
|
|
path = self._path(key)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_bytes(data)
|
|
|
|
def get(self, key: str) -> bytes | None:
|
|
path = self._path(key)
|
|
return path.read_bytes() if path.is_file() else None
|
|
|
|
def exists(self, key: str) -> bool:
|
|
return self._path(key).is_file()
|
|
|
|
def keys(self, prefix: str) -> list[str]:
|
|
folder = self.root / prefix
|
|
if not folder.is_dir():
|
|
return []
|
|
return [
|
|
path.relative_to(self.root).as_posix()
|
|
for path in folder.rglob("*")
|
|
if path.is_file()
|
|
]
|
|
|
|
|
|
class S3Blobs:
|
|
"""Keys as objects under a prefix in an S3-compatible bucket (MinIO locally)."""
|
|
|
|
def __init__(
|
|
self, client: Any, bucket: str, prefix: str = "", create: bool = False
|
|
) -> None:
|
|
self._client = client
|
|
self.bucket = bucket
|
|
self.prefix = prefix.lstrip("/")
|
|
if create:
|
|
self._ensure_bucket()
|
|
|
|
def _ensure_bucket(self) -> None:
|
|
from botocore.exceptions import ClientError
|
|
|
|
try:
|
|
self._client.head_bucket(Bucket=self.bucket)
|
|
except ClientError:
|
|
self._client.create_bucket(Bucket=self.bucket)
|
|
|
|
def put(self, key: str, data: bytes, content_type: str) -> None:
|
|
self._client.put_object(
|
|
Bucket=self.bucket,
|
|
Key=self.prefix + key,
|
|
Body=data,
|
|
ContentType=content_type,
|
|
)
|
|
|
|
def get(self, key: str) -> bytes | None:
|
|
from botocore.exceptions import ClientError
|
|
|
|
try:
|
|
response = self._client.get_object(
|
|
Bucket=self.bucket, Key=self.prefix + key
|
|
)
|
|
except ClientError as exc:
|
|
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
|
return None
|
|
raise
|
|
data: bytes = response["Body"].read()
|
|
return data
|
|
|
|
def exists(self, key: str) -> bool:
|
|
from botocore.exceptions import ClientError
|
|
|
|
try:
|
|
self._client.head_object(Bucket=self.bucket, Key=self.prefix + key)
|
|
except ClientError as exc:
|
|
if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}:
|
|
return False
|
|
raise
|
|
return True
|
|
|
|
def keys(self, prefix: str) -> list[str]:
|
|
found: list[str] = []
|
|
pages = self._client.get_paginator("list_objects_v2").paginate(
|
|
Bucket=self.bucket, Prefix=self.prefix + prefix
|
|
)
|
|
for page in pages:
|
|
for item in page.get("Contents", []):
|
|
found.append(str(item["Key"])[len(self.prefix) :])
|
|
return found
|
|
|
|
|
|
class Store:
|
|
def __init__(self, blobs: Blobs | str | Path) -> None:
|
|
self.blobs: Blobs = (
|
|
VolumeBlobs(blobs) if isinstance(blobs, (str, Path)) else blobs
|
|
)
|
|
|
|
@staticmethod
|
|
def _case(tenant: str, case: str) -> str:
|
|
return f"tenants/{slug(tenant, 'Tenant')}/cases/{slug(case, 'Case')}"
|
|
|
|
def _execution(self, tenant: str, case: str, execution_id: str) -> str:
|
|
return (
|
|
f"{self._case(tenant, case)}/executions/{slug(execution_id, 'Execution')}"
|
|
)
|
|
|
|
def keep(
|
|
self,
|
|
*,
|
|
tenant: str,
|
|
case: str,
|
|
execution_id: str,
|
|
work: str,
|
|
status: str,
|
|
request: str,
|
|
files: Mapping[str, str],
|
|
guide: str | None = None,
|
|
project_id: str | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Keep one Execution's files and record it. Written once: an Execution
|
|
already kept is refused."""
|
|
folder = self._execution(tenant, case, execution_id)
|
|
paths = {name: file_name(name) for name in files}
|
|
if self.blobs.exists(f"{folder}/record.json"):
|
|
raise ValueError(f"Execution {execution_id} is already kept")
|
|
for name, path in paths.items():
|
|
self.blobs.put(
|
|
f"{folder}/files/{path}", files[name].encode("utf-8"), media_type(name)
|
|
)
|
|
record = {
|
|
"tenant": tenant,
|
|
"customer_case": case,
|
|
"execution_id": execution_id,
|
|
"work": work,
|
|
"status": status,
|
|
"request": request,
|
|
"project_id": project_id,
|
|
"guide": guide if guide in files else None,
|
|
"files": sorted(files),
|
|
"kept_at": dt.datetime.now(dt.UTC).isoformat(),
|
|
"kept_ns": time.time_ns(),
|
|
}
|
|
# The record last: an Execution is kept once its record is.
|
|
self.blobs.put(
|
|
f"{folder}/record.json",
|
|
json.dumps(record, indent=2).encode("utf-8"),
|
|
"application/json",
|
|
)
|
|
return record
|
|
|
|
def record(
|
|
self, tenant: str, case: str, execution_id: str
|
|
) -> dict[str, Any] | None:
|
|
try:
|
|
data = self.blobs.get(
|
|
f"{self._execution(tenant, case, execution_id)}/record.json"
|
|
)
|
|
except ValueError:
|
|
return None
|
|
if data is None:
|
|
return None
|
|
record: dict[str, Any] = json.loads(data)
|
|
return record
|
|
|
|
def file(
|
|
self, tenant: str, case: str, execution_id: str, name: str
|
|
) -> tuple[bytes, str] | None:
|
|
"""A kept file's content and media type, or None."""
|
|
try:
|
|
key = (
|
|
f"{self._execution(tenant, case, execution_id)}/files/{file_name(name)}"
|
|
)
|
|
data = self.blobs.get(key)
|
|
except ValueError:
|
|
return None
|
|
return None if data is None else (data, media_type(name))
|
|
|
|
def files(self, tenant: str, case: str, execution_id: str) -> dict[str, str]:
|
|
"""Every file kept for an Execution, by name."""
|
|
record = self.record(tenant, case, execution_id) or {}
|
|
found: dict[str, str] = {}
|
|
for name in record.get("files", []):
|
|
kept = self.file(tenant, case, execution_id, name)
|
|
if kept is not None:
|
|
found[name] = kept[0].decode("utf-8")
|
|
return found
|
|
|
|
def case(self, tenant: str, case: str) -> list[dict[str, Any]]:
|
|
"""The Customer case's Executions, newest first."""
|
|
try:
|
|
prefix = f"{self._case(tenant, case)}/executions/"
|
|
except ValueError:
|
|
return []
|
|
records = []
|
|
for key in self.blobs.keys(prefix):
|
|
# <id>/record.json; a delivered file of that name is under files/.
|
|
parts = key[len(prefix) :].split("/")
|
|
if len(parts) == 2 and parts[1] == "record.json":
|
|
data = self.blobs.get(key)
|
|
if data is not None:
|
|
records.append(json.loads(data))
|
|
return sorted(records, key=lambda r: r.get("kept_ns", 0), reverse=True)
|
|
|
|
def latest(self, tenant: str, case: str, work: str) -> dict[str, Any] | None:
|
|
"""The newest Execution of `work` in the Customer case that completed."""
|
|
return next(
|
|
(
|
|
record
|
|
for record in self.case(tenant, case)
|
|
if record["work"] == work and record["status"] == "completed"
|
|
),
|
|
None,
|
|
)
|
|
|
|
|
|
def store_from_settings(settings: OrchestrationSettings) -> Store | None:
|
|
"""The store the settings name; None when nothing is to be kept."""
|
|
if settings.STORE_BACKEND == "s3":
|
|
import boto3
|
|
|
|
if not settings.S3_BUCKET:
|
|
raise ValueError("STORE_BACKEND=s3 needs S3_BUCKET")
|
|
client = boto3.client(
|
|
"s3",
|
|
endpoint_url=settings.S3_ENDPOINT_URL or None,
|
|
region_name=settings.S3_REGION,
|
|
)
|
|
return Store(
|
|
S3Blobs(client, settings.S3_BUCKET, settings.S3_PREFIX, create=True)
|
|
)
|
|
if settings.STORE_BACKEND != "volume":
|
|
raise ValueError(
|
|
f"STORE_BACKEND must be volume or s3, not {settings.STORE_BACKEND!r}"
|
|
)
|
|
return Store(VolumeBlobs(settings.STORE_DIR)) if settings.STORE_DIR else None
|