"""What agents delivered, kept by tenant, Customer case and Execution (ADR-0004). tenants//cases//executions//record.json /files/ An Execution is written once. Its files are what the agent delivered, by the names it gave them; its record says what it was. The same layout is kept in a folder or under a prefix in an S3-compatible bucket. """ from __future__ import annotations import datetime as dt import json import re import time from collections.abc import Mapping from pathlib import Path, PurePosixPath from typing import TYPE_CHECKING, Any, Protocol if TYPE_CHECKING: from orchestration.config import OrchestrationSettings _SLUG = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") _MEDIA_TYPES = { ".md": "text/markdown; charset=utf-8", ".json": "application/json", ".yaml": "application/yaml", ".yml": "application/yaml", ".html": "text/html; charset=utf-8", ".csv": "text/csv; charset=utf-8", } _TEXT = "text/plain; charset=utf-8" def slug(value: str, what: str) -> str: """`value`, when it can name a folder: letters, digits, dot, dash, underscore.""" if not _SLUG.match(value or "") or value in {".", ".."}: raise ValueError(f"{what} {value!r} must be letters, digits, '.', '-' or '_'") return value def file_name(name: str) -> PurePosixPath: """A delivered file's name as a path inside its Execution, or ValueError.""" path = PurePosixPath(name) if ( not name or "\\" in name or path.is_absolute() or any(part in {"", ".", ".."} for part in name.split("/")) ): raise ValueError(f"{name!r} is not a file name inside an Execution") return path def media_type(name: str) -> str: return _MEDIA_TYPES.get(PurePosixPath(name).suffix.lower(), _TEXT) class Blobs(Protocol): """Where a store's keys are kept: a folder, or a bucket.""" def put(self, key: str, data: bytes, content_type: str) -> None: ... def get(self, key: str) -> bytes | None: ... def exists(self, key: str) -> bool: ... def keys(self, prefix: str) -> list[str]: ... class VolumeBlobs: """Keys as files under a folder.""" def __init__(self, root: str | Path) -> None: self.root = Path(root) def _path(self, key: str) -> Path: path = (self.root / key).resolve() if self.root.resolve() not in path.parents: raise ValueError(f"{key!r} is outside the store") return path def put(self, key: str, data: bytes, content_type: str) -> None: path = self._path(key) path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(data) def get(self, key: str) -> bytes | None: path = self._path(key) return path.read_bytes() if path.is_file() else None def exists(self, key: str) -> bool: return self._path(key).is_file() def keys(self, prefix: str) -> list[str]: folder = self.root / prefix if not folder.is_dir(): return [] return [ path.relative_to(self.root).as_posix() for path in folder.rglob("*") if path.is_file() ] class S3Blobs: """Keys as objects under a prefix in an S3-compatible bucket (MinIO locally).""" def __init__( self, client: Any, bucket: str, prefix: str = "", create: bool = False ) -> None: self._client = client self.bucket = bucket self.prefix = prefix.lstrip("/") if create: self._ensure_bucket() def _ensure_bucket(self) -> None: from botocore.exceptions import ClientError try: self._client.head_bucket(Bucket=self.bucket) except ClientError: self._client.create_bucket(Bucket=self.bucket) def put(self, key: str, data: bytes, content_type: str) -> None: self._client.put_object( Bucket=self.bucket, Key=self.prefix + key, Body=data, ContentType=content_type, ) def get(self, key: str) -> bytes | None: from botocore.exceptions import ClientError try: response = self._client.get_object( Bucket=self.bucket, Key=self.prefix + key ) except ClientError as exc: if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}: return None raise data: bytes = response["Body"].read() return data def exists(self, key: str) -> bool: from botocore.exceptions import ClientError try: self._client.head_object(Bucket=self.bucket, Key=self.prefix + key) except ClientError as exc: if exc.response.get("Error", {}).get("Code") in {"NoSuchKey", "404"}: return False raise return True def keys(self, prefix: str) -> list[str]: found: list[str] = [] pages = self._client.get_paginator("list_objects_v2").paginate( Bucket=self.bucket, Prefix=self.prefix + prefix ) for page in pages: for item in page.get("Contents", []): found.append(str(item["Key"])[len(self.prefix) :]) return found class Store: def __init__(self, blobs: Blobs | str | Path) -> None: self.blobs: Blobs = ( VolumeBlobs(blobs) if isinstance(blobs, (str, Path)) else blobs ) @staticmethod def _case(tenant: str, case: str) -> str: return f"tenants/{slug(tenant, 'Tenant')}/cases/{slug(case, 'Case')}" def _execution(self, tenant: str, case: str, execution_id: str) -> str: return ( f"{self._case(tenant, case)}/executions/{slug(execution_id, 'Execution')}" ) def keep( self, *, tenant: str, case: str, execution_id: str, work: str, status: str, request: str, files: Mapping[str, str], guide: str | None = None, project_id: str | None = None, ) -> dict[str, Any]: """Keep one Execution's files and record it. Written once: an Execution already kept is refused.""" folder = self._execution(tenant, case, execution_id) paths = {name: file_name(name) for name in files} if self.blobs.exists(f"{folder}/record.json"): raise ValueError(f"Execution {execution_id} is already kept") for name, path in paths.items(): self.blobs.put( f"{folder}/files/{path}", files[name].encode("utf-8"), media_type(name) ) record = { "tenant": tenant, "customer_case": case, "execution_id": execution_id, "work": work, "status": status, "request": request, "project_id": project_id, "guide": guide if guide in files else None, "files": sorted(files), "kept_at": dt.datetime.now(dt.UTC).isoformat(), "kept_ns": time.time_ns(), } # The record last: an Execution is kept once its record is. self.blobs.put( f"{folder}/record.json", json.dumps(record, indent=2).encode("utf-8"), "application/json", ) return record def record( self, tenant: str, case: str, execution_id: str ) -> dict[str, Any] | None: try: data = self.blobs.get( f"{self._execution(tenant, case, execution_id)}/record.json" ) except ValueError: return None if data is None: return None record: dict[str, Any] = json.loads(data) return record def file( self, tenant: str, case: str, execution_id: str, name: str ) -> tuple[bytes, str] | None: """A kept file's content and media type, or None.""" try: key = ( f"{self._execution(tenant, case, execution_id)}/files/{file_name(name)}" ) data = self.blobs.get(key) except ValueError: return None return None if data is None else (data, media_type(name)) def files(self, tenant: str, case: str, execution_id: str) -> dict[str, str]: """Every file kept for an Execution, by name.""" record = self.record(tenant, case, execution_id) or {} found: dict[str, str] = {} for name in record.get("files", []): kept = self.file(tenant, case, execution_id, name) if kept is not None: found[name] = kept[0].decode("utf-8") return found def case(self, tenant: str, case: str) -> list[dict[str, Any]]: """The Customer case's Executions, newest first.""" try: prefix = f"{self._case(tenant, case)}/executions/" except ValueError: return [] records = [] for key in self.blobs.keys(prefix): # /record.json; a delivered file of that name is under files/. parts = key[len(prefix) :].split("/") if len(parts) == 2 and parts[1] == "record.json": data = self.blobs.get(key) if data is not None: records.append(json.loads(data)) return sorted(records, key=lambda r: r.get("kept_ns", 0), reverse=True) def latest(self, tenant: str, case: str, work: str) -> dict[str, Any] | None: """The newest Execution of `work` in the Customer case that completed.""" return next( ( record for record in self.case(tenant, case) if record["work"] == work and record["status"] == "completed" ), None, ) def store_from_settings(settings: OrchestrationSettings) -> Store | None: """The store the settings name; None when nothing is to be kept.""" if settings.STORE_BACKEND == "s3": import boto3 if not settings.S3_BUCKET: raise ValueError("STORE_BACKEND=s3 needs S3_BUCKET") client = boto3.client( "s3", endpoint_url=settings.S3_ENDPOINT_URL or None, region_name=settings.S3_REGION, ) return Store( S3Blobs(client, settings.S3_BUCKET, settings.S3_PREFIX, create=True) ) if settings.STORE_BACKEND != "volume": raise ValueError( f"STORE_BACKEND must be volume or s3, not {settings.STORE_BACKEND!r}" ) return Store(VolumeBlobs(settings.STORE_DIR)) if settings.STORE_DIR else None