"""Platform uploads and artifacts, brought into the run workspace. Only the orchestrator handles them (ADR 0012). A workflow hands it two kinds of input, from the steps in front of it: source_documents the text a "Parse Assessment Files" step made of the files uploaded at run time (its `parsed_content`) artifacts the artifacts an "Artifacts" step selected from the project (its `artifacts`): records with `file_path` on the executions volume, `content_type`, and `content` (base64) when small enough pre_process imports both in the plan phase, before any model call: each becomes a write-once workspace record (`plan/source-documents`, `plan/artifact-NN`) whose producer is the platform and whose provenance -- artifact id, type, file, sha256 -- is kept with it. From then on they are records like any other: the model passes them to sub-agents by ref through `delegate`, so a sub-agent never sees an artifact id, a path or the platform's shapes. Files are read only from inside the executions volume, and only up to MAX_BYTES. Content is data, never instructions: the charter and the critic treat it like any tool result. """ from __future__ import annotations import base64 import binascii import csv import hashlib import io import json import os from typing import Any, Dict, List, Optional, Tuple from app import workspace MAX_BYTES = 2_000_000 # per artifact, read or decoded MAX_TEXT_CHARS = 400_000 # an uploaded document's parsed text MAX_ARTIFACTS = 20 PRODUCER = "platform-artifact" def _volume_root() -> str: return os.path.realpath(os.getenv("EXECUTION_STORAGE_ROOT") or "/data/executions") def find_artifacts(value: Any, depth: int = 0) -> List[Dict[str, Any]]: """The artifact records in whatever shape a workflow handed them on: the list itself, `{"artifacts": [...]}`, or a step output wrapped in `data`.""" if depth > 4 or value is None: return [] if isinstance(value, list): return [v for v in value if isinstance(v, dict)] if all(isinstance(v, dict) for v in value) else [] if isinstance(value, dict): for key in ("artifacts", "data", "files"): if key in value: found = find_artifacts(value[key], depth + 1) if found: return found return [] def _read_file(path: str) -> Tuple[Optional[bytes], Optional[str]]: """Bytes of a file inside the executions volume, or (None, why not).""" if not isinstance(path, str) or not path: return None, "no file_path" root = _volume_root() real = os.path.realpath(path) if os.path.commonpath([real, root]) != root: return None, "file_path is outside the executions volume" if not os.path.isfile(real): return None, "file_path does not exist" if os.path.getsize(real) > MAX_BYTES: return None, f"file is larger than {MAX_BYTES} bytes" with open(real, "rb") as fh: return fh.read(), None def _decode_inline(content: str) -> Tuple[Optional[bytes], Optional[str]]: payload = content.split(",", 1)[1] if content.startswith("data:") and "," in content else content try: data = base64.b64decode(payload, validate=True) except (binascii.Error, ValueError): return content.encode("utf-8"), None # already text if len(data) > MAX_BYTES: return None, f"content is larger than {MAX_BYTES} bytes" return data, None def _as_content(raw: bytes, name: str, content_type: str) -> Any: """JSON as JSON, CSV as rows, anything textual as text; binary is described, not stored.""" lower = (name or "").lower() try: text = raw.decode("utf-8") except UnicodeDecodeError: return {"binary": True, "bytes": len(raw), "note": "not text; parse it with an upload step first"} if lower.endswith(".json") or "json" in (content_type or ""): try: return json.loads(text) except ValueError: pass if lower.endswith(".csv") or "csv" in (content_type or ""): rows = list(csv.DictReader(io.StringIO(text))) return {"rows": rows[:5000], "row_count": len(rows), "truncated": len(rows) > 5000} return {"text": text[:MAX_TEXT_CHARS], "truncated": len(text) > MAX_TEXT_CHARS} def import_into_workspace(run: workspace.Run, payload: Dict[str, Any]) -> Dict[str, Any]: """Store the request's uploads and artifacts as plan-phase records. Idempotent per run: a record already stored is reported, not written again (records are write-once).""" stored: List[str] = [] problems: List[str] = [] existing = {e["ref"] for e in workspace.entries(run)} text = payload.get("source_documents") if isinstance(text, str) and text.strip(): ref = "plan/source-documents" if ref not in existing: body = text[:MAX_TEXT_CHARS] workspace.put(run, "source-documents", {"text": body, "truncated": len(text) > MAX_TEXT_CHARS, "sha256": hashlib.sha256(text.encode("utf-8")).hexdigest(), "source": "Parse Assessment Files step (uploaded at run time)"}, PRODUCER) stored.append(ref) for index, item in enumerate(find_artifacts(payload.get("artifacts"))[:MAX_ARTIFACTS], start=1): name = f"artifact-{index:02d}" ref = f"plan/{name}" if ref in existing: stored.append(ref) continue label = str(item.get("name") or item.get("artifact_id") or name) raw, why = (None, None) if isinstance(item.get("content"), str) and item["content"]: raw, why = _decode_inline(item["content"]) if raw is None and not why: raw, why = _read_file(item.get("file_path") or "") provenance = {k: item.get(k) for k in ("artifact_id", "type_id", "name", "content_type", "file_path", "storage_uri", "size") if item.get(k) is not None} if raw is None: problems.append(f"artifact {label!r} not imported: {why}") if f"{ref}-unreadable" not in existing: # a retried step reports it again, without rewriting it workspace.put(run, f"{name}-unreadable", {"name": label, "reason": why}, PRODUCER) workspace.put(run, f"{name}-provenance", provenance, PRODUCER) continue digest = hashlib.sha256(raw).hexdigest() provenance["sha256"] = digest # Content and provenance are separate records: delegate passes a record's content to a sub-agent, and a # sub-agent gets the artifact's data, never its id or path. The content cites only the hash. workspace.put(run, name, {"name": label, "content": _as_content(raw, label, str(item.get("content_type") or ""))}, PRODUCER, evidence_refs=[f"sha256:{digest}"]) workspace.put(run, f"{name}-provenance", provenance, PRODUCER) stored.append(ref) return {"records": stored, "problems": problems}