decomposer: generate deliverable files for Define the normalized ingestion contract and pluggable source-connector interface for SCM, CMS, and other content sources.; Implement a GitHub SCM connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of Markdown, plain-text, and source files with webhook and revision metadata.; Implement a SharePoint CMS connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of PDF, DOCX, and HTML files with webhook and revision metadata.; Implement ingestion orchestration and triggers; Normalize, govern, and publish ingested content to the shared knowledge store.; Add deployment/configuration, tests, and documentation for the ingestion agent.; Validate end-to-end ingestion and downstream content availability.

kab_ingestion/orchestrator.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+import uuid
+from dataclasses import dataclass
+from .models import IngestionBatch
+
+@dataclass
+class RunResult:
+    run_id: str
+    status: str
+    batch: IngestionBatch | None = None
+    attempts: int = 0
+    dead_letter: list[dict] | None = None
+
+class Orchestrator:
+    def __init__(self, connectors, cursor_store, publisher, max_attempts=3):
+        self.connectors, self.cursors, self.publisher, self.max_attempts = connectors, cursor_store, publisher, max_attempts
+
+    def run(self, connector_name, tenant_id, mode='incremental', idempotency_key=None):
+        connector = self.connectors[connector_name]
+        cursor = self.cursors.load(tenant_id, connector_name)
+        batch = None
+        for attempt in range(1, self.max_attempts + 1):
+            try:
+                batch = connector.full(tenant_id) if mode == 'full' else connector.incremental(tenant_id, cursor)
+                self.publisher.publish(batch)
+                if batch.next_cursor: self.cursors.save(batch.next_cursor)
+                return RunResult(str(uuid.uuid4()), 'succeeded', batch, attempt, [])
+            except Exception as exc:
+                if attempt == self.max_attempts:
+                    return RunResult(str(uuid.uuid4()), 'dead_lettered', batch, attempt, [{'error': type(exc).__name__}])
+        raise AssertionError('unreachable')