decomposer: generate deliverable files for Define the normalized ingestion contract and pluggable source-connector interface for SCM, CMS, and other content sources.; Implement a GitHub SCM connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of Markdown, plain-text, and source files with webhook and revision metadata.; Implement a SharePoint CMS connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of PDF, DOCX, and HTML files with webhook and revision metadata.; Implement ingestion orchestration and triggers; Normalize, govern, and publish ingested content to the shared knowledge store.; Add deployment/configuration, tests, and documentation for the ingestion agent.; Validate end-to-end ingestion and downstream content availability.

kab_ingestion/__init__.py

@@ -0,0 +1,5 @@
+"""KAB content ingestion agent."""
+
+from .models import NormalizedDocument, SyncCursor
+
+__all__ = ["NormalizedDocument", "SyncCursor"]

kab_ingestion/__main__.py

@@ -0,0 +1 @@
+print('KAB ingestion adapters are deployed by the platform runtime; use Orchestrator with injected ports.')

kab_ingestion/github.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+import hashlib
+from datetime import datetime, timezone
+from .models import ACL, Change, IngestionBatch, NormalizedDocument, Provenance, SyncCursor
+
+class GitHubConnector:
+    name = "github"
+    def __init__(self, api, secret_provider, config: dict):
+        self.api, self.secrets, self.config = api, secret_provider, config
+        self.scope = f"{config['owner']}/{config['repo']}:{config.get('path_prefix','')}"
+
+    def _headers(self):
+        return {"Authorization": f"Bearer {self.secrets.get(self.config['credential_ref'])}", "Accept": "application/vnd.github+json"}
+
+    def _doc(self, tenant_id, item, event_id=None):
+        body = item.get("content", "")
+        digest = hashlib.sha256(body.encode()).hexdigest()
+        now = datetime.now(timezone.utc)
+        return NormalizedDocument(document_id=f"github:{item['sha']}", tenant_id=tenant_id,
+            title=item['path'].rsplit('/', 1)[-1], body=body, mime_type=item.get('mime_type','text/plain'),
+            source_path=item['path'], content_hash=digest, updated_at=now, acl=ACL(**item.get('acl', {})),
+            provenance=Provenance(connector=self.name, source_uri=item['html_url'], source_id=item['sha'],
+              revision_id=item.get('commit_sha'), revision_time=item.get('commit_time'), retrieved_at=now, webhook_event_id=event_id))
+
+    def full(self, tenant_id):
+        items = self.api.list_files(self.config['owner'], self.config['repo'], self.config.get('path_prefix',''), self._headers())
+        docs = [self._doc(tenant_id, x) for x in items if x.get('mime_type') in self.config.get('mime_types', ['text/markdown','text/plain','text/x-python'])]
+        return IngestionBatch(run_id=hashlib.sha256(self.scope.encode()).hexdigest()[:16], tenant_id=tenant_id, documents=docs,
+            next_cursor=SyncCursor(connector=self.name, scope_fingerprint=self.scope, value=items[-1].get('commit_sha') if items else None, version=1, updated_at=datetime.now(timezone.utc)))
+
+    def incremental(self, tenant_id, cursor):
+        items = self.api.list_changed(self.config['owner'], self.config['repo'], cursor.value if cursor else None, self.config.get('path_prefix',''), self._headers())
+        batch = self.full(tenant_id)
+        allowed = {x['path'] for x in items}
+        batch.documents = [d for d in batch.documents if d.source_path in allowed]
+        if cursor: batch.next_cursor.version = cursor.version + 1
+        return batch
+
+    def changes_from_webhook(self, payload):
+        return [Change(path=x['filename'], source_id=payload['after'], revision_id=payload['after'], deleted=x['status']=='removed') for x in payload.get('commits', []) for x in x.get('modified', []) + x.get('removed', [])]

kab_ingestion/governance.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+from datetime import datetime, timedelta, timezone
+
+class GovernedPublisher:
+    def __init__(self, store, audit, retention_days=30): self.store, self.audit, self.retention_days = store, audit, retention_days
+    def publish(self, batch):
+        if any(d.tenant_id != batch.tenant_id for d in batch.documents): raise PermissionError('tenant mismatch')
+        self.store.upsert(batch.documents)
+        self.audit.append({'tenant_id': batch.tenant_id, 'run_id': batch.run_id, 'action': 'publish', 'count': len(batch.documents), 'at': datetime.now(timezone.utc).isoformat()})
+    def purge_expired_tombstones(self, docs):
+        cutoff = datetime.now(timezone.utc) - timedelta(days=self.retention_days)
+        ids = [d.document_id for d in docs if d.deleted and d.updated_at < cutoff]
+        if ids: self.store.delete(ids, docs[0].tenant_id)
+        return ids

kab_ingestion/models.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+from datetime import datetime
+from typing import Literal
+from pydantic import BaseModel, Field
+
+class ACL(BaseModel):
+    principals: list[str] = Field(default_factory=list)
+    groups: list[str] = Field(default_factory=list)
+    public: bool = False
+
+class Provenance(BaseModel):
+    connector: str
+    source_uri: str
+    source_id: str
+    revision_id: str | None = None
+    revision_time: datetime | None = None
+    retrieved_at: datetime
+    webhook_event_id: str | None = None
+
+class NormalizedDocument(BaseModel):
+    document_id: str
+    tenant_id: str
+    title: str
+    body: str
+    mime_type: str
+    language: str | None = None
+    source_path: str
+    content_hash: str
+    updated_at: datetime
+    deleted: bool = False
+    acl: ACL
+    provenance: Provenance
+    metadata: dict[str, str] = Field(default_factory=dict)
+
+class SyncCursor(BaseModel):
+    connector: str
+    scope_fingerprint: str
+    value: str | None = None
+    version: int = 0
+    updated_at: datetime
+
+class Change(BaseModel):
+    path: str
+    source_id: str
+    revision_id: str | None = None
+    deleted: bool = False
+
+class IngestionBatch(BaseModel):
+    run_id: str
+    tenant_id: str
+    documents: list[NormalizedDocument]
+    next_cursor: SyncCursor | None = None
+    errors: list[dict[str, str]] = Field(default_factory=list)

kab_ingestion/orchestrator.py

@@ -0,0 +1,31 @@
+from __future__ import annotations
+import uuid
+from dataclasses import dataclass
+from .models import IngestionBatch
+
+@dataclass
+class RunResult:
+    run_id: str
+    status: str
+    batch: IngestionBatch | None = None
+    attempts: int = 0
+    dead_letter: list[dict] | None = None
+
+class Orchestrator:
+    def __init__(self, connectors, cursor_store, publisher, max_attempts=3):
+        self.connectors, self.cursors, self.publisher, self.max_attempts = connectors, cursor_store, publisher, max_attempts
+
+    def run(self, connector_name, tenant_id, mode='incremental', idempotency_key=None):
+        connector = self.connectors[connector_name]
+        cursor = self.cursors.load(tenant_id, connector_name)
+        batch = None
+        for attempt in range(1, self.max_attempts + 1):
+            try:
+                batch = connector.full(tenant_id) if mode == 'full' else connector.incremental(tenant_id, cursor)
+                self.publisher.publish(batch)
+                if batch.next_cursor: self.cursors.save(batch.next_cursor)
+                return RunResult(str(uuid.uuid4()), 'succeeded', batch, attempt, [])
+            except Exception as exc:
+                if attempt == self.max_attempts:
+                    return RunResult(str(uuid.uuid4()), 'dead_lettered', batch, attempt, [{'error': type(exc).__name__}])
+        raise AssertionError('unreachable')

kab_ingestion/ports.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+from typing import Protocol
+from .models import Change, IngestionBatch, SyncCursor
+
+class Connector(Protocol):
+    name: str
+    def full(self, tenant_id: str) -> IngestionBatch: ...
+    def incremental(self, tenant_id: str, cursor: SyncCursor | None) -> IngestionBatch: ...
+    def changes_from_webhook(self, payload: dict) -> list[Change]: ...
+
+class HTTPClient(Protocol):
+    def get(self, url: str, *, headers: dict[str, str], params: dict | None = None): ...
+
+class SecretProvider(Protocol):
+    def get(self, ref: str) -> str: ...
+
+class CursorStore(Protocol):
+    def load(self, tenant_id: str, connector: str) -> SyncCursor | None: ...
+    def save(self, cursor: SyncCursor) -> None: ...
+
+class KnowledgeStore(Protocol):
+    def upsert(self, documents: list) -> None: ...
+    def delete(self, document_ids: list[str], tenant_id: str) -> None: ...

kab_ingestion/sharepoint.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+import hashlib
+from datetime import datetime, timezone
+from .models import ACL, Change, IngestionBatch, NormalizedDocument, Provenance, SyncCursor
+
+class SharePointConnector:
+    name = "sharepoint"
+    def __init__(self, graph, secret_provider, config):
+        self.graph, self.secrets, self.config = graph, secret_provider, config
+        self.scope = ":".join(config[x] for x in ('site_id','drive_id')) + ':' + config.get('folder_path','')
+
+    def _headers(self): return {"Authorization": f"Bearer {self.secrets.get(self.config['credential_ref'])}"}
+    def _doc(self, tenant_id, item, event_id=None):
+        body = item.get('text', '')
+        now = datetime.now(timezone.utc)
+        return NormalizedDocument(document_id=f"sharepoint:{item['id']}", tenant_id=tenant_id, title=item['name'], body=body,
+          mime_type=item['mime_type'], source_path=item['web_url'], content_hash=hashlib.sha256(body.encode()).hexdigest(),
+          updated_at=item['last_modified'], acl=ACL(**item.get('acl', {})), provenance=Provenance(connector=self.name,
+          source_uri=item['web_url'], source_id=item['id'], revision_id=item.get('e_tag'), revision_time=item['last_modified'],
+          retrieved_at=now, webhook_event_id=event_id))
+
+    def full(self, tenant_id):
+        items = self.graph.list_files(self.config, self._headers())
+        allowed = set(self.config.get('mime_types', ['application/pdf','application/vnd.openxmlformats-officedocument.wordprocessingml.document','text/html']))
+        docs = [self._doc(tenant_id, x) for x in items if x['mime_type'] in allowed]
+        return IngestionBatch(run_id=hashlib.sha256(self.scope.encode()).hexdigest()[:16], tenant_id=tenant_id, documents=docs,
+          next_cursor=SyncCursor(connector=self.name, scope_fingerprint=self.scope, value=items[-1].get('delta_token') if items else None, version=1, updated_at=datetime.now(timezone.utc)))
+
+    def incremental(self, tenant_id, cursor):
+        items, token = self.graph.delta(self.config, cursor.value if cursor else None, self._headers())
+        docs = [self._doc(tenant_id, x) for x in items if not x.get('deleted')]
+        return IngestionBatch(run_id=hashlib.sha256(self.scope.encode()).hexdigest()[:16], tenant_id=tenant_id, documents=docs,
+          next_cursor=SyncCursor(connector=self.name, scope_fingerprint=self.scope, value=token, version=(cursor.version+1 if cursor else 1), updated_at=datetime.now(timezone.utc)))
+
+    def changes_from_webhook(self, payload):
+        return [Change(path=r['resource'], source_id=r['resource'], revision_id=r.get('resourceData', {}).get('id')) for r in payload.get('value', [])]

kab_ingestion/triggers.py

@@ -0,0 +1,17 @@
+from dataclasses import dataclass
+
+@dataclass(frozen=True)
+class Trigger:
+    connector: str
+    tenant_id: str
+    mode: str = 'incremental'
+    idempotency_key: str = ''
+    source: str = 'on_demand'
+
+class TriggerValidator:
+    @staticmethod
+    def validate(trigger):
+        if trigger.mode not in ('full', 'incremental'): raise ValueError('invalid mode')
+        if trigger.source not in ('on_demand', 'schedule', 'webhook'): raise ValueError('invalid source')
+        if not trigger.tenant_id or not trigger.connector: raise ValueError('tenant and connector required')
+        return trigger