decomposer: generate deliverable files for Define the normalized ingestion contract and pluggable source-connector interface for SCM, CMS, and other content sources.; Implement a GitHub SCM connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of Markdown, plain-text, and source files with webhook and revision metadata.; Implement a SharePoint CMS connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of PDF, DOCX, and HTML files with webhook and revision metadata.; Implement ingestion orchestration and triggers; Normalize, govern, and publish ingested content to the shared knowledge store.; Add deployment/configuration, tests, and documentation for the ingestion agent.; Validate end-to-end ingestion and downstream content availability.

kab_ingestion/github.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+import hashlib
+from datetime import datetime, timezone
+from .models import ACL, Change, IngestionBatch, NormalizedDocument, Provenance, SyncCursor
+
+class GitHubConnector:
+    name = "github"
+    def __init__(self, api, secret_provider, config: dict):
+        self.api, self.secrets, self.config = api, secret_provider, config
+        self.scope = f"{config['owner']}/{config['repo']}:{config.get('path_prefix','')}"
+
+    def _headers(self):
+        return {"Authorization": f"Bearer {self.secrets.get(self.config['credential_ref'])}", "Accept": "application/vnd.github+json"}
+
+    def _doc(self, tenant_id, item, event_id=None):
+        body = item.get("content", "")
+        digest = hashlib.sha256(body.encode()).hexdigest()
+        now = datetime.now(timezone.utc)
+        return NormalizedDocument(document_id=f"github:{item['sha']}", tenant_id=tenant_id,
+            title=item['path'].rsplit('/', 1)[-1], body=body, mime_type=item.get('mime_type','text/plain'),
+            source_path=item['path'], content_hash=digest, updated_at=now, acl=ACL(**item.get('acl', {})),
+            provenance=Provenance(connector=self.name, source_uri=item['html_url'], source_id=item['sha'],
+              revision_id=item.get('commit_sha'), revision_time=item.get('commit_time'), retrieved_at=now, webhook_event_id=event_id))
+
+    def full(self, tenant_id):
+        items = self.api.list_files(self.config['owner'], self.config['repo'], self.config.get('path_prefix',''), self._headers())
+        docs = [self._doc(tenant_id, x) for x in items if x.get('mime_type') in self.config.get('mime_types', ['text/markdown','text/plain','text/x-python'])]
+        return IngestionBatch(run_id=hashlib.sha256(self.scope.encode()).hexdigest()[:16], tenant_id=tenant_id, documents=docs,
+            next_cursor=SyncCursor(connector=self.name, scope_fingerprint=self.scope, value=items[-1].get('commit_sha') if items else None, version=1, updated_at=datetime.now(timezone.utc)))
+
+    def incremental(self, tenant_id, cursor):
+        items = self.api.list_changed(self.config['owner'], self.config['repo'], cursor.value if cursor else None, self.config.get('path_prefix',''), self._headers())
+        batch = self.full(tenant_id)
+        allowed = {x['path'] for x in items}
+        batch.documents = [d for d in batch.documents if d.source_path in allowed]
+        if cursor: batch.next_cursor.version = cursor.version + 1
+        return batch
+
+    def changes_from_webhook(self, payload):
+        return [Change(path=x['filename'], source_id=payload['after'], revision_id=payload['after'], deleted=x['status']=='removed') for x in payload.get('commits', []) for x in x.get('modified', []) + x.get('removed', [])]