decomposer: generate deliverable files for Define the normalized ingestion contract and pluggable source-connector interface for SCM, CMS, and other content sources.; Implement a GitHub SCM connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of Markdown, plain-text, and source files with webhook and revision metadata.; Implement a SharePoint CMS connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of PDF, DOCX, and HTML files with webhook and revision metadata.; Implement ingestion orchestration and triggers; Normalize, govern, and publish ingested content to the shared knowledge store.; Add deployment/configuration, tests, and documentation for the ingestion agent.; Validate end-to-end ingestion and downstream content availability.

kab_ingestion/github.py

@@ -1,40 +1,33 @@
-from __future__ import annotations
-import hashlib
-from datetime import datetime, timezone
-from .models import ACL, Change, IngestionBatch, NormalizedDocument, Provenance, SyncCursor
+from typing import Any
+from .models import *
+from .ports import HTTP, SecretStore
+from .util import *
 
 class GitHubConnector:
-    name = "github"
-    def __init__(self, api, secret_provider, config: dict):
-        self.api, self.secrets, self.config = api, secret_provider, config
-        self.scope = f"{config['owner']}/{config['repo']}:{config.get('path_prefix','')}"
-
-    def _headers(self):
-        return {"Authorization": f"Bearer {self.secrets.get(self.config['credential_ref'])}", "Accept": "application/vnd.github+json"}
-
-    def _doc(self, tenant_id, item, event_id=None):
-        body = item.get("content", "")
-        digest = hashlib.sha256(body.encode()).hexdigest()
-        now = datetime.now(timezone.utc)
-        return NormalizedDocument(document_id=f"github:{item['sha']}", tenant_id=tenant_id,
-            title=item['path'].rsplit('/', 1)[-1], body=body, mime_type=item.get('mime_type','text/plain'),
-            source_path=item['path'], content_hash=digest, updated_at=now, acl=ACL(**item.get('acl', {})),
-            provenance=Provenance(connector=self.name, source_uri=item['html_url'], source_id=item['sha'],
-              revision_id=item.get('commit_sha'), revision_time=item.get('commit_time'), retrieved_at=now, webhook_event_id=event_id))
-
-    def full(self, tenant_id):
-        items = self.api.list_files(self.config['owner'], self.config['repo'], self.config.get('path_prefix',''), self._headers())
-        docs = [self._doc(tenant_id, x) for x in items if x.get('mime_type') in self.config.get('mime_types', ['text/markdown','text/plain','text/x-python'])]
-        return IngestionBatch(run_id=hashlib.sha256(self.scope.encode()).hexdigest()[:16], tenant_id=tenant_id, documents=docs,
-            next_cursor=SyncCursor(connector=self.name, scope_fingerprint=self.scope, value=items[-1].get('commit_sha') if items else None, version=1, updated_at=datetime.now(timezone.utc)))
-
-    def incremental(self, tenant_id, cursor):
-        items = self.api.list_changed(self.config['owner'], self.config['repo'], cursor.value if cursor else None, self.config.get('path_prefix',''), self._headers())
-        batch = self.full(tenant_id)
-        allowed = {x['path'] for x in items}
-        batch.documents = [d for d in batch.documents if d.source_path in allowed]
-        if cursor: batch.next_cursor.version = cursor.version + 1
-        return batch
-
-    def changes_from_webhook(self, payload):
-        return [Change(path=x['filename'], source_id=payload['after'], revision_id=payload['after'], deleted=x['status']=='removed') for x in payload.get('commits', []) for x in x.get('modified', []) + x.get('removed', [])]
+    """Scoped GitHub REST connector; HTTP and secrets are injected."""
+    def __init__(self, http: HTTP, secrets: SecretStore, clock=now): self.http, self.secrets, self.clock = http, secrets, clock
+    def _base(self, c): return f"https://api.github.com/repos/{c.scope['owner']}/{c.scope['repo']}"
+    def _doc(self, c, item, token, event_id=None):
+        path, raw = item["path"], self.http.request("GET", item["download_url"], headers=json_headers(token))
+        content = raw.get("content", "")
+        mime = "text/markdown" if path.lower().endswith((".md", ".markdown")) else "text/plain"
+        prov = Provenance("github", f"{c.scope['owner']}/{c.scope['repo']}:{path}", item.get("html_url", item["download_url"]), item.get("sha"), self.clock(), event_id)
+        return Document(stable_id(c.tenant_id,"github",prov.source_id), c.tenant_id, path.rsplit('/',1)[-1], content, mime, digest(content), item.get("last_commit"), prov, ACL(c.tenant_id, tuple(c.options.get("principals", [])), tuple(c.options.get("groups", []))), {"path":path,"branch":c.scope.get("ref","")})
+    def full(self, c):
+        require_scope(c.scope,("owner","repo")); token=self.secrets.get(c.secret_ref)
+        data=self.http.request("GET", self._base(c)+"/git/trees/"+c.scope.get("ref","HEAD"), headers=json_headers(token), params={"recursive":"1"})
+        paths=c.scope.get("paths", []); allowed=lambda p: (not paths or any(p==x or p.startswith(x.rstrip('/')+'/') for x in paths))
+        items=[x for x in data.get("tree",[]) if x.get("type")=="blob" and allowed(x["path"]) and x["path"].lower().endswith(tuple(c.scope.get("extensions", [".md",".markdown",".txt",".py",".js",".ts",".java",".go"]))) ]
+        docs=[self._doc(c,{**x,"download_url":f"https://raw.githubusercontent.com/{c.scope['owner']}/{c.scope['repo']}/{c.scope.get('ref','HEAD')}/{x['path']}"},token) for x in items]
+        return docs, SyncCursor(c.connector_id,"full",data.get("sha"),updated_at=self.clock())
+    def incremental(self,c,cursor,changes=None):
+        if changes is None: changes=[]
+        token=self.secrets.get(c.secret_ref); docs=[]
+        for ch in changes:
+            if ch.kind in ("deleted","removed"): continue
+            item=self.http.request("GET",self._base(c)+"/contents/"+ch.item_id,headers=json_headers(token),params={"ref":c.scope.get("ref","HEAD")})
+            if item.get("type")=="file": docs.append(self._doc(c,item,token,ch.event_id))
+        return docs, SyncCursor(c.connector_id,"incremental",revision=changes[-1].revision if changes else cursor.revision,updated_at=self.clock())
+    def webhook(self,c,payload,headers):
+        event=payload.get("head_commit",{}).get("id") or payload.get("after")
+        return [Change(x.get("filename",""),"deleted" if payload.get("deleted") else "modified",event,event) for x in payload.get("commits", [{}])[0].get("modified",[])] if payload.get("commits") else [Change(payload.get("path",""),"modified",event,event)]