decomposer: generate deliverable files for Define the normalized ingestion contract and pluggable source-connector interface for SCM, CMS, and other content sources.; Implement a GitHub SCM connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of Markdown, plain-text, and source files with webhook and revision metadata.; Implement a SharePoint CMS connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of PDF, DOCX, and HTML files with webhook and revision metadata.; Implement ingestion orchestration and triggers; Normalize, govern, and publish ingested content to the shared knowledge store.; Add deployment/configuration, tests, and documentation for the ingestion agent.; Validate end-to-end ingestion and downstream content availability.

kab_ingestion/sharepoint.py

@@ -1,36 +1,26 @@
-from __future__ import annotations
-import hashlib
-from datetime import datetime, timezone
-from .models import ACL, Change, IngestionBatch, NormalizedDocument, Provenance, SyncCursor
+from .models import *
+from .ports import HTTP, SecretStore
+from .util import *
 
 class SharePointConnector:
-    name = "sharepoint"
-    def __init__(self, graph, secret_provider, config):
-        self.graph, self.secrets, self.config = graph, secret_provider, config
-        self.scope = ":".join(config[x] for x in ('site_id','drive_id')) + ':' + config.get('folder_path','')
-
-    def _headers(self): return {"Authorization": f"Bearer {self.secrets.get(self.config['credential_ref'])}"}
-    def _doc(self, tenant_id, item, event_id=None):
-        body = item.get('text', '')
-        now = datetime.now(timezone.utc)
-        return NormalizedDocument(document_id=f"sharepoint:{item['id']}", tenant_id=tenant_id, title=item['name'], body=body,
-          mime_type=item['mime_type'], source_path=item['web_url'], content_hash=hashlib.sha256(body.encode()).hexdigest(),
-          updated_at=item['last_modified'], acl=ACL(**item.get('acl', {})), provenance=Provenance(connector=self.name,
-          source_uri=item['web_url'], source_id=item['id'], revision_id=item.get('e_tag'), revision_time=item['last_modified'],
-          retrieved_at=now, webhook_event_id=event_id))
-
-    def full(self, tenant_id):
-        items = self.graph.list_files(self.config, self._headers())
-        allowed = set(self.config.get('mime_types', ['application/pdf','application/vnd.openxmlformats-officedocument.wordprocessingml.document','text/html']))
-        docs = [self._doc(tenant_id, x) for x in items if x['mime_type'] in allowed]
-        return IngestionBatch(run_id=hashlib.sha256(self.scope.encode()).hexdigest()[:16], tenant_id=tenant_id, documents=docs,
-          next_cursor=SyncCursor(connector=self.name, scope_fingerprint=self.scope, value=items[-1].get('delta_token') if items else None, version=1, updated_at=datetime.now(timezone.utc)))
-
-    def incremental(self, tenant_id, cursor):
-        items, token = self.graph.delta(self.config, cursor.value if cursor else None, self._headers())
-        docs = [self._doc(tenant_id, x) for x in items if not x.get('deleted')]
-        return IngestionBatch(run_id=hashlib.sha256(self.scope.encode()).hexdigest()[:16], tenant_id=tenant_id, documents=docs,
-          next_cursor=SyncCursor(connector=self.name, scope_fingerprint=self.scope, value=token, version=(cursor.version+1 if cursor else 1), updated_at=datetime.now(timezone.utc)))
-
-    def changes_from_webhook(self, payload):
-        return [Change(path=r['resource'], source_id=r['resource'], revision_id=r.get('resourceData', {}).get('id')) for r in payload.get('value', [])]
+    """Microsoft Graph drive connector for PDF, DOCX, and HTML files."""
+    def __init__(self,http:HTTP,secrets:SecretStore,clock=now): self.http,self.secrets,self.clock=http,secrets,clock
+    def _url(self,c,path): return "https://graph.microsoft.com/v1.0/sites/"+c.scope["site_id"]+path
+    def _doc(self,c,item,token,event=None):
+        content=self.http.request("GET",item["@microsoft.graph.downloadUrl"],headers={}).get("content","")
+        ext=item["name"].lower().rsplit('.',1)[-1]; mime={"pdf":"application/pdf","docx":"application/vnd.openxmlformats-officedocument.wordprocessingml.document","html":"text/html","htm":"text/html"}[ext]
+        p=Provenance("sharepoint",item["id"],item.get("webUrl",""),item.get("eTag"),self.clock(),event)
+        acl=ACL(c.tenant_id,tuple(c.options.get("principals",[])),tuple(c.options.get("groups",[])))
+        return Document(stable_id(c.tenant_id,"sharepoint",item["id"]),c.tenant_id,item["name"],content,mime,digest(content),item.get("lastModifiedDateTime"),p,acl,{"library_id":c.scope.get("library_id"),"folder":c.scope.get("folder")})
+    def full(self,c):
+        require_scope(c.scope,("site_id","library_id")); token=self.secrets.get(c.secret_ref); path=f"/drives/{c.scope['library_id']}/root/children"
+        data=self.http.request("GET",self._url(c,path),headers=json_headers(token)); wanted=tuple(c.scope.get("extensions",[".pdf",".docx",".html"]))
+        docs=[self._doc(c,x,token) for x in data.get("value",[]) if x.get("file") and x["name"].lower().endswith(wanted) and (not c.scope.get("folder") or x.get("parentReference",{}).get("path","").endswith(c.scope["folder"]))]
+        return docs,SyncCursor(c.connector_id,"full",delta_token=data.get("@odata.deltaLink"),updated_at=self.clock())
+    def incremental(self,c,cursor,changes=None):
+        token=self.secrets.get(c.secret_ref); data=self.http.request("GET",cursor.delta_token or self._url(c,f"/drives/{c.scope['library_id']}/root/delta"),headers=json_headers(token)); docs=[]; deleted=[]
+        for x in data.get("value",[]):
+            if "deleted" in x: deleted.append(x["id"])
+            elif x.get("file"): docs.append(self._doc(c,x,token))
+        return docs,SyncCursor(c.connector_id,"incremental",delta_token=data.get("@odata.deltaLink",cursor.delta_token),updated_at=self.clock())
+    def webhook(self,c,payload,headers): return [Change(x.get("resource",""),"modified",x.get("sequenceNumber"),payload.get("subscriptionId")) for x in payload.get("value",[])]