decomposer: generate deliverable files for Define the normalized ingestion contract and pluggable source-connector interface for SCM, CMS, and other content sources.; Implement a GitHub SCM connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of Markdown, plain-text, and source files with webhook and revision metadata.; Implement a SharePoint CMS connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of PDF, DOCX, and HTML files with webhook and revision metadata.; Implement ingestion orchestration and triggers; Normalize, govern, and publish ingested content to the shared knowledge store.; Add deployment/configuration, tests, and documentation for the ingestion agent.; Validate end-to-end ingestion and downstream content availability.

tests/test_connectors.py

@@ -1,17 +1,21 @@
-from datetime import datetime, timezone
 from kab_ingestion.github import GitHubConnector
 from kab_ingestion.sharepoint import SharePointConnector
+from kab_ingestion.models import ConnectorConfig, SyncCursor
 class Secrets:
- def get(self, ref): return 'token'
-class GH:
- def list_files(self,*a): return [{'path':'docs/a.md','content':'hello','mime_type':'text/markdown','sha':'1','html_url':'u'}]
- def list_changed(self,*a): return self.list_files()
-class SP:
- def list_files(self,*a): return [{'id':'1','name':'a.html','text':'x','mime_type':'text/html','web_url':'u','last_modified':datetime.now(timezone.utc)}]
- def delta(self,*a): return self.list_files(), 'next'
-def test_github_scope_and_normalization():
- c=GitHubConnector(GH(),Secrets(),{'owner':'o','repo':'r','path_prefix':'docs/','credential_ref':'s'})
- assert c.full('t').documents[0].document_id == 'github:1'
-def test_sharepoint_normalization():
- c=SharePointConnector(SP(),Secrets(),{'site_id':'s','drive_id':'d','credential_ref':'x'})
- assert c.full('t').documents[0].mime_type == 'text/html'
+ def get(self,r): return "token"
+class HTTP:
+ def __init__(self, responses): self.responses=responses
+ def request(self,*a,**k): return self.responses.pop(0)
+def gh():
+ c=ConnectorConfig("gh","t1","s",{"owner":"o","repo":"r","ref":"main"})
+ h=HTTP([{"tree":[{"type":"blob","path":"a.md","sha":"1"}]},{"content":"hello"}])
+ return GitHubConnector(h,Secrets()).full(c)
+def test_github_full():
+ docs,c=gh(); assert docs[0].mime_type=="text/markdown" and docs[0].acl.tenant_id=="t1"
+def test_github_scope_required():
+ try: GitHubConnector(HTTP([]),Secrets()).full(ConnectorConfig("x","t","s",{})); assert False
+ except ValueError: assert True
+def test_sharepoint_full():
+ c=ConnectorConfig("sp","t1","s",{"site_id":"s","library_id":"l"})
+ item={"id":"i","name":"a.pdf","eTag":"e","webUrl":"u","@microsoft.graph.downloadUrl":"raw","file":{}}
+ docs,_=SharePointConnector(HTTP([{"value":[item]},{"content":"pdf"}]),Secrets()).full(c); assert docs[0].mime_type=="application/pdf"