decomposer: generate deliverable files for Define the normalized ingestion contract and pluggable source-connector interface for SCM, CMS, and other content sources.; Implement a GitHub SCM connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of Markdown, plain-text, and source files with webhook and revision metadata.; Implement a SharePoint CMS connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of PDF, DOCX, and HTML files with webhook and revision metadata.; Implement ingestion orchestration and triggers; Normalize, govern, and publish ingested content to the shared knowledge store.; Add deployment/configuration, tests, and documentation for the ingestion agent.; Validate end-to-end ingestion and downstream content availability.

tests/test_connectors.py

@@ -1,17 +1,21 @@
-from datetime import datetime, timezone
 from kab_ingestion.github import GitHubConnector
 from kab_ingestion.sharepoint import SharePointConnector
+from kab_ingestion.models import ConnectorConfig, SyncCursor
 class Secrets:
- def get(self, ref): return 'token'
-class GH:
- def list_files(self,*a): return [{'path':'docs/a.md','content':'hello','mime_type':'text/markdown','sha':'1','html_url':'u'}]
- def list_changed(self,*a): return self.list_files()
-class SP:
- def list_files(self,*a): return [{'id':'1','name':'a.html','text':'x','mime_type':'text/html','web_url':'u','last_modified':datetime.now(timezone.utc)}]
- def delta(self,*a): return self.list_files(), 'next'
-def test_github_scope_and_normalization():
- c=GitHubConnector(GH(),Secrets(),{'owner':'o','repo':'r','path_prefix':'docs/','credential_ref':'s'})
- assert c.full('t').documents[0].document_id == 'github:1'
-def test_sharepoint_normalization():
- c=SharePointConnector(SP(),Secrets(),{'site_id':'s','drive_id':'d','credential_ref':'x'})
- assert c.full('t').documents[0].mime_type == 'text/html'
+ def get(self,r): return "token"
+class HTTP:
+ def __init__(self, responses): self.responses=responses
+ def request(self,*a,**k): return self.responses.pop(0)
+def gh():
+ c=ConnectorConfig("gh","t1","s",{"owner":"o","repo":"r","ref":"main"})
+ h=HTTP([{"tree":[{"type":"blob","path":"a.md","sha":"1"}]},{"content":"hello"}])
+ return GitHubConnector(h,Secrets()).full(c)
+def test_github_full():
+ docs,c=gh(); assert docs[0].mime_type=="text/markdown" and docs[0].acl.tenant_id=="t1"
+def test_github_scope_required():
+ try: GitHubConnector(HTTP([]),Secrets()).full(ConnectorConfig("x","t","s",{})); assert False
+ except ValueError: assert True
+def test_sharepoint_full():
+ c=ConnectorConfig("sp","t1","s",{"site_id":"s","library_id":"l"})
+ item={"id":"i","name":"a.pdf","eTag":"e","webUrl":"u","@microsoft.graph.downloadUrl":"raw","file":{}}
+ docs,_=SharePointConnector(HTTP([{"value":[item]},{"content":"pdf"}]),Secrets()).full(c); assert docs[0].mime_type=="application/pdf"

tests/test_orchestration.py

@@ -1,12 +1,16 @@
 from kab_ingestion.orchestrator import Orchestrator
+from kab_ingestion.models import *
 class C:
- def incremental(self,t,c): return type('B',(),{'next_cursor':None})()
- def full(self,t): return self.incremental(t,None)
-class Cur:
- def load(self,*a): return None
- def save(self,*a): pass
-class Pub:
- def publish(self,b): pass
-def test_run_success():
- r=Orchestrator({'x':C()},Cur(),Pub()).run('x','t')
- assert r.status == 'succeeded'
+ def full(self,c): return [],SyncCursor(c.connector_id,"full")
+ def incremental(self,c,cur,changes=None): return [],cur
+ def webhook(self,c,p,h): return []
+class P:
+ def upsert(self,d): self.d=list(d)
+ def delete(self,i,t): pass
+def test_run_is_idempotent_shape():
+ p=P(); r=Orchestrator({"x":C()},p).run(ConnectorConfig("x","t","s",{}),mode="full",run_id="fixed")
+ assert r.status=="succeeded" and r.run_id=="fixed"
+def test_validation_rejects_cross_tenant():
+ from kab_ingestion.validation import validate_documents
+ d=Document("i","other","x","x","text/plain","h",None,Provenance("x","i","u"),ACL("other"))
+ assert not validate_documents([d],"t")["passed"]