decomposer: generate deliverable files for Define the normalized ingestion contract and pluggable source-connector interface for SCM, CMS, and other content sources.; Implement a GitHub SCM connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of Markdown, plain-text, and source files with webhook and revision metadata.; Implement a SharePoint CMS connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of PDF, DOCX, and HTML files with webhook and revision metadata.; Implement ingestion orchestration and triggers; Normalize, govern, and publish ingested content to the shared knowledge store.; Add deployment/configuration, tests, and documentation for the ingestion agent.; Validate end-to-end ingestion and downstream content availability.

kab_ingestion/__init__.py

@@ -1,5 +1,4 @@
 """KAB content ingestion agent."""
+from .models import Document, ConnectorConfig, SyncCursor
 
-from .models import NormalizedDocument, SyncCursor
-
-__all__ = ["NormalizedDocument", "SyncCursor"]
+__all__ = ["Document", "ConnectorConfig", "SyncCursor"]

kab_ingestion/__main__.py

@@ -1 +1 @@
-print('KAB ingestion adapters are deployed by the platform runtime; use Orchestrator with injected ports.')
+print('Use the host scheduler or webhook adapter to invoke Orchestrator.run().')

kab_ingestion/github.py

@@ -1,40 +1,33 @@
-from __future__ import annotations
-import hashlib
-from datetime import datetime, timezone
-from .models import ACL, Change, IngestionBatch, NormalizedDocument, Provenance, SyncCursor
+from typing import Any
+from .models import *
+from .ports import HTTP, SecretStore
+from .util import *
 
 class GitHubConnector:
-    name = "github"
-    def __init__(self, api, secret_provider, config: dict):
-        self.api, self.secrets, self.config = api, secret_provider, config
-        self.scope = f"{config['owner']}/{config['repo']}:{config.get('path_prefix','')}"
-
-    def _headers(self):
-        return {"Authorization": f"Bearer {self.secrets.get(self.config['credential_ref'])}", "Accept": "application/vnd.github+json"}
-
-    def _doc(self, tenant_id, item, event_id=None):
-        body = item.get("content", "")
-        digest = hashlib.sha256(body.encode()).hexdigest()
-        now = datetime.now(timezone.utc)
-        return NormalizedDocument(document_id=f"github:{item['sha']}", tenant_id=tenant_id,
-            title=item['path'].rsplit('/', 1)[-1], body=body, mime_type=item.get('mime_type','text/plain'),
-            source_path=item['path'], content_hash=digest, updated_at=now, acl=ACL(**item.get('acl', {})),
-            provenance=Provenance(connector=self.name, source_uri=item['html_url'], source_id=item['sha'],
-              revision_id=item.get('commit_sha'), revision_time=item.get('commit_time'), retrieved_at=now, webhook_event_id=event_id))
-
-    def full(self, tenant_id):
-        items = self.api.list_files(self.config['owner'], self.config['repo'], self.config.get('path_prefix',''), self._headers())
-        docs = [self._doc(tenant_id, x) for x in items if x.get('mime_type') in self.config.get('mime_types', ['text/markdown','text/plain','text/x-python'])]
-        return IngestionBatch(run_id=hashlib.sha256(self.scope.encode()).hexdigest()[:16], tenant_id=tenant_id, documents=docs,
-            next_cursor=SyncCursor(connector=self.name, scope_fingerprint=self.scope, value=items[-1].get('commit_sha') if items else None, version=1, updated_at=datetime.now(timezone.utc)))
-
-    def incremental(self, tenant_id, cursor):
-        items = self.api.list_changed(self.config['owner'], self.config['repo'], cursor.value if cursor else None, self.config.get('path_prefix',''), self._headers())
-        batch = self.full(tenant_id)
-        allowed = {x['path'] for x in items}
-        batch.documents = [d for d in batch.documents if d.source_path in allowed]
-        if cursor: batch.next_cursor.version = cursor.version + 1
-        return batch
-
-    def changes_from_webhook(self, payload):
-        return [Change(path=x['filename'], source_id=payload['after'], revision_id=payload['after'], deleted=x['status']=='removed') for x in payload.get('commits', []) for x in x.get('modified', []) + x.get('removed', [])]
+    """Scoped GitHub REST connector; HTTP and secrets are injected."""
+    def __init__(self, http: HTTP, secrets: SecretStore, clock=now): self.http, self.secrets, self.clock = http, secrets, clock
+    def _base(self, c): return f"https://api.github.com/repos/{c.scope['owner']}/{c.scope['repo']}"
+    def _doc(self, c, item, token, event_id=None):
+        path, raw = item["path"], self.http.request("GET", item["download_url"], headers=json_headers(token))
+        content = raw.get("content", "")
+        mime = "text/markdown" if path.lower().endswith((".md", ".markdown")) else "text/plain"
+        prov = Provenance("github", f"{c.scope['owner']}/{c.scope['repo']}:{path}", item.get("html_url", item["download_url"]), item.get("sha"), self.clock(), event_id)
+        return Document(stable_id(c.tenant_id,"github",prov.source_id), c.tenant_id, path.rsplit('/',1)[-1], content, mime, digest(content), item.get("last_commit"), prov, ACL(c.tenant_id, tuple(c.options.get("principals", [])), tuple(c.options.get("groups", []))), {"path":path,"branch":c.scope.get("ref","")})
+    def full(self, c):
+        require_scope(c.scope,("owner","repo")); token=self.secrets.get(c.secret_ref)
+        data=self.http.request("GET", self._base(c)+"/git/trees/"+c.scope.get("ref","HEAD"), headers=json_headers(token), params={"recursive":"1"})
+        paths=c.scope.get("paths", []); allowed=lambda p: (not paths or any(p==x or p.startswith(x.rstrip('/')+'/') for x in paths))
+        items=[x for x in data.get("tree",[]) if x.get("type")=="blob" and allowed(x["path"]) and x["path"].lower().endswith(tuple(c.scope.get("extensions", [".md",".markdown",".txt",".py",".js",".ts",".java",".go"]))) ]
+        docs=[self._doc(c,{**x,"download_url":f"https://raw.githubusercontent.com/{c.scope['owner']}/{c.scope['repo']}/{c.scope.get('ref','HEAD')}/{x['path']}"},token) for x in items]
+        return docs, SyncCursor(c.connector_id,"full",data.get("sha"),updated_at=self.clock())
+    def incremental(self,c,cursor,changes=None):
+        if changes is None: changes=[]
+        token=self.secrets.get(c.secret_ref); docs=[]
+        for ch in changes:
+            if ch.kind in ("deleted","removed"): continue
+            item=self.http.request("GET",self._base(c)+"/contents/"+ch.item_id,headers=json_headers(token),params={"ref":c.scope.get("ref","HEAD")})
+            if item.get("type")=="file": docs.append(self._doc(c,item,token,ch.event_id))
+        return docs, SyncCursor(c.connector_id,"incremental",revision=changes[-1].revision if changes else cursor.revision,updated_at=self.clock())
+    def webhook(self,c,payload,headers):
+        event=payload.get("head_commit",{}).get("id") or payload.get("after")
+        return [Change(x.get("filename",""),"deleted" if payload.get("deleted") else "modified",event,event) for x in payload.get("commits", [{}])[0].get("modified",[])] if payload.get("commits") else [Change(payload.get("path",""),"modified",event,event)]

kab_ingestion/governance.py

@@ -1,14 +1,11 @@
-from __future__ import annotations
-from datetime import datetime, timedelta, timezone
+from .models import Document
+from .util import now
 
-class GovernedPublisher:
-    def __init__(self, store, audit, retention_days=30): self.store, self.audit, self.retention_days = store, audit, retention_days
-    def publish(self, batch):
-        if any(d.tenant_id != batch.tenant_id for d in batch.documents): raise PermissionError('tenant mismatch')
-        self.store.upsert(batch.documents)
-        self.audit.append({'tenant_id': batch.tenant_id, 'run_id': batch.run_id, 'action': 'publish', 'count': len(batch.documents), 'at': datetime.now(timezone.utc).isoformat()})
-    def purge_expired_tombstones(self, docs):
-        cutoff = datetime.now(timezone.utc) - timedelta(days=self.retention_days)
-        ids = [d.document_id for d in docs if d.deleted and d.updated_at < cutoff]
-        if ids: self.store.delete(ids, docs[0].tenant_id)
-        return ids
+def publish(batch, publisher, tenant_id):
+    accepted=[d for d in batch if d.tenant_id==tenant_id and d.acl.tenant_id==tenant_id and not d.deleted]
+    publisher.upsert(accepted)
+    return {"published":len(accepted),"rejected":len(batch)-len(accepted),"tenant_id":tenant_id,"at":now()}
+
+def deletion(ids,publisher,tenant_id): publisher.delete(ids,tenant_id); return {"deleted":len(ids),"tenant_id":tenant_id,"at":now()}
+
+def audit(event,run_id,tenant_id): return {"event":event,"run_id":run_id,"tenant_id":tenant_id,"at":now()}

kab_ingestion/models.py

@@ -1,53 +1,57 @@
-from __future__ import annotations
-from datetime import datetime
-from typing import Literal
-from pydantic import BaseModel, Field
+from dataclasses import dataclass, field
+from typing import Any
 
-class ACL(BaseModel):
-    principals: list[str] = Field(default_factory=list)
-    groups: list[str] = Field(default_factory=list)
-    public: bool = False
+@dataclass(frozen=True)
+class ACL:
+    tenant_id: str
+    principals: tuple[str, ...] = ()
+    groups: tuple[str, ...] = ()
+    visibility: str = "restricted"
 
-class Provenance(BaseModel):
-    connector: str
-    source_uri: str
+@dataclass(frozen=True)
+class Provenance:
+    source_type: str
     source_id: str
-    revision_id: str | None = None
-    revision_time: datetime | None = None
-    retrieved_at: datetime
+    canonical_url: str
+    revision: str | None = None
+    retrieved_at: str | None = None
     webhook_event_id: str | None = None
+    connector_version: str = "1.0"
 
-class NormalizedDocument(BaseModel):
-    document_id: str
+@dataclass(frozen=True)
+class Document:
+    id: str
     tenant_id: str
     title: str
-    body: str
+    content: str
     mime_type: str
-    language: str | None = None
-    source_path: str
     content_hash: str
-    updated_at: datetime
-    deleted: bool = False
-    acl: ACL
+    modified_at: str | None
     provenance: Provenance
-    metadata: dict[str, str] = Field(default_factory=dict)
-
-class SyncCursor(BaseModel):
-    connector: str
-    scope_fingerprint: str
-    value: str | None = None
-    version: int = 0
-    updated_at: datetime
-
-class Change(BaseModel):
-    path: str
-    source_id: str
-    revision_id: str | None = None
+    acl: ACL
+    metadata: dict[str, Any] = field(default_factory=dict)
     deleted: bool = False
 
-class IngestionBatch(BaseModel):
-    run_id: str
+@dataclass(frozen=True)
+class ConnectorConfig:
+    connector_id: str
     tenant_id: str
-    documents: list[NormalizedDocument]
-    next_cursor: SyncCursor | None = None
-    errors: list[dict[str, str]] = Field(default_factory=list)
+    secret_ref: str
+    scope: dict[str, Any]
+    options: dict[str, Any] = field(default_factory=dict)
+
+@dataclass(frozen=True)
+class SyncCursor:
+    connector_id: str
+    mode: str
+    revision: str | None = None
+    delta_token: str | None = None
+    updated_at: str | None = None
+
+@dataclass(frozen=True)
+class Change:
+    item_id: str
+    kind: str
+    revision: str | None = None
+    event_id: str | None = None
+    payload: dict[str, Any] = field(default_factory=dict)

kab_ingestion/orchestrator.py

@@ -1,31 +1,21 @@
-from __future__ import annotations
-import uuid
-from dataclasses import dataclass
-from .models import IngestionBatch
+from dataclasses import dataclass, field
+from .models import *
+from .ports import Connector, Publisher
+from .util import now
 
 @dataclass
 class RunResult:
-    run_id: str
-    status: str
-    batch: IngestionBatch | None = None
-    attempts: int = 0
-    dead_letter: list[dict] | None = None
+    run_id: str; status: str; count: int=0; errors: list[str]=field(default_factory=list); cursor: SyncCursor|None=None
 
 class Orchestrator:
-    def __init__(self, connectors, cursor_store, publisher, max_attempts=3):
-        self.connectors, self.cursors, self.publisher, self.max_attempts = connectors, cursor_store, publisher, max_attempts
-
-    def run(self, connector_name, tenant_id, mode='incremental', idempotency_key=None):
-        connector = self.connectors[connector_name]
-        cursor = self.cursors.load(tenant_id, connector_name)
-        batch = None
-        for attempt in range(1, self.max_attempts + 1):
+    def __init__(self, connectors: dict[str,Connector], publisher: Publisher, max_retries=3): self.connectors,self.publisher,self.max_retries=connectors,publisher,max_retries
+    def run(self, config, cursor=None, mode="incremental", changes=None, run_id=None):
+        run_id=run_id or f"{config.connector_id}:{now()}"
+        for attempt in range(self.max_retries):
             try:
-                batch = connector.full(tenant_id) if mode == 'full' else connector.incremental(tenant_id, cursor)
-                self.publisher.publish(batch)
-                if batch.next_cursor: self.cursors.save(batch.next_cursor)
-                return RunResult(str(uuid.uuid4()), 'succeeded', batch, attempt, [])
-            except Exception as exc:
-                if attempt == self.max_attempts:
-                    return RunResult(str(uuid.uuid4()), 'dead_lettered', batch, attempt, [{'error': type(exc).__name__}])
-        raise AssertionError('unreachable')
+                docs,new_cursor=(self.connectors[config.connector_id].full(config) if mode=="full" else self.connectors[config.connector_id].incremental(config,cursor or SyncCursor(config.connector_id,"incremental"),changes))
+                self.publisher.upsert(docs); return RunResult(run_id,"succeeded",len(docs),cursor=new_cursor)
+            except Exception as e:
+                if attempt==self.max_retries-1: return RunResult(run_id,"dead_letter",errors=[str(e)])
+        return RunResult(run_id,"dead_letter",errors=["unreachable"])
+    def webhook(self,config,payload,headers): return self.connectors[config.connector_id].webhook(config,payload,headers)

kab_ingestion/ports.py

@@ -1,23 +1,17 @@
-from __future__ import annotations
-from typing import Protocol
-from .models import Change, IngestionBatch, SyncCursor
+from typing import Protocol, Iterable, Any
+from .models import ConnectorConfig, SyncCursor, Change, Document
 
-class Connector(Protocol):
-    name: str
-    def full(self, tenant_id: str) -> IngestionBatch: ...
-    def incremental(self, tenant_id: str, cursor: SyncCursor | None) -> IngestionBatch: ...
-    def changes_from_webhook(self, payload: dict) -> list[Change]: ...
+class HTTP(Protocol):
+    def request(self, method: str, url: str, *, headers: dict[str, str], params: dict[str, Any] | None = None) -> dict[str, Any]: ...
 
-class HTTPClient(Protocol):
-    def get(self, url: str, *, headers: dict[str, str], params: dict | None = None): ...
-
-class SecretProvider(Protocol):
+class SecretStore(Protocol):
     def get(self, ref: str) -> str: ...
 
-class CursorStore(Protocol):
-    def load(self, tenant_id: str, connector: str) -> SyncCursor | None: ...
-    def save(self, cursor: SyncCursor) -> None: ...
+class Publisher(Protocol):
+    def upsert(self, documents: Iterable[Document]) -> None: ...
+    def delete(self, ids: Iterable[str], tenant_id: str) -> None: ...
 
-class KnowledgeStore(Protocol):
-    def upsert(self, documents: list) -> None: ...
-    def delete(self, document_ids: list[str], tenant_id: str) -> None: ...
+class Connector(Protocol):
+    def full(self, config: ConnectorConfig) -> tuple[list[Document], SyncCursor]: ...
+    def incremental(self, config: ConnectorConfig, cursor: SyncCursor, changes: list[Change] | None = None) -> tuple[list[Document], SyncCursor]: ...
+    def webhook(self, config: ConnectorConfig, payload: dict[str, Any], headers: dict[str, str]) -> list[Change]: ...

kab_ingestion/sharepoint.py

@@ -1,36 +1,26 @@
-from __future__ import annotations
-import hashlib
-from datetime import datetime, timezone
-from .models import ACL, Change, IngestionBatch, NormalizedDocument, Provenance, SyncCursor
+from .models import *
+from .ports import HTTP, SecretStore
+from .util import *
 
 class SharePointConnector:
-    name = "sharepoint"
-    def __init__(self, graph, secret_provider, config):
-        self.graph, self.secrets, self.config = graph, secret_provider, config
-        self.scope = ":".join(config[x] for x in ('site_id','drive_id')) + ':' + config.get('folder_path','')
-
-    def _headers(self): return {"Authorization": f"Bearer {self.secrets.get(self.config['credential_ref'])}"}
-    def _doc(self, tenant_id, item, event_id=None):
-        body = item.get('text', '')
-        now = datetime.now(timezone.utc)
-        return NormalizedDocument(document_id=f"sharepoint:{item['id']}", tenant_id=tenant_id, title=item['name'], body=body,
-          mime_type=item['mime_type'], source_path=item['web_url'], content_hash=hashlib.sha256(body.encode()).hexdigest(),
-          updated_at=item['last_modified'], acl=ACL(**item.get('acl', {})), provenance=Provenance(connector=self.name,
-          source_uri=item['web_url'], source_id=item['id'], revision_id=item.get('e_tag'), revision_time=item['last_modified'],
-          retrieved_at=now, webhook_event_id=event_id))
-
-    def full(self, tenant_id):
-        items = self.graph.list_files(self.config, self._headers())
-        allowed = set(self.config.get('mime_types', ['application/pdf','application/vnd.openxmlformats-officedocument.wordprocessingml.document','text/html']))
-        docs = [self._doc(tenant_id, x) for x in items if x['mime_type'] in allowed]
-        return IngestionBatch(run_id=hashlib.sha256(self.scope.encode()).hexdigest()[:16], tenant_id=tenant_id, documents=docs,
-          next_cursor=SyncCursor(connector=self.name, scope_fingerprint=self.scope, value=items[-1].get('delta_token') if items else None, version=1, updated_at=datetime.now(timezone.utc)))
-
-    def incremental(self, tenant_id, cursor):
-        items, token = self.graph.delta(self.config, cursor.value if cursor else None, self._headers())
-        docs = [self._doc(tenant_id, x) for x in items if not x.get('deleted')]
-        return IngestionBatch(run_id=hashlib.sha256(self.scope.encode()).hexdigest()[:16], tenant_id=tenant_id, documents=docs,
-          next_cursor=SyncCursor(connector=self.name, scope_fingerprint=self.scope, value=token, version=(cursor.version+1 if cursor else 1), updated_at=datetime.now(timezone.utc)))
-
-    def changes_from_webhook(self, payload):
-        return [Change(path=r['resource'], source_id=r['resource'], revision_id=r.get('resourceData', {}).get('id')) for r in payload.get('value', [])]
+    """Microsoft Graph drive connector for PDF, DOCX, and HTML files."""
+    def __init__(self,http:HTTP,secrets:SecretStore,clock=now): self.http,self.secrets,self.clock=http,secrets,clock
+    def _url(self,c,path): return "https://graph.microsoft.com/v1.0/sites/"+c.scope["site_id"]+path
+    def _doc(self,c,item,token,event=None):
+        content=self.http.request("GET",item["@microsoft.graph.downloadUrl"],headers={}).get("content","")
+        ext=item["name"].lower().rsplit('.',1)[-1]; mime={"pdf":"application/pdf","docx":"application/vnd.openxmlformats-officedocument.wordprocessingml.document","html":"text/html","htm":"text/html"}[ext]
+        p=Provenance("sharepoint",item["id"],item.get("webUrl",""),item.get("eTag"),self.clock(),event)
+        acl=ACL(c.tenant_id,tuple(c.options.get("principals",[])),tuple(c.options.get("groups",[])))
+        return Document(stable_id(c.tenant_id,"sharepoint",item["id"]),c.tenant_id,item["name"],content,mime,digest(content),item.get("lastModifiedDateTime"),p,acl,{"library_id":c.scope.get("library_id"),"folder":c.scope.get("folder")})
+    def full(self,c):
+        require_scope(c.scope,("site_id","library_id")); token=self.secrets.get(c.secret_ref); path=f"/drives/{c.scope['library_id']}/root/children"
+        data=self.http.request("GET",self._url(c,path),headers=json_headers(token)); wanted=tuple(c.scope.get("extensions",[".pdf",".docx",".html"]))
+        docs=[self._doc(c,x,token) for x in data.get("value",[]) if x.get("file") and x["name"].lower().endswith(wanted) and (not c.scope.get("folder") or x.get("parentReference",{}).get("path","").endswith(c.scope["folder"]))]
+        return docs,SyncCursor(c.connector_id,"full",delta_token=data.get("@odata.deltaLink"),updated_at=self.clock())
+    def incremental(self,c,cursor,changes=None):
+        token=self.secrets.get(c.secret_ref); data=self.http.request("GET",cursor.delta_token or self._url(c,f"/drives/{c.scope['library_id']}/root/delta"),headers=json_headers(token)); docs=[]; deleted=[]
+        for x in data.get("value",[]):
+            if "deleted" in x: deleted.append(x["id"])
+            elif x.get("file"): docs.append(self._doc(c,x,token))
+        return docs,SyncCursor(c.connector_id,"incremental",delta_token=data.get("@odata.deltaLink",cursor.delta_token),updated_at=self.clock())
+    def webhook(self,c,payload,headers): return [Change(x.get("resource",""),"modified",x.get("sequenceNumber"),payload.get("subscriptionId")) for x in payload.get("value",[])]

kab_ingestion/util.py

@@ -0,0 +1,18 @@
+import hashlib, json
+from datetime import datetime, timezone
+
+def now() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+def digest(value: bytes | str) -> str:
+    return hashlib.sha256(value if isinstance(value, bytes) else value.encode()).hexdigest()
+
+def stable_id(tenant: str, source: str, item: str) -> str:
+    return digest(f"{tenant}:{source}:{item}")[:32]
+
+def require_scope(scope: dict, keys: tuple[str, ...]) -> None:
+    if any(not scope.get(key) for key in keys):
+        raise ValueError(f"missing required scope: {', '.join(keys)}")
+
+def json_headers(token: str) -> dict[str, str]:
+    return {"Authorization": f"Bearer {token}", "Accept": "application/json"}

kab_ingestion/validation.py

@@ -0,0 +1,8 @@
+from .models import Document
+
+def validate_documents(documents,tenant_id):
+    errors=[]
+    for d in documents:
+        if d.tenant_id!=tenant_id or d.acl.tenant_id!=tenant_id: errors.append(d.id+": tenant isolation failure")
+        if not d.provenance.source_id or not d.content_hash: errors.append(d.id+": missing provenance/hash")
+    return {"passed":not errors,"errors":errors,"count":len(documents)}