decomposer: generate deliverable files for Define the normalized ingestion contract and pluggable source-connector interface for SCM, CMS, and other content sources.; Implement a GitHub SCM connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of Markdown, plain-text, and source files with webhook and revision metadata.; Implement a SharePoint CMS connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of PDF, DOCX, and HTML files with webhook and revision metadata.; Implement ingestion orchestration and triggers; Normalize, govern, and publish ingested content to the shared knowledge store.; Add deployment/configuration, tests, and documentation for the ingestion agent.; Validate end-to-end ingestion and downstream content availability.

kab_ingestion/models.py

@@ -0,0 +1,53 @@
+from __future__ import annotations
+from datetime import datetime
+from typing import Literal
+from pydantic import BaseModel, Field
+
+class ACL(BaseModel):
+    principals: list[str] = Field(default_factory=list)
+    groups: list[str] = Field(default_factory=list)
+    public: bool = False
+
+class Provenance(BaseModel):
+    connector: str
+    source_uri: str
+    source_id: str
+    revision_id: str | None = None
+    revision_time: datetime | None = None
+    retrieved_at: datetime
+    webhook_event_id: str | None = None
+
+class NormalizedDocument(BaseModel):
+    document_id: str
+    tenant_id: str
+    title: str
+    body: str
+    mime_type: str
+    language: str | None = None
+    source_path: str
+    content_hash: str
+    updated_at: datetime
+    deleted: bool = False
+    acl: ACL
+    provenance: Provenance
+    metadata: dict[str, str] = Field(default_factory=dict)
+
+class SyncCursor(BaseModel):
+    connector: str
+    scope_fingerprint: str
+    value: str | None = None
+    version: int = 0
+    updated_at: datetime
+
+class Change(BaseModel):
+    path: str
+    source_id: str
+    revision_id: str | None = None
+    deleted: bool = False
+
+class IngestionBatch(BaseModel):
+    run_id: str
+    tenant_id: str
+    documents: list[NormalizedDocument]
+    next_cursor: SyncCursor | None = None
+    errors: list[dict[str, str]] = Field(default_factory=list)