decomposer: generate deliverable files for Define the normalized ingestion contract and pluggable source-connector interface for SCM, CMS, and other content sources.; Implement a GitHub SCM connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of Markdown, plain-text, and source files with webhook and revision metadata.; Implement a SharePoint CMS connector that conforms to the normalized ingestion contract and supports scoped full and incremental ingestion of PDF, DOCX, and HTML files with webhook and revision metadata.; Implement ingestion orchestration and triggers; Normalize, govern, and publish ingested content to the shared knowledge store.; Add deployment/configuration, tests, and documentation for the ingestion agent.; Validate end-to-end ingestion and downstream content availability.

kab_ingestion/models.py

@@ -1,53 +1,57 @@
-from __future__ import annotations
-from datetime import datetime
-from typing import Literal
-from pydantic import BaseModel, Field
+from dataclasses import dataclass, field
+from typing import Any
 
-class ACL(BaseModel):
-    principals: list[str] = Field(default_factory=list)
-    groups: list[str] = Field(default_factory=list)
-    public: bool = False
+@dataclass(frozen=True)
+class ACL:
+    tenant_id: str
+    principals: tuple[str, ...] = ()
+    groups: tuple[str, ...] = ()
+    visibility: str = "restricted"
 
-class Provenance(BaseModel):
-    connector: str
-    source_uri: str
+@dataclass(frozen=True)
+class Provenance:
+    source_type: str
     source_id: str
-    revision_id: str | None = None
-    revision_time: datetime | None = None
-    retrieved_at: datetime
+    canonical_url: str
+    revision: str | None = None
+    retrieved_at: str | None = None
     webhook_event_id: str | None = None
+    connector_version: str = "1.0"
 
-class NormalizedDocument(BaseModel):
-    document_id: str
+@dataclass(frozen=True)
+class Document:
+    id: str
     tenant_id: str
     title: str
-    body: str
+    content: str
     mime_type: str
-    language: str | None = None
-    source_path: str
     content_hash: str
-    updated_at: datetime
-    deleted: bool = False
-    acl: ACL
+    modified_at: str | None
     provenance: Provenance
-    metadata: dict[str, str] = Field(default_factory=dict)
-
-class SyncCursor(BaseModel):
-    connector: str
-    scope_fingerprint: str
-    value: str | None = None
-    version: int = 0
-    updated_at: datetime
-
-class Change(BaseModel):
-    path: str
-    source_id: str
-    revision_id: str | None = None
+    acl: ACL
+    metadata: dict[str, Any] = field(default_factory=dict)
     deleted: bool = False
 
-class IngestionBatch(BaseModel):
-    run_id: str
+@dataclass(frozen=True)
+class ConnectorConfig:
+    connector_id: str
     tenant_id: str
-    documents: list[NormalizedDocument]
-    next_cursor: SyncCursor | None = None
-    errors: list[dict[str, str]] = Field(default_factory=list)
+    secret_ref: str
+    scope: dict[str, Any]
+    options: dict[str, Any] = field(default_factory=dict)
+
+@dataclass(frozen=True)
+class SyncCursor:
+    connector_id: str
+    mode: str
+    revision: str | None = None
+    delta_token: str | None = None
+    updated_at: str | None = None
+
+@dataclass(frozen=True)
+class Change:
+    item_id: str
+    kind: str
+    revision: str | None = None
+    event_id: str | None = None
+    payload: dict[str, Any] = field(default_factory=dict)