from typing import Any from .models import * from .ports import HTTP, SecretStore from .util import * class GitHubConnector: """Scoped GitHub REST connector; HTTP and secrets are injected.""" def __init__(self, http: HTTP, secrets: SecretStore, clock=now): self.http, self.secrets, self.clock = http, secrets, clock def _base(self, c): return f"https://api.github.com/repos/{c.scope['owner']}/{c.scope['repo']}" def _doc(self, c, item, token, event_id=None): path, raw = item["path"], self.http.request("GET", item["download_url"], headers=json_headers(token)) content = raw.get("content", "") mime = "text/markdown" if path.lower().endswith((".md", ".markdown")) else "text/plain" prov = Provenance("github", f"{c.scope['owner']}/{c.scope['repo']}:{path}", item.get("html_url", item["download_url"]), item.get("sha"), self.clock(), event_id) return Document(stable_id(c.tenant_id,"github",prov.source_id), c.tenant_id, path.rsplit('/',1)[-1], content, mime, digest(content), item.get("last_commit"), prov, ACL(c.tenant_id, tuple(c.options.get("principals", [])), tuple(c.options.get("groups", []))), {"path":path,"branch":c.scope.get("ref","")}) def full(self, c): require_scope(c.scope,("owner","repo")); token=self.secrets.get(c.secret_ref) data=self.http.request("GET", self._base(c)+"/git/trees/"+c.scope.get("ref","HEAD"), headers=json_headers(token), params={"recursive":"1"}) paths=c.scope.get("paths", []); allowed=lambda p: (not paths or any(p==x or p.startswith(x.rstrip('/')+'/') for x in paths)) items=[x for x in data.get("tree",[]) if x.get("type")=="blob" and allowed(x["path"]) and x["path"].lower().endswith(tuple(c.scope.get("extensions", [".md",".markdown",".txt",".py",".js",".ts",".java",".go"]))) ] docs=[self._doc(c,{**x,"download_url":f"https://raw.githubusercontent.com/{c.scope['owner']}/{c.scope['repo']}/{c.scope.get('ref','HEAD')}/{x['path']}"},token) for x in items] return docs, SyncCursor(c.connector_id,"full",data.get("sha"),updated_at=self.clock()) def incremental(self,c,cursor,changes=None): if changes is None: changes=[] token=self.secrets.get(c.secret_ref); docs=[] for ch in changes: if ch.kind in ("deleted","removed"): continue item=self.http.request("GET",self._base(c)+"/contents/"+ch.item_id,headers=json_headers(token),params={"ref":c.scope.get("ref","HEAD")}) if item.get("type")=="file": docs.append(self._doc(c,item,token,ch.event_id)) return docs, SyncCursor(c.connector_id,"incremental",revision=changes[-1].revision if changes else cursor.revision,updated_at=self.clock()) def webhook(self,c,payload,headers): event=payload.get("head_commit",{}).get("id") or payload.get("after") return [Change(x.get("filename",""),"deleted" if payload.get("deleted") else "modified",event,event) for x in payload.get("commits", [{}])[0].get("modified",[])] if payload.get("commits") else [Change(payload.get("path",""),"modified",event,event)]