34 lines
3.0 KiB
Python
34 lines
3.0 KiB
Python
from typing import Any
|
|
from .models import *
|
|
from .ports import HTTP, SecretStore
|
|
from .util import *
|
|
|
|
class GitHubConnector:
|
|
"""Scoped GitHub REST connector; HTTP and secrets are injected."""
|
|
def __init__(self, http: HTTP, secrets: SecretStore, clock=now): self.http, self.secrets, self.clock = http, secrets, clock
|
|
def _base(self, c): return f"https://api.github.com/repos/{c.scope['owner']}/{c.scope['repo']}"
|
|
def _doc(self, c, item, token, event_id=None):
|
|
path, raw = item["path"], self.http.request("GET", item["download_url"], headers=json_headers(token))
|
|
content = raw.get("content", "")
|
|
mime = "text/markdown" if path.lower().endswith((".md", ".markdown")) else "text/plain"
|
|
prov = Provenance("github", f"{c.scope['owner']}/{c.scope['repo']}:{path}", item.get("html_url", item["download_url"]), item.get("sha"), self.clock(), event_id)
|
|
return Document(stable_id(c.tenant_id,"github",prov.source_id), c.tenant_id, path.rsplit('/',1)[-1], content, mime, digest(content), item.get("last_commit"), prov, ACL(c.tenant_id, tuple(c.options.get("principals", [])), tuple(c.options.get("groups", []))), {"path":path,"branch":c.scope.get("ref","")})
|
|
def full(self, c):
|
|
require_scope(c.scope,("owner","repo")); token=self.secrets.get(c.secret_ref)
|
|
data=self.http.request("GET", self._base(c)+"/git/trees/"+c.scope.get("ref","HEAD"), headers=json_headers(token), params={"recursive":"1"})
|
|
paths=c.scope.get("paths", []); allowed=lambda p: (not paths or any(p==x or p.startswith(x.rstrip('/')+'/') for x in paths))
|
|
items=[x for x in data.get("tree",[]) if x.get("type")=="blob" and allowed(x["path"]) and x["path"].lower().endswith(tuple(c.scope.get("extensions", [".md",".markdown",".txt",".py",".js",".ts",".java",".go"]))) ]
|
|
docs=[self._doc(c,{**x,"download_url":f"https://raw.githubusercontent.com/{c.scope['owner']}/{c.scope['repo']}/{c.scope.get('ref','HEAD')}/{x['path']}"},token) for x in items]
|
|
return docs, SyncCursor(c.connector_id,"full",data.get("sha"),updated_at=self.clock())
|
|
def incremental(self,c,cursor,changes=None):
|
|
if changes is None: changes=[]
|
|
token=self.secrets.get(c.secret_ref); docs=[]
|
|
for ch in changes:
|
|
if ch.kind in ("deleted","removed"): continue
|
|
item=self.http.request("GET",self._base(c)+"/contents/"+ch.item_id,headers=json_headers(token),params={"ref":c.scope.get("ref","HEAD")})
|
|
if item.get("type")=="file": docs.append(self._doc(c,item,token,ch.event_id))
|
|
return docs, SyncCursor(c.connector_id,"incremental",revision=changes[-1].revision if changes else cursor.revision,updated_at=self.clock())
|
|
def webhook(self,c,payload,headers):
|
|
event=payload.get("head_commit",{}).get("id") or payload.get("after")
|
|
return [Change(x.get("filename",""),"deleted" if payload.get("deleted") else "modified",event,event) for x in payload.get("commits", [{}])[0].get("modified",[])] if payload.get("commits") else [Change(payload.get("path",""),"modified",event,event)]
|