From 85165394fbd049d2facb409406e9039ff46f2bfa Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Fri, 27 Dec 2024 05:03:42 +0800 Subject: [PATCH 1/7] feat(github): add github api driver --- drivers/all.go | 1 + drivers/github/driver.go | 410 +++++++++++++++++++++++++++++++++++++++ drivers/github/meta.go | 33 ++++ drivers/github/types.go | 50 +++++ drivers/github/util.go | 57 ++++++ 5 files changed, 551 insertions(+) create mode 100644 drivers/github/driver.go create mode 100644 drivers/github/meta.go create mode 100644 drivers/github/types.go create mode 100644 drivers/github/util.go diff --git a/drivers/all.go b/drivers/all.go index 4c4ef5c147b..8b253a08558 100644 --- a/drivers/all.go +++ b/drivers/all.go @@ -24,6 +24,7 @@ import ( _ "github.com/alist-org/alist/v3/drivers/dropbox" _ "github.com/alist-org/alist/v3/drivers/febbox" _ "github.com/alist-org/alist/v3/drivers/ftp" + _ "github.com/alist-org/alist/v3/drivers/github" _ "github.com/alist-org/alist/v3/drivers/google_drive" _ "github.com/alist-org/alist/v3/drivers/google_photo" _ "github.com/alist-org/alist/v3/drivers/halalcloud" diff --git a/drivers/github/driver.go b/drivers/github/driver.go new file mode 100644 index 00000000000..39a5fff19ae --- /dev/null +++ b/drivers/github/driver.go @@ -0,0 +1,410 @@ +package github + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "github.com/alist-org/alist/v3/drivers/base" + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/errs" + "github.com/alist-org/alist/v3/internal/model" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "io" + stdpath "path" + "strconv" + "strings" + "text/template" +) + +type Github struct { + model.Storage + Addition + client *resty.Client + mkdirMsgTmpl *template.Template + deleteMsgTmpl *template.Template + putMsgTmpl *template.Template +} + +func (d *Github) Config() driver.Config { + return config +} + +func (d *Github) GetAddition() driver.Additional { + return &d.Addition +} + +func (d *Github) Init(ctx context.Context) error { + d.RootFolderPath = utils.FixAndCleanPath(d.RootFolderPath) + if d.CommitterName != "" && d.CommitterEmail == "" { + return errors.New("committer email is required") + } + if d.CommitterName == "" && d.CommitterEmail != "" { + return errors.New("committer name is required") + } + if d.AuthorName != "" && d.AuthorEmail == "" { + return errors.New("author email is required") + } + if d.AuthorName == "" && d.AuthorEmail != "" { + return errors.New("author name is required") + } + var err error + d.mkdirMsgTmpl, err = template.New("mkdirCommitMsgTemplate").Parse(d.MkdirCommitMsg) + if err != nil { + return err + } + d.deleteMsgTmpl, err = template.New("deleteCommitMsgTemplate").Parse(d.DeleteCommitMsg) + if err != nil { + return err + } + d.putMsgTmpl, err = template.New("putCommitMsgTemplate").Parse(d.PutCommitMsg) + if err != nil { + return err + } + d.client = base.NewRestyClient(). + SetHeader("Accept", "application/vnd.github.object+json"). + SetHeader("Authorization", "Bearer "+d.Token). + SetHeader("X-GitHub-Api-Version", "2022-11-28") + return nil +} + +func (d *Github) Drop(ctx context.Context) error { + return nil +} + +func (d *Github) List(ctx context.Context, dir model.Obj, args model.ListArgs) ([]model.Obj, error) { + obj, err := d.get(dir.GetPath()) + if err != nil { + return nil, err + } + if obj.Entries == nil { + return nil, errs.NotFolder + } + ret := make([]model.Obj, 0, len(obj.Entries)) + for _, entry := range obj.Entries { + if entry.Name != ".gitkeep" { + ret = append(ret, entry.toModelObj()) + } + } + return ret, nil +} + +func (d *Github) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { + obj, err := d.get(file.GetPath()) + if err != nil { + return nil, err + } + return &model.Link{ + URL: obj.DownloadURL, + }, nil +} + +func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + commitMessage, err := getMessage(d.mkdirMsgTmpl, &MessageTemplateVars{ + UserName: ctx.Value("user").(*model.User).Username, + ObjName: dirName, + ObjPath: stdpath.Join(parentDir.GetPath(), dirName), + ParentName: parentDir.GetName(), + ParentPath: parentDir.GetPath(), + }, "mkdir") + if err != nil { + return err + } + parent, err := d.get(parentDir.GetPath()) + if err != nil { + return err + } + if parent.Entries == nil { + return errs.NotFolder + } + // if parent folder contains .gitkeep only, mark it and delete .gitkeep later + gitKeepSha := "" + if len(parent.Entries) == 1 && parent.Entries[0].Name == ".gitkeep" { + gitKeepSha = parent.Entries[0].Sha + } + + if err = d.createGitKeep(stdpath.Join(parentDir.GetPath(), dirName), commitMessage); err != nil { + return err + } + if gitKeepSha != "" { + err = d.delete(stdpath.Join(parentDir.GetPath(), ".gitkeep"), gitKeepSha, commitMessage) + } + return err +} + +func (d *Github) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotSupport +} + +func (d *Github) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { + return nil, errs.NotSupport +} + +func (d *Github) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { + return nil, errs.NotSupport +} + +func (d *Github) Remove(ctx context.Context, obj model.Obj) error { + parentDir := stdpath.Dir(obj.GetPath()) + commitMessage, err := getMessage(d.deleteMsgTmpl, &MessageTemplateVars{ + UserName: ctx.Value("user").(*model.User).Username, + ObjName: obj.GetName(), + ObjPath: obj.GetPath(), + ParentName: stdpath.Base(parentDir), + ParentPath: parentDir, + }, "remove") + if err != nil { + return err + } + parent, err := d.get(parentDir) + if err != nil { + return err + } + if parent.Entries == nil { + return errs.ObjectNotFound + } + sha := "" + isDir := false + for _, entry := range parent.Entries { + if entry.Name == obj.GetName() { + sha = entry.Sha + isDir = entry.Type == "dir" + break + } + } + if isDir { + return d.rmdir(obj.GetPath(), commitMessage) + } + if sha == "" { + return errs.ObjectNotFound + } + // if deleted file is the only child of its parent, create .gitkeep to retain empty folder + if parentDir != "/" && len(parent.Entries) == 1 { + if err = d.createGitKeep(parentDir, commitMessage); err != nil { + return err + } + } + return d.delete(obj.GetPath(), sha, commitMessage) +} + +func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + commitMessage, err := getMessage(d.putMsgTmpl, &MessageTemplateVars{ + UserName: ctx.Value("user").(*model.User).Username, + ObjName: stream.GetName(), + ObjPath: stdpath.Join(dstDir.GetPath(), stream.GetName()), + ParentName: dstDir.GetName(), + ParentPath: dstDir.GetPath(), + }, "upload") + if err != nil { + return nil, err + } + parent, err := d.get(dstDir.GetPath()) + if err != nil { + return nil, err + } + if parent.Entries == nil { + return nil, errs.NotFolder + } + uploadSha := "" + for _, entry := range parent.Entries { + if entry.Name == stream.GetName() { + uploadSha = entry.Sha + break + } + } + // if parent folder contains .gitkeep only, mark it and delete .gitkeep later + gitKeepSha := "" + if len(parent.Entries) == 1 && parent.Entries[0].Name == ".gitkeep" { + gitKeepSha = parent.Entries[0].Sha + } + + path := stdpath.Join(dstDir.GetPath(), stream.GetName()) + resp, err := d.put(path, uploadSha, commitMessage, ctx, stream, up) + if err != nil { + return nil, err + } + if gitKeepSha != "" { + err = d.delete(stdpath.Join(dstDir.GetPath(), ".gitkeep"), gitKeepSha, commitMessage) + } + return resp.Content.toModelObj(), err +} + +var _ driver.Driver = (*Github)(nil) + +func (d *Github) getApiUrl(path string) string { + path = utils.FixAndCleanPath(path) + return fmt.Sprintf("https://api.github.com/repos/%s/%s/contents%s", d.Owner, d.Repo, path) +} + +func (d *Github) get(path string) (*Object, error) { + req := d.client.R() + if d.Ref != "" { + req = req.SetQueryParam("ref", d.Ref) + } + res, err := req.Get(d.getApiUrl(path)) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toErr(res) + } + var resp Object + err = utils.Json.Unmarshal(res.Body(), &resp) + return &resp, err +} + +func (d *Github) createGitKeep(path, message string) error { + body := map[string]interface{}{ + "message": message, + "content": "", + } + if d.Ref != "" { + body["branch"] = d.Ref + } + d.addCommitterAndAuthor(&body) + + res, err := d.client.R().SetBody(body).Put(d.getApiUrl(stdpath.Join(path, ".gitkeep"))) + if err != nil { + return err + } + if res.StatusCode() != 200 && res.StatusCode() != 201 { + return toErr(res) + } + return nil +} + +func (d *Github) put(path, sha, message string, ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) (*PutResp, error) { + beforeContent := strings.Builder{} + beforeContent.WriteString("{\"message\":\"") + beforeContent.WriteString(message) + beforeContent.WriteString("\",") + if sha != "" { + beforeContent.WriteString("\"sha\":\"") + beforeContent.WriteString(sha) + beforeContent.WriteString("\",") + } + if d.Ref != "" { + beforeContent.WriteString("\"branch\":\"") + beforeContent.WriteString(d.Ref) + beforeContent.WriteString("\",") + } + if d.CommitterName != "" { + beforeContent.WriteString("\"committer\":{\"name\":\"") + beforeContent.WriteString(d.CommitterName) + beforeContent.WriteString("\",\"email\":\"") + beforeContent.WriteString(d.CommitterEmail) + beforeContent.WriteString("\"},") + } + if d.AuthorName != "" { + beforeContent.WriteString("\"author\":{\"name\":\"") + beforeContent.WriteString(d.AuthorName) + beforeContent.WriteString("\",\"email\":\"") + beforeContent.WriteString(d.AuthorEmail) + beforeContent.WriteString("\"},") + } + beforeContent.WriteString("\"content\":\"") + + length := int64(beforeContent.Len()) + calculateBase64Length(stream.GetSize()) + 2 + beforeContentReader := strings.NewReader(beforeContent.String()) + contentReader, contentWriter := io.Pipe() + go func() { + encoder := base64.NewEncoder(base64.StdEncoding, contentWriter) + if _, err := io.Copy(encoder, stream); err != nil { + _ = contentWriter.CloseWithError(err) + return + } + _ = encoder.Close() + _ = contentWriter.Close() + }() + afterContentReader := strings.NewReader("\"}") + res, err := d.client.R(). + SetHeader("Content-Length", strconv.FormatInt(length, 10)). + SetBody(&ReaderWithCtx{ + Reader: io.MultiReader(beforeContentReader, contentReader, afterContentReader), + Ctx: ctx, + Length: length, + Progress: up, + }). + Put(d.getApiUrl(path)) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 && res.StatusCode() != 201 { + return nil, toErr(res) + } + var resp PutResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Github) delete(path, sha, message string) error { + body := map[string]interface{}{ + "message": message, + "sha": sha, + } + if d.Ref != "" { + body["branch"] = d.Ref + } + d.addCommitterAndAuthor(&body) + res, err := d.client.R().SetBody(body).Delete(d.getApiUrl(path)) + if err != nil { + return err + } + if res.StatusCode() != 200 { + return toErr(res) + } + return nil +} + +func (d *Github) rmdir(path, message string) error { + for { // the number of sub-items returned pre call is limited to a maximum of 1000 + obj, err := d.get(path) + if err != nil { // until 404 + return nil + } + if obj.Type != "dir" || obj.Entries == nil { + return errs.NotFolder + } + if len(obj.Entries) == 0 { // maybe never access + return nil + } + if err = d.clearSub(obj, path, message); err != nil { + return err + } + } +} + +func (d *Github) clearSub(obj *Object, path, message string) error { + for _, entry := range obj.Entries { + var err error + if entry.Type == "dir" { + err = d.rmdir(stdpath.Join(path, entry.Name), message) + } else { + err = d.delete(stdpath.Join(path, entry.Name), entry.Sha, message) + } + if err != nil { + return err + } + } + return nil +} + +func (d *Github) addCommitterAndAuthor(m *map[string]interface{}) { + if d.CommitterName != "" { + committer := map[string]string{ + "name": d.CommitterName, + "email": d.CommitterEmail, + } + (*m)["committer"] = committer + } + if d.AuthorName != "" { + author := map[string]string{ + "name": d.AuthorName, + "email": d.AuthorEmail, + } + (*m)["author"] = author + } +} diff --git a/drivers/github/meta.go b/drivers/github/meta.go new file mode 100644 index 00000000000..78b30989a37 --- /dev/null +++ b/drivers/github/meta.go @@ -0,0 +1,33 @@ +package github + +import ( + "github.com/alist-org/alist/v3/internal/driver" + "github.com/alist-org/alist/v3/internal/op" +) + +type Addition struct { + driver.RootPath + Token string `json:"token" type:"string" required:"true"` + Owner string `json:"owner" type:"string" required:"true"` + Repo string `json:"repo" type:"string" required:"true"` + Ref string `json:"ref" type:"string"` + CommitterName string `json:"committer_name" type:"string"` + CommitterEmail string `json:"committer_email" type:"string"` + AuthorName string `json:"author_name" type:"string"` + AuthorEmail string `json:"author_email" type:"string"` + MkdirCommitMsg string `json:"mkdir_commit_message" type:"text" default:"{{.UserName}} mkdir {{.ObjPath}}"` + DeleteCommitMsg string `json:"delete_commit_message" type:"text" default:"{{.UserName}} remove {{.ObjPath}}"` + PutCommitMsg string `json:"put_commit_message" type:"text" default:"{{.UserName}} upload {{.ObjPath}}"` +} + +var config = driver.Config{ + Name: "GitHub API", + LocalSort: true, + DefaultRoot: "/", +} + +func init() { + op.RegisterDriver(func() driver.Driver { + return &Github{} + }) +} diff --git a/drivers/github/types.go b/drivers/github/types.go new file mode 100644 index 00000000000..206a04589f6 --- /dev/null +++ b/drivers/github/types.go @@ -0,0 +1,50 @@ +package github + +import ( + "github.com/alist-org/alist/v3/internal/model" + "time" +) + +type Links struct { + Git string `json:"git"` + Html string `json:"html"` + Self string `json:"self"` +} + +type Object struct { + Type string `json:"type"` + Encoding string `json:"encoding" required:"false"` + Size int64 `json:"size"` + Name string `json:"name"` + Path string `json:"path"` + Content string `json:"Content" required:"false"` + Sha string `json:"sha"` + URL string `json:"url"` + GitURL string `json:"git_url"` + HtmlURL string `json:"html_url"` + DownloadURL string `json:"download_url"` + Entries []Object `json:"entries" required:"false"` + Links Links `json:"_links"` + SubmoduleGitURL string `json:"submodule_git_url" required:"false"` + Target string `json:"target" required:"false"` +} + +func (o *Object) toModelObj() *model.Object { + return &model.Object{ + Name: o.Name, + Size: o.Size, + Modified: time.Unix(0, 0), + IsFolder: o.Type == "dir", + } +} + +type PutResp struct { + Content Object `json:"Content"` + Commit interface{} `json:"commit"` +} + +type ErrResp struct { + Message string `json:"message"` + DocumentationURL string `json:"documentation_url"` + Status string `json:"status"` +} diff --git a/drivers/github/util.go b/drivers/github/util.go new file mode 100644 index 00000000000..aad2e8f806b --- /dev/null +++ b/drivers/github/util.go @@ -0,0 +1,57 @@ +package github + +import ( + "context" + "errors" + "fmt" + "github.com/alist-org/alist/v3/pkg/utils" + "github.com/go-resty/resty/v2" + "io" + "math" + "strings" + "text/template" +) + +type ReaderWithCtx struct { + Reader io.Reader + Ctx context.Context + Length int64 + Progress func(percentage float64) + offset int64 +} + +func (r *ReaderWithCtx) Read(p []byte) (int, error) { + n, err := r.Reader.Read(p) + r.offset += int64(n) + r.Progress(math.Min(100.0, float64(r.offset)/float64(r.Length))) + return n, err +} + +type MessageTemplateVars struct { + UserName string + ObjName string + ObjPath string + ParentName string + ParentPath string +} + +func getMessage(tmpl *template.Template, vars *MessageTemplateVars, defaultOpStr string) (string, error) { + sb := strings.Builder{} + if err := tmpl.Execute(&sb, vars); err != nil { + return fmt.Sprintf("%s %s %s", vars.UserName, defaultOpStr, vars.ObjPath), err + } + return sb.String(), nil +} + +func calculateBase64Length(inputLength int64) int64 { + return 4 * ((inputLength + 2) / 3) +} + +func toErr(res *resty.Response) error { + var errMsg ErrResp + if err := utils.Json.Unmarshal(res.Body(), &errMsg); err != nil { + return errors.New(res.Status()) + } else { + return fmt.Errorf("%s: %s", res.Status(), errMsg.Message) + } +} From bccf5925520f7e1f85af28d5768f50e8c9dbde90 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Fri, 27 Dec 2024 17:22:12 +0800 Subject: [PATCH 2/7] fix: filter submodule operation --- drivers/github/driver.go | 86 +++++++++++++++++++++------------------- 1 file changed, 46 insertions(+), 40 deletions(-) diff --git a/drivers/github/driver.go b/drivers/github/driver.go index 39a5fff19ae..26620cc6d39 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -95,22 +95,15 @@ func (d *Github) Link(ctx context.Context, file model.Obj, args model.LinkArgs) if err != nil { return nil, err } + if obj.Type == "submodule" { + return nil, errors.New("cannot download a submodule") + } return &model.Link{ URL: obj.DownloadURL, }, nil } func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { - commitMessage, err := getMessage(d.mkdirMsgTmpl, &MessageTemplateVars{ - UserName: ctx.Value("user").(*model.User).Username, - ObjName: dirName, - ObjPath: stdpath.Join(parentDir.GetPath(), dirName), - ParentName: parentDir.GetName(), - ParentPath: parentDir.GetPath(), - }, "mkdir") - if err != nil { - return err - } parent, err := d.get(parentDir.GetPath()) if err != nil { return err @@ -124,6 +117,16 @@ func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin gitKeepSha = parent.Entries[0].Sha } + commitMessage, err := getMessage(d.mkdirMsgTmpl, &MessageTemplateVars{ + UserName: ctx.Value("user").(*model.User).Username, + ObjName: dirName, + ObjPath: stdpath.Join(parentDir.GetPath(), dirName), + ParentName: parentDir.GetName(), + ParentPath: parentDir.GetPath(), + }, "mkdir") + if err != nil { + return err + } if err = d.createGitKeep(stdpath.Join(parentDir.GetPath(), dirName), commitMessage); err != nil { return err } @@ -147,16 +150,6 @@ func (d *Github) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, func (d *Github) Remove(ctx context.Context, obj model.Obj) error { parentDir := stdpath.Dir(obj.GetPath()) - commitMessage, err := getMessage(d.deleteMsgTmpl, &MessageTemplateVars{ - UserName: ctx.Value("user").(*model.User).Username, - ObjName: obj.GetName(), - ObjPath: obj.GetPath(), - ParentName: stdpath.Base(parentDir), - ParentPath: parentDir, - }, "remove") - if err != nil { - return err - } parent, err := d.get(parentDir) if err != nil { return err @@ -165,20 +158,33 @@ func (d *Github) Remove(ctx context.Context, obj model.Obj) error { return errs.ObjectNotFound } sha := "" - isDir := false + objType := "" for _, entry := range parent.Entries { if entry.Name == obj.GetName() { sha = entry.Sha - isDir = entry.Type == "dir" + objType = entry.Type break } } - if isDir { - return d.rmdir(obj.GetPath(), commitMessage) - } if sha == "" { return errs.ObjectNotFound } + if objType == "submodule" { + return errors.New("cannot delete a submodule") + } + commitMessage, err := getMessage(d.deleteMsgTmpl, &MessageTemplateVars{ + UserName: ctx.Value("user").(*model.User).Username, + ObjName: obj.GetName(), + ObjPath: obj.GetPath(), + ParentName: stdpath.Base(parentDir), + ParentPath: parentDir, + }, "remove") + if err != nil { + return err + } + if objType == "dir" { + return d.rmdir(obj.GetPath(), commitMessage) + } // if deleted file is the only child of its parent, create .gitkeep to retain empty folder if parentDir != "/" && len(parent.Entries) == 1 { if err = d.createGitKeep(parentDir, commitMessage); err != nil { @@ -189,16 +195,6 @@ func (d *Github) Remove(ctx context.Context, obj model.Obj) error { } func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { - commitMessage, err := getMessage(d.putMsgTmpl, &MessageTemplateVars{ - UserName: ctx.Value("user").(*model.User).Username, - ObjName: stream.GetName(), - ObjPath: stdpath.Join(dstDir.GetPath(), stream.GetName()), - ParentName: dstDir.GetName(), - ParentPath: dstDir.GetPath(), - }, "upload") - if err != nil { - return nil, err - } parent, err := d.get(dstDir.GetPath()) if err != nil { return nil, err @@ -219,6 +215,16 @@ func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr gitKeepSha = parent.Entries[0].Sha } + commitMessage, err := getMessage(d.putMsgTmpl, &MessageTemplateVars{ + UserName: ctx.Value("user").(*model.User).Username, + ObjName: stream.GetName(), + ObjPath: stdpath.Join(dstDir.GetPath(), stream.GetName()), + ParentName: dstDir.GetName(), + ParentPath: dstDir.GetPath(), + }, "upload") + if err != nil { + return nil, err + } path := stdpath.Join(dstDir.GetPath(), stream.GetName()) resp, err := d.put(path, uploadSha, commitMessage, ctx, stream, up) if err != nil { @@ -232,7 +238,7 @@ func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr var _ driver.Driver = (*Github)(nil) -func (d *Github) getApiUrl(path string) string { +func (d *Github) getContentApiUrl(path string) string { path = utils.FixAndCleanPath(path) return fmt.Sprintf("https://api.github.com/repos/%s/%s/contents%s", d.Owner, d.Repo, path) } @@ -242,7 +248,7 @@ func (d *Github) get(path string) (*Object, error) { if d.Ref != "" { req = req.SetQueryParam("ref", d.Ref) } - res, err := req.Get(d.getApiUrl(path)) + res, err := req.Get(d.getContentApiUrl(path)) if err != nil { return nil, err } @@ -264,7 +270,7 @@ func (d *Github) createGitKeep(path, message string) error { } d.addCommitterAndAuthor(&body) - res, err := d.client.R().SetBody(body).Put(d.getApiUrl(stdpath.Join(path, ".gitkeep"))) + res, err := d.client.R().SetBody(body).Put(d.getContentApiUrl(stdpath.Join(path, ".gitkeep"))) if err != nil { return err } @@ -326,7 +332,7 @@ func (d *Github) put(path, sha, message string, ctx context.Context, stream mode Length: length, Progress: up, }). - Put(d.getApiUrl(path)) + Put(d.getContentApiUrl(path)) if err != nil { return nil, err } @@ -349,7 +355,7 @@ func (d *Github) delete(path, sha, message string) error { body["branch"] = d.Ref } d.addCommitterAndAuthor(&body) - res, err := d.client.R().SetBody(body).Delete(d.getApiUrl(path)) + res, err := d.client.R().SetBody(body).Delete(d.getContentApiUrl(path)) if err != nil { return err } From 6dbc38664ff257941d811a7c358cf0958c137165 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sat, 28 Dec 2024 04:07:36 +0800 Subject: [PATCH 3/7] feat: rename, copy and move, but with bugs --- drivers/github/driver.go | 744 +++++++++++++++++++++++++++++++++++---- drivers/github/meta.go | 3 + drivers/github/types.go | 49 +++ drivers/github/util.go | 50 +++ 4 files changed, 768 insertions(+), 78 deletions(-) diff --git a/drivers/github/driver.go b/drivers/github/driver.go index 26620cc6d39..068553312c6 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -15,6 +15,7 @@ import ( stdpath "path" "strconv" "strings" + "sync" "text/template" ) @@ -25,6 +26,11 @@ type Github struct { mkdirMsgTmpl *template.Template deleteMsgTmpl *template.Template putMsgTmpl *template.Template + renameMsgTmpl *template.Template + copyMsgTmpl *template.Template + moveMsgTmpl *template.Template + isOnBranch bool + commitMutex sync.Mutex } func (d *Github) Config() driver.Config { @@ -62,10 +68,29 @@ func (d *Github) Init(ctx context.Context) error { if err != nil { return err } + d.renameMsgTmpl, err = template.New("renameCommitMsgTemplate").Parse(d.RenameCommitMsg) + if err != nil { + return err + } + d.copyMsgTmpl, err = template.New("copyCommitMsgTemplate").Parse(d.CopyCommitMsg) + if err != nil { + return err + } + d.moveMsgTmpl, err = template.New("moveCommitMsgTemplate").Parse(d.MoveCommitMsg) + if err != nil { + return err + } d.client = base.NewRestyClient(). SetHeader("Accept", "application/vnd.github.object+json"). SetHeader("Authorization", "Bearer "+d.Token). SetHeader("X-GitHub-Api-Version", "2022-11-28") + if d.Ref == "" { + // TODO Get default branch + d.isOnBranch = true + } else { + _, err = d.getBranchHead() + d.isOnBranch = err == nil + } return nil } @@ -81,13 +106,30 @@ func (d *Github) List(ctx context.Context, dir model.Obj, args model.ListArgs) ( if obj.Entries == nil { return nil, errs.NotFolder } - ret := make([]model.Obj, 0, len(obj.Entries)) - for _, entry := range obj.Entries { - if entry.Name != ".gitkeep" { - ret = append(ret, entry.toModelObj()) + if len(obj.Entries) >= 1000 { + tree, err := d.getTree(obj.Sha) + if err != nil { + return nil, err + } + if tree.Truncated { + return nil, fmt.Errorf("tree %s is truncated", dir.GetPath()) + } + ret := make([]model.Obj, 0, len(tree.Trees)) + for _, t := range tree.Trees { + if t.Path != ".gitkeep" { + ret = append(ret, t.toModelObj()) + } + } + return ret, nil + } else { + ret := make([]model.Obj, 0, len(obj.Entries)) + for _, entry := range obj.Entries { + if entry.Name != ".gitkeep" { + ret = append(ret, entry.toModelObj()) + } } + return ret, nil } - return ret, nil } func (d *Github) Link(ctx context.Context, file model.Obj, args model.LinkArgs) (*model.Link, error) { @@ -104,6 +146,9 @@ func (d *Github) Link(ctx context.Context, file model.Obj, args model.LinkArgs) } func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName string) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } parent, err := d.get(parentDir.GetPath()) if err != nil { return err @@ -136,19 +181,326 @@ func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin return err } -func (d *Github) Move(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { - return nil, errs.NotSupport +func (d *Github) Move(ctx context.Context, srcObj, dstDir model.Obj) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return errors.New("cannot move parent dir to child") + } + + var rootSha string + if strings.HasPrefix(dstDir.GetPath(), stdpath.Dir(srcObj.GetPath())) { // /aa/1 -> /aa/bb/ + dstOldSha, dstNewSha, ancestorOldSha, srcParentTree, err := d.copyWithoutRenewTree(srcObj, dstDir) + if err != nil { + return err + } + + srcParentPath := stdpath.Dir(srcObj.GetPath()) + dstRest := dstDir.GetPath()[len(srcParentPath):] + if dstRest[0] == '/' { + dstRest = dstRest[1:] + } + dstNextName, _, _ := strings.Cut(dstRest, "/") + dstNextPath := stdpath.Join(srcParentPath, dstNextName) + dstNextTreeSha, err := d.renewParentTrees(dstDir.GetPath(), dstOldSha, dstNewSha, dstNextPath) + if err != nil { + return err + } + var delSrc, dstNextTree *TreeObjReq = nil, nil + for _, t := range srcParentTree.Trees { + if t.Path == dstNextName { + dstNextTree = &t.TreeObjReq + dstNextTree.Sha = dstNextTreeSha + } + if t.Path == srcObj.GetName() { + delSrc = &t.TreeObjReq + delSrc.Sha = nil + } + if delSrc != nil && dstNextTree != nil { + break + } + } + if delSrc == nil || dstNextTree == nil { + return errs.ObjectNotFound + } + ancestorNewSha, err := d.newTree(ancestorOldSha, []interface{}{*delSrc, *dstNextTree}) + if err != nil { + return err + } + rootSha, err = d.renewParentTrees(srcParentPath, ancestorOldSha, ancestorNewSha, "/") + if err != nil { + return err + } + } else if strings.HasPrefix(srcObj.GetPath(), dstDir.GetPath()) { // /aa/bb/1 -> /aa/ + srcParentPath := stdpath.Dir(srcObj.GetPath()) + srcParentTree, srcParentOldSha, err := d.getTreeDirectly(srcParentPath) + if err != nil { + return err + } + var src *TreeObjReq = nil + for _, t := range srcParentTree.Trees { + if t.Path == srcObj.GetName() { + if t.Type == "commit" { + return errors.New("cannot move a submodule") + } + src = &t.TreeObjReq + break + } + } + if src == nil { + return errs.ObjectNotFound + } + + delSrc := *src + delSrc.Sha = nil + delSrcTree := make([]interface{}, 0, 2) + delSrcTree = append(delSrcTree, delSrc) + if len(srcParentTree.Trees) == 1 { + delSrcTree = append(delSrcTree, map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }) + } + srcParentNewSha, err := d.newTree(srcParentOldSha, delSrcTree) + if err != nil { + return err + } + srcRest := srcObj.GetPath()[len(dstDir.GetPath()):] + if srcRest[0] == '/' { + srcRest = srcRest[1:] + } + srcNextName, _, ok := strings.Cut(srcRest, "/") + if !ok { // /aa/1 -> /aa/ + return errors.New("cannot move in place") + } + srcNextPath := stdpath.Join(dstDir.GetPath(), srcNextName) + srcNextTreeSha, err := d.renewParentTrees(srcParentPath, srcParentOldSha, srcParentNewSha, srcNextPath) + if err != nil { + return err + } + + ancestorTree, ancestorOldSha, err := d.getTreeDirectly(dstDir.GetPath()) + if err != nil { + return err + } + var srcNextTree *TreeObjReq = nil + for _, t := range ancestorTree.Trees { + if t.Path == srcNextName { + srcNextTree = &t.TreeObjReq + srcNextTree.Sha = srcNextTreeSha + break + } + } + if srcNextTree == nil { + return errs.ObjectNotFound + } + ancestorNewSha, err := d.newTree(ancestorOldSha, []interface{}{*srcNextTree, *src}) + if err != nil { + return err + } + rootSha, err = d.renewParentTrees(dstDir.GetPath(), ancestorOldSha, ancestorNewSha, "/") + if err != nil { + return err + } + } else { // /aa/1 -> /bb/ + // do copy + dstOldSha, dstNewSha, srcParentOldSha, srcParentTree, err := d.copyWithoutRenewTree(srcObj, dstDir) + if err != nil { + return err + } + + // delete src object and create new tree + var srcNewTree *TreeObjReq = nil + for _, t := range srcParentTree.Trees { + if t.Path == srcObj.GetName() { + srcNewTree = &t.TreeObjReq + srcNewTree.Sha = nil + break + } + } + if srcNewTree == nil { + return errs.ObjectNotFound + } + delSrcTree := make([]interface{}, 0, 2) + delSrcTree = append(delSrcTree, *srcNewTree) + if len(srcParentTree.Trees) == 1 { + delSrcTree = append(delSrcTree, map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }) + } + srcParentNewSha, err := d.newTree(srcParentOldSha, delSrcTree) + if err != nil { + return err + } + + // renew but the common ancestor of srcPath and dstPath + ancestor, srcChildName, dstChildName, _, _ := getPathCommonAncestor(srcObj.GetPath(), dstDir.GetPath()) + dstNextTreeSha, err := d.renewParentTrees(dstDir.GetPath(), dstOldSha, dstNewSha, stdpath.Join(ancestor, dstChildName)) + if err != nil { + return err + } + srcNextTreeSha, err := d.renewParentTrees(stdpath.Dir(srcObj.GetPath()), srcParentOldSha, srcParentNewSha, stdpath.Join(ancestor, srcChildName)) + if err != nil { + return err + } + + // renew the tree of the last common ancestor + ancestorTree, ancestorOldSha, err := d.getTreeDirectly(ancestor) + if err != nil { + return err + } + newTree := make([]interface{}, 2) + srcBind := false + dstBind := false + for _, t := range ancestorTree.Trees { + if t.Path == srcChildName { + t.Sha = srcNextTreeSha + newTree[0] = t.TreeObjReq + srcBind = true + } + if t.Path == dstChildName { + t.Sha = dstNextTreeSha + newTree[1] = t.TreeObjReq + dstBind = true + } + if srcBind && dstBind { + break + } + } + if !srcBind || !dstBind { + return errs.ObjectNotFound + } + ancestorNewSha, err := d.newTree(ancestorOldSha, newTree) + if err != nil { + return err + } + // renew until root + rootSha, err = d.renewParentTrees(ancestor, ancestorOldSha, ancestorNewSha, "/") + if err != nil { + return err + } + } + + // commit + message, err := getMessage(d.moveMsgTmpl, &MessageTemplateVars{ + UserName: ctx.Value("user").(*model.User).Username, + ObjName: srcObj.GetName(), + ObjPath: srcObj.GetPath(), + ParentName: stdpath.Base(stdpath.Dir(srcObj.GetPath())), + ParentPath: stdpath.Dir(srcObj.GetPath()), + TargetName: stdpath.Base(dstDir.GetPath()), + TargetPath: dstDir.GetPath(), + }, "move") + if err != nil { + return err + } + commit, err := d.commit(message, rootSha) + if err != nil { + return err + } + return d.updateHead(commit) } -func (d *Github) Rename(ctx context.Context, srcObj model.Obj, newName string) (model.Obj, error) { - return nil, errs.NotSupport +func (d *Github) Rename(ctx context.Context, srcObj model.Obj, newName string) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + parentDir := stdpath.Dir(srcObj.GetPath()) + tree, _, err := d.getTreeDirectly(parentDir) + if err != nil { + return err + } + newTree := make([]interface{}, 2) + operated := false + for _, t := range tree.Trees { + if t.Path == srcObj.GetName() { + if t.Type == "commit" { + return errors.New("cannot rename a submodule") + } + delCopy := t.TreeObjReq + delCopy.Sha = nil + newTree[0] = delCopy + t.Path = newName + newTree[1] = t.TreeObjReq + operated = true + break + } + } + if !operated { + return errs.ObjectNotFound + } + newSha, err := d.newTree(tree.Sha, newTree) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(parentDir, tree.Sha, newSha, "/") + if err != nil { + return err + } + message, err := getMessage(d.renameMsgTmpl, &MessageTemplateVars{ + UserName: ctx.Value("user").(*model.User).Username, + ObjName: srcObj.GetName(), + ObjPath: srcObj.GetPath(), + ParentName: stdpath.Base(parentDir), + ParentPath: parentDir, + TargetName: newName, + TargetPath: parentDir, + }, "rename") + if err != nil { + return err + } + commit, err := d.commit(message, rootSha) + if err != nil { + return err + } + return d.updateHead(commit) } -func (d *Github) Copy(ctx context.Context, srcObj, dstDir model.Obj) (model.Obj, error) { - return nil, errs.NotSupport +func (d *Github) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } + if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { + return errors.New("cannot copy parent dir to child") + } + + dstSha, newSha, _, _, err := d.copyWithoutRenewTree(srcObj, dstDir) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(dstDir.GetPath(), dstSha, newSha, "/") + if err != nil { + return err + } + message, err := getMessage(d.copyMsgTmpl, &MessageTemplateVars{ + UserName: ctx.Value("user").(*model.User).Username, + ObjName: srcObj.GetName(), + ObjPath: srcObj.GetPath(), + ParentName: stdpath.Base(stdpath.Dir(srcObj.GetPath())), + ParentPath: stdpath.Dir(srcObj.GetPath()), + TargetName: stdpath.Base(dstDir.GetPath()), + TargetPath: dstDir.GetPath(), + }, "copy") + if err != nil { + return err + } + commit, err := d.commit(message, rootSha) + if err != nil { + return err + } + return d.updateHead(commit) } func (d *Github) Remove(ctx context.Context, obj model.Obj) error { + if !d.isOnBranch { + return errors.New("cannot write to non-branch reference") + } parentDir := stdpath.Dir(obj.GetPath()) parent, err := d.get(parentDir) if err != nil { @@ -157,21 +509,6 @@ func (d *Github) Remove(ctx context.Context, obj model.Obj) error { if parent.Entries == nil { return errs.ObjectNotFound } - sha := "" - objType := "" - for _, entry := range parent.Entries { - if entry.Name == obj.GetName() { - sha = entry.Sha - objType = entry.Type - break - } - } - if sha == "" { - return errs.ObjectNotFound - } - if objType == "submodule" { - return errors.New("cannot delete a submodule") - } commitMessage, err := getMessage(d.deleteMsgTmpl, &MessageTemplateVars{ UserName: ctx.Value("user").(*model.User).Username, ObjName: obj.GetName(), @@ -179,22 +516,77 @@ func (d *Github) Remove(ctx context.Context, obj model.Obj) error { ParentName: stdpath.Base(parentDir), ParentPath: parentDir, }, "remove") - if err != nil { - return err - } - if objType == "dir" { - return d.rmdir(obj.GetPath(), commitMessage) - } - // if deleted file is the only child of its parent, create .gitkeep to retain empty folder - if parentDir != "/" && len(parent.Entries) == 1 { - if err = d.createGitKeep(parentDir, commitMessage); err != nil { + if len(parent.Entries) >= 1000 { + tree, err := d.getTree(parent.Sha) + if err != nil { + return err + } + if tree.Truncated { + return fmt.Errorf("tree %s is truncated", parentDir) + } + var del *TreeObjReq = nil + for _, t := range tree.Trees { + if t.Path == obj.GetName() { + if t.Type == "commit" { + return errors.New("cannot remove a submodule") + } + del = &t.TreeObjReq + del.Sha = nil + break + } + } + if del == nil { + return errs.ObjectNotFound + } + newSha, err := d.newTree(parent.Sha, []interface{}{*del}) + if err != nil { return err } + rootSha, err := d.renewParentTrees(parentDir, parent.Sha, newSha, "/") + if err != nil { + return err + } + commit, err := d.commit(commitMessage, rootSha) + if err != nil { + return err + } + return d.updateHead(commit) + } else { + sha := "" + objType := "" + for _, entry := range parent.Entries { + if entry.Name == obj.GetName() { + sha = entry.Sha + objType = entry.Type + break + } + } + if sha == "" { + return errs.ObjectNotFound + } + if objType == "submodule" { + return errors.New("cannot delete a submodule") + } + if err != nil { + return err + } + if objType == "dir" { + return d.rmdir(parentDir, parent.Sha, sha, commitMessage) + } + // if deleted file is the only child of its parent, create .gitkeep to retain empty folder + if parentDir != "/" && len(parent.Entries) == 1 { + if err = d.createGitKeep(parentDir, commitMessage); err != nil { + return err + } + } + return d.delete(obj.GetPath(), sha, commitMessage) } - return d.delete(obj.GetPath(), sha, commitMessage) } func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { + if !d.isOnBranch { + return nil, errors.New("cannot write to non-branch reference") + } parent, err := d.get(dstDir.GetPath()) if err != nil { return nil, err @@ -203,10 +595,32 @@ func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr return nil, errs.NotFolder } uploadSha := "" - for _, entry := range parent.Entries { - if entry.Name == stream.GetName() { - uploadSha = entry.Sha - break + if len(parent.Entries) >= 1000 { + tree, err := d.getTree(parent.Sha) + if err != nil { + return nil, err + } + if tree.Truncated { + return nil, fmt.Errorf("tree %s is truncated", dstDir.GetPath()) + } + for _, t := range tree.Trees { + if t.Path == stream.GetName() { + if t.Type == "commit" { + return nil, errors.New("cannot replace a submodule") + } + uploadSha = t.Sha.(string) + break + } + } + } else { + for _, entry := range parent.Entries { + if entry.Name == stream.GetName() { + if entry.Type == "submodule" { + return nil, errors.New("cannot replace a submodule") + } + uploadSha = entry.Sha + break + } } } // if parent folder contains .gitkeep only, mark it and delete .gitkeep later @@ -244,11 +658,7 @@ func (d *Github) getContentApiUrl(path string) string { } func (d *Github) get(path string) (*Object, error) { - req := d.client.R() - if d.Ref != "" { - req = req.SetQueryParam("ref", d.Ref) - } - res, err := req.Get(d.getContentApiUrl(path)) + res, err := d.client.R().SetQueryParam("ref", d.Ref).Get(d.getContentApiUrl(path)) if err != nil { return nil, err } @@ -264,9 +674,7 @@ func (d *Github) createGitKeep(path, message string) error { body := map[string]interface{}{ "message": message, "content": "", - } - if d.Ref != "" { - body["branch"] = d.Ref + "branch": d.Ref, } d.addCommitterAndAuthor(&body) @@ -284,17 +692,14 @@ func (d *Github) put(path, sha, message string, ctx context.Context, stream mode beforeContent := strings.Builder{} beforeContent.WriteString("{\"message\":\"") beforeContent.WriteString(message) + beforeContent.WriteString("\",\"branch\":\"") + beforeContent.WriteString(d.Ref) beforeContent.WriteString("\",") if sha != "" { beforeContent.WriteString("\"sha\":\"") beforeContent.WriteString(sha) beforeContent.WriteString("\",") } - if d.Ref != "" { - beforeContent.WriteString("\"branch\":\"") - beforeContent.WriteString(d.Ref) - beforeContent.WriteString("\",") - } if d.CommitterName != "" { beforeContent.WriteString("\"committer\":{\"name\":\"") beforeContent.WriteString(d.CommitterName) @@ -350,9 +755,7 @@ func (d *Github) delete(path, sha, message string) error { body := map[string]interface{}{ "message": message, "sha": sha, - } - if d.Ref != "" { - body["branch"] = d.Ref + "branch": d.Ref, } d.addCommitterAndAuthor(&body) res, err := d.client.R().SetBody(body).Delete(d.getContentApiUrl(path)) @@ -365,39 +768,224 @@ func (d *Github) delete(path, sha, message string) error { return nil } -func (d *Github) rmdir(path, message string) error { - for { // the number of sub-items returned pre call is limited to a maximum of 1000 - obj, err := d.get(path) - if err != nil { // until 404 - return nil - } - if obj.Type != "dir" || obj.Entries == nil { - return errs.NotFolder - } - if len(obj.Entries) == 0 { // maybe never access - return nil - } - if err = d.clearSub(obj, path, message); err != nil { - return err +func (d *Github) rmdir(parentPath, parentSha, sha, message string) error { + tree, err := d.getTree(parentSha) + if err != nil { + return err + } + if tree.Truncated { + return fmt.Errorf("tree %s is truncated", parentPath) + } + var newTree *TreeObjReq = nil + for _, t := range tree.Trees { + if t.Sha != sha { + newTree = &t.TreeObjReq + newTree.Sha = nil + break } } + if newTree == nil { + return errs.ObjectNotFound + } + newTreeSlice := make([]interface{}, 0, 2) + newTreeSlice = append(newTreeSlice, *newTree) + if len(tree.Trees) == 1 { + newTreeSlice = append(newTreeSlice, map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }) + } + sha, err = d.newTree(parentSha, newTreeSlice) + if err != nil { + return err + } + + sha, err = d.renewParentTrees(parentPath, parentSha, sha, "/") + if err != nil { + return err + } + commit, err := d.commit(message, sha) + if err != nil { + return err + } + return d.updateHead(commit) } -func (d *Github) clearSub(obj *Object, path, message string) error { - for _, entry := range obj.Entries { - var err error - if entry.Type == "dir" { - err = d.rmdir(stdpath.Join(path, entry.Name), message) - } else { - err = d.delete(stdpath.Join(path, entry.Name), entry.Sha, message) +func (d *Github) renewParentTrees(path, prevSha, curSha, until string) (string, error) { + for path != until { + path = stdpath.Dir(path) + tree, sha, err := d.getTreeDirectly(path) + if err != nil { + return "", err + } + var newTree *TreeObjReq = nil + for _, t := range tree.Trees { + if t.Sha == prevSha { + newTree = &t.TreeObjReq + newTree.Sha = curSha + break + } } + if newTree == nil { + return "", errs.ObjectNotFound + } + curSha, err = d.newTree(sha, []interface{}{*newTree}) if err != nil { - return err + return "", err } + prevSha = sha + } + return curSha, nil +} + +func (d *Github) getTree(sha string) (*TreeResp, error) { + res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/trees/%s", d.Owner, d.Repo, sha)) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toErr(res) + } + var resp TreeResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (d *Github) getTreeDirectly(path string) (*TreeResp, string, error) { + p, err := d.get(path) + if err != nil { + return nil, "", err + } + if p.Entries == nil { + return nil, "", fmt.Errorf("%s is not a folder", path) + } + tree, err := d.getTree(p.Sha) + if err != nil { + return nil, "", err + } + if tree.Truncated { + return nil, "", fmt.Errorf("tree %s is truncated", path) + } + return tree, p.Sha, nil +} + +func (d *Github) newTree(baseSha string, tree []interface{}) (string, error) { + res, err := d.client.R(). + SetBody(&TreeReq{ + BaseTree: baseSha, + Trees: tree, + }). + Post(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/trees", d.Owner, d.Repo)) + if err != nil { + return "", err + } + if res.StatusCode() != 201 { + return "", toErr(res) + } + var resp TreeResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return "", err + } + return resp.Sha, nil +} + +func (d *Github) commit(message, treeSha string) (string, error) { + d.commitMutex.Lock() + defer d.commitMutex.Unlock() + oldCommit, err := d.getBranchHead() + body := map[string]interface{}{ + "message": message, + "tree": treeSha, + "parents": []string{oldCommit}, + } + d.addCommitterAndAuthor(&body) + res, err := d.client.R().SetBody(body).Post(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/commits", d.Owner, d.Repo)) + if err != nil { + return "", err + } + if res.StatusCode() != 201 { + return "", toErr(res) + } + var resp CommitResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return "", err + } + return resp.Sha, nil +} + +func (d *Github) getBranchHead() (string, error) { + res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s/branches/%s", d.Owner, d.Repo, d.Ref)) + if err != nil { + return "", err + } + if res.StatusCode() != 200 { + return "", toErr(res) + } + var resp BranchResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return "", err + } + return resp.Commit.Sha, nil +} + +func (d *Github) updateHead(sha string) error { + res, err := d.client.R(). + SetBody(&UpdateRefReq{ + Sha: sha, + Force: false, + }). + Patch(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/refs/heads/%s", d.Owner, d.Repo, d.Ref)) + if err != nil { + return err + } + if res.StatusCode() != 200 { + return toErr(res) } return nil } +func (d *Github) copyWithoutRenewTree(srcObj, dstDir model.Obj) (dstSha, newSha, srcParentSha string, srcParentTree *TreeResp, err error) { + dst, err := d.get(dstDir.GetPath()) + if err != nil { + return "", "", "", nil, err + } + dstSha = dst.Sha + srcParentPath := stdpath.Dir(srcObj.GetPath()) + srcParentTree, srcParentSha, err = d.getTreeDirectly(srcParentPath) + if err != nil { + return "", "", "", nil, err + } + var src *TreeObjReq = nil + for _, t := range srcParentTree.Trees { + if t.Path == srcObj.GetName() { + if t.Type == "commit" { + return "", "", "", nil, errors.New("cannot copy a submodule") + } + src = &t.TreeObjReq + break + } + } + if src == nil { + return "", "", "", nil, errs.ObjectNotFound + } + + delGitKeep := TreeObjReq{ + Path: ".gitkeep", + Mode: "100644", + Type: "blob", + Sha: nil, + } + newSha, err = d.newTree(dstSha, []interface{}{*src, delGitKeep}) + if err != nil { + return "", "", "", nil, err + } + return dstSha, newSha, srcParentSha, srcParentTree, nil +} + func (d *Github) addCommitterAndAuthor(m *map[string]interface{}) { if d.CommitterName != "" { committer := map[string]string{ diff --git a/drivers/github/meta.go b/drivers/github/meta.go index 78b30989a37..8e98b6fd2d9 100644 --- a/drivers/github/meta.go +++ b/drivers/github/meta.go @@ -18,6 +18,9 @@ type Addition struct { MkdirCommitMsg string `json:"mkdir_commit_message" type:"text" default:"{{.UserName}} mkdir {{.ObjPath}}"` DeleteCommitMsg string `json:"delete_commit_message" type:"text" default:"{{.UserName}} remove {{.ObjPath}}"` PutCommitMsg string `json:"put_commit_message" type:"text" default:"{{.UserName}} upload {{.ObjPath}}"` + RenameCommitMsg string `json:"rename_commit_message" type:"text" default:"{{.UserName}} rename {{.ObjPath}} to {{.TargetName}}"` + CopyCommitMsg string `json:"copy_commit_message" type:"text" default:"{{.UserName}} copy {{.ObjPath}} to {{.TargetPath}}"` + MoveCommitMsg string `json:"move_commit_message" type:"text" default:"{{.UserName}} move {{.ObjPath}} to {{.TargetPath}}"` } var config = driver.Config{ diff --git a/drivers/github/types.go b/drivers/github/types.go index 206a04589f6..d1516d8e766 100644 --- a/drivers/github/types.go +++ b/drivers/github/types.go @@ -48,3 +48,52 @@ type ErrResp struct { DocumentationURL string `json:"documentation_url"` Status string `json:"status"` } + +type TreeObjReq struct { + Path string `json:"path"` + Mode string `json:"mode"` + Type string `json:"type"` + Sha interface{} `json:"sha"` + Size int64 `json:"size" required:"false"` +} + +func (o *TreeObjReq) toModelObj() *model.Object { + return &model.Object{ + Name: o.Path, + Size: o.Size, + Modified: time.Unix(0, 0), + IsFolder: o.Type == "tree", + } +} + +type TreeObjResp struct { + TreeObjReq + Size int64 `json:"size" required:"false"` + URL string `json:"url"` +} + +type TreeResp struct { + Sha string `json:"sha"` + URL string `json:"url"` + Trees []TreeObjResp `json:"tree"` + Truncated bool `json:"truncated"` +} + +type TreeReq struct { + BaseTree string `json:"base_tree"` + Trees []interface{} `json:"tree"` +} + +type CommitResp struct { + Sha string `json:"sha"` +} + +type BranchResp struct { + Name string `json:"name"` + Commit CommitResp `json:"commit"` +} + +type UpdateRefReq struct { + Sha string `json:"sha"` + Force bool `json:"force"` +} diff --git a/drivers/github/util.go b/drivers/github/util.go index aad2e8f806b..7718c3936f9 100644 --- a/drivers/github/util.go +++ b/drivers/github/util.go @@ -33,6 +33,8 @@ type MessageTemplateVars struct { ObjPath string ParentName string ParentPath string + TargetName string + TargetPath string } func getMessage(tmpl *template.Template, vars *MessageTemplateVars, defaultOpStr string) (string, error) { @@ -55,3 +57,51 @@ func toErr(res *resty.Response) error { return fmt.Errorf("%s: %s", res.Status(), errMsg.Message) } } + +// Example input: +// a = /aaa/bbb/ccc +// b = /aaa/b11/ddd/ccc +// +// Output: +// ancestor = /aaa +// aChildName = bbb +// bChildName = b11 +// aRest = bbb/ccc +// bRest = b11/ddd/ccc +func getPathCommonAncestor(a, b string) (ancestor, aChildName, bChildName, aRest, bRest string) { + a = utils.FixAndCleanPath(a) + b = utils.FixAndCleanPath(b) + idx := 1 + for idx < len(a) && idx < len(b) { + if a[idx] != b[idx] { + break + } + idx++ + } + aNextIdx := idx + for aNextIdx < len(a) { + if a[aNextIdx] == '/' { + break + } + aNextIdx++ + } + bNextIdx := idx + for bNextIdx < len(b) { + if b[bNextIdx] == '/' { + break + } + bNextIdx++ + } + for idx > 0 { + if a[idx] == '/' { + break + } + idx-- + } + ancestor = utils.FixAndCleanPath(a[:idx]) + aChildName = a[idx+1 : aNextIdx] + bChildName = b[idx+1 : bNextIdx] + aRest = a[idx+1:] + bRest = b[idx+1:] + return ancestor, aChildName, bChildName, aRest, bRest +} From bf94f89903bdcbb666d4371aceede150e17f0a2c Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sat, 28 Dec 2024 16:04:34 +0800 Subject: [PATCH 4/7] fix: move and copy returns 422 --- drivers/github/driver.go | 203 +++++++++++++++------------------------ drivers/github/types.go | 19 ++-- drivers/github/util.go | 9 ++ 3 files changed, 98 insertions(+), 133 deletions(-) diff --git a/drivers/github/driver.go b/drivers/github/driver.go index 068553312c6..07f71aaf3ef 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -11,6 +11,7 @@ import ( "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" + log "github.com/sirupsen/logrus" "io" stdpath "path" "strconv" @@ -83,9 +84,15 @@ func (d *Github) Init(ctx context.Context) error { d.client = base.NewRestyClient(). SetHeader("Accept", "application/vnd.github.object+json"). SetHeader("Authorization", "Bearer "+d.Token). - SetHeader("X-GitHub-Api-Version", "2022-11-28") + SetHeader("X-GitHub-Api-Version", "2022-11-28"). + SetLogger(log.StandardLogger()). + SetDebug(false) if d.Ref == "" { - // TODO Get default branch + repo, err := d.getRepo() + if err != nil { + return err + } + d.Ref = repo.DefaultBranch d.isOnBranch = true } else { _, err = d.getBranchHead() @@ -163,7 +170,7 @@ func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin } commitMessage, err := getMessage(d.mkdirMsgTmpl, &MessageTemplateVars{ - UserName: ctx.Value("user").(*model.User).Username, + UserName: getUsername(ctx), ObjName: dirName, ObjPath: stdpath.Join(parentDir.GetPath(), dirName), ParentName: parentDir.GetName(), @@ -389,7 +396,7 @@ func (d *Github) Move(ctx context.Context, srcObj, dstDir model.Obj) error { // commit message, err := getMessage(d.moveMsgTmpl, &MessageTemplateVars{ - UserName: ctx.Value("user").(*model.User).Username, + UserName: getUsername(ctx), ObjName: srcObj.GetName(), ObjPath: srcObj.GetPath(), ParentName: stdpath.Base(stdpath.Dir(srcObj.GetPath())), @@ -444,7 +451,7 @@ func (d *Github) Rename(ctx context.Context, srcObj model.Obj, newName string) e return err } message, err := getMessage(d.renameMsgTmpl, &MessageTemplateVars{ - UserName: ctx.Value("user").(*model.User).Username, + UserName: getUsername(ctx), ObjName: srcObj.GetName(), ObjPath: srcObj.GetPath(), ParentName: stdpath.Base(parentDir), @@ -479,7 +486,7 @@ func (d *Github) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { return err } message, err := getMessage(d.copyMsgTmpl, &MessageTemplateVars{ - UserName: ctx.Value("user").(*model.User).Username, + UserName: getUsername(ctx), ObjName: srcObj.GetName(), ObjPath: srcObj.GetPath(), ParentName: stdpath.Base(stdpath.Dir(srcObj.GetPath())), @@ -502,85 +509,54 @@ func (d *Github) Remove(ctx context.Context, obj model.Obj) error { return errors.New("cannot write to non-branch reference") } parentDir := stdpath.Dir(obj.GetPath()) - parent, err := d.get(parentDir) + tree, treeSha, err := d.getTreeDirectly(parentDir) if err != nil { return err } - if parent.Entries == nil { + var del *TreeObjReq = nil + for _, t := range tree.Trees { + if t.Path == obj.GetName() { + if t.Type == "commit" { + return errors.New("cannot remove a submodule") + } + del = &t.TreeObjReq + del.Sha = nil + break + } + } + if del == nil { return errs.ObjectNotFound } + newTree := make([]interface{}, 0, 2) + newTree = append(newTree, *del) + if len(tree.Trees) == 1 { // completely emptying the repository will get a 404 + newTree = append(newTree, map[string]string{ + "path": ".gitkeep", + "mode": "100644", + "type": "blob", + "content": "", + }) + } + newSha, err := d.newTree(treeSha, newTree) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(parentDir, treeSha, newSha, "/") + if err != nil { + return err + } commitMessage, err := getMessage(d.deleteMsgTmpl, &MessageTemplateVars{ - UserName: ctx.Value("user").(*model.User).Username, + UserName: getUsername(ctx), ObjName: obj.GetName(), ObjPath: obj.GetPath(), ParentName: stdpath.Base(parentDir), ParentPath: parentDir, }, "remove") - if len(parent.Entries) >= 1000 { - tree, err := d.getTree(parent.Sha) - if err != nil { - return err - } - if tree.Truncated { - return fmt.Errorf("tree %s is truncated", parentDir) - } - var del *TreeObjReq = nil - for _, t := range tree.Trees { - if t.Path == obj.GetName() { - if t.Type == "commit" { - return errors.New("cannot remove a submodule") - } - del = &t.TreeObjReq - del.Sha = nil - break - } - } - if del == nil { - return errs.ObjectNotFound - } - newSha, err := d.newTree(parent.Sha, []interface{}{*del}) - if err != nil { - return err - } - rootSha, err := d.renewParentTrees(parentDir, parent.Sha, newSha, "/") - if err != nil { - return err - } - commit, err := d.commit(commitMessage, rootSha) - if err != nil { - return err - } - return d.updateHead(commit) - } else { - sha := "" - objType := "" - for _, entry := range parent.Entries { - if entry.Name == obj.GetName() { - sha = entry.Sha - objType = entry.Type - break - } - } - if sha == "" { - return errs.ObjectNotFound - } - if objType == "submodule" { - return errors.New("cannot delete a submodule") - } - if err != nil { - return err - } - if objType == "dir" { - return d.rmdir(parentDir, parent.Sha, sha, commitMessage) - } - // if deleted file is the only child of its parent, create .gitkeep to retain empty folder - if parentDir != "/" && len(parent.Entries) == 1 { - if err = d.createGitKeep(parentDir, commitMessage); err != nil { - return err - } - } - return d.delete(obj.GetPath(), sha, commitMessage) + commit, err := d.commit(commitMessage, rootSha) + if err != nil { + return err } + return d.updateHead(commit) } func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { @@ -630,7 +606,7 @@ func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr } commitMessage, err := getMessage(d.putMsgTmpl, &MessageTemplateVars{ - UserName: ctx.Value("user").(*model.User).Username, + UserName: getUsername(ctx), ObjName: stream.GetName(), ObjPath: stdpath.Join(dstDir.GetPath(), stream.GetName()), ParentName: dstDir.GetName(), @@ -768,51 +744,6 @@ func (d *Github) delete(path, sha, message string) error { return nil } -func (d *Github) rmdir(parentPath, parentSha, sha, message string) error { - tree, err := d.getTree(parentSha) - if err != nil { - return err - } - if tree.Truncated { - return fmt.Errorf("tree %s is truncated", parentPath) - } - var newTree *TreeObjReq = nil - for _, t := range tree.Trees { - if t.Sha != sha { - newTree = &t.TreeObjReq - newTree.Sha = nil - break - } - } - if newTree == nil { - return errs.ObjectNotFound - } - newTreeSlice := make([]interface{}, 0, 2) - newTreeSlice = append(newTreeSlice, *newTree) - if len(tree.Trees) == 1 { - newTreeSlice = append(newTreeSlice, map[string]string{ - "path": ".gitkeep", - "mode": "100644", - "type": "blob", - "content": "", - }) - } - sha, err = d.newTree(parentSha, newTreeSlice) - if err != nil { - return err - } - - sha, err = d.renewParentTrees(parentPath, parentSha, sha, "/") - if err != nil { - return err - } - commit, err := d.commit(message, sha) - if err != nil { - return err - } - return d.updateHead(commit) -} - func (d *Github) renewParentTrees(path, prevSha, curSha, until string) (string, error) { for path != until { path = stdpath.Dir(path) @@ -953,6 +884,9 @@ func (d *Github) copyWithoutRenewTree(srcObj, dstDir model.Obj) (dstSha, newSha, if err != nil { return "", "", "", nil, err } + if dst.Entries == nil { + return "", "", "", nil, errs.NotFolder + } dstSha = dst.Sha srcParentPath := stdpath.Dir(srcObj.GetPath()) srcParentTree, srcParentSha, err = d.getTreeDirectly(srcParentPath) @@ -973,19 +907,38 @@ func (d *Github) copyWithoutRenewTree(srcObj, dstDir model.Obj) (dstSha, newSha, return "", "", "", nil, errs.ObjectNotFound } - delGitKeep := TreeObjReq{ - Path: ".gitkeep", - Mode: "100644", - Type: "blob", - Sha: nil, + newTree := make([]interface{}, 0, 2) + newTree = append(newTree, *src) + if len(dst.Entries) == 1 && dst.Entries[0].Name == ".gitkeep" { + newTree = append(newTree, TreeObjReq{ + Path: ".gitkeep", + Mode: "100644", + Type: "blob", + Sha: nil, + }) } - newSha, err = d.newTree(dstSha, []interface{}{*src, delGitKeep}) + newSha, err = d.newTree(dstSha, newTree) if err != nil { return "", "", "", nil, err } return dstSha, newSha, srcParentSha, srcParentTree, nil } +func (d *Github) getRepo() (*RepoResp, error) { + res, err := d.client.R().Get(fmt.Sprintf("https://api.github.com/repos/%s/%s", d.Owner, d.Repo)) + if err != nil { + return nil, err + } + if res.StatusCode() != 200 { + return nil, toErr(res) + } + var resp RepoResp + if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { + return nil, err + } + return &resp, nil +} + func (d *Github) addCommitterAndAuthor(m *map[string]interface{}) { if d.CommitterName != "" { committer := map[string]string{ diff --git a/drivers/github/types.go b/drivers/github/types.go index d1516d8e766..d55cf25e5cb 100644 --- a/drivers/github/types.go +++ b/drivers/github/types.go @@ -54,10 +54,15 @@ type TreeObjReq struct { Mode string `json:"mode"` Type string `json:"type"` Sha interface{} `json:"sha"` - Size int64 `json:"size" required:"false"` } -func (o *TreeObjReq) toModelObj() *model.Object { +type TreeObjResp struct { + TreeObjReq + Size int64 `json:"size" required:"false"` + URL string `json:"url"` +} + +func (o *TreeObjResp) toModelObj() *model.Object { return &model.Object{ Name: o.Path, Size: o.Size, @@ -66,12 +71,6 @@ func (o *TreeObjReq) toModelObj() *model.Object { } } -type TreeObjResp struct { - TreeObjReq - Size int64 `json:"size" required:"false"` - URL string `json:"url"` -} - type TreeResp struct { Sha string `json:"sha"` URL string `json:"url"` @@ -97,3 +96,7 @@ type UpdateRefReq struct { Sha string `json:"sha"` Force bool `json:"force"` } + +type RepoResp struct { + DefaultBranch string `json:"default_branch"` +} diff --git a/drivers/github/util.go b/drivers/github/util.go index 7718c3936f9..46d5834bd71 100644 --- a/drivers/github/util.go +++ b/drivers/github/util.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/alist-org/alist/v3/internal/model" "github.com/alist-org/alist/v3/pkg/utils" "github.com/go-resty/resty/v2" "io" @@ -105,3 +106,11 @@ func getPathCommonAncestor(a, b string) (ancestor, aChildName, bChildName, aRest bRest = b[idx+1:] return ancestor, aChildName, bChildName, aRest, bRest } + +func getUsername(ctx context.Context) string { + user, ok := ctx.Value("user").(*model.User) + if !ok { + return "" + } + return user.Username +} From 714da9545e8e1202cfb88e4c054fbe573776d1c5 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sat, 28 Dec 2024 17:14:36 +0800 Subject: [PATCH 5/7] fix: change TargetPath in rename msg from parent path to new self path --- drivers/github/driver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/github/driver.go b/drivers/github/driver.go index 07f71aaf3ef..f48fa9a313c 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -457,7 +457,7 @@ func (d *Github) Rename(ctx context.Context, srcObj model.Obj, newName string) e ParentName: stdpath.Base(parentDir), ParentPath: parentDir, TargetName: newName, - TargetPath: parentDir, + TargetPath: stdpath.Join(parentDir, newName), }, "rename") if err != nil { return err From 4bda846c2839db9110644e129287f9be6cb00095 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Sat, 28 Dec 2024 17:37:55 +0800 Subject: [PATCH 6/7] fix: add non-commit mutex --- drivers/github/driver.go | 70 +++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 41 deletions(-) diff --git a/drivers/github/driver.go b/drivers/github/driver.go index f48fa9a313c..30f707510bf 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -407,11 +407,7 @@ func (d *Github) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if err != nil { return err } - commit, err := d.commit(message, rootSha) - if err != nil { - return err - } - return d.updateHead(commit) + return d.commit(message, rootSha) } func (d *Github) Rename(ctx context.Context, srcObj model.Obj, newName string) error { @@ -462,11 +458,7 @@ func (d *Github) Rename(ctx context.Context, srcObj model.Obj, newName string) e if err != nil { return err } - commit, err := d.commit(message, rootSha) - if err != nil { - return err - } - return d.updateHead(commit) + return d.commit(message, rootSha) } func (d *Github) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { @@ -497,11 +489,7 @@ func (d *Github) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { if err != nil { return err } - commit, err := d.commit(message, rootSha) - if err != nil { - return err - } - return d.updateHead(commit) + return d.commit(message, rootSha) } func (d *Github) Remove(ctx context.Context, obj model.Obj) error { @@ -552,11 +540,7 @@ func (d *Github) Remove(ctx context.Context, obj model.Obj) error { ParentName: stdpath.Base(parentDir), ParentPath: parentDir, }, "remove") - commit, err := d.commit(commitMessage, rootSha) - if err != nil { - return err - } - return d.updateHead(commit) + return d.commit(commitMessage, rootSha) } func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { @@ -647,6 +631,8 @@ func (d *Github) get(path string) (*Object, error) { } func (d *Github) createGitKeep(path, message string) error { + d.commitMutex.Lock() + defer d.commitMutex.Unlock() body := map[string]interface{}{ "message": message, "content": "", @@ -705,6 +691,8 @@ func (d *Github) put(path, sha, message string, ctx context.Context, stream mode _ = contentWriter.Close() }() afterContentReader := strings.NewReader("\"}") + d.commitMutex.Lock() + defer d.commitMutex.Unlock() res, err := d.client.R(). SetHeader("Content-Length", strconv.FormatInt(length, 10)). SetBody(&ReaderWithCtx{ @@ -728,6 +716,8 @@ func (d *Github) put(path, sha, message string, ctx context.Context, stream mode } func (d *Github) delete(path, sha, message string) error { + d.commitMutex.Lock() + defer d.commitMutex.Unlock() body := map[string]interface{}{ "message": message, "sha": sha, @@ -824,7 +814,7 @@ func (d *Github) newTree(baseSha string, tree []interface{}) (string, error) { return resp.Sha, nil } -func (d *Github) commit(message, treeSha string) (string, error) { +func (d *Github) commit(message, treeSha string) error { d.commitMutex.Lock() defer d.commitMutex.Unlock() oldCommit, err := d.getBranchHead() @@ -836,16 +826,30 @@ func (d *Github) commit(message, treeSha string) (string, error) { d.addCommitterAndAuthor(&body) res, err := d.client.R().SetBody(body).Post(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/commits", d.Owner, d.Repo)) if err != nil { - return "", err + return err } if res.StatusCode() != 201 { - return "", toErr(res) + return toErr(res) } var resp CommitResp if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { - return "", err + return err } - return resp.Sha, nil + + // update branch head + res, err = d.client.R(). + SetBody(&UpdateRefReq{ + Sha: resp.Sha, + Force: false, + }). + Patch(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/refs/heads/%s", d.Owner, d.Repo, d.Ref)) + if err != nil { + return err + } + if res.StatusCode() != 200 { + return toErr(res) + } + return nil } func (d *Github) getBranchHead() (string, error) { @@ -863,22 +867,6 @@ func (d *Github) getBranchHead() (string, error) { return resp.Commit.Sha, nil } -func (d *Github) updateHead(sha string) error { - res, err := d.client.R(). - SetBody(&UpdateRefReq{ - Sha: sha, - Force: false, - }). - Patch(fmt.Sprintf("https://api.github.com/repos/%s/%s/git/refs/heads/%s", d.Owner, d.Repo, d.Ref)) - if err != nil { - return err - } - if res.StatusCode() != 200 { - return toErr(res) - } - return nil -} - func (d *Github) copyWithoutRenewTree(srcObj, dstDir model.Obj) (dstSha, newSha, srcParentSha string, srcParentTree *TreeResp, err error) { dst, err := d.get(dstDir.GetPath()) if err != nil { From d07a7b9bb47c52c8f2408c1c8e228ca8331ba761 Mon Sep 17 00:00:00 2001 From: KirCute <951206789@qq.com> Date: Mon, 30 Dec 2024 17:58:26 +0800 Subject: [PATCH 7/7] pref(github): use net/http to put blob --- drivers/github/driver.go | 181 ++++++++++++++++++--------------------- drivers/github/types.go | 6 +- drivers/github/util.go | 7 +- 3 files changed, 88 insertions(+), 106 deletions(-) diff --git a/drivers/github/driver.go b/drivers/github/driver.go index 30f707510bf..ea8f62762ed 100644 --- a/drivers/github/driver.go +++ b/drivers/github/driver.go @@ -13,8 +13,8 @@ import ( "github.com/go-resty/resty/v2" log "github.com/sirupsen/logrus" "io" + "net/http" stdpath "path" - "strconv" "strings" "sync" "text/template" @@ -156,6 +156,8 @@ func (d *Github) MakeDir(ctx context.Context, parentDir model.Obj, dirName strin if !d.isOnBranch { return errors.New("cannot write to non-branch reference") } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() parent, err := d.get(parentDir.GetPath()) if err != nil { return err @@ -195,6 +197,8 @@ func (d *Github) Move(ctx context.Context, srcObj, dstDir model.Obj) error { if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { return errors.New("cannot move parent dir to child") } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() var rootSha string if strings.HasPrefix(dstDir.GetPath(), stdpath.Dir(srcObj.GetPath())) { // /aa/1 -> /aa/bb/ @@ -414,6 +418,8 @@ func (d *Github) Rename(ctx context.Context, srcObj model.Obj, newName string) e if !d.isOnBranch { return errors.New("cannot write to non-branch reference") } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() parentDir := stdpath.Dir(srcObj.GetPath()) tree, _, err := d.getTreeDirectly(parentDir) if err != nil { @@ -468,6 +474,8 @@ func (d *Github) Copy(ctx context.Context, srcObj, dstDir model.Obj) error { if strings.HasPrefix(dstDir.GetPath(), srcObj.GetPath()) { return errors.New("cannot copy parent dir to child") } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() dstSha, newSha, _, _, err := d.copyWithoutRenewTree(srcObj, dstDir) if err != nil { @@ -496,6 +504,8 @@ func (d *Github) Remove(ctx context.Context, obj model.Obj) error { if !d.isOnBranch { return errors.New("cannot write to non-branch reference") } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() parentDir := stdpath.Dir(obj.GetPath()) tree, treeSha, err := d.getTreeDirectly(parentDir) if err != nil { @@ -540,53 +550,51 @@ func (d *Github) Remove(ctx context.Context, obj model.Obj) error { ParentName: stdpath.Base(parentDir), ParentPath: parentDir, }, "remove") + if err != nil { + return err + } return d.commit(commitMessage, rootSha) } -func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) (model.Obj, error) { +func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStreamer, up driver.UpdateProgress) error { if !d.isOnBranch { - return nil, errors.New("cannot write to non-branch reference") + return errors.New("cannot write to non-branch reference") + } + blob, err := d.putBlob(ctx, stream, up) + if err != nil { + return err } + d.commitMutex.Lock() + defer d.commitMutex.Unlock() parent, err := d.get(dstDir.GetPath()) if err != nil { - return nil, err + return err } if parent.Entries == nil { - return nil, errs.NotFolder - } - uploadSha := "" - if len(parent.Entries) >= 1000 { - tree, err := d.getTree(parent.Sha) - if err != nil { - return nil, err - } - if tree.Truncated { - return nil, fmt.Errorf("tree %s is truncated", dstDir.GetPath()) - } - for _, t := range tree.Trees { - if t.Path == stream.GetName() { - if t.Type == "commit" { - return nil, errors.New("cannot replace a submodule") - } - uploadSha = t.Sha.(string) - break - } - } - } else { - for _, entry := range parent.Entries { - if entry.Name == stream.GetName() { - if entry.Type == "submodule" { - return nil, errors.New("cannot replace a submodule") - } - uploadSha = entry.Sha - break - } - } + return errs.NotFolder } - // if parent folder contains .gitkeep only, mark it and delete .gitkeep later - gitKeepSha := "" + newTree := make([]interface{}, 0, 2) + newTree = append(newTree, TreeObjReq{ + Path: stream.GetName(), + Mode: "100644", + Type: "blob", + Sha: blob, + }) if len(parent.Entries) == 1 && parent.Entries[0].Name == ".gitkeep" { - gitKeepSha = parent.Entries[0].Sha + newTree = append(newTree, TreeObjReq{ + Path: ".gitkeep", + Mode: "100644", + Type: "blob", + Sha: nil, + }) + } + newSha, err := d.newTree(parent.Sha, newTree) + if err != nil { + return err + } + rootSha, err := d.renewParentTrees(dstDir.GetPath(), parent.Sha, newSha, "/") + if err != nil { + return err } commitMessage, err := getMessage(d.putMsgTmpl, &MessageTemplateVars{ @@ -597,17 +605,9 @@ func (d *Github) Put(ctx context.Context, dstDir model.Obj, stream model.FileStr ParentPath: dstDir.GetPath(), }, "upload") if err != nil { - return nil, err - } - path := stdpath.Join(dstDir.GetPath(), stream.GetName()) - resp, err := d.put(path, uploadSha, commitMessage, ctx, stream, up) - if err != nil { - return nil, err - } - if gitKeepSha != "" { - err = d.delete(stdpath.Join(dstDir.GetPath(), ".gitkeep"), gitKeepSha, commitMessage) + return err } - return resp.Content.toModelObj(), err + return d.commit(commitMessage, rootSha) } var _ driver.Driver = (*Github)(nil) @@ -631,8 +631,6 @@ func (d *Github) get(path string) (*Object, error) { } func (d *Github) createGitKeep(path, message string) error { - d.commitMutex.Lock() - defer d.commitMutex.Unlock() body := map[string]interface{}{ "message": message, "content": "", @@ -650,36 +648,11 @@ func (d *Github) createGitKeep(path, message string) error { return nil } -func (d *Github) put(path, sha, message string, ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) (*PutResp, error) { - beforeContent := strings.Builder{} - beforeContent.WriteString("{\"message\":\"") - beforeContent.WriteString(message) - beforeContent.WriteString("\",\"branch\":\"") - beforeContent.WriteString(d.Ref) - beforeContent.WriteString("\",") - if sha != "" { - beforeContent.WriteString("\"sha\":\"") - beforeContent.WriteString(sha) - beforeContent.WriteString("\",") - } - if d.CommitterName != "" { - beforeContent.WriteString("\"committer\":{\"name\":\"") - beforeContent.WriteString(d.CommitterName) - beforeContent.WriteString("\",\"email\":\"") - beforeContent.WriteString(d.CommitterEmail) - beforeContent.WriteString("\"},") - } - if d.AuthorName != "" { - beforeContent.WriteString("\"author\":{\"name\":\"") - beforeContent.WriteString(d.AuthorName) - beforeContent.WriteString("\",\"email\":\"") - beforeContent.WriteString(d.AuthorEmail) - beforeContent.WriteString("\"},") - } - beforeContent.WriteString("\"content\":\"") - - length := int64(beforeContent.Len()) + calculateBase64Length(stream.GetSize()) + 2 - beforeContentReader := strings.NewReader(beforeContent.String()) +func (d *Github) putBlob(ctx context.Context, stream model.FileStreamer, up driver.UpdateProgress) (string, error) { + beforeContent := "{\"encoding\":\"base64\",\"content\":\"" + afterContent := "\"}" + length := int64(len(beforeContent)) + calculateBase64Length(stream.GetSize()) + int64(len(afterContent)) + beforeContentReader := strings.NewReader(beforeContent) contentReader, contentWriter := io.Pipe() go func() { encoder := base64.NewEncoder(base64.StdEncoding, contentWriter) @@ -690,34 +663,46 @@ func (d *Github) put(path, sha, message string, ctx context.Context, stream mode _ = encoder.Close() _ = contentWriter.Close() }() - afterContentReader := strings.NewReader("\"}") - d.commitMutex.Lock() - defer d.commitMutex.Unlock() - res, err := d.client.R(). - SetHeader("Content-Length", strconv.FormatInt(length, 10)). - SetBody(&ReaderWithCtx{ + afterContentReader := strings.NewReader(afterContent) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, + fmt.Sprintf("https://api.github.com/repos/%s/%s/git/blobs", d.Owner, d.Repo), + &ReaderWithProgress{ Reader: io.MultiReader(beforeContentReader, contentReader, afterContentReader), - Ctx: ctx, Length: length, Progress: up, - }). - Put(d.getContentApiUrl(path)) + }) if err != nil { - return nil, err + return "", err } - if res.StatusCode() != 200 && res.StatusCode() != 201 { - return nil, toErr(res) + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("Authorization", "Bearer "+d.Token) + req.Header.Set("X-GitHub-Api-Version", "2022-11-28") + req.ContentLength = length + + res, err := base.HttpClient.Do(req) + if err != nil { + return "", err } - var resp PutResp - if err = utils.Json.Unmarshal(res.Body(), &resp); err != nil { - return nil, err + resBody, err := io.ReadAll(res.Body) + if err != nil { + return "", err } - return &resp, nil + if res.StatusCode != 201 { + var errMsg ErrResp + if err = utils.Json.Unmarshal(resBody, &errMsg); err != nil { + return "", errors.New(res.Status) + } else { + return "", fmt.Errorf("%s: %s", res.Status, errMsg.Message) + } + } + var resp PutBlobResp + if err = utils.Json.Unmarshal(resBody, &resp); err != nil { + return "", err + } + return resp.Sha, nil } func (d *Github) delete(path, sha, message string) error { - d.commitMutex.Lock() - defer d.commitMutex.Unlock() body := map[string]interface{}{ "message": message, "sha": sha, @@ -815,8 +800,6 @@ func (d *Github) newTree(baseSha string, tree []interface{}) (string, error) { } func (d *Github) commit(message, treeSha string) error { - d.commitMutex.Lock() - defer d.commitMutex.Unlock() oldCommit, err := d.getBranchHead() body := map[string]interface{}{ "message": message, diff --git a/drivers/github/types.go b/drivers/github/types.go index d55cf25e5cb..425f89795a7 100644 --- a/drivers/github/types.go +++ b/drivers/github/types.go @@ -38,9 +38,9 @@ func (o *Object) toModelObj() *model.Object { } } -type PutResp struct { - Content Object `json:"Content"` - Commit interface{} `json:"commit"` +type PutBlobResp struct { + URL string `json:"url"` + Sha string `json:"sha"` } type ErrResp struct { diff --git a/drivers/github/util.go b/drivers/github/util.go index 46d5834bd71..1e7f7fdbf36 100644 --- a/drivers/github/util.go +++ b/drivers/github/util.go @@ -13,18 +13,17 @@ import ( "text/template" ) -type ReaderWithCtx struct { +type ReaderWithProgress struct { Reader io.Reader - Ctx context.Context Length int64 Progress func(percentage float64) offset int64 } -func (r *ReaderWithCtx) Read(p []byte) (int, error) { +func (r *ReaderWithProgress) Read(p []byte) (int, error) { n, err := r.Reader.Read(p) r.offset += int64(n) - r.Progress(math.Min(100.0, float64(r.offset)/float64(r.Length))) + r.Progress(math.Min(100.0, float64(r.offset)/float64(r.Length)*100.0)) return n, err }