From 09663493543a2ce15fc90fbb3808dda5b87335e5 Mon Sep 17 00:00:00 2001 From: Lunny Xiao Date: Wed, 30 Jun 2021 15:23:49 +0800 Subject: [PATCH] Make the github migration less rate limit waiting to get comment per page from repository but not per issue (#16070) * Make the github migration less rate limit waiting to get comment per page from repository but not per issue * Fix lint * adjust Downloader interface * Fix missed reviews * Fix test * Remove unused struct --- modules/migrations/base/downloader.go | 10 ++- modules/migrations/base/null_downloader.go | 9 ++- modules/migrations/base/retry_downloader.go | 7 +- modules/migrations/gitea_downloader.go | 16 ++-- modules/migrations/gitea_downloader_test.go | 4 +- modules/migrations/github.go | 87 ++++++++++++++++++++- modules/migrations/github_test.go | 4 +- modules/migrations/gitlab.go | 7 +- modules/migrations/gitlab_test.go | 4 +- modules/migrations/gogs.go | 7 +- modules/migrations/gogs_test.go | 4 +- modules/migrations/migrate.go | 77 ++++++++++++------ modules/migrations/restore.go | 14 ++-- 13 files changed, 194 insertions(+), 56 deletions(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 919f4b52a..2388b2dd6 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -11,6 +11,13 @@ import ( "code.gitea.io/gitea/modules/structs" ) +// GetCommentOptions represents an options for get comment +type GetCommentOptions struct { + IssueNumber int64 + Page int + PageSize int +} + // Downloader downloads the site repo informations type Downloader interface { SetContext(context.Context) @@ -20,7 +27,8 @@ type Downloader interface { GetReleases() ([]*Release, error) GetLabels() ([]*Label, error) GetIssues(page, perPage int) ([]*Issue, bool, error) - GetComments(issueNumber int64) ([]*Comment, error) + GetComments(opts GetCommentOptions) ([]*Comment, bool, error) + SupportGetRepoComments() bool GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) GetReviews(pullRequestNumber int64) ([]*Review, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) diff --git a/modules/migrations/base/null_downloader.go b/modules/migrations/base/null_downloader.go index a93c20339..53a536709 100644 --- a/modules/migrations/base/null_downloader.go +++ b/modules/migrations/base/null_downloader.go @@ -51,8 +51,8 @@ func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { } // GetComments returns comments according issueNumber -func (n NullDownloader) GetComments(issueNumber int64) ([]*Comment, error) { - return nil, &ErrNotSupported{Entity: "Comments"} +func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) { + return nil, false, &ErrNotSupported{Entity: "Comments"} } // GetPullRequests returns pull requests according page and perPage @@ -80,3 +80,8 @@ func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) ( } return remoteAddr, nil } + +// SupportGetRepoComments return true if it supports get repo comments +func (n NullDownloader) SupportGetRepoComments() bool { + return false +} diff --git a/modules/migrations/base/retry_downloader.go b/modules/migrations/base/retry_downloader.go index 82a038b98..e6c80038f 100644 --- a/modules/migrations/base/retry_downloader.go +++ b/modules/migrations/base/retry_downloader.go @@ -150,18 +150,19 @@ func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { } // GetComments returns a repository's comments with retry -func (d *RetryDownloader) GetComments(issueNumber int64) ([]*Comment, error) { +func (d *RetryDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, error) { var ( comments []*Comment + isEnd bool err error ) err = d.retry(func() error { - comments, err = d.Downloader.GetComments(issueNumber) + comments, isEnd, err = d.Downloader.GetComments(opts) return err }) - return comments, err + return comments, isEnd, err } // GetPullRequests returns a repository's pull requests with retry diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index 40820ae37..665466ffe 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -435,37 +435,37 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err } // GetComments returns comments according issueNumber -func (g *GiteaDownloader) GetComments(index int64) ([]*base.Comment, error) { +func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { var allComments = make([]*base.Comment, 0, g.maxPerPage) // for i := 1; ; i++ { // make sure gitea can shutdown gracefully select { case <-g.ctx.Done(): - return nil, nil + return nil, false, nil default: } - comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, index, gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ + comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.IssueNumber, gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ // PageSize: g.maxPerPage, // Page: i, }}) if err != nil { - return nil, fmt.Errorf("error while listing comments for issue #%d. Error: %v", index, err) + return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.IssueNumber, err) } for _, comment := range comments { reactions, err := g.getCommentReactions(comment.ID) if err != nil { - log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", index, comment.ID, g.repoOwner, g.repoName, err) + log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err) if err2 := models.CreateRepositoryNotice( - fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", index, comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { + fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { log.Error("create repository notice failed: ", err2) } } allComments = append(allComments, &base.Comment{ - IssueIndex: index, + IssueIndex: opts.IssueNumber, PosterID: comment.Poster.ID, PosterName: comment.Poster.UserName, PosterEmail: comment.Poster.Email, @@ -481,7 +481,7 @@ func (g *GiteaDownloader) GetComments(index int64) ([]*base.Comment, error) { // break // } //} - return allComments, nil + return allComments, true, nil } // GetPullRequests returns pull requests according page and perPage diff --git a/modules/migrations/gitea_downloader_test.go b/modules/migrations/gitea_downloader_test.go index babf03828..f62b19897 100644 --- a/modules/migrations/gitea_downloader_test.go +++ b/modules/migrations/gitea_downloader_test.go @@ -224,7 +224,9 @@ func TestGiteaDownloadRepo(t *testing.T) { Closed: &closed2, }, issues[1]) - comments, err := downloader.GetComments(4) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + IssueNumber: 4, + }) assert.NoError(t, err) assert.Len(t, comments, 2) assert.EqualValues(t, 1598975370, comments[0].Created.Unix()) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 8a3f5d34c..9b897662d 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -11,6 +11,7 @@ import ( "io" "net/http" "net/url" + "strconv" "strings" "time" @@ -450,8 +451,22 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, return allIssues, len(issues) < perPage, nil } +// SupportGetRepoComments return true if it supports get repo comments +func (g *GithubDownloaderV3) SupportGetRepoComments() bool { + return true +} + // GetComments returns comments according issueNumber -func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, error) { +func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + if opts.IssueNumber > 0 { + comments, err := g.getComments(opts.IssueNumber) + return comments, false, err + } + + return g.GetAllComments(opts.Page, opts.PageSize) +} + +func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, error) { var ( allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" @@ -519,6 +534,75 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er return allComments, nil } +// GetAllComments returns repository comments according page and perPageSize +func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) { + var ( + allComments = make([]*base.Comment, 0, perPage) + created = "created" + asc = "asc" + ) + opt := &github.IssueListCommentsOptions{ + Sort: &created, + Direction: &asc, + ListOptions: github.ListOptions{ + Page: page, + PerPage: perPage, + }, + } + + g.sleep() + comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt) + if err != nil { + return nil, false, fmt.Errorf("error while listing repos: %v", err) + } + log.Trace("Request get comments %d/%d, but in fact get %d", perPage, page, len(comments)) + g.rate = &resp.Rate + for _, comment := range comments { + var email string + if comment.User.Email != nil { + email = *comment.User.Email + } + + // get reactions + var reactions []*base.Reaction + for i := 1; ; i++ { + g.sleep() + res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ + Page: i, + PerPage: g.maxPerPage, + }) + if err != nil { + return nil, false, err + } + g.rate = &resp.Rate + if len(res) == 0 { + break + } + for _, reaction := range res { + reactions = append(reactions, &base.Reaction{ + UserID: reaction.User.GetID(), + UserName: reaction.User.GetLogin(), + Content: reaction.GetContent(), + }) + } + } + idx := strings.LastIndex(*comment.IssueURL, "/") + issueIndex, _ := strconv.ParseInt((*comment.IssueURL)[idx+1:], 10, 64) + allComments = append(allComments, &base.Comment{ + IssueIndex: issueIndex, + PosterID: *comment.User.ID, + PosterName: *comment.User.Login, + PosterEmail: email, + Content: *comment.Body, + Created: *comment.CreatedAt, + Updated: *comment.UpdatedAt, + Reactions: reactions, + }) + } + + return allComments, len(allComments) < perPage, nil +} + // GetPullRequests returns pull requests according page and perPage func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { if perPage > g.maxPerPage { @@ -539,6 +623,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } + log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs)) g.rate = &resp.Rate for _, pr := range prs { var body string diff --git a/modules/migrations/github_test.go b/modules/migrations/github_test.go index 5bd980a9d..e0ee2fea8 100644 --- a/modules/migrations/github_test.go +++ b/modules/migrations/github_test.go @@ -240,7 +240,9 @@ func TestGitHubDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, err := downloader.GetComments(2) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + IssueNumber: 2, + }) assert.NoError(t, err) assert.Len(t, comments, 2) assert.EqualValues(t, []*base.Comment{ diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index a697075ff..c83989f77 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -430,7 +430,8 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er // GetComments returns comments according issueNumber // TODO: figure out how to transfer comment reactions -func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) { +func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + var issueNumber = opts.IssueNumber var allComments = make([]*base.Comment, 0, g.maxPerPage) var page = 1 @@ -457,7 +458,7 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro } if err != nil { - return nil, fmt.Errorf("error while listing comments: %v %v", g.repoID, err) + return nil, false, fmt.Errorf("error while listing comments: %v %v", g.repoID, err) } for _, comment := range comments { // Flatten comment threads @@ -490,7 +491,7 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro } page = resp.NextPage } - return allComments, nil + return allComments, true, nil } // GetPullRequests returns pull requests according page and perPage diff --git a/modules/migrations/gitlab_test.go b/modules/migrations/gitlab_test.go index 32ed6d807..6a77ff3c2 100644 --- a/modules/migrations/gitlab_test.go +++ b/modules/migrations/gitlab_test.go @@ -204,7 +204,9 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, issues) - comments, err := downloader.GetComments(2) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + IssueNumber: 2, + }) assert.NoError(t, err) assert.Len(t, comments, 4) assert.EqualValues(t, []*base.Comment{ diff --git a/modules/migrations/gogs.go b/modules/migrations/gogs.go index b61690793..d689b0da1 100644 --- a/modules/migrations/gogs.go +++ b/modules/migrations/gogs.go @@ -227,12 +227,13 @@ func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, } // GetComments returns comments according issueNumber -func (g *GogsDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) { +func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + var issueNumber = opts.IssueNumber var allComments = make([]*base.Comment, 0, 100) comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, issueNumber) if err != nil { - return nil, fmt.Errorf("error while listing repos: %v", err) + return nil, false, fmt.Errorf("error while listing repos: %v", err) } for _, comment := range comments { if len(comment.Body) == 0 || comment.Poster == nil { @@ -249,7 +250,7 @@ func (g *GogsDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) }) } - return allComments, nil + return allComments, true, nil } // GetTopics return repository topics diff --git a/modules/migrations/gogs_test.go b/modules/migrations/gogs_test.go index d173952b0..4e384036d 100644 --- a/modules/migrations/gogs_test.go +++ b/modules/migrations/gogs_test.go @@ -103,7 +103,9 @@ func TestGogsDownloadRepo(t *testing.T) { }, issues) // downloader.GetComments() - comments, err := downloader.GetComments(1) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + IssueNumber: 1, + }) assert.NoError(t, err) assert.Len(t, comments, 1) assert.EqualValues(t, []*base.Comment{ diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 3cdf68ab6..0a507d9c3 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -292,6 +292,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts reviewBatchSize = uploader.MaxBatchInsertSize("review") ) + supportAllComments := downloader.SupportGetRepoComments() + if opts.Issues { log.Trace("migrating issues and comments") messenger("repo.migrate.migrating_issues") @@ -311,11 +313,13 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts return err } - if opts.Comments { + if opts.Comments && !supportAllComments { var allComments = make([]*base.Comment, 0, commentBatchSize) for _, issue := range issues { log.Trace("migrating issue %d's comments", issue.Number) - comments, err := downloader.GetComments(issue.Number) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + IssueNumber: issue.Number, + }) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -366,30 +370,34 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts } if opts.Comments { - // plain comments - var allComments = make([]*base.Comment, 0, commentBatchSize) - for _, pr := range prs { - log.Trace("migrating pull request %d's comments", pr.Number) - comments, err := downloader.GetComments(pr.Number) - if err != nil { - if !base.IsErrNotSupported(err) { + if !supportAllComments { + // plain comments + var allComments = make([]*base.Comment, 0, commentBatchSize) + for _, pr := range prs { + log.Trace("migrating pull request %d's comments", pr.Number) + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + IssueNumber: pr.Number, + }) + if err != nil { + if !base.IsErrNotSupported(err) { + return err + } + log.Warn("migrating comments is not supported, ignored") + } + + allComments = append(allComments, comments...) + + if len(allComments) >= commentBatchSize { + if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { + return err + } + allComments = allComments[commentBatchSize:] + } + } + if len(allComments) > 0 { + if err = uploader.CreateComments(allComments...); err != nil { return err } - log.Warn("migrating comments is not supported, ignored") - } - - allComments = append(allComments, comments...) - - if len(allComments) >= commentBatchSize { - if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil { - return err - } - allComments = allComments[commentBatchSize:] - } - } - if len(allComments) > 0 { - if err = uploader.CreateComments(allComments...); err != nil { - return err } } @@ -439,6 +447,27 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts } } + if opts.Comments && supportAllComments { + log.Trace("migrating comments") + for i := 1; ; i++ { + comments, isEnd, err := downloader.GetComments(base.GetCommentOptions{ + Page: i, + PageSize: commentBatchSize, + }) + if err != nil { + return err + } + + if err := uploader.CreateComments(comments...); err != nil { + return err + } + + if isEnd { + break + } + } + } + return uploader.Finish() } diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 5b44811d5..6177f80cb 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -212,27 +212,27 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, } // GetComments returns comments according issueNumber -func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) { +func (r *RepositoryRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { var comments = make([]*base.Comment, 0, 10) - p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber)) + p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", opts.IssueNumber)) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { - return nil, nil + return nil, false, nil } - return nil, err + return nil, false, err } bs, err := ioutil.ReadFile(p) if err != nil { - return nil, err + return nil, false, err } err = yaml.Unmarshal(bs, &comments) if err != nil { - return nil, err + return nil, false, err } - return comments, nil + return comments, false, nil } // GetPullRequests returns pull requests according page and perPage