realaravinth/bench-forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix bug isEnd detection on getIssues/getPullRequests (#13299)

Browse source
This commit is contained in:
Lunny Xiao 2020-10-25 13:11:03 +08:00 committed by GitHub
parent eab5837142
commit d130cd147f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 50 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
modules/migrations/github.go
View file

@ -65,23 +65,25 @@ func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
// GithubDownloaderV3 implements a Downloader interface to get repository informations // GithubDownloaderV3 implements a Downloader interface to get repository informations
// from github via APIv3 // from github via APIv3
type GithubDownloaderV3 struct { type GithubDownloaderV3 struct {
ctx context.Context ctx context.Context
client *github.Client client *github.Client
repoOwner string repoOwner string
repoName string repoName string
userName string userName string
password string password string
rate *github.Rate rate *github.Rate
maxPerPage int
} }
// NewGithubDownloaderV3 creates a github Downloader via github v3 API // NewGithubDownloaderV3 creates a github Downloader via github v3 API
func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 { func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
var downloader = GithubDownloaderV3{ var downloader = GithubDownloaderV3{
userName: userName, userName: userName,
password: password, password: password,
ctx: ctx, ctx: ctx,
repoOwner: repoOwner, repoOwner: repoOwner,
repoName: repoName, repoName: repoName,
maxPerPage: 100,
} }
client := &http.Client{ client := &http.Client{
@ -177,7 +179,7 @@ func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
// GetMilestones returns milestones // GetMilestones returns milestones
func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) { func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
var perPage = 100 var perPage = g.maxPerPage
var milestones = make([]*base.Milestone, 0, perPage) var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
g.sleep() g.sleep()
@ -233,7 +235,7 @@ func convertGithubLabel(label *github.Label) *base.Label {
// GetLabels returns labels // GetLabels returns labels
func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) { func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
var perPage = 100 var perPage = g.maxPerPage
var labels = make([]*base.Label, 0, perPage) var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
g.sleep() g.sleep()
@ -304,7 +306,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
// GetReleases returns releases // GetReleases returns releases
func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) { func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
var perPage = 100 var perPage = g.maxPerPage
var releases = make([]*base.Release, 0, perPage) var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
g.sleep() g.sleep()
@ -342,6 +344,9 @@ func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, err
// GetIssues returns issues according start and limit // GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &github.IssueListByRepoOptions{ opt := &github.IssueListByRepoOptions{
Sort: "created", Sort: "created",
Direction: "asc", Direction: "asc",
@ -429,7 +434,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
// GetComments returns comments according issueNumber // GetComments returns comments according issueNumber
func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, error) { func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, error) {
var ( var (
allComments = make([]*base.Comment, 0, 100) allComments = make([]*base.Comment, 0, g.maxPerPage)
created = "created" created = "created"
asc = "asc" asc = "asc"
) )
@ -437,7 +442,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
Sort: &created, Sort: &created,
Direction: &asc, Direction: &asc,
ListOptions: github.ListOptions{ ListOptions: github.ListOptions{
PerPage: 100, PerPage: g.maxPerPage,
}, },
} }
for { for {
@ -459,7 +464,7 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
g.sleep() g.sleep()
res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{ res, resp, err := g.client.Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
Page: i, Page: i,
PerPage: 100, PerPage: g.maxPerPage,
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -497,6 +502,9 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
// GetPullRequests returns pull requests according page and perPage // GetPullRequests returns pull requests according page and perPage
func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &github.PullRequestListOptions{ opt := &github.PullRequestListOptions{
Sort: "created", Sort: "created",
Direction: "asc", Direction: "asc",
@ -650,7 +658,7 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
g.sleep() g.sleep()
res, resp, err := g.client.Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{ res, resp, err := g.client.Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
Page: i, Page: i,
PerPage: 100, PerPage: g.maxPerPage,
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -687,9 +695,9 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
// GetReviews returns pull requests review // GetReviews returns pull requests review
func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
var allReviews = make([]*base.Review, 0, 100) var allReviews = make([]*base.Review, 0, g.maxPerPage)
opt := &github.ListOptions{ opt := &github.ListOptions{
PerPage: 100, PerPage: g.maxPerPage,
} }
for { for {
g.sleep() g.sleep()
@ -703,7 +711,7 @@ func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review
r.IssueIndex = pullRequestNumber r.IssueIndex = pullRequestNumber
// retrieve all review comments // retrieve all review comments
opt2 := &github.ListOptions{ opt2 := &github.ListOptions{
PerPage: 100, PerPage: g.maxPerPage,
} }
for { for {
g.sleep() g.sleep()

30
modules/migrations/gitlab.go
View file

@ -68,6 +68,7 @@ type GitlabDownloader struct {
repoName string repoName string
issueCount int64 issueCount int64
fetchPRcomments bool fetchPRcomments bool
maxPerPage int
} }
// NewGitlabDownloader creates a gitlab Downloader via gitlab API // NewGitlabDownloader creates a gitlab Downloader via gitlab API
@ -99,10 +100,11 @@ func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, passw
} }
return &GitlabDownloader{ return &GitlabDownloader{
ctx: ctx, ctx: ctx,
client: gitlabClient, client: gitlabClient,
repoID: gr.ID, repoID: gr.ID,
repoName: gr.Name, repoName: gr.Name,
maxPerPage: 100,
}, nil }, nil
} }
@ -159,7 +161,7 @@ func (g *GitlabDownloader) GetTopics() ([]string, error) {
// GetMilestones returns milestones // GetMilestones returns milestones
func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) { func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
var perPage = 100 var perPage = g.maxPerPage
var state = "all" var state = "all"
var milestones = make([]*base.Milestone, 0, perPage) var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
@ -230,7 +232,7 @@ func (g *GitlabDownloader) normalizeColor(val string) string {
// GetLabels returns labels // GetLabels returns labels
func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) { func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
var perPage = 100 var perPage = g.maxPerPage
var labels = make([]*base.Label, 0, perPage) var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{ ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{
@ -281,7 +283,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
// GetReleases returns releases // GetReleases returns releases
func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
var perPage = 100 var perPage = g.maxPerPage
var releases = make([]*base.Release, 0, perPage) var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ { for i := 1; ; i++ {
ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{ ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{
@ -330,6 +332,10 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
state := "all" state := "all"
sort := "asc" sort := "asc"
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &gitlab.ListProjectIssuesOptions{ opt := &gitlab.ListProjectIssuesOptions{
State: &state, State: &state,
Sort: &sort, Sort: &sort,
@ -401,7 +407,7 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er
// GetComments returns comments according issueNumber // GetComments returns comments according issueNumber
// TODO: figure out how to transfer comment reactions // TODO: figure out how to transfer comment reactions
func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) { func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
var allComments = make([]*base.Comment, 0, 100) var allComments = make([]*base.Comment, 0, g.maxPerPage)
var page = 1 var page = 1
var realIssueNumber int64 var realIssueNumber int64
@ -415,14 +421,14 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro
realIssueNumber = issueNumber realIssueNumber = issueNumber
comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListIssueDiscussionsOptions{ comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListIssueDiscussionsOptions{
Page: page, Page: page,
PerPage: 100, PerPage: g.maxPerPage,
}, nil, gitlab.WithContext(g.ctx)) }, nil, gitlab.WithContext(g.ctx))
} else { } else {
// If this is a PR, we need to figure out the Gitlab/original PR ID to be passed below // If this is a PR, we need to figure out the Gitlab/original PR ID to be passed below
realIssueNumber = issueNumber - g.issueCount realIssueNumber = issueNumber - g.issueCount
comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListMergeRequestDiscussionsOptions{ comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListMergeRequestDiscussionsOptions{
Page: page, Page: page,
PerPage: 100, PerPage: g.maxPerPage,
}, nil, gitlab.WithContext(g.ctx)) }, nil, gitlab.WithContext(g.ctx))
} }
@ -465,6 +471,10 @@ func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, erro
// GetPullRequests returns pull requests according page and perPage // GetPullRequests returns pull requests according page and perPage
func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &gitlab.ListProjectMergeRequestsOptions{ opt := &gitlab.ListProjectMergeRequestsOptions{
ListOptions: gitlab.ListOptions{ ListOptions: gitlab.ListOptions{
PerPage: perPage, PerPage: perPage,