realaravinth/bench-forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

improve github downloader on migrations (#7049)

Browse source 
* improve github downloader on migrations

* fix tests

* fix  uppercase function parameters
This commit is contained in:
Lunny Xiao 2019-05-31 04:26:57 +08:00 committed by techknowlogick
parent 43cf2f3b55
commit 7d12ec2abd
5 changed files with 141 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
modules/migrations/base/downloader.go
View file

@ -11,9 +11,9 @@ type Downloader interface {
GetMilestones() ([]*Milestone, error) GetMilestones() ([]*Milestone, error)
GetReleases() ([]*Release, error) GetReleases() ([]*Release, error)
GetLabels() ([]*Label, error) GetLabels() ([]*Label, error)
GetIssues(start, limit int) ([]*Issue, error) GetIssues(page, perPage int) ([]*Issue, bool, error)
GetComments(issueNumber int64) ([]*Comment, error) GetComments(issueNumber int64) ([]*Comment, error)
GetPullRequests(start, limit int) ([]*PullRequest, error) GetPullRequests(page, perPage int) ([]*PullRequest, error)
} }
// DownloaderFactory defines an interface to match a downloader implementation and create a downloader // DownloaderFactory defines an interface to match a downloader implementation and create a downloader

8
modules/migrations/git.go
View file

@ -53,9 +53,9 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
return nil, ErrNotSupported return nil, ErrNotSupported
} }
// GetIssues returns issues according start and limit // GetIssues returns issues according page and perPage
func (g *PlainGitDownloader) GetIssues(start, limit int) ([]*base.Issue, error) { func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
return nil, ErrNotSupported return nil, false, ErrNotSupported
} }
// GetComments returns comments according issueNumber // GetComments returns comments according issueNumber
@ -63,7 +63,7 @@ func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, er
return nil, ErrNotSupported return nil, ErrNotSupported
} }
// GetPullRequests returns pull requests according start and limit // GetPullRequests returns pull requests according page and perPage
func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) { func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
return nil, ErrNotSupported return nil, ErrNotSupported
} }

48
modules/migrations/github.go
View file

@ -272,21 +272,22 @@ func convertGithubReactions(reactions *github.Reactions) *base.Reactions {
} }
// GetIssues returns issues according start and limit // GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(start, limit int) ([]*base.Issue, error) { func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
var perPage = 100
opt := &github.IssueListByRepoOptions{ opt := &github.IssueListByRepoOptions{
Sort: "created", Sort: "created",
Direction: "asc", Direction: "asc",
State: "all", State: "all",
ListOptions: github.ListOptions{ ListOptions: github.ListOptions{
PerPage: perPage, PerPage: perPage,
Page: page,
}, },
} }
var allIssues = make([]*base.Issue, 0, limit)
for { var allIssues = make([]*base.Issue, 0, perPage)
issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil { if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err) return nil, false, fmt.Errorf("error while listing repos: %v", err)
} }
for _, issue := range issues { for _, issue := range issues {
if issue.IsPullRequest() { if issue.IsPullRequest() {
@ -327,16 +328,9 @@ func (g *GithubDownloaderV3) GetIssues(start, limit int) ([]*base.Issue, error)
Closed: issue.ClosedAt, Closed: issue.ClosedAt,
IsLocked: *issue.Locked, IsLocked: *issue.Locked,
}) })
if len(allIssues) >= limit {
return allIssues, nil
} }
}
if resp.NextPage == 0 { return allIssues, len(issues) < perPage, nil
break
}
opt.Page = resp.NextPage
}
return allIssues, nil
} }
// GetComments returns comments according issueNumber // GetComments returns comments according issueNumber
@ -379,19 +373,20 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
return allComments, nil return allComments, nil
} }
// GetPullRequests returns pull requests according start and limit // GetPullRequests returns pull requests according page and perPage
func (g *GithubDownloaderV3) GetPullRequests(start, limit int) ([]*base.PullRequest, error) { func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) {
opt := &github.PullRequestListOptions{ opt := &github.PullRequestListOptions{
Sort: "created", Sort: "created",
Direction: "asc", Direction: "asc",
State: "all", State: "all",
ListOptions: github.ListOptions{ ListOptions: github.ListOptions{
PerPage: 100, PerPage: perPage,
Page: page,
}, },
} }
var allPRs = make([]*base.PullRequest, 0, 100) var allPRs = make([]*base.PullRequest, 0, perPage)
for {
prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt) prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil { if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err) return nil, fmt.Errorf("error while listing repos: %v", err)
} }
@ -462,14 +457,7 @@ func (g *GithubDownloaderV3) GetPullRequests(start, limit int) ([]*base.PullRequ
}, },
PatchURL: *pr.PatchURL, PatchURL: *pr.PatchURL,
}) })
if len(allPRs) >= limit { }
return allPRs, nil
}
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
}
return allPRs, nil return allPRs, nil
} }

6
modules/migrations/github_test.go
View file

@ -166,9 +166,11 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, releases[len(releases)-1:]) }, releases[len(releases)-1:])
// downloader.GetIssues() // downloader.GetIssues()
issues, err := downloader.GetIssues(0, 3) issues, isEnd, err := downloader.GetIssues(1, 8)
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, 3, len(issues)) assert.EqualValues(t, 3, len(issues))
assert.False(t, isEnd)
var ( var (
closed1 = time.Date(2018, 10, 23, 02, 57, 43, 0, time.UTC) closed1 = time.Date(2018, 10, 23, 02, 57, 43, 0, time.UTC)
) )
@ -319,7 +321,7 @@ something like in the latest 15days could be enough don't you think ?
}, comments[:3]) }, comments[:3])
// downloader.GetPullRequests() // downloader.GetPullRequests()
prs, err := downloader.GetPullRequests(0, 3) prs, err := downloader.GetPullRequests(1, 3)
assert.NoError(t, err) assert.NoError(t, err)
assert.EqualValues(t, 3, len(prs)) assert.EqualValues(t, 3, len(prs))

10
modules/migrations/migrate.go
View file

@ -128,8 +128,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.Issues { if opts.Issues {
log.Trace("migrating issues and comments") log.Trace("migrating issues and comments")
for { for i := 1; ; i++ {
issues, err := downloader.GetIssues(0, 100) issues, isEnd, err := downloader.GetIssues(i, 100)
if err != nil { if err != nil {
return err return err
} }
@ -160,7 +160,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
} }
} }
if len(issues) < 100 { if isEnd {
break break
} }
} }
@ -168,8 +168,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.PullRequests { if opts.PullRequests {
log.Trace("migrating pull requests and comments") log.Trace("migrating pull requests and comments")
for { for i := 1; ; i++ {
prs, err := downloader.GetPullRequests(0, 100) prs, err := downloader.GetPullRequests(i, 100)
if err != nil { if err != nil {
return err return err
} }