realaravinth/bench-forgejo
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

improve github downloader on migrations (#7049)

Browse source 
* improve github downloader on migrations

* fix tests

* fix  uppercase function parameters
This commit is contained in:
Lunny Xiao 2019-05-31 04:26:57 +08:00 committed by techknowlogick
parent 43cf2f3b55
commit 7d12ec2abd
5 changed files with 141 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
modules/migrations/base/downloader.go
View file

@ -11,9 +11,9 @@ type Downloader interface {
GetMilestones() ([]*Milestone, error)
GetReleases() ([]*Release, error)
GetLabels() ([]*Label, error)
GetIssues(start, limit int) ([]*Issue, error)
GetIssues(page, perPage int) ([]*Issue, bool, error)
GetComments(issueNumber int64) ([]*Comment, error)
GetPullRequests(start, limit int) ([]*PullRequest, error)
GetPullRequests(page, perPage int) ([]*PullRequest, error)
}
// DownloaderFactory defines an interface to match a downloader implementation and create a downloader

8
modules/migrations/git.go
View file

@ -53,9 +53,9 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
return nil, ErrNotSupported
}
// GetIssues returns issues according start and limit
func (g *PlainGitDownloader) GetIssues(start, limit int) ([]*base.Issue, error) {
return nil, ErrNotSupported
// GetIssues returns issues according page and perPage
func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
return nil, false, ErrNotSupported
}
// GetComments returns comments according issueNumber
@ -63,7 +63,7 @@ func (g *PlainGitDownloader) GetComments(issueNumber int64) ([]*base.Comment, er
return nil, ErrNotSupported
}
// GetPullRequests returns pull requests according start and limit
// GetPullRequests returns pull requests according page and perPage
func (g *PlainGitDownloader) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
return nil, ErrNotSupported
}

46
modules/migrations/github.go
View file

@ -272,21 +272,22 @@ func convertGithubReactions(reactions *github.Reactions) *base.Reactions {
}
// GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(start, limit int) ([]*base.Issue, error) {
var perPage = 100
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
opt := &github.IssueListByRepoOptions{
Sort: "created",
Direction: "asc",
State: "all",
ListOptions: github.ListOptions{
PerPage: perPage,
Page: page,
},
}
var allIssues = make([]*base.Issue, 0, limit)
for {
issues, resp, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
var allIssues = make([]*base.Issue, 0, perPage)
issues, _, err := g.client.Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
for _, issue := range issues {
if issue.IsPullRequest() {
@ -327,16 +328,9 @@ func (g *GithubDownloaderV3) GetIssues(start, limit int) ([]*base.Issue, error)
Closed: issue.ClosedAt,
IsLocked: *issue.Locked,
})
if len(allIssues) >= limit {
return allIssues, nil
}
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
}
return allIssues, nil
return allIssues, len(issues) < perPage, nil
}
// GetComments returns comments according issueNumber
@ -379,19 +373,20 @@ func (g *GithubDownloaderV3) GetComments(issueNumber int64) ([]*base.Comment, er
return allComments, nil
}
// GetPullRequests returns pull requests according start and limit
func (g *GithubDownloaderV3) GetPullRequests(start, limit int) ([]*base.PullRequest, error) {
// GetPullRequests returns pull requests according page and perPage
func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) {
opt := &github.PullRequestListOptions{
Sort: "created",
Direction: "asc",
State: "all",
ListOptions: github.ListOptions{
PerPage: 100,
PerPage: perPage,
Page: page,
},
}
var allPRs = make([]*base.PullRequest, 0, 100)
for {
prs, resp, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
var allPRs = make([]*base.PullRequest, 0, perPage)
prs, _, err := g.client.PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
@ -462,14 +457,7 @@ func (g *GithubDownloaderV3) GetPullRequests(start, limit int) ([]*base.PullRequ
},
PatchURL: *pr.PatchURL,
})
if len(allPRs) >= limit {
return allPRs, nil
}
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
}
return allPRs, nil
}

6
modules/migrations/github_test.go
View file

@ -166,9 +166,11 @@ func TestGitHubDownloadRepo(t *testing.T) {
}, releases[len(releases)-1:])
// downloader.GetIssues()
issues, err := downloader.GetIssues(0, 3)
issues, isEnd, err := downloader.GetIssues(1, 8)
assert.NoError(t, err)
assert.EqualValues(t, 3, len(issues))
assert.False(t, isEnd)
var (
closed1 = time.Date(2018, 10, 23, 02, 57, 43, 0, time.UTC)
)
@ -319,7 +321,7 @@ something like in the latest 15days could be enough don't you think ?
}, comments[:3])
// downloader.GetPullRequests()
prs, err := downloader.GetPullRequests(0, 3)
prs, err := downloader.GetPullRequests(1, 3)
assert.NoError(t, err)
assert.EqualValues(t, 3, len(prs))

10
modules/migrations/migrate.go
View file

@ -128,8 +128,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.Issues {
log.Trace("migrating issues and comments")
for {
issues, err := downloader.GetIssues(0, 100)
for i := 1; ; i++ {
issues, isEnd, err := downloader.GetIssues(i, 100)
if err != nil {
return err
}
@ -160,7 +160,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
}
}
if len(issues) < 100 {
if isEnd {
break
}
}
@ -168,8 +168,8 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
if opts.PullRequests {
log.Trace("migrating pull requests and comments")
for {
prs, err := downloader.GetPullRequests(0, 100)
for i := 1; ; i++ {
prs, err := downloader.GetPullRequests(i, 100)
if err != nil {
return err
}