bench-forgejo/modules/migrations/gitlab.go
Gernot Eger a3fe9d87f2
Set the base url when migrating from Gitlab using access token or username without password ()
When migrating from gitlab, set the baseUrl in NewGitlabDownloader when using an access token or username without password

Fix 
2020-06-11 16:41:01 +01:00

562 lines
14 KiB
Go

// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"errors"
"fmt"
"net/url"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/structs"
"github.com/xanzy/go-gitlab"
)
var (
_ base.Downloader = &GitlabDownloader{}
_ base.DownloaderFactory = &GitlabDownloaderFactory{}
)
func init() {
RegisterDownloaderFactory(&GitlabDownloaderFactory{})
}
// GitlabDownloaderFactory defines a gitlab downloader factory
type GitlabDownloaderFactory struct {
}
// Match returns true if the migration remote URL matched this downloader factory
func (f *GitlabDownloaderFactory) Match(opts base.MigrateOptions) (bool, error) {
var matched bool
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return false, err
}
if strings.EqualFold(u.Host, "gitlab.com") && opts.AuthUsername != "" {
matched = true
}
return matched, nil
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GitlabDownloaderFactory) New(opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return nil, err
}
baseURL := u.Scheme + "://" + u.Host
repoNameSpace := strings.TrimPrefix(u.Path, "/")
log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
return NewGitlabDownloader(baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword), nil
}
// GitServiceType returns the type of git service
func (f *GitlabDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.GitlabService
}
// GitlabDownloader implements a Downloader interface to get repository informations
// from gitlab via go-gitlab
// - issueCount is incremented in GetIssues() to ensure PR and Issue numbers do not overlap,
// because Gitlab has individual Issue and Pull Request numbers.
// - issueSeen, working alongside issueCount, is checked in GetComments() to see whether we
// need to fetch the Issue or PR comments, as Gitlab stores them separately.
type GitlabDownloader struct {
ctx context.Context
client *gitlab.Client
repoID int
repoName string
issueCount int64
fetchPRcomments bool
}
// NewGitlabDownloader creates a gitlab Downloader via gitlab API
// Use either a username/password, personal token entered into the username field, or anonymous/public access
// Note: Public access only allows very basic access
func NewGitlabDownloader(baseURL, repoPath, username, password string) *GitlabDownloader {
var gitlabClient *gitlab.Client
var err error
if username != "" {
if password == "" {
gitlabClient, err = gitlab.NewClient(username, gitlab.WithBaseURL(baseURL))
} else {
gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
}
}
if err != nil {
log.Trace("Error logging into gitlab: %v", err)
return nil
}
// Grab and store project/repo ID here, due to issues using the URL escaped path
gr, _, err := gitlabClient.Projects.GetProject(repoPath, nil, nil)
if err != nil {
log.Trace("Error retrieving project: %v", err)
return nil
}
if gr == nil {
log.Trace("Error getting project, project is nil")
return nil
}
return &GitlabDownloader{
ctx: context.Background(),
client: gitlabClient,
repoID: gr.ID,
repoName: gr.Name,
}
}
// SetContext set context
func (g *GitlabDownloader) SetContext(ctx context.Context) {
g.ctx = ctx
}
// GetRepoInfo returns a repository information
func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) {
if g == nil {
return nil, errors.New("error: GitlabDownloader is nil")
}
gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil)
if err != nil {
return nil, err
}
var private bool
switch gr.Visibility {
case gitlab.InternalVisibility:
private = true
case gitlab.PrivateVisibility:
private = true
}
var owner string
if gr.Owner == nil {
log.Trace("gr.Owner is nil, trying to get owner from Namespace")
if gr.Namespace != nil && gr.Namespace.Kind == "user" {
owner = gr.Namespace.Path
}
} else {
owner = gr.Owner.Username
}
// convert gitlab repo to stand Repo
return &base.Repository{
Owner: owner,
Name: gr.Name,
IsPrivate: private,
Description: gr.Description,
OriginalURL: gr.WebURL,
CloneURL: gr.HTTPURLToRepo,
}, nil
}
// GetTopics return gitlab topics
func (g *GitlabDownloader) GetTopics() ([]string, error) {
if g == nil {
return nil, errors.New("error: GitlabDownloader is nil")
}
gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil)
if err != nil {
return nil, err
}
return gr.TagList, err
}
// GetMilestones returns milestones
func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
if g == nil {
return nil, errors.New("error: GitlabDownloader is nil")
}
var perPage = 100
var state = "all"
var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
ms, _, err := g.client.Milestones.ListMilestones(g.repoID, &gitlab.ListMilestonesOptions{
State: &state,
ListOptions: gitlab.ListOptions{
Page: i,
PerPage: perPage,
}}, nil)
if err != nil {
return nil, err
}
for _, m := range ms {
var desc string
if m.Description != "" {
desc = m.Description
}
var state = "open"
var closedAt *time.Time
if m.State != "" {
state = m.State
if state == "closed" {
closedAt = m.UpdatedAt
}
}
var deadline *time.Time
if m.DueDate != nil {
deadlineParsed, err := time.Parse("2006-01-02", m.DueDate.String())
if err != nil {
log.Trace("Error parsing Milestone DueDate time")
deadline = nil
} else {
deadline = &deadlineParsed
}
}
milestones = append(milestones, &base.Milestone{
Title: m.Title,
Description: desc,
Deadline: deadline,
State: state,
Created: *m.CreatedAt,
Updated: m.UpdatedAt,
Closed: closedAt,
})
}
if len(ms) < perPage {
break
}
}
return milestones, nil
}
// GetLabels returns labels
func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
if g == nil {
return nil, errors.New("error: GitlabDownloader is nil")
}
var perPage = 100
var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{
Page: i,
PerPage: perPage,
}, nil)
if err != nil {
return nil, err
}
for _, label := range ls {
baseLabel := &base.Label{
Name: label.Name,
Color: strings.TrimLeft(label.Color, "#)"),
Description: label.Description,
}
labels = append(labels, baseLabel)
}
if len(ls) < perPage {
break
}
}
return labels, nil
}
func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release {
r := &base.Release{
TagName: rel.TagName,
TargetCommitish: rel.Commit.ID,
Name: rel.Name,
Body: rel.Description,
Created: *rel.CreatedAt,
PublisherID: int64(rel.Author.ID),
PublisherName: rel.Author.Username,
}
for k, asset := range rel.Assets.Links {
r.Assets = append(r.Assets, base.ReleaseAsset{
URL: asset.URL,
Name: asset.Name,
ContentType: &rel.Assets.Sources[k].Format,
})
}
return r
}
// GetReleases returns releases
func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
var perPage = 100
var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{
Page: i,
PerPage: perPage,
}, nil)
if err != nil {
return nil, err
}
for _, release := range ls {
releases = append(releases, g.convertGitlabRelease(release))
}
if len(ls) < perPage {
break
}
}
return releases, nil
}
// GetIssues returns issues according start and limit
// Note: issue label description and colors are not supported by the go-gitlab library at this time
// TODO: figure out how to transfer issue reactions
func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
state := "all"
sort := "asc"
opt := &gitlab.ListProjectIssuesOptions{
State: &state,
Sort: &sort,
ListOptions: gitlab.ListOptions{
PerPage: perPage,
Page: page,
},
}
var allIssues = make([]*base.Issue, 0, perPage)
issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil)
if err != nil {
return nil, false, fmt.Errorf("error while listing issues: %v", err)
}
for _, issue := range issues {
var labels = make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, &base.Label{
Name: l,
})
}
var milestone string
if issue.Milestone != nil {
milestone = issue.Milestone.Title
}
allIssues = append(allIssues, &base.Issue{
Title: issue.Title,
Number: int64(issue.IID),
PosterID: int64(issue.Author.ID),
PosterName: issue.Author.Username,
Content: issue.Description,
Milestone: milestone,
State: issue.State,
Created: *issue.CreatedAt,
Labels: labels,
Closed: issue.ClosedAt,
IsLocked: issue.DiscussionLocked,
Updated: *issue.UpdatedAt,
})
// increment issueCount, to be used in GetPullRequests()
g.issueCount++
}
return allIssues, len(issues) < perPage, nil
}
// GetComments returns comments according issueNumber
func (g *GitlabDownloader) GetComments(issueNumber int64) ([]*base.Comment, error) {
var allComments = make([]*base.Comment, 0, 100)
var page = 1
var realIssueNumber int64
for {
var comments []*gitlab.Discussion
var resp *gitlab.Response
var err error
// fetchPRcomments decides whether to fetch Issue or PR comments
if !g.fetchPRcomments {
realIssueNumber = issueNumber
comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListIssueDiscussionsOptions{
Page: page,
PerPage: 100,
}, nil)
} else {
// If this is a PR, we need to figure out the Gitlab/original PR ID to be passed below
realIssueNumber = issueNumber - g.issueCount
comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListMergeRequestDiscussionsOptions{
Page: page,
PerPage: 100,
}, nil)
}
if err != nil {
return nil, fmt.Errorf("error while listing comments: %v %v", g.repoID, err)
}
for _, comment := range comments {
// Flatten comment threads
if !comment.IndividualNote {
for _, note := range comment.Notes {
allComments = append(allComments, &base.Comment{
IssueIndex: realIssueNumber,
PosterID: int64(note.Author.ID),
PosterName: note.Author.Username,
PosterEmail: note.Author.Email,
Content: note.Body,
Created: *note.CreatedAt,
})
}
} else {
c := comment.Notes[0]
allComments = append(allComments, &base.Comment{
IssueIndex: realIssueNumber,
PosterID: int64(c.Author.ID),
PosterName: c.Author.Username,
PosterEmail: c.Author.Email,
Content: c.Body,
Created: *c.CreatedAt,
})
}
}
if resp.NextPage == 0 {
break
}
page = resp.NextPage
}
return allComments, nil
}
// GetPullRequests returns pull requests according page and perPage
func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, error) {
opt := &gitlab.ListProjectMergeRequestsOptions{
ListOptions: gitlab.ListOptions{
PerPage: perPage,
Page: page,
},
}
// Set fetchPRcomments to true here, so PR comments are fetched instead of Issue comments
g.fetchPRcomments = true
var allPRs = make([]*base.PullRequest, 0, perPage)
prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil)
if err != nil {
return nil, fmt.Errorf("error while listing merge requests: %v", err)
}
for _, pr := range prs {
var labels = make([]*base.Label, 0, len(pr.Labels))
for _, l := range pr.Labels {
labels = append(labels, &base.Label{
Name: l,
})
}
var merged bool
if pr.State == "merged" {
merged = true
pr.State = "closed"
}
var mergeTime = pr.MergedAt
if merged && pr.MergedAt == nil {
mergeTime = pr.UpdatedAt
}
var closeTime = pr.ClosedAt
if merged && pr.ClosedAt == nil {
closeTime = pr.UpdatedAt
}
var locked bool
if pr.State == "locked" {
locked = true
}
var milestone string
if pr.Milestone != nil {
milestone = pr.Milestone.Title
}
// Add the PR ID to the Issue Count because PR and Issues share ID space in Gitea
newPRNumber := g.issueCount + int64(pr.IID)
allPRs = append(allPRs, &base.PullRequest{
Title: pr.Title,
Number: newPRNumber,
OriginalNumber: int64(pr.IID),
PosterName: pr.Author.Username,
PosterID: int64(pr.Author.ID),
Content: pr.Description,
Milestone: milestone,
State: pr.State,
Created: *pr.CreatedAt,
Closed: closeTime,
Labels: labels,
Merged: merged,
MergeCommitSHA: pr.MergeCommitSHA,
MergedTime: mergeTime,
IsLocked: locked,
Head: base.PullRequestBranch{
Ref: pr.SourceBranch,
SHA: pr.SHA,
RepoName: g.repoName,
OwnerName: pr.Author.Username,
CloneURL: pr.WebURL,
},
Base: base.PullRequestBranch{
Ref: pr.TargetBranch,
SHA: pr.DiffRefs.BaseSha,
RepoName: g.repoName,
OwnerName: pr.Author.Username,
},
PatchURL: pr.WebURL + ".patch",
})
}
return allPRs, nil
}
// GetReviews returns pull requests review
func (g *GitlabDownloader) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
state, _, err := g.client.MergeRequestApprovals.GetApprovalState(g.repoID, int(pullRequestNumber))
if err != nil {
return nil, err
}
// GitLab's Approvals are equivalent to Gitea's approve reviews
approvers := make(map[int]string)
for i := range state.Rules {
for u := range state.Rules[i].ApprovedBy {
approvers[state.Rules[i].ApprovedBy[u].ID] = state.Rules[i].ApprovedBy[u].Username
}
}
var reviews = make([]*base.Review, 0, len(approvers))
for id, name := range approvers {
reviews = append(reviews, &base.Review{
ReviewerID: int64(id),
ReviewerName: name,
// GitLab API doesn't return a creation date
CreatedAt: time.Now(),
// All we get are approvals
State: base.ReviewStateApproved,
})
}
return reviews, nil
}