From 8a0ff92fb35bbce9d1be8cbbbb16333212506d60 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Mon, 5 Jul 2021 18:46:00 +0200 Subject: [PATCH 01/14] Use context to simplify logic. --- modules/migrations/base/downloader.go | 8 +-- modules/migrations/base/issue.go | 1 + modules/migrations/base/null_downloader.go | 4 +- modules/migrations/base/pullrequest.go | 2 +- modules/migrations/base/retry_downloader.go | 4 +- modules/migrations/gitea_downloader.go | 30 +++++++--- modules/migrations/gitea_downloader_test.go | 3 +- modules/migrations/github.go | 19 ++++-- modules/migrations/github_test.go | 10 +++- modules/migrations/gitlab.go | 66 +++++++++++++-------- modules/migrations/gitlab_test.go | 36 +++++++---- modules/migrations/gogs.go | 7 ++- modules/migrations/gogs_test.go | 2 +- modules/migrations/migrate.go | 18 +----- modules/migrations/restore.go | 20 ++++++- 15 files changed, 145 insertions(+), 85 deletions(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 2388b2dd6e77..3d0d662950cc 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -13,9 +13,9 @@ import ( // GetCommentOptions represents an options for get comment type GetCommentOptions struct { - IssueNumber int64 - Page int - PageSize int + Context interface{} + Page int + PageSize int } // Downloader downloads the site repo informations @@ -30,7 +30,7 @@ type Downloader interface { GetComments(opts GetCommentOptions) ([]*Comment, bool, error) SupportGetRepoComments() bool GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) - GetReviews(pullRequestNumber int64) ([]*Review, error) + GetReviews(pullRequestContext interface{}) ([]*Review, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) } diff --git a/modules/migrations/base/issue.go b/modules/migrations/base/issue.go index 8b1b4612444d..bd0486baf9f7 100644 --- a/modules/migrations/base/issue.go +++ b/modules/migrations/base/issue.go @@ -25,4 +25,5 @@ type Issue struct { Labels []*Label Reactions []*Reaction Assignees []string + Context interface{} `yaml:"-"` } diff --git a/modules/migrations/base/null_downloader.go b/modules/migrations/base/null_downloader.go index 53a536709d1f..9ba11aa74778 100644 --- a/modules/migrations/base/null_downloader.go +++ b/modules/migrations/base/null_downloader.go @@ -50,7 +50,7 @@ func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) { return nil, false, &ErrNotSupported{Entity: "Issues"} } -// GetComments returns comments according issueNumber +// GetComments returns comments according the options func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) { return nil, false, &ErrNotSupported{Entity: "Comments"} } @@ -61,7 +61,7 @@ func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool } // GetReviews returns pull requests review -func (n NullDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) { +func (n NullDownloader) GetReviews(pullRequestContext interface{}) ([]*Review, error) { return nil, &ErrNotSupported{Entity: "Reviews"} } diff --git a/modules/migrations/base/pullrequest.go b/modules/migrations/base/pullrequest.go index 6411137d0aa3..d62a69be36f2 100644 --- a/modules/migrations/base/pullrequest.go +++ b/modules/migrations/base/pullrequest.go @@ -13,7 +13,6 @@ import ( // PullRequest defines a standard pull request information type PullRequest struct { Number int64 - OriginalNumber int64 `yaml:"original_number"` Title string PosterName string `yaml:"poster_name"` PosterID int64 `yaml:"poster_id"` @@ -34,6 +33,7 @@ type PullRequest struct { Assignees []string IsLocked bool `yaml:"is_locked"` Reactions []*Reaction + Context interface{} `yaml:"-"` } // IsForkPullRequest returns true if the pull request from a forked repository but not the same repository diff --git a/modules/migrations/base/retry_downloader.go b/modules/migrations/base/retry_downloader.go index e6c80038f181..f303526bca9a 100644 --- a/modules/migrations/base/retry_downloader.go +++ b/modules/migrations/base/retry_downloader.go @@ -182,14 +182,14 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo } // GetReviews returns pull requests reviews -func (d *RetryDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) { +func (d *RetryDownloader) GetReviews(pullRequestContext interface{}) ([]*Review, error) { var ( reviews []*Review err error ) err = d.retry(func() error { - reviews, err = d.Downloader.GetReviews(pullRequestNumber) + reviews, err = d.Downloader.GetReviews(pullRequestContext) return err }) diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index 665466ffeffd..920589a113af 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -424,6 +424,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err Labels: labels, Assignees: assignees, IsLocked: issue.IsLocked, + Context: issue.Index, }) } @@ -436,6 +437,11 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err // GetComments returns comments according issueNumber func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + issueNumber, ok := opts.Context.(int64) + if !ok { + return nil, false, fmt.Errorf("unexpected comment context: %+v", opts.Context) + } + var allComments = make([]*base.Comment, 0, g.maxPerPage) // for i := 1; ; i++ { @@ -446,26 +452,26 @@ func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comm default: } - comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.IssueNumber, gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ + comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, issueNumber, gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ // PageSize: g.maxPerPage, // Page: i, }}) if err != nil { - return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.IssueNumber, err) + return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", issueNumber, err) } for _, comment := range comments { reactions, err := g.getCommentReactions(comment.ID) if err != nil { - log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err) + log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", issueNumber, comment.ID, g.repoOwner, g.repoName, err) if err2 := models.CreateRepositoryNotice( - fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { + fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", issueNumber, comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { log.Error("create repository notice failed: ", err2) } } allComments = append(allComments, &base.Comment{ - IssueIndex: opts.IssueNumber, + IssueIndex: issueNumber, PosterID: comment.Poster.ID, PosterName: comment.Poster.UserName, PosterEmail: comment.Poster.Email, @@ -595,6 +601,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques RepoName: g.repoName, OwnerName: g.repoOwner, }, + Context: pr.Index, }) } @@ -606,12 +613,17 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques } // GetReviews returns pull requests review -func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) { +func (g *GiteaDownloader) GetReviews(context interface{}) ([]*base.Review, error) { if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil { log.Info("GiteaDownloader: instance to old, skip GetReviews") return nil, nil } + issueNumber, ok := context.(int64) + if !ok { + return nil, fmt.Errorf("unexpected context: %+v", context) + } + var allReviews = make([]*base.Review, 0, g.maxPerPage) for i := 1; ; i++ { @@ -622,7 +634,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) { default: } - prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, index, gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ + prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, issueNumber, gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ Page: i, PageSize: g.maxPerPage, }}) @@ -632,7 +644,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) { for _, pr := range prl { - rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, index, pr.ID) + rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, issueNumber, pr.ID) if err != nil { return nil, err } @@ -658,7 +670,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) { allReviews = append(allReviews, &base.Review{ ID: pr.ID, - IssueIndex: index, + IssueIndex: issueNumber, ReviewerID: pr.Reviewer.ID, ReviewerName: pr.Reviewer.UserName, Official: pr.Official, diff --git a/modules/migrations/gitea_downloader_test.go b/modules/migrations/gitea_downloader_test.go index f62b19897c63..cddaeca6a084 100644 --- a/modules/migrations/gitea_downloader_test.go +++ b/modules/migrations/gitea_downloader_test.go @@ -225,7 +225,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, issues[1]) comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: 4, + Context: 4, }) assert.NoError(t, err) assert.Len(t, comments, 2) @@ -337,7 +337,6 @@ func TestGiteaDownloadRepo(t *testing.T) { func assertEqualPulls(t *testing.T, pullExp, pullGet *base.PullRequest) { assertEqualIssue(t, pull2issue(pullExp), pull2issue(pullGet)) - assert.EqualValues(t, 0, pullGet.OriginalNumber) assert.EqualValues(t, pullExp.PatchURL, pullGet.PatchURL) assert.EqualValues(t, pullExp.Merged, pullGet.Merged) assert.EqualValues(t, pullExp.MergedTime.Unix(), pullGet.MergedTime.Unix()) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 9b897662d030..d250ebf27705 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -445,6 +445,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, Reactions: reactions, Closed: issue.ClosedAt, IsLocked: *issue.Locked, + Context: int64(*issue.Number), }) } @@ -458,8 +459,8 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool { // GetComments returns comments according issueNumber func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - if opts.IssueNumber > 0 { - comments, err := g.getComments(opts.IssueNumber) + if issueNumber, ok := opts.Context.(int64); ok { + comments, err := g.getComments(issueNumber) return comments, false, err } @@ -735,6 +736,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq }, PatchURL: *pr.PatchURL, Reactions: reactions, + Context: int64(*pr.Number), }) } @@ -798,28 +800,33 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques } // GetReviews returns pull requests review -func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { +func (g *GithubDownloaderV3) GetReviews(context interface{}) ([]*base.Review, error) { + issueNumber, ok := context.(int64) + if !ok { + return nil, fmt.Errorf("unexpected context: %+v", context) + } + var allReviews = make([]*base.Review, 0, g.maxPerPage) opt := &github.ListOptions{ PerPage: g.maxPerPage, } for { g.sleep() - reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), opt) + reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } g.rate = &resp.Rate for _, review := range reviews { r := convertGithubReview(review) - r.IssueIndex = pullRequestNumber + r.IssueIndex = issueNumber // retrieve all review comments opt2 := &github.ListOptions{ PerPage: g.maxPerPage, } for { g.sleep() - reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), review.GetID(), opt2) + reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), review.GetID(), opt2) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } diff --git a/modules/migrations/github_test.go b/modules/migrations/github_test.go index e0ee2fea8447..17ba235af29e 100644 --- a/modules/migrations/github_test.go +++ b/modules/migrations/github_test.go @@ -184,7 +184,8 @@ func TestGitHubDownloadRepo(t *testing.T) { Content: "+1", }, }, - Closed: &closed1, + Closed: &closed1, + Context: 1, }, { Number: 2, @@ -235,13 +236,14 @@ func TestGitHubDownloadRepo(t *testing.T) { Content: "+1", }, }, - Closed: &closed2, + Closed: &closed2, + Context: 2, }, }, issues) // downloader.GetComments() comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: 2, + Context: 2, }) assert.NoError(t, err) assert.Len(t, comments, 2) @@ -317,6 +319,7 @@ func TestGitHubDownloadRepo(t *testing.T) { Merged: true, MergedTime: &merged1, MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2", + Context: 3, }, { Number: 4, @@ -363,6 +366,7 @@ func TestGitHubDownloadRepo(t *testing.T) { Content: "+1", }, }, + Context: 4, }, }, prs) diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index c83989f771fb..910dcd390f6f 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -60,17 +60,14 @@ func (f *GitlabDownloaderFactory) GitServiceType() structs.GitServiceType { // from gitlab via go-gitlab // - issueCount is incremented in GetIssues() to ensure PR and Issue numbers do not overlap, // because Gitlab has individual Issue and Pull Request numbers. -// - issueSeen, working alongside issueCount, is checked in GetComments() to see whether we -// need to fetch the Issue or PR comments, as Gitlab stores them separately. type GitlabDownloader struct { base.NullDownloader - ctx context.Context - client *gitlab.Client - repoID int - repoName string - issueCount int64 - fetchPRcomments bool - maxPerPage int + ctx context.Context + client *gitlab.Client + repoID int + repoName string + issueCount int64 + maxPerPage int } // NewGitlabDownloader creates a gitlab Downloader via gitlab API @@ -350,6 +347,12 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { return releases, nil } +type gitlabIssueContext struct { + OriginalID int + MigratedID int64 + IsMergeRequest bool +} + // GetIssues returns issues according start and limit // Note: issue label description and colors are not supported by the go-gitlab library at this time func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { @@ -419,6 +422,11 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er Closed: issue.ClosedAt, IsLocked: issue.DiscussionLocked, Updated: *issue.UpdatedAt, + Context: gitlabIssueContext{ + OriginalID: int(issue.IID), + MigratedID: int64(issue.IID), + IsMergeRequest: false, + }, }) // increment issueCount, to be used in GetPullRequests() @@ -431,27 +439,26 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er // GetComments returns comments according issueNumber // TODO: figure out how to transfer comment reactions func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - var issueNumber = opts.IssueNumber + context, ok := opts.Context.(gitlabIssueContext) + if !ok { + return nil, false, fmt.Errorf("unexpected context: %+v", opts.Context) + } + var allComments = make([]*base.Comment, 0, g.maxPerPage) var page = 1 - var realIssueNumber int64 for { var comments []*gitlab.Discussion var resp *gitlab.Response var err error - // fetchPRcomments decides whether to fetch Issue or PR comments - if !g.fetchPRcomments { - realIssueNumber = issueNumber - comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListIssueDiscussionsOptions{ + if !context.IsMergeRequest { + comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, context.OriginalID, &gitlab.ListIssueDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) } else { - // If this is a PR, we need to figure out the Gitlab/original PR ID to be passed below - realIssueNumber = issueNumber - g.issueCount - comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(realIssueNumber), &gitlab.ListMergeRequestDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, context.OriginalID, &gitlab.ListMergeRequestDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) @@ -465,7 +472,7 @@ func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com if !comment.IndividualNote { for _, note := range comment.Notes { allComments = append(allComments, &base.Comment{ - IssueIndex: realIssueNumber, + IssueIndex: int64(context.MigratedID), PosterID: int64(note.Author.ID), PosterName: note.Author.Username, PosterEmail: note.Author.Email, @@ -476,7 +483,7 @@ func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com } else { c := comment.Notes[0] allComments = append(allComments, &base.Comment{ - IssueIndex: realIssueNumber, + IssueIndex: int64(context.MigratedID), PosterID: int64(c.Author.ID), PosterName: c.Author.Username, PosterEmail: c.Author.Email, @@ -507,9 +514,6 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque }, } - // Set fetchPRcomments to true here, so PR comments are fetched instead of Issue comments - g.fetchPRcomments = true - var allPRs = make([]*base.PullRequest, 0, perPage) prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(g.ctx)) @@ -573,7 +577,6 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque allPRs = append(allPRs, &base.PullRequest{ Title: pr.Title, Number: newPRNumber, - OriginalNumber: int64(pr.IID), PosterName: pr.Author.Username, PosterID: int64(pr.Author.ID), Content: pr.Description, @@ -601,6 +604,11 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque OwnerName: pr.Author.Username, }, PatchURL: pr.WebURL + ".patch", + Context: gitlabIssueContext{ + OriginalID: int(pr.IID), + MigratedID: newPRNumber, + IsMergeRequest: true, + }, }) } @@ -608,8 +616,13 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests review -func (g *GitlabDownloader) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { - state, resp, err := g.client.MergeRequestApprovals.GetApprovalState(g.repoID, int(pullRequestNumber), gitlab.WithContext(g.ctx)) +func (g *GitlabDownloader) GetReviews(context interface{}) ([]*base.Review, error) { + issueContext, ok := context.(gitlabIssueContext) + if !ok { + return nil, fmt.Errorf("unexpected context: %+v", context) + } + + state, resp, err := g.client.MergeRequestApprovals.GetApprovalState(g.repoID, int(issueContext.OriginalID), gitlab.WithContext(g.ctx)) if err != nil { if resp != nil && resp.StatusCode == 404 { log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error())) @@ -629,6 +642,7 @@ func (g *GitlabDownloader) GetReviews(pullRequestNumber int64) ([]*base.Review, var reviews = make([]*base.Review, 0, len(approvers)) for id, name := range approvers { reviews = append(reviews, &base.Review{ + IssueIndex: issueContext.MigratedID, ReviewerID: int64(id), ReviewerName: name, // GitLab API doesn't return a creation date diff --git a/modules/migrations/gitlab_test.go b/modules/migrations/gitlab_test.go index 6a77ff3c230a..392ff3a3f15d 100644 --- a/modules/migrations/gitlab_test.go +++ b/modules/migrations/gitlab_test.go @@ -153,6 +153,10 @@ func TestGitlabDownloadRepo(t *testing.T) { Content: "open_mouth", }}, Closed: &closed1, + Context: gitlabIssueContext{ + OriginalID: 1, + IsMergeRequest: false, + }, }, { Number: 2, @@ -201,11 +205,19 @@ func TestGitlabDownloadRepo(t *testing.T) { Content: "hearts", }}, Closed: &closed2, + Context: gitlabIssueContext{ + OriginalID: 2, + IsMergeRequest: false, + }, }, }, issues) comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: 2, + Context: gitlabIssueContext{ + OriginalID: 2, + MigratedID: 2, + IsMergeRequest: false, + }, }) assert.NoError(t, err) assert.Len(t, comments, 4) @@ -250,15 +262,14 @@ func TestGitlabDownloadRepo(t *testing.T) { assert.EqualValues(t, []*base.PullRequest{ { - Number: 4, - OriginalNumber: 2, - Title: "Test branch", - Content: "do not merge this PR", - Milestone: "1.0.0", - PosterID: 1241334, - PosterName: "lafriks", - State: "opened", - Created: time.Date(2019, 11, 28, 15, 56, 54, 104000000, time.UTC), + Number: 4, + Title: "Test branch", + Content: "do not merge this PR", + Milestone: "1.0.0", + PosterID: 1241334, + PosterName: "lafriks", + State: "opened", + Created: time.Date(2019, 11, 28, 15, 56, 54, 104000000, time.UTC), Labels: []*base.Label{ { Name: "bug", @@ -291,6 +302,11 @@ func TestGitlabDownloadRepo(t *testing.T) { Merged: false, MergedTime: nil, MergeCommitSHA: "", + Context: gitlabIssueContext{ + OriginalID: 2, + MigratedID: 4, + IsMergeRequest: true, + }, }, }, prs) diff --git a/modules/migrations/gogs.go b/modules/migrations/gogs.go index d689b0da1155..cd2b63101fdc 100644 --- a/modules/migrations/gogs.go +++ b/modules/migrations/gogs.go @@ -228,7 +228,11 @@ func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, // GetComments returns comments according issueNumber func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - var issueNumber = opts.IssueNumber + issueNumber, ok := opts.Context.(int64) + if !ok { + return nil, false, fmt.Errorf("unexpected context: %+v", opts.Context) + } + var allComments = make([]*base.Comment, 0, 100) comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, issueNumber) @@ -302,6 +306,7 @@ func convertGogsIssue(issue *gogs.Issue) *base.Issue { Created: issue.Created, Labels: labels, Closed: closed, + Context: issue.Index, } } diff --git a/modules/migrations/gogs_test.go b/modules/migrations/gogs_test.go index 4e384036d702..201bf6b9f7bb 100644 --- a/modules/migrations/gogs_test.go +++ b/modules/migrations/gogs_test.go @@ -104,7 +104,7 @@ func TestGogsDownloadRepo(t *testing.T) { // downloader.GetComments() comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: 1, + Context: 1, }) assert.NoError(t, err) assert.Len(t, comments, 1) diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go index 0a507d9c3341..7d5aa9670b1e 100644 --- a/modules/migrations/migrate.go +++ b/modules/migrations/migrate.go @@ -318,7 +318,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts for _, issue := range issues { log.Trace("migrating issue %d's comments", issue.Number) comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: issue.Number, + Context: issue.Context, }) if err != nil { if !base.IsErrNotSupported(err) { @@ -376,7 +376,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts for _, pr := range prs { log.Trace("migrating pull request %d's comments", pr.Number) comments, _, err := downloader.GetComments(base.GetCommentOptions{ - IssueNumber: pr.Number, + Context: pr.Context, }) if err != nil { if !base.IsErrNotSupported(err) { @@ -404,14 +404,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts // migrate reviews var allReviews = make([]*base.Review, 0, reviewBatchSize) for _, pr := range prs { - number := pr.Number - - // on gitlab migrations pull number change - if pr.OriginalNumber > 0 { - number = pr.OriginalNumber - } - - reviews, err := downloader.GetReviews(number) + reviews, err := downloader.GetReviews(pr.Context) if err != nil { if !base.IsErrNotSupported(err) { return err @@ -419,11 +412,6 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts log.Warn("migrating reviews is not supported, ignored") break } - if pr.OriginalNumber > 0 { - for i := range reviews { - reviews[i].IssueIndex = pr.Number - } - } allReviews = append(allReviews, reviews...) diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 6177f80cbbca..432941f913f8 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -208,13 +208,21 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, if err != nil { return nil, false, err } + for _, issue := range issues { + issue.Context = issue.Number + } return issues, true, nil } // GetComments returns comments according issueNumber func (r *RepositoryRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + issueNumber, ok := opts.Context.(int64) + if !ok { + return nil, false, fmt.Errorf("unexpected context: %+v", opts.Context) + } + var comments = make([]*base.Comment, 0, 10) - p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", opts.IssueNumber)) + p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber)) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -258,14 +266,20 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq } for _, pr := range pulls { pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL) + pr.Context = pr.Number } return pulls, true, nil } // GetReviews returns pull requests review -func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) { +func (r *RepositoryRestorer) GetReviews(context interface{}) ([]*base.Review, error) { + issueNumber, ok := context.(int64) + if !ok { + return nil, fmt.Errorf("unexpected context: %+v", context) + } + var reviews = make([]*base.Review, 0, 10) - p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", pullRequestNumber)) + p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", issueNumber)) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { From c0661b983ee0ec1d362de4860c63ec7d9fb1705c Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 6 Jul 2021 15:32:53 +0000 Subject: [PATCH 02/14] Added migration from OneDev. --- modules/migrations/gitea_uploader.go | 3 + modules/migrations/onedev.go | 559 +++++++++++++++++++++++++++ modules/structs/repo.go | 4 + options/locale/locale_en-US.ini | 1 + public/img/svg/gitea-onedev.svg | 1 + templates/repo/migrate/onedev.tmpl | 113 ++++++ web_src/svg/gitea-onedev.svg | 42 ++ 7 files changed, 723 insertions(+) create mode 100644 modules/migrations/onedev.go create mode 100644 public/img/svg/gitea-onedev.svg create mode 100644 templates/repo/migrate/onedev.tmpl create mode 100644 web_src/svg/gitea-onedev.svg diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go index 2b18098b7f16..fd4ce799d3c0 100644 --- a/modules/migrations/gitea_uploader.go +++ b/modules/migrations/gitea_uploader.go @@ -555,6 +555,9 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR // download patch file err := func() error { + if pr.PatchURL == "" { + return nil + } // pr.PatchURL maybe a local file ret, err := uri.Open(pr.PatchURL) if err != nil { diff --git a/modules/migrations/onedev.go b/modules/migrations/onedev.go new file mode 100644 index 000000000000..17e6eeea9aee --- /dev/null +++ b/modules/migrations/onedev.go @@ -0,0 +1,559 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "code.gitea.io/gitea/modules/log" + "code.gitea.io/gitea/modules/migrations/base" + "code.gitea.io/gitea/modules/structs" + + jsoniter "github.com/json-iterator/go" +) + +var ( + _ base.Downloader = &OneDevDownloader{} + _ base.DownloaderFactory = &OneDevDownloaderFactory{} +) + +func init() { + RegisterDownloaderFactory(&OneDevDownloaderFactory{}) +} + +// OneDevDownloaderFactory defines a onedev downloader factory +type OneDevDownloaderFactory struct { +} + +// New returns a Downloader related to this factory according MigrateOptions +func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) { + u, err := url.Parse(opts.CloneAddr) + if err != nil { + return nil, err + } + + repoName := "" + + fields := strings.Split(strings.Trim(u.Path, "/"), "/") + if len(fields) == 2 && fields[0] == "projects" { + repoName = fields[1] + } else if len(fields) == 1 { + repoName = fields[0] + } else { + return nil, fmt.Errorf("invalid path: %s", u.Path) + } + + u.Path = "" + u.Fragment = "" + + log.Trace("Create onedev downloader. BaseURL: %v RepoName: %s", u, repoName) + + return NewOneDevDownloader(ctx, u, opts.AuthUsername, opts.AuthPassword, repoName), nil +} + +// GitServiceType returns the type of git service +func (f *OneDevDownloaderFactory) GitServiceType() structs.GitServiceType { + return structs.OneDevService +} + +type onedevUser struct { + ID int64 `json:"id"` + Name string `json:"name"` + Email string `json:"email"` +} + +// OneDevDownloader implements a Downloader interface to get repository informations +// from OneDev via API +type OneDevDownloader struct { + base.NullDownloader + ctx context.Context + client *http.Client + baseURL *url.URL + repoName string + repoID int64 + maxIssueIndex int64 + userMap map[int64]*onedevUser + milestoneMap map[int64]string +} + +// SetContext set context +func (d *OneDevDownloader) SetContext(ctx context.Context) { + d.ctx = ctx +} + +// NewOneDevDownloader creates a gogs Downloader via gogs API +func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader { + var downloader = &OneDevDownloader{ + ctx: ctx, + baseURL: baseURL, + repoName: repoName, + client: &http.Client{ + Transport: &http.Transport{ + Proxy: func(req *http.Request) (*url.URL, error) { + if len(username) > 0 && len(password) > 0 { + req.SetBasicAuth(username, password) + } + return nil, nil + }, + }, + }, + userMap: make(map[int64]*onedevUser), + milestoneMap: make(map[int64]string), + } + + return downloader +} + +func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error { + u, err := d.baseURL.Parse(endpoint) + if err != nil { + return err + } + + if parameter != nil { + query := u.Query() + for k, v := range parameter { + query.Set(k, v) + } + u.RawQuery = query.Encode() + } + + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return err + } + + resp, err := d.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + decoder := jsoniter.NewDecoder(resp.Body) + return decoder.Decode(&result) +} + +// GetRepoInfo returns a repository information +func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) { + info := make([]struct { + ID int64 `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + }, 0, 1) + + err := d.callAPI( + "/api/projects", + map[string]string{ + "query": `"Name" is "` + d.repoName + `"`, + "offset": "0", + "count": "1", + }, + &info, + ) + if err != nil { + return nil, err + } + if len(info) != 1 { + return nil, fmt.Errorf("Project %s not found", d.repoName) + } + + d.repoID = info[0].ID + + cloneURL, err := d.baseURL.Parse(info[0].Name) + if err != nil { + return nil, err + } + originalURL, err := d.baseURL.Parse("/projects/" + info[0].Name) + if err != nil { + return nil, err + } + + return &base.Repository{ + Name: info[0].Name, + Description: info[0].Description, + CloneURL: cloneURL.String(), + OriginalURL: originalURL.String(), + }, nil +} + +// GetMilestones returns milestones +func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { + rawMilestones := make([]struct { + ID int64 `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + DueDate *time.Time `json:"dueDate"` + Closed bool `json:"closed"` + }, 0, 100) + + endpoint := fmt.Sprintf("/api/projects/%d/milestones", d.repoID) + + t := time.Now() + + var milestones = make([]*base.Milestone, 0, 100) + offset := 0 + for { + err := d.callAPI( + endpoint, + map[string]string{ + "offset": strconv.Itoa(offset), + "count": "100", + }, + &rawMilestones, + ) + if err != nil { + return nil, err + } + if len(rawMilestones) == 0 { + break + } + offset += 100 + + for _, milestone := range rawMilestones { + d.milestoneMap[milestone.ID] = milestone.Name + closed := &t + if !milestone.Closed { + closed = nil + } + + milestones = append(milestones, &base.Milestone{ + Title: milestone.Name, + Description: milestone.Description, + Deadline: milestone.DueDate, + Created: t, + Updated: &t, + Closed: closed, + }) + } + } + return milestones, nil +} + +// GetLabels returns labels +func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) { + return nil, nil +} + +type onedevIssueContext struct { + OriginalID int64 + MigratedID int64 + IsPullRequest bool +} + +// GetIssues returns issues according start and limit, perPage is not supported +func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { + rawIssues := make([]struct { + ID int64 `json:"id"` + Number int64 `json:"number"` + State string `json:"state"` + Title string `json:"title"` + Description string `json:"description"` + MilestoneID int64 `json:"milestoneId"` + SubmitterID int64 `json:"submitterId"` + SubmitDate time.Time `json:"submitDate"` + }, 0, perPage) + + err := d.callAPI( + "/api/issues", + map[string]string{ + "query": `"Project" is "` + d.repoName + `"`, + "offset": strconv.Itoa((page - 1) * perPage), + "count": strconv.Itoa(perPage), + }, + &rawIssues, + ) + if err != nil { + return nil, false, err + } + + issues := make([]*base.Issue, 0, len(rawIssues)) + for _, issue := range rawIssues { + state := strings.ToLower(issue.State) + if state == "released" { + state = "closed" + } + poster := d.tryGetUser(issue.SubmitterID) + issues = append(issues, &base.Issue{ + Title: issue.Title, + Number: issue.Number, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: issue.Description, + Milestone: d.milestoneMap[issue.MilestoneID], + State: state, + Created: issue.SubmitDate, + //Closed: closed, + Context: onedevIssueContext{ + OriginalID: issue.ID, + MigratedID: issue.Number, + IsPullRequest: false, + }, + }) + + if d.maxIssueIndex < issue.Number { + d.maxIssueIndex = issue.Number + } + } + + return issues, len(issues) == 0, nil +} + +// GetComments returns comments according issueNumber +func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { + context, ok := opts.Context.(onedevIssueContext) + if !ok { + return nil, false, fmt.Errorf("unexpected comment context: %+v", opts.Context) + } + + rawComments := make([]struct { + Date time.Time `json:"date"` + UserID int64 `json:"userId"` + Content string `json:"content"` + }, 0, 100) + + var endpoint string + if context.IsPullRequest { + endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", context.OriginalID) + } else { + endpoint = fmt.Sprintf("/api/issues/%d/comments", context.OriginalID) + } + + err := d.callAPI( + endpoint, + nil, + &rawComments, + ) + if err != nil { + return nil, false, err + } + + rawChanges := make([]struct { + Date time.Time `json:"date"` + UserID int64 `json:"userId"` + Data map[string]interface{} `json:"data"` + }, 0, 100) + + if context.IsPullRequest { + endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", context.OriginalID) + } else { + endpoint = fmt.Sprintf("/api/issues/%d/changes", context.OriginalID) + } + + err = d.callAPI( + endpoint, + nil, + &rawChanges, + ) + if err != nil { + return nil, false, err + } + + comments := make([]*base.Comment, 0, len(rawComments)+len(rawChanges)) + for _, comment := range rawComments { + if len(comment.Content) == 0 { + continue + } + poster := d.tryGetUser(comment.UserID) + comments = append(comments, &base.Comment{ + IssueIndex: context.MigratedID, + PosterID: poster.ID, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: comment.Content, + Created: comment.Date, + Updated: comment.Date, + }) + } + for _, change := range rawChanges { + contentV, ok := change.Data["content"] + if !ok { + contentV, ok = change.Data["comment"] + if !ok { + continue + } + } + content, ok := contentV.(string) + if !ok || len(content) == 0 { + continue + } + + poster := d.tryGetUser(change.UserID) + comments = append(comments, &base.Comment{ + IssueIndex: context.MigratedID, + PosterID: poster.ID, + PosterName: poster.Name, + PosterEmail: poster.Email, + Content: content, + Created: change.Date, + Updated: change.Date, + }) + } + + return comments, false, nil +} + +// GetPullRequests returns pull requests according page and perPage +func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { + rawPullRequests := make([]struct { + ID int64 `json:"id"` + Number int64 `json:"number"` + Title string `json:"title"` + SubmitterID int64 `json:"submitterId"` + SubmitDate time.Time `json:"submitDate"` + Description string `json:"description"` + TargetBranch string `json:"targetBranch"` + SourceBranch string `json:"sourceBranch"` + BaseCommitHash string `json:"baseCommitHash"` + CloseInfo *struct { + Date *time.Time `json:"date"` + Status string `json:"status"` + } + }, 0, perPage) + + err := d.callAPI( + "/api/pull-requests", + map[string]string{ + "query": `"Target Project" is "` + d.repoName + `"`, + "offset": strconv.Itoa((page - 1) * perPage), + "count": strconv.Itoa(perPage), + }, + &rawPullRequests, + ) + if err != nil { + return nil, false, err + } + + pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests)) + for _, pr := range rawPullRequests { + state := "open" + merged := false + var closeTime *time.Time + var mergedTime *time.Time + if pr.CloseInfo != nil { + state = "closed" + closeTime = pr.CloseInfo.Date + if pr.CloseInfo.Status == "MERGED" { // "DISCARDED" + merged = true + mergedTime = pr.CloseInfo.Date + } + } + poster := d.tryGetUser(pr.SubmitterID) + + number := pr.Number + d.maxIssueIndex + pullRequests = append(pullRequests, &base.PullRequest{ + Title: pr.Title, + Number: number, + PosterName: poster.Name, + PosterID: poster.ID, + Content: pr.Description, + State: state, + Created: pr.SubmitDate, + Closed: closeTime, + Merged: merged, + MergedTime: mergedTime, + Head: base.PullRequestBranch{ + Ref: pr.SourceBranch, + SHA: pr.BaseCommitHash, + RepoName: d.repoName, + }, + Base: base.PullRequestBranch{ + Ref: pr.TargetBranch, + SHA: "", + RepoName: d.repoName, + }, + Context: onedevIssueContext{ + OriginalID: pr.ID, + MigratedID: number, + IsPullRequest: true, + }, + }) + } + + return pullRequests, len(pullRequests) == 0, nil +} + +// GetReviews returns pull requests review +func (d *OneDevDownloader) GetReviews(context interface{}) ([]*base.Review, error) { + issueContext, ok := context.(onedevIssueContext) + if !ok { + return nil, fmt.Errorf("unexpected context: %+v", context) + } + + rawReviews := make([]struct { + ID int64 `json:"id"` + UserID int64 `json:"userId"` + Result *struct { + Commit string `json:"commit"` + Approved bool `json:"approved"` + Comment string `json:"comment"` + } + }, 0, 100) + + err := d.callAPI( + fmt.Sprintf("/api/pull-requests/%d/reviews", issueContext.OriginalID), + nil, + &rawReviews, + ) + if err != nil { + return nil, err + } + + var reviews = make([]*base.Review, 0, len(rawReviews)) + for _, review := range rawReviews { + state := base.ReviewStatePending + content := "" + if review.Result != nil { + if len(review.Result.Comment) > 0 { + state = base.ReviewStateCommented + content = review.Result.Comment + } + if review.Result.Approved { + state = base.ReviewStateApproved + } + } + + poster := d.tryGetUser(review.UserID) + reviews = append(reviews, &base.Review{ + IssueIndex: issueContext.MigratedID, + ReviewerID: poster.ID, + ReviewerName: poster.Name, + CreatedAt: time.Now(), + Content: content, + State: state, + }) + } + + return reviews, nil +} + +// GetTopics return repository topics +func (d *OneDevDownloader) GetTopics() ([]string, error) { + return []string{}, nil +} + +func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser { + user, ok := d.userMap[userID] + if !ok { + err := d.callAPI( + fmt.Sprintf("/api/users/%d", userID), + nil, + &user, + ) + if err != nil { + user = &onedevUser{ + Name: fmt.Sprintf("User %d", userID), + } + } + d.userMap[userID] = user + } + + return user +} diff --git a/modules/structs/repo.go b/modules/structs/repo.go index cef864c0205b..bd13825ffaf9 100644 --- a/modules/structs/repo.go +++ b/modules/structs/repo.go @@ -246,6 +246,7 @@ const ( GiteaService // 3 gitea service GitlabService // 4 gitlab service GogsService // 5 gogs service + OneDevService // 6 onedev service ) // Name represents the service type's name @@ -265,6 +266,8 @@ func (gt GitServiceType) Title() string { return "GitLab" case GogsService: return "Gogs" + case OneDevService: + return "OneDev" case PlainGitService: return "Git" } @@ -320,5 +323,6 @@ var ( GitlabService, GiteaService, GogsService, + OneDevService, } ) diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini index dcdfa611ec95..9aaa335c008e 100644 --- a/options/locale/locale_en-US.ini +++ b/options/locale/locale_en-US.ini @@ -886,6 +886,7 @@ migrate.git.description = Migrating or Mirroring git data from Git services migrate.gitlab.description = Migrating data from GitLab.com or Self-Hosted gitlab server. migrate.gitea.description = Migrating data from Gitea.com or Self-Hosted Gitea server. migrate.gogs.description = Migrating data from notabug.org or other Self-Hosted Gogs server. +migrate.onedev.description = Migrating data from code.onedev.io or Self-Hosted OneDev server. migrate.migrating_git = Migrating Git Data migrate.migrating_topics = Migrating Topics migrate.migrating_milestones = Migrating Milestones diff --git a/public/img/svg/gitea-onedev.svg b/public/img/svg/gitea-onedev.svg new file mode 100644 index 000000000000..1f0d1d8363b4 --- /dev/null +++ b/public/img/svg/gitea-onedev.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/templates/repo/migrate/onedev.tmpl b/templates/repo/migrate/onedev.tmpl new file mode 100644 index 000000000000..95ac9dd9d772 --- /dev/null +++ b/templates/repo/migrate/onedev.tmpl @@ -0,0 +1,113 @@ +{{template "base/head" .}} +
+
+
+
+ {{.CsrfTokenHtml}} +

+ {{.i18n.Tr "repo.migrate.migrate" .service.Title}} + +

+
+ {{template "base/alert" .}} +
+ + + + {{.i18n.Tr "repo.migrate.clone_address_desc"}}{{if .ContextUser.CanImportLocal}} {{.i18n.Tr "repo.migrate.clone_local_path"}}{{end}} + +
+ +
+ + +
+ +
+ + +
+ + {{template "repo/migrate/options" .}} + +
+
+ +
+ + +
+
+ + +
+
+
+ +
+ + +
+
+
+ +
+ +
+ +
+ + + {{avatar .ContextUser 28 "mini"}} + {{.ContextUser.ShortName 40}} + + {{svg "octicon-triangle-down" 14 "dropdown icon"}} +
+
+ {{avatar .SignedUser 28 "mini"}} + {{.SignedUser.ShortName 40}} +
+ {{range .Orgs}} +
+ {{avatar . 28 "mini"}} + {{.ShortName 40}} +
+ {{end}} +
+
+
+ +
+ + +
+
+ +
+ {{if .IsForcedPrivate}} + + + {{else}} + + + {{end}} +
+
+
+ + +
+ +
+ + + {{.i18n.Tr "cancel"}} +
+
+
+
+
+
+{{template "base/footer" .}} diff --git a/web_src/svg/gitea-onedev.svg b/web_src/svg/gitea-onedev.svg new file mode 100644 index 000000000000..490c22fc8e2f --- /dev/null +++ b/web_src/svg/gitea-onedev.svg @@ -0,0 +1,42 @@ + + + + + + + + + + + \ No newline at end of file From e17edc0dd64bd0d73abc68b648e9dfcf7ce9e4d2 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 6 Jul 2021 16:26:14 +0000 Subject: [PATCH 03/14] Use "Pull Requests". --- templates/repo/migrate/onedev.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/templates/repo/migrate/onedev.tmpl b/templates/repo/migrate/onedev.tmpl index 95ac9dd9d772..03f9c6fd38e6 100644 --- a/templates/repo/migrate/onedev.tmpl +++ b/templates/repo/migrate/onedev.tmpl @@ -46,7 +46,7 @@
- +
From 1394141cdd5dfd6cdeb4c64b59be89dc75ef5430 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 6 Jul 2021 16:30:09 +0000 Subject: [PATCH 04/14] Fixed test. --- modules/migrations/gitea_downloader_test.go | 4 ++-- modules/migrations/github_test.go | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/migrations/gitea_downloader_test.go b/modules/migrations/gitea_downloader_test.go index cddaeca6a084..71f0c692b029 100644 --- a/modules/migrations/gitea_downloader_test.go +++ b/modules/migrations/gitea_downloader_test.go @@ -225,7 +225,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, issues[1]) comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: 4, + Context: int64(4), }) assert.NoError(t, err) assert.Len(t, comments, 2) @@ -299,7 +299,7 @@ func TestGiteaDownloadRepo(t *testing.T) { PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", }, prs[1]) - reviews, err := downloader.GetReviews(7) + reviews, err := downloader.GetReviews(int64(7)) assert.NoError(t, err) if assert.Len(t, reviews, 3) { assert.EqualValues(t, 689, reviews[0].ReviewerID) diff --git a/modules/migrations/github_test.go b/modules/migrations/github_test.go index 17ba235af29e..d6080860d15d 100644 --- a/modules/migrations/github_test.go +++ b/modules/migrations/github_test.go @@ -185,7 +185,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, Closed: &closed1, - Context: 1, + Context: int64(1), }, { Number: 2, @@ -237,13 +237,13 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, Closed: &closed2, - Context: 2, + Context: int64(2), }, }, issues) // downloader.GetComments() comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: 2, + Context: int64(2), }) assert.NoError(t, err) assert.Len(t, comments, 2) @@ -319,7 +319,7 @@ func TestGitHubDownloadRepo(t *testing.T) { Merged: true, MergedTime: &merged1, MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2", - Context: 3, + Context: int64(3), }, { Number: 4, @@ -366,11 +366,11 @@ func TestGitHubDownloadRepo(t *testing.T) { Content: "+1", }, }, - Context: 4, + Context: int64(4), }, }, prs) - reviews, err := downloader.GetReviews(3) + reviews, err := downloader.GetReviews(int64(3)) assert.NoError(t, err) assert.EqualValues(t, []*base.Review{ { @@ -402,7 +402,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) - reviews, err = downloader.GetReviews(4) + reviews, err = downloader.GetReviews(int64(4)) assert.NoError(t, err) assert.EqualValues(t, []*base.Review{ { From 15b903e11dc881d7d643e9bdee0aca707f235bef Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Wed, 7 Jul 2021 07:46:00 +0200 Subject: [PATCH 05/14] Changed comments. --- modules/migrations/onedev.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/migrations/onedev.go b/modules/migrations/onedev.go index 17e6eeea9aee..83e4b56430a3 100644 --- a/modules/migrations/onedev.go +++ b/modules/migrations/onedev.go @@ -29,11 +29,11 @@ func init() { RegisterDownloaderFactory(&OneDevDownloaderFactory{}) } -// OneDevDownloaderFactory defines a onedev downloader factory +// OneDevDownloaderFactory defines a downloader factory type OneDevDownloaderFactory struct { } -// New returns a Downloader related to this factory according MigrateOptions +// New returns a downloader related to this factory according MigrateOptions func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) { u, err := url.Parse(opts.CloneAddr) if err != nil { @@ -71,7 +71,7 @@ type onedevUser struct { } // OneDevDownloader implements a Downloader interface to get repository informations -// from OneDev via API +// from OneDev type OneDevDownloader struct { base.NullDownloader ctx context.Context @@ -89,7 +89,7 @@ func (d *OneDevDownloader) SetContext(ctx context.Context) { d.ctx = ctx } -// NewOneDevDownloader creates a gogs Downloader via gogs API +// NewOneDevDownloader creates a new downloader func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader { var downloader = &OneDevDownloader{ ctx: ctx, @@ -306,7 +306,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er return issues, len(issues) == 0, nil } -// GetComments returns comments according issueNumber +// GetComments returns requested comments func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { context, ok := opts.Context.(onedevIssueContext) if !ok { @@ -400,7 +400,7 @@ func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com return comments, false, nil } -// GetPullRequests returns pull requests according page and perPage +// GetPullRequests returns requested pull requests func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { rawPullRequests := make([]struct { ID int64 `json:"id"` @@ -480,7 +480,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque return pullRequests, len(pullRequests) == 0, nil } -// GetReviews returns pull requests review +// GetReviews returns requested pull requests reviews func (d *OneDevDownloader) GetReviews(context interface{}) ([]*base.Review, error) { issueContext, ok := context.(onedevIssueContext) if !ok { From 5962cf6ad0bb2ab93bdd4eb19fb59f710b32a9ac Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sun, 8 Aug 2021 17:16:00 +0200 Subject: [PATCH 06/14] Use typed context. --- modules/migrations/base/downloader.go | 4 +- modules/migrations/base/issue.go | 23 ++++++++++- modules/migrations/base/null_downloader.go | 2 +- modules/migrations/base/pullrequest.go | 2 +- modules/migrations/base/retry_downloader.go | 2 +- modules/migrations/gitea_downloader.go | 32 ++++++--------- modules/migrations/gitea_downloader_test.go | 4 +- modules/migrations/github.go | 27 ++++++------- modules/migrations/github_test.go | 14 +++---- modules/migrations/gitlab.go | 39 ++++++++++--------- modules/migrations/gitlab_test.go | 16 ++++---- modules/migrations/gogs.go | 11 ++---- modules/migrations/gogs_test.go | 2 +- modules/migrations/onedev.go | 43 +++++++++++---------- modules/migrations/restore.go | 20 +++------- 15 files changed, 119 insertions(+), 122 deletions(-) diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go index 3d0d662950cc..583d5a787b1a 100644 --- a/modules/migrations/base/downloader.go +++ b/modules/migrations/base/downloader.go @@ -13,7 +13,7 @@ import ( // GetCommentOptions represents an options for get comment type GetCommentOptions struct { - Context interface{} + Context IssueContext Page int PageSize int } @@ -30,7 +30,7 @@ type Downloader interface { GetComments(opts GetCommentOptions) ([]*Comment, bool, error) SupportGetRepoComments() bool GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) - GetReviews(pullRequestContext interface{}) ([]*Review, error) + GetReviews(pullRequestContext IssueContext) ([]*Review, error) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) } diff --git a/modules/migrations/base/issue.go b/modules/migrations/base/issue.go index bd0486baf9f7..42fd312db6fd 100644 --- a/modules/migrations/base/issue.go +++ b/modules/migrations/base/issue.go @@ -7,6 +7,27 @@ package base import "time" +// IssueContext is used to map between local and foreign issue/PR ids. +type IssueContext interface { + LocalID() int64 + ForeignID() int64 +} + +// BasicIssueContext is a 1:1 mapping between local and foreign ids. +type BasicIssueContext struct { + ID int64 +} + +// LocalID gets the local id. +func (c BasicIssueContext) LocalID() int64 { + return c.ID +} + +// ForeignID gets the foreign id. +func (c BasicIssueContext) ForeignID() int64 { + return c.ID +} + // Issue is a standard issue information type Issue struct { Number int64 @@ -25,5 +46,5 @@ type Issue struct { Labels []*Label Reactions []*Reaction Assignees []string - Context interface{} `yaml:"-"` + Context IssueContext `yaml:"-"` } diff --git a/modules/migrations/base/null_downloader.go b/modules/migrations/base/null_downloader.go index 9ba11aa74778..c64d0e263379 100644 --- a/modules/migrations/base/null_downloader.go +++ b/modules/migrations/base/null_downloader.go @@ -61,7 +61,7 @@ func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool } // GetReviews returns pull requests review -func (n NullDownloader) GetReviews(pullRequestContext interface{}) ([]*Review, error) { +func (n NullDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) { return nil, &ErrNotSupported{Entity: "Reviews"} } diff --git a/modules/migrations/base/pullrequest.go b/modules/migrations/base/pullrequest.go index d62a69be36f2..84b302d18fad 100644 --- a/modules/migrations/base/pullrequest.go +++ b/modules/migrations/base/pullrequest.go @@ -33,7 +33,7 @@ type PullRequest struct { Assignees []string IsLocked bool `yaml:"is_locked"` Reactions []*Reaction - Context interface{} `yaml:"-"` + Context IssueContext `yaml:"-"` } // IsForkPullRequest returns true if the pull request from a forked repository but not the same repository diff --git a/modules/migrations/base/retry_downloader.go b/modules/migrations/base/retry_downloader.go index f303526bca9a..623bfc86b526 100644 --- a/modules/migrations/base/retry_downloader.go +++ b/modules/migrations/base/retry_downloader.go @@ -182,7 +182,7 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo } // GetReviews returns pull requests reviews -func (d *RetryDownloader) GetReviews(pullRequestContext interface{}) ([]*Review, error) { +func (d *RetryDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) { var ( reviews []*Review err error diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index 920589a113af..f2eec1200c74 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -424,7 +424,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err Labels: labels, Assignees: assignees, IsLocked: issue.IsLocked, - Context: issue.Index, + Context: base.BasicIssueContext{ID: issue.Index}, }) } @@ -437,11 +437,6 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err // GetComments returns comments according issueNumber func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - issueNumber, ok := opts.Context.(int64) - if !ok { - return nil, false, fmt.Errorf("unexpected comment context: %+v", opts.Context) - } - var allComments = make([]*base.Comment, 0, g.maxPerPage) // for i := 1; ; i++ { @@ -452,26 +447,26 @@ func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comm default: } - comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, issueNumber, gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ + comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{ // PageSize: g.maxPerPage, // Page: i, }}) if err != nil { - return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", issueNumber, err) + return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.Context.ForeignID(), err) } for _, comment := range comments { reactions, err := g.getCommentReactions(comment.ID) if err != nil { - log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", issueNumber, comment.ID, g.repoOwner, g.repoName, err) + log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err) if err2 := models.CreateRepositoryNotice( - fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", issueNumber, comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { + fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil { log.Error("create repository notice failed: ", err2) } } allComments = append(allComments, &base.Comment{ - IssueIndex: issueNumber, + IssueIndex: opts.Context.LocalID(), PosterID: comment.Poster.ID, PosterName: comment.Poster.UserName, PosterEmail: comment.Poster.Email, @@ -601,7 +596,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques RepoName: g.repoName, OwnerName: g.repoOwner, }, - Context: pr.Index, + Context: base.BasicIssueContext{ID: pr.Index}, }) } @@ -613,17 +608,12 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques } // GetReviews returns pull requests review -func (g *GiteaDownloader) GetReviews(context interface{}) ([]*base.Review, error) { +func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil { log.Info("GiteaDownloader: instance to old, skip GetReviews") return nil, nil } - issueNumber, ok := context.(int64) - if !ok { - return nil, fmt.Errorf("unexpected context: %+v", context) - } - var allReviews = make([]*base.Review, 0, g.maxPerPage) for i := 1; ; i++ { @@ -634,7 +624,7 @@ func (g *GiteaDownloader) GetReviews(context interface{}) ([]*base.Review, error default: } - prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, issueNumber, gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ + prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, context.ForeignID(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{ Page: i, PageSize: g.maxPerPage, }}) @@ -644,7 +634,7 @@ func (g *GiteaDownloader) GetReviews(context interface{}) ([]*base.Review, error for _, pr := range prl { - rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, issueNumber, pr.ID) + rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, context.ForeignID(), pr.ID) if err != nil { return nil, err } @@ -670,7 +660,7 @@ func (g *GiteaDownloader) GetReviews(context interface{}) ([]*base.Review, error allReviews = append(allReviews, &base.Review{ ID: pr.ID, - IssueIndex: issueNumber, + IssueIndex: context.LocalID(), ReviewerID: pr.Reviewer.ID, ReviewerName: pr.Reviewer.UserName, Official: pr.Official, diff --git a/modules/migrations/gitea_downloader_test.go b/modules/migrations/gitea_downloader_test.go index 71f0c692b029..b9652754539c 100644 --- a/modules/migrations/gitea_downloader_test.go +++ b/modules/migrations/gitea_downloader_test.go @@ -225,7 +225,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, issues[1]) comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: int64(4), + Context: base.BasicIssueContext{ID: 4}, }) assert.NoError(t, err) assert.Len(t, comments, 2) @@ -299,7 +299,7 @@ func TestGiteaDownloadRepo(t *testing.T) { PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", }, prs[1]) - reviews, err := downloader.GetReviews(int64(7)) + reviews, err := downloader.GetReviews(base.BasicIssueContext{ID: 7}) assert.NoError(t, err) if assert.Len(t, reviews, 3) { assert.EqualValues(t, 689, reviews[0].ReviewerID) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index d250ebf27705..c490639502cf 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -445,7 +445,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, Reactions: reactions, Closed: issue.ClosedAt, IsLocked: *issue.Locked, - Context: int64(*issue.Number), + Context: base.BasicIssueContext{ID: int64(*issue.Number)}, }) } @@ -459,15 +459,15 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool { // GetComments returns comments according issueNumber func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - if issueNumber, ok := opts.Context.(int64); ok { - comments, err := g.getComments(issueNumber) + if opts.Context != nil { + comments, err := g.getComments(opts.Context) return comments, false, err } return g.GetAllComments(opts.Page, opts.PageSize) } -func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, error) { +func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*base.Comment, error) { var ( allComments = make([]*base.Comment, 0, g.maxPerPage) created = "created" @@ -482,7 +482,7 @@ func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, er } for { g.sleep() - comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt) + comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } @@ -517,7 +517,7 @@ func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, er } } allComments = append(allComments, &base.Comment{ - IssueIndex: issueNumber, + IssueIndex: issueContext.LocalID(), PosterID: *comment.User.ID, PosterName: *comment.User.Login, PosterEmail: email, @@ -736,7 +736,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq }, PatchURL: *pr.PatchURL, Reactions: reactions, - Context: int64(*pr.Number), + Context: base.BasicIssueContext{ID: int64(*pr.Number)}, }) } @@ -800,33 +800,28 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques } // GetReviews returns pull requests review -func (g *GithubDownloaderV3) GetReviews(context interface{}) ([]*base.Review, error) { - issueNumber, ok := context.(int64) - if !ok { - return nil, fmt.Errorf("unexpected context: %+v", context) - } - +func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Review, error) { var allReviews = make([]*base.Review, 0, g.maxPerPage) opt := &github.ListOptions{ PerPage: g.maxPerPage, } for { g.sleep() - reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt) + reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } g.rate = &resp.Rate for _, review := range reviews { r := convertGithubReview(review) - r.IssueIndex = issueNumber + r.IssueIndex = context.LocalID() // retrieve all review comments opt2 := &github.ListOptions{ PerPage: g.maxPerPage, } for { g.sleep() - reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), review.GetID(), opt2) + reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2) if err != nil { return nil, fmt.Errorf("error while listing repos: %v", err) } diff --git a/modules/migrations/github_test.go b/modules/migrations/github_test.go index d6080860d15d..4b5f1ef6785c 100644 --- a/modules/migrations/github_test.go +++ b/modules/migrations/github_test.go @@ -185,7 +185,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, Closed: &closed1, - Context: int64(1), + Context: base.BasicIssueContext{ID: 1}, }, { Number: 2, @@ -237,13 +237,13 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, Closed: &closed2, - Context: int64(2), + Context: base.BasicIssueContext{ID: 2}, }, }, issues) // downloader.GetComments() comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: int64(2), + Context: base.BasicIssueContext{ID: 2}, }) assert.NoError(t, err) assert.Len(t, comments, 2) @@ -319,7 +319,7 @@ func TestGitHubDownloadRepo(t *testing.T) { Merged: true, MergedTime: &merged1, MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2", - Context: int64(3), + Context: base.BasicIssueContext{ID: 3}, }, { Number: 4, @@ -366,11 +366,11 @@ func TestGitHubDownloadRepo(t *testing.T) { Content: "+1", }, }, - Context: int64(4), + Context: base.BasicIssueContext{ID: 4}, }, }, prs) - reviews, err := downloader.GetReviews(int64(3)) + reviews, err := downloader.GetReviews(base.BasicIssueContext{ID: 3}) assert.NoError(t, err) assert.EqualValues(t, []*base.Review{ { @@ -402,7 +402,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) - reviews, err = downloader.GetReviews(int64(4)) + reviews, err = downloader.GetReviews(base.BasicIssueContext{ID: 4}) assert.NoError(t, err) assert.EqualValues(t, []*base.Review{ { diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go index 910dcd390f6f..981616db9793 100644 --- a/modules/migrations/gitlab.go +++ b/modules/migrations/gitlab.go @@ -348,11 +348,19 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) { } type gitlabIssueContext struct { - OriginalID int - MigratedID int64 + foreignID int64 + localID int64 IsMergeRequest bool } +func (c gitlabIssueContext) LocalID() int64 { + return c.localID +} + +func (c gitlabIssueContext) ForeignID() int64 { + return c.foreignID +} + // GetIssues returns issues according start and limit // Note: issue label description and colors are not supported by the go-gitlab library at this time func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { @@ -423,8 +431,8 @@ func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er IsLocked: issue.DiscussionLocked, Updated: *issue.UpdatedAt, Context: gitlabIssueContext{ - OriginalID: int(issue.IID), - MigratedID: int64(issue.IID), + foreignID: int64(issue.IID), + localID: int64(issue.IID), IsMergeRequest: false, }, }) @@ -453,12 +461,12 @@ func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com var resp *gitlab.Response var err error if !context.IsMergeRequest { - comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, context.OriginalID, &gitlab.ListIssueDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListIssueDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) } else { - comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, context.OriginalID, &gitlab.ListMergeRequestDiscussionsOptions{ + comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListMergeRequestDiscussionsOptions{ Page: page, PerPage: g.maxPerPage, }, nil, gitlab.WithContext(g.ctx)) @@ -472,7 +480,7 @@ func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com if !comment.IndividualNote { for _, note := range comment.Notes { allComments = append(allComments, &base.Comment{ - IssueIndex: int64(context.MigratedID), + IssueIndex: context.LocalID(), PosterID: int64(note.Author.ID), PosterName: note.Author.Username, PosterEmail: note.Author.Email, @@ -483,7 +491,7 @@ func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com } else { c := comment.Notes[0] allComments = append(allComments, &base.Comment{ - IssueIndex: int64(context.MigratedID), + IssueIndex: context.LocalID(), PosterID: int64(c.Author.ID), PosterName: c.Author.Username, PosterEmail: c.Author.Email, @@ -605,8 +613,8 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque }, PatchURL: pr.WebURL + ".patch", Context: gitlabIssueContext{ - OriginalID: int(pr.IID), - MigratedID: newPRNumber, + foreignID: int64(pr.IID), + localID: newPRNumber, IsMergeRequest: true, }, }) @@ -616,13 +624,8 @@ func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns pull requests review -func (g *GitlabDownloader) GetReviews(context interface{}) ([]*base.Review, error) { - issueContext, ok := context.(gitlabIssueContext) - if !ok { - return nil, fmt.Errorf("unexpected context: %+v", context) - } - - state, resp, err := g.client.MergeRequestApprovals.GetApprovalState(g.repoID, int(issueContext.OriginalID), gitlab.WithContext(g.ctx)) +func (g *GitlabDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { + state, resp, err := g.client.MergeRequestApprovals.GetApprovalState(g.repoID, int(context.ForeignID()), gitlab.WithContext(g.ctx)) if err != nil { if resp != nil && resp.StatusCode == 404 { log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error())) @@ -642,7 +645,7 @@ func (g *GitlabDownloader) GetReviews(context interface{}) ([]*base.Review, erro var reviews = make([]*base.Review, 0, len(approvers)) for id, name := range approvers { reviews = append(reviews, &base.Review{ - IssueIndex: issueContext.MigratedID, + IssueIndex: context.LocalID(), ReviewerID: int64(id), ReviewerName: name, // GitLab API doesn't return a creation date diff --git a/modules/migrations/gitlab_test.go b/modules/migrations/gitlab_test.go index 392ff3a3f15d..8e0f551995bf 100644 --- a/modules/migrations/gitlab_test.go +++ b/modules/migrations/gitlab_test.go @@ -154,7 +154,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }}, Closed: &closed1, Context: gitlabIssueContext{ - OriginalID: 1, + foreignID: 1, IsMergeRequest: false, }, }, @@ -206,7 +206,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }}, Closed: &closed2, Context: gitlabIssueContext{ - OriginalID: 2, + foreignID: 2, IsMergeRequest: false, }, }, @@ -214,8 +214,8 @@ func TestGitlabDownloadRepo(t *testing.T) { comments, _, err := downloader.GetComments(base.GetCommentOptions{ Context: gitlabIssueContext{ - OriginalID: 2, - MigratedID: 2, + foreignID: 2, + localID: 2, IsMergeRequest: false, }, }) @@ -303,14 +303,14 @@ func TestGitlabDownloadRepo(t *testing.T) { MergedTime: nil, MergeCommitSHA: "", Context: gitlabIssueContext{ - OriginalID: 2, - MigratedID: 4, + foreignID: 2, + localID: 4, IsMergeRequest: true, }, }, }, prs) - rvs, err := downloader.GetReviews(1) + rvs, err := downloader.GetReviews(base.BasicIssueContext{ID: 1}) assert.NoError(t, err) if assert.Len(t, rvs, 2) { for i := range rvs { @@ -327,7 +327,7 @@ func TestGitlabDownloadRepo(t *testing.T) { } } } - rvs, err = downloader.GetReviews(2) + rvs, err = downloader.GetReviews(base.BasicIssueContext{ID: 2}) assert.NoError(t, err) if assert.Len(t, prs, 1) { assert.EqualValues(t, 4575606, rvs[0].ReviewerID) diff --git a/modules/migrations/gogs.go b/modules/migrations/gogs.go index cd2b63101fdc..25dc66fdeacb 100644 --- a/modules/migrations/gogs.go +++ b/modules/migrations/gogs.go @@ -228,14 +228,9 @@ func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, // GetComments returns comments according issueNumber func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - issueNumber, ok := opts.Context.(int64) - if !ok { - return nil, false, fmt.Errorf("unexpected context: %+v", opts.Context) - } - var allComments = make([]*base.Comment, 0, 100) - comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, issueNumber) + comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID()) if err != nil { return nil, false, fmt.Errorf("error while listing repos: %v", err) } @@ -244,7 +239,7 @@ func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comme continue } allComments = append(allComments, &base.Comment{ - IssueIndex: issueNumber, + IssueIndex: opts.Context.LocalID(), PosterID: comment.Poster.ID, PosterName: comment.Poster.Login, PosterEmail: comment.Poster.Email, @@ -306,7 +301,7 @@ func convertGogsIssue(issue *gogs.Issue) *base.Issue { Created: issue.Created, Labels: labels, Closed: closed, - Context: issue.Index, + Context: base.BasicIssueContext{ID: issue.Index}, } } diff --git a/modules/migrations/gogs_test.go b/modules/migrations/gogs_test.go index 201bf6b9f7bb..79c61faee568 100644 --- a/modules/migrations/gogs_test.go +++ b/modules/migrations/gogs_test.go @@ -104,7 +104,7 @@ func TestGogsDownloadRepo(t *testing.T) { // downloader.GetComments() comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: 1, + Context: base.BasicIssueContext{ID: 1}, }) assert.NoError(t, err) assert.Len(t, comments, 1) diff --git a/modules/migrations/onedev.go b/modules/migrations/onedev.go index 83e4b56430a3..443c998c1bde 100644 --- a/modules/migrations/onedev.go +++ b/modules/migrations/onedev.go @@ -243,11 +243,19 @@ func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) { } type onedevIssueContext struct { - OriginalID int64 - MigratedID int64 + foreignID int64 + localID int64 IsPullRequest bool } +func (c onedevIssueContext) LocalID() int64 { + return c.localID +} + +func (c onedevIssueContext) ForeignID() int64 { + return c.foreignID +} + // GetIssues returns issues according start and limit, perPage is not supported func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { rawIssues := make([]struct { @@ -292,8 +300,8 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er Created: issue.SubmitDate, //Closed: closed, Context: onedevIssueContext{ - OriginalID: issue.ID, - MigratedID: issue.Number, + foreignID: issue.ID, + localID: issue.Number, IsPullRequest: false, }, }) @@ -321,9 +329,9 @@ func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com var endpoint string if context.IsPullRequest { - endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", context.OriginalID) + endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", context.ForeignID()) } else { - endpoint = fmt.Sprintf("/api/issues/%d/comments", context.OriginalID) + endpoint = fmt.Sprintf("/api/issues/%d/comments", context.ForeignID()) } err := d.callAPI( @@ -342,9 +350,9 @@ func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com }, 0, 100) if context.IsPullRequest { - endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", context.OriginalID) + endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", context.ForeignID()) } else { - endpoint = fmt.Sprintf("/api/issues/%d/changes", context.OriginalID) + endpoint = fmt.Sprintf("/api/issues/%d/changes", context.ForeignID()) } err = d.callAPI( @@ -363,7 +371,7 @@ func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com } poster := d.tryGetUser(comment.UserID) comments = append(comments, &base.Comment{ - IssueIndex: context.MigratedID, + IssueIndex: context.LocalID(), PosterID: poster.ID, PosterName: poster.Name, PosterEmail: poster.Email, @@ -387,7 +395,7 @@ func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com poster := d.tryGetUser(change.UserID) comments = append(comments, &base.Comment{ - IssueIndex: context.MigratedID, + IssueIndex: context.LocalID(), PosterID: poster.ID, PosterName: poster.Name, PosterEmail: poster.Email, @@ -470,8 +478,8 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque RepoName: d.repoName, }, Context: onedevIssueContext{ - OriginalID: pr.ID, - MigratedID: number, + foreignID: pr.ID, + localID: number, IsPullRequest: true, }, }) @@ -481,12 +489,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque } // GetReviews returns requested pull requests reviews -func (d *OneDevDownloader) GetReviews(context interface{}) ([]*base.Review, error) { - issueContext, ok := context.(onedevIssueContext) - if !ok { - return nil, fmt.Errorf("unexpected context: %+v", context) - } - +func (d *OneDevDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { rawReviews := make([]struct { ID int64 `json:"id"` UserID int64 `json:"userId"` @@ -498,7 +501,7 @@ func (d *OneDevDownloader) GetReviews(context interface{}) ([]*base.Review, erro }, 0, 100) err := d.callAPI( - fmt.Sprintf("/api/pull-requests/%d/reviews", issueContext.OriginalID), + fmt.Sprintf("/api/pull-requests/%d/reviews", context.ForeignID()), nil, &rawReviews, ) @@ -522,7 +525,7 @@ func (d *OneDevDownloader) GetReviews(context interface{}) ([]*base.Review, erro poster := d.tryGetUser(review.UserID) reviews = append(reviews, &base.Review{ - IssueIndex: issueContext.MigratedID, + IssueIndex: context.LocalID(), ReviewerID: poster.ID, ReviewerName: poster.Name, CreatedAt: time.Now(), diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 432941f913f8..347a7f2f1e4e 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -209,20 +209,15 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, return nil, false, err } for _, issue := range issues { - issue.Context = issue.Number + issue.Context = base.BasicIssueContext{ID: issue.Number} } return issues, true, nil } // GetComments returns comments according issueNumber func (r *RepositoryRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { - issueNumber, ok := opts.Context.(int64) - if !ok { - return nil, false, fmt.Errorf("unexpected context: %+v", opts.Context) - } - var comments = make([]*base.Comment, 0, 10) - p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber)) + p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", opts.Context.ForeignID())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { @@ -266,20 +261,15 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq } for _, pr := range pulls { pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL) - pr.Context = pr.Number + pr.Context = base.BasicIssueContext{ID: pr.Number} } return pulls, true, nil } // GetReviews returns pull requests review -func (r *RepositoryRestorer) GetReviews(context interface{}) ([]*base.Review, error) { - issueNumber, ok := context.(int64) - if !ok { - return nil, fmt.Errorf("unexpected context: %+v", context) - } - +func (r *RepositoryRestorer) GetReviews(context base.IssueContext) ([]*base.Review, error) { var reviews = make([]*base.Review, 0, 10) - p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", issueNumber)) + p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", context.ForeignID())) _, err := os.Stat(p) if err != nil { if os.IsNotExist(err) { From 86d648fa9fbee9b956a14f25b82ba0e4b3894e50 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 12 Aug 2021 14:43:41 +0000 Subject: [PATCH 07/14] Added suggestions. --- modules/migrations/base/issue.go | 8 +++----- modules/migrations/gitea_downloader.go | 4 ++-- modules/migrations/gitea_downloader_test.go | 4 ++-- modules/migrations/github.go | 4 ++-- modules/migrations/github_test.go | 14 +++++++------- modules/migrations/gitlab_test.go | 4 ++-- modules/migrations/gogs.go | 2 +- modules/migrations/gogs_test.go | 2 +- modules/migrations/onedev.go | 2 +- modules/migrations/restore.go | 4 ++-- 10 files changed, 23 insertions(+), 25 deletions(-) diff --git a/modules/migrations/base/issue.go b/modules/migrations/base/issue.go index 42fd312db6fd..7addd1336a28 100644 --- a/modules/migrations/base/issue.go +++ b/modules/migrations/base/issue.go @@ -14,18 +14,16 @@ type IssueContext interface { } // BasicIssueContext is a 1:1 mapping between local and foreign ids. -type BasicIssueContext struct { - ID int64 -} +type BasicIssueContext int64 // LocalID gets the local id. func (c BasicIssueContext) LocalID() int64 { - return c.ID + return int64(c) } // ForeignID gets the foreign id. func (c BasicIssueContext) ForeignID() int64 { - return c.ID + return int64(c) } // Issue is a standard issue information diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go index f2eec1200c74..d46b09eda80a 100644 --- a/modules/migrations/gitea_downloader.go +++ b/modules/migrations/gitea_downloader.go @@ -424,7 +424,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err Labels: labels, Assignees: assignees, IsLocked: issue.IsLocked, - Context: base.BasicIssueContext{ID: issue.Index}, + Context: base.BasicIssueContext(issue.Index), }) } @@ -596,7 +596,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques RepoName: g.repoName, OwnerName: g.repoOwner, }, - Context: base.BasicIssueContext{ID: pr.Index}, + Context: base.BasicIssueContext(pr.Index), }) } diff --git a/modules/migrations/gitea_downloader_test.go b/modules/migrations/gitea_downloader_test.go index b9652754539c..f9ee5c9ffce8 100644 --- a/modules/migrations/gitea_downloader_test.go +++ b/modules/migrations/gitea_downloader_test.go @@ -225,7 +225,7 @@ func TestGiteaDownloadRepo(t *testing.T) { }, issues[1]) comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: base.BasicIssueContext{ID: 4}, + Context: base.BasicIssueContext(4), }) assert.NoError(t, err) assert.Len(t, comments, 2) @@ -299,7 +299,7 @@ func TestGiteaDownloadRepo(t *testing.T) { PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch", }, prs[1]) - reviews, err := downloader.GetReviews(base.BasicIssueContext{ID: 7}) + reviews, err := downloader.GetReviews(base.BasicIssueContext(7)) assert.NoError(t, err) if assert.Len(t, reviews, 3) { assert.EqualValues(t, 689, reviews[0].ReviewerID) diff --git a/modules/migrations/github.go b/modules/migrations/github.go index 3e7ec2ca56a6..ba92bc7e9511 100644 --- a/modules/migrations/github.go +++ b/modules/migrations/github.go @@ -445,7 +445,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, Reactions: reactions, Closed: issue.ClosedAt, IsLocked: *issue.Locked, - Context: base.BasicIssueContext{ID: int64(*issue.Number)}, + Context: base.BasicIssueContext(*issue.Number), }) } @@ -736,7 +736,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq }, PatchURL: *pr.PatchURL, Reactions: reactions, - Context: base.BasicIssueContext{ID: int64(*pr.Number)}, + Context: base.BasicIssueContext(*pr.Number), }) } diff --git a/modules/migrations/github_test.go b/modules/migrations/github_test.go index 4b5f1ef6785c..0d8a581d44f7 100644 --- a/modules/migrations/github_test.go +++ b/modules/migrations/github_test.go @@ -185,7 +185,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, Closed: &closed1, - Context: base.BasicIssueContext{ID: 1}, + Context: base.BasicIssueContext(1), }, { Number: 2, @@ -237,13 +237,13 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, Closed: &closed2, - Context: base.BasicIssueContext{ID: 2}, + Context: base.BasicIssueContext(2), }, }, issues) // downloader.GetComments() comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: base.BasicIssueContext{ID: 2}, + Context: base.BasicIssueContext(2), }) assert.NoError(t, err) assert.Len(t, comments, 2) @@ -319,7 +319,7 @@ func TestGitHubDownloadRepo(t *testing.T) { Merged: true, MergedTime: &merged1, MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2", - Context: base.BasicIssueContext{ID: 3}, + Context: base.BasicIssueContext(3), }, { Number: 4, @@ -366,11 +366,11 @@ func TestGitHubDownloadRepo(t *testing.T) { Content: "+1", }, }, - Context: base.BasicIssueContext{ID: 4}, + Context: base.BasicIssueContext(4), }, }, prs) - reviews, err := downloader.GetReviews(base.BasicIssueContext{ID: 3}) + reviews, err := downloader.GetReviews(base.BasicIssueContext(3)) assert.NoError(t, err) assert.EqualValues(t, []*base.Review{ { @@ -402,7 +402,7 @@ func TestGitHubDownloadRepo(t *testing.T) { }, }, reviews) - reviews, err = downloader.GetReviews(base.BasicIssueContext{ID: 4}) + reviews, err = downloader.GetReviews(base.BasicIssueContext(4)) assert.NoError(t, err) assert.EqualValues(t, []*base.Review{ { diff --git a/modules/migrations/gitlab_test.go b/modules/migrations/gitlab_test.go index 8e0f551995bf..b1708f53aae2 100644 --- a/modules/migrations/gitlab_test.go +++ b/modules/migrations/gitlab_test.go @@ -310,7 +310,7 @@ func TestGitlabDownloadRepo(t *testing.T) { }, }, prs) - rvs, err := downloader.GetReviews(base.BasicIssueContext{ID: 1}) + rvs, err := downloader.GetReviews(base.BasicIssueContext(1)) assert.NoError(t, err) if assert.Len(t, rvs, 2) { for i := range rvs { @@ -327,7 +327,7 @@ func TestGitlabDownloadRepo(t *testing.T) { } } } - rvs, err = downloader.GetReviews(base.BasicIssueContext{ID: 2}) + rvs, err = downloader.GetReviews(base.BasicIssueContext(2)) assert.NoError(t, err) if assert.Len(t, prs, 1) { assert.EqualValues(t, 4575606, rvs[0].ReviewerID) diff --git a/modules/migrations/gogs.go b/modules/migrations/gogs.go index da644d07be6e..4af3952070ee 100644 --- a/modules/migrations/gogs.go +++ b/modules/migrations/gogs.go @@ -301,7 +301,7 @@ func convertGogsIssue(issue *gogs.Issue) *base.Issue { Created: issue.Created, Labels: labels, Closed: closed, - Context: base.BasicIssueContext{ID: issue.Index}, + Context: base.BasicIssueContext(issue.Index), } } diff --git a/modules/migrations/gogs_test.go b/modules/migrations/gogs_test.go index 79c61faee568..febff2b6596c 100644 --- a/modules/migrations/gogs_test.go +++ b/modules/migrations/gogs_test.go @@ -104,7 +104,7 @@ func TestGogsDownloadRepo(t *testing.T) { // downloader.GetComments() comments, _, err := downloader.GetComments(base.GetCommentOptions{ - Context: base.BasicIssueContext{ID: 1}, + Context: base.BasicIssueContext(1), }) assert.NoError(t, err) assert.Len(t, comments, 1) diff --git a/modules/migrations/onedev.go b/modules/migrations/onedev.go index 443c998c1bde..bc40501b595b 100644 --- a/modules/migrations/onedev.go +++ b/modules/migrations/onedev.go @@ -126,7 +126,7 @@ func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, u.RawQuery = query.Encode() } - req, err := http.NewRequest("GET", u.String(), nil) + req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil) if err != nil { return err } diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go index 347a7f2f1e4e..6287d601c2a0 100644 --- a/modules/migrations/restore.go +++ b/modules/migrations/restore.go @@ -209,7 +209,7 @@ func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, return nil, false, err } for _, issue := range issues { - issue.Context = base.BasicIssueContext{ID: issue.Number} + issue.Context = base.BasicIssueContext(issue.Number) } return issues, true, nil } @@ -261,7 +261,7 @@ func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullReq } for _, pr := range pulls { pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL) - pr.Context = base.BasicIssueContext{ID: pr.Number} + pr.Context = base.BasicIssueContext(pr.Number) } return pulls, true, nil } From eeb652b7393c4f74348b858ca36359a5dccc7d1f Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 12 Aug 2021 15:44:18 +0000 Subject: [PATCH 08/14] Fixed broken pull requests. --- modules/migrations/onedev.go | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/modules/migrations/onedev.go b/modules/migrations/onedev.go index bc40501b595b..85fb0aaad89e 100644 --- a/modules/migrations/onedev.go +++ b/modules/migrations/onedev.go @@ -13,11 +13,10 @@ import ( "strings" "time" + "code.gitea.io/gitea/modules/json" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/migrations/base" "code.gitea.io/gitea/modules/structs" - - jsoniter "github.com/json-iterator/go" ) var ( @@ -137,7 +136,7 @@ func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, } defer resp.Body.Close() - decoder := jsoniter.NewDecoder(resp.Body) + decoder := json.NewDecoder(resp.Body) return decoder.Decode(&result) } @@ -441,6 +440,21 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests)) for _, pr := range rawPullRequests { + var mergePreview struct { + TargetHeadCommitHash string `json:"targetHeadCommitHash"` + HeadCommitHash string `json:"headCommitHash"` + MergeStrategy string `json:"mergeStrategy"` + MergeCommitHash string `json:"mergeCommitHash"` + } + err := d.callAPI( + fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID), + nil, + &mergePreview, + ) + if err != nil { + return nil, false, err + } + state := "open" merged := false var closeTime *time.Time @@ -469,12 +483,12 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque MergedTime: mergedTime, Head: base.PullRequestBranch{ Ref: pr.SourceBranch, - SHA: pr.BaseCommitHash, + SHA: mergePreview.HeadCommitHash, RepoName: d.repoName, }, Base: base.PullRequestBranch{ Ref: pr.TargetBranch, - SHA: "", + SHA: mergePreview.TargetHeadCommitHash, RepoName: d.repoName, }, Context: onedevIssueContext{ From 3afbe25c6598744dca7a180a3f5de771c73e4a3d Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 12 Aug 2021 16:13:13 +0000 Subject: [PATCH 09/14] Migrate labels. --- modules/migrations/onedev.go | 50 ++++++++++++++++++++++++++++-- templates/repo/migrate/onedev.tmpl | 8 +++-- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/modules/migrations/onedev.go b/modules/migrations/onedev.go index 85fb0aaad89e..45b94acc78e0 100644 --- a/modules/migrations/onedev.go +++ b/modules/migrations/onedev.go @@ -238,7 +238,32 @@ func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { // GetLabels returns labels func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) { - return nil, nil + return []*base.Label{ + { + Name: "Bug", + Color: "f64e60", + }, + { + Name: "Build Failure", + Color: "f64e60", + }, + { + Name: "Discussion", + Color: "8950fc", + }, + { + Name: "Improvement", + Color: "1bc5bd", + }, + { + Name: "New Feature", + Color: "1bc5bd", + }, + { + Name: "Support Request", + Color: "8950fc", + }, + }, nil } type onedevIssueContext struct { @@ -283,6 +308,27 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er issues := make([]*base.Issue, 0, len(rawIssues)) for _, issue := range rawIssues { + fields := make([]struct { + Name string `json:"name"` + Value string `json:"value"` + }, 0, 10) + err := d.callAPI( + fmt.Sprintf("/api/issues/%d/fields", issue.ID), + nil, + &fields, + ) + if err != nil { + return nil, false, err + } + + var label *base.Label + for _, field := range fields { + if field.Name == "Type" { + label = &base.Label{Name: field.Value} + break + } + } + state := strings.ToLower(issue.State) if state == "released" { state = "closed" @@ -297,7 +343,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er Milestone: d.milestoneMap[issue.MilestoneID], State: state, Created: issue.SubmitDate, - //Closed: closed, + Labels: []*base.Label{label}, Context: onedevIssueContext{ foreignID: issue.ID, localID: issue.Number, diff --git a/templates/repo/migrate/onedev.tmpl b/templates/repo/migrate/onedev.tmpl index 03f9c6fd38e6..def366f9d8da 100644 --- a/templates/repo/migrate/onedev.tmpl +++ b/templates/repo/migrate/onedev.tmpl @@ -38,12 +38,16 @@
- - + +
+
+ + +
From ee381c09db27a798cb1019c227484c5a82be6fc5 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Thu, 12 Aug 2021 20:21:12 +0000 Subject: [PATCH 10/14] Added tests. --- modules/migrations/onedev.go | 7 +- modules/migrations/onedev_test.go | 167 ++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+), 6 deletions(-) create mode 100644 modules/migrations/onedev_test.go diff --git a/modules/migrations/onedev.go b/modules/migrations/onedev.go index 45b94acc78e0..3fc0a47a264b 100644 --- a/modules/migrations/onedev.go +++ b/modules/migrations/onedev.go @@ -195,8 +195,6 @@ func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { endpoint := fmt.Sprintf("/api/projects/%d/milestones", d.repoID) - t := time.Now() - var milestones = make([]*base.Milestone, 0, 100) offset := 0 for { @@ -218,7 +216,7 @@ func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { for _, milestone := range rawMilestones { d.milestoneMap[milestone.ID] = milestone.Name - closed := &t + closed := milestone.DueDate if !milestone.Closed { closed = nil } @@ -227,8 +225,6 @@ func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) { Title: milestone.Name, Description: milestone.Description, Deadline: milestone.DueDate, - Created: t, - Updated: &t, Closed: closed, }) } @@ -588,7 +584,6 @@ func (d *OneDevDownloader) GetReviews(context base.IssueContext) ([]*base.Review IssueIndex: context.LocalID(), ReviewerID: poster.ID, ReviewerName: poster.Name, - CreatedAt: time.Now(), Content: content, State: state, }) diff --git a/modules/migrations/onedev_test.go b/modules/migrations/onedev_test.go new file mode 100644 index 000000000000..f3ec7e540afd --- /dev/null +++ b/modules/migrations/onedev_test.go @@ -0,0 +1,167 @@ +// Copyright 2021 The Gitea Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package migrations + +import ( + "context" + "fmt" + "net/http" + "net/url" + "testing" + "time" + + "code.gitea.io/gitea/modules/migrations/base" + + "github.com/stretchr/testify/assert" +) + +func TestOneDevDownloadRepo(t *testing.T) { + resp, err := http.Get("https://code.onedev.io/projects/go-gitea-test_repo") + if err != nil || resp.StatusCode != 200 { + t.Skipf("Can't access test repo, skipping %s", t.Name()) + } + + u, _ := url.Parse("https://code.onedev.io") + downloader := NewOneDevDownloader(context.Background(), u, "", "", "go-gitea-test_repo") + if err != nil { + t.Fatal(fmt.Sprintf("NewOneDevDownloader is nil: %v", err)) + } + repo, err := downloader.GetRepoInfo() + assert.NoError(t, err) + assert.EqualValues(t, &base.Repository{ + Name: "go-gitea-test_repo", + Owner: "", + Description: "Test repository for testing migration from OpenDev to gitea", + CloneURL: "https://code.onedev.io/go-gitea-test_repo", + OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo", + }, repo) + + milestones, err := downloader.GetMilestones() + assert.NoError(t, err) + assert.Len(t, milestones, 2) + assertMilestoneEqual(t, "", "1.0.0", + "2021-05-04 00:00:00.000 +0000 UTC", + "", + "", + "2021-05-04 00:00:00.000 +0000 UTC", + "", milestones[0]) + assertMilestoneEqual(t, "next things?", "1.1.0", + "", + "", + "", + "", + "", milestones[1]) + + labels, err := downloader.GetLabels() + assert.NoError(t, err) + assert.Len(t, labels, 6) + + issues, isEnd, err := downloader.GetIssues(1, 2) + assert.NoError(t, err) + assert.Len(t, issues, 2) + assert.False(t, isEnd) + + assert.EqualValues(t, []*base.Issue{ + { + Number: 4, + Title: "Hi there", + Content: "an issue not assigned to a milestone", + PosterName: "User 336", + State: "open", + Created: time.Date(2021, 8, 9, 22, 56, 16, 734000000, time.UTC), + Updated: time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), + Labels: []*base.Label{ + { + Name: "Improvement", + }, + }, + Context: onedevIssueContext{ + foreignID: 398, + localID: 4, + IsPullRequest: false, + }, + }, + { + Number: 3, + Title: "Add an awesome feature", + Content: "just another issue to test against", + PosterName: "User 336", + State: "open", + Milestone: "1.1.0", + Created: time.Date(2021, 8, 9, 22, 55, 49, 878000000, time.UTC), + Updated: time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), + Labels: []*base.Label{ + { + Name: "New Feature", + }, + }, + Context: onedevIssueContext{ + foreignID: 397, + localID: 3, + IsPullRequest: false, + }, + }, + }, issues) + + comments, _, err := downloader.GetComments(base.GetCommentOptions{ + Context: onedevIssueContext{ + foreignID: 398, + localID: 4, + IsPullRequest: false, + }, + }) + assert.NoError(t, err) + assert.Len(t, comments, 1) + assert.EqualValues(t, []*base.Comment{ + { + IssueIndex: 4, + PosterName: "User 336", + Created: time.Date(2021, 8, 9, 22, 56, 31, 128000000, time.UTC), + Updated: time.Date(2021, 8, 9, 22, 56, 31, 128000000, time.UTC), + Content: "it has a comment\r\n\r\nEDIT: that got edited", + }, + }, comments) + + prs, _, err := downloader.GetPullRequests(1, 1) + assert.NoError(t, err) + assert.Len(t, prs, 1) + + assert.EqualValues(t, []*base.PullRequest{ + { + Number: 5, + Title: "Pull to add a new file", + Content: "just do some git stuff", + PosterName: "User 336", + State: "open", + Created: time.Date(2021, 8, 9, 23, 1, 16, 025000000, time.UTC), + Updated: time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), + Head: base.PullRequestBranch{ + Ref: "branch-for-a-pull", + SHA: "343deffe3526b9bc84e873743ff7f6e6d8b827c0", + RepoName: "go-gitea-test_repo", + }, + Base: base.PullRequestBranch{ + Ref: "master", + SHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2", + RepoName: "go-gitea-test_repo", + }, + Context: onedevIssueContext{ + foreignID: 186, + localID: 5, + IsPullRequest: true, + }, + }, + }, prs) + + rvs, err := downloader.GetReviews(onedevIssueContext{ + foreignID: 186, + localID: 5, + }) + assert.NoError(t, err) + assert.Len(t, rvs, 1) + assert.Equal(t, "User 317", rvs[0].ReviewerName) + assert.Equal(t, "PENDING", rvs[0].State) + assert.Empty(t, rvs[0].Content) +} From b78e222c7b29269d7ba3cce95483a27f5333afd0 Mon Sep 17 00:00:00 2001 From: 6543 <6543@obermui.de> Date: Fri, 13 Aug 2021 00:51:15 +0200 Subject: [PATCH 11/14] Update modules/migrations/onedev_test.go --- modules/migrations/onedev_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/migrations/onedev_test.go b/modules/migrations/onedev_test.go index f3ec7e540afd..fae9e2d7707b 100644 --- a/modules/migrations/onedev_test.go +++ b/modules/migrations/onedev_test.go @@ -33,7 +33,7 @@ func TestOneDevDownloadRepo(t *testing.T) { assert.EqualValues(t, &base.Repository{ Name: "go-gitea-test_repo", Owner: "", - Description: "Test repository for testing migration from OpenDev to gitea", + Description: "Test repository for testing migration from OneDev to gitea", CloneURL: "https://code.onedev.io/go-gitea-test_repo", OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo", }, repo) From 0602dc8b32fdb4a2f4c8a0bc0d67e5968fa67e89 Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Fri, 13 Aug 2021 07:47:00 +0200 Subject: [PATCH 12/14] Use unix constructor. --- modules/migrations/gitlab_test.go | 2 ++ modules/migrations/onedev.go | 2 ++ modules/migrations/onedev_test.go | 52 ++++++++++++++++--------------- 3 files changed, 31 insertions(+), 25 deletions(-) diff --git a/modules/migrations/gitlab_test.go b/modules/migrations/gitlab_test.go index b1708f53aae2..0cb596fd9484 100644 --- a/modules/migrations/gitlab_test.go +++ b/modules/migrations/gitlab_test.go @@ -155,6 +155,7 @@ func TestGitlabDownloadRepo(t *testing.T) { Closed: &closed1, Context: gitlabIssueContext{ foreignID: 1, + localID: 1, IsMergeRequest: false, }, }, @@ -207,6 +208,7 @@ func TestGitlabDownloadRepo(t *testing.T) { Closed: &closed2, Context: gitlabIssueContext{ foreignID: 2, + localID: 2, IsMergeRequest: false, }, }, diff --git a/modules/migrations/onedev.go b/modules/migrations/onedev.go index 3fc0a47a264b..66e508adf3d1 100644 --- a/modules/migrations/onedev.go +++ b/modules/migrations/onedev.go @@ -339,6 +339,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er Milestone: d.milestoneMap[issue.MilestoneID], State: state, Created: issue.SubmitDate, + Updated: issue.SubmitDate, Labels: []*base.Label{label}, Context: onedevIssueContext{ foreignID: issue.ID, @@ -520,6 +521,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque Content: pr.Description, State: state, Created: pr.SubmitDate, + Updated: pr.SubmitDate, Closed: closeTime, Merged: merged, MergedTime: mergedTime, diff --git a/modules/migrations/onedev_test.go b/modules/migrations/onedev_test.go index fae9e2d7707b..48d56c3e22b3 100644 --- a/modules/migrations/onedev_test.go +++ b/modules/migrations/onedev_test.go @@ -41,18 +41,18 @@ func TestOneDevDownloadRepo(t *testing.T) { milestones, err := downloader.GetMilestones() assert.NoError(t, err) assert.Len(t, milestones, 2) - assertMilestoneEqual(t, "", "1.0.0", - "2021-05-04 00:00:00.000 +0000 UTC", - "", - "", - "2021-05-04 00:00:00.000 +0000 UTC", - "", milestones[0]) - assertMilestoneEqual(t, "next things?", "1.1.0", - "", - "", - "", - "", - "", milestones[1]) + deadline := time.Unix(1620086400, 0) + assert.EqualValues(t, []*base.Milestone{ + { + Title: "1.0.0", + Deadline: &deadline, + Closed: &deadline, + }, + { + Title: "1.1.0", + Description: "next things?", + }, + }, milestones) labels, err := downloader.GetLabels() assert.NoError(t, err) @@ -62,7 +62,6 @@ func TestOneDevDownloadRepo(t *testing.T) { assert.NoError(t, err) assert.Len(t, issues, 2) assert.False(t, isEnd) - assert.EqualValues(t, []*base.Issue{ { Number: 4, @@ -70,8 +69,8 @@ func TestOneDevDownloadRepo(t *testing.T) { Content: "an issue not assigned to a milestone", PosterName: "User 336", State: "open", - Created: time.Date(2021, 8, 9, 22, 56, 16, 734000000, time.UTC), - Updated: time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), + Created: time.Unix(1628549776, 734000000), + Updated: time.Unix(1628549776, 734000000), Labels: []*base.Label{ { Name: "Improvement", @@ -90,8 +89,8 @@ func TestOneDevDownloadRepo(t *testing.T) { PosterName: "User 336", State: "open", Milestone: "1.1.0", - Created: time.Date(2021, 8, 9, 22, 55, 49, 878000000, time.UTC), - Updated: time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), + Created: time.Unix(1628549749, 878000000), + Updated: time.Unix(1628549749, 878000000), Labels: []*base.Label{ { Name: "New Feature", @@ -118,8 +117,8 @@ func TestOneDevDownloadRepo(t *testing.T) { { IssueIndex: 4, PosterName: "User 336", - Created: time.Date(2021, 8, 9, 22, 56, 31, 128000000, time.UTC), - Updated: time.Date(2021, 8, 9, 22, 56, 31, 128000000, time.UTC), + Created: time.Unix(1628549791, 128000000), + Updated: time.Unix(1628549791, 128000000), Content: "it has a comment\r\n\r\nEDIT: that got edited", }, }, comments) @@ -127,7 +126,6 @@ func TestOneDevDownloadRepo(t *testing.T) { prs, _, err := downloader.GetPullRequests(1, 1) assert.NoError(t, err) assert.Len(t, prs, 1) - assert.EqualValues(t, []*base.PullRequest{ { Number: 5, @@ -135,8 +133,8 @@ func TestOneDevDownloadRepo(t *testing.T) { Content: "just do some git stuff", PosterName: "User 336", State: "open", - Created: time.Date(2021, 8, 9, 23, 1, 16, 025000000, time.UTC), - Updated: time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), + Created: time.Unix(1628550076, 25000000), + Updated: time.Unix(1628550076, 25000000), Head: base.PullRequestBranch{ Ref: "branch-for-a-pull", SHA: "343deffe3526b9bc84e873743ff7f6e6d8b827c0", @@ -161,7 +159,11 @@ func TestOneDevDownloadRepo(t *testing.T) { }) assert.NoError(t, err) assert.Len(t, rvs, 1) - assert.Equal(t, "User 317", rvs[0].ReviewerName) - assert.Equal(t, "PENDING", rvs[0].State) - assert.Empty(t, rvs[0].Content) + assert.EqualValues(t, []*base.Review{ + { + IssueIndex: 5, + ReviewerName: "User 317", + State: "PENDING", + }, + }, rvs) } From b6ba145c5d579806736a1727c256cc210710375a Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Sat, 14 Aug 2021 10:10:53 +0000 Subject: [PATCH 13/14] Updated comments. --- modules/migrations/onedev.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/migrations/onedev.go b/modules/migrations/onedev.go index 66e508adf3d1..e60265895fbc 100644 --- a/modules/migrations/onedev.go +++ b/modules/migrations/onedev.go @@ -140,7 +140,7 @@ func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, return decoder.Decode(&result) } -// GetRepoInfo returns a repository information +// GetRepoInfo returns repository information func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) { info := make([]struct { ID int64 `json:"id"` @@ -276,7 +276,7 @@ func (c onedevIssueContext) ForeignID() int64 { return c.foreignID } -// GetIssues returns issues according start and limit, perPage is not supported +// GetIssues returns issues func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) { rawIssues := make([]struct { ID int64 `json:"id"` @@ -356,7 +356,7 @@ func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, er return issues, len(issues) == 0, nil } -// GetComments returns requested comments +// GetComments returns comments func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) { context, ok := opts.Context.(onedevIssueContext) if !ok { @@ -447,10 +447,10 @@ func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Com }) } - return comments, false, nil + return comments, true, nil } -// GetPullRequests returns requested pull requests +// GetPullRequests returns pull requests func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) { rawPullRequests := make([]struct { ID int64 `json:"id"` @@ -546,7 +546,7 @@ func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullReque return pullRequests, len(pullRequests) == 0, nil } -// GetReviews returns requested pull requests reviews +// GetReviews returns pull requests reviews func (d *OneDevDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) { rawReviews := make([]struct { ID int64 `json:"id"` From 94b1dcc13488e5f0a11cfe4ef919382e40d6056f Mon Sep 17 00:00:00 2001 From: KN4CK3R Date: Tue, 17 Aug 2021 20:08:24 +0000 Subject: [PATCH 14/14] Added missing service. --- modules/convert/utils.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/modules/convert/utils.go b/modules/convert/utils.go index 69de306689e0..a0463d7b1006 100644 --- a/modules/convert/utils.go +++ b/modules/convert/utils.go @@ -33,6 +33,8 @@ func ToGitServiceType(value string) structs.GitServiceType { return structs.GitlabService case "gogs": return structs.GogsService + case "onedev": + return structs.OneDevService default: return structs.PlainGitService }