Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add migrate from OneDev #16356

Merged
merged 26 commits into from
Aug 21, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
8a0ff92
Use context to simplify logic.
KN4CK3R Jul 5, 2021
c0661b9
Added migration from OneDev.
KN4CK3R Jul 6, 2021
e17edc0
Use "Pull Requests".
KN4CK3R Jul 6, 2021
1394141
Fixed test.
KN4CK3R Jul 6, 2021
15b903e
Changed comments.
KN4CK3R Jul 7, 2021
5962cf6
Use typed context.
KN4CK3R Aug 8, 2021
2febe35
Merge branch 'main' of https://github.com/go-gitea/gitea into feature…
KN4CK3R Aug 8, 2021
86d648f
Added suggestions.
KN4CK3R Aug 12, 2021
f8ab470
Merge branch 'main' of https://github.com/go-gitea/gitea into feature…
KN4CK3R Aug 12, 2021
eeb652b
Fixed broken pull requests.
KN4CK3R Aug 12, 2021
3afbe25
Migrate labels.
KN4CK3R Aug 12, 2021
ee381c0
Added tests.
KN4CK3R Aug 12, 2021
b78e222
Update modules/migrations/onedev_test.go
6543 Aug 12, 2021
2ce860b
Merge branch 'main' into feature-migrate-onedev
6543 Aug 12, 2021
0602dc8
Use unix constructor.
KN4CK3R Aug 13, 2021
2580cae
Merge branch 'main' into feature-migrate-onedev
6543 Aug 13, 2021
b6ba145
Updated comments.
KN4CK3R Aug 14, 2021
23212aa
Merge branch 'main' of https://github.com/go-gitea/gitea into feature…
KN4CK3R Aug 15, 2021
94b1dcc
Added missing service.
KN4CK3R Aug 17, 2021
2b5b150
Merge branch 'main' of https://github.com/go-gitea/gitea into feature…
KN4CK3R Aug 17, 2021
783c378
Merge branch 'main' of https://github.com/go-gitea/gitea into feature…
KN4CK3R Aug 18, 2021
f34557e
Merge branch 'main' into feature-migrate-onedev
6543 Aug 18, 2021
e870a16
Merge branch 'main' into feature-migrate-onedev
6543 Aug 20, 2021
76dd48e
Merge branch 'main' into feature-migrate-onedev
techknowlogick Aug 21, 2021
91d0141
Merge branch 'main' into feature-migrate-onedev
6543 Aug 21, 2021
a4ce603
Merge branch 'main' into feature-migrate-onedev
6543 Aug 21, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions modules/convert/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ func ToGitServiceType(value string) structs.GitServiceType {
return structs.GitlabService
case "gogs":
return structs.GogsService
case "onedev":
return structs.OneDevService
default:
return structs.PlainGitService
}
Expand Down
8 changes: 4 additions & 4 deletions modules/migrations/base/downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,9 @@ import (

// GetCommentOptions represents an options for get comment
type GetCommentOptions struct {
IssueNumber int64
Page int
PageSize int
Context IssueContext
Page int
PageSize int
}

// Downloader downloads the site repo information
Expand All @@ -30,7 +30,7 @@ type Downloader interface {
GetComments(opts GetCommentOptions) ([]*Comment, bool, error)
SupportGetRepoComments() bool
GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
GetReviews(pullRequestNumber int64) ([]*Review, error)
GetReviews(pullRequestContext IssueContext) ([]*Review, error)
FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
}

Expand Down
20 changes: 20 additions & 0 deletions modules/migrations/base/issue.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,25 @@ package base

import "time"

// IssueContext is used to map between local and foreign issue/PR ids.
type IssueContext interface {
LocalID() int64
ForeignID() int64
}

// BasicIssueContext is a 1:1 mapping between local and foreign ids.
type BasicIssueContext int64

// LocalID gets the local id.
func (c BasicIssueContext) LocalID() int64 {
return int64(c)
}

// ForeignID gets the foreign id.
func (c BasicIssueContext) ForeignID() int64 {
return int64(c)
}

// Issue is a standard issue information
type Issue struct {
Number int64
Expand All @@ -25,4 +44,5 @@ type Issue struct {
Labels []*Label
Reactions []*Reaction
Assignees []string
Context IssueContext `yaml:"-"`
}
4 changes: 2 additions & 2 deletions modules/migrations/base/null_downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
return nil, false, &ErrNotSupported{Entity: "Issues"}
}

// GetComments returns comments according issueNumber
// GetComments returns comments according the options
func (n NullDownloader) GetComments(GetCommentOptions) ([]*Comment, bool, error) {
return nil, false, &ErrNotSupported{Entity: "Comments"}
}
Expand All @@ -61,7 +61,7 @@ func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool
}

// GetReviews returns pull requests review
func (n NullDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
func (n NullDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
return nil, &ErrNotSupported{Entity: "Reviews"}
}

Expand Down
2 changes: 1 addition & 1 deletion modules/migrations/base/pullrequest.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ import (
// PullRequest defines a standard pull request information
type PullRequest struct {
Number int64
OriginalNumber int64 `yaml:"original_number"`
6543 marked this conversation as resolved.
Show resolved Hide resolved
Title string
PosterName string `yaml:"poster_name"`
PosterID int64 `yaml:"poster_id"`
Expand All @@ -34,6 +33,7 @@ type PullRequest struct {
Assignees []string
IsLocked bool `yaml:"is_locked"`
Reactions []*Reaction
Context IssueContext `yaml:"-"`
}

// IsForkPullRequest returns true if the pull request from a forked repository but not the same repository
Expand Down
4 changes: 2 additions & 2 deletions modules/migrations/base/retry_downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -182,14 +182,14 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo
}

// GetReviews returns pull requests reviews
func (d *RetryDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
func (d *RetryDownloader) GetReviews(pullRequestContext IssueContext) ([]*Review, error) {
var (
reviews []*Review
err error
)

err = d.retry(func() error {
reviews, err = d.Downloader.GetReviews(pullRequestNumber)
reviews, err = d.Downloader.GetReviews(pullRequestContext)
return err
})

Expand Down
20 changes: 11 additions & 9 deletions modules/migrations/gitea_downloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -444,6 +444,7 @@ func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, err
Labels: labels,
Assignees: assignees,
IsLocked: issue.IsLocked,
Context: base.BasicIssueContext(issue.Index),
})
}

Expand All @@ -466,26 +467,26 @@ func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comm
default:
}

comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.IssueNumber, gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{
comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{
// PageSize: g.maxPerPage,
// Page: i,
}})
if err != nil {
return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.IssueNumber, err)
return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.Context.ForeignID(), err)
}

for _, comment := range comments {
reactions, err := g.getCommentReactions(comment.ID)
if err != nil {
log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err)
log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)
if err2 := models.CreateRepositoryNotice(
fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.IssueNumber, comment.ID, g.repoOwner, g.repoName, err)); err2 != nil {
fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil {
log.Error("create repository notice failed: ", err2)
}
}

allComments = append(allComments, &base.Comment{
IssueIndex: opts.IssueNumber,
IssueIndex: opts.Context.LocalID(),
PosterID: comment.Poster.ID,
PosterName: comment.Poster.UserName,
PosterEmail: comment.Poster.Email,
Expand Down Expand Up @@ -615,6 +616,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
RepoName: g.repoName,
OwnerName: g.repoOwner,
},
Context: base.BasicIssueContext(pr.Index),
})
}

Expand All @@ -626,7 +628,7 @@ func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullReques
}

// GetReviews returns pull requests review
func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {
func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil {
log.Info("GiteaDownloader: instance to old, skip GetReviews")
return nil, nil
Expand All @@ -642,7 +644,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {
default:
}

prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, index, gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{
prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, context.ForeignID(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{
Page: i,
PageSize: g.maxPerPage,
}})
Expand All @@ -652,7 +654,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {

for _, pr := range prl {

rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, index, pr.ID)
rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, context.ForeignID(), pr.ID)
if err != nil {
return nil, err
}
Expand All @@ -678,7 +680,7 @@ func (g *GiteaDownloader) GetReviews(index int64) ([]*base.Review, error) {

allReviews = append(allReviews, &base.Review{
ID: pr.ID,
IssueIndex: index,
IssueIndex: context.LocalID(),
ReviewerID: pr.Reviewer.ID,
ReviewerName: pr.Reviewer.UserName,
Official: pr.Official,
Expand Down
4 changes: 2 additions & 2 deletions modules/migrations/gitea_downloader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
}, issues)

comments, _, err := downloader.GetComments(base.GetCommentOptions{
IssueNumber: 4,
Context: base.BasicIssueContext(4),
})
assert.NoError(t, err)
assertCommentsEqual(t, []*base.Comment{
Expand Down Expand Up @@ -265,7 +265,7 @@ func TestGiteaDownloadRepo(t *testing.T) {
PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch",
}, prs[1])

reviews, err := downloader.GetReviews(7)
reviews, err := downloader.GetReviews(base.BasicIssueContext(7))
assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{
{
Expand Down
3 changes: 3 additions & 0 deletions modules/migrations/gitea_uploader.go
Original file line number Diff line number Diff line change
Expand Up @@ -609,6 +609,9 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR

// download patch file
err := func() error {
if pr.PatchURL == "" {
6543 marked this conversation as resolved.
Show resolved Hide resolved
return nil
}
// pr.PatchURL maybe a local file
ret, err := uri.Open(pr.PatchURL)
if err != nil {
Expand Down
20 changes: 11 additions & 9 deletions modules/migrations/github.go
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
Closed: issue.ClosedAt,
IsLocked: issue.GetLocked(),
Assignees: assignees,
Context: base.BasicIssueContext(*issue.Number),
})
}

Expand All @@ -441,15 +442,15 @@ func (g *GithubDownloaderV3) SupportGetRepoComments() bool {

// GetComments returns comments according issueNumber
func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
if opts.IssueNumber > 0 {
comments, err := g.getComments(opts.IssueNumber)
if opts.Context != nil {
comments, err := g.getComments(opts.Context)
return comments, false, err
}

return g.GetAllComments(opts.Page, opts.PageSize)
}

func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, error) {
func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*base.Comment, error) {
var (
allComments = make([]*base.Comment, 0, g.maxPerPage)
created = "created"
Expand All @@ -464,7 +465,7 @@ func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, er
}
for {
g.sleep()
comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueNumber), opt)
comments, resp, err := g.client.Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
Expand Down Expand Up @@ -495,7 +496,7 @@ func (g *GithubDownloaderV3) getComments(issueNumber int64) ([]*base.Comment, er
}

allComments = append(allComments, &base.Comment{
IssueIndex: issueNumber,
IssueIndex: issueContext.LocalID(),
PosterID: comment.GetUser().GetID(),
PosterName: comment.GetUser().GetLogin(),
PosterEmail: comment.GetUser().GetEmail(),
Expand Down Expand Up @@ -661,6 +662,7 @@ func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullReq
},
PatchURL: pr.GetPatchURL(),
Reactions: reactions,
Context: base.BasicIssueContext(*pr.Number),
})
}

Expand Down Expand Up @@ -724,28 +726,28 @@ func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullReques
}

// GetReviews returns pull requests review
func (g *GithubDownloaderV3) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Review, error) {
var allReviews = make([]*base.Review, 0, g.maxPerPage)
opt := &github.ListOptions{
PerPage: g.maxPerPage,
}
for {
g.sleep()
reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), opt)
reviews, resp, err := g.client.PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
g.rate = &resp.Rate
for _, review := range reviews {
r := convertGithubReview(review)
r.IssueIndex = pullRequestNumber
r.IssueIndex = context.LocalID()
// retrieve all review comments
opt2 := &github.ListOptions{
PerPage: g.maxPerPage,
}
for {
g.sleep()
reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(pullRequestNumber), review.GetID(), opt2)
reviewComments, resp, err := g.client.PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
Expand Down
8 changes: 5 additions & 3 deletions modules/migrations/github_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ func TestGitHubDownloadRepo(t *testing.T) {

// downloader.GetComments()
comments, _, err := downloader.GetComments(base.GetCommentOptions{
IssueNumber: 2,
Context: base.BasicIssueContext(2),
})
assert.NoError(t, err)
assertCommentsEqual(t, []*base.Comment{
Expand Down Expand Up @@ -286,6 +286,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
Merged: true,
MergedTime: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
Context: base.BasicIssueContext(3),
},
{
Number: 4,
Expand Down Expand Up @@ -332,10 +333,11 @@ func TestGitHubDownloadRepo(t *testing.T) {
Content: "+1",
},
},
Context: base.BasicIssueContext(4),
},
}, prs)

reviews, err := downloader.GetReviews(3)
reviews, err := downloader.GetReviews(base.BasicIssueContext(3))
assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{
{
Expand Down Expand Up @@ -367,7 +369,7 @@ func TestGitHubDownloadRepo(t *testing.T) {
},
}, reviews)

reviews, err = downloader.GetReviews(4)
reviews, err = downloader.GetReviews(base.BasicIssueContext(4))
assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{
{
Expand Down
Loading