Add support to import repository data from an exported data of Github

cmd/restore_repo.go

@@ -23,6 +23,15 @@ var CmdRestoreRepository = cli.Command{
 	Description: "This is a command for restoring the repository data.",
 	Action:      runRestoreRepository,
 	Flags: []cli.Flag{
+		cli.StringFlag{
+			Name:  "data_type, d",
+			Value: "gitea",
+			Usage: "The data type that will be imported, default is gitea, options are gitea, github",
+		},
+		cli.StringFlag{
+			Name:  "repo_filepath, f",
+			Usage: "Repository compressed data path to restore from",
+		},
 		cli.StringFlag{
 			Name:  "repo_dir, r",
 			Value: "./data",
@@ -62,6 +71,8 @@ func runRestoreRepository(c *cli.Context) error {
 	}
 	statusCode, errStr := private.RestoreRepo(
 		ctx,
+		c.String("data_type"),
+		c.String("repo_filepath"),
 		c.String("repo_dir"),
 		c.String("owner_name"),
 		c.String("repo_name"),

models/issues/comment.go

@@ -180,6 +180,16 @@ func (t CommentType) String() string {
 	return commentStrings[t]
 }
 
+// GetCommentTypeByString return the comment type by given comment type string
+func GetCommentTypeByString(s string) CommentType {
+	for i, cs := range commentStrings {
+		if cs == s {
+			return CommentType(i)
+		}
+	}
+	return -1
+}
+
 // RoleDescriptor defines comment tag type
 type RoleDescriptor int
 

models/issues/label.go

@@ -301,15 +301,15 @@ func DeleteLabel(id, labelID int64) error {
 // GetLabelByID returns a label by given ID.
 func GetLabelByID(ctx context.Context, labelID int64) (*Label, error) {
 	if labelID <= 0 {
-		return nil, ErrLabelNotExist{labelID}
+		return nil, ErrLabelNotExist{LabelID: labelID}
 	}
 
 	l := &Label{}
 	has, err := db.GetEngine(ctx).ID(labelID).Get(l)
 	if err != nil {
 		return nil, err
 	} else if !has {
-		return nil, ErrLabelNotExist{l.ID}
+		return nil, ErrLabelNotExist{LabelID: l.ID}
 	}
 	return l, nil
 }
@@ -435,6 +435,18 @@ func CountLabelsByRepoID(repoID int64) (int64, error) {
 	return db.GetEngine(db.DefaultContext).Where("repo_id = ?", repoID).Count(&Label{})
 }
 
+// GetLabelByRepoIDAndName get label from repo
+func GetLabelByRepoIDAndName(repoID int64, name string) (*Label, error) {
+	var label Label
+	has, err := db.GetEngine(db.DefaultContext).Where("repo_id = ? AND name = ?", repoID, name).Get(&label)
+	if err != nil {
+		return nil, err
+	} else if !has {
+		return nil, ErrLabelNotExist{}
+	}
+	return &label, nil
+}
+
 // ________
 // \_____  \_______  ____
 //  /   |   \_  __ \/ ___\

models/migrate.go

@@ -91,6 +91,16 @@ func insertIssue(ctx context.Context, issue *issues_model.Issue) error {
 		}
 	}
 
+	for _, attach := range issue.Attachments {
+		attach.IssueID = issue.ID
+	}
+
+	if len(issue.Attachments) > 0 {
+		if _, err := sess.Insert(issue.Attachments); err != nil {
+			return err
+		}
+	}
+
 	return nil
 }
 
@@ -124,6 +134,16 @@ func InsertIssueComments(comments []*issues_model.Comment) error {
 				return err
 			}
 		}
+
+		for _, attach := range comment.Attachments {
+			attach.IssueID = comment.IssueID
+			attach.CommentID = comment.ID
+		}
+		if len(comment.Attachments) > 0 {
+			if err := db.Insert(ctx, comment.Attachments); err != nil {
+				return err
+			}
+		}
 	}
 
 	for issueID := range issueIDs {

modules/git/diff.go

@@ -52,6 +52,7 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff
 	if err != nil {
 		return err
 	}
+
 	fileArgs := make([]string, 0)
 	if len(file) > 0 {
 		fileArgs = append(fileArgs, "--", file)

modules/migration/asset.go

@@ -0,0 +1,25 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migration
+
+import (
+	"io"
+	"time"
+)
+
+// Asset represents an asset for issue, comment, release
+type Asset struct {
+	ID            int64
+	Name          string
+	ContentType   *string `yaml:"content_type"`
+	Size          *int
+	DownloadCount *int `yaml:"download_count"`
+	Created       time.Time
+	Updated       time.Time
+	DownloadURL   *string `yaml:"download_url"`
+	OriginalURL   string
+	// if DownloadURL is nil, the function should be invoked
+	DownloadFunc func() (io.ReadCloser, error) `yaml:"-"`
+}

modules/migration/comment.go

@@ -14,9 +14,47 @@ type Commentable interface {
 	GetContext() DownloaderContext
 }
 
+/*
+"comment",
+	"reopen",
+	"close",
+	"issue_ref",
+	"commit_ref",
+	"comment_ref",
+	"pull_ref",
+	"label",
+	"milestone",
+	"assignees",
+	"change_title",
+	"delete_branch",
+	"start_tracking",
+	"stop_tracking",
+	"add_time_manual",
+	"cancel_tracking",
+	"added_deadline",
+	"modified_deadline",
+	"removed_deadline",
+	"add_dependency",
+	"remove_dependency",
+	"code",
+	"review",
+	"lock",
+	"unlock",
+	"change_target_branch",
+	"delete_time_manual",
+	"review_request",
+	"merge_pull",
+	"pull_push",
+	"project",
+	"project_board",
+	"dismiss_review",
+	"change_issue_ref",
+*/
+
 // Comment is a standard comment information
 type Comment struct {
-	IssueIndex  int64 `yaml:"issue_index"`
+	Type        string // empty means comment
+	IssueIndex  int64  `yaml:"issue_index"`
 	Index       int64
 	PosterID    int64  `yaml:"poster_id"`
 	PosterName  string `yaml:"poster_name"`
@@ -25,6 +63,7 @@ type Comment struct {
 	Updated     time.Time
 	Content     string
 	Reactions   []*Reaction
+	Assets      []*Asset
 }
 
 // GetExternalName ExternalUserMigrated interface

modules/migration/downloader.go

@@ -11,6 +11,20 @@ import (
 	"code.gitea.io/gitea/modules/structs"
 )
 
+// GetCommentOptions represents an options for get comment
+type GetCommentOptions struct {
+	Commentable
+	Page     int
+	PageSize int
+}
+
+// GetReviewOptions represents an options for get reviews
+type GetReviewOptions struct {
+	Reviewable
+	Page     int
+	PageSize int
+}
+
 // Downloader downloads the site repo information
 type Downloader interface {
 	SetContext(context.Context)
@@ -20,12 +34,14 @@ type Downloader interface {
 	GetReleases() ([]*Release, error)
 	GetLabels() ([]*Label, error)
 	GetIssues(page, perPage int) ([]*Issue, bool, error)
-	GetComments(commentable Commentable) ([]*Comment, bool, error)
+	GetComments(opts GetCommentOptions) ([]*Comment, bool, error)
 	GetAllComments(page, perPage int) ([]*Comment, bool, error)
 	SupportGetRepoComments() bool
 	GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
-	GetReviews(reviewable Reviewable) ([]*Review, error)
+	GetReviews(reviewable Reviewable) ([]*Review, bool, error)
+	SupportGetRepoReviews() bool
 	FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
+	CleanUp()
 }
 
 // DownloaderFactory defines an interface to match a downloader implementation and create a downloader

modules/migration/issue.go

@@ -9,21 +9,22 @@ import "time"
 
 // Issue is a standard issue information
 type Issue struct {
-	Number       int64             `json:"number"`
-	PosterID     int64             `yaml:"poster_id" json:"poster_id"`
-	PosterName   string            `yaml:"poster_name" json:"poster_name"`
-	PosterEmail  string            `yaml:"poster_email" json:"poster_email"`
-	Title        string            `json:"title"`
-	Content      string            `json:"content"`
-	Ref          string            `json:"ref"`
-	Milestone    string            `json:"milestone"`
-	State        string            `json:"state"` // closed, open
-	IsLocked     bool              `yaml:"is_locked" json:"is_locked"`
-	Created      time.Time         `json:"created"`
-	Updated      time.Time         `json:"updated"`
-	Closed       *time.Time        `json:"closed"`
-	Labels       []*Label          `json:"labels"`
-	Reactions    []*Reaction       `json:"reactions"`
+	Number       int64       `json:"number"`
+	PosterID     int64       `yaml:"poster_id" json:"poster_id"`
+	PosterName   string      `yaml:"poster_name" json:"poster_name"`
+	PosterEmail  string      `yaml:"poster_email" json:"poster_email"`
+	Title        string      `json:"title"`
+	Content      string      `json:"content"`
+	Ref          string      `json:"ref"`
+	Milestone    string      `json:"milestone"`
+	State        string      `json:"state"` // closed, open
+	IsLocked     bool        `yaml:"is_locked" json:"is_locked"`
+	Created      time.Time   `json:"created"`
+	Updated      time.Time   `json:"updated"`
+	Closed       *time.Time  `json:"closed"`
+	Labels       []*Label    `json:"labels"`
+	Reactions    []*Reaction `json:"reactions"`
+	Assets       []*Asset
 	Assignees    []string          `json:"assignees"`
 	ForeignIndex int64             `json:"foreign_id"`
 	Context      DownloaderContext `yaml:"-"`

modules/migration/null_downloader.go

@@ -48,7 +48,7 @@ func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
 }
 
 // GetComments returns comments of an issue or PR
-func (n NullDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) {
+func (n NullDownloader) GetComments(commentable GetCommentOptions) ([]*Comment, bool, error) {
 	return nil, false, ErrNotSupported{Entity: "Comments"}
 }
 
@@ -63,8 +63,8 @@ func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool
 }
 
 // GetReviews returns pull requests review
-func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) {
-	return nil, ErrNotSupported{Entity: "Reviews"}
+func (n NullDownloader) GetReviews(reviewable Reviewable) ([]*Review, bool, error) {
+	return nil, false, ErrNotSupported{Entity: "Reviews"}
 }
 
 // FormatCloneURL add authentication into remote URLs
@@ -87,3 +87,12 @@ func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (
 func (n NullDownloader) SupportGetRepoComments() bool {
 	return false
 }
+
+// SupportGetRepoReviews return true if it supports get repo pullrequest reviews
+func (n NullDownloader) SupportGetRepoReviews() bool {
+	return false
+}
+
+// CleanUp clean the downloader temporary resources
+func (n NullDownloader) CleanUp() {
+}

modules/migration/pullrequest.go

@@ -37,6 +37,7 @@ type PullRequest struct {
 	Reactions      []*Reaction
 	ForeignIndex   int64
 	Context        DownloaderContext `yaml:"-"`
+	Assets         []*Asset
 	EnsuredSafe    bool              `yaml:"ensured_safe"`
 }
 

modules/migration/release.go

@@ -5,25 +5,9 @@
 package migration
 
 import (
-	"io"
 	"time"
 )
 
-// ReleaseAsset represents a release asset
-type ReleaseAsset struct {
-	ID            int64
-	Name          string
-	ContentType   *string `yaml:"content_type"`
-	Size          *int
-	DownloadCount *int `yaml:"download_count"`
-	Created       time.Time
-	Updated       time.Time
-
-	DownloadURL *string `yaml:"download_url"` // SECURITY: It is the responsibility of downloader to make sure this is safe
-	// if DownloadURL is nil, the function should be invoked
-	DownloadFunc func() (io.ReadCloser, error) `yaml:"-"` // SECURITY: It is the responsibility of downloader to make sure this is safe
-}
-
 // Release represents a release
 type Release struct {
 	TagName         string `yaml:"tag_name"`         // SECURITY: This must pass git.IsValidRefPattern
@@ -35,7 +19,7 @@ type Release struct {
 	PublisherID     int64  `yaml:"publisher_id"`
 	PublisherName   string `yaml:"publisher_name"`
 	PublisherEmail  string `yaml:"publisher_email"`
-	Assets          []*ReleaseAsset
+	Assets          []*Asset
 	Created         time.Time
 	Published       time.Time
 }

modules/migration/retry_downloader.go

@@ -148,15 +148,15 @@ func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
 }
 
 // GetComments returns a repository's comments with retry
-func (d *RetryDownloader) GetComments(commentable Commentable) ([]*Comment, bool, error) {
+func (d *RetryDownloader) GetComments(opts GetCommentOptions) ([]*Comment, bool, error) {
 	var (
 		comments []*Comment
 		isEnd    bool
 		err      error
 	)
 
 	err = d.retry(func() error {
-		comments, isEnd, err = d.Downloader.GetComments(commentable)
+		comments, isEnd, err = d.Downloader.GetComments(opts)
 		return err
 	})
 
@@ -180,16 +180,17 @@ func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bo
 }
 
 // GetReviews returns pull requests reviews
-func (d *RetryDownloader) GetReviews(reviewable Reviewable) ([]*Review, error) {
+func (d *RetryDownloader) GetReviews(reviwable Reviewable) ([]*Review, bool, error) {
 	var (
 		reviews []*Review
+		isEnd   bool
 		err     error
 	)
 
 	err = d.retry(func() error {
-		reviews, err = d.Downloader.GetReviews(reviewable)
+		reviews, isEnd, err = d.Downloader.GetReviews(reviwable)
 		return err
 	})
 
-	return reviews, err
+	return reviews, isEnd, err
 }