Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244)

* Dump github/gitlab repository data to a local directory

* Fix lint

* Adjust directory structure

* Allow migration special units

* Allow migration ignore release assets

* Fix lint

* Add restore repository

* stage the changes

* Merge

* Fix lint

* Update the interface

* Add some restore methods

* Finish restore

* Add comments

* Fix restore

* Add a token flag

* Fix bug

* Fix test

* Fix test

* Fix bug

* Fix bug

* Fix lint

* Fix restore

* refactor downloader

* fmt

* Fix bug isEnd detection on getIssues

* Refactor maxPerPage

* Remove unused codes

* Remove unused codes

* Fix bug

* Fix restore

* Fix dump

* Uploader should not depend downloader

* use release attachment name but not id

* Fix restore bug

* Fix lint

* Fix restore bug

* Add a method of DownloadFunc for base.Release to make uploader not depend on downloader

* fix Release yml marshal

* Fix trace information

* Fix bug when dump & restore

* Save relative path on yml file

* Fix bug

* Use relative path

* Update docs

* Use git service string but not int

* Recognize clone addr to service type

modules/migrations/base/comment.go

@@ -9,10 +9,10 @@ import "time"
 
 // Comment is a standard comment information
 type Comment struct {
-	IssueIndex  int64
-	PosterID    int64
-	PosterName  string
-	PosterEmail string
+	IssueIndex  int64  `yaml:"issue_index"`
+	PosterID    int64  `yaml:"poster_id"`
+	PosterName  string `yaml:"poster_name"`
+	PosterEmail string `yaml:"poster_email"`
 	Created     time.Time
 	Updated     time.Time
 	Content     string

modules/migrations/base/downloader.go

@@ -7,20 +7,13 @@ package base
 
 import (
 	"context"
-	"io"
 	"time"
 
 	"code.gitea.io/gitea/modules/structs"
 )
 
-// AssetDownloader downloads an asset (attachment) for a release
-type AssetDownloader interface {
-	GetAsset(relTag string, relID, id int64) (io.ReadCloser, error)
-}
-
 // Downloader downloads the site repo informations
 type Downloader interface {
-	AssetDownloader
 	SetContext(context.Context)
 	GetRepoInfo() (*Repository, error)
 	GetTopics() ([]string, error)

modules/migrations/base/issue.go

@@ -10,15 +10,15 @@ import "time"
 // Issue is a standard issue information
 type Issue struct {
 	Number      int64
-	PosterID    int64
-	PosterName  string
-	PosterEmail string
+	PosterID    int64  `yaml:"poster_id"`
+	PosterName  string `yaml:"poster_name"`
+	PosterEmail string `yaml:"poster_email"`
 	Title       string
 	Content     string
 	Ref         string
 	Milestone   string
 	State       string // closed, open
-	IsLocked    bool
+	IsLocked    bool   `yaml:"is_locked"`
 	Created     time.Time
 	Updated     time.Time
 	Closed      *time.Time

modules/migrations/base/options.go

@@ -31,5 +31,6 @@ type MigrateOptions struct {
 	Releases        bool
 	Comments        bool
 	PullRequests    bool
+	ReleaseAssets   bool
 	MigrateToRepoID int64
 }

modules/migrations/base/pullrequest.go

@@ -13,11 +13,11 @@ import (
 // PullRequest defines a standard pull request information
 type PullRequest struct {
 	Number         int64
-	OriginalNumber int64
+	OriginalNumber int64 `yaml:"original_number"`
 	Title          string
-	PosterName     string
-	PosterID       int64
-	PosterEmail    string
+	PosterName     string `yaml:"poster_name"`
+	PosterID       int64  `yaml:"poster_id"`
+	PosterEmail    string `yaml:"poster_email"`
 	Content        string
 	Milestone      string
 	State          string
@@ -25,14 +25,14 @@ type PullRequest struct {
 	Updated        time.Time
 	Closed         *time.Time
 	Labels         []*Label
-	PatchURL       string
+	PatchURL       string `yaml:"patch_url"`
 	Merged         bool
-	MergedTime     *time.Time
-	MergeCommitSHA string
+	MergedTime     *time.Time `yaml:"merged_time"`
+	MergeCommitSHA string     `yaml:"merge_commit_sha"`
 	Head           PullRequestBranch
 	Base           PullRequestBranch
 	Assignees      []string
-	IsLocked       bool
+	IsLocked       bool `yaml:"is_locked"`
 	Reactions      []*Reaction
 }
 
@@ -43,11 +43,11 @@ func (p *PullRequest) IsForkPullRequest() bool {
 
 // PullRequestBranch represents a pull request branch
 type PullRequestBranch struct {
-	CloneURL  string
+	CloneURL  string `yaml:"clone_url"`
 	Ref       string
 	SHA       string
-	RepoName  string
-	OwnerName string
+	RepoName  string `yaml:"repo_name"`
+	OwnerName string `yaml:"owner_name"`
 }
 
 // RepoPath returns pull request repo path

modules/migrations/base/reaction.go

@@ -6,7 +6,7 @@ package base
 
 // Reaction represents a reaction to an issue/pr/comment.
 type Reaction struct {
-	UserID   int64
-	UserName string
+	UserID   int64  `yaml:"user_id"`
+	UserName string `yaml:"user_name"`
 	Content  string
 }

modules/migrations/base/release.go

@@ -4,32 +4,37 @@
 
 package base
 
-import "time"
+import (
+	"io"
+	"time"
+)
 
 // ReleaseAsset represents a release asset
 type ReleaseAsset struct {
 	ID            int64
 	Name          string
-	ContentType   *string
+	ContentType   *string `yaml:"content_type"`
 	Size          *int
-	DownloadCount *int
+	DownloadCount *int `yaml:"download_count"`
 	Created       time.Time
 	Updated       time.Time
-	DownloadURL   *string
+	DownloadURL   *string `yaml:"download_url"`
+	// if DownloadURL is nil, the function should be invoked
+	DownloadFunc func() (io.ReadCloser, error) `yaml:"-"`
 }
 
 // Release represents a release
 type Release struct {
-	TagName         string
-	TargetCommitish string
+	TagName         string `yaml:"tag_name"`
+	TargetCommitish string `yaml:"target_commitish"`
 	Name            string
 	Body            string
 	Draft           bool
 	Prerelease      bool
-	PublisherID     int64
-	PublisherName   string
-	PublisherEmail  string
-	Assets          []ReleaseAsset
+	PublisherID     int64  `yaml:"publisher_id"`
+	PublisherName   string `yaml:"publisher_name"`
+	PublisherEmail  string `yaml:"publisher_email"`
+	Assets          []*ReleaseAsset
 	Created         time.Time
 	Published       time.Time
 }

modules/migrations/base/repo.go

@@ -9,10 +9,10 @@ package base
 type Repository struct {
 	Name          string
 	Owner         string
-	IsPrivate     bool
-	IsMirror      bool
+	IsPrivate     bool `yaml:"is_private"`
+	IsMirror      bool `yaml:"is_mirror"`
 	Description   string
-	CloneURL      string
-	OriginalURL   string
+	CloneURL      string `yaml:"clone_url"`
+	OriginalURL   string `yaml:"original_url"`
 	DefaultBranch string
 }

modules/migrations/base/review.go

@@ -17,29 +17,29 @@ const (
 // Review is a standard review information
 type Review struct {
 	ID           int64
-	IssueIndex   int64
-	ReviewerID   int64
-	ReviewerName string
+	IssueIndex   int64  `yaml:"issue_index"`
+	ReviewerID   int64  `yaml:"reviewer_id"`
+	ReviewerName string `yaml:"reviewer_name"`
 	Official     bool
-	CommitID     string
+	CommitID     string `yaml:"commit_id"`
 	Content      string
-	CreatedAt    time.Time
-	State        string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
+	CreatedAt    time.Time `yaml:"created_at"`
+	State        string    // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
 	Comments     []*ReviewComment
 }
 
 // ReviewComment represents a review comment
 type ReviewComment struct {
 	ID        int64
-	InReplyTo int64
+	InReplyTo int64 `yaml:"in_reply_to"`
 	Content   string
-	TreePath  string
-	DiffHunk  string
+	TreePath  string `yaml:"tree_path"`
+	DiffHunk  string `yaml:"diff_hunk"`
 	Position  int
 	Line      int
-	CommitID  string
-	PosterID  int64
+	CommitID  string `yaml:"commit_id"`
+	PosterID  int64  `yaml:"poster_id"`
 	Reactions []*Reaction
-	CreatedAt time.Time
-	UpdatedAt time.Time
+	CreatedAt time.Time `yaml:"created_at"`
+	UpdatedAt time.Time `yaml:"updated_at"`
 }

modules/migrations/base/uploader.go

@@ -11,7 +11,7 @@ type Uploader interface {
 	CreateRepo(repo *Repository, opts MigrateOptions) error
 	CreateTopics(topic ...string) error
 	CreateMilestones(milestones ...*Milestone) error
-	CreateReleases(downloader Downloader, releases ...*Release) error
+	CreateReleases(releases ...*Release) error
 	SyncTags() error
 	CreateLabels(labels ...*Label) error
 	CreateIssues(issues ...*Issue) error
@@ -19,5 +19,6 @@ type Uploader interface {
 	CreatePullRequests(prs ...*PullRequest) error
 	CreateReviews(reviews ...*Review) error
 	Rollback() error
+	Finish() error
 	Close()
 }