forked from forgejo/forgejo
Add support to migrate from gogs (#14342)
Add support to migrate gogs: * issues * comments * labels * milestones * wiki Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com> Co-authored-by: Andrew Thornton <art27@cantab.net>
This commit is contained in:
6543
GitHub
parent
b5570d3e68
commit
81c833d92d
55 changed files with 2782 additions and 353 deletions
|
|
@ -7,7 +7,6 @@ package base
|
|||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"code.gitea.io/gitea/modules/structs"
|
||||
)
|
||||
|
|
@ -24,6 +23,7 @@ type Downloader interface {
|
|||
GetComments(issueNumber int64) ([]*Comment, error)
|
||||
GetPullRequests(page, perPage int) ([]*PullRequest, bool, error)
|
||||
GetReviews(pullRequestNumber int64) ([]*Review, error)
|
||||
FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error)
|
||||
}
|
||||
|
||||
// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
|
||||
|
|
@ -31,213 +31,3 @@ type DownloaderFactory interface {
|
|||
New(ctx context.Context, opts MigrateOptions) (Downloader, error)
|
||||
GitServiceType() structs.GitServiceType
|
||||
}
|
||||
|
||||
var (
|
||||
_ Downloader = &RetryDownloader{}
|
||||
)
|
||||
|
||||
// RetryDownloader retry the downloads
|
||||
type RetryDownloader struct {
|
||||
Downloader
|
||||
ctx context.Context
|
||||
RetryTimes int // the total execute times
|
||||
RetryDelay int // time to delay seconds
|
||||
}
|
||||
|
||||
// NewRetryDownloader creates a retry downloader
|
||||
func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
|
||||
return &RetryDownloader{
|
||||
Downloader: downloader,
|
||||
ctx: ctx,
|
||||
RetryTimes: retryTimes,
|
||||
RetryDelay: retryDelay,
|
||||
}
|
||||
}
|
||||
|
||||
// SetContext set context
|
||||
func (d *RetryDownloader) SetContext(ctx context.Context) {
|
||||
d.ctx = ctx
|
||||
d.Downloader.SetContext(ctx)
|
||||
}
|
||||
|
||||
// GetRepoInfo returns a repository information with retry
|
||||
func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
repo *Repository
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if repo, err = d.Downloader.GetRepoInfo(); err == nil {
|
||||
return repo, nil
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetTopics returns a repository's topics with retry
|
||||
func (d *RetryDownloader) GetTopics() ([]string, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
topics []string
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if topics, err = d.Downloader.GetTopics(); err == nil {
|
||||
return topics, nil
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetMilestones returns a repository's milestones with retry
|
||||
func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
milestones []*Milestone
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if milestones, err = d.Downloader.GetMilestones(); err == nil {
|
||||
return milestones, nil
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetReleases returns a repository's releases with retry
|
||||
func (d *RetryDownloader) GetReleases() ([]*Release, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
releases []*Release
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if releases, err = d.Downloader.GetReleases(); err == nil {
|
||||
return releases, nil
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetLabels returns a repository's labels with retry
|
||||
func (d *RetryDownloader) GetLabels() ([]*Label, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
labels []*Label
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if labels, err = d.Downloader.GetLabels(); err == nil {
|
||||
return labels, nil
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetIssues returns a repository's issues with retry
|
||||
func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
issues []*Issue
|
||||
isEnd bool
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if issues, isEnd, err = d.Downloader.GetIssues(page, perPage); err == nil {
|
||||
return issues, isEnd, nil
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, false, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
// GetComments returns a repository's comments with retry
|
||||
func (d *RetryDownloader) GetComments(issueNumber int64) ([]*Comment, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
comments []*Comment
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if comments, err = d.Downloader.GetComments(issueNumber); err == nil {
|
||||
return comments, nil
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetPullRequests returns a repository's pull requests with retry
|
||||
func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
prs []*PullRequest
|
||||
err error
|
||||
isEnd bool
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage); err == nil {
|
||||
return prs, isEnd, nil
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, false, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
// GetReviews returns pull requests reviews
|
||||
func (d *RetryDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
reviews []*Review
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if reviews, err = d.Downloader.GetReviews(pullRequestNumber); err == nil {
|
||||
return reviews, nil
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
26
modules/migrations/base/error.go
Normal file
26
modules/migrations/base/error.go
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
import "fmt"
|
||||
|
||||
// ErrNotSupported represents status if a downloader do not supported something.
|
||||
type ErrNotSupported struct {
|
||||
Entity string
|
||||
}
|
||||
|
||||
// IsErrNotSupported checks if an error is an ErrNotSupported
|
||||
func IsErrNotSupported(err error) bool {
|
||||
_, ok := err.(ErrNotSupported)
|
||||
return ok
|
||||
}
|
||||
|
||||
// Error return error message
|
||||
func (err ErrNotSupported) Error() string {
|
||||
if len(err.Entity) != 0 {
|
||||
return fmt.Sprintf("'%s' not supported", err.Entity)
|
||||
}
|
||||
return "not supported"
|
||||
}
|
||||
|
|
@ -15,5 +15,5 @@ type Milestone struct {
|
|||
Created time.Time
|
||||
Updated *time.Time
|
||||
Closed *time.Time
|
||||
State string
|
||||
State string // open, closed
|
||||
}
|
||||
|
|
|
|||
82
modules/migrations/base/null_downloader.go
Normal file
82
modules/migrations/base/null_downloader.go
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/url"
|
||||
)
|
||||
|
||||
// NullDownloader implements a blank downloader
|
||||
type NullDownloader struct {
|
||||
}
|
||||
|
||||
var (
|
||||
_ Downloader = &NullDownloader{}
|
||||
)
|
||||
|
||||
// SetContext set context
|
||||
func (n NullDownloader) SetContext(_ context.Context) {}
|
||||
|
||||
// GetRepoInfo returns a repository information
|
||||
func (n NullDownloader) GetRepoInfo() (*Repository, error) {
|
||||
return nil, &ErrNotSupported{Entity: "RepoInfo"}
|
||||
}
|
||||
|
||||
// GetTopics return repository topics
|
||||
func (n NullDownloader) GetTopics() ([]string, error) {
|
||||
return nil, &ErrNotSupported{Entity: "Topics"}
|
||||
}
|
||||
|
||||
// GetMilestones returns milestones
|
||||
func (n NullDownloader) GetMilestones() ([]*Milestone, error) {
|
||||
return nil, &ErrNotSupported{Entity: "Milestones"}
|
||||
}
|
||||
|
||||
// GetReleases returns releases
|
||||
func (n NullDownloader) GetReleases() ([]*Release, error) {
|
||||
return nil, &ErrNotSupported{Entity: "Releases"}
|
||||
}
|
||||
|
||||
// GetLabels returns labels
|
||||
func (n NullDownloader) GetLabels() ([]*Label, error) {
|
||||
return nil, &ErrNotSupported{Entity: "Labels"}
|
||||
}
|
||||
|
||||
// GetIssues returns issues according start and limit
|
||||
func (n NullDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
|
||||
return nil, false, &ErrNotSupported{Entity: "Issues"}
|
||||
}
|
||||
|
||||
// GetComments returns comments according issueNumber
|
||||
func (n NullDownloader) GetComments(issueNumber int64) ([]*Comment, error) {
|
||||
return nil, &ErrNotSupported{Entity: "Comments"}
|
||||
}
|
||||
|
||||
// GetPullRequests returns pull requests according page and perPage
|
||||
func (n NullDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
|
||||
return nil, false, &ErrNotSupported{Entity: "PullRequests"}
|
||||
}
|
||||
|
||||
// GetReviews returns pull requests review
|
||||
func (n NullDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
|
||||
return nil, &ErrNotSupported{Entity: "Reviews"}
|
||||
}
|
||||
|
||||
// FormatCloneURL add authentification into remote URLs
|
||||
func (n NullDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
|
||||
if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
|
||||
u, err := url.Parse(remoteAddr)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
|
||||
if len(opts.AuthToken) > 0 {
|
||||
u.User = url.UserPassword("oauth2", opts.AuthToken)
|
||||
}
|
||||
return u.String(), nil
|
||||
}
|
||||
return remoteAddr, nil
|
||||
}
|
||||
247
modules/migrations/base/retry_downloader.go
Normal file
247
modules/migrations/base/retry_downloader.go
Normal file
|
|
@ -0,0 +1,247 @@
|
|||
// Copyright 2021 The Gitea Authors. All rights reserved.
|
||||
// Use of this source code is governed by a MIT-style
|
||||
// license that can be found in the LICENSE file.
|
||||
|
||||
package base
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
_ Downloader = &RetryDownloader{}
|
||||
)
|
||||
|
||||
// RetryDownloader retry the downloads
|
||||
type RetryDownloader struct {
|
||||
Downloader
|
||||
ctx context.Context
|
||||
RetryTimes int // the total execute times
|
||||
RetryDelay int // time to delay seconds
|
||||
}
|
||||
|
||||
// NewRetryDownloader creates a retry downloader
|
||||
func NewRetryDownloader(ctx context.Context, downloader Downloader, retryTimes, retryDelay int) *RetryDownloader {
|
||||
return &RetryDownloader{
|
||||
Downloader: downloader,
|
||||
ctx: ctx,
|
||||
RetryTimes: retryTimes,
|
||||
RetryDelay: retryDelay,
|
||||
}
|
||||
}
|
||||
|
||||
// SetContext set context
|
||||
func (d *RetryDownloader) SetContext(ctx context.Context) {
|
||||
d.ctx = ctx
|
||||
d.Downloader.SetContext(ctx)
|
||||
}
|
||||
|
||||
// GetRepoInfo returns a repository information with retry
|
||||
func (d *RetryDownloader) GetRepoInfo() (*Repository, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
repo *Repository
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if repo, err = d.Downloader.GetRepoInfo(); err == nil {
|
||||
return repo, nil
|
||||
}
|
||||
if IsErrNotSupported(err) {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetTopics returns a repository's topics with retry
|
||||
func (d *RetryDownloader) GetTopics() ([]string, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
topics []string
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if topics, err = d.Downloader.GetTopics(); err == nil {
|
||||
return topics, nil
|
||||
}
|
||||
if IsErrNotSupported(err) {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetMilestones returns a repository's milestones with retry
|
||||
func (d *RetryDownloader) GetMilestones() ([]*Milestone, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
milestones []*Milestone
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if milestones, err = d.Downloader.GetMilestones(); err == nil {
|
||||
return milestones, nil
|
||||
}
|
||||
if IsErrNotSupported(err) {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetReleases returns a repository's releases with retry
|
||||
func (d *RetryDownloader) GetReleases() ([]*Release, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
releases []*Release
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if releases, err = d.Downloader.GetReleases(); err == nil {
|
||||
return releases, nil
|
||||
}
|
||||
if IsErrNotSupported(err) {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetLabels returns a repository's labels with retry
|
||||
func (d *RetryDownloader) GetLabels() ([]*Label, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
labels []*Label
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if labels, err = d.Downloader.GetLabels(); err == nil {
|
||||
return labels, nil
|
||||
}
|
||||
if IsErrNotSupported(err) {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetIssues returns a repository's issues with retry
|
||||
func (d *RetryDownloader) GetIssues(page, perPage int) ([]*Issue, bool, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
issues []*Issue
|
||||
isEnd bool
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if issues, isEnd, err = d.Downloader.GetIssues(page, perPage); err == nil {
|
||||
return issues, isEnd, nil
|
||||
}
|
||||
if IsErrNotSupported(err) {
|
||||
return nil, false, err
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, false, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
// GetComments returns a repository's comments with retry
|
||||
func (d *RetryDownloader) GetComments(issueNumber int64) ([]*Comment, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
comments []*Comment
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if comments, err = d.Downloader.GetComments(issueNumber); err == nil {
|
||||
return comments, nil
|
||||
}
|
||||
if IsErrNotSupported(err) {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// GetPullRequests returns a repository's pull requests with retry
|
||||
func (d *RetryDownloader) GetPullRequests(page, perPage int) ([]*PullRequest, bool, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
prs []*PullRequest
|
||||
err error
|
||||
isEnd bool
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if prs, isEnd, err = d.Downloader.GetPullRequests(page, perPage); err == nil {
|
||||
return prs, isEnd, nil
|
||||
}
|
||||
if IsErrNotSupported(err) {
|
||||
return nil, false, err
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, false, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
// GetReviews returns pull requests reviews
|
||||
func (d *RetryDownloader) GetReviews(pullRequestNumber int64) ([]*Review, error) {
|
||||
var (
|
||||
times = d.RetryTimes
|
||||
reviews []*Review
|
||||
err error
|
||||
)
|
||||
for ; times > 0; times-- {
|
||||
if reviews, err = d.Downloader.GetReviews(pullRequestNumber); err == nil {
|
||||
return reviews, nil
|
||||
}
|
||||
if IsErrNotSupported(err) {
|
||||
return nil, err
|
||||
}
|
||||
select {
|
||||
case <-d.ctx.Done():
|
||||
return nil, d.ctx.Err()
|
||||
case <-time.After(time.Second * time.Duration(d.RetryDelay)):
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue