domi/forgejo-domi
1
0
Fork 0
forked from forgejo/forgejo
Code Pull requests Activity

Git migration UX (#12619)

Browse source 
* Initial work

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Implementation

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Fix gitlab and token cloning

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Imports and JS

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Fix test

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Linting

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Generate swagger

Signed-off-by: jolheiser <john.olheiser@gmail.com>

* Move mirror toggle and rename options

Signed-off-by: jolheiser <john.olheiser@gmail.com>

Co-authored-by: Lauris BH <lauris@nix.lv>
Co-authored-by: Lunny Xiao <xiaolunwen@gmail.com>
This commit is contained in:
John Olheiser 2020-08-27 20:36:37 -05:00 • committed by GitHub
parent ed2f6e137b
commit 211321fb93
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 273 additions and 181 deletions
Download patch file Download diff file Expand all files Collapse all files

2
modules/auth/repo_form.go
View file

@ -56,8 +56,10 @@ func (f *CreateRepoForm) Validate(ctx *macaron.Context, errs binding.Errors) bin
type MigrateRepoForm struct {
// required: true
CloneAddr string `json:"clone_addr" binding:"Required"`
Service int `json:"service"`
AuthUsername string `json:"auth_username"`
AuthPassword string `json:"auth_password"`
AuthToken string `json:"auth_token"`
// required: true
UID int64 `json:"uid" binding:"Required"`
// required: true

8
modules/migrations/base/downloader.go
View file

@ -7,13 +7,20 @@ package base
import (
"context"
"io"
"time"
"code.gitea.io/gitea/modules/structs"
)
// AssetDownloader downloads an asset (attachment) for a release
type AssetDownloader interface {
GetAsset(tag string, id int64) (io.ReadCloser, error)
}
// Downloader downloads the site repo informations
type Downloader interface {
AssetDownloader
SetContext(context.Context)
GetRepoInfo() (*Repository, error)
GetTopics() ([]string, error)
@ -28,7 +35,6 @@ type Downloader interface {
// DownloaderFactory defines an interface to match a downloader implementation and create a downloader
type DownloaderFactory interface {
Match(opts MigrateOptions) (bool, error)
New(opts MigrateOptions) (Downloader, error)
GitServiceType() structs.GitServiceType
}

2
modules/migrations/base/release.go
View file

@ -8,7 +8,7 @@ import "time"
// ReleaseAsset represents a release asset
type ReleaseAsset struct {
URL string
ID int64
Name string
ContentType *string
Size *int

2
modules/migrations/base/uploader.go
View file

@ -11,7 +11,7 @@ type Uploader interface {
CreateRepo(repo *Repository, opts MigrateOptions) error
CreateTopics(topic ...string) error
CreateMilestones(milestones ...*Milestone) error
CreateReleases(releases ...*Release) error
CreateReleases(downloader Downloader, releases ...*Release) error
SyncTags() error
CreateLabels(labels ...*Label) error
CreateIssues(issues ...*Issue) error

6
modules/migrations/git.go
View file

@ -6,6 +6,7 @@ package migrations
import (
"context"
"io"
"code.gitea.io/gitea/modules/migrations/base"
)
@ -64,6 +65,11 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
return nil, ErrNotSupported
}
// GetAsset returns an asset
func (g *PlainGitDownloader) GetAsset(_ string, _ int64) (io.ReadCloser, error) {
return nil, ErrNotSupported
}
// GetIssues returns issues according page and perPage
func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
return nil, false, ErrNotSupported

13
modules/migrations/gitea.go
View file

@ -93,12 +93,15 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
}
var remoteAddr = repo.CloneURL
if len(opts.AuthUsername) > 0 {
if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
u, err := url.Parse(repo.CloneURL)
if err != nil {
return err
}
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
if len(opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", opts.AuthToken)
}
remoteAddr = u.String()
}
@ -210,7 +213,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
}
// CreateReleases creates releases
func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases ...*base.Release) error {
var rels = make([]*models.Release, 0, len(releases))
for _, release := range releases {
var rel = models.Release{
@ -269,13 +272,11 @@ func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
// download attachment
err = func() error {
resp, err := http.Get(asset.URL)
rc, err := downloader.GetAsset(rel.TagName, asset.ID)
if err != nil {
return err
}
defer resp.Body.Close()
_, err = storage.Attachments.Save(attach.RelativePath(), resp.Body)
_, err = storage.Attachments.Save(attach.RelativePath(), rc)
return err
}()
if err != nil {

2
modules/migrations/gitea_test.go
View file

@ -26,7 +26,7 @@ func TestGiteaUploadRepo(t *testing.T) {
user := models.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
var (
downloader = NewGithubDownloaderV3("", "", "go-xorm", "builder")
downloader = NewGithubDownloaderV3("", "", "", "go-xorm", "builder")
repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05")
uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
)

63
modules/migrations/github.go
View file

@ -6,8 +6,11 @@
package migrations
import (
"bytes"
"context"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"strings"
@ -37,16 +40,6 @@ func init() {
type GithubDownloaderV3Factory struct {
}
// Match returns ture if the migration remote URL matched this downloader factory
func (f *GithubDownloaderV3Factory) Match(opts base.MigrateOptions) (bool, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return false, err
}
return strings.EqualFold(u.Host, "github.com") && opts.AuthUsername != "", nil
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
@ -60,7 +53,7 @@ func (f *GithubDownloaderV3Factory) New(opts base.MigrateOptions) (base.Download
log.Trace("Create github downloader: %s/%s", oldOwner, oldName)
return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, oldOwner, oldName), nil
return NewGithubDownloaderV3(opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
}
// GitServiceType returns the type of git service
@ -81,7 +74,7 @@ type GithubDownloaderV3 struct {
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *GithubDownloaderV3 {
func NewGithubDownloaderV3(userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
var downloader = GithubDownloaderV3{
userName: userName,
password: password,
@ -90,23 +83,19 @@ func NewGithubDownloaderV3(userName, password, repoOwner, repoName string) *Gith
repoName: repoName,
}
var client *http.Client
if userName != "" {
if password == "" {
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: userName},
)
client = oauth2.NewClient(downloader.ctx, ts)
} else {
client = &http.Client{
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return nil, nil
},
},
}
}
client := &http.Client{
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return nil, nil
},
},
}
if token != "" {
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
)
client = oauth2.NewClient(downloader.ctx, ts)
}
downloader.client = github.NewClient(client)
return &downloader
@ -290,10 +279,8 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
}
for _, asset := range rel.Assets {
u, _ := url.Parse(*asset.BrowserDownloadURL)
u.User = url.UserPassword(g.userName, g.password)
r.Assets = append(r.Assets, base.ReleaseAsset{
URL: u.String(),
ID: *asset.ID,
Name: *asset.Name,
ContentType: asset.ContentType,
Size: asset.Size,
@ -331,6 +318,18 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
return releases, nil
}
// GetAsset returns an asset
func (g *GithubDownloaderV3) GetAsset(_ string, id int64) (io.ReadCloser, error) {
asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, id, http.DefaultClient)
if err != nil {
return nil, err
}
if asset == nil {
return ioutil.NopCloser(bytes.NewBufferString(redir)), nil
}
return asset, nil
}
// GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
opt := &github.IssueListByRepoOptions{

2
modules/migrations/github_test.go
View file

@ -64,7 +64,7 @@ func assertLabelEqual(t *testing.T, name, color, description string, label *base
func TestGitHubDownloadRepo(t *testing.T) {
GithubLimitRateRemaining = 3 //Wait at 3 remaining since we could have 3 CI in //
downloader := NewGithubDownloaderV3(os.Getenv("GITHUB_READ_TOKEN"), "", "go-gitea", "test_repo")
downloader := NewGithubDownloaderV3("", "", os.Getenv("GITHUB_READ_TOKEN"), "go-gitea", "test_repo")
err := downloader.RefreshRate()
assert.NoError(t, err)

57
modules/migrations/gitlab.go
View file

@ -8,6 +8,8 @@ import (
"context"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
@ -32,21 +34,6 @@ func init() {
type GitlabDownloaderFactory struct {
}
// Match returns true if the migration remote URL matched this downloader factory
func (f *GitlabDownloaderFactory) Match(opts base.MigrateOptions) (bool, error) {
var matched bool
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return false, err
}
if strings.EqualFold(u.Host, "gitlab.com") && opts.AuthUsername != "" {
matched = true
}
return matched, nil
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GitlabDownloaderFactory) New(opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
@ -56,10 +43,11 @@ func (f *GitlabDownloaderFactory) New(opts base.MigrateOptions) (base.Downloader
baseURL := u.Scheme + "://" + u.Host
repoNameSpace := strings.TrimPrefix(u.Path, "/")
repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git")
log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
return NewGitlabDownloader(baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword), nil
return NewGitlabDownloader(baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword, opts.AuthToken), nil
}
// GitServiceType returns the type of git service
@ -85,15 +73,13 @@ type GitlabDownloader struct {
// NewGitlabDownloader creates a gitlab Downloader via gitlab API
// Use either a username/password, personal token entered into the username field, or anonymous/public access
// Note: Public access only allows very basic access
func NewGitlabDownloader(baseURL, repoPath, username, password string) *GitlabDownloader {
func NewGitlabDownloader(baseURL, repoPath, username, password, token string) *GitlabDownloader {
var gitlabClient *gitlab.Client
var err error
if username != "" {
if password == "" {
gitlabClient, err = gitlab.NewClient(username, gitlab.WithBaseURL(baseURL))
} else {
gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
}
if token != "" {
gitlabClient, err = gitlab.NewClient(token, gitlab.WithBaseURL(baseURL))
} else {
gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
}
if err != nil {
@ -271,7 +257,7 @@ func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
}
func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release {
var zero int
r := &base.Release{
TagName: rel.TagName,
TargetCommitish: rel.Commit.ID,
@ -284,9 +270,11 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
for k, asset := range rel.Assets.Links {
r.Assets = append(r.Assets, base.ReleaseAsset{
URL: asset.URL,
Name: asset.Name,
ContentType: &rel.Assets.Sources[k].Format,
ID: int64(asset.ID),
Name: asset.Name,
ContentType: &rel.Assets.Sources[k].Format,
Size: &zero,
DownloadCount: &zero,
})
}
return r
@ -315,6 +303,21 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
return releases, nil
}
// GetAsset returns an asset
func (g *GitlabDownloader) GetAsset(tag string, id int64) (io.ReadCloser, error) {
link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, tag, int(id))
if err != nil {
return nil, err
}
resp, err := http.Get(link.URL)
if err != nil {
return nil, err
}
// resp.Body is closed by the uploader
return resp.Body, nil
}
// GetIssues returns issues according start and limit
// Note: issue label description and colors are not supported by the go-gitlab library at this time
// TODO: figure out how to transfer issue reactions

2
modules/migrations/gitlab_test.go
View file

@ -27,7 +27,7 @@ func TestGitlabDownloadRepo(t *testing.T) {
t.Skipf("Can't access test repo, skipping %s", t.Name())
}
downloader := NewGitlabDownloader("https://gitlab.com", "gitea/test_repo", gitlabPersonalAccessToken, "")
downloader := NewGitlabDownloader("https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken)
if downloader == nil {
t.Fatal("NewGitlabDownloader is nil")
}

13
modules/migrations/migrate.go
View file

@ -13,7 +13,6 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
)
// MigrateOptions is equal to base.MigrateOptions
@ -33,18 +32,15 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
var (
downloader base.Downloader
uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
theFactory base.DownloaderFactory
err error
)
for _, factory := range factories {
if match, err := factory.Match(opts); err != nil {
return nil, err
} else if match {
if factory.GitServiceType() == opts.GitServiceType {
downloader, err = factory.New(opts)
if err != nil {
return nil, err
}
theFactory = factory
break
}
}
@ -57,11 +53,8 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
opts.Comments = false
opts.Issues = false
opts.PullRequests = false
opts.GitServiceType = structs.PlainGitService
downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr)
log.Trace("Will migrate from git: %s", opts.OriginalURL)
} else if opts.GitServiceType == structs.NotMigrated {
opts.GitServiceType = theFactory.GitServiceType()
}
uploader.gitServiceType = opts.GitServiceType
@ -169,7 +162,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
relBatchSize = len(releases)
}
if err := uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
if err := uploader.CreateReleases(downloader, releases[:relBatchSize]...); err != nil {
return err
}
releases = releases[relBatchSize:]

27
modules/structs/repo.go
View file

@ -218,6 +218,32 @@ func (gt GitServiceType) Name() string {
return ""
}
// Title represents the service type's proper title
func (gt GitServiceType) Title() string {
switch gt {
case GithubService:
return "GitHub"
case GiteaService:
return "Gitea"
case GitlabService:
return "GitLab"
case GogsService:
return "Gogs"
case PlainGitService:
return "Git"
}
return ""
}
// TokenAuth represents whether a service type supports token-based auth
func (gt GitServiceType) TokenAuth() bool {
switch gt {
case GithubService, GiteaService, GitlabService:
return true
}
return false
}
var (
// SupportedFullGitService represents all git services supported to migrate issues/labels/prs and etc.
// TODO: add to this list after new git service added
@ -233,6 +259,7 @@ type MigrateRepoOption struct {
CloneAddr string `json:"clone_addr" binding:"Required"`
AuthUsername string `json:"auth_username"`
AuthPassword string `json:"auth_password"`
AuthToken string `json:"auth_token"`
// required: true
UID int `json:"uid" binding:"Required"`
// required: true