domi/forgejo-domi
1
0
Fork 0
forked from forgejo/forgejo
Code Pull requests Activity

Move migrations into services and base into modules/migration (#17663)

Browse source 
* Move migrtions into services and base into modules/migration

* Fix imports

* Fix lint
This commit is contained in:
Lunny Xiao 2021-11-16 23:25:33 +08:00 • committed by GitHub
parent 48ccd325a1
commit 7e1ae38097
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
50 changed files with 51 additions and 52 deletions
Download patch file Download diff file Expand all files Collapse all files

637
services/migrations/dump.go Normal file
View file

@ -0,0 +1,637 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/structs"
"gopkg.in/yaml.v2"
)
var (
_ base.Uploader = &RepositoryDumper{}
)
// RepositoryDumper implements an Uploader to the local directory
type RepositoryDumper struct {
ctx context.Context
baseDir string
repoOwner string
repoName string
opts base.MigrateOptions
milestoneFile *os.File
labelFile *os.File
releaseFile *os.File
issueFile *os.File
commentFiles map[int64]*os.File
pullrequestFile *os.File
reviewFiles map[int64]*os.File
gitRepo *git.Repository
prHeadCache map[string]struct{}
}
// NewRepositoryDumper creates an gitea Uploader
func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) (*RepositoryDumper, error) {
baseDir = filepath.Join(baseDir, repoOwner, repoName)
if err := os.MkdirAll(baseDir, os.ModePerm); err != nil {
return nil, err
}
return &RepositoryDumper{
ctx: ctx,
opts: opts,
baseDir: baseDir,
repoOwner: repoOwner,
repoName: repoName,
prHeadCache: make(map[string]struct{}),
commentFiles: make(map[int64]*os.File),
reviewFiles: make(map[int64]*os.File),
}, nil
}
// MaxBatchInsertSize returns the table's max batch insert size
func (g *RepositoryDumper) MaxBatchInsertSize(tp string) int {
return 1000
}
func (g *RepositoryDumper) gitPath() string {
return filepath.Join(g.baseDir, "git")
}
func (g *RepositoryDumper) wikiPath() string {
return filepath.Join(g.baseDir, "wiki")
}
func (g *RepositoryDumper) commentDir() string {
return filepath.Join(g.baseDir, "comments")
}
func (g *RepositoryDumper) reviewDir() string {
return filepath.Join(g.baseDir, "reviews")
}
func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) {
if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword)
if len(g.opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", g.opts.AuthToken)
}
remoteAddr = u.String()
}
return remoteAddr, nil
}
// CreateRepo creates a repository
func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
f, err := os.Create(filepath.Join(g.baseDir, "repo.yml"))
if err != nil {
return err
}
defer f.Close()
bs, err := yaml.Marshal(map[string]interface{}{
"name": repo.Name,
"owner": repo.Owner,
"description": repo.Description,
"clone_addr": opts.CloneAddr,
"original_url": repo.OriginalURL,
"is_private": opts.Private,
"service_type": opts.GitServiceType,
"wiki": opts.Wiki,
"issues": opts.Issues,
"milestones": opts.Milestones,
"labels": opts.Labels,
"releases": opts.Releases,
"comments": opts.Comments,
"pulls": opts.PullRequests,
"assets": opts.ReleaseAssets,
})
if err != nil {
return err
}
if _, err := f.Write(bs); err != nil {
return err
}
repoPath := g.gitPath()
if err := os.MkdirAll(repoPath, os.ModePerm); err != nil {
return err
}
migrateTimeout := 2 * time.Hour
remoteAddr, err := g.setURLToken(repo.CloneURL)
if err != nil {
return err
}
err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
})
if err != nil {
return fmt.Errorf("Clone: %v", err)
}
if opts.Wiki {
wikiPath := g.wikiPath()
wikiRemotePath := repository.WikiRemoteURL(remoteAddr)
if len(wikiRemotePath) > 0 {
if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil {
return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
if err := git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
Mirror: true,
Quiet: true,
Timeout: migrateTimeout,
Branch: "master",
}); err != nil {
log.Warn("Clone wiki: %v", err)
if err := os.RemoveAll(wikiPath); err != nil {
return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
}
}
}
}
g.gitRepo, err = git.OpenRepository(g.gitPath())
return err
}
// Close closes this uploader
func (g *RepositoryDumper) Close() {
if g.gitRepo != nil {
g.gitRepo.Close()
}
if g.milestoneFile != nil {
g.milestoneFile.Close()
}
if g.labelFile != nil {
g.labelFile.Close()
}
if g.releaseFile != nil {
g.releaseFile.Close()
}
if g.issueFile != nil {
g.issueFile.Close()
}
for _, f := range g.commentFiles {
f.Close()
}
if g.pullrequestFile != nil {
g.pullrequestFile.Close()
}
for _, f := range g.reviewFiles {
f.Close()
}
}
// CreateTopics creates topics
func (g *RepositoryDumper) CreateTopics(topics ...string) error {
f, err := os.Create(filepath.Join(g.baseDir, "topic.yml"))
if err != nil {
return err
}
defer f.Close()
bs, err := yaml.Marshal(map[string]interface{}{
"topics": topics,
})
if err != nil {
return err
}
if _, err := f.Write(bs); err != nil {
return err
}
return nil
}
// CreateMilestones creates milestones
func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error {
var err error
if g.milestoneFile == nil {
g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml"))
if err != nil {
return err
}
}
bs, err := yaml.Marshal(milestones)
if err != nil {
return err
}
if _, err := g.milestoneFile.Write(bs); err != nil {
return err
}
return nil
}
// CreateLabels creates labels
func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error {
var err error
if g.labelFile == nil {
g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml"))
if err != nil {
return err
}
}
bs, err := yaml.Marshal(labels)
if err != nil {
return err
}
if _, err := g.labelFile.Write(bs); err != nil {
return err
}
return nil
}
// CreateReleases creates releases
func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error {
if g.opts.ReleaseAssets {
for _, release := range releases {
attachDir := filepath.Join("release_assets", release.TagName)
if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil {
return err
}
for _, asset := range release.Assets {
attachLocalPath := filepath.Join(attachDir, asset.Name)
// download attachment
err := func(attachPath string) error {
var rc io.ReadCloser
var err error
if asset.DownloadURL == nil {
rc, err = asset.DownloadFunc()
if err != nil {
return err
}
} else {
resp, err := http.Get(*asset.DownloadURL)
if err != nil {
return err
}
rc = resp.Body
}
defer rc.Close()
fw, err := os.Create(attachPath)
if err != nil {
return fmt.Errorf("Create: %v", err)
}
defer fw.Close()
_, err = io.Copy(fw, rc)
return err
}(filepath.Join(g.baseDir, attachLocalPath))
if err != nil {
return err
}
asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source
}
}
}
var err error
if g.releaseFile == nil {
g.releaseFile, err = os.Create(filepath.Join(g.baseDir, "release.yml"))
if err != nil {
return err
}
}
bs, err := yaml.Marshal(releases)
if err != nil {
return err
}
if _, err := g.releaseFile.Write(bs); err != nil {
return err
}
return nil
}
// SyncTags syncs releases with tags in the database
func (g *RepositoryDumper) SyncTags() error {
return nil
}
// CreateIssues creates issues
func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error {
var err error
if g.issueFile == nil {
g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml"))
if err != nil {
return err
}
}
bs, err := yaml.Marshal(issues)
if err != nil {
return err
}
if _, err := g.issueFile.Write(bs); err != nil {
return err
}
return nil
}
func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File, itemsMap map[int64][]interface{}) error {
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return err
}
for number, items := range itemsMap {
var err error
itemFile := itemFiles[number]
if itemFile == nil {
itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number)))
if err != nil {
return err
}
itemFiles[number] = itemFile
}
bs, err := yaml.Marshal(items)
if err != nil {
return err
}
if _, err := itemFile.Write(bs); err != nil {
return err
}
}
return nil
}
// CreateComments creates comments of issues
func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error {
var commentsMap = make(map[int64][]interface{}, len(comments))
for _, comment := range comments {
commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment)
}
return g.createItems(g.commentDir(), g.commentFiles, commentsMap)
}
// CreatePullRequests creates pull requests
func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
for _, pr := range prs {
// download patch file
err := func() error {
u, err := g.setURLToken(pr.PatchURL)
if err != nil {
return err
}
resp, err := http.Get(u)
if err != nil {
return err
}
defer resp.Body.Close()
pullDir := filepath.Join(g.gitPath(), "pulls")
if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
return err
}
fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))
f, err := os.Create(fPath)
if err != nil {
return err
}
defer f.Close()
if _, err = io.Copy(f, resp.Body); err != nil {
return err
}
pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number)
return nil
}()
if err != nil {
return err
}
// set head information
pullHead := filepath.Join(g.gitPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
return err
}
p, err := os.Create(filepath.Join(pullHead, "head"))
if err != nil {
return err
}
_, err = p.WriteString(pr.Head.SHA)
p.Close()
if err != nil {
return err
}
if pr.IsForkPullRequest() && pr.State != "closed" {
if pr.Head.OwnerName != "" {
remote := pr.Head.OwnerName
_, ok := g.prHeadCache[remote]
if !ok {
// git remote add
// TODO: how to handle private CloneURL?
err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
if err != nil {
log.Error("AddRemote failed: %s", err)
} else {
g.prHeadCache[remote] = struct{}{}
ok = true
}
}
if ok {
_, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.gitPath())
if err != nil {
log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
} else {
// a new branch name with <original_owner_name/original_branchname> will be created to as new head branch
ref := path.Join(pr.Head.OwnerName, pr.Head.Ref)
headBranch := filepath.Join(g.gitPath(), "refs", "heads", ref)
if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
return err
}
b, err := os.Create(headBranch)
if err != nil {
return err
}
_, err = b.WriteString(pr.Head.SHA)
b.Close()
if err != nil {
return err
}
pr.Head.Ref = ref
}
}
}
}
// whatever it's a forked repo PR, we have to change head info as the same as the base info
pr.Head.OwnerName = pr.Base.OwnerName
pr.Head.RepoName = pr.Base.RepoName
}
var err error
if g.pullrequestFile == nil {
if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil {
return err
}
g.pullrequestFile, err = os.Create(filepath.Join(g.baseDir, "pull_request.yml"))
if err != nil {
return err
}
}
bs, err := yaml.Marshal(prs)
if err != nil {
return err
}
if _, err := g.pullrequestFile.Write(bs); err != nil {
return err
}
return nil
}
// CreateReviews create pull request reviews
func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error {
var reviewsMap = make(map[int64][]interface{}, len(reviews))
for _, review := range reviews {
reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review)
}
return g.createItems(g.reviewDir(), g.reviewFiles, reviewsMap)
}
// Rollback when migrating failed, this will rollback all the changes.
func (g *RepositoryDumper) Rollback() error {
g.Close()
return os.RemoveAll(g.baseDir)
}
// Finish when migrating succeed, this will update something.
func (g *RepositoryDumper) Finish() error {
return nil
}
// DumpRepository dump repository according MigrateOptions to a local directory
func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error {
downloader, err := newDownloader(ctx, ownerName, opts)
if err != nil {
return err
}
uploader, err := NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts)
if err != nil {
return err
}
if err := migrateRepository(downloader, uploader, opts, nil); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)
}
return err
}
return nil
}
func updateOptionsUnits(opts *base.MigrateOptions, units []string) {
if len(units) == 0 {
opts.Wiki = true
opts.Issues = true
opts.Milestones = true
opts.Labels = true
opts.Releases = true
opts.Comments = true
opts.PullRequests = true
opts.ReleaseAssets = true
} else {
for _, unit := range units {
switch strings.ToLower(unit) {
case "wiki":
opts.Wiki = true
case "issues":
opts.Issues = true
case "milestones":
opts.Milestones = true
case "labels":
opts.Labels = true
case "releases":
opts.Releases = true
case "release_assets":
opts.ReleaseAssets = true
case "comments":
opts.Comments = true
case "pull_requests":
opts.PullRequests = true
}
}
}
}
// RestoreRepository restore a repository from the disk directory
func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string, units []string) error {
doer, err := models.GetAdminUser()
if err != nil {
return err
}
var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName)
downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName)
if err != nil {
return err
}
opts, err := downloader.getRepoOptions()
if err != nil {
return err
}
tp, _ := strconv.Atoi(opts["service_type"])
var migrateOpts = base.MigrateOptions{
GitServiceType: structs.GitServiceType(tp),
}
updateOptionsUnits(&migrateOpts, units)
if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)
}
return err
}
return updateMigrationPosterIDByGitService(ctx, structs.GitServiceType(tp))
}

29
services/migrations/error.go Normal file
View file

@ -0,0 +1,29 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"errors"
"github.com/google/go-github/v39/github"
)
var (
// ErrRepoNotCreated returns the error that repository not created
ErrRepoNotCreated = errors.New("repository is not created yet")
)
// IsRateLimitError returns true if the err is github.RateLimitError
func IsRateLimitError(err error) bool {
_, ok := err.(*github.RateLimitError)
return ok
}
// IsTwoFactorAuthError returns true if the err is github.TwoFactorAuthError
func IsTwoFactorAuthError(err error) bool {
_, ok := err.(*github.TwoFactorAuthError)
return ok
}

51
services/migrations/git.go Normal file
View file

@ -0,0 +1,51 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
base "code.gitea.io/gitea/modules/migration"
)
var (
_ base.Downloader = &PlainGitDownloader{}
)
// PlainGitDownloader implements a Downloader interface to clone git from a http/https URL
type PlainGitDownloader struct {
base.NullDownloader
ownerName string
repoName string
remoteURL string
}
// NewPlainGitDownloader creates a git Downloader
func NewPlainGitDownloader(ownerName, repoName, remoteURL string) *PlainGitDownloader {
return &PlainGitDownloader{
ownerName: ownerName,
repoName: repoName,
remoteURL: remoteURL,
}
}
// SetContext set context
func (g *PlainGitDownloader) SetContext(ctx context.Context) {
}
// GetRepoInfo returns a repository information
func (g *PlainGitDownloader) GetRepoInfo() (*base.Repository, error) {
// convert github repo to stand Repo
return &base.Repository{
Owner: g.ownerName,
Name: g.repoName,
CloneURL: g.remoteURL,
}, nil
}
// GetTopics return empty string slice
func (g PlainGitDownloader) GetTopics() ([]string, error) {
return []string{}, nil
}

72
services/migrations/gitbucket.go Normal file
View file

@ -0,0 +1,72 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"net/url"
"strings"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/structs"
)
var (
_ base.Downloader = &GitBucketDownloader{}
_ base.DownloaderFactory = &GitBucketDownloaderFactory{}
)
func init() {
RegisterDownloaderFactory(&GitBucketDownloaderFactory{})
}
// GitBucketDownloaderFactory defines a GitBucket downloader factory
type GitBucketDownloaderFactory struct {
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GitBucketDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return nil, err
}
baseURL := u.Scheme + "://" + u.Host
fields := strings.Split(u.Path, "/")
oldOwner := fields[1]
oldName := strings.TrimSuffix(fields[2], ".git")
return NewGitBucketDownloader(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
}
// GitServiceType returns the type of git service
func (f *GitBucketDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.GitBucketService
}
// GitBucketDownloader implements a Downloader interface to get repository information
// from GitBucket via GithubDownloader
type GitBucketDownloader struct {
*GithubDownloaderV3
}
// NewGitBucketDownloader creates a GitBucket downloader
func NewGitBucketDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GitBucketDownloader {
githubDownloader := NewGithubDownloaderV3(ctx, baseURL, userName, password, token, repoOwner, repoName)
githubDownloader.SkipReactions = true
return &GitBucketDownloader{
githubDownloader,
}
}
// SupportGetRepoComments return true if it supports get repo comments
func (g *GitBucketDownloader) SupportGetRepoComments() bool {
return false
}
// GetReviews is not supported
func (g *GitBucketDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
return nil, &base.ErrNotSupported{Entity: "Reviews"}
}

699
services/migrations/gitea_downloader.go Normal file
View file

@ -0,0 +1,699 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"strings"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
gitea_sdk "code.gitea.io/sdk/gitea"
)
var (
_ base.Downloader = &GiteaDownloader{}
_ base.DownloaderFactory = &GiteaDownloaderFactory{}
)
func init() {
RegisterDownloaderFactory(&GiteaDownloaderFactory{})
}
// GiteaDownloaderFactory defines a gitea downloader factory
type GiteaDownloaderFactory struct {
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GiteaDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return nil, err
}
baseURL := u.Scheme + "://" + u.Host
repoNameSpace := strings.TrimPrefix(u.Path, "/")
repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git")
path := strings.Split(repoNameSpace, "/")
if len(path) < 2 {
return nil, fmt.Errorf("invalid path: %s", repoNameSpace)
}
repoPath := strings.Join(path[len(path)-2:], "/")
if len(path) > 2 {
subPath := strings.Join(path[:len(path)-2], "/")
baseURL += "/" + subPath
}
log.Trace("Create gitea downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
return NewGiteaDownloader(ctx, baseURL, repoPath, opts.AuthUsername, opts.AuthPassword, opts.AuthToken)
}
// GitServiceType returns the type of git service
func (f *GiteaDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.GiteaService
}
// GiteaDownloader implements a Downloader interface to get repository information's
type GiteaDownloader struct {
base.NullDownloader
ctx context.Context
client *gitea_sdk.Client
repoOwner string
repoName string
pagination bool
maxPerPage int
}
// NewGiteaDownloader creates a gitea Downloader via gitea API
// Use either a username/password or personal token. token is preferred
// Note: Public access only allows very basic access
func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GiteaDownloader, error) {
giteaClient, err := gitea_sdk.NewClient(
baseURL,
gitea_sdk.SetToken(token),
gitea_sdk.SetBasicAuth(username, password),
gitea_sdk.SetContext(ctx),
gitea_sdk.SetHTTPClient(&http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}),
)
if err != nil {
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
return nil, err
}
path := strings.Split(repoPath, "/")
paginationSupport := true
if err = giteaClient.CheckServerVersionConstraint(">=1.12"); err != nil {
paginationSupport = false
}
// set small maxPerPage since we can only guess
// (default would be 50 but this can differ)
maxPerPage := 10
// gitea instances >=1.13 can tell us what maximum they have
apiConf, _, err := giteaClient.GetGlobalAPISettings()
if err != nil {
log.Info("Unable to get global API settings. Ignoring these.")
log.Debug("giteaClient.GetGlobalAPISettings. Error: %v", err)
}
if apiConf != nil {
maxPerPage = apiConf.MaxResponseItems
}
return &GiteaDownloader{
ctx: ctx,
client: giteaClient,
repoOwner: path[0],
repoName: path[1],
pagination: paginationSupport,
maxPerPage: maxPerPage,
}, nil
}
// SetContext set context
func (g *GiteaDownloader) SetContext(ctx context.Context) {
g.ctx = ctx
}
// GetRepoInfo returns a repository information
func (g *GiteaDownloader) GetRepoInfo() (*base.Repository, error) {
if g == nil {
return nil, errors.New("error: GiteaDownloader is nil")
}
repo, _, err := g.client.GetRepo(g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
return &base.Repository{
Name: repo.Name,
Owner: repo.Owner.UserName,
IsPrivate: repo.Private,
Description: repo.Description,
CloneURL: repo.CloneURL,
OriginalURL: repo.HTMLURL,
DefaultBranch: repo.DefaultBranch,
}, nil
}
// GetTopics return gitea topics
func (g *GiteaDownloader) GetTopics() ([]string, error) {
topics, _, err := g.client.ListRepoTopics(g.repoOwner, g.repoName, gitea_sdk.ListRepoTopicsOptions{})
return topics, err
}
// GetMilestones returns milestones
func (g *GiteaDownloader) GetMilestones() ([]*base.Milestone, error) {
var milestones = make([]*base.Milestone, 0, g.maxPerPage)
for i := 1; ; i++ {
// make sure gitea can shutdown gracefully
select {
case <-g.ctx.Done():
return nil, nil
default:
}
ms, _, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName, gitea_sdk.ListMilestoneOption{
ListOptions: gitea_sdk.ListOptions{
PageSize: g.maxPerPage,
Page: i,
},
State: gitea_sdk.StateAll,
})
if err != nil {
return nil, err
}
for i := range ms {
// old gitea instances dont have this information
createdAT := time.Time{}
var updatedAT *time.Time
if ms[i].Closed != nil {
createdAT = *ms[i].Closed
updatedAT = ms[i].Closed
}
// new gitea instances (>=1.13) do
if !ms[i].Created.IsZero() {
createdAT = ms[i].Created
}
if ms[i].Updated != nil && !ms[i].Updated.IsZero() {
updatedAT = ms[i].Updated
}
milestones = append(milestones, &base.Milestone{
Title: ms[i].Title,
Description: ms[i].Description,
Deadline: ms[i].Deadline,
Created: createdAT,
Updated: updatedAT,
Closed: ms[i].Closed,
State: string(ms[i].State),
})
}
if !g.pagination || len(ms) < g.maxPerPage {
break
}
}
return milestones, nil
}
func (g *GiteaDownloader) convertGiteaLabel(label *gitea_sdk.Label) *base.Label {
return &base.Label{
Name: label.Name,
Color: label.Color,
Description: label.Description,
}
}
// GetLabels returns labels
func (g *GiteaDownloader) GetLabels() ([]*base.Label, error) {
var labels = make([]*base.Label, 0, g.maxPerPage)
for i := 1; ; i++ {
// make sure gitea can shutdown gracefully
select {
case <-g.ctx.Done():
return nil, nil
default:
}
ls, _, err := g.client.ListRepoLabels(g.repoOwner, g.repoName, gitea_sdk.ListLabelsOptions{ListOptions: gitea_sdk.ListOptions{
PageSize: g.maxPerPage,
Page: i,
}})
if err != nil {
return nil, err
}
for i := range ls {
labels = append(labels, g.convertGiteaLabel(ls[i]))
}
if !g.pagination || len(ls) < g.maxPerPage {
break
}
}
return labels, nil
}
func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Release {
r := &base.Release{
TagName: rel.TagName,
TargetCommitish: rel.Target,
Name: rel.Title,
Body: rel.Note,
Draft: rel.IsDraft,
Prerelease: rel.IsPrerelease,
PublisherID: rel.Publisher.ID,
PublisherName: rel.Publisher.UserName,
PublisherEmail: rel.Publisher.Email,
Published: rel.PublishedAt,
Created: rel.CreatedAt,
}
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
for _, asset := range rel.Attachments {
size := int(asset.Size)
dlCount := int(asset.DownloadCount)
r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: asset.ID,
Name: asset.Name,
Size: &size,
DownloadCount: &dlCount,
Created: asset.Created,
DownloadURL: &asset.DownloadURL,
DownloadFunc: func() (io.ReadCloser, error) {
asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, rel.ID, asset.ID)
if err != nil {
return nil, err
}
// FIXME: for a private download?
req, err := http.NewRequest("GET", asset.DownloadURL, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
// resp.Body is closed by the uploader
return resp.Body, nil
},
})
}
return r
}
// GetReleases returns releases
func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) {
var releases = make([]*base.Release, 0, g.maxPerPage)
for i := 1; ; i++ {
// make sure gitea can shutdown gracefully
select {
case <-g.ctx.Done():
return nil, nil
default:
}
rl, _, err := g.client.ListReleases(g.repoOwner, g.repoName, gitea_sdk.ListReleasesOptions{ListOptions: gitea_sdk.ListOptions{
PageSize: g.maxPerPage,
Page: i,
}})
if err != nil {
return nil, err
}
for i := range rl {
releases = append(releases, g.convertGiteaRelease(rl[i]))
}
if !g.pagination || len(rl) < g.maxPerPage {
break
}
}
return releases, nil
}
func (g *GiteaDownloader) getIssueReactions(index int64) ([]*base.Reaction, error) {
var reactions []*base.Reaction
if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {
log.Info("GiteaDownloader: instance to old, skip getIssueReactions")
return reactions, nil
}
rl, _, err := g.client.GetIssueReactions(g.repoOwner, g.repoName, index)
if err != nil {
return nil, err
}
for _, reaction := range rl {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.ID,
UserName: reaction.User.UserName,
Content: reaction.Reaction,
})
}
return reactions, nil
}
func (g *GiteaDownloader) getCommentReactions(commentID int64) ([]*base.Reaction, error) {
var reactions []*base.Reaction
if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {
log.Info("GiteaDownloader: instance to old, skip getCommentReactions")
return reactions, nil
}
rl, _, err := g.client.GetIssueCommentReactions(g.repoOwner, g.repoName, commentID)
if err != nil {
return nil, err
}
for i := range rl {
reactions = append(reactions, &base.Reaction{
UserID: rl[i].User.ID,
UserName: rl[i].User.UserName,
Content: rl[i].Reaction,
})
}
return reactions, nil
}
// GetIssues returns issues according start and limit
func (g *GiteaDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
var allIssues = make([]*base.Issue, 0, perPage)
issues, _, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gitea_sdk.ListIssueOption{
ListOptions: gitea_sdk.ListOptions{Page: page, PageSize: perPage},
State: gitea_sdk.StateAll,
Type: gitea_sdk.IssueTypeIssue,
})
if err != nil {
return nil, false, fmt.Errorf("error while listing issues: %v", err)
}
for _, issue := range issues {
var labels = make([]*base.Label, 0, len(issue.Labels))
for i := range issue.Labels {
labels = append(labels, g.convertGiteaLabel(issue.Labels[i]))
}
var milestone string
if issue.Milestone != nil {
milestone = issue.Milestone.Title
}
reactions, err := g.getIssueReactions(issue.Index)
if err != nil {
log.Warn("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err)
if err2 := models.CreateRepositoryNotice(
fmt.Sprintf("Unable to load reactions during migrating issue #%d to %s/%s. Error: %v", issue.Index, g.repoOwner, g.repoName, err)); err2 != nil {
log.Error("create repository notice failed: ", err2)
}
}
var assignees []string
for i := range issue.Assignees {
assignees = append(assignees, issue.Assignees[i].UserName)
}
allIssues = append(allIssues, &base.Issue{
Title: issue.Title,
Number: issue.Index,
PosterID: issue.Poster.ID,
PosterName: issue.Poster.UserName,
PosterEmail: issue.Poster.Email,
Content: issue.Body,
Milestone: milestone,
State: string(issue.State),
Created: issue.Created,
Updated: issue.Updated,
Closed: issue.Closed,
Reactions: reactions,
Labels: labels,
Assignees: assignees,
IsLocked: issue.IsLocked,
Context: base.BasicIssueContext(issue.Index),
})
}
isEnd := len(issues) < perPage
if !g.pagination {
isEnd = len(issues) == 0
}
return allIssues, isEnd, nil
}
// GetComments returns comments according issueNumber
func (g *GiteaDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
var allComments = make([]*base.Comment, 0, g.maxPerPage)
for i := 1; ; i++ {
// make sure gitea can shutdown gracefully
select {
case <-g.ctx.Done():
return nil, false, nil
default:
}
comments, _, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID(), gitea_sdk.ListIssueCommentOptions{ListOptions: gitea_sdk.ListOptions{
PageSize: g.maxPerPage,
Page: i,
}})
if err != nil {
return nil, false, fmt.Errorf("error while listing comments for issue #%d. Error: %v", opts.Context.ForeignID(), err)
}
for _, comment := range comments {
reactions, err := g.getCommentReactions(comment.ID)
if err != nil {
log.Warn("Unable to load comment reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)
if err2 := models.CreateRepositoryNotice(
fmt.Sprintf("Unable to load reactions during migrating issue #%d for comment %d to %s/%s. Error: %v", opts.Context.ForeignID(), comment.ID, g.repoOwner, g.repoName, err)); err2 != nil {
log.Error("create repository notice failed: ", err2)
}
}
allComments = append(allComments, &base.Comment{
IssueIndex: opts.Context.LocalID(),
PosterID: comment.Poster.ID,
PosterName: comment.Poster.UserName,
PosterEmail: comment.Poster.Email,
Content: comment.Body,
Created: comment.Created,
Updated: comment.Updated,
Reactions: reactions,
})
}
if !g.pagination || len(comments) < g.maxPerPage {
break
}
}
return allComments, true, nil
}
// GetPullRequests returns pull requests according page and perPage
func (g *GiteaDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
var allPRs = make([]*base.PullRequest, 0, perPage)
prs, _, err := g.client.ListRepoPullRequests(g.repoOwner, g.repoName, gitea_sdk.ListPullRequestsOptions{
ListOptions: gitea_sdk.ListOptions{
Page: page,
PageSize: perPage,
},
State: gitea_sdk.StateAll,
})
if err != nil {
return nil, false, fmt.Errorf("error while listing pull requests (page: %d, pagesize: %d). Error: %v", page, perPage, err)
}
for _, pr := range prs {
var milestone string
if pr.Milestone != nil {
milestone = pr.Milestone.Title
}
var labels = make([]*base.Label, 0, len(pr.Labels))
for i := range pr.Labels {
labels = append(labels, g.convertGiteaLabel(pr.Labels[i]))
}
var (
headUserName string
headRepoName string
headCloneURL string
headRef string
headSHA string
)
if pr.Head != nil {
if pr.Head.Repository != nil {
headUserName = pr.Head.Repository.Owner.UserName
headRepoName = pr.Head.Repository.Name
headCloneURL = pr.Head.Repository.CloneURL
}
headSHA = pr.Head.Sha
headRef = pr.Head.Ref
}
var mergeCommitSHA string
if pr.MergedCommitID != nil {
mergeCommitSHA = *pr.MergedCommitID
}
reactions, err := g.getIssueReactions(pr.Index)
if err != nil {
log.Warn("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err)
if err2 := models.CreateRepositoryNotice(
fmt.Sprintf("Unable to load reactions during migrating pull #%d to %s/%s. Error: %v", pr.Index, g.repoOwner, g.repoName, err)); err2 != nil {
log.Error("create repository notice failed: ", err2)
}
}
var assignees []string
for i := range pr.Assignees {
assignees = append(assignees, pr.Assignees[i].UserName)
}
createdAt := time.Time{}
if pr.Created != nil {
createdAt = *pr.Created
}
updatedAt := time.Time{}
if pr.Created != nil {
updatedAt = *pr.Updated
}
closedAt := pr.Closed
if pr.Merged != nil && closedAt == nil {
closedAt = pr.Merged
}
allPRs = append(allPRs, &base.PullRequest{
Title: pr.Title,
Number: pr.Index,
PosterID: pr.Poster.ID,
PosterName: pr.Poster.UserName,
PosterEmail: pr.Poster.Email,
Content: pr.Body,
State: string(pr.State),
Created: createdAt,
Updated: updatedAt,
Closed: closedAt,
Labels: labels,
Milestone: milestone,
Reactions: reactions,
Assignees: assignees,
Merged: pr.HasMerged,
MergedTime: pr.Merged,
MergeCommitSHA: mergeCommitSHA,
IsLocked: pr.IsLocked,
PatchURL: pr.PatchURL,
Head: base.PullRequestBranch{
Ref: headRef,
SHA: headSHA,
RepoName: headRepoName,
OwnerName: headUserName,
CloneURL: headCloneURL,
},
Base: base.PullRequestBranch{
Ref: pr.Base.Ref,
SHA: pr.Base.Sha,
RepoName: g.repoName,
OwnerName: g.repoOwner,
},
Context: base.BasicIssueContext(pr.Index),
})
}
isEnd := len(prs) < perPage
if !g.pagination {
isEnd = len(prs) == 0
}
return allPRs, isEnd, nil
}
// GetReviews returns pull requests review
func (g *GiteaDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
if err := g.client.CheckServerVersionConstraint(">=1.12"); err != nil {
log.Info("GiteaDownloader: instance to old, skip GetReviews")
return nil, nil
}
var allReviews = make([]*base.Review, 0, g.maxPerPage)
for i := 1; ; i++ {
// make sure gitea can shutdown gracefully
select {
case <-g.ctx.Done():
return nil, nil
default:
}
prl, _, err := g.client.ListPullReviews(g.repoOwner, g.repoName, context.ForeignID(), gitea_sdk.ListPullReviewsOptions{ListOptions: gitea_sdk.ListOptions{
Page: i,
PageSize: g.maxPerPage,
}})
if err != nil {
return nil, err
}
for _, pr := range prl {
rcl, _, err := g.client.ListPullReviewComments(g.repoOwner, g.repoName, context.ForeignID(), pr.ID)
if err != nil {
return nil, err
}
var reviewComments []*base.ReviewComment
for i := range rcl {
line := int(rcl[i].LineNum)
if rcl[i].OldLineNum > 0 {
line = int(rcl[i].OldLineNum) * -1
}
reviewComments = append(reviewComments, &base.ReviewComment{
ID: rcl[i].ID,
Content: rcl[i].Body,
TreePath: rcl[i].Path,
DiffHunk: rcl[i].DiffHunk,
Line: line,
CommitID: rcl[i].CommitID,
PosterID: rcl[i].Reviewer.ID,
CreatedAt: rcl[i].Created,
UpdatedAt: rcl[i].Updated,
})
}
allReviews = append(allReviews, &base.Review{
ID: pr.ID,
IssueIndex: context.LocalID(),
ReviewerID: pr.Reviewer.ID,
ReviewerName: pr.Reviewer.UserName,
Official: pr.Official,
CommitID: pr.CommitID,
Content: pr.Body,
CreatedAt: pr.Submitted,
State: string(pr.State),
Comments: reviewComments,
})
}
if len(prl) < g.maxPerPage {
break
}
}
return allReviews, nil
}

318
services/migrations/gitea_downloader_test.go Normal file
View file

@ -0,0 +1,318 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"net/http"
"os"
"sort"
"testing"
"time"
base "code.gitea.io/gitea/modules/migration"
"github.com/stretchr/testify/assert"
)
func TestGiteaDownloadRepo(t *testing.T) {
// Skip tests if Gitea token is not found
giteaToken := os.Getenv("GITEA_TOKEN")
if giteaToken == "" {
t.Skip("skipped test because GITEA_TOKEN was not in the environment")
}
resp, err := http.Get("https://gitea.com/gitea")
if err != nil || resp.StatusCode != 200 {
t.Skipf("Can't reach https://gitea.com, skipping %s", t.Name())
}
downloader, err := NewGiteaDownloader(context.Background(), "https://gitea.com", "gitea/test_repo", "", "", giteaToken)
if downloader == nil {
t.Fatal("NewGitlabDownloader is nil")
}
if !assert.NoError(t, err) {
t.Fatal("NewGitlabDownloader error occur")
}
repo, err := downloader.GetRepoInfo()
assert.NoError(t, err)
assertRepositoryEqual(t, &base.Repository{
Name: "test_repo",
Owner: "gitea",
IsPrivate: false,
Description: "Test repository for testing migration from gitea to gitea",
CloneURL: "https://gitea.com/gitea/test_repo.git",
OriginalURL: "https://gitea.com/gitea/test_repo",
DefaultBranch: "master",
}, repo)
topics, err := downloader.GetTopics()
assert.NoError(t, err)
sort.Strings(topics)
assert.EqualValues(t, []string{"ci", "gitea", "migration", "test"}, topics)
labels, err := downloader.GetLabels()
assert.NoError(t, err)
assertLabelsEqual(t, []*base.Label{
{
Name: "Bug",
Color: "e11d21",
},
{
Name: "Enhancement",
Color: "207de5",
},
{
Name: "Feature",
Color: "0052cc",
Description: "a feature request",
},
{
Name: "Invalid",
Color: "d4c5f9",
},
{
Name: "Question",
Color: "fbca04",
},
{
Name: "Valid",
Color: "53e917",
},
}, labels)
milestones, err := downloader.GetMilestones()
assert.NoError(t, err)
assertMilestonesEqual(t, []*base.Milestone{
{
Title: "V2 Finalize",
Created: time.Unix(0, 0),
Deadline: timePtr(time.Unix(1599263999, 0)),
Updated: timePtr(time.Unix(0, 0)),
State: "open",
},
{
Title: "V1",
Description: "Generate Content",
Created: time.Unix(0, 0),
Updated: timePtr(time.Unix(0, 0)),
Closed: timePtr(time.Unix(1598985406, 0)),
State: "closed",
},
}, milestones)
releases, err := downloader.GetReleases()
assert.NoError(t, err)
assertReleasesEqual(t, []*base.Release{
{
Name: "Second Release",
TagName: "v2-rc1",
TargetCommitish: "master",
Body: "this repo has:\r\n* reactions\r\n* wiki\r\n* issues (open/closed)\r\n* pulls (open/closed/merged) (external/internal)\r\n* pull reviews\r\n* projects\r\n* milestones\r\n* labels\r\n* releases\r\n\r\nto test migration against",
Draft: false,
Prerelease: true,
Created: time.Date(2020, 9, 1, 18, 2, 43, 0, time.UTC),
Published: time.Date(2020, 9, 1, 18, 2, 43, 0, time.UTC),
PublisherID: 689,
PublisherName: "6543",
PublisherEmail: "6543@obermui.de",
},
{
Name: "First Release",
TagName: "V1",
TargetCommitish: "master",
Body: "as title",
Draft: false,
Prerelease: false,
Created: time.Date(2020, 9, 1, 17, 30, 32, 0, time.UTC),
Published: time.Date(2020, 9, 1, 17, 30, 32, 0, time.UTC),
PublisherID: 689,
PublisherName: "6543",
PublisherEmail: "6543@obermui.de",
},
}, releases)
issues, isEnd, err := downloader.GetIssues(1, 50)
assert.NoError(t, err)
assert.True(t, isEnd)
assert.Len(t, issues, 7)
assert.EqualValues(t, "open", issues[0].State)
issues, isEnd, err = downloader.GetIssues(3, 2)
assert.NoError(t, err)
assert.False(t, isEnd)
assertIssuesEqual(t, []*base.Issue{
{
Number: 4,
Title: "what is this repo about?",
Content: "",
Milestone: "V1",
PosterID: -1,
PosterName: "Ghost",
PosterEmail: "",
State: "closed",
IsLocked: true,
Created: time.Unix(1598975321, 0),
Updated: time.Unix(1598975400, 0),
Labels: []*base.Label{{
Name: "Question",
Color: "fbca04",
Description: "",
}},
Reactions: []*base.Reaction{
{
UserID: 689,
UserName: "6543",
Content: "gitea",
},
{
UserID: 689,
UserName: "6543",
Content: "laugh",
},
},
Closed: timePtr(time.Date(2020, 9, 1, 15, 49, 34, 0, time.UTC)),
},
{
Number: 2,
Title: "Spam",
Content: ":(",
Milestone: "",
PosterID: 689,
PosterName: "6543",
PosterEmail: "6543@obermui.de",
State: "closed",
IsLocked: false,
Created: time.Unix(1598919780, 0),
Updated: time.Unix(1598969497, 0),
Labels: []*base.Label{{
Name: "Invalid",
Color: "d4c5f9",
Description: "",
}},
Closed: timePtr(time.Unix(1598969497, 0)),
},
}, issues)
comments, _, err := downloader.GetComments(base.GetCommentOptions{
Context: base.BasicIssueContext(4),
})
assert.NoError(t, err)
assertCommentsEqual(t, []*base.Comment{
{
IssueIndex: 4,
PosterID: 689,
PosterName: "6543",
PosterEmail: "6543@obermui.de",
Created: time.Unix(1598975370, 0),
Updated: time.Unix(1599070865, 0),
Content: "a really good question!\n\nIt is the used as TESTSET for gitea2gitea repo migration function",
},
{
IssueIndex: 4,
PosterID: -1,
PosterName: "Ghost",
PosterEmail: "",
Created: time.Unix(1598975393, 0),
Updated: time.Unix(1598975393, 0),
Content: "Oh!",
},
}, comments)
prs, isEnd, err := downloader.GetPullRequests(1, 50)
assert.NoError(t, err)
assert.True(t, isEnd)
assert.Len(t, prs, 6)
prs, isEnd, err = downloader.GetPullRequests(1, 3)
assert.NoError(t, err)
assert.False(t, isEnd)
assert.Len(t, prs, 3)
assertPullRequestEqual(t, &base.PullRequest{
Number: 12,
PosterID: 689,
PosterName: "6543",
PosterEmail: "6543@obermui.de",
Title: "Dont Touch",
Content: "\r\nadd dont touch note",
Milestone: "V2 Finalize",
State: "closed",
IsLocked: false,
Created: time.Unix(1598982759, 0),
Updated: time.Unix(1599023425, 0),
Closed: timePtr(time.Unix(1598982934, 0)),
Assignees: []string{"techknowlogick"},
Base: base.PullRequestBranch{
CloneURL: "",
Ref: "master",
SHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
RepoName: "test_repo",
OwnerName: "gitea",
},
Head: base.PullRequestBranch{
CloneURL: "https://gitea.com/6543-forks/test_repo.git",
Ref: "refs/pull/12/head",
SHA: "b6ab5d9ae000b579a5fff03f92c486da4ddf48b6",
RepoName: "test_repo",
OwnerName: "6543-forks",
},
Merged: true,
MergedTime: timePtr(time.Unix(1598982934, 0)),
MergeCommitSHA: "827aa28a907853e5ddfa40c8f9bc52471a2685fd",
PatchURL: "https://gitea.com/gitea/test_repo/pulls/12.patch",
}, prs[1])
reviews, err := downloader.GetReviews(base.BasicIssueContext(7))
assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{
{
ID: 1770,
IssueIndex: 7,
ReviewerID: 689,
ReviewerName: "6543",
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
CreatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
State: "COMMENT", // TODO
Comments: []*base.ReviewComment{
{
ID: 116561,
InReplyTo: 0,
Content: "is one `\\newline` to less?",
TreePath: "README.md",
DiffHunk: "@@ -2,3 +2,3 @@\n \n-Test repository for testing migration from gitea 2 gitea\n\\ No newline at end of file\n+Test repository for testing migration from gitea 2 gitea",
Position: 0,
Line: 4,
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
PosterID: 689,
Reactions: nil,
CreatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
UpdatedAt: time.Date(2020, 9, 1, 16, 12, 58, 0, time.UTC),
},
},
},
{
ID: 1771,
IssueIndex: 7,
ReviewerID: 9,
ReviewerName: "techknowlogick",
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
CreatedAt: time.Date(2020, 9, 1, 17, 6, 47, 0, time.UTC),
State: "REQUEST_CHANGES", // TODO
Content: "I think this needs some changes",
},
{
ID: 1772,
IssueIndex: 7,
ReviewerID: 9,
ReviewerName: "techknowlogick",
CommitID: "187ece0cb6631e2858a6872e5733433bb3ca3b03",
CreatedAt: time.Date(2020, 9, 1, 17, 19, 51, 0, time.UTC),
State: base.ReviewStateApproved,
Official: true,
Content: "looks good",
},
}, reviews)
}

981
services/migrations/gitea_uploader.go Normal file
View file

@ -0,0 +1,981 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"sync"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/git"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
repo_module "code.gitea.io/gitea/modules/repository"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/uri"
"code.gitea.io/gitea/services/pull"
gouuid "github.com/google/uuid"
)
var (
_ base.Uploader = &GiteaLocalUploader{}
)
// GiteaLocalUploader implements an Uploader to gitea sites
type GiteaLocalUploader struct {
ctx context.Context
doer *models.User
repoOwner string
repoName string
repo *models.Repository
labels sync.Map
milestones sync.Map
issues sync.Map
gitRepo *git.Repository
prHeadCache map[string]struct{}
userMap map[int64]int64 // external user id mapping to user id
prCache map[int64]*models.PullRequest
gitServiceType structs.GitServiceType
}
// NewGiteaLocalUploader creates an gitea Uploader via gitea API v1
func NewGiteaLocalUploader(ctx context.Context, doer *models.User, repoOwner, repoName string) *GiteaLocalUploader {
return &GiteaLocalUploader{
ctx: ctx,
doer: doer,
repoOwner: repoOwner,
repoName: repoName,
prHeadCache: make(map[string]struct{}),
userMap: make(map[int64]int64),
prCache: make(map[int64]*models.PullRequest),
}
}
// MaxBatchInsertSize returns the table's max batch insert size
func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
switch tp {
case "issue":
return db.MaxBatchInsertSize(new(models.Issue))
case "comment":
return db.MaxBatchInsertSize(new(models.Comment))
case "milestone":
return db.MaxBatchInsertSize(new(models.Milestone))
case "label":
return db.MaxBatchInsertSize(new(models.Label))
case "release":
return db.MaxBatchInsertSize(new(models.Release))
case "pullrequest":
return db.MaxBatchInsertSize(new(models.PullRequest))
}
return 10
}
// CreateRepo creates a repository
func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
owner, err := models.GetUserByName(g.repoOwner)
if err != nil {
return err
}
var r *models.Repository
if opts.MigrateToRepoID <= 0 {
r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{
Name: g.repoName,
Description: repo.Description,
OriginalURL: repo.OriginalURL,
GitServiceType: opts.GitServiceType,
IsPrivate: opts.Private,
IsMirror: opts.Mirror,
Status: models.RepositoryBeingMigrated,
})
} else {
r, err = models.GetRepositoryByID(opts.MigrateToRepoID)
}
if err != nil {
return err
}
r.DefaultBranch = repo.DefaultBranch
r, err = repo_module.MigrateRepositoryGitData(g.ctx, owner, r, base.MigrateOptions{
RepoName: g.repoName,
Description: repo.Description,
OriginalURL: repo.OriginalURL,
GitServiceType: opts.GitServiceType,
Mirror: repo.IsMirror,
LFS: opts.LFS,
LFSEndpoint: opts.LFSEndpoint,
CloneAddr: repo.CloneURL,
Private: repo.IsPrivate,
Wiki: opts.Wiki,
Releases: opts.Releases, // if didn't get releases, then sync them from tags
MirrorInterval: opts.MirrorInterval,
})
g.repo = r
if err != nil {
return err
}
g.gitRepo, err = git.OpenRepository(r.RepoPath())
return err
}
// Close closes this uploader
func (g *GiteaLocalUploader) Close() {
if g.gitRepo != nil {
g.gitRepo.Close()
}
}
// CreateTopics creates topics
func (g *GiteaLocalUploader) CreateTopics(topics ...string) error {
// ignore topics to long for the db
c := 0
for i := range topics {
if len(topics[i]) <= 50 {
topics[c] = topics[i]
c++
}
}
topics = topics[:c]
return models.SaveTopics(g.repo.ID, topics...)
}
// CreateMilestones creates milestones
func (g *GiteaLocalUploader) CreateMilestones(milestones ...*base.Milestone) error {
var mss = make([]*models.Milestone, 0, len(milestones))
for _, milestone := range milestones {
var deadline timeutil.TimeStamp
if milestone.Deadline != nil {
deadline = timeutil.TimeStamp(milestone.Deadline.Unix())
}
if deadline == 0 {
deadline = timeutil.TimeStamp(time.Date(9999, 1, 1, 0, 0, 0, 0, setting.DefaultUILocation).Unix())
}
if milestone.Created.IsZero() {
if milestone.Updated != nil {
milestone.Created = *milestone.Updated
} else if milestone.Deadline != nil {
milestone.Created = *milestone.Deadline
} else {
milestone.Created = time.Now()
}
}
if milestone.Updated == nil || milestone.Updated.IsZero() {
milestone.Updated = &milestone.Created
}
var ms = models.Milestone{
RepoID: g.repo.ID,
Name: milestone.Title,
Content: milestone.Description,
IsClosed: milestone.State == "closed",
CreatedUnix: timeutil.TimeStamp(milestone.Created.Unix()),
UpdatedUnix: timeutil.TimeStamp(milestone.Updated.Unix()),
DeadlineUnix: deadline,
}
if ms.IsClosed && milestone.Closed != nil {
ms.ClosedDateUnix = timeutil.TimeStamp(milestone.Closed.Unix())
}
mss = append(mss, &ms)
}
err := models.InsertMilestones(mss...)
if err != nil {
return err
}
for _, ms := range mss {
g.milestones.Store(ms.Name, ms.ID)
}
return nil
}
// CreateLabels creates labels
func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
var lbs = make([]*models.Label, 0, len(labels))
for _, label := range labels {
lbs = append(lbs, &models.Label{
RepoID: g.repo.ID,
Name: label.Name,
Description: label.Description,
Color: fmt.Sprintf("#%s", label.Color),
})
}
err := models.NewLabels(lbs...)
if err != nil {
return err
}
for _, lb := range lbs {
g.labels.Store(lb.Name, lb)
}
return nil
}
// CreateReleases creates releases
func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
var rels = make([]*models.Release, 0, len(releases))
for _, release := range releases {
if release.Created.IsZero() {
if !release.Published.IsZero() {
release.Created = release.Published
} else {
release.Created = time.Now()
}
}
var rel = models.Release{
RepoID: g.repo.ID,
TagName: release.TagName,
LowerTagName: strings.ToLower(release.TagName),
Target: release.TargetCommitish,
Title: release.Name,
Sha1: release.TargetCommitish,
Note: release.Body,
IsDraft: release.Draft,
IsPrerelease: release.Prerelease,
IsTag: false,
CreatedUnix: timeutil.TimeStamp(release.Created.Unix()),
}
userid, ok := g.userMap[release.PublisherID]
tp := g.gitServiceType.Name()
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", release.PublisherID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[release.PublisherID] = userid
}
}
if userid > 0 {
rel.PublisherID = userid
} else {
rel.PublisherID = g.doer.ID
rel.OriginalAuthor = release.PublisherName
rel.OriginalAuthorID = release.PublisherID
}
// calc NumCommits if no draft
if !release.Draft {
commit, err := g.gitRepo.GetTagCommit(rel.TagName)
if err != nil {
return fmt.Errorf("GetTagCommit[%v]: %v", rel.TagName, err)
}
rel.NumCommits, err = commit.CommitsCount()
if err != nil {
return fmt.Errorf("CommitsCount: %v", err)
}
}
for _, asset := range release.Assets {
if asset.Created.IsZero() {
if !asset.Updated.IsZero() {
asset.Created = asset.Updated
} else {
asset.Created = release.Created
}
}
var attach = models.Attachment{
UUID: gouuid.New().String(),
Name: asset.Name,
DownloadCount: int64(*asset.DownloadCount),
Size: int64(*asset.Size),
CreatedUnix: timeutil.TimeStamp(asset.Created.Unix()),
}
// download attachment
err := func() error {
// asset.DownloadURL maybe a local file
var rc io.ReadCloser
var err error
if asset.DownloadFunc != nil {
rc, err = asset.DownloadFunc()
if err != nil {
return err
}
} else if asset.DownloadURL != nil {
rc, err = uri.Open(*asset.DownloadURL)
if err != nil {
return err
}
}
if rc == nil {
return nil
}
_, err = storage.Attachments.Save(attach.RelativePath(), rc, int64(*asset.Size))
rc.Close()
return err
}()
if err != nil {
return err
}
rel.Attachments = append(rel.Attachments, &attach)
}
rels = append(rels, &rel)
}
return models.InsertReleases(rels...)
}
// SyncTags syncs releases with tags in the database
func (g *GiteaLocalUploader) SyncTags() error {
return repo_module.SyncReleasesWithTags(g.repo, g.gitRepo)
}
// CreateIssues creates issues
func (g *GiteaLocalUploader) CreateIssues(issues ...*base.Issue) error {
var iss = make([]*models.Issue, 0, len(issues))
for _, issue := range issues {
var labels []*models.Label
for _, label := range issue.Labels {
lb, ok := g.labels.Load(label.Name)
if ok {
labels = append(labels, lb.(*models.Label))
}
}
var milestoneID int64
if issue.Milestone != "" {
milestone, ok := g.milestones.Load(issue.Milestone)
if ok {
milestoneID = milestone.(int64)
}
}
if issue.Created.IsZero() {
if issue.Closed != nil {
issue.Created = *issue.Closed
} else {
issue.Created = time.Now()
}
}
if issue.Updated.IsZero() {
if issue.Closed != nil {
issue.Updated = *issue.Closed
} else {
issue.Updated = time.Now()
}
}
var is = models.Issue{
RepoID: g.repo.ID,
Repo: g.repo,
Index: issue.Number,
Title: issue.Title,
Content: issue.Content,
Ref: issue.Ref,
IsClosed: issue.State == "closed",
IsLocked: issue.IsLocked,
MilestoneID: milestoneID,
Labels: labels,
CreatedUnix: timeutil.TimeStamp(issue.Created.Unix()),
UpdatedUnix: timeutil.TimeStamp(issue.Updated.Unix()),
}
userid, ok := g.userMap[issue.PosterID]
tp := g.gitServiceType.Name()
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", issue.PosterID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[issue.PosterID] = userid
}
}
if userid > 0 {
is.PosterID = userid
} else {
is.PosterID = g.doer.ID
is.OriginalAuthor = issue.PosterName
is.OriginalAuthorID = issue.PosterID
}
if issue.Closed != nil {
is.ClosedUnix = timeutil.TimeStamp(issue.Closed.Unix())
}
// add reactions
for _, reaction := range issue.Reactions {
userid, ok := g.userMap[reaction.UserID]
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[reaction.UserID] = userid
}
}
var res = models.Reaction{
Type: reaction.Content,
CreatedUnix: timeutil.TimeStampNow(),
}
if userid > 0 {
res.UserID = userid
} else {
res.UserID = g.doer.ID
res.OriginalAuthorID = reaction.UserID
res.OriginalAuthor = reaction.UserName
}
is.Reactions = append(is.Reactions, &res)
}
iss = append(iss, &is)
}
if len(iss) > 0 {
if err := models.InsertIssues(iss...); err != nil {
return err
}
for _, is := range iss {
g.issues.Store(is.Index, is)
}
}
return nil
}
// CreateComments creates comments of issues
func (g *GiteaLocalUploader) CreateComments(comments ...*base.Comment) error {
var cms = make([]*models.Comment, 0, len(comments))
for _, comment := range comments {
var issue *models.Issue
issueInter, ok := g.issues.Load(comment.IssueIndex)
if !ok {
var err error
issue, err = models.GetIssueByIndex(g.repo.ID, comment.IssueIndex)
if err != nil {
return err
}
g.issues.Store(comment.IssueIndex, issue)
} else {
issue = issueInter.(*models.Issue)
}
userid, ok := g.userMap[comment.PosterID]
tp := g.gitServiceType.Name()
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", comment.PosterID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[comment.PosterID] = userid
}
}
if comment.Created.IsZero() {
comment.Created = time.Unix(int64(issue.CreatedUnix), 0)
}
if comment.Updated.IsZero() {
comment.Updated = comment.Created
}
cm := models.Comment{
IssueID: issue.ID,
Type: models.CommentTypeComment,
Content: comment.Content,
CreatedUnix: timeutil.TimeStamp(comment.Created.Unix()),
UpdatedUnix: timeutil.TimeStamp(comment.Updated.Unix()),
}
if userid > 0 {
cm.PosterID = userid
} else {
cm.PosterID = g.doer.ID
cm.OriginalAuthor = comment.PosterName
cm.OriginalAuthorID = comment.PosterID
}
// add reactions
for _, reaction := range comment.Reactions {
userid, ok := g.userMap[reaction.UserID]
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[reaction.UserID] = userid
}
}
var res = models.Reaction{
Type: reaction.Content,
CreatedUnix: timeutil.TimeStampNow(),
}
if userid > 0 {
res.UserID = userid
} else {
res.UserID = g.doer.ID
res.OriginalAuthorID = reaction.UserID
res.OriginalAuthor = reaction.UserName
}
cm.Reactions = append(cm.Reactions, &res)
}
cms = append(cms, &cm)
}
if len(cms) == 0 {
return nil
}
return models.InsertIssueComments(cms)
}
// CreatePullRequests creates pull requests
func (g *GiteaLocalUploader) CreatePullRequests(prs ...*base.PullRequest) error {
var gprs = make([]*models.PullRequest, 0, len(prs))
for _, pr := range prs {
gpr, err := g.newPullRequest(pr)
if err != nil {
return err
}
userid, ok := g.userMap[pr.PosterID]
tp := g.gitServiceType.Name()
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[pr.PosterID] = userid
}
}
if userid > 0 {
gpr.Issue.PosterID = userid
} else {
gpr.Issue.PosterID = g.doer.ID
gpr.Issue.OriginalAuthor = pr.PosterName
gpr.Issue.OriginalAuthorID = pr.PosterID
}
gprs = append(gprs, gpr)
}
if err := models.InsertPullRequests(gprs...); err != nil {
return err
}
for _, pr := range gprs {
g.issues.Store(pr.Issue.Index, pr.Issue)
pull.AddToTaskQueue(pr)
}
return nil
}
func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullRequest, error) {
var labels []*models.Label
for _, label := range pr.Labels {
lb, ok := g.labels.Load(label.Name)
if ok {
labels = append(labels, lb.(*models.Label))
}
}
var milestoneID int64
if pr.Milestone != "" {
milestone, ok := g.milestones.Load(pr.Milestone)
if ok {
milestoneID = milestone.(int64)
}
}
// download patch file
err := func() error {
if pr.PatchURL == "" {
return nil
}
// pr.PatchURL maybe a local file
ret, err := uri.Open(pr.PatchURL)
if err != nil {
return err
}
defer ret.Close()
pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
return err
}
f, err := os.Create(filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number)))
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(f, ret)
return err
}()
if err != nil {
return nil, err
}
// set head information
pullHead := filepath.Join(g.repo.RepoPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
return nil, err
}
p, err := os.Create(filepath.Join(pullHead, "head"))
if err != nil {
return nil, err
}
_, err = p.WriteString(pr.Head.SHA)
p.Close()
if err != nil {
return nil, err
}
var head = "unknown repository"
if pr.IsForkPullRequest() && pr.State != "closed" {
if pr.Head.OwnerName != "" {
remote := pr.Head.OwnerName
_, ok := g.prHeadCache[remote]
if !ok {
// git remote add
err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
if err != nil {
log.Error("AddRemote failed: %s", err)
} else {
g.prHeadCache[remote] = struct{}{}
ok = true
}
}
if ok {
_, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.repo.RepoPath())
if err != nil {
log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
} else {
headBranch := filepath.Join(g.repo.RepoPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
return nil, err
}
b, err := os.Create(headBranch)
if err != nil {
return nil, err
}
_, err = b.WriteString(pr.Head.SHA)
b.Close()
if err != nil {
return nil, err
}
head = pr.Head.OwnerName + "/" + pr.Head.Ref
}
}
}
} else {
head = pr.Head.Ref
// Ensure the closed PR SHA still points to an existing ref
_, err = git.NewCommand("rev-list", "--quiet", "-1", pr.Head.SHA).RunInDir(g.repo.RepoPath())
if err != nil {
if pr.Head.SHA != "" {
// Git update-ref remove bad references with a relative path
log.Warn("Deprecated local head, removing : %v", pr.Head.SHA)
relPath := pr.GetGitRefName()
_, err = git.NewCommand("update-ref", "--no-deref", "-d", relPath).RunInDir(g.repo.RepoPath())
} else {
// The SHA is empty, remove the head file
log.Warn("Empty reference, removing : %v", pullHead)
err = os.Remove(filepath.Join(pullHead, "head"))
}
if err != nil {
log.Error("Cannot remove local head ref, %v", err)
}
}
}
if pr.Created.IsZero() {
if pr.Closed != nil {
pr.Created = *pr.Closed
} else if pr.MergedTime != nil {
pr.Created = *pr.MergedTime
} else {
pr.Created = time.Now()
}
}
if pr.Updated.IsZero() {
pr.Updated = pr.Created
}
var issue = models.Issue{
RepoID: g.repo.ID,
Repo: g.repo,
Title: pr.Title,
Index: pr.Number,
Content: pr.Content,
MilestoneID: milestoneID,
IsPull: true,
IsClosed: pr.State == "closed",
IsLocked: pr.IsLocked,
Labels: labels,
CreatedUnix: timeutil.TimeStamp(pr.Created.Unix()),
UpdatedUnix: timeutil.TimeStamp(pr.Updated.Unix()),
}
tp := g.gitServiceType.Name()
userid, ok := g.userMap[pr.PosterID]
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", pr.PosterID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[pr.PosterID] = userid
}
}
if userid > 0 {
issue.PosterID = userid
} else {
issue.PosterID = g.doer.ID
issue.OriginalAuthor = pr.PosterName
issue.OriginalAuthorID = pr.PosterID
}
// add reactions
for _, reaction := range pr.Reactions {
userid, ok := g.userMap[reaction.UserID]
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", reaction.UserID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[reaction.UserID] = userid
}
}
var res = models.Reaction{
Type: reaction.Content,
CreatedUnix: timeutil.TimeStampNow(),
}
if userid > 0 {
res.UserID = userid
} else {
res.UserID = g.doer.ID
res.OriginalAuthorID = reaction.UserID
res.OriginalAuthor = reaction.UserName
}
issue.Reactions = append(issue.Reactions, &res)
}
var pullRequest = models.PullRequest{
HeadRepoID: g.repo.ID,
HeadBranch: head,
BaseRepoID: g.repo.ID,
BaseBranch: pr.Base.Ref,
MergeBase: pr.Base.SHA,
Index: pr.Number,
HasMerged: pr.Merged,
Issue: &issue,
}
if pullRequest.Issue.IsClosed && pr.Closed != nil {
pullRequest.Issue.ClosedUnix = timeutil.TimeStamp(pr.Closed.Unix())
}
if pullRequest.HasMerged && pr.MergedTime != nil {
pullRequest.MergedUnix = timeutil.TimeStamp(pr.MergedTime.Unix())
pullRequest.MergedCommitID = pr.MergeCommitSHA
pullRequest.MergerID = g.doer.ID
}
// TODO: assignees
return &pullRequest, nil
}
func convertReviewState(state string) models.ReviewType {
switch state {
case base.ReviewStatePending:
return models.ReviewTypePending
case base.ReviewStateApproved:
return models.ReviewTypeApprove
case base.ReviewStateChangesRequested:
return models.ReviewTypeReject
case base.ReviewStateCommented:
return models.ReviewTypeComment
default:
return models.ReviewTypePending
}
}
// CreateReviews create pull request reviews
func (g *GiteaLocalUploader) CreateReviews(reviews ...*base.Review) error {
var cms = make([]*models.Review, 0, len(reviews))
for _, review := range reviews {
var issue *models.Issue
issueInter, ok := g.issues.Load(review.IssueIndex)
if !ok {
var err error
issue, err = models.GetIssueByIndex(g.repo.ID, review.IssueIndex)
if err != nil {
return err
}
g.issues.Store(review.IssueIndex, issue)
} else {
issue = issueInter.(*models.Issue)
}
userid, ok := g.userMap[review.ReviewerID]
tp := g.gitServiceType.Name()
if !ok && tp != "" {
var err error
userid, err = models.GetUserIDByExternalUserID(tp, fmt.Sprintf("%v", review.ReviewerID))
if err != nil {
log.Error("GetUserIDByExternalUserID: %v", err)
}
if userid > 0 {
g.userMap[review.ReviewerID] = userid
}
}
if review.CreatedAt.IsZero() {
review.CreatedAt = time.Unix(int64(issue.CreatedUnix), 0)
}
var cm = models.Review{
Type: convertReviewState(review.State),
IssueID: issue.ID,
Content: review.Content,
Official: review.Official,
CreatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
UpdatedUnix: timeutil.TimeStamp(review.CreatedAt.Unix()),
}
if userid > 0 {
cm.ReviewerID = userid
} else {
cm.ReviewerID = g.doer.ID
cm.OriginalAuthor = review.ReviewerName
cm.OriginalAuthorID = review.ReviewerID
}
// get pr
pr, ok := g.prCache[issue.ID]
if !ok {
var err error
pr, err = models.GetPullRequestByIssueIDWithNoAttributes(issue.ID)
if err != nil {
return err
}
g.prCache[issue.ID] = pr
}
for _, comment := range review.Comments {
line := comment.Line
if line != 0 {
comment.Position = 1
} else {
_, _, line, _ = git.ParseDiffHunkString(comment.DiffHunk)
}
headCommitID, err := g.gitRepo.GetRefCommitID(pr.GetGitRefName())
if err != nil {
log.Warn("GetRefCommitID[%s]: %v, the review comment will be ignored", pr.GetGitRefName(), err)
continue
}
var patch string
reader, writer := io.Pipe()
defer func() {
_ = reader.Close()
_ = writer.Close()
}()
go func() {
if err := git.GetRepoRawDiffForFile(g.gitRepo, pr.MergeBase, headCommitID, git.RawDiffNormal, comment.TreePath, writer); err != nil {
// We should ignore the error since the commit maybe removed when force push to the pull request
log.Warn("GetRepoRawDiffForFile failed when migrating [%s, %s, %s, %s]: %v", g.gitRepo.Path, pr.MergeBase, headCommitID, comment.TreePath, err)
}
_ = writer.Close()
}()
patch, _ = git.CutDiffAroundLine(reader, int64((&models.Comment{Line: int64(line + comment.Position - 1)}).UnsignedLine()), line < 0, setting.UI.CodeCommentLines)
if comment.CreatedAt.IsZero() {
comment.CreatedAt = review.CreatedAt
}
if comment.UpdatedAt.IsZero() {
comment.UpdatedAt = comment.CreatedAt
}
var c = models.Comment{
Type: models.CommentTypeCode,
PosterID: comment.PosterID,
IssueID: issue.ID,
Content: comment.Content,
Line: int64(line + comment.Position - 1),
TreePath: comment.TreePath,
CommitSHA: comment.CommitID,
Patch: patch,
CreatedUnix: timeutil.TimeStamp(comment.CreatedAt.Unix()),
UpdatedUnix: timeutil.TimeStamp(comment.UpdatedAt.Unix()),
}
if userid > 0 {
c.PosterID = userid
} else {
c.PosterID = g.doer.ID
c.OriginalAuthor = review.ReviewerName
c.OriginalAuthorID = review.ReviewerID
}
cm.Comments = append(cm.Comments, &c)
}
cms = append(cms, &cm)
}
return models.InsertReviews(cms)
}
// Rollback when migrating failed, this will rollback all the changes.
func (g *GiteaLocalUploader) Rollback() error {
if g.repo != nil && g.repo.ID > 0 {
g.gitRepo.Close()
if err := models.DeleteRepository(g.doer, g.repo.OwnerID, g.repo.ID); err != nil {
return err
}
}
return nil
}
// Finish when migrating success, this will do some status update things.
func (g *GiteaLocalUploader) Finish() error {
if g.repo == nil || g.repo.ID <= 0 {
return ErrRepoNotCreated
}
// update issue_index
if err := models.RecalculateIssueIndexForRepo(g.repo.ID); err != nil {
return err
}
g.repo.Status = models.RepositoryReady
return models.UpdateRepositoryCols(g.repo, "status")
}

115
services/migrations/gitea_uploader_test.go Normal file
View file

@ -0,0 +1,115 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"testing"
"time"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/graceful"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"github.com/stretchr/testify/assert"
)
func TestGiteaUploadRepo(t *testing.T) {
// FIXME: Since no accesskey or user/password will trigger rate limit of github, just skip
t.Skip()
unittest.PrepareTestEnv(t)
user := unittest.AssertExistsAndLoadBean(t, &models.User{ID: 1}).(*models.User)
var (
downloader = NewGithubDownloaderV3(context.Background(), "https://github.com", "", "", "", "go-xorm", "builder")
repoName = "builder-" + time.Now().Format("2006-01-02-15-04-05")
uploader = NewGiteaLocalUploader(graceful.GetManager().HammerContext(), user, user.Name, repoName)
)
err := migrateRepository(downloader, uploader, base.MigrateOptions{
CloneAddr: "https://github.com/go-xorm/builder",
RepoName: repoName,
AuthUsername: "",
Wiki: true,
Issues: true,
Milestones: true,
Labels: true,
Releases: true,
Comments: true,
PullRequests: true,
Private: true,
Mirror: false,
}, nil)
assert.NoError(t, err)
repo := unittest.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)
assert.True(t, repo.HasWiki())
assert.EqualValues(t, models.RepositoryReady, repo.Status)
milestones, _, err := models.GetMilestones(models.GetMilestonesOption{
RepoID: repo.ID,
State: structs.StateOpen,
})
assert.NoError(t, err)
assert.Len(t, milestones, 1)
milestones, _, err = models.GetMilestones(models.GetMilestonesOption{
RepoID: repo.ID,
State: structs.StateClosed,
})
assert.NoError(t, err)
assert.Empty(t, milestones)
labels, err := models.GetLabelsByRepoID(repo.ID, "", db.ListOptions{})
assert.NoError(t, err)
assert.Len(t, labels, 12)
releases, err := models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
ListOptions: db.ListOptions{
PageSize: 10,
Page: 0,
},
IncludeTags: true,
})
assert.NoError(t, err)
assert.Len(t, releases, 8)
releases, err = models.GetReleasesByRepoID(repo.ID, models.FindReleasesOptions{
ListOptions: db.ListOptions{
PageSize: 10,
Page: 0,
},
IncludeTags: false,
})
assert.NoError(t, err)
assert.Len(t, releases, 1)
issues, err := models.Issues(&models.IssuesOptions{
RepoIDs: []int64{repo.ID},
IsPull: util.OptionalBoolFalse,
SortType: "oldest",
})
assert.NoError(t, err)
assert.Len(t, issues, 15)
assert.NoError(t, issues[0].LoadDiscussComments())
assert.Empty(t, issues[0].Comments)
pulls, _, err := models.PullRequests(repo.ID, &models.PullRequestsOptions{
SortType: "oldest",
})
assert.NoError(t, err)
assert.Len(t, pulls, 30)
assert.NoError(t, pulls[0].LoadIssue())
assert.NoError(t, pulls[0].Issue.LoadDiscussComments())
assert.Len(t, pulls[0].Issue.Comments, 2)
}

836
services/migrations/github.go Normal file
View file

@ -0,0 +1,836 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
"github.com/google/go-github/v39/github"
"golang.org/x/oauth2"
)
var (
_ base.Downloader = &GithubDownloaderV3{}
_ base.DownloaderFactory = &GithubDownloaderV3Factory{}
// GithubLimitRateRemaining limit to wait for new rate to apply
GithubLimitRateRemaining = 0
)
func init() {
RegisterDownloaderFactory(&GithubDownloaderV3Factory{})
}
// GithubDownloaderV3Factory defines a github downloader v3 factory
type GithubDownloaderV3Factory struct {
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GithubDownloaderV3Factory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return nil, err
}
baseURL := u.Scheme + "://" + u.Host
fields := strings.Split(u.Path, "/")
oldOwner := fields[1]
oldName := strings.TrimSuffix(fields[2], ".git")
log.Trace("Create github downloader: %s/%s", oldOwner, oldName)
return NewGithubDownloaderV3(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, oldOwner, oldName), nil
}
// GitServiceType returns the type of git service
func (f *GithubDownloaderV3Factory) GitServiceType() structs.GitServiceType {
return structs.GithubService
}
// GithubDownloaderV3 implements a Downloader interface to get repository information
// from github via APIv3
type GithubDownloaderV3 struct {
base.NullDownloader
ctx context.Context
clients []*github.Client
repoOwner string
repoName string
userName string
password string
rates []*github.Rate
curClientIdx int
maxPerPage int
SkipReactions bool
}
// NewGithubDownloaderV3 creates a github Downloader via github v3 API
func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GithubDownloaderV3 {
var downloader = GithubDownloaderV3{
userName: userName,
password: password,
ctx: ctx,
repoOwner: repoOwner,
repoName: repoName,
maxPerPage: 100,
}
if token != "" {
tokens := strings.Split(token, ",")
for _, token := range tokens {
token = strings.TrimSpace(token)
ts := oauth2.StaticTokenSource(
&oauth2.Token{AccessToken: token},
)
var client = &http.Client{
Transport: &oauth2.Transport{
Base: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: func(req *http.Request) (*url.URL, error) {
return proxy.Proxy()(req)
},
},
Source: oauth2.ReuseTokenSource(nil, ts),
},
}
downloader.addClient(client, baseURL)
}
} else {
var client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return proxy.Proxy()(req)
},
},
}
downloader.addClient(client, baseURL)
}
return &downloader
}
func (g *GithubDownloaderV3) addClient(client *http.Client, baseURL string) {
githubClient := github.NewClient(client)
if baseURL != "https://github.com" {
githubClient, _ = github.NewEnterpriseClient(baseURL, baseURL, client)
}
g.clients = append(g.clients, githubClient)
g.rates = append(g.rates, nil)
}
// SetContext set context
func (g *GithubDownloaderV3) SetContext(ctx context.Context) {
g.ctx = ctx
}
func (g *GithubDownloaderV3) waitAndPickClient() {
var recentIdx int
var maxRemaining int
for i := 0; i < len(g.clients); i++ {
if g.rates[i] != nil && g.rates[i].Remaining > maxRemaining {
maxRemaining = g.rates[i].Remaining
recentIdx = i
}
}
g.curClientIdx = recentIdx // if no max remain, it will always pick the first client.
for g.rates[g.curClientIdx] != nil && g.rates[g.curClientIdx].Remaining <= GithubLimitRateRemaining {
timer := time.NewTimer(time.Until(g.rates[g.curClientIdx].Reset.Time))
select {
case <-g.ctx.Done():
util.StopTimer(timer)
return
case <-timer.C:
}
err := g.RefreshRate()
if err != nil {
log.Error("g.getClient().RateLimits: %s", err)
}
}
}
// RefreshRate update the current rate (doesn't count in rate limit)
func (g *GithubDownloaderV3) RefreshRate() error {
rates, _, err := g.getClient().RateLimits(g.ctx)
if err != nil {
// if rate limit is not enabled, ignore it
if strings.Contains(err.Error(), "404") {
g.setRate(nil)
return nil
}
return err
}
g.setRate(rates.GetCore())
return nil
}
func (g *GithubDownloaderV3) getClient() *github.Client {
return g.clients[g.curClientIdx]
}
func (g *GithubDownloaderV3) setRate(rate *github.Rate) {
g.rates[g.curClientIdx] = rate
}
// GetRepoInfo returns a repository information
func (g *GithubDownloaderV3) GetRepoInfo() (*base.Repository, error) {
g.waitAndPickClient()
gr, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
g.setRate(&resp.Rate)
// convert github repo to stand Repo
return &base.Repository{
Owner: g.repoOwner,
Name: gr.GetName(),
IsPrivate: gr.GetPrivate(),
Description: gr.GetDescription(),
OriginalURL: gr.GetHTMLURL(),
CloneURL: gr.GetCloneURL(),
DefaultBranch: gr.GetDefaultBranch(),
}, nil
}
// GetTopics return github topics
func (g *GithubDownloaderV3) GetTopics() ([]string, error) {
g.waitAndPickClient()
r, resp, err := g.getClient().Repositories.Get(g.ctx, g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
g.setRate(&resp.Rate)
return r.Topics, nil
}
// GetMilestones returns milestones
func (g *GithubDownloaderV3) GetMilestones() ([]*base.Milestone, error) {
var perPage = g.maxPerPage
var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
g.waitAndPickClient()
ms, resp, err := g.getClient().Issues.ListMilestones(g.ctx, g.repoOwner, g.repoName,
&github.MilestoneListOptions{
State: "all",
ListOptions: github.ListOptions{
Page: i,
PerPage: perPage,
}})
if err != nil {
return nil, err
}
g.setRate(&resp.Rate)
for _, m := range ms {
var state = "open"
if m.State != nil {
state = *m.State
}
milestones = append(milestones, &base.Milestone{
Title: m.GetTitle(),
Description: m.GetDescription(),
Deadline: m.DueOn,
State: state,
Created: m.GetCreatedAt(),
Updated: m.UpdatedAt,
Closed: m.ClosedAt,
})
}
if len(ms) < perPage {
break
}
}
return milestones, nil
}
func convertGithubLabel(label *github.Label) *base.Label {
return &base.Label{
Name: label.GetName(),
Color: label.GetColor(),
Description: label.GetDescription(),
}
}
// GetLabels returns labels
func (g *GithubDownloaderV3) GetLabels() ([]*base.Label, error) {
var perPage = g.maxPerPage
var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
g.waitAndPickClient()
ls, resp, err := g.getClient().Issues.ListLabels(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
})
if err != nil {
return nil, err
}
g.setRate(&resp.Rate)
for _, label := range ls {
labels = append(labels, convertGithubLabel(label))
}
if len(ls) < perPage {
break
}
}
return labels, nil
}
func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease) *base.Release {
r := &base.Release{
Name: rel.GetName(),
TagName: rel.GetTagName(),
TargetCommitish: rel.GetTargetCommitish(),
Draft: rel.GetDraft(),
Prerelease: rel.GetPrerelease(),
Created: rel.GetCreatedAt().Time,
PublisherID: rel.GetAuthor().GetID(),
PublisherName: rel.GetAuthor().GetLogin(),
PublisherEmail: rel.GetAuthor().GetEmail(),
Body: rel.GetBody(),
}
if rel.PublishedAt != nil {
r.Published = rel.PublishedAt.Time
}
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
for _, asset := range rel.Assets {
var assetID = *asset.ID // Don't optimize this, for closure we need a local variable
r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: asset.GetID(),
Name: asset.GetName(),
ContentType: asset.ContentType,
Size: asset.Size,
DownloadCount: asset.DownloadCount,
Created: asset.CreatedAt.Time,
Updated: asset.UpdatedAt.Time,
DownloadFunc: func() (io.ReadCloser, error) {
g.waitAndPickClient()
asset, redirectURL, err := g.getClient().Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, assetID, nil)
if err != nil {
return nil, err
}
if err := g.RefreshRate(); err != nil {
log.Error("g.getClient().RateLimits: %s", err)
}
if asset == nil {
if redirectURL != "" {
g.waitAndPickClient()
req, err := http.NewRequestWithContext(g.ctx, "GET", redirectURL, nil)
if err != nil {
return nil, err
}
resp, err := httpClient.Do(req)
err1 := g.RefreshRate()
if err1 != nil {
log.Error("g.getClient().RateLimits: %s", err1)
}
if err != nil {
return nil, err
}
return resp.Body, nil
}
return nil, fmt.Errorf("No release asset found for %d", assetID)
}
return asset, nil
},
})
}
return r
}
// GetReleases returns releases
func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
var perPage = g.maxPerPage
var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
g.waitAndPickClient()
ls, resp, err := g.getClient().Repositories.ListReleases(g.ctx, g.repoOwner, g.repoName,
&github.ListOptions{
Page: i,
PerPage: perPage,
})
if err != nil {
return nil, err
}
g.setRate(&resp.Rate)
for _, release := range ls {
releases = append(releases, g.convertGithubRelease(release))
}
if len(ls) < perPage {
break
}
}
return releases, nil
}
// GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &github.IssueListByRepoOptions{
Sort: "created",
Direction: "asc",
State: "all",
ListOptions: github.ListOptions{
PerPage: perPage,
Page: page,
},
}
var allIssues = make([]*base.Issue, 0, perPage)
g.waitAndPickClient()
issues, resp, err := g.getClient().Issues.ListByRepo(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))
g.setRate(&resp.Rate)
for _, issue := range issues {
if issue.IsPullRequest() {
continue
}
var labels = make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, convertGithubLabel(l))
}
// get reactions
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
g.waitAndPickClient()
res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, issue.GetNumber(), &github.ListOptions{
Page: i,
PerPage: perPage,
})
if err != nil {
return nil, false, err
}
g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
}
var assignees []string
for i := range issue.Assignees {
assignees = append(assignees, issue.Assignees[i].GetLogin())
}
allIssues = append(allIssues, &base.Issue{
Title: *issue.Title,
Number: int64(*issue.Number),
PosterID: issue.GetUser().GetID(),
PosterName: issue.GetUser().GetLogin(),
PosterEmail: issue.GetUser().GetEmail(),
Content: issue.GetBody(),
Milestone: issue.GetMilestone().GetTitle(),
State: issue.GetState(),
Created: issue.GetCreatedAt(),
Updated: issue.GetUpdatedAt(),
Labels: labels,
Reactions: reactions,
Closed: issue.ClosedAt,
IsLocked: issue.GetLocked(),
Assignees: assignees,
Context: base.BasicIssueContext(*issue.Number),
})
}
return allIssues, len(issues) < perPage, nil
}
// SupportGetRepoComments return true if it supports get repo comments
func (g *GithubDownloaderV3) SupportGetRepoComments() bool {
return true
}
// GetComments returns comments according issueNumber
func (g *GithubDownloaderV3) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
if opts.Context != nil {
comments, err := g.getComments(opts.Context)
return comments, false, err
}
return g.GetAllComments(opts.Page, opts.PageSize)
}
func (g *GithubDownloaderV3) getComments(issueContext base.IssueContext) ([]*base.Comment, error) {
var (
allComments = make([]*base.Comment, 0, g.maxPerPage)
created = "created"
asc = "asc"
)
opt := &github.IssueListCommentsOptions{
Sort: &created,
Direction: &asc,
ListOptions: github.ListOptions{
PerPage: g.maxPerPage,
},
}
for {
g.waitAndPickClient()
comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, int(issueContext.ForeignID()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
g.setRate(&resp.Rate)
for _, comment := range comments {
// get reactions
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
g.waitAndPickClient()
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
Page: i,
PerPage: g.maxPerPage,
})
if err != nil {
return nil, err
}
g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
}
allComments = append(allComments, &base.Comment{
IssueIndex: issueContext.LocalID(),
PosterID: comment.GetUser().GetID(),
PosterName: comment.GetUser().GetLogin(),
PosterEmail: comment.GetUser().GetEmail(),
Content: comment.GetBody(),
Created: comment.GetCreatedAt(),
Updated: comment.GetUpdatedAt(),
Reactions: reactions,
})
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
}
return allComments, nil
}
// GetAllComments returns repository comments according page and perPageSize
func (g *GithubDownloaderV3) GetAllComments(page, perPage int) ([]*base.Comment, bool, error) {
var (
allComments = make([]*base.Comment, 0, perPage)
created = "created"
asc = "asc"
)
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &github.IssueListCommentsOptions{
Sort: &created,
Direction: &asc,
ListOptions: github.ListOptions{
Page: page,
PerPage: perPage,
},
}
g.waitAndPickClient()
comments, resp, err := g.getClient().Issues.ListComments(g.ctx, g.repoOwner, g.repoName, 0, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
var isEnd = resp.NextPage == 0
log.Trace("Request get comments %d/%d, but in fact get %d, next page is %d", perPage, page, len(comments), resp.NextPage)
g.setRate(&resp.Rate)
for _, comment := range comments {
// get reactions
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
g.waitAndPickClient()
res, resp, err := g.getClient().Reactions.ListIssueCommentReactions(g.ctx, g.repoOwner, g.repoName, comment.GetID(), &github.ListOptions{
Page: i,
PerPage: g.maxPerPage,
})
if err != nil {
return nil, false, err
}
g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
}
idx := strings.LastIndex(*comment.IssueURL, "/")
issueIndex, _ := strconv.ParseInt((*comment.IssueURL)[idx+1:], 10, 64)
allComments = append(allComments, &base.Comment{
IssueIndex: issueIndex,
PosterID: comment.GetUser().GetID(),
PosterName: comment.GetUser().GetLogin(),
PosterEmail: comment.GetUser().GetEmail(),
Content: comment.GetBody(),
Created: comment.GetCreatedAt(),
Updated: comment.GetUpdatedAt(),
Reactions: reactions,
})
}
return allComments, isEnd, nil
}
// GetPullRequests returns pull requests according page and perPage
func (g *GithubDownloaderV3) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &github.PullRequestListOptions{
Sort: "created",
Direction: "asc",
State: "all",
ListOptions: github.ListOptions{
PerPage: perPage,
Page: page,
},
}
var allPRs = make([]*base.PullRequest, 0, perPage)
g.waitAndPickClient()
prs, resp, err := g.getClient().PullRequests.List(g.ctx, g.repoOwner, g.repoName, opt)
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
log.Trace("Request get pull requests %d/%d, but in fact get %d", perPage, page, len(prs))
g.setRate(&resp.Rate)
for _, pr := range prs {
var labels = make([]*base.Label, 0, len(pr.Labels))
for _, l := range pr.Labels {
labels = append(labels, convertGithubLabel(l))
}
// get reactions
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
g.waitAndPickClient()
res, resp, err := g.getClient().Reactions.ListIssueReactions(g.ctx, g.repoOwner, g.repoName, pr.GetNumber(), &github.ListOptions{
Page: i,
PerPage: perPage,
})
if err != nil {
return nil, false, err
}
g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
}
// download patch and saved as tmp file
g.waitAndPickClient()
allPRs = append(allPRs, &base.PullRequest{
Title: pr.GetTitle(),
Number: int64(pr.GetNumber()),
PosterID: pr.GetUser().GetID(),
PosterName: pr.GetUser().GetLogin(),
PosterEmail: pr.GetUser().GetEmail(),
Content: pr.GetBody(),
Milestone: pr.GetMilestone().GetTitle(),
State: pr.GetState(),
Created: pr.GetCreatedAt(),
Updated: pr.GetUpdatedAt(),
Closed: pr.ClosedAt,
Labels: labels,
Merged: pr.MergedAt != nil,
MergeCommitSHA: pr.GetMergeCommitSHA(),
MergedTime: pr.MergedAt,
IsLocked: pr.ActiveLockReason != nil,
Head: base.PullRequestBranch{
Ref: pr.GetHead().GetRef(),
SHA: pr.GetHead().GetSHA(),
OwnerName: pr.GetHead().GetUser().GetLogin(),
RepoName: pr.GetHead().GetRepo().GetName(),
CloneURL: pr.GetHead().GetRepo().GetCloneURL(),
},
Base: base.PullRequestBranch{
Ref: pr.GetBase().GetRef(),
SHA: pr.GetBase().GetSHA(),
RepoName: pr.GetBase().GetRepo().GetName(),
OwnerName: pr.GetBase().GetUser().GetLogin(),
},
PatchURL: pr.GetPatchURL(),
Reactions: reactions,
Context: base.BasicIssueContext(*pr.Number),
})
}
return allPRs, len(prs) < perPage, nil
}
func convertGithubReview(r *github.PullRequestReview) *base.Review {
return &base.Review{
ID: r.GetID(),
ReviewerID: r.GetUser().GetID(),
ReviewerName: r.GetUser().GetLogin(),
CommitID: r.GetCommitID(),
Content: r.GetBody(),
CreatedAt: r.GetSubmittedAt(),
State: r.GetState(),
}
}
func (g *GithubDownloaderV3) convertGithubReviewComments(cs []*github.PullRequestComment) ([]*base.ReviewComment, error) {
var rcs = make([]*base.ReviewComment, 0, len(cs))
for _, c := range cs {
// get reactions
var reactions []*base.Reaction
if !g.SkipReactions {
for i := 1; ; i++ {
g.waitAndPickClient()
res, resp, err := g.getClient().Reactions.ListPullRequestCommentReactions(g.ctx, g.repoOwner, g.repoName, c.GetID(), &github.ListOptions{
Page: i,
PerPage: g.maxPerPage,
})
if err != nil {
return nil, err
}
g.setRate(&resp.Rate)
if len(res) == 0 {
break
}
for _, reaction := range res {
reactions = append(reactions, &base.Reaction{
UserID: reaction.User.GetID(),
UserName: reaction.User.GetLogin(),
Content: reaction.GetContent(),
})
}
}
}
rcs = append(rcs, &base.ReviewComment{
ID: c.GetID(),
InReplyTo: c.GetInReplyTo(),
Content: c.GetBody(),
TreePath: c.GetPath(),
DiffHunk: c.GetDiffHunk(),
Position: c.GetPosition(),
CommitID: c.GetCommitID(),
PosterID: c.GetUser().GetID(),
Reactions: reactions,
CreatedAt: c.GetCreatedAt(),
UpdatedAt: c.GetUpdatedAt(),
})
}
return rcs, nil
}
// GetReviews returns pull requests review
func (g *GithubDownloaderV3) GetReviews(context base.IssueContext) ([]*base.Review, error) {
var allReviews = make([]*base.Review, 0, g.maxPerPage)
opt := &github.ListOptions{
PerPage: g.maxPerPage,
}
for {
g.waitAndPickClient()
reviews, resp, err := g.getClient().PullRequests.ListReviews(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), opt)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
g.setRate(&resp.Rate)
for _, review := range reviews {
r := convertGithubReview(review)
r.IssueIndex = context.LocalID()
// retrieve all review comments
opt2 := &github.ListOptions{
PerPage: g.maxPerPage,
}
for {
g.waitAndPickClient()
reviewComments, resp, err := g.getClient().PullRequests.ListReviewComments(g.ctx, g.repoOwner, g.repoName, int(context.ForeignID()), review.GetID(), opt2)
if err != nil {
return nil, fmt.Errorf("error while listing repos: %v", err)
}
g.setRate(&resp.Rate)
cs, err := g.convertGithubReviewComments(reviewComments)
if err != nil {
return nil, err
}
r.Comments = append(r.Comments, cs...)
if resp.NextPage == 0 {
break
}
opt2.Page = resp.NextPage
}
allReviews = append(allReviews, r)
}
if resp.NextPage == 0 {
break
}
opt.Page = resp.NextPage
}
return allReviews, nil
}

430
services/migrations/github_test.go Normal file
View file

@ -0,0 +1,430 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"os"
"testing"
"time"
base "code.gitea.io/gitea/modules/migration"
"github.com/stretchr/testify/assert"
)
func TestGitHubDownloadRepo(t *testing.T) {
GithubLimitRateRemaining = 3 //Wait at 3 remaining since we could have 3 CI in //
downloader := NewGithubDownloaderV3(context.Background(), "https://github.com", "", "", os.Getenv("GITHUB_READ_TOKEN"), "go-gitea", "test_repo")
err := downloader.RefreshRate()
assert.NoError(t, err)
repo, err := downloader.GetRepoInfo()
assert.NoError(t, err)
assertRepositoryEqual(t, &base.Repository{
Name: "test_repo",
Owner: "go-gitea",
Description: "Test repository for testing migration from github to gitea",
CloneURL: "https://github.com/go-gitea/test_repo.git",
OriginalURL: "https://github.com/go-gitea/test_repo",
DefaultBranch: "master",
}, repo)
topics, err := downloader.GetTopics()
assert.NoError(t, err)
assert.Contains(t, topics, "gitea")
milestones, err := downloader.GetMilestones()
assert.NoError(t, err)
assertMilestonesEqual(t, []*base.Milestone{
{
Title: "1.0.0",
Description: "Milestone 1.0.0",
Deadline: timePtr(time.Date(2019, 11, 11, 8, 0, 0, 0, time.UTC)),
Created: time.Date(2019, 11, 12, 19, 37, 8, 0, time.UTC),
Updated: timePtr(time.Date(2019, 11, 12, 21, 56, 17, 0, time.UTC)),
Closed: timePtr(time.Date(2019, 11, 12, 19, 45, 49, 0, time.UTC)),
State: "closed",
},
{
Title: "1.1.0",
Description: "Milestone 1.1.0",
Deadline: timePtr(time.Date(2019, 11, 12, 8, 0, 0, 0, time.UTC)),
Created: time.Date(2019, 11, 12, 19, 37, 25, 0, time.UTC),
Updated: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
Closed: timePtr(time.Date(2019, 11, 12, 19, 45, 46, 0, time.UTC)),
State: "closed",
},
}, milestones)
labels, err := downloader.GetLabels()
assert.NoError(t, err)
assertLabelsEqual(t, []*base.Label{
{
Name: "bug",
Color: "d73a4a",
Description: "Something isn't working",
},
{
Name: "documentation",
Color: "0075ca",
Description: "Improvements or additions to documentation",
},
{
Name: "duplicate",
Color: "cfd3d7",
Description: "This issue or pull request already exists",
},
{
Name: "enhancement",
Color: "a2eeef",
Description: "New feature or request",
},
{
Name: "good first issue",
Color: "7057ff",
Description: "Good for newcomers",
},
{
Name: "help wanted",
Color: "008672",
Description: "Extra attention is needed",
},
{
Name: "invalid",
Color: "e4e669",
Description: "This doesn't seem right",
},
{
Name: "question",
Color: "d876e3",
Description: "Further information is requested",
},
{
Name: "wontfix",
Color: "ffffff",
Description: "This will not be worked on",
},
}, labels)
releases, err := downloader.GetReleases()
assert.NoError(t, err)
assertReleasesEqual(t, []*base.Release{
{
TagName: "v0.9.99",
TargetCommitish: "master",
Name: "First Release",
Body: "A test release",
Created: time.Date(2019, 11, 9, 16, 49, 21, 0, time.UTC),
Published: time.Date(2019, 11, 12, 20, 12, 10, 0, time.UTC),
PublisherID: 1669571,
PublisherName: "mrsdizzie",
},
}, releases)
// downloader.GetIssues()
issues, isEnd, err := downloader.GetIssues(1, 2)
assert.NoError(t, err)
assert.False(t, isEnd)
assertIssuesEqual(t, []*base.Issue{
{
Number: 1,
Title: "Please add an animated gif icon to the merge button",
Content: "I just want the merge button to hurt my eyes a little. \xF0\x9F\x98\x9D ",
Milestone: "1.0.0",
PosterID: 18600385,
PosterName: "guillep2k",
State: "closed",
Created: time.Date(2019, 11, 9, 17, 0, 29, 0, time.UTC),
Updated: time.Date(2019, 11, 12, 20, 29, 53, 0, time.UTC),
Labels: []*base.Label{
{
Name: "bug",
Color: "d73a4a",
Description: "Something isn't working",
},
{
Name: "good first issue",
Color: "7057ff",
Description: "Good for newcomers",
},
},
Reactions: []*base.Reaction{
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "+1",
},
},
Closed: timePtr(time.Date(2019, 11, 12, 20, 22, 22, 0, time.UTC)),
},
{
Number: 2,
Title: "Test issue",
Content: "This is test issue 2, do not touch!",
Milestone: "1.1.0",
PosterID: 1669571,
PosterName: "mrsdizzie",
State: "closed",
Created: time.Date(2019, 11, 12, 21, 0, 6, 0, time.UTC),
Updated: time.Date(2019, 11, 12, 22, 7, 14, 0, time.UTC),
Labels: []*base.Label{
{
Name: "duplicate",
Color: "cfd3d7",
Description: "This issue or pull request already exists",
},
},
Reactions: []*base.Reaction{
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "heart",
},
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "laugh",
},
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "-1",
},
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "confused",
},
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "hooray",
},
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "+1",
},
},
Closed: timePtr(time.Date(2019, 11, 12, 21, 1, 31, 0, time.UTC)),
},
}, issues)
// downloader.GetComments()
comments, _, err := downloader.GetComments(base.GetCommentOptions{
Context: base.BasicIssueContext(2),
})
assert.NoError(t, err)
assertCommentsEqual(t, []*base.Comment{
{
IssueIndex: 2,
PosterID: 1669571,
PosterName: "mrsdizzie",
Created: time.Date(2019, 11, 12, 21, 0, 13, 0, time.UTC),
Updated: time.Date(2019, 11, 12, 21, 0, 13, 0, time.UTC),
Content: "This is a comment",
Reactions: []*base.Reaction{
{
UserID: 1669571,
UserName: "mrsdizzie",
Content: "+1",
},
},
},
{
IssueIndex: 2,
PosterID: 1669571,
PosterName: "mrsdizzie",
Created: time.Date(2019, 11, 12, 22, 7, 14, 0, time.UTC),
Updated: time.Date(2019, 11, 12, 22, 7, 14, 0, time.UTC),
Content: "A second comment",
Reactions: nil,
},
}, comments)
// downloader.GetPullRequests()
prs, _, err := downloader.GetPullRequests(1, 2)
assert.NoError(t, err)
assertPullRequestsEqual(t, []*base.PullRequest{
{
Number: 3,
Title: "Update README.md",
Content: "add warning to readme",
Milestone: "1.1.0",
PosterID: 1669571,
PosterName: "mrsdizzie",
State: "closed",
Created: time.Date(2019, 11, 12, 21, 21, 43, 0, time.UTC),
Updated: time.Date(2019, 11, 12, 21, 39, 28, 0, time.UTC),
Labels: []*base.Label{
{
Name: "documentation",
Color: "0075ca",
Description: "Improvements or additions to documentation",
},
},
PatchURL: "https://github.com/go-gitea/test_repo/pull/3.patch",
Head: base.PullRequestBranch{
Ref: "master",
CloneURL: "https://github.com/mrsdizzie/test_repo.git",
SHA: "076160cf0b039f13e5eff19619932d181269414b",
RepoName: "test_repo",
OwnerName: "mrsdizzie",
},
Base: base.PullRequestBranch{
Ref: "master",
SHA: "72866af952e98d02a73003501836074b286a78f6",
OwnerName: "go-gitea",
RepoName: "test_repo",
},
Closed: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
Merged: true,
MergedTime: timePtr(time.Date(2019, 11, 12, 21, 39, 27, 0, time.UTC)),
MergeCommitSHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
Context: base.BasicIssueContext(3),
},
{
Number: 4,
Title: "Test branch",
Content: "do not merge this PR",
Milestone: "1.0.0",
PosterID: 1669571,
PosterName: "mrsdizzie",
State: "open",
Created: time.Date(2019, 11, 12, 21, 54, 18, 0, time.UTC),
Updated: time.Date(2020, 1, 4, 11, 30, 1, 0, time.UTC),
Labels: []*base.Label{
{
Name: "bug",
Color: "d73a4a",
Description: "Something isn't working",
},
},
PatchURL: "https://github.com/go-gitea/test_repo/pull/4.patch",
Head: base.PullRequestBranch{
Ref: "test-branch",
SHA: "2be9101c543658591222acbee3eb799edfc3853d",
RepoName: "test_repo",
OwnerName: "mrsdizzie",
CloneURL: "https://github.com/mrsdizzie/test_repo.git",
},
Base: base.PullRequestBranch{
Ref: "master",
SHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
OwnerName: "go-gitea",
RepoName: "test_repo",
},
Merged: false,
MergeCommitSHA: "565d1208f5fffdc1c5ae1a2436491eb9a5e4ebae",
Reactions: []*base.Reaction{
{
UserID: 81045,
UserName: "lunny",
Content: "heart",
},
{
UserID: 81045,
UserName: "lunny",
Content: "+1",
},
},
Context: base.BasicIssueContext(4),
},
}, prs)
reviews, err := downloader.GetReviews(base.BasicIssueContext(3))
assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{
{
ID: 315859956,
IssueIndex: 3,
ReviewerID: 42128690,
ReviewerName: "jolheiser",
CommitID: "076160cf0b039f13e5eff19619932d181269414b",
CreatedAt: time.Date(2019, 11, 12, 21, 35, 24, 0, time.UTC),
State: base.ReviewStateApproved,
},
{
ID: 315860062,
IssueIndex: 3,
ReviewerID: 1824502,
ReviewerName: "zeripath",
CommitID: "076160cf0b039f13e5eff19619932d181269414b",
CreatedAt: time.Date(2019, 11, 12, 21, 35, 36, 0, time.UTC),
State: base.ReviewStateApproved,
},
{
ID: 315861440,
IssueIndex: 3,
ReviewerID: 165205,
ReviewerName: "lafriks",
CommitID: "076160cf0b039f13e5eff19619932d181269414b",
CreatedAt: time.Date(2019, 11, 12, 21, 38, 00, 0, time.UTC),
State: base.ReviewStateApproved,
},
}, reviews)
reviews, err = downloader.GetReviews(base.BasicIssueContext(4))
assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{
{
ID: 338338740,
IssueIndex: 4,
ReviewerID: 81045,
ReviewerName: "lunny",
CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
CreatedAt: time.Date(2020, 01, 04, 05, 33, 18, 0, time.UTC),
State: base.ReviewStateApproved,
Comments: []*base.ReviewComment{
{
ID: 363017488,
Content: "This is a good pull request.",
TreePath: "README.md",
DiffHunk: "@@ -1,2 +1,4 @@\n # test_repo\n Test repository for testing migration from github to gitea\n+",
Position: 3,
CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
PosterID: 81045,
CreatedAt: time.Date(2020, 01, 04, 05, 33, 06, 0, time.UTC),
UpdatedAt: time.Date(2020, 01, 04, 05, 33, 18, 0, time.UTC),
},
},
},
{
ID: 338339651,
IssueIndex: 4,
ReviewerID: 81045,
ReviewerName: "lunny",
CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
CreatedAt: time.Date(2020, 01, 04, 06, 07, 06, 0, time.UTC),
State: base.ReviewStateChangesRequested,
Content: "Don't add more reviews",
},
{
ID: 338349019,
IssueIndex: 4,
ReviewerID: 81045,
ReviewerName: "lunny",
CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
CreatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
State: base.ReviewStateCommented,
Comments: []*base.ReviewComment{
{
ID: 363029944,
Content: "test a single comment.",
TreePath: "LICENSE",
DiffHunk: "@@ -19,3 +19,5 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\n LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\n OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\n SOFTWARE.\n+",
Position: 4,
CommitID: "2be9101c543658591222acbee3eb799edfc3853d",
PosterID: 81045,
CreatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
UpdatedAt: time.Date(2020, 01, 04, 11, 21, 41, 0, time.UTC),
},
},
},
}, reviews)
}

678
services/migrations/gitlab.go Normal file
View file

@ -0,0 +1,678 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"crypto/tls"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"github.com/xanzy/go-gitlab"
)
var (
_ base.Downloader = &GitlabDownloader{}
_ base.DownloaderFactory = &GitlabDownloaderFactory{}
)
func init() {
RegisterDownloaderFactory(&GitlabDownloaderFactory{})
}
// GitlabDownloaderFactory defines a gitlab downloader factory
type GitlabDownloaderFactory struct {
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GitlabDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return nil, err
}
baseURL := u.Scheme + "://" + u.Host
repoNameSpace := strings.TrimPrefix(u.Path, "/")
repoNameSpace = strings.TrimSuffix(repoNameSpace, ".git")
log.Trace("Create gitlab downloader. BaseURL: %s RepoName: %s", baseURL, repoNameSpace)
return NewGitlabDownloader(ctx, baseURL, repoNameSpace, opts.AuthUsername, opts.AuthPassword, opts.AuthToken)
}
// GitServiceType returns the type of git service
func (f *GitlabDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.GitlabService
}
// GitlabDownloader implements a Downloader interface to get repository information
// from gitlab via go-gitlab
// - issueCount is incremented in GetIssues() to ensure PR and Issue numbers do not overlap,
// because Gitlab has individual Issue and Pull Request numbers.
type GitlabDownloader struct {
base.NullDownloader
ctx context.Context
client *gitlab.Client
repoID int
repoName string
issueCount int64
maxPerPage int
}
// NewGitlabDownloader creates a gitlab Downloader via gitlab API
// Use either a username/password, personal token entered into the username field, or anonymous/public access
// Note: Public access only allows very basic access
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}))
// Only use basic auth if token is blank and password is NOT
// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
if token == "" && password != "" {
gitlabClient, err = gitlab.NewBasicAuthClient(username, password, gitlab.WithBaseURL(baseURL))
}
if err != nil {
log.Trace("Error logging into gitlab: %v", err)
return nil, err
}
// split namespace and subdirectory
pathParts := strings.Split(strings.Trim(repoPath, "/"), "/")
var resp *gitlab.Response
u, _ := url.Parse(baseURL)
for len(pathParts) >= 2 {
_, resp, err = gitlabClient.Version.GetVersion()
if err == nil || resp != nil && resp.StatusCode == 401 {
err = nil // if no authentication given, this still should work
break
}
u.Path = path.Join(u.Path, pathParts[0])
baseURL = u.String()
pathParts = pathParts[1:]
_ = gitlab.WithBaseURL(baseURL)(gitlabClient)
repoPath = strings.Join(pathParts, "/")
}
if err != nil {
log.Trace("Error could not get gitlab version: %v", err)
return nil, err
}
log.Trace("gitlab downloader: use BaseURL: '%s' and RepoPath: '%s'", baseURL, repoPath)
// Grab and store project/repo ID here, due to issues using the URL escaped path
gr, _, err := gitlabClient.Projects.GetProject(repoPath, nil, nil, gitlab.WithContext(ctx))
if err != nil {
log.Trace("Error retrieving project: %v", err)
return nil, err
}
if gr == nil {
log.Trace("Error getting project, project is nil")
return nil, errors.New("Error getting project, project is nil")
}
return &GitlabDownloader{
ctx: ctx,
client: gitlabClient,
repoID: gr.ID,
repoName: gr.Name,
maxPerPage: 100,
}, nil
}
// SetContext set context
func (g *GitlabDownloader) SetContext(ctx context.Context) {
g.ctx = ctx
}
// GetRepoInfo returns a repository information
func (g *GitlabDownloader) GetRepoInfo() (*base.Repository, error) {
gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))
if err != nil {
return nil, err
}
var private bool
switch gr.Visibility {
case gitlab.InternalVisibility:
private = true
case gitlab.PrivateVisibility:
private = true
}
var owner string
if gr.Owner == nil {
log.Trace("gr.Owner is nil, trying to get owner from Namespace")
if gr.Namespace != nil && gr.Namespace.Kind == "user" {
owner = gr.Namespace.Path
}
} else {
owner = gr.Owner.Username
}
// convert gitlab repo to stand Repo
return &base.Repository{
Owner: owner,
Name: gr.Name,
IsPrivate: private,
Description: gr.Description,
OriginalURL: gr.WebURL,
CloneURL: gr.HTTPURLToRepo,
DefaultBranch: gr.DefaultBranch,
}, nil
}
// GetTopics return gitlab topics
func (g *GitlabDownloader) GetTopics() ([]string, error) {
gr, _, err := g.client.Projects.GetProject(g.repoID, nil, nil, gitlab.WithContext(g.ctx))
if err != nil {
return nil, err
}
return gr.TagList, err
}
// GetMilestones returns milestones
func (g *GitlabDownloader) GetMilestones() ([]*base.Milestone, error) {
var perPage = g.maxPerPage
var state = "all"
var milestones = make([]*base.Milestone, 0, perPage)
for i := 1; ; i++ {
ms, _, err := g.client.Milestones.ListMilestones(g.repoID, &gitlab.ListMilestonesOptions{
State: &state,
ListOptions: gitlab.ListOptions{
Page: i,
PerPage: perPage,
}}, nil, gitlab.WithContext(g.ctx))
if err != nil {
return nil, err
}
for _, m := range ms {
var desc string
if m.Description != "" {
desc = m.Description
}
var state = "open"
var closedAt *time.Time
if m.State != "" {
state = m.State
if state == "closed" {
closedAt = m.UpdatedAt
}
}
var deadline *time.Time
if m.DueDate != nil {
deadlineParsed, err := time.Parse("2006-01-02", m.DueDate.String())
if err != nil {
log.Trace("Error parsing Milestone DueDate time")
deadline = nil
} else {
deadline = &deadlineParsed
}
}
milestones = append(milestones, &base.Milestone{
Title: m.Title,
Description: desc,
Deadline: deadline,
State: state,
Created: *m.CreatedAt,
Updated: m.UpdatedAt,
Closed: closedAt,
})
}
if len(ms) < perPage {
break
}
}
return milestones, nil
}
func (g *GitlabDownloader) normalizeColor(val string) string {
val = strings.TrimLeft(val, "#")
val = strings.ToLower(val)
if len(val) == 3 {
c := []rune(val)
val = fmt.Sprintf("%c%c%c%c%c%c", c[0], c[0], c[1], c[1], c[2], c[2])
}
if len(val) != 6 {
return ""
}
return val
}
// GetLabels returns labels
func (g *GitlabDownloader) GetLabels() ([]*base.Label, error) {
var perPage = g.maxPerPage
var labels = make([]*base.Label, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Labels.ListLabels(g.repoID, &gitlab.ListLabelsOptions{ListOptions: gitlab.ListOptions{
Page: i,
PerPage: perPage,
}}, nil, gitlab.WithContext(g.ctx))
if err != nil {
return nil, err
}
for _, label := range ls {
baseLabel := &base.Label{
Name: label.Name,
Color: g.normalizeColor(label.Color),
Description: label.Description,
}
labels = append(labels, baseLabel)
}
if len(ls) < perPage {
break
}
}
return labels, nil
}
func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Release {
var zero int
r := &base.Release{
TagName: rel.TagName,
TargetCommitish: rel.Commit.ID,
Name: rel.Name,
Body: rel.Description,
Created: *rel.CreatedAt,
PublisherID: int64(rel.Author.ID),
PublisherName: rel.Author.Username,
}
httpClient := &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: proxy.Proxy(),
},
}
for k, asset := range rel.Assets.Links {
r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: int64(asset.ID),
Name: asset.Name,
ContentType: &rel.Assets.Sources[k].Format,
Size: &zero,
DownloadCount: &zero,
DownloadFunc: func() (io.ReadCloser, error) {
link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, asset.ID, gitlab.WithContext(g.ctx))
if err != nil {
return nil, err
}
req, err := http.NewRequest("GET", link.URL, nil)
if err != nil {
return nil, err
}
req = req.WithContext(g.ctx)
resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
// resp.Body is closed by the uploader
return resp.Body, nil
},
})
}
return r
}
// GetReleases returns releases
func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
var perPage = g.maxPerPage
var releases = make([]*base.Release, 0, perPage)
for i := 1; ; i++ {
ls, _, err := g.client.Releases.ListReleases(g.repoID, &gitlab.ListReleasesOptions{
Page: i,
PerPage: perPage,
}, nil, gitlab.WithContext(g.ctx))
if err != nil {
return nil, err
}
for _, release := range ls {
releases = append(releases, g.convertGitlabRelease(release))
}
if len(ls) < perPage {
break
}
}
return releases, nil
}
type gitlabIssueContext struct {
foreignID int64
localID int64
IsMergeRequest bool
}
func (c gitlabIssueContext) LocalID() int64 {
return c.localID
}
func (c gitlabIssueContext) ForeignID() int64 {
return c.foreignID
}
// GetIssues returns issues according start and limit
// Note: issue label description and colors are not supported by the go-gitlab library at this time
func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
state := "all"
sort := "asc"
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &gitlab.ListProjectIssuesOptions{
State: &state,
Sort: &sort,
ListOptions: gitlab.ListOptions{
PerPage: perPage,
Page: page,
},
}
var allIssues = make([]*base.Issue, 0, perPage)
issues, _, err := g.client.Issues.ListProjectIssues(g.repoID, opt, nil, gitlab.WithContext(g.ctx))
if err != nil {
return nil, false, fmt.Errorf("error while listing issues: %v", err)
}
for _, issue := range issues {
var labels = make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, &base.Label{
Name: l,
})
}
var milestone string
if issue.Milestone != nil {
milestone = issue.Milestone.Title
}
var reactions []*base.Reaction
var awardPage = 1
for {
awards, _, err := g.client.AwardEmoji.ListIssueAwardEmoji(g.repoID, issue.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx))
if err != nil {
return nil, false, fmt.Errorf("error while listing issue awards: %v", err)
}
for i := range awards {
reactions = append(reactions, g.awardToReaction(awards[i]))
}
if len(awards) < perPage {
break
}
awardPage++
}
allIssues = append(allIssues, &base.Issue{
Title: issue.Title,
Number: int64(issue.IID),
PosterID: int64(issue.Author.ID),
PosterName: issue.Author.Username,
Content: issue.Description,
Milestone: milestone,
State: issue.State,
Created: *issue.CreatedAt,
Labels: labels,
Reactions: reactions,
Closed: issue.ClosedAt,
IsLocked: issue.DiscussionLocked,
Updated: *issue.UpdatedAt,
Context: gitlabIssueContext{
foreignID: int64(issue.IID),
localID: int64(issue.IID),
IsMergeRequest: false,
},
})
// increment issueCount, to be used in GetPullRequests()
g.issueCount++
}
return allIssues, len(issues) < perPage, nil
}
// GetComments returns comments according issueNumber
// TODO: figure out how to transfer comment reactions
func (g *GitlabDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
context, ok := opts.Context.(gitlabIssueContext)
if !ok {
return nil, false, fmt.Errorf("unexpected context: %+v", opts.Context)
}
var allComments = make([]*base.Comment, 0, g.maxPerPage)
var page = 1
for {
var comments []*gitlab.Discussion
var resp *gitlab.Response
var err error
if !context.IsMergeRequest {
comments, resp, err = g.client.Discussions.ListIssueDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListIssueDiscussionsOptions{
Page: page,
PerPage: g.maxPerPage,
}, nil, gitlab.WithContext(g.ctx))
} else {
comments, resp, err = g.client.Discussions.ListMergeRequestDiscussions(g.repoID, int(context.ForeignID()), &gitlab.ListMergeRequestDiscussionsOptions{
Page: page,
PerPage: g.maxPerPage,
}, nil, gitlab.WithContext(g.ctx))
}
if err != nil {
return nil, false, fmt.Errorf("error while listing comments: %v %v", g.repoID, err)
}
for _, comment := range comments {
// Flatten comment threads
if !comment.IndividualNote {
for _, note := range comment.Notes {
allComments = append(allComments, &base.Comment{
IssueIndex: context.LocalID(),
PosterID: int64(note.Author.ID),
PosterName: note.Author.Username,
PosterEmail: note.Author.Email,
Content: note.Body,
Created: *note.CreatedAt,
})
}
} else {
c := comment.Notes[0]
allComments = append(allComments, &base.Comment{
IssueIndex: context.LocalID(),
PosterID: int64(c.Author.ID),
PosterName: c.Author.Username,
PosterEmail: c.Author.Email,
Content: c.Body,
Created: *c.CreatedAt,
})
}
}
if resp.NextPage == 0 {
break
}
page = resp.NextPage
}
return allComments, true, nil
}
// GetPullRequests returns pull requests according page and perPage
func (g *GitlabDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
if perPage > g.maxPerPage {
perPage = g.maxPerPage
}
opt := &gitlab.ListProjectMergeRequestsOptions{
ListOptions: gitlab.ListOptions{
PerPage: perPage,
Page: page,
},
}
var allPRs = make([]*base.PullRequest, 0, perPage)
prs, _, err := g.client.MergeRequests.ListProjectMergeRequests(g.repoID, opt, nil, gitlab.WithContext(g.ctx))
if err != nil {
return nil, false, fmt.Errorf("error while listing merge requests: %v", err)
}
for _, pr := range prs {
var labels = make([]*base.Label, 0, len(pr.Labels))
for _, l := range pr.Labels {
labels = append(labels, &base.Label{
Name: l,
})
}
var merged bool
if pr.State == "merged" {
merged = true
pr.State = "closed"
}
var mergeTime = pr.MergedAt
if merged && pr.MergedAt == nil {
mergeTime = pr.UpdatedAt
}
var closeTime = pr.ClosedAt
if merged && pr.ClosedAt == nil {
closeTime = pr.UpdatedAt
}
var locked bool
if pr.State == "locked" {
locked = true
}
var milestone string
if pr.Milestone != nil {
milestone = pr.Milestone.Title
}
var reactions []*base.Reaction
var awardPage = 1
for {
awards, _, err := g.client.AwardEmoji.ListMergeRequestAwardEmoji(g.repoID, pr.IID, &gitlab.ListAwardEmojiOptions{Page: awardPage, PerPage: perPage}, gitlab.WithContext(g.ctx))
if err != nil {
return nil, false, fmt.Errorf("error while listing merge requests awards: %v", err)
}
for i := range awards {
reactions = append(reactions, g.awardToReaction(awards[i]))
}
if len(awards) < perPage {
break
}
awardPage++
}
// Add the PR ID to the Issue Count because PR and Issues share ID space in Gitea
newPRNumber := g.issueCount + int64(pr.IID)
allPRs = append(allPRs, &base.PullRequest{
Title: pr.Title,
Number: newPRNumber,
PosterName: pr.Author.Username,
PosterID: int64(pr.Author.ID),
Content: pr.Description,
Milestone: milestone,
State: pr.State,
Created: *pr.CreatedAt,
Closed: closeTime,
Labels: labels,
Merged: merged,
MergeCommitSHA: pr.MergeCommitSHA,
MergedTime: mergeTime,
IsLocked: locked,
Reactions: reactions,
Head: base.PullRequestBranch{
Ref: pr.SourceBranch,
SHA: pr.SHA,
RepoName: g.repoName,
OwnerName: pr.Author.Username,
CloneURL: pr.WebURL,
},
Base: base.PullRequestBranch{
Ref: pr.TargetBranch,
SHA: pr.DiffRefs.BaseSha,
RepoName: g.repoName,
OwnerName: pr.Author.Username,
},
PatchURL: pr.WebURL + ".patch",
Context: gitlabIssueContext{
foreignID: int64(pr.IID),
localID: newPRNumber,
IsMergeRequest: true,
},
})
}
return allPRs, len(prs) < perPage, nil
}
// GetReviews returns pull requests review
func (g *GitlabDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
approvals, resp, err := g.client.MergeRequestApprovals.GetConfiguration(g.repoID, int(context.ForeignID()), gitlab.WithContext(g.ctx))
if err != nil {
if resp != nil && resp.StatusCode == 404 {
log.Error(fmt.Sprintf("GitlabDownloader: while migrating a error occurred: '%s'", err.Error()))
return []*base.Review{}, nil
}
return nil, err
}
var reviews = make([]*base.Review, 0, len(approvals.ApprovedBy))
for _, user := range approvals.ApprovedBy {
reviews = append(reviews, &base.Review{
IssueIndex: context.LocalID(),
ReviewerID: int64(user.User.ID),
ReviewerName: user.User.Username,
CreatedAt: *approvals.UpdatedAt,
// All we get are approvals
State: base.ReviewStateApproved,
})
}
return reviews, nil
}
func (g *GitlabDownloader) awardToReaction(award *gitlab.AwardEmoji) *base.Reaction {
return &base.Reaction{
UserID: int64(award.User.ID),
UserName: award.User.Username,
Content: award.Name,
}
}

334
services/migrations/gitlab_test.go Normal file
View file

@ -0,0 +1,334 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"net/http"
"os"
"testing"
"time"
base "code.gitea.io/gitea/modules/migration"
"github.com/stretchr/testify/assert"
)
func TestGitlabDownloadRepo(t *testing.T) {
// Skip tests if Gitlab token is not found
gitlabPersonalAccessToken := os.Getenv("GITLAB_READ_TOKEN")
if gitlabPersonalAccessToken == "" {
t.Skip("skipped test because GITLAB_READ_TOKEN was not in the environment")
}
resp, err := http.Get("https://gitlab.com/gitea/test_repo")
if err != nil || resp.StatusCode != 200 {
t.Skipf("Can't access test repo, skipping %s", t.Name())
}
downloader, err := NewGitlabDownloader(context.Background(), "https://gitlab.com", "gitea/test_repo", "", "", gitlabPersonalAccessToken)
if err != nil {
t.Fatal(fmt.Sprintf("NewGitlabDownloader is nil: %v", err))
}
repo, err := downloader.GetRepoInfo()
assert.NoError(t, err)
// Repo Owner is blank in Gitlab Group repos
assertRepositoryEqual(t, &base.Repository{
Name: "test_repo",
Owner: "",
Description: "Test repository for testing migration from gitlab to gitea",
CloneURL: "https://gitlab.com/gitea/test_repo.git",
OriginalURL: "https://gitlab.com/gitea/test_repo",
DefaultBranch: "master",
}, repo)
topics, err := downloader.GetTopics()
assert.NoError(t, err)
assert.True(t, len(topics) == 2)
assert.EqualValues(t, []string{"migration", "test"}, topics)
milestones, err := downloader.GetMilestones()
assert.NoError(t, err)
assertMilestonesEqual(t, []*base.Milestone{
{
Title: "1.1.0",
Created: time.Date(2019, 11, 28, 8, 42, 44, 575000000, time.UTC),
Updated: timePtr(time.Date(2019, 11, 28, 8, 42, 44, 575000000, time.UTC)),
State: "active",
},
{
Title: "1.0.0",
Created: time.Date(2019, 11, 28, 8, 42, 30, 301000000, time.UTC),
Updated: timePtr(time.Date(2019, 11, 28, 15, 57, 52, 401000000, time.UTC)),
Closed: timePtr(time.Date(2019, 11, 28, 15, 57, 52, 401000000, time.UTC)),
State: "closed",
},
}, milestones)
labels, err := downloader.GetLabels()
assert.NoError(t, err)
assertLabelsEqual(t, []*base.Label{
{
Name: "bug",
Color: "d9534f",
},
{
Name: "confirmed",
Color: "d9534f",
},
{
Name: "critical",
Color: "d9534f",
},
{
Name: "discussion",
Color: "428bca",
},
{
Name: "documentation",
Color: "f0ad4e",
},
{
Name: "duplicate",
Color: "7f8c8d",
},
{
Name: "enhancement",
Color: "5cb85c",
},
{
Name: "suggestion",
Color: "428bca",
},
{
Name: "support",
Color: "f0ad4e",
},
}, labels)
releases, err := downloader.GetReleases()
assert.NoError(t, err)
assertReleasesEqual(t, []*base.Release{
{
TagName: "v0.9.99",
TargetCommitish: "0720a3ec57c1f843568298117b874319e7deee75",
Name: "First Release",
Body: "A test release",
Created: time.Date(2019, 11, 28, 9, 9, 48, 840000000, time.UTC),
PublisherID: 1241334,
PublisherName: "lafriks",
},
}, releases)
issues, isEnd, err := downloader.GetIssues(1, 2)
assert.NoError(t, err)
assert.False(t, isEnd)
assertIssuesEqual(t, []*base.Issue{
{
Number: 1,
Title: "Please add an animated gif icon to the merge button",
Content: "I just want the merge button to hurt my eyes a little. :stuck_out_tongue_closed_eyes:",
Milestone: "1.0.0",
PosterID: 1241334,
PosterName: "lafriks",
State: "closed",
Created: time.Date(2019, 11, 28, 8, 43, 35, 459000000, time.UTC),
Updated: time.Date(2019, 11, 28, 8, 46, 23, 304000000, time.UTC),
Labels: []*base.Label{
{
Name: "bug",
},
{
Name: "discussion",
},
},
Reactions: []*base.Reaction{
{
UserID: 1241334,
UserName: "lafriks",
Content: "thumbsup",
},
{
UserID: 1241334,
UserName: "lafriks",
Content: "open_mouth",
}},
Closed: timePtr(time.Date(2019, 11, 28, 8, 46, 23, 275000000, time.UTC)),
},
{
Number: 2,
Title: "Test issue",
Content: "This is test issue 2, do not touch!",
Milestone: "1.1.0",
PosterID: 1241334,
PosterName: "lafriks",
State: "closed",
Created: time.Date(2019, 11, 28, 8, 44, 46, 277000000, time.UTC),
Updated: time.Date(2019, 11, 28, 8, 45, 44, 987000000, time.UTC),
Labels: []*base.Label{
{
Name: "duplicate",
},
},
Reactions: []*base.Reaction{
{
UserID: 1241334,
UserName: "lafriks",
Content: "thumbsup",
},
{
UserID: 1241334,
UserName: "lafriks",
Content: "thumbsdown",
},
{
UserID: 1241334,
UserName: "lafriks",
Content: "laughing",
},
{
UserID: 1241334,
UserName: "lafriks",
Content: "tada",
},
{
UserID: 1241334,
UserName: "lafriks",
Content: "confused",
},
{
UserID: 1241334,
UserName: "lafriks",
Content: "hearts",
}},
Closed: timePtr(time.Date(2019, 11, 28, 8, 45, 44, 959000000, time.UTC)),
},
}, issues)
comments, _, err := downloader.GetComments(base.GetCommentOptions{
Context: gitlabIssueContext{
foreignID: 2,
localID: 2,
IsMergeRequest: false,
},
})
assert.NoError(t, err)
assertCommentsEqual(t, []*base.Comment{
{
IssueIndex: 2,
PosterID: 1241334,
PosterName: "lafriks",
Created: time.Date(2019, 11, 28, 8, 44, 52, 501000000, time.UTC),
Content: "This is a comment",
Reactions: nil,
},
{
IssueIndex: 2,
PosterID: 1241334,
PosterName: "lafriks",
Created: time.Date(2019, 11, 28, 8, 45, 2, 329000000, time.UTC),
Content: "changed milestone to %2",
Reactions: nil,
},
{
IssueIndex: 2,
PosterID: 1241334,
PosterName: "lafriks",
Created: time.Date(2019, 11, 28, 8, 45, 45, 7000000, time.UTC),
Content: "closed",
Reactions: nil,
},
{
IssueIndex: 2,
PosterID: 1241334,
PosterName: "lafriks",
Created: time.Date(2019, 11, 28, 8, 45, 53, 501000000, time.UTC),
Content: "A second comment",
Reactions: nil,
},
}, comments)
prs, _, err := downloader.GetPullRequests(1, 1)
assert.NoError(t, err)
assertPullRequestsEqual(t, []*base.PullRequest{
{
Number: 4,
Title: "Test branch",
Content: "do not merge this PR",
Milestone: "1.0.0",
PosterID: 1241334,
PosterName: "lafriks",
State: "opened",
Created: time.Date(2019, 11, 28, 15, 56, 54, 104000000, time.UTC),
Labels: []*base.Label{
{
Name: "bug",
},
},
Reactions: []*base.Reaction{{
UserID: 4575606,
UserName: "real6543",
Content: "thumbsup",
}, {
UserID: 4575606,
UserName: "real6543",
Content: "tada",
}},
PatchURL: "https://gitlab.com/gitea/test_repo/-/merge_requests/2.patch",
Head: base.PullRequestBranch{
Ref: "feat/test",
CloneURL: "https://gitlab.com/gitea/test_repo/-/merge_requests/2",
SHA: "9f733b96b98a4175276edf6a2e1231489c3bdd23",
RepoName: "test_repo",
OwnerName: "lafriks",
},
Base: base.PullRequestBranch{
Ref: "master",
SHA: "",
OwnerName: "lafriks",
RepoName: "test_repo",
},
Closed: nil,
Merged: false,
MergedTime: nil,
MergeCommitSHA: "",
Context: gitlabIssueContext{
foreignID: 2,
localID: 4,
IsMergeRequest: true,
},
},
}, prs)
rvs, err := downloader.GetReviews(base.BasicIssueContext(1))
assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{
{
ReviewerID: 4102996,
ReviewerName: "zeripath",
CreatedAt: time.Date(2019, 11, 28, 16, 02, 8, 377000000, time.UTC),
State: "APPROVED",
},
{
ReviewerID: 527793,
ReviewerName: "axifive",
CreatedAt: time.Date(2019, 11, 28, 16, 02, 8, 377000000, time.UTC),
State: "APPROVED",
},
}, rvs)
rvs, err = downloader.GetReviews(base.BasicIssueContext(2))
assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{
{
ReviewerID: 4575606,
ReviewerName: "real6543",
CreatedAt: time.Date(2020, 04, 19, 19, 24, 21, 108000000, time.UTC),
State: "APPROVED",
},
}, rvs)
}

315
services/migrations/gogs.go Normal file
View file

@ -0,0 +1,315 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"crypto/tls"
"fmt"
"net/http"
"net/url"
"strings"
"time"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"github.com/gogs/go-gogs-client"
)
var (
_ base.Downloader = &GogsDownloader{}
_ base.DownloaderFactory = &GogsDownloaderFactory{}
)
func init() {
RegisterDownloaderFactory(&GogsDownloaderFactory{})
}
// GogsDownloaderFactory defines a gogs downloader factory
type GogsDownloaderFactory struct {
}
// New returns a Downloader related to this factory according MigrateOptions
func (f *GogsDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return nil, err
}
baseURL := u.Scheme + "://" + u.Host
repoNameSpace := strings.TrimSuffix(u.Path, ".git")
repoNameSpace = strings.Trim(repoNameSpace, "/")
fields := strings.Split(repoNameSpace, "/")
if len(fields) < 2 {
return nil, fmt.Errorf("invalid path: %s", repoNameSpace)
}
log.Trace("Create gogs downloader. BaseURL: %s RepoOwner: %s RepoName: %s", baseURL, fields[0], fields[1])
return NewGogsDownloader(ctx, baseURL, opts.AuthUsername, opts.AuthPassword, opts.AuthToken, fields[0], fields[1]), nil
}
// GitServiceType returns the type of git service
func (f *GogsDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.GogsService
}
// GogsDownloader implements a Downloader interface to get repository information
// from gogs via API
type GogsDownloader struct {
base.NullDownloader
ctx context.Context
client *gogs.Client
baseURL string
repoOwner string
repoName string
userName string
password string
openIssuesFinished bool
openIssuesPages int
transport http.RoundTripper
}
// SetContext set context
func (g *GogsDownloader) SetContext(ctx context.Context) {
g.ctx = ctx
}
// NewGogsDownloader creates a gogs Downloader via gogs API
func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token, repoOwner, repoName string) *GogsDownloader {
var downloader = GogsDownloader{
ctx: ctx,
baseURL: baseURL,
userName: userName,
password: password,
repoOwner: repoOwner,
repoName: repoName,
}
var client *gogs.Client
if len(token) != 0 {
client = gogs.NewClient(baseURL, token)
downloader.userName = token
} else {
downloader.transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
return proxy.Proxy()(req)
},
}
client = gogs.NewClient(baseURL, "")
client.SetHTTPClient(&http.Client{
Transport: &downloader,
})
}
downloader.client = client
return &downloader
}
// RoundTrip wraps the provided request within this downloader's context and passes it to our internal http.Transport.
// This implements http.RoundTripper and makes the gogs client requests cancellable even though it is not cancellable itself
func (g *GogsDownloader) RoundTrip(req *http.Request) (*http.Response, error) {
return g.transport.RoundTrip(req.WithContext(g.ctx))
}
// GetRepoInfo returns a repository information
func (g *GogsDownloader) GetRepoInfo() (*base.Repository, error) {
gr, err := g.client.GetRepo(g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
// convert gogs repo to stand Repo
return &base.Repository{
Owner: g.repoOwner,
Name: g.repoName,
IsPrivate: gr.Private,
Description: gr.Description,
CloneURL: gr.CloneURL,
OriginalURL: gr.HTMLURL,
DefaultBranch: gr.DefaultBranch,
}, nil
}
// GetMilestones returns milestones
func (g *GogsDownloader) GetMilestones() ([]*base.Milestone, error) {
var perPage = 100
var milestones = make([]*base.Milestone, 0, perPage)
ms, err := g.client.ListRepoMilestones(g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
for _, m := range ms {
milestones = append(milestones, &base.Milestone{
Title: m.Title,
Description: m.Description,
Deadline: m.Deadline,
State: string(m.State),
Closed: m.Closed,
})
}
return milestones, nil
}
// GetLabels returns labels
func (g *GogsDownloader) GetLabels() ([]*base.Label, error) {
var perPage = 100
var labels = make([]*base.Label, 0, perPage)
ls, err := g.client.ListRepoLabels(g.repoOwner, g.repoName)
if err != nil {
return nil, err
}
for _, label := range ls {
labels = append(labels, convertGogsLabel(label))
}
return labels, nil
}
// GetIssues returns issues according start and limit, perPage is not supported
func (g *GogsDownloader) GetIssues(page, _ int) ([]*base.Issue, bool, error) {
var state string
if g.openIssuesFinished {
state = string(gogs.STATE_CLOSED)
page -= g.openIssuesPages
} else {
state = string(gogs.STATE_OPEN)
g.openIssuesPages = page
}
issues, isEnd, err := g.getIssues(page, state)
if err != nil {
return nil, false, err
}
if isEnd {
if g.openIssuesFinished {
return issues, true, nil
}
g.openIssuesFinished = true
}
return issues, false, nil
}
func (g *GogsDownloader) getIssues(page int, state string) ([]*base.Issue, bool, error) {
var allIssues = make([]*base.Issue, 0, 10)
issues, err := g.client.ListRepoIssues(g.repoOwner, g.repoName, gogs.ListIssueOption{
Page: page,
State: state,
})
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
for _, issue := range issues {
if issue.PullRequest != nil {
continue
}
allIssues = append(allIssues, convertGogsIssue(issue))
}
return allIssues, len(issues) == 0, nil
}
// GetComments returns comments according issueNumber
func (g *GogsDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
var allComments = make([]*base.Comment, 0, 100)
comments, err := g.client.ListIssueComments(g.repoOwner, g.repoName, opts.Context.ForeignID())
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
for _, comment := range comments {
if len(comment.Body) == 0 || comment.Poster == nil {
continue
}
allComments = append(allComments, &base.Comment{
IssueIndex: opts.Context.LocalID(),
PosterID: comment.Poster.ID,
PosterName: comment.Poster.Login,
PosterEmail: comment.Poster.Email,
Content: comment.Body,
Created: comment.Created,
Updated: comment.Updated,
})
}
return allComments, true, nil
}
// GetTopics return repository topics
func (g *GogsDownloader) GetTopics() ([]string, error) {
return []string{}, nil
}
// FormatCloneURL add authentification into remote URLs
func (g *GogsDownloader) FormatCloneURL(opts MigrateOptions, remoteAddr string) (string, error) {
if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
if len(opts.AuthToken) != 0 {
u.User = url.UserPassword(opts.AuthToken, "")
} else {
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
}
return u.String(), nil
}
return remoteAddr, nil
}
func convertGogsIssue(issue *gogs.Issue) *base.Issue {
var milestone string
if issue.Milestone != nil {
milestone = issue.Milestone.Title
}
var labels = make([]*base.Label, 0, len(issue.Labels))
for _, l := range issue.Labels {
labels = append(labels, convertGogsLabel(l))
}
var closed *time.Time
if issue.State == gogs.STATE_CLOSED {
// gogs client haven't provide closed, so we use updated instead
closed = &issue.Updated
}
return &base.Issue{
Title: issue.Title,
Number: issue.Index,
PosterID: issue.Poster.ID,
PosterName: issue.Poster.Login,
PosterEmail: issue.Poster.Email,
Content: issue.Body,
Milestone: milestone,
State: string(issue.State),
Created: issue.Created,
Updated: issue.Updated,
Labels: labels,
Closed: closed,
Context: base.BasicIssueContext(issue.Index),
}
}
func convertGogsLabel(label *gogs.Label) *base.Label {
return &base.Label{
Name: label.Name,
Color: label.Color,
}
}

142
services/migrations/gogs_test.go Normal file
View file

@ -0,0 +1,142 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"net/http"
"os"
"testing"
"time"
base "code.gitea.io/gitea/modules/migration"
"github.com/stretchr/testify/assert"
)
func TestGogsDownloadRepo(t *testing.T) {
// Skip tests if Gogs token is not found
gogsPersonalAccessToken := os.Getenv("GOGS_READ_TOKEN")
if len(gogsPersonalAccessToken) == 0 {
t.Skip("skipped test because GOGS_READ_TOKEN was not in the environment")
}
resp, err := http.Get("https://try.gogs.io/lunnytest/TESTREPO")
if err != nil || resp.StatusCode/100 != 2 {
// skip and don't run test
t.Skipf("visit test repo failed, ignored")
return
}
downloader := NewGogsDownloader(context.Background(), "https://try.gogs.io", "", "", gogsPersonalAccessToken, "lunnytest", "TESTREPO")
repo, err := downloader.GetRepoInfo()
assert.NoError(t, err)
assertRepositoryEqual(t, &base.Repository{
Name: "TESTREPO",
Owner: "lunnytest",
Description: "",
CloneURL: "https://try.gogs.io/lunnytest/TESTREPO.git",
OriginalURL: "https://try.gogs.io/lunnytest/TESTREPO",
DefaultBranch: "master",
}, repo)
milestones, err := downloader.GetMilestones()
assert.NoError(t, err)
assertMilestonesEqual(t, []*base.Milestone{
{
Title: "1.0",
State: "open",
},
}, milestones)
labels, err := downloader.GetLabels()
assert.NoError(t, err)
assertLabelsEqual(t, []*base.Label{
{
Name: "bug",
Color: "ee0701",
},
{
Name: "duplicate",
Color: "cccccc",
},
{
Name: "enhancement",
Color: "84b6eb",
},
{
Name: "help wanted",
Color: "128a0c",
},
{
Name: "invalid",
Color: "e6e6e6",
},
{
Name: "question",
Color: "cc317c",
},
{
Name: "wontfix",
Color: "ffffff",
},
}, labels)
// downloader.GetIssues()
issues, isEnd, err := downloader.GetIssues(1, 8)
assert.NoError(t, err)
assert.False(t, isEnd)
assertIssuesEqual(t, []*base.Issue{
{
Number: 1,
PosterID: 5331,
PosterName: "lunny",
PosterEmail: "xiaolunwen@gmail.com",
Title: "test",
Content: "test",
Milestone: "",
State: "open",
Created: time.Date(2019, 06, 11, 8, 16, 44, 0, time.UTC),
Updated: time.Date(2019, 10, 26, 11, 07, 2, 0, time.UTC),
Labels: []*base.Label{
{
Name: "bug",
Color: "ee0701",
},
},
},
}, issues)
// downloader.GetComments()
comments, _, err := downloader.GetComments(base.GetCommentOptions{
Context: base.BasicIssueContext(1),
})
assert.NoError(t, err)
assertCommentsEqual(t, []*base.Comment{
{
IssueIndex: 1,
PosterID: 5331,
PosterName: "lunny",
PosterEmail: "xiaolunwen@gmail.com",
Created: time.Date(2019, 06, 11, 8, 19, 50, 0, time.UTC),
Updated: time.Date(2019, 06, 11, 8, 19, 50, 0, time.UTC),
Content: "1111",
},
{
IssueIndex: 1,
PosterID: 15822,
PosterName: "clacplouf",
PosterEmail: "test1234@dbn.re",
Created: time.Date(2019, 10, 26, 11, 7, 2, 0, time.UTC),
Updated: time.Date(2019, 10, 26, 11, 7, 2, 0, time.UTC),
Content: "88888888",
},
}, comments)
// downloader.GetPullRequests()
_, _, err = downloader.GetPullRequests(1, 3)
assert.Error(t, err)
}

266
services/migrations/main_test.go Normal file
View file

@ -0,0 +1,266 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"path/filepath"
"testing"
"time"
"code.gitea.io/gitea/models/unittest"
base "code.gitea.io/gitea/modules/migration"
"github.com/stretchr/testify/assert"
)
func TestMain(m *testing.M) {
unittest.MainTest(m, filepath.Join("..", ".."))
}
func timePtr(t time.Time) *time.Time {
return &t
}
func assertTimeEqual(t *testing.T, expected, actual time.Time) {
assert.Equal(t, expected.UTC(), actual.UTC())
}
func assertTimePtrEqual(t *testing.T, expected, actual *time.Time) {
if expected == nil {
assert.Nil(t, actual)
} else {
assertTimeEqual(t, *expected, *actual)
}
}
func assertCommentEqual(t *testing.T, expected, actual *base.Comment) {
assert.Equal(t, expected.IssueIndex, actual.IssueIndex)
assert.Equal(t, expected.PosterID, actual.PosterID)
assert.Equal(t, expected.PosterName, actual.PosterName)
assert.Equal(t, expected.PosterEmail, actual.PosterEmail)
assertTimeEqual(t, expected.Created, actual.Created)
assertTimeEqual(t, expected.Updated, actual.Updated)
assert.Equal(t, expected.Content, actual.Content)
assertReactionsEqual(t, expected.Reactions, actual.Reactions)
}
func assertCommentsEqual(t *testing.T, expected, actual []*base.Comment) {
if assert.Len(t, actual, len(expected)) {
for i := range expected {
assertCommentEqual(t, expected[i], actual[i])
}
}
}
func assertLabelEqual(t *testing.T, expected, actual *base.Label) {
assert.Equal(t, expected.Name, actual.Name)
assert.Equal(t, expected.Color, actual.Color)
assert.Equal(t, expected.Description, actual.Description)
}
func assertLabelsEqual(t *testing.T, expected, actual []*base.Label) {
if assert.Len(t, actual, len(expected)) {
for i := range expected {
assertLabelEqual(t, expected[i], actual[i])
}
}
}
func assertMilestoneEqual(t *testing.T, expected, actual *base.Milestone) {
assert.Equal(t, expected.Title, actual.Title)
assert.Equal(t, expected.Description, actual.Description)
assertTimePtrEqual(t, expected.Deadline, actual.Deadline)
assertTimeEqual(t, expected.Created, actual.Created)
assertTimePtrEqual(t, expected.Updated, actual.Updated)
assertTimePtrEqual(t, expected.Closed, actual.Closed)
assert.Equal(t, expected.State, actual.State)
}
func assertMilestonesEqual(t *testing.T, expected, actual []*base.Milestone) {
if assert.Len(t, actual, len(expected)) {
for i := range expected {
assertMilestoneEqual(t, expected[i], actual[i])
}
}
}
func assertIssueEqual(t *testing.T, expected, actual *base.Issue) {
assert.Equal(t, expected.Number, actual.Number)
assert.Equal(t, expected.PosterID, actual.PosterID)
assert.Equal(t, expected.PosterName, actual.PosterName)
assert.Equal(t, expected.PosterEmail, actual.PosterEmail)
assert.Equal(t, expected.Title, actual.Title)
assert.Equal(t, expected.Content, actual.Content)
assert.Equal(t, expected.Ref, actual.Ref)
assert.Equal(t, expected.Milestone, actual.Milestone)
assert.Equal(t, expected.State, actual.State)
assert.Equal(t, expected.IsLocked, actual.IsLocked)
assertTimeEqual(t, expected.Created, actual.Created)
assertTimeEqual(t, expected.Updated, actual.Updated)
assertTimePtrEqual(t, expected.Closed, actual.Closed)
assertLabelsEqual(t, expected.Labels, actual.Labels)
assertReactionsEqual(t, expected.Reactions, actual.Reactions)
assert.ElementsMatch(t, expected.Assignees, actual.Assignees)
}
func assertIssuesEqual(t *testing.T, expected, actual []*base.Issue) {
if assert.Len(t, actual, len(expected)) {
for i := range expected {
assertIssueEqual(t, expected[i], actual[i])
}
}
}
func assertPullRequestEqual(t *testing.T, expected, actual *base.PullRequest) {
assert.Equal(t, expected.Number, actual.Number)
assert.Equal(t, expected.Title, actual.Title)
assert.Equal(t, expected.PosterID, actual.PosterID)
assert.Equal(t, expected.PosterName, actual.PosterName)
assert.Equal(t, expected.PosterEmail, actual.PosterEmail)
assert.Equal(t, expected.Content, actual.Content)
assert.Equal(t, expected.Milestone, actual.Milestone)
assert.Equal(t, expected.State, actual.State)
assertTimeEqual(t, expected.Created, actual.Created)
assertTimeEqual(t, expected.Updated, actual.Updated)
assertTimePtrEqual(t, expected.Closed, actual.Closed)
assertLabelsEqual(t, expected.Labels, actual.Labels)
assert.Equal(t, expected.PatchURL, actual.PatchURL)
assert.Equal(t, expected.Merged, actual.Merged)
assertTimePtrEqual(t, expected.MergedTime, actual.MergedTime)
assert.Equal(t, expected.MergeCommitSHA, actual.MergeCommitSHA)
assertPullRequestBranchEqual(t, expected.Head, actual.Head)
assertPullRequestBranchEqual(t, expected.Base, actual.Base)
assert.ElementsMatch(t, expected.Assignees, actual.Assignees)
assert.Equal(t, expected.IsLocked, actual.IsLocked)
assertReactionsEqual(t, expected.Reactions, actual.Reactions)
}
func assertPullRequestsEqual(t *testing.T, expected, actual []*base.PullRequest) {
if assert.Len(t, actual, len(expected)) {
for i := range expected {
assertPullRequestEqual(t, expected[i], actual[i])
}
}
}
func assertPullRequestBranchEqual(t *testing.T, expected, actual base.PullRequestBranch) {
assert.Equal(t, expected.CloneURL, actual.CloneURL)
assert.Equal(t, expected.Ref, actual.Ref)
assert.Equal(t, expected.SHA, actual.SHA)
assert.Equal(t, expected.RepoName, actual.RepoName)
assert.Equal(t, expected.OwnerName, actual.OwnerName)
}
func assertReactionEqual(t *testing.T, expected, actual *base.Reaction) {
assert.Equal(t, expected.UserID, actual.UserID)
assert.Equal(t, expected.UserName, actual.UserName)
assert.Equal(t, expected.Content, actual.Content)
}
func assertReactionsEqual(t *testing.T, expected, actual []*base.Reaction) {
if assert.Len(t, actual, len(expected)) {
for i := range expected {
assertReactionEqual(t, expected[i], actual[i])
}
}
}
func assertReleaseAssetEqual(t *testing.T, expected, actual *base.ReleaseAsset) {
assert.Equal(t, expected.ID, actual.ID)
assert.Equal(t, expected.Name, actual.Name)
assert.Equal(t, expected.ContentType, actual.ContentType)
assert.Equal(t, expected.Size, actual.Size)
assert.Equal(t, expected.DownloadCount, actual.DownloadCount)
assertTimeEqual(t, expected.Created, actual.Created)
assertTimeEqual(t, expected.Updated, actual.Updated)
assert.Equal(t, expected.DownloadURL, actual.DownloadURL)
}
func assertReleaseAssetsEqual(t *testing.T, expected, actual []*base.ReleaseAsset) {
if assert.Len(t, actual, len(expected)) {
for i := range expected {
assertReleaseAssetEqual(t, expected[i], actual[i])
}
}
}
func assertReleaseEqual(t *testing.T, expected, actual *base.Release) {
assert.Equal(t, expected.TagName, actual.TagName)
assert.Equal(t, expected.TargetCommitish, actual.TargetCommitish)
assert.Equal(t, expected.Name, actual.Name)
assert.Equal(t, expected.Body, actual.Body)
assert.Equal(t, expected.Draft, actual.Draft)
assert.Equal(t, expected.Prerelease, actual.Prerelease)
assert.Equal(t, expected.PublisherID, actual.PublisherID)
assert.Equal(t, expected.PublisherName, actual.PublisherName)
assert.Equal(t, expected.PublisherEmail, actual.PublisherEmail)
assertReleaseAssetsEqual(t, expected.Assets, actual.Assets)
assertTimeEqual(t, expected.Created, actual.Created)
assertTimeEqual(t, expected.Published, actual.Published)
}
func assertReleasesEqual(t *testing.T, expected, actual []*base.Release) {
if assert.Len(t, actual, len(expected)) {
for i := range expected {
assertReleaseEqual(t, expected[i], actual[i])
}
}
}
func assertRepositoryEqual(t *testing.T, expected, actual *base.Repository) {
assert.Equal(t, expected.Name, actual.Name)
assert.Equal(t, expected.Owner, actual.Owner)
assert.Equal(t, expected.IsPrivate, actual.IsPrivate)
assert.Equal(t, expected.IsMirror, actual.IsMirror)
assert.Equal(t, expected.Description, actual.Description)
assert.Equal(t, expected.CloneURL, actual.CloneURL)
assert.Equal(t, expected.OriginalURL, actual.OriginalURL)
assert.Equal(t, expected.DefaultBranch, actual.DefaultBranch)
}
func assertReviewEqual(t *testing.T, expected, actual *base.Review) {
assert.Equal(t, expected.ID, actual.ID)
assert.Equal(t, expected.IssueIndex, actual.IssueIndex)
assert.Equal(t, expected.ReviewerID, actual.ReviewerID)
assert.Equal(t, expected.ReviewerName, actual.ReviewerName)
assert.Equal(t, expected.Official, actual.Official)
assert.Equal(t, expected.CommitID, actual.CommitID)
assert.Equal(t, expected.Content, actual.Content)
assertTimeEqual(t, expected.CreatedAt, actual.CreatedAt)
assert.Equal(t, expected.State, actual.State)
assertReviewCommentsEqual(t, expected.Comments, actual.Comments)
}
func assertReviewsEqual(t *testing.T, expected, actual []*base.Review) {
if assert.Len(t, actual, len(expected)) {
for i := range expected {
assertReviewEqual(t, expected[i], actual[i])
}
}
}
func assertReviewCommentEqual(t *testing.T, expected, actual *base.ReviewComment) {
assert.Equal(t, expected.ID, actual.ID)
assert.Equal(t, expected.InReplyTo, actual.InReplyTo)
assert.Equal(t, expected.Content, actual.Content)
assert.Equal(t, expected.TreePath, actual.TreePath)
assert.Equal(t, expected.DiffHunk, actual.DiffHunk)
assert.Equal(t, expected.Position, actual.Position)
assert.Equal(t, expected.Line, actual.Line)
assert.Equal(t, expected.CommitID, actual.CommitID)
assert.Equal(t, expected.PosterID, actual.PosterID)
assertReactionsEqual(t, expected.Reactions, actual.Reactions)
assertTimeEqual(t, expected.CreatedAt, actual.CreatedAt)
assertTimeEqual(t, expected.UpdatedAt, actual.UpdatedAt)
}
func assertReviewCommentsEqual(t *testing.T, expected, actual []*base.ReviewComment) {
if assert.Len(t, actual, len(expected)) {
for i := range expected {
assertReviewCommentEqual(t, expected[i], actual[i])
}
}
}

476
services/migrations/migrate.go Normal file
View file

@ -0,0 +1,476 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Copyright 2018 Jonas Franz. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"net"
"net/url"
"path/filepath"
"strings"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/matchlist"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
)
// MigrateOptions is equal to base.MigrateOptions
type MigrateOptions = base.MigrateOptions
var (
factories []base.DownloaderFactory
allowList *matchlist.Matchlist
blockList *matchlist.Matchlist
)
// RegisterDownloaderFactory registers a downloader factory
func RegisterDownloaderFactory(factory base.DownloaderFactory) {
factories = append(factories, factory)
}
// IsMigrateURLAllowed checks if an URL is allowed to be migrated from
func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
// Remote address can be HTTP/HTTPS/Git URL or local path.
u, err := url.Parse(remoteURL)
if err != nil {
return &models.ErrInvalidCloneAddr{IsURLError: true}
}
if u.Scheme == "file" || u.Scheme == "" {
if !doer.CanImportLocal() {
return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsPermissionDenied: true, LocalPath: true}
}
isAbs := filepath.IsAbs(u.Host + u.Path)
if !isAbs {
return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
}
isDir, err := util.IsDir(u.Host + u.Path)
if err != nil {
log.Error("Unable to check if %s is a directory: %v", u.Host+u.Path, err)
return err
}
if !isDir {
return &models.ErrInvalidCloneAddr{Host: "<LOCAL_FILESYSTEM>", IsInvalidPath: true, LocalPath: true}
}
return nil
}
if u.Scheme == "git" && u.Port() != "" && (strings.Contains(remoteURL, "%0d") || strings.Contains(remoteURL, "%0a")) {
return &models.ErrInvalidCloneAddr{Host: u.Host, IsURLError: true}
}
if u.Opaque != "" || u.Scheme != "" && u.Scheme != "http" && u.Scheme != "https" && u.Scheme != "git" {
return &models.ErrInvalidCloneAddr{Host: u.Host, IsProtocolInvalid: true, IsPermissionDenied: true, IsURLError: true}
}
host := strings.ToLower(u.Host)
if len(setting.Migrations.AllowedDomains) > 0 {
if !allowList.Match(host) {
return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
}
} else {
if blockList.Match(host) {
return &models.ErrInvalidCloneAddr{Host: u.Host, IsPermissionDenied: true}
}
}
if !setting.Migrations.AllowLocalNetworks {
addrList, err := net.LookupIP(strings.Split(u.Host, ":")[0])
if err != nil {
return &models.ErrInvalidCloneAddr{Host: u.Host, NotResolvedIP: true}
}
for _, addr := range addrList {
if util.IsIPPrivate(addr) || !addr.IsGlobalUnicast() {
return &models.ErrInvalidCloneAddr{Host: u.Host, PrivateNet: addr.String(), IsPermissionDenied: true}
}
}
}
return nil
}
// MigrateRepository migrate repository according MigrateOptions
func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*models.Repository, error) {
err := IsMigrateURLAllowed(opts.CloneAddr, doer)
if err != nil {
return nil, err
}
if opts.LFS && len(opts.LFSEndpoint) > 0 {
err := IsMigrateURLAllowed(opts.LFSEndpoint, doer)
if err != nil {
return nil, err
}
}
downloader, err := newDownloader(ctx, ownerName, opts)
if err != nil {
return nil, err
}
var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
uploader.gitServiceType = opts.GitServiceType
if err := migrateRepository(downloader, uploader, opts, messenger); err != nil {
if err1 := uploader.Rollback(); err1 != nil {
log.Error("rollback failed: %v", err1)
}
if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
log.Error("create respotiry notice failed: ", err2)
}
return nil, err
}
return uploader.repo, nil
}
func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) {
var (
downloader base.Downloader
err error
)
for _, factory := range factories {
if factory.GitServiceType() == opts.GitServiceType {
downloader, err = factory.New(ctx, opts)
if err != nil {
return nil, err
}
break
}
}
if downloader == nil {
opts.Wiki = true
opts.Milestones = false
opts.Labels = false
opts.Releases = false
opts.Comments = false
opts.Issues = false
opts.PullRequests = false
downloader = NewPlainGitDownloader(ownerName, opts.RepoName, opts.CloneAddr)
log.Trace("Will migrate from git: %s", opts.OriginalURL)
}
if setting.Migrations.MaxAttempts > 1 {
downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
}
return downloader, nil
}
// migrateRepository will download information and then upload it to Uploader, this is a simple
// process for small repository. For a big repository, save all the data to disk
// before upload is better
func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
if messenger == nil {
messenger = base.NilMessenger
}
repo, err := downloader.GetRepoInfo()
if err != nil {
if !base.IsErrNotSupported(err) {
return err
}
log.Info("migrating repo infos is not supported, ignored")
}
repo.IsPrivate = opts.Private
repo.IsMirror = opts.Mirror
if opts.Description != "" {
repo.Description = opts.Description
}
if repo.CloneURL, err = downloader.FormatCloneURL(opts, repo.CloneURL); err != nil {
return err
}
log.Trace("migrating git data from %s", repo.CloneURL)
messenger("repo.migrate.migrating_git")
if err = uploader.CreateRepo(repo, opts); err != nil {
return err
}
defer uploader.Close()
log.Trace("migrating topics")
messenger("repo.migrate.migrating_topics")
topics, err := downloader.GetTopics()
if err != nil {
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating topics is not supported, ignored")
}
if len(topics) != 0 {
if err = uploader.CreateTopics(topics...); err != nil {
return err
}
}
if opts.Milestones {
log.Trace("migrating milestones")
messenger("repo.migrate.migrating_milestones")
milestones, err := downloader.GetMilestones()
if err != nil {
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating milestones is not supported, ignored")
}
msBatchSize := uploader.MaxBatchInsertSize("milestone")
for len(milestones) > 0 {
if len(milestones) < msBatchSize {
msBatchSize = len(milestones)
}
if err := uploader.CreateMilestones(milestones...); err != nil {
return err
}
milestones = milestones[msBatchSize:]
}
}
if opts.Labels {
log.Trace("migrating labels")
messenger("repo.migrate.migrating_labels")
labels, err := downloader.GetLabels()
if err != nil {
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating labels is not supported, ignored")
}
lbBatchSize := uploader.MaxBatchInsertSize("label")
for len(labels) > 0 {
if len(labels) < lbBatchSize {
lbBatchSize = len(labels)
}
if err := uploader.CreateLabels(labels...); err != nil {
return err
}
labels = labels[lbBatchSize:]
}
}
if opts.Releases {
log.Trace("migrating releases")
messenger("repo.migrate.migrating_releases")
releases, err := downloader.GetReleases()
if err != nil {
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating releases is not supported, ignored")
}
relBatchSize := uploader.MaxBatchInsertSize("release")
for len(releases) > 0 {
if len(releases) < relBatchSize {
relBatchSize = len(releases)
}
if err = uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
return err
}
releases = releases[relBatchSize:]
}
// Once all releases (if any) are inserted, sync any remaining non-release tags
if err = uploader.SyncTags(); err != nil {
return err
}
}
var (
commentBatchSize = uploader.MaxBatchInsertSize("comment")
reviewBatchSize = uploader.MaxBatchInsertSize("review")
)
supportAllComments := downloader.SupportGetRepoComments()
if opts.Issues {
log.Trace("migrating issues and comments")
messenger("repo.migrate.migrating_issues")
var issueBatchSize = uploader.MaxBatchInsertSize("issue")
for i := 1; ; i++ {
issues, isEnd, err := downloader.GetIssues(i, issueBatchSize)
if err != nil {
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating issues is not supported, ignored")
break
}
if err := uploader.CreateIssues(issues...); err != nil {
return err
}
if opts.Comments && !supportAllComments {
var allComments = make([]*base.Comment, 0, commentBatchSize)
for _, issue := range issues {
log.Trace("migrating issue %d's comments", issue.Number)
comments, _, err := downloader.GetComments(base.GetCommentOptions{
Context: issue.Context,
})
if err != nil {
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating comments is not supported, ignored")
}
allComments = append(allComments, comments...)
if len(allComments) >= commentBatchSize {
if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
return err
}
allComments = allComments[commentBatchSize:]
}
}
if len(allComments) > 0 {
if err = uploader.CreateComments(allComments...); err != nil {
return err
}
}
}
if isEnd {
break
}
}
}
if opts.PullRequests {
log.Trace("migrating pull requests and comments")
messenger("repo.migrate.migrating_pulls")
var prBatchSize = uploader.MaxBatchInsertSize("pullrequest")
for i := 1; ; i++ {
prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize)
if err != nil {
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating pull requests is not supported, ignored")
break
}
if err := uploader.CreatePullRequests(prs...); err != nil {
return err
}
if opts.Comments {
if !supportAllComments {
// plain comments
var allComments = make([]*base.Comment, 0, commentBatchSize)
for _, pr := range prs {
log.Trace("migrating pull request %d's comments", pr.Number)
comments, _, err := downloader.GetComments(base.GetCommentOptions{
Context: pr.Context,
})
if err != nil {
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating comments is not supported, ignored")
}
allComments = append(allComments, comments...)
if len(allComments) >= commentBatchSize {
if err = uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
return err
}
allComments = allComments[commentBatchSize:]
}
}
if len(allComments) > 0 {
if err = uploader.CreateComments(allComments...); err != nil {
return err
}
}
}
// migrate reviews
var allReviews = make([]*base.Review, 0, reviewBatchSize)
for _, pr := range prs {
reviews, err := downloader.GetReviews(pr.Context)
if err != nil {
if !base.IsErrNotSupported(err) {
return err
}
log.Warn("migrating reviews is not supported, ignored")
break
}
allReviews = append(allReviews, reviews...)
if len(allReviews) >= reviewBatchSize {
if err = uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
return err
}
allReviews = allReviews[reviewBatchSize:]
}
}
if len(allReviews) > 0 {
if err = uploader.CreateReviews(allReviews...); err != nil {
return err
}
}
}
if isEnd {
break
}
}
}
if opts.Comments && supportAllComments {
log.Trace("migrating comments")
for i := 1; ; i++ {
comments, isEnd, err := downloader.GetComments(base.GetCommentOptions{
Page: i,
PageSize: commentBatchSize,
})
if err != nil {
return err
}
if err := uploader.CreateComments(comments...); err != nil {
return err
}
if isEnd {
break
}
}
}
return uploader.Finish()
}
// Init migrations service
func Init() error {
var err error
allowList, err = matchlist.NewMatchlist(setting.Migrations.AllowedDomains...)
if err != nil {
return fmt.Errorf("init migration allowList domains failed: %v", err)
}
blockList, err = matchlist.NewMatchlist(setting.Migrations.BlockedDomains...)
if err != nil {
return fmt.Errorf("init migration blockList domains failed: %v", err)
}
return nil
}

74
services/migrations/migrate_test.go Normal file
View file

@ -0,0 +1,74 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"path/filepath"
"testing"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/unittest"
"code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert"
)
func TestMigrateWhiteBlocklist(t *testing.T) {
assert.NoError(t, unittest.PrepareTestDatabase())
adminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user1"}).(*models.User)
nonAdminUser := unittest.AssertExistsAndLoadBean(t, &models.User{Name: "user2"}).(*models.User)
setting.Migrations.AllowedDomains = []string{"github.com"}
assert.NoError(t, Init())
err := IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
assert.Error(t, err)
err = IsMigrateURLAllowed("https://github.com/go-gitea/gitea.git", nonAdminUser)
assert.NoError(t, err)
err = IsMigrateURLAllowed("https://gITHUb.com/go-gitea/gitea.git", nonAdminUser)
assert.NoError(t, err)
setting.Migrations.AllowedDomains = []string{}
setting.Migrations.BlockedDomains = []string{"github.com"}
assert.NoError(t, Init())
err = IsMigrateURLAllowed("https://gitlab.com/gitlab/gitlab.git", nonAdminUser)
assert.NoError(t, err)
err = IsMigrateURLAllowed("https://github.com/go-gitea/gitea.git", nonAdminUser)
assert.Error(t, err)
err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser)
assert.Error(t, err)
setting.Migrations.AllowLocalNetworks = true
err = IsMigrateURLAllowed("https://10.0.0.1/go-gitea/gitea.git", nonAdminUser)
assert.NoError(t, err)
old := setting.ImportLocalPaths
setting.ImportLocalPaths = false
err = IsMigrateURLAllowed("/home/foo/bar/goo", adminUser)
assert.Error(t, err)
setting.ImportLocalPaths = true
abs, err := filepath.Abs(".")
assert.NoError(t, err)
err = IsMigrateURLAllowed(abs, adminUser)
assert.NoError(t, err)
err = IsMigrateURLAllowed(abs, nonAdminUser)
assert.Error(t, err)
nonAdminUser.AllowImportLocal = true
err = IsMigrateURLAllowed(abs, nonAdminUser)
assert.NoError(t, err)
setting.ImportLocalPaths = old
}

619
services/migrations/onedev.go Normal file
View file

@ -0,0 +1,619 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"net/http"
"net/url"
"strconv"
"strings"
"time"
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
base "code.gitea.io/gitea/modules/migration"
"code.gitea.io/gitea/modules/structs"
)
var (
_ base.Downloader = &OneDevDownloader{}
_ base.DownloaderFactory = &OneDevDownloaderFactory{}
)
func init() {
RegisterDownloaderFactory(&OneDevDownloaderFactory{})
}
// OneDevDownloaderFactory defines a downloader factory
type OneDevDownloaderFactory struct {
}
// New returns a downloader related to this factory according MigrateOptions
func (f *OneDevDownloaderFactory) New(ctx context.Context, opts base.MigrateOptions) (base.Downloader, error) {
u, err := url.Parse(opts.CloneAddr)
if err != nil {
return nil, err
}
repoName := ""
fields := strings.Split(strings.Trim(u.Path, "/"), "/")
if len(fields) == 2 && fields[0] == "projects" {
repoName = fields[1]
} else if len(fields) == 1 {
repoName = fields[0]
} else {
return nil, fmt.Errorf("invalid path: %s", u.Path)
}
u.Path = ""
u.Fragment = ""
log.Trace("Create onedev downloader. BaseURL: %v RepoName: %s", u, repoName)
return NewOneDevDownloader(ctx, u, opts.AuthUsername, opts.AuthPassword, repoName), nil
}
// GitServiceType returns the type of git service
func (f *OneDevDownloaderFactory) GitServiceType() structs.GitServiceType {
return structs.OneDevService
}
type onedevUser struct {
ID int64 `json:"id"`
Name string `json:"name"`
Email string `json:"email"`
}
// OneDevDownloader implements a Downloader interface to get repository informations
// from OneDev
type OneDevDownloader struct {
base.NullDownloader
ctx context.Context
client *http.Client
baseURL *url.URL
repoName string
repoID int64
maxIssueIndex int64
userMap map[int64]*onedevUser
milestoneMap map[int64]string
}
// SetContext set context
func (d *OneDevDownloader) SetContext(ctx context.Context) {
d.ctx = ctx
}
// NewOneDevDownloader creates a new downloader
func NewOneDevDownloader(ctx context.Context, baseURL *url.URL, username, password, repoName string) *OneDevDownloader {
var downloader = &OneDevDownloader{
ctx: ctx,
baseURL: baseURL,
repoName: repoName,
client: &http.Client{
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
if len(username) > 0 && len(password) > 0 {
req.SetBasicAuth(username, password)
}
return nil, nil
},
},
},
userMap: make(map[int64]*onedevUser),
milestoneMap: make(map[int64]string),
}
return downloader
}
func (d *OneDevDownloader) callAPI(endpoint string, parameter map[string]string, result interface{}) error {
u, err := d.baseURL.Parse(endpoint)
if err != nil {
return err
}
if parameter != nil {
query := u.Query()
for k, v := range parameter {
query.Set(k, v)
}
u.RawQuery = query.Encode()
}
req, err := http.NewRequestWithContext(d.ctx, "GET", u.String(), nil)
if err != nil {
return err
}
resp, err := d.client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
decoder := json.NewDecoder(resp.Body)
return decoder.Decode(&result)
}
// GetRepoInfo returns repository information
func (d *OneDevDownloader) GetRepoInfo() (*base.Repository, error) {
info := make([]struct {
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
}, 0, 1)
err := d.callAPI(
"/api/projects",
map[string]string{
"query": `"Name" is "` + d.repoName + `"`,
"offset": "0",
"count": "1",
},
&info,
)
if err != nil {
return nil, err
}
if len(info) != 1 {
return nil, fmt.Errorf("Project %s not found", d.repoName)
}
d.repoID = info[0].ID
cloneURL, err := d.baseURL.Parse(info[0].Name)
if err != nil {
return nil, err
}
originalURL, err := d.baseURL.Parse("/projects/" + info[0].Name)
if err != nil {
return nil, err
}
return &base.Repository{
Name: info[0].Name,
Description: info[0].Description,
CloneURL: cloneURL.String(),
OriginalURL: originalURL.String(),
}, nil
}
// GetMilestones returns milestones
func (d *OneDevDownloader) GetMilestones() ([]*base.Milestone, error) {
rawMilestones := make([]struct {
ID int64 `json:"id"`
Name string `json:"name"`
Description string `json:"description"`
DueDate *time.Time `json:"dueDate"`
Closed bool `json:"closed"`
}, 0, 100)
endpoint := fmt.Sprintf("/api/projects/%d/milestones", d.repoID)
var milestones = make([]*base.Milestone, 0, 100)
offset := 0
for {
err := d.callAPI(
endpoint,
map[string]string{
"offset": strconv.Itoa(offset),
"count": "100",
},
&rawMilestones,
)
if err != nil {
return nil, err
}
if len(rawMilestones) == 0 {
break
}
offset += 100
for _, milestone := range rawMilestones {
d.milestoneMap[milestone.ID] = milestone.Name
closed := milestone.DueDate
if !milestone.Closed {
closed = nil
}
milestones = append(milestones, &base.Milestone{
Title: milestone.Name,
Description: milestone.Description,
Deadline: milestone.DueDate,
Closed: closed,
})
}
}
return milestones, nil
}
// GetLabels returns labels
func (d *OneDevDownloader) GetLabels() ([]*base.Label, error) {
return []*base.Label{
{
Name: "Bug",
Color: "f64e60",
},
{
Name: "Build Failure",
Color: "f64e60",
},
{
Name: "Discussion",
Color: "8950fc",
},
{
Name: "Improvement",
Color: "1bc5bd",
},
{
Name: "New Feature",
Color: "1bc5bd",
},
{
Name: "Support Request",
Color: "8950fc",
},
}, nil
}
type onedevIssueContext struct {
foreignID int64
localID int64
IsPullRequest bool
}
func (c onedevIssueContext) LocalID() int64 {
return c.localID
}
func (c onedevIssueContext) ForeignID() int64 {
return c.foreignID
}
// GetIssues returns issues
func (d *OneDevDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
rawIssues := make([]struct {
ID int64 `json:"id"`
Number int64 `json:"number"`
State string `json:"state"`
Title string `json:"title"`
Description string `json:"description"`
MilestoneID int64 `json:"milestoneId"`
SubmitterID int64 `json:"submitterId"`
SubmitDate time.Time `json:"submitDate"`
}, 0, perPage)
err := d.callAPI(
"/api/issues",
map[string]string{
"query": `"Project" is "` + d.repoName + `"`,
"offset": strconv.Itoa((page - 1) * perPage),
"count": strconv.Itoa(perPage),
},
&rawIssues,
)
if err != nil {
return nil, false, err
}
issues := make([]*base.Issue, 0, len(rawIssues))
for _, issue := range rawIssues {
fields := make([]struct {
Name string `json:"name"`
Value string `json:"value"`
}, 0, 10)
err := d.callAPI(
fmt.Sprintf("/api/issues/%d/fields", issue.ID),
nil,
&fields,
)
if err != nil {
return nil, false, err
}
var label *base.Label
for _, field := range fields {
if field.Name == "Type" {
label = &base.Label{Name: field.Value}
break
}
}
state := strings.ToLower(issue.State)
if state == "released" {
state = "closed"
}
poster := d.tryGetUser(issue.SubmitterID)
issues = append(issues, &base.Issue{
Title: issue.Title,
Number: issue.Number,
PosterName: poster.Name,
PosterEmail: poster.Email,
Content: issue.Description,
Milestone: d.milestoneMap[issue.MilestoneID],
State: state,
Created: issue.SubmitDate,
Updated: issue.SubmitDate,
Labels: []*base.Label{label},
Context: onedevIssueContext{
foreignID: issue.ID,
localID: issue.Number,
IsPullRequest: false,
},
})
if d.maxIssueIndex < issue.Number {
d.maxIssueIndex = issue.Number
}
}
return issues, len(issues) == 0, nil
}
// GetComments returns comments
func (d *OneDevDownloader) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
context, ok := opts.Context.(onedevIssueContext)
if !ok {
return nil, false, fmt.Errorf("unexpected comment context: %+v", opts.Context)
}
rawComments := make([]struct {
Date time.Time `json:"date"`
UserID int64 `json:"userId"`
Content string `json:"content"`
}, 0, 100)
var endpoint string
if context.IsPullRequest {
endpoint = fmt.Sprintf("/api/pull-requests/%d/comments", context.ForeignID())
} else {
endpoint = fmt.Sprintf("/api/issues/%d/comments", context.ForeignID())
}
err := d.callAPI(
endpoint,
nil,
&rawComments,
)
if err != nil {
return nil, false, err
}
rawChanges := make([]struct {
Date time.Time `json:"date"`
UserID int64 `json:"userId"`
Data map[string]interface{} `json:"data"`
}, 0, 100)
if context.IsPullRequest {
endpoint = fmt.Sprintf("/api/pull-requests/%d/changes", context.ForeignID())
} else {
endpoint = fmt.Sprintf("/api/issues/%d/changes", context.ForeignID())
}
err = d.callAPI(
endpoint,
nil,
&rawChanges,
)
if err != nil {
return nil, false, err
}
comments := make([]*base.Comment, 0, len(rawComments)+len(rawChanges))
for _, comment := range rawComments {
if len(comment.Content) == 0 {
continue
}
poster := d.tryGetUser(comment.UserID)
comments = append(comments, &base.Comment{
IssueIndex: context.LocalID(),
PosterID: poster.ID,
PosterName: poster.Name,
PosterEmail: poster.Email,
Content: comment.Content,
Created: comment.Date,
Updated: comment.Date,
})
}
for _, change := range rawChanges {
contentV, ok := change.Data["content"]
if !ok {
contentV, ok = change.Data["comment"]
if !ok {
continue
}
}
content, ok := contentV.(string)
if !ok || len(content) == 0 {
continue
}
poster := d.tryGetUser(change.UserID)
comments = append(comments, &base.Comment{
IssueIndex: context.LocalID(),
PosterID: poster.ID,
PosterName: poster.Name,
PosterEmail: poster.Email,
Content: content,
Created: change.Date,
Updated: change.Date,
})
}
return comments, true, nil
}
// GetPullRequests returns pull requests
func (d *OneDevDownloader) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
rawPullRequests := make([]struct {
ID int64 `json:"id"`
Number int64 `json:"number"`
Title string `json:"title"`
SubmitterID int64 `json:"submitterId"`
SubmitDate time.Time `json:"submitDate"`
Description string `json:"description"`
TargetBranch string `json:"targetBranch"`
SourceBranch string `json:"sourceBranch"`
BaseCommitHash string `json:"baseCommitHash"`
CloseInfo *struct {
Date *time.Time `json:"date"`
Status string `json:"status"`
}
}, 0, perPage)
err := d.callAPI(
"/api/pull-requests",
map[string]string{
"query": `"Target Project" is "` + d.repoName + `"`,
"offset": strconv.Itoa((page - 1) * perPage),
"count": strconv.Itoa(perPage),
},
&rawPullRequests,
)
if err != nil {
return nil, false, err
}
pullRequests := make([]*base.PullRequest, 0, len(rawPullRequests))
for _, pr := range rawPullRequests {
var mergePreview struct {
TargetHeadCommitHash string `json:"targetHeadCommitHash"`
HeadCommitHash string `json:"headCommitHash"`
MergeStrategy string `json:"mergeStrategy"`
MergeCommitHash string `json:"mergeCommitHash"`
}
err := d.callAPI(
fmt.Sprintf("/api/pull-requests/%d/merge-preview", pr.ID),
nil,
&mergePreview,
)
if err != nil {
return nil, false, err
}
state := "open"
merged := false
var closeTime *time.Time
var mergedTime *time.Time
if pr.CloseInfo != nil {
state = "closed"
closeTime = pr.CloseInfo.Date
if pr.CloseInfo.Status == "MERGED" { // "DISCARDED"
merged = true
mergedTime = pr.CloseInfo.Date
}
}
poster := d.tryGetUser(pr.SubmitterID)
number := pr.Number + d.maxIssueIndex
pullRequests = append(pullRequests, &base.PullRequest{
Title: pr.Title,
Number: number,
PosterName: poster.Name,
PosterID: poster.ID,
Content: pr.Description,
State: state,
Created: pr.SubmitDate,
Updated: pr.SubmitDate,
Closed: closeTime,
Merged: merged,
MergedTime: mergedTime,
Head: base.PullRequestBranch{
Ref: pr.SourceBranch,
SHA: mergePreview.HeadCommitHash,
RepoName: d.repoName,
},
Base: base.PullRequestBranch{
Ref: pr.TargetBranch,
SHA: mergePreview.TargetHeadCommitHash,
RepoName: d.repoName,
},
Context: onedevIssueContext{
foreignID: pr.ID,
localID: number,
IsPullRequest: true,
},
})
}
return pullRequests, len(pullRequests) == 0, nil
}
// GetReviews returns pull requests reviews
func (d *OneDevDownloader) GetReviews(context base.IssueContext) ([]*base.Review, error) {
rawReviews := make([]struct {
ID int64 `json:"id"`
UserID int64 `json:"userId"`
Result *struct {
Commit string `json:"commit"`
Approved bool `json:"approved"`
Comment string `json:"comment"`
}
}, 0, 100)
err := d.callAPI(
fmt.Sprintf("/api/pull-requests/%d/reviews", context.ForeignID()),
nil,
&rawReviews,
)
if err != nil {
return nil, err
}
var reviews = make([]*base.Review, 0, len(rawReviews))
for _, review := range rawReviews {
state := base.ReviewStatePending
content := ""
if review.Result != nil {
if len(review.Result.Comment) > 0 {
state = base.ReviewStateCommented
content = review.Result.Comment
}
if review.Result.Approved {
state = base.ReviewStateApproved
}
}
poster := d.tryGetUser(review.UserID)
reviews = append(reviews, &base.Review{
IssueIndex: context.LocalID(),
ReviewerID: poster.ID,
ReviewerName: poster.Name,
Content: content,
State: state,
})
}
return reviews, nil
}
// GetTopics return repository topics
func (d *OneDevDownloader) GetTopics() ([]string, error) {
return []string{}, nil
}
func (d *OneDevDownloader) tryGetUser(userID int64) *onedevUser {
user, ok := d.userMap[userID]
if !ok {
err := d.callAPI(
fmt.Sprintf("/api/users/%d", userID),
nil,
&user,
)
if err != nil {
user = &onedevUser{
Name: fmt.Sprintf("User %d", userID),
}
}
d.userMap[userID] = user
}
return user
}

164
services/migrations/onedev_test.go Normal file
View file

@ -0,0 +1,164 @@
// Copyright 2021 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"net/http"
"net/url"
"testing"
"time"
base "code.gitea.io/gitea/modules/migration"
"github.com/stretchr/testify/assert"
)
func TestOneDevDownloadRepo(t *testing.T) {
resp, err := http.Get("https://code.onedev.io/projects/go-gitea-test_repo")
if err != nil || resp.StatusCode != 200 {
t.Skipf("Can't access test repo, skipping %s", t.Name())
}
u, _ := url.Parse("https://code.onedev.io")
downloader := NewOneDevDownloader(context.Background(), u, "", "", "go-gitea-test_repo")
if err != nil {
t.Fatal(fmt.Sprintf("NewOneDevDownloader is nil: %v", err))
}
repo, err := downloader.GetRepoInfo()
assert.NoError(t, err)
assertRepositoryEqual(t, &base.Repository{
Name: "go-gitea-test_repo",
Owner: "",
Description: "Test repository for testing migration from OneDev to gitea",
CloneURL: "https://code.onedev.io/go-gitea-test_repo",
OriginalURL: "https://code.onedev.io/projects/go-gitea-test_repo",
}, repo)
milestones, err := downloader.GetMilestones()
assert.NoError(t, err)
deadline := time.Unix(1620086400, 0)
assertMilestonesEqual(t, []*base.Milestone{
{
Title: "1.0.0",
Deadline: &deadline,
Closed: &deadline,
},
{
Title: "1.1.0",
Description: "next things?",
},
}, milestones)
labels, err := downloader.GetLabels()
assert.NoError(t, err)
assert.Len(t, labels, 6)
issues, isEnd, err := downloader.GetIssues(1, 2)
assert.NoError(t, err)
assert.False(t, isEnd)
assertIssuesEqual(t, []*base.Issue{
{
Number: 4,
Title: "Hi there",
Content: "an issue not assigned to a milestone",
PosterName: "User 336",
State: "open",
Created: time.Unix(1628549776, 734000000),
Updated: time.Unix(1628549776, 734000000),
Labels: []*base.Label{
{
Name: "Improvement",
},
},
Context: onedevIssueContext{
foreignID: 398,
localID: 4,
IsPullRequest: false,
},
},
{
Number: 3,
Title: "Add an awesome feature",
Content: "just another issue to test against",
PosterName: "User 336",
State: "open",
Milestone: "1.1.0",
Created: time.Unix(1628549749, 878000000),
Updated: time.Unix(1628549749, 878000000),
Labels: []*base.Label{
{
Name: "New Feature",
},
},
Context: onedevIssueContext{
foreignID: 397,
localID: 3,
IsPullRequest: false,
},
},
}, issues)
comments, _, err := downloader.GetComments(base.GetCommentOptions{
Context: onedevIssueContext{
foreignID: 398,
localID: 4,
IsPullRequest: false,
},
})
assert.NoError(t, err)
assertCommentsEqual(t, []*base.Comment{
{
IssueIndex: 4,
PosterName: "User 336",
Created: time.Unix(1628549791, 128000000),
Updated: time.Unix(1628549791, 128000000),
Content: "it has a comment\n\nEDIT: that got edited",
},
}, comments)
prs, _, err := downloader.GetPullRequests(1, 1)
assert.NoError(t, err)
assertPullRequestsEqual(t, []*base.PullRequest{
{
Number: 5,
Title: "Pull to add a new file",
Content: "just do some git stuff",
PosterName: "User 336",
State: "open",
Created: time.Unix(1628550076, 25000000),
Updated: time.Unix(1628550076, 25000000),
Head: base.PullRequestBranch{
Ref: "branch-for-a-pull",
SHA: "343deffe3526b9bc84e873743ff7f6e6d8b827c0",
RepoName: "go-gitea-test_repo",
},
Base: base.PullRequestBranch{
Ref: "master",
SHA: "f32b0a9dfd09a60f616f29158f772cedd89942d2",
RepoName: "go-gitea-test_repo",
},
Context: onedevIssueContext{
foreignID: 186,
localID: 5,
IsPullRequest: true,
},
},
}, prs)
rvs, err := downloader.GetReviews(onedevIssueContext{
foreignID: 186,
localID: 5,
})
assert.NoError(t, err)
assertReviewsEqual(t, []*base.Review{
{
IssueIndex: 5,
ReviewerName: "User 317",
State: "PENDING",
},
}, rvs)
}

290
services/migrations/restore.go Normal file
View file

@ -0,0 +1,290 @@
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"fmt"
"os"
"path/filepath"
"strconv"
base "code.gitea.io/gitea/modules/migration"
"gopkg.in/yaml.v2"
)
// RepositoryRestorer implements an Downloader from the local directory
type RepositoryRestorer struct {
base.NullDownloader
ctx context.Context
baseDir string
repoOwner string
repoName string
}
// NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder
func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) (*RepositoryRestorer, error) {
baseDir, err := filepath.Abs(baseDir)
if err != nil {
return nil, err
}
return &RepositoryRestorer{
ctx: ctx,
baseDir: baseDir,
repoOwner: owner,
repoName: repoName,
}, nil
}
func (r *RepositoryRestorer) commentDir() string {
return filepath.Join(r.baseDir, "comments")
}
func (r *RepositoryRestorer) reviewDir() string {
return filepath.Join(r.baseDir, "reviews")
}
// SetContext set context
func (r *RepositoryRestorer) SetContext(ctx context.Context) {
r.ctx = ctx
}
func (r *RepositoryRestorer) getRepoOptions() (map[string]string, error) {
p := filepath.Join(r.baseDir, "repo.yml")
bs, err := os.ReadFile(p)
if err != nil {
return nil, err
}
var opts = make(map[string]string)
err = yaml.Unmarshal(bs, &opts)
if err != nil {
return nil, err
}
return opts, nil
}
// GetRepoInfo returns a repository information
func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) {
opts, err := r.getRepoOptions()
if err != nil {
return nil, err
}
isPrivate, _ := strconv.ParseBool(opts["is_private"])
return &base.Repository{
Owner: r.repoOwner,
Name: r.repoName,
IsPrivate: isPrivate,
Description: opts["description"],
OriginalURL: opts["original_url"],
CloneURL: filepath.Join(r.baseDir, "git"),
DefaultBranch: opts["default_branch"],
}, nil
}
// GetTopics return github topics
func (r *RepositoryRestorer) GetTopics() ([]string, error) {
p := filepath.Join(r.baseDir, "topic.yml")
var topics = struct {
Topics []string `yaml:"topics"`
}{}
bs, err := os.ReadFile(p)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(bs, &topics)
if err != nil {
return nil, err
}
return topics.Topics, nil
}
// GetMilestones returns milestones
func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) {
var milestones = make([]*base.Milestone, 0, 10)
p := filepath.Join(r.baseDir, "milestone.yml")
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
bs, err := os.ReadFile(p)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(bs, &milestones)
if err != nil {
return nil, err
}
return milestones, nil
}
// GetReleases returns releases
func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) {
var releases = make([]*base.Release, 0, 10)
p := filepath.Join(r.baseDir, "release.yml")
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
bs, err := os.ReadFile(p)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(bs, &releases)
if err != nil {
return nil, err
}
for _, rel := range releases {
for _, asset := range rel.Assets {
if asset.DownloadURL != nil {
*asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL)
}
}
}
return releases, nil
}
// GetLabels returns labels
func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) {
var labels = make([]*base.Label, 0, 10)
p := filepath.Join(r.baseDir, "label.yml")
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
bs, err := os.ReadFile(p)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(bs, &labels)
if err != nil {
return nil, err
}
return labels, nil
}
// GetIssues returns issues according start and limit
func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
var issues = make([]*base.Issue, 0, 10)
p := filepath.Join(r.baseDir, "issue.yml")
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, true, nil
}
return nil, false, err
}
bs, err := os.ReadFile(p)
if err != nil {
return nil, false, err
}
err = yaml.Unmarshal(bs, &issues)
if err != nil {
return nil, false, err
}
for _, issue := range issues {
issue.Context = base.BasicIssueContext(issue.Number)
}
return issues, true, nil
}
// GetComments returns comments according issueNumber
func (r *RepositoryRestorer) GetComments(opts base.GetCommentOptions) ([]*base.Comment, bool, error) {
var comments = make([]*base.Comment, 0, 10)
p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", opts.Context.ForeignID()))
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, false, nil
}
return nil, false, err
}
bs, err := os.ReadFile(p)
if err != nil {
return nil, false, err
}
err = yaml.Unmarshal(bs, &comments)
if err != nil {
return nil, false, err
}
return comments, false, nil
}
// GetPullRequests returns pull requests according page and perPage
func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
var pulls = make([]*base.PullRequest, 0, 10)
p := filepath.Join(r.baseDir, "pull_request.yml")
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, true, nil
}
return nil, false, err
}
bs, err := os.ReadFile(p)
if err != nil {
return nil, false, err
}
err = yaml.Unmarshal(bs, &pulls)
if err != nil {
return nil, false, err
}
for _, pr := range pulls {
pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL)
pr.Context = base.BasicIssueContext(pr.Number)
}
return pulls, true, nil
}
// GetReviews returns pull requests review
func (r *RepositoryRestorer) GetReviews(context base.IssueContext) ([]*base.Review, error) {
var reviews = make([]*base.Review, 0, 10)
p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", context.ForeignID()))
_, err := os.Stat(p)
if err != nil {
if os.IsNotExist(err) {
return nil, nil
}
return nil, err
}
bs, err := os.ReadFile(p)
if err != nil {
return nil, err
}
err = yaml.Unmarshal(bs, &reviews)
if err != nil {
return nil, err
}
return reviews, nil
}

76
services/migrations/update.go Normal file
View file

@ -0,0 +1,76 @@
// Copyright 2019 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package migrations
import (
"context"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/db"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/structs"
)
// UpdateMigrationPosterID updates all migrated repositories' issues and comments posterID
func UpdateMigrationPosterID(ctx context.Context) error {
for _, gitService := range structs.SupportedFullGitService {
select {
case <-ctx.Done():
log.Warn("UpdateMigrationPosterID aborted before %s", gitService.Name())
return db.ErrCancelledf("during UpdateMigrationPosterID before %s", gitService.Name())
default:
}
if err := updateMigrationPosterIDByGitService(ctx, gitService); err != nil {
log.Error("updateMigrationPosterIDByGitService failed: %v", err)
}
}
return nil
}
func updateMigrationPosterIDByGitService(ctx context.Context, tp structs.GitServiceType) error {
provider := tp.Name()
if len(provider) == 0 {
return nil
}
const batchSize = 100
var start int
for {
select {
case <-ctx.Done():
log.Warn("UpdateMigrationPosterIDByGitService(%s) cancelled", tp.Name())
return nil
default:
}
users, err := models.FindExternalUsersByProvider(models.FindExternalUserOptions{
Provider: provider,
Start: start,
Limit: batchSize,
})
if err != nil {
return err
}
for _, user := range users {
select {
case <-ctx.Done():
log.Warn("UpdateMigrationPosterIDByGitService(%s) cancelled", tp.Name())
return nil
default:
}
externalUserID := user.ExternalID
if err := models.UpdateMigrationsByType(tp, externalUserID, user.UserID); err != nil {
log.Error("UpdateMigrationsByType type %s external user id %v to local user id %v failed: %v", tp.Name(), user.ExternalID, user.UserID, err)
}
}
if len(users) < batchSize {
break
}
start += len(users)
}
return nil
}