domi/forgejo-domi
1
0
Fork 0
forked from forgejo/forgejo
Code Pull requests Activity

Dump github/gitlab/gitea repository data to a local directory and restore to gitea (#12244)

Browse source 
* Dump github/gitlab repository data to a local directory

* Fix lint

* Adjust directory structure

* Allow migration special units

* Allow migration ignore release assets

* Fix lint

* Add restore repository

* stage the changes

* Merge

* Fix lint

* Update the interface

* Add some restore methods

* Finish restore

* Add comments

* Fix restore

* Add a token flag

* Fix bug

* Fix test

* Fix test

* Fix bug

* Fix bug

* Fix lint

* Fix restore

* refactor downloader

* fmt

* Fix bug isEnd detection on getIssues

* Refactor maxPerPage

* Remove unused codes

* Remove unused codes

* Fix bug

* Fix restore

* Fix dump

* Uploader should not depend downloader

* use release attachment name but not id

* Fix restore bug

* Fix lint

* Fix restore bug

* Add a method of DownloadFunc for base.Release to make uploader not depend on downloader

* fix Release yml marshal

* Fix trace information

* Fix bug when dump & restore

* Save relative path on yml file

* Fix bug

* Use relative path

* Update docs

* Use git service string but not int

* Recognize clone addr to service type
This commit is contained in:
Lunny Xiao 2020-12-27 11:34:19 +08:00 • committed by GitHub
parent 212fa340cf
commit dd08853b10
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
29 changed files with 1491 additions and 232 deletions
Download patch file Download diff file Expand all files Collapse all files

60
modules/migrations/gitea_uploader.go
View file

@ -10,7 +10,6 @@ import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"os"
"path/filepath"
@ -28,6 +27,7 @@ import (
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/uri"
"code.gitea.io/gitea/services/pull"
gouuid "github.com/google/uuid"
@ -86,6 +86,22 @@ func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
return 10
}
func fullURL(opts base.MigrateOptions, remoteAddr string) (string, error) {
var fullRemoteAddr = remoteAddr
if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
u, err := url.Parse(remoteAddr)
if err != nil {
return "", err
}
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
if len(opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", opts.AuthToken)
}
fullRemoteAddr = u.String()
}
return fullRemoteAddr, nil
}
// CreateRepo creates a repository
func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
owner, err := models.GetUserByName(g.repoOwner)
@ -93,19 +109,10 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
return err
}
var remoteAddr = repo.CloneURL
if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
u, err := url.Parse(repo.CloneURL)
if err != nil {
return err
}
u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
if len(opts.AuthToken) > 0 {
u.User = url.UserPassword("oauth2", opts.AuthToken)
}
remoteAddr = u.String()
remoteAddr, err := fullURL(opts, repo.CloneURL)
if err != nil {
return err
}
var r *models.Repository
if opts.MigrateToRepoID <= 0 {
r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{
@ -224,7 +231,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
}
// CreateReleases creates releases
func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases ...*base.Release) error {
func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
var rels = make([]*models.Release, 0, len(releases))
for _, release := range releases {
var rel = models.Release{
@ -283,25 +290,27 @@ func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases
// download attachment
err = func() error {
// asset.DownloadURL maybe a local file
var rc io.ReadCloser
if asset.DownloadURL == nil {
rc, err = downloader.GetAsset(rel.TagName, rel.ID, asset.ID)
rc, err = asset.DownloadFunc()
if err != nil {
return err
}
} else {
resp, err := http.Get(*asset.DownloadURL)
rc, err = uri.Open(*asset.DownloadURL)
if err != nil {
return err
}
rc = resp.Body
}
defer rc.Close()
_, err = storage.Attachments.Save(attach.RelativePath(), rc)
return err
}()
if err != nil {
return err
}
rel.Attachments = append(rel.Attachments, &attach)
}
@ -559,11 +568,12 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
// download patch file
err := func() error {
resp, err := http.Get(pr.PatchURL)
// pr.PatchURL maybe a local file
ret, err := uri.Open(pr.PatchURL)
if err != nil {
return err
}
defer resp.Body.Close()
defer ret.Close()
pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
return err
@ -573,7 +583,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
return err
}
defer f.Close()
_, err = io.Copy(f, resp.Body)
_, err = io.Copy(f, ret)
return err
}()
if err != nil {
@ -859,3 +869,13 @@ func (g *GiteaLocalUploader) Rollback() error {
}
return nil
}
// Finish when migrating success, this will do some status update things.
func (g *GiteaLocalUploader) Finish() error {
if g.repo == nil || g.repo.ID <= 0 {
return ErrRepoNotCreated
}
g.repo.Status = models.RepositoryReady
return models.UpdateRepositoryCols(g.repo, "status")
}