Split migrations folder (#21549)

There are too many files in `models/migrations` folder so that I split
them into sub folders.

models/migrations/v1_15/main_test.go

@@ -0,0 +1,15 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/migrations/base"
+)
+
+func TestMain(m *testing.M) {
+	base.MainTest(m)
+}

models/migrations/v1_15/v178.go

@@ -0,0 +1,18 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"xorm.io/xorm"
+)
+
+func AddLFSMirrorColumns(x *xorm.Engine) error {
+	type Mirror struct {
+		LFS         bool   `xorm:"lfs_enabled NOT NULL DEFAULT false"`
+		LFSEndpoint string `xorm:"lfs_endpoint TEXT"`
+	}
+
+	return x.Sync2(new(Mirror))
+}

models/migrations/v1_15/v179.go

@@ -0,0 +1,29 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"code.gitea.io/gitea/models/migrations/base"
+
+	"xorm.io/xorm"
+	"xorm.io/xorm/schemas"
+)
+
+func ConvertAvatarURLToText(x *xorm.Engine) error {
+	dbType := x.Dialect().URI().DBType
+	if dbType == schemas.SQLITE { // For SQLITE, varchar or char will always be represented as TEXT
+		return nil
+	}
+
+	// Some oauth2 providers may give very long avatar urls (i.e. Google)
+	return base.ModifyColumn(x, "external_login_user", &schemas.Column{
+		Name: "avatar_url",
+		SQLType: schemas.SQLType{
+			Name: schemas.Text,
+		},
+		Nullable:       true,
+		DefaultIsEmpty: true,
+	})
+}

models/migrations/v1_15/v180.go

@@ -0,0 +1,122 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"code.gitea.io/gitea/modules/json"
+	"code.gitea.io/gitea/modules/util"
+
+	"xorm.io/builder"
+	"xorm.io/xorm"
+)
+
+func DeleteMigrationCredentials(x *xorm.Engine) (err error) {
+	// Task represents a task
+	type Task struct {
+		ID             int64
+		DoerID         int64 `xorm:"index"` // operator
+		OwnerID        int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero
+		RepoID         int64 `xorm:"index"`
+		Type           int
+		Status         int `xorm:"index"`
+		StartTime      int64
+		EndTime        int64
+		PayloadContent string `xorm:"TEXT"`
+		Errors         string `xorm:"TEXT"` // if task failed, saved the error reason
+		Created        int64  `xorm:"created"`
+	}
+
+	const TaskTypeMigrateRepo = 0
+	const TaskStatusStopped = 2
+
+	const batchSize = 100
+
+	// only match migration tasks, that are not pending or running
+	cond := builder.Eq{
+		"type": TaskTypeMigrateRepo,
+	}.And(builder.Gte{
+		"status": TaskStatusStopped,
+	})
+
+	sess := x.NewSession()
+	defer sess.Close()
+
+	for start := 0; ; start += batchSize {
+		tasks := make([]*Task, 0, batchSize)
+		if err = sess.Limit(batchSize, start).Where(cond, 0).Find(&tasks); err != nil {
+			return
+		}
+		if len(tasks) == 0 {
+			break
+		}
+		if err = sess.Begin(); err != nil {
+			return
+		}
+		for _, t := range tasks {
+			if t.PayloadContent, err = removeCredentials(t.PayloadContent); err != nil {
+				return
+			}
+			if _, err = sess.ID(t.ID).Cols("payload_content").Update(t); err != nil {
+				return
+			}
+		}
+		if err = sess.Commit(); err != nil {
+			return
+		}
+	}
+	return err
+}
+
+func removeCredentials(payload string) (string, error) {
+	// MigrateOptions defines the way a repository gets migrated
+	// this is for internal usage by migrations module and func who interact with it
+	type MigrateOptions struct {
+		// required: true
+		CloneAddr             string `json:"clone_addr" binding:"Required"`
+		CloneAddrEncrypted    string `json:"clone_addr_encrypted,omitempty"`
+		AuthUsername          string `json:"auth_username"`
+		AuthPassword          string `json:"-"`
+		AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"`
+		AuthToken             string `json:"-"`
+		AuthTokenEncrypted    string `json:"auth_token_encrypted,omitempty"`
+		// required: true
+		UID int `json:"uid" binding:"Required"`
+		// required: true
+		RepoName        string `json:"repo_name" binding:"Required"`
+		Mirror          bool   `json:"mirror"`
+		LFS             bool   `json:"lfs"`
+		LFSEndpoint     string `json:"lfs_endpoint"`
+		Private         bool   `json:"private"`
+		Description     string `json:"description"`
+		OriginalURL     string
+		GitServiceType  int
+		Wiki            bool
+		Issues          bool
+		Milestones      bool
+		Labels          bool
+		Releases        bool
+		Comments        bool
+		PullRequests    bool
+		ReleaseAssets   bool
+		MigrateToRepoID int64
+		MirrorInterval  string `json:"mirror_interval"`
+	}
+
+	var opts MigrateOptions
+	err := json.Unmarshal([]byte(payload), &opts)
+	if err != nil {
+		return "", err
+	}
+
+	opts.AuthPassword = ""
+	opts.AuthToken = ""
+	opts.CloneAddr = util.SanitizeCredentialURLs(opts.CloneAddr)
+
+	confBytes, err := json.Marshal(opts)
+	if err != nil {
+		return "", err
+	}
+	return string(confBytes), nil
+}

models/migrations/v1_15/v181.go

@@ -0,0 +1,93 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"strings"
+
+	"xorm.io/xorm"
+)
+
+func AddPrimaryEmail2EmailAddress(x *xorm.Engine) (err error) {
+	type User struct {
+		ID       int64  `xorm:"pk autoincr"`
+		Email    string `xorm:"NOT NULL"`
+		IsActive bool   `xorm:"INDEX"` // Activate primary email
+	}
+
+	type EmailAddress1 struct {
+		ID          int64  `xorm:"pk autoincr"`
+		UID         int64  `xorm:"INDEX NOT NULL"`
+		Email       string `xorm:"UNIQUE NOT NULL"`
+		LowerEmail  string
+		IsActivated bool
+		IsPrimary   bool `xorm:"DEFAULT(false) NOT NULL"`
+	}
+
+	// Add lower_email and is_primary columns
+	if err = x.Table("email_address").Sync2(new(EmailAddress1)); err != nil {
+		return
+	}
+
+	if _, err = x.Exec("UPDATE email_address SET lower_email=LOWER(email), is_primary=?", false); err != nil {
+		return
+	}
+
+	type EmailAddress struct {
+		ID          int64  `xorm:"pk autoincr"`
+		UID         int64  `xorm:"INDEX NOT NULL"`
+		Email       string `xorm:"UNIQUE NOT NULL"`
+		LowerEmail  string `xorm:"UNIQUE NOT NULL"`
+		IsActivated bool
+		IsPrimary   bool `xorm:"DEFAULT(false) NOT NULL"`
+	}
+
+	// change lower_email as unique
+	if err = x.Sync2(new(EmailAddress)); err != nil {
+		return
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+
+	const batchSize = 100
+
+	for start := 0; ; start += batchSize {
+		users := make([]*User, 0, batchSize)
+		if err = sess.Limit(batchSize, start).Find(&users); err != nil {
+			return
+		}
+		if len(users) == 0 {
+			break
+		}
+
+		for _, user := range users {
+			var exist bool
+			exist, err = sess.Where("email=?", user.Email).Table("email_address").Exist()
+			if err != nil {
+				return
+			}
+			if !exist {
+				if _, err = sess.Insert(&EmailAddress{
+					UID:         user.ID,
+					Email:       user.Email,
+					LowerEmail:  strings.ToLower(user.Email),
+					IsActivated: user.IsActive,
+					IsPrimary:   true,
+				}); err != nil {
+					return
+				}
+			} else {
+				if _, err = sess.Where("email=?", user.Email).Cols("is_primary").Update(&EmailAddress{
+					IsPrimary: true,
+				}); err != nil {
+					return
+				}
+			}
+		}
+	}
+
+	return nil
+}

models/migrations/v1_15/v181_test.go

@@ -0,0 +1,56 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"strings"
+	"testing"
+
+	"code.gitea.io/gitea/models/migrations/base"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_AddPrimaryEmail2EmailAddress(t *testing.T) {
+	type User struct {
+		ID       int64
+		Email    string
+		IsActive bool
+	}
+
+	// Prepare and load the testing database
+	x, deferable := base.PrepareTestEnv(t, 0, new(User))
+	if x == nil || t.Failed() {
+		defer deferable()
+		return
+	}
+	defer deferable()
+
+	err := AddPrimaryEmail2EmailAddress(x)
+	assert.NoError(t, err)
+
+	type EmailAddress struct {
+		ID          int64  `xorm:"pk autoincr"`
+		UID         int64  `xorm:"INDEX NOT NULL"`
+		Email       string `xorm:"UNIQUE NOT NULL"`
+		LowerEmail  string `xorm:"UNIQUE NOT NULL"`
+		IsActivated bool
+		IsPrimary   bool `xorm:"DEFAULT(false) NOT NULL"`
+	}
+
+	users := make([]User, 0, 20)
+	err = x.Find(&users)
+	assert.NoError(t, err)
+
+	for _, user := range users {
+		var emailAddress EmailAddress
+		has, err := x.Where("lower_email=?", strings.ToLower(user.Email)).Get(&emailAddress)
+		assert.NoError(t, err)
+		assert.True(t, has)
+		assert.True(t, emailAddress.IsPrimary)
+		assert.EqualValues(t, user.IsActive, emailAddress.IsActivated)
+		assert.EqualValues(t, user.ID, emailAddress.UID)
+	}
+}

models/migrations/v1_15/v182.go

@@ -0,0 +1,42 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"xorm.io/xorm"
+)
+
+func AddIssueResourceIndexTable(x *xorm.Engine) error {
+	type ResourceIndex struct {
+		GroupID  int64 `xorm:"pk"`
+		MaxIndex int64 `xorm:"index"`
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Table("issue_index").Sync2(new(ResourceIndex)); err != nil {
+		return err
+	}
+
+	// Remove data we're goint to rebuild
+	if _, err := sess.Table("issue_index").Where("1=1").Delete(&ResourceIndex{}); err != nil {
+		return err
+	}
+
+	// Create current data for all repositories with issues and PRs
+	if _, err := sess.Exec("INSERT INTO issue_index (group_id, max_index) " +
+		"SELECT max_data.repo_id, max_data.max_index " +
+		"FROM ( SELECT issue.repo_id AS repo_id, max(issue.`index`) AS max_index " +
+		"FROM issue GROUP BY issue.repo_id) AS max_data"); err != nil {
+		return err
+	}
+
+	return sess.Commit()
+}

models/migrations/v1_15/v182_test.go

@@ -0,0 +1,61 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"testing"
+
+	"code.gitea.io/gitea/models/migrations/base"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_AddIssueResourceIndexTable(t *testing.T) {
+	// Create the models used in the migration
+	type Issue struct {
+		ID     int64 `xorm:"pk autoincr"`
+		RepoID int64 `xorm:"UNIQUE(s)"`
+		Index  int64 `xorm:"UNIQUE(s)"`
+	}
+
+	// Prepare and load the testing database
+	x, deferable := base.PrepareTestEnv(t, 0, new(Issue))
+	if x == nil || t.Failed() {
+		defer deferable()
+		return
+	}
+	defer deferable()
+
+	// Run the migration
+	if err := AddIssueResourceIndexTable(x); err != nil {
+		assert.NoError(t, err)
+		return
+	}
+
+	type ResourceIndex struct {
+		GroupID  int64 `xorm:"pk"`
+		MaxIndex int64 `xorm:"index"`
+	}
+
+	start := 0
+	const batchSize = 1000
+	for {
+		indexes := make([]ResourceIndex, 0, batchSize)
+		err := x.Table("issue_index").Limit(batchSize, start).Find(&indexes)
+		assert.NoError(t, err)
+
+		for _, idx := range indexes {
+			var maxIndex int
+			has, err := x.SQL("SELECT max(`index`) FROM issue WHERE repo_id = ?", idx.GroupID).Get(&maxIndex)
+			assert.NoError(t, err)
+			assert.True(t, has)
+			assert.EqualValues(t, maxIndex, idx.MaxIndex)
+		}
+		if len(indexes) < batchSize {
+			break
+		}
+		start += len(indexes)
+	}
+}

models/migrations/v1_15/v183.go

@@ -0,0 +1,39 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"fmt"
+	"time"
+
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func CreatePushMirrorTable(x *xorm.Engine) error {
+	type PushMirror struct {
+		ID         int64 `xorm:"pk autoincr"`
+		RepoID     int64 `xorm:"INDEX"`
+		RemoteName string
+
+		Interval       time.Duration
+		CreatedUnix    timeutil.TimeStamp `xorm:"created"`
+		LastUpdateUnix timeutil.TimeStamp `xorm:"INDEX last_update"`
+		LastError      string             `xorm:"text"`
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Sync2(new(PushMirror)); err != nil {
+		return fmt.Errorf("Sync2: %w", err)
+	}
+
+	return sess.Commit()
+}

models/migrations/v1_15/v184.go

@@ -0,0 +1,72 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"context"
+	"fmt"
+
+	"code.gitea.io/gitea/models/migrations/base"
+	"code.gitea.io/gitea/modules/setting"
+
+	"xorm.io/xorm"
+)
+
+func RenameTaskErrorsToMessage(x *xorm.Engine) error {
+	type Task struct {
+		Errors string `xorm:"TEXT"` // if task failed, saved the error reason
+		Type   int
+		Status int `xorm:"index"`
+	}
+
+	// This migration maybe rerun so that we should check if it has been run
+	messageExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "task", "message")
+	if err != nil {
+		return err
+	}
+
+	if messageExist {
+		errorsExist, err := x.Dialect().IsColumnExist(x.DB(), context.Background(), "task", "errors")
+		if err != nil {
+			return err
+		}
+		if !errorsExist {
+			return nil
+		}
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Sync2(new(Task)); err != nil {
+		return fmt.Errorf("error on Sync2: %w", err)
+	}
+
+	if messageExist {
+		// if both errors and message exist, drop message at first
+		if err := base.DropTableColumns(sess, "task", "message"); err != nil {
+			return err
+		}
+	}
+
+	switch {
+	case setting.Database.UseMySQL:
+		if _, err := sess.Exec("ALTER TABLE `task` CHANGE errors message text"); err != nil {
+			return err
+		}
+	case setting.Database.UseMSSQL:
+		if _, err := sess.Exec("sp_rename 'task.errors', 'message', 'COLUMN'"); err != nil {
+			return err
+		}
+	default:
+		if _, err := sess.Exec("ALTER TABLE `task` RENAME COLUMN errors TO message"); err != nil {
+			return err
+		}
+	}
+	return sess.Commit()
+}

models/migrations/v1_15/v185.go

@@ -0,0 +1,22 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"xorm.io/xorm"
+)
+
+func AddRepoArchiver(x *xorm.Engine) error {
+	// RepoArchiver represents all archivers
+	type RepoArchiver struct {
+		ID          int64 `xorm:"pk autoincr"`
+		RepoID      int64 `xorm:"index unique(s)"`
+		Type        int   `xorm:"unique(s)"`
+		Status      int
+		CommitID    string `xorm:"VARCHAR(40) unique(s)"`
+		CreatedUnix int64  `xorm:"INDEX NOT NULL created"`
+	}
+	return x.Sync2(new(RepoArchiver))
+}

models/migrations/v1_15/v186.go

@@ -0,0 +1,26 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"code.gitea.io/gitea/modules/timeutil"
+
+	"xorm.io/xorm"
+)
+
+func CreateProtectedTagTable(x *xorm.Engine) error {
+	type ProtectedTag struct {
+		ID               int64 `xorm:"pk autoincr"`
+		RepoID           int64
+		NamePattern      string
+		AllowlistUserIDs []int64 `xorm:"JSON TEXT"`
+		AllowlistTeamIDs []int64 `xorm:"JSON TEXT"`
+
+		CreatedUnix timeutil.TimeStamp `xorm:"created"`
+		UpdatedUnix timeutil.TimeStamp `xorm:"updated"`
+	}
+
+	return x.Sync2(new(ProtectedTag))
+}

models/migrations/v1_15/v187.go

@@ -0,0 +1,48 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import (
+	"code.gitea.io/gitea/models/migrations/base"
+
+	"xorm.io/xorm"
+)
+
+func DropWebhookColumns(x *xorm.Engine) error {
+	// Make sure the columns exist before dropping them
+	type Webhook struct {
+		Signature string `xorm:"TEXT"`
+		IsSSL     bool   `xorm:"is_ssl"`
+	}
+	if err := x.Sync2(new(Webhook)); err != nil {
+		return err
+	}
+
+	type HookTask struct {
+		Typ         string `xorm:"VARCHAR(16) index"`
+		URL         string `xorm:"TEXT"`
+		Signature   string `xorm:"TEXT"`
+		HTTPMethod  string `xorm:"http_method"`
+		ContentType int
+		IsSSL       bool
+	}
+	if err := x.Sync2(new(HookTask)); err != nil {
+		return err
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+	if err := base.DropTableColumns(sess, "webhook", "signature", "is_ssl"); err != nil {
+		return err
+	}
+	if err := base.DropTableColumns(sess, "hook_task", "typ", "url", "signature", "http_method", "content_type", "is_ssl"); err != nil {
+		return err
+	}
+
+	return sess.Commit()
+}

models/migrations/v1_15/v188.go

@@ -0,0 +1,15 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package v1_15 //nolint
+
+import "xorm.io/xorm"
+
+func AddKeyIsVerified(x *xorm.Engine) error {
+	type GPGKey struct {
+		Verified bool `xorm:"NOT NULL DEFAULT false"`
+	}
+
+	return x.Sync(new(GPGKey))
+}