domi/forgejo-domi
1
0
Fork 0
forked from forgejo/forgejo
Code Pull requests Activity

Add a storage layer for attachments (#11387)

Browse source 
* Add a storage layer for attachments

* Fix some bug

* fix test

* Fix copyright head and lint

* Fix bug

* Add setting for minio and flags for migrate-storage

* Add documents

* fix lint

* Add test for minio store type on attachments

* fix test

* fix test

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Add warning when storage migrated successfully

* Fix drone

* fix test

* rebase

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* remove log on xorm

* Fi download bug

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* refactor the codes

* add trace

* Fix test

* Add URL function to serve attachments directly from S3/Minio

* Add ability to enable/disable redirection in attachment configuration

* Fix typo

* Add a storage layer for attachments

* Add setting for minio and flags for migrate-storage

* fix lint

* Add test for minio store type on attachments

* Apply suggestions from code review

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>

* Fix drone

* fix test

* Fix test

* display the error on console

* Move minio test to amd64 since minio docker don't support arm64

* don't change unrelated files

* Fix lint

* Fix build

* update go.mod and go.sum

* Use github.com/minio/minio-go/v6

* Remove unused function

* Upgrade minio to v7 and some other improvements

* fix lint

* Fix go mod

Co-authored-by: guillep2k <18600385+guillep2k@users.noreply.github.com>
Co-authored-by: Tyler <tystuyfzand@gmail.com>
This commit is contained in:
Lunny Xiao 2020-08-18 12:23:45 +08:00 • committed by GitHub
parent 02fbe1e5dc
commit 62e6c9bc6c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
330 changed files with 62099 additions and 331 deletions
Download patch file Download diff file Expand all files Collapse all files

14
models/admin.go
View file

@ -1,4 +1,5 @@
// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2020 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
@ -8,6 +9,7 @@ import (
"fmt"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
@ -65,6 +67,18 @@ func RemoveAllWithNotice(title, path string) {
removeAllWithNotice(x, title, path)
}
// RemoveStorageWithNotice removes a file from the storage and
// creates a system notice when error occurs.
func RemoveStorageWithNotice(bucket storage.ObjectStorage, title, path string) {
if err := bucket.Delete(path); err != nil {
desc := fmt.Sprintf("%s [%s]: %v", title, path, err)
log.Warn(title+" [%s]: %v", path, err)
if err = createNotice(x, NoticeRepository, desc); err != nil {
log.Error("CreateRepositoryNotice: %v", err)
}
}
}
func removeAllWithNotice(e Engine, title, path string) {
if err := util.RemoveAll(path); err != nil {
desc := fmt.Sprintf("%s [%s]: %v", title, path, err)

63
models/attachment.go
View file

@ -5,15 +5,15 @@
package models
import (
"bytes"
"fmt"
"io"
"os"
"path"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
gouuid "github.com/google/uuid"
"xorm.io/xorm"
@ -56,15 +56,14 @@ func (a *Attachment) APIFormat() *api.Attachment {
}
}
// AttachmentLocalPath returns where attachment is stored in local file
// system based on given UUID.
func AttachmentLocalPath(uuid string) string {
return path.Join(setting.AttachmentPath, uuid[0:1], uuid[1:2], uuid)
// AttachmentRelativePath returns the relative path
func AttachmentRelativePath(uuid string) string {
return path.Join(uuid[0:1], uuid[1:2], uuid)
}
// LocalPath returns where attachment is stored in local file system.
func (a *Attachment) LocalPath() string {
return AttachmentLocalPath(a.UUID)
// RelativePath returns the relative path of the attachment
func (a *Attachment) RelativePath() string {
return AttachmentRelativePath(a.UUID)
}
// DownloadURL returns the download url of the attached file
@ -100,29 +99,11 @@ func (a *Attachment) LinkedRepository() (*Repository, UnitType, error) {
func NewAttachment(attach *Attachment, buf []byte, file io.Reader) (_ *Attachment, err error) {
attach.UUID = gouuid.New().String()
localPath := attach.LocalPath()
if err = os.MkdirAll(path.Dir(localPath), os.ModePerm); err != nil {
return nil, fmt.Errorf("MkdirAll: %v", err)
}
fw, err := os.Create(localPath)
size, err := storage.Attachments.Save(attach.RelativePath(), io.MultiReader(bytes.NewReader(buf), file))
if err != nil {
return nil, fmt.Errorf("Create: %v", err)
}
defer fw.Close()
if _, err = fw.Write(buf); err != nil {
return nil, fmt.Errorf("Write: %v", err)
} else if _, err = io.Copy(fw, file); err != nil {
return nil, fmt.Errorf("Copy: %v", err)
}
// Update file size
var fi os.FileInfo
if fi, err = fw.Stat(); err != nil {
return nil, fmt.Errorf("file size: %v", err)
}
attach.Size = fi.Size()
attach.Size = size
if _, err := x.Insert(attach); err != nil {
return nil, err
@ -238,7 +219,7 @@ func DeleteAttachments(attachments []*Attachment, remove bool) (int, error) {
if remove {
for i, a := range attachments {
if err := util.Remove(a.LocalPath()); err != nil {
if err := storage.Attachments.Delete(a.RelativePath()); err != nil {
return i, err
}
}
@ -290,3 +271,25 @@ func DeleteAttachmentsByRelease(releaseID int64) error {
_, err := x.Where("release_id = ?", releaseID).Delete(&Attachment{})
return err
}
// IterateAttachment iterates attachments; it should not be used when Gitea is servicing users.
func IterateAttachment(f func(attach *Attachment) error) error {
var start int
const batchSize = 100
for {
var attachments = make([]*Attachment, 0, batchSize)
if err := x.Limit(batchSize, start).Find(&attachments); err != nil {
return err
}
if len(attachments) == 0 {
return nil
}
start += len(attachments)
for _, attach := range attachments {
if err := f(attach); err != nil {
return err
}
}
}
}

3
models/issue.go
View file

@ -1983,8 +1983,9 @@ func deleteIssuesByRepoID(sess Engine, repoID int64) (attachmentPaths []string,
Find(&attachments); err != nil {
return
}
for j := range attachments {
attachmentPaths = append(attachmentPaths, attachments[j].LocalPath())
attachmentPaths = append(attachmentPaths, attachments[j].RelativePath())
}
if _, err = sess.In("issue_id", deleteCond).

4
models/migrations/v112.go
View file

@ -6,7 +6,7 @@ package migrations
import (
"fmt"
"path"
"path/filepath"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@ -31,7 +31,7 @@ func removeAttachmentMissedRepo(x *xorm.Engine) error {
for i := 0; i < len(attachments); i++ {
uuid := attachments[i].UUID
if err = util.RemoveAll(path.Join(setting.AttachmentPath, uuid[0:1], uuid[1:2], uuid)); err != nil {
if err = util.RemoveAll(filepath.Join(setting.Attachment.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
fmt.Printf("Error: %v", err)
}
}

18
models/migrations/v96.go
View file

@ -5,7 +5,7 @@
package migrations
import (
"path"
"path/filepath"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
@ -14,7 +14,6 @@ import (
)
func deleteOrphanedAttachments(x *xorm.Engine) error {
type Attachment struct {
ID int64 `xorm:"pk autoincr"`
UUID string `xorm:"uuid UNIQUE"`
@ -23,12 +22,6 @@ func deleteOrphanedAttachments(x *xorm.Engine) error {
CommentID int64
}
// AttachmentLocalPath returns where attachment is stored in local file
// system based on given UUID.
AttachmentLocalPath := func(uuid string) string {
return path.Join(setting.AttachmentPath, uuid[0:1], uuid[1:2], uuid)
}
sess := x.NewSession()
defer sess.Close()
@ -53,12 +46,15 @@ func deleteOrphanedAttachments(x *xorm.Engine) error {
for _, attachment := range attachements {
ids = append(ids, attachment.ID)
}
if _, err := sess.In("id", ids).Delete(new(Attachment)); err != nil {
return err
if len(ids) > 0 {
if _, err := sess.In("id", ids).Delete(new(Attachment)); err != nil {
return err
}
}
for _, attachment := range attachements {
if err := util.RemoveAll(AttachmentLocalPath(attachment.UUID)); err != nil {
uuid := attachment.UUID
if err := util.RemoveAll(filepath.Join(setting.Attachment.Path, uuid[0:1], uuid[1:2], uuid)); err != nil {
return err
}
}

7
models/repo.go
View file

@ -32,6 +32,7 @@ import (
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/options"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
api "code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
"code.gitea.io/gitea/modules/util"
@ -1595,7 +1596,7 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
}
releaseAttachments := make([]string, 0, len(attachments))
for i := 0; i < len(attachments); i++ {
releaseAttachments = append(releaseAttachments, attachments[i].LocalPath())
releaseAttachments = append(releaseAttachments, attachments[i].RelativePath())
}
if _, err = sess.Exec("UPDATE `user` SET num_stars=num_stars-1 WHERE id IN (SELECT `uid` FROM `star` WHERE repo_id = ?)", repo.ID); err != nil {
@ -1720,12 +1721,12 @@ func DeleteRepository(doer *User, uid, repoID int64) error {
// Remove issue attachment files.
for i := range attachmentPaths {
removeAllWithNotice(x, "Delete issue attachment", attachmentPaths[i])
RemoveStorageWithNotice(storage.Attachments, "Delete issue attachment", attachmentPaths[i])
}
// Remove release attachment files.
for i := range releaseAttachments {
removeAllWithNotice(x, "Delete release attachment", releaseAttachments[i])
RemoveStorageWithNotice(storage.Attachments, "Delete release attachment", releaseAttachments[i])
}
if len(repo.Avatar) > 0 {

6
models/unit_tests.go
View file

@ -15,6 +15,7 @@ import (
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/util"
"github.com/stretchr/testify/assert"
@ -67,6 +68,11 @@ func MainTest(m *testing.M, pathToGiteaRoot string) {
fatalTestError("url.Parse: %v\n", err)
}
setting.Attachment.Path = filepath.Join(setting.AppDataPath, "attachments")
if err = storage.Init(); err != nil {
fatalTestError("storage.Init: %v\n", err)
}
if err = util.RemoveAll(setting.RepoRootPath); err != nil {
fatalTestError("util.RemoveAll: %v\n", err)
}