domi/forgejo-domi
1
0
Fork 0
forked from forgejo/forgejo
Code Pull requests Activity

Make minio package support legacy MD5 checksum (#23768) (#23770)

Browse source 
Backport #23768 (no source code conflict, only some unrelated
docs/test-ini conflicts)

Some storages like:

 * https://developers.cloudflare.com/r2/api/s3/api/
 * https://www.backblaze.com/b2/docs/s3_compatible_api.html

They do not support "x-amz-checksum-algorithm" header

But minio recently uses that header with CRC32C by default. So we have
to tell minio to use legacy MD5 checksum.
This commit is contained in:
wxiaoguang 2023-03-29 00:02:13 +08:00 • committed by GitHub
parent 428d26d4a8
commit b73d1ac1eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 73 additions and 23 deletions
Download patch file Download diff file Expand all files Collapse all files

21
modules/storage/minio.go
View file

@ -6,6 +6,7 @@ package storage
import (
"context"
"crypto/tls"
"fmt"
"io"
"net/http"
"net/url"
@ -52,10 +53,12 @@ type MinioStorageConfig struct {
BasePath string `ini:"MINIO_BASE_PATH"`
UseSSL bool `ini:"MINIO_USE_SSL"`
InsecureSkipVerify bool `ini:"MINIO_INSECURE_SKIP_VERIFY"`
ChecksumAlgorithm string `ini:"MINIO_CHECKSUM_ALGORITHM"`
}
// MinioStorage returns a minio bucket storage
type MinioStorage struct {
cfg *MinioStorageConfig
ctx context.Context
client *minio.Client
bucket string
@ -90,6 +93,10 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
}
config := configInterface.(MinioStorageConfig)
if config.ChecksumAlgorithm != "" && config.ChecksumAlgorithm != "default" && config.ChecksumAlgorithm != "md5" {
return nil, fmt.Errorf("invalid minio checksum algorithm: %s", config.ChecksumAlgorithm)
}
log.Info("Creating Minio storage at %s:%s with base path %s", config.Endpoint, config.Bucket, config.BasePath)
minioClient, err := minio.New(config.Endpoint, &minio.Options{
@ -112,6 +119,7 @@ func NewMinioStorage(ctx context.Context, cfg interface{}) (ObjectStorage, error
}
return &MinioStorage{
cfg: &config,
ctx: ctx,
client: minioClient,
bucket: config.Bucket,
@ -123,7 +131,7 @@ func (m *MinioStorage) buildMinioPath(p string) string {
return strings.TrimPrefix(path.Join(m.basePath, path.Clean("/" + strings.ReplaceAll(p, "\\", "/"))[1:]), "/")
}
// Open open a file
// Open opens a file
func (m *MinioStorage) Open(path string) (Object, error) {
opts := minio.GetObjectOptions{}
object, err := m.client.GetObject(m.ctx, m.bucket, m.buildMinioPath(path), opts)
@ -133,7 +141,7 @@ func (m *MinioStorage) Open(path string) (Object, error) {
return &minioObject{object}, nil
}
// Save save a file to minio
// Save saves a file to minio
func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error) {
uploadInfo, err := m.client.PutObject(
m.ctx,
@ -141,7 +149,14 @@ func (m *MinioStorage) Save(path string, r io.Reader, size int64) (int64, error)
m.buildMinioPath(path),
r,
size,
minio.PutObjectOptions{ContentType: "application/octet-stream"},
minio.PutObjectOptions{
ContentType: "application/octet-stream",
// some storages like:
// * https://developers.cloudflare.com/r2/api/s3/api/
// * https://www.backblaze.com/b2/docs/s3_compatible_api.html
// do not support "x-amz-checksum-algorithm" header, so use legacy MD5 checksum
SendContentMd5: m.cfg.ChecksumAlgorithm == "md5",
},
)
if err != nil {
return 0, convertMinioErr(err)