domi/forgejo-domi
1
0
Fork 0
forked from forgejo/forgejo
Code Pull requests Activity

Update Vendor (#16325)

Browse source 
* Add Dependencie Update Script

* update gitea.com/lunny/levelqueue

* update github.com/PuerkitoBio/goquery

* update github.com/alecthomas/chroma

* update github.com/blevesearch/bleve/v2

* update github.com/caddyserver/certmagic

* update github.com/go-enry/go-enry/v2

* update github.com/go-redis/redis/v8

* update github.com/hashicorp/golang-lru

* update github.com/klauspost/compress

* update github.com/markbates/goth

* update github.com/mholt/archiver/v3

* update github.com/microcosm-cc/bluemonday

* update github.com/minio/minio-go/v7

* update github.com/olivere/elastic/v7

* update github.com/xanzy/go-gitlab

* update github.com/yuin/goldmark
This commit is contained in:
6543 2021-07-04 04:06:10 +02:00 • committed by GitHub
parent 65ae46bc20
commit fae07cbc8f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
319 changed files with 33568 additions and 21050 deletions
Download patch file Download diff file Expand all files Collapse all files

5
vendor/github.com/minio/minio-go/v7/Makefile generated vendored
View file

@ -21,7 +21,10 @@ test:
@GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./...
examples:
@$(foreach v,$(wildcard examples/s3/*), go build -o ${TMPDIR}/$(basename $(v)) $(v) || exit 1;)
@echo "Building s3 examples"
@cd ./examples/s3 && $(foreach v,$(wildcard examples/s3/*.go),go build -mod=mod -o ${TMPDIR}/$(basename $(v)) $(notdir $(v)) || exit 1;)
@echo "Building minio examples"
@cd ./examples/minio && $(foreach v,$(wildcard examples/minio/*.go),go build -mod=mod -o ${TMPDIR}/$(basename $(v)) $(notdir $(v)) || exit 1;)
functional-test:
@GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go

4
vendor/github.com/minio/minio-go/v7/README.md generated vendored
View file

@ -104,12 +104,12 @@ func main() {
contentType := "application/zip"
// Upload the zip file with FPutObject
n, err := minioClient.FPutObject(ctx, bucketName, objectName, filePath, minio.PutObjectOptions{ContentType: contentType})
info, err := minioClient.FPutObject(ctx, bucketName, objectName, filePath, minio.PutObjectOptions{ContentType: contentType})
if err != nil {
log.Fatalln(err)
}
log.Printf("Successfully uploaded %s of size %d\n", objectName, n)
log.Printf("Successfully uploaded %s of size %d\n", objectName, info.Size)
}
```

2
vendor/github.com/minio/minio-go/v7/api-bucket-policy.go generated vendored
View file

@ -63,7 +63,7 @@ func (c Client) putBucketPolicy(ctx context.Context, bucketName, policy string)
return err
}
if resp != nil {
if resp.StatusCode != http.StatusNoContent {
if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, bucketName, "")
}
}

79
vendor/github.com/minio/minio-go/v7/api-bucket-replication.go generated vendored
View file

@ -20,9 +20,12 @@ package minio
import (
"bytes"
"context"
"encoding/json"
"encoding/xml"
"io/ioutil"
"net/http"
"net/url"
"time"
"github.com/minio/minio-go/v7/pkg/replication"
"github.com/minio/minio-go/v7/pkg/s3utils"
@ -147,3 +150,79 @@ func (c Client) getBucketReplication(ctx context.Context, bucketName string) (cf
return cfg, nil
}
// GetBucketReplicationMetrics fetches bucket replication status metrics
func (c Client) GetBucketReplicationMetrics(ctx context.Context, bucketName string) (s replication.Metrics, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return s, err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("replication-metrics", "")
// Execute GET on bucket to get replication config.
resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
})
defer closeResponse(resp)
if err != nil {
return s, err
}
if resp.StatusCode != http.StatusOK {
return s, httpRespToErrorResponse(resp, bucketName, "")
}
respBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return s, err
}
if err := json.Unmarshal(respBytes, &s); err != nil {
return s, err
}
return s, nil
}
// ResetBucketReplication kicks off replication of previously replicated objects if ExistingObjectReplication
// is enabled in the replication config
func (c Client) ResetBucketReplication(ctx context.Context, bucketName string, olderThan time.Duration) (resetID string, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return "", err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("replication-reset", "")
if olderThan > 0 {
urlValues.Set("older-than", olderThan.String())
}
// Execute GET on bucket to get replication config.
resp, err := c.executeMethod(ctx, http.MethodPut, requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
})
defer closeResponse(resp)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", httpRespToErrorResponse(resp, bucketName, "")
}
respBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
if err := json.Unmarshal(respBytes, &resetID); err != nil {
return "", err
}
return resetID, nil
}

5
vendor/github.com/minio/minio-go/v7/api-compose-object.go generated vendored
View file

@ -220,6 +220,9 @@ func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBuck
if dstOpts.Internal.SourceETag != "" {
headers.Set(minIOBucketSourceETag, dstOpts.Internal.SourceETag)
}
if dstOpts.Internal.ReplicationRequest {
headers.Set(minIOBucketReplicationRequest, "")
}
if len(dstOpts.UserTags) != 0 {
headers.Set(amzTaggingHeader, s3utils.TagEncode(dstOpts.UserTags))
}
@ -510,7 +513,7 @@ func (c Client) ComposeObject(ctx context.Context, dst CopyDestOptions, srcs ...
// 4. Make final complete-multipart request.
uploadInfo, err := c.completeMultipartUpload(ctx, dst.Bucket, dst.Object, uploadID,
completeMultipartUpload{Parts: objParts})
completeMultipartUpload{Parts: objParts}, PutObjectOptions{})
if err != nil {
return UploadInfo{}, err
}

55
vendor/github.com/minio/minio-go/v7/api-list.go generated vendored
View file

@ -58,12 +58,12 @@ func (c Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
/// Bucket Read Operations.
func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix string, recursive, metadata bool, maxKeys int) <-chan ObjectInfo {
func (c Client) listObjectsV2(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo {
// Allocate new list objects channel.
objectStatCh := make(chan ObjectInfo, 1)
// Default listing is delimited at "/"
delimiter := "/"
if recursive {
if opts.Recursive {
// If recursive we do not delimit.
delimiter = ""
}
@ -81,7 +81,7 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix stri
}
// Validate incoming object prefix.
if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil {
if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
defer close(objectStatCh)
objectStatCh <- ObjectInfo{
Err: err,
@ -96,8 +96,8 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix stri
var continuationToken string
for {
// Get list of objects a maximum of 1000 per request.
result, err := c.listObjectsV2Query(ctx, bucketName, objectPrefix, continuationToken,
fetchOwner, metadata, delimiter, maxKeys)
result, err := c.listObjectsV2Query(ctx, bucketName, opts.Prefix, continuationToken,
fetchOwner, opts.WithMetadata, delimiter, opts.MaxKeys, opts.headers)
if err != nil {
objectStatCh <- ObjectInfo{
Err: err,
@ -153,7 +153,7 @@ func (c Client) listObjectsV2(ctx context.Context, bucketName, objectPrefix stri
// ?prefix - Limits the response to keys that begin with the specified prefix.
// ?max-keys - Sets the maximum number of keys returned in the response body.
// ?metadata - Specifies if we want metadata for the objects as part of list operation.
func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int) (ListBucketV2Result, error) {
func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix, continuationToken string, fetchOwner, metadata bool, delimiter string, maxkeys int, headers http.Header) (ListBucketV2Result, error) {
// Validate bucket name.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return ListBucketV2Result{}, err
@ -202,6 +202,7 @@ func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
customHeader: headers,
})
defer closeResponse(resp)
if err != nil {
@ -246,12 +247,12 @@ func (c Client) listObjectsV2Query(ctx context.Context, bucketName, objectPrefix
return listBucketResult, nil
}
func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix string, recursive bool, maxKeys int) <-chan ObjectInfo {
func (c Client) listObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo {
// Allocate new list objects channel.
objectStatCh := make(chan ObjectInfo, 1)
// Default listing is delimited at "/"
delimiter := "/"
if recursive {
if opts.Recursive {
// If recursive we do not delimit.
delimiter = ""
}
@ -264,7 +265,7 @@ func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix string
return objectStatCh
}
// Validate incoming object prefix.
if err := s3utils.CheckValidObjectNamePrefix(objectPrefix); err != nil {
if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
defer close(objectStatCh)
objectStatCh <- ObjectInfo{
Err: err,
@ -279,7 +280,7 @@ func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix string
marker := ""
for {
// Get list of objects a maximum of 1000 per request.
result, err := c.listObjectsQuery(ctx, bucketName, objectPrefix, marker, delimiter, maxKeys)
result, err := c.listObjectsQuery(ctx, bucketName, opts.Prefix, marker, delimiter, opts.MaxKeys, opts.headers)
if err != nil {
objectStatCh <- ObjectInfo{
Err: err,
@ -326,12 +327,12 @@ func (c Client) listObjects(ctx context.Context, bucketName, objectPrefix string
return objectStatCh
}
func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix string, recursive bool, maxKeys int) <-chan ObjectInfo {
func (c Client) listObjectVersions(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo {
// Allocate new list objects channel.
resultCh := make(chan ObjectInfo, 1)
// Default listing is delimited at "/"
delimiter := "/"
if recursive {
if opts.Recursive {
// If recursive we do not delimit.
delimiter = ""
}
@ -346,7 +347,7 @@ func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix strin
}
// Validate incoming object prefix.
if err := s3utils.CheckValidObjectNamePrefix(prefix); err != nil {
if err := s3utils.CheckValidObjectNamePrefix(opts.Prefix); err != nil {
defer close(resultCh)
resultCh <- ObjectInfo{
Err: err,
@ -365,7 +366,7 @@ func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix strin
for {
// Get list of objects a maximum of 1000 per request.
result, err := c.listObjectVersionsQuery(ctx, bucketName, prefix, keyMarker, versionIDMarker, delimiter, maxKeys)
result, err := c.listObjectVersionsQuery(ctx, bucketName, opts.Prefix, keyMarker, versionIDMarker, delimiter, opts.MaxKeys, opts.headers)
if err != nil {
resultCh <- ObjectInfo{
Err: err,
@ -438,7 +439,7 @@ func (c Client) listObjectVersions(ctx context.Context, bucketName, prefix strin
// ?delimiter - A delimiter is a character you use to group keys.
// ?prefix - Limits the response to keys that begin with the specified prefix.
// ?max-keys - Sets the maximum number of keys returned in the response body.
func (c Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix, keyMarker, versionIDMarker, delimiter string, maxkeys int) (ListVersionsResult, error) {
func (c Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix, keyMarker, versionIDMarker, delimiter string, maxkeys int, headers http.Header) (ListVersionsResult, error) {
// Validate bucket name.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return ListVersionsResult{}, err
@ -483,6 +484,7 @@ func (c Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix,
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
customHeader: headers,
})
defer closeResponse(resp)
if err != nil {
@ -534,7 +536,7 @@ func (c Client) listObjectVersionsQuery(ctx context.Context, bucketName, prefix,
// ?delimiter - A delimiter is a character you use to group keys.
// ?prefix - Limits the response to keys that begin with the specified prefix.
// ?max-keys - Sets the maximum number of keys returned in the response body.
func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) {
func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int, headers http.Header) (ListBucketResult, error) {
// Validate bucket name.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return ListBucketResult{}, err
@ -571,6 +573,7 @@ func (c Client) listObjectsQuery(ctx context.Context, bucketName, objectPrefix,
bucketName: bucketName,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
customHeader: headers,
})
defer closeResponse(resp)
if err != nil {
@ -629,6 +632,18 @@ type ListObjectsOptions struct {
// Use the deprecated list objects V1 API
UseV1 bool
headers http.Header
}
// Set adds a key value pair to the options. The
// key-value pair will be part of the HTTP GET request
// headers.
func (o *ListObjectsOptions) Set(key, value string) {
if o.headers == nil {
o.headers = make(http.Header)
}
o.headers.Set(key, value)
}
// ListObjects returns objects list after evaluating the passed options.
@ -640,22 +655,22 @@ type ListObjectsOptions struct {
//
func (c Client) ListObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo {
if opts.WithVersions {
return c.listObjectVersions(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys)
return c.listObjectVersions(ctx, bucketName, opts)
}
// Use legacy list objects v1 API
if opts.UseV1 {
return c.listObjects(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys)
return c.listObjects(ctx, bucketName, opts)
}
// Check whether this is snowball region, if yes ListObjectsV2 doesn't work, fallback to listObjectsV1.
if location, ok := c.bucketLocCache.Get(bucketName); ok {
if location == "snowball" {
return c.listObjects(ctx, bucketName, opts.Prefix, opts.Recursive, opts.MaxKeys)
return c.listObjects(ctx, bucketName, opts)
}
}
return c.listObjectsV2(ctx, bucketName, opts.Prefix, opts.Recursive, opts.WithMetadata, opts.MaxKeys)
return c.listObjectsV2(ctx, bucketName, opts)
}
// ListIncompleteUploads - List incompletely uploaded multipart objects.

6
vendor/github.com/minio/minio-go/v7/api-put-object-common.go generated vendored
View file

@ -57,17 +57,17 @@ func isReadAt(reader io.Reader) (ok bool) {
return
}
// optimalPartInfo - calculate the optimal part info for a given
// OptimalPartInfo - calculate the optimal part info for a given
// object size.
//
// NOTE: Assumption here is that for any object to be uploaded to any S3 compatible
// object storage it will have the following parameters as constants.
//
// maxPartsCount - 10000
// minPartSize - 128MiB
// minPartSize - 16MiB
// maxMultipartPutObjectSize - 5TiB
//
func optimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) {
func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) {
// object size is '-1' set it to 5TiB.
var unknownSize bool
if objectSize == -1 {

15
vendor/github.com/minio/minio-go/v7/api-put-object-multipart.go generated vendored
View file

@ -32,6 +32,7 @@ import (
"strconv"
"strings"
"github.com/google/uuid"
"github.com/minio/minio-go/v7/pkg/encrypt"
"github.com/minio/minio-go/v7/pkg/s3utils"
)
@ -72,7 +73,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje
var complMultipartUpload completeMultipartUpload
// Calculate the optimal parts info for a given size.
totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize)
totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize)
if err != nil {
return UploadInfo{}, err
}
@ -175,7 +176,7 @@ func (c Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obje
// Sort all completed parts.
sort.Sort(completedParts(complMultipartUpload.Parts))
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload)
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{})
if err != nil {
return UploadInfo{}, err
}
@ -198,6 +199,13 @@ func (c Client) initiateMultipartUpload(ctx context.Context, bucketName, objectN
urlValues := make(url.Values)
urlValues.Set("uploads", "")
if opts.Internal.SourceVersionID != "" {
if _, err := uuid.Parse(opts.Internal.SourceVersionID); err != nil {
return initiateMultipartUploadResult{}, errInvalidArgument(err.Error())
}
urlValues.Set("versionId", opts.Internal.SourceVersionID)
}
// Set ContentType header.
customHeader := opts.Header()
@ -301,7 +309,7 @@ func (c Client) uploadPart(ctx context.Context, bucketName, objectName, uploadID
// completeMultipartUpload - Completes a multipart upload by assembling previously uploaded parts.
func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectName, uploadID string,
complete completeMultipartUpload) (UploadInfo, error) {
complete completeMultipartUpload, opts PutObjectOptions) (UploadInfo, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return UploadInfo{}, err
@ -328,6 +336,7 @@ func (c Client) completeMultipartUpload(ctx context.Context, bucketName, objectN
contentBody: completeMultipartUploadBuffer,
contentLength: int64(len(completeMultipartUploadBytes)),
contentSHA256Hex: sum256Hex(completeMultipartUploadBytes),
customHeader: opts.Header(),
}
// Execute POST to complete multipart upload for an objectName.

8
vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go generated vendored
View file

@ -101,7 +101,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa
}
// Calculate the optimal parts info for a given size.
totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize)
totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(size, opts.PartSize)
if err != nil {
return UploadInfo{}, err
}
@ -231,7 +231,7 @@ func (c Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketNa
// Sort all completed parts.
sort.Sort(completedParts(complMultipartUpload.Parts))
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload)
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{})
if err != nil {
return UploadInfo{}, err
}
@ -251,7 +251,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu
}
// Calculate the optimal parts info for a given size.
totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size, opts.PartSize)
totalPartsCount, partSize, lastPartSize, err := OptimalPartInfo(size, opts.PartSize)
if err != nil {
return UploadInfo{}, err
}
@ -358,7 +358,7 @@ func (c Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, bu
// Sort all completed parts.
sort.Sort(completedParts(complMultipartUpload.Parts))
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload)
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{})
if err != nil {
return UploadInfo{}, err
}

16
vendor/github.com/minio/minio-go/v7/api-put-object.go generated vendored
View file

@ -55,10 +55,11 @@ func (r ReplicationStatus) Empty() bool {
// AdvancedPutOptions for internal use - to be utilized by replication, ILM transition
// implementation on MinIO server
type AdvancedPutOptions struct {
SourceVersionID string
SourceETag string
ReplicationStatus ReplicationStatus
SourceMTime time.Time
SourceVersionID string
SourceETag string
ReplicationStatus ReplicationStatus
SourceMTime time.Time
ReplicationRequest bool
}
// PutObjectOptions represents options specified by user for PutObject call
@ -152,6 +153,9 @@ func (opts PutObjectOptions) Header() (header http.Header) {
if opts.Internal.SourceETag != "" {
header.Set(minIOBucketSourceETag, opts.Internal.SourceETag)
}
if opts.Internal.ReplicationRequest {
header.Set(minIOBucketReplicationRequest, "")
}
if len(opts.UserTags) != 0 {
header.Set(amzTaggingHeader, s3utils.TagEncode(opts.UserTags))
}
@ -269,7 +273,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName
var complMultipartUpload completeMultipartUpload
// Calculate the optimal parts info for a given size.
totalPartsCount, partSize, _, err := optimalPartInfo(-1, opts.PartSize)
totalPartsCount, partSize, _, err := OptimalPartInfo(-1, opts.PartSize)
if err != nil {
return UploadInfo{}, err
}
@ -356,7 +360,7 @@ func (c Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketName
// Sort all completed parts.
sort.Sort(completedParts(complMultipartUpload.Parts))
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload)
uploadInfo, err := c.completeMultipartUpload(ctx, bucketName, objectName, uploadID, complMultipartUpload, PutObjectOptions{})
if err != nil {
return UploadInfo{}, err
}

4
vendor/github.com/minio/minio-go/v7/api-remove.go generated vendored
View file

@ -64,6 +64,7 @@ type AdvancedRemoveOptions struct {
ReplicationDeleteMarker bool
ReplicationStatus ReplicationStatus
ReplicationMTime time.Time
ReplicationRequest bool
}
// RemoveObjectOptions represents options specified by user for RemoveObject call
@ -112,6 +113,9 @@ func (c Client) removeObject(ctx context.Context, bucketName, objectName string,
if !opts.Internal.ReplicationStatus.Empty() {
headers.Set(amzBucketReplicationStatus, string(opts.Internal.ReplicationStatus))
}
if opts.Internal.ReplicationRequest {
headers.Set(minIOBucketReplicationRequest, "")
}
// Execute DELETE on objectName.
resp, err := c.executeMethod(ctx, http.MethodDelete, requestMetadata{
bucketName: bucketName,

5
vendor/github.com/minio/minio-go/v7/api.go generated vendored
View file

@ -108,7 +108,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v7.0.10"
libraryVersion = "v7.0.12"
)
// User Agent should always following the below style.
@ -544,9 +544,6 @@ func (c Client) executeMethod(ctx context.Context, method string, metadata reque
// Indicate to our routine to exit cleanly upon return.
defer cancel()
// Blank indentifier is kept here on purpose since 'range' without
// blank identifiers is only supported since go1.4
// https://golang.org/doc/go1.4#forrange.
for range c.newRetryTimer(retryCtx, reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter) {
// Retry executes the following function body if request has an
// error until maxRetries have been exhausted, retry attempts are

5
vendor/github.com/minio/minio-go/v7/constants.go generated vendored
View file

@ -23,9 +23,9 @@ package minio
// a part in a multipart upload may not be uploaded.
const absMinPartSize = 1024 * 1024 * 5
// minPartSize - minimum part size 128MiB per object after which
// minPartSize - minimum part size 16MiB per object after which
// putObject behaves internally as multipart.
const minPartSize = 1024 * 1024 * 128
const minPartSize = 1024 * 1024 * 16
// maxPartsCount - maximum number of parts for a single multipart session.
const maxPartsCount = 10000
@ -88,4 +88,5 @@ const (
minIOBucketSourceETag = "X-Minio-Source-Etag"
minIOBucketReplicationDeleteMarker = "X-Minio-Source-DeleteMarker"
minIOBucketReplicationProxyRequest = "X-Minio-Source-Proxy-Request"
minIOBucketReplicationRequest = "X-Minio-Source-Replication-Request"
)

8
vendor/github.com/minio/minio-go/v7/core.go generated vendored
View file

@ -46,13 +46,13 @@ func NewCore(endpoint string, opts *Options) (*Core, error) {
// ListObjects - List all the objects at a prefix, optionally with marker and delimiter
// you can further filter the results.
func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListBucketResult, err error) {
return c.listObjectsQuery(context.Background(), bucket, prefix, marker, delimiter, maxKeys)
return c.listObjectsQuery(context.Background(), bucket, prefix, marker, delimiter, maxKeys, nil)
}
// ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses
// continuationToken instead of marker to support iteration over the results.
func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) {
return c.listObjectsV2Query(context.Background(), bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys)
return c.listObjectsV2Query(context.Background(), bucketName, objectPrefix, continuationToken, fetchOwner, false, delimiter, maxkeys, nil)
}
// CopyObject - copies an object from source object to destination object on server side.
@ -97,10 +97,10 @@ func (c Core) ListObjectParts(ctx context.Context, bucket, object, uploadID stri
}
// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object.
func (c Core) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []CompletePart) (string, error) {
func (c Core) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, parts []CompletePart, opts PutObjectOptions) (string, error) {
res, err := c.completeMultipartUpload(ctx, bucket, object, uploadID, completeMultipartUpload{
Parts: parts,
})
}, opts)
return res.ETag, err
}

11812
vendor/github.com/minio/minio-go/v7/functional_tests.go generated vendored Normal file
View file

File diff suppressed because it is too large Load diff

6
vendor/github.com/minio/minio-go/v7/go.mod generated vendored
View file

@ -1,8 +1,9 @@
module github.com/minio/minio-go/v7
go 1.12
go 1.14
require (
github.com/dustin/go-humanize v1.0.0
github.com/google/uuid v1.1.1
github.com/json-iterator/go v1.1.10
github.com/klauspost/cpuid v1.3.1 // indirect
@ -13,9 +14,10 @@ require (
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/rs/xid v1.2.1
github.com/sirupsen/logrus v1.8.1
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/stretchr/testify v1.4.0 // indirect
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f
golang.org/x/net v0.0.0-20200707034311-ab3426394381
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae // indirect
golang.org/x/text v0.3.3 // indirect

17
vendor/github.com/minio/minio-go/v7/go.sum generated vendored
View file

@ -1,6 +1,8 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@ -10,7 +12,6 @@ github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs=
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
@ -25,11 +26,9 @@ github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKU
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
@ -37,30 +36,32 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s=
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f h1:aZp0e2vLN4MToVqnjNEYEtrEA8RH8U8FN1CU7JgqsPU=
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

48
vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go generated vendored
View file

@ -59,6 +59,10 @@ const (
defaultECSRoleEndpoint = "http://169.254.170.2"
defaultSTSRoleEndpoint = "https://sts.amazonaws.com"
defaultIAMSecurityCredsPath = "/latest/meta-data/iam/security-credentials/"
tokenRequestTTLHeader = "X-aws-ec2-metadata-token-ttl-seconds"
tokenPath = "/latest/api/token"
tokenTTL = "21600"
tokenRequestHeader = "X-aws-ec2-metadata-token"
)
// NewIAM returns a pointer to a new Credentials object wrapping the IAM.
@ -75,6 +79,7 @@ func NewIAM(endpoint string) *Credentials {
// Error will be returned if the request fails, or unable to extract
// the desired
func (m *IAM) Retrieve() (Value, error) {
token := os.Getenv("AWS_CONTAINER_AUTHORIZATION_TOKEN")
var roleCreds ec2RoleCredRespBody
var err error
@ -120,7 +125,7 @@ func (m *IAM) Retrieve() (Value, error) {
os.Getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"))
}
roleCreds, err = getEcsTaskCredentials(m.Client, endpoint)
roleCreds, err = getEcsTaskCredentials(m.Client, endpoint, token)
case len(os.Getenv("AWS_CONTAINER_CREDENTIALS_FULL_URI")) > 0:
if len(endpoint) == 0 {
@ -134,7 +139,7 @@ func (m *IAM) Retrieve() (Value, error) {
}
}
roleCreds, err = getEcsTaskCredentials(m.Client, endpoint)
roleCreds, err = getEcsTaskCredentials(m.Client, endpoint, token)
default:
roleCreds, err = getCredentials(m.Client, endpoint)
@ -192,11 +197,14 @@ func getIAMRoleURL(endpoint string) (*url.URL, error) {
// with the current EC2 service. If there are no credentials,
// or there is an error making or receiving the request.
// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
func listRoleNames(client *http.Client, u *url.URL) ([]string, error) {
func listRoleNames(client *http.Client, u *url.URL, token string) ([]string, error) {
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
if token != "" {
req.Header.Add(tokenRequestHeader, token)
}
resp, err := client.Do(req)
if err != nil {
return nil, err
@ -219,12 +227,16 @@ func listRoleNames(client *http.Client, u *url.URL) ([]string, error) {
return credsList, nil
}
func getEcsTaskCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) {
func getEcsTaskCredentials(client *http.Client, endpoint string, token string) (ec2RoleCredRespBody, error) {
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
if err != nil {
return ec2RoleCredRespBody{}, err
}
if token != "" {
req.Header.Set("Authorization", token)
}
resp, err := client.Do(req)
if err != nil {
return ec2RoleCredRespBody{}, err
@ -242,12 +254,35 @@ func getEcsTaskCredentials(client *http.Client, endpoint string) (ec2RoleCredRes
return respCreds, nil
}
func fetchIMDSToken(client *http.Client, endpoint string) (string, error) {
req, err := http.NewRequest(http.MethodPut, endpoint+tokenPath, nil)
if err != nil {
return "", err
}
req.Header.Add(tokenRequestTTLHeader, tokenTTL)
resp, err := client.Do(req)
if err != nil {
return "", err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", errors.New(resp.Status)
}
return string(data), nil
}
// getCredentials - obtains the credentials from the IAM role name associated with
// the current EC2 service.
//
// If the credentials cannot be found, or there is an error
// reading the response an error will be returned.
func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody, error) {
// https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html
token, _ := fetchIMDSToken(client, endpoint)
// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
u, err := getIAMRoleURL(endpoint)
@ -256,7 +291,7 @@ func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody,
}
// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html
roleNames, err := listRoleNames(client, u)
roleNames, err := listRoleNames(client, u, token)
if err != nil {
return ec2RoleCredRespBody{}, err
}
@ -280,6 +315,9 @@ func getCredentials(client *http.Client, endpoint string) (ec2RoleCredRespBody,
if err != nil {
return ec2RoleCredRespBody{}, err
}
if token != "" {
req.Header.Add(tokenRequestHeader, token)
}
resp, err := client.Do(req)
if err != nil {

13
vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go generated vendored
View file

@ -81,10 +81,15 @@ func (n NoncurrentVersionTransition) IsDaysNull() bool {
return n.NoncurrentDays == ExpirationDays(0)
}
// IsStorageClassEmpty returns true if storage class field is empty
func (n NoncurrentVersionTransition) IsStorageClassEmpty() bool {
return n.StorageClass == ""
}
// MarshalXML is extended to leave out
// <NoncurrentVersionTransition></NoncurrentVersionTransition> tags
func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
if n.IsDaysNull() {
if n.IsDaysNull() || n.IsStorageClassEmpty() {
return nil
}
type noncurrentVersionTransitionWrapper NoncurrentVersionTransition
@ -137,9 +142,9 @@ func (t Transition) MarshalXML(en *xml.Encoder, startElement xml.StartElement) e
// And And Rule for LifecycleTag, to be used in LifecycleRuleFilter
type And struct {
XMLName xml.Name `xml:"And,omitempty" json:"-"`
Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"`
Tags []Tag `xml:"Tag,omitempty" json:"Tags,omitempty"`
XMLName xml.Name `xml:"And" json:"-"`
Prefix string `xml:"Prefix" json:"Prefix,omitempty"`
Tags []Tag `xml:"Tag" json:"Tags,omitempty"`
}
// IsEmpty returns true if Tags field is null

174
vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go generated vendored
View file

@ -27,7 +27,7 @@ import (
"github.com/rs/xid"
)
var errInvalidFilter = fmt.Errorf("Invalid filter")
var errInvalidFilter = fmt.Errorf("invalid filter")
// OptionType specifies operation to be performed on config
type OptionType string
@ -46,19 +46,21 @@ const (
// Options represents options to set a replication configuration rule
type Options struct {
Op OptionType
ID string
Prefix string
RuleStatus string
Priority string
TagString string
StorageClass string
RoleArn string
DestBucket string
IsTagSet bool
IsSCSet bool
ReplicateDeletes string // replicate versioned deletes
ReplicateDeleteMarkers string // replicate soft deletes
Op OptionType
ID string
Prefix string
RuleStatus string
Priority string
TagString string
StorageClass string
RoleArn string
DestBucket string
IsTagSet bool
IsSCSet bool
ReplicateDeletes string // replicate versioned deletes
ReplicateDeleteMarkers string // replicate soft deletes
ReplicaSync string // replicate replica metadata modifications
ExistingObjectReplicate string
}
// Tags returns a slice of tags for a rule
@ -71,7 +73,7 @@ func (opts Options) Tags() ([]Tag, error) {
}
kv := strings.SplitN(tok, "=", 2)
if len(kv) != 2 {
return []Tag{}, fmt.Errorf("Tags should be entered as comma separated k=v pairs")
return []Tag{}, fmt.Errorf("tags should be entered as comma separated k=v pairs")
}
tagList = append(tagList, Tag{
Key: kv[0],
@ -102,7 +104,7 @@ func (c *Config) AddRule(opts Options) error {
return err
}
if opts.RoleArn != c.Role && c.Role != "" {
return fmt.Errorf("Role ARN does not match existing configuration")
return fmt.Errorf("role ARN does not match existing configuration")
}
var status Status
// toggle rule status for edit option
@ -112,7 +114,7 @@ func (c *Config) AddRule(opts Options) error {
case "disable":
status = Disabled
default:
return fmt.Errorf("Rule state should be either [enable|disable]")
return fmt.Errorf("rule state should be either [enable|disable]")
}
tags, err := opts.Tags()
@ -142,7 +144,7 @@ func (c *Config) AddRule(opts Options) error {
arnStr = c.Role
}
if arnStr == "" {
return fmt.Errorf("Role ARN required")
return fmt.Errorf("role ARN required")
}
tokens := strings.Split(arnStr, ":")
if len(tokens) != 6 {
@ -183,7 +185,28 @@ func (c *Config) AddRule(opts Options) error {
return fmt.Errorf("ReplicateDeletes should be either enable|disable")
}
}
var replicaSync Status
// replica sync is by default Enabled, unless specified.
switch opts.ReplicaSync {
case "enable", "":
replicaSync = Enabled
case "disable":
replicaSync = Disabled
default:
return fmt.Errorf("replica metadata sync should be either [enable|disable]")
}
var existingStatus Status
if opts.ExistingObjectReplicate != "" {
switch opts.ExistingObjectReplicate {
case "enable":
existingStatus = Enabled
case "disable", "":
existingStatus = Disabled
default:
return fmt.Errorf("existingObjectReplicate should be either enable|disable")
}
}
newRule := Rule{
ID: opts.ID,
Priority: priority,
@ -200,9 +223,13 @@ func (c *Config) AddRule(opts Options) error {
// However AWS leaves this configurable https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-for-metadata-changes.html
SourceSelectionCriteria: SourceSelectionCriteria{
ReplicaModifications: ReplicaModifications{
Status: Enabled,
Status: replicaSync,
},
},
// By default disable existing object replication unless selected
ExistingObjectReplication: ExistingObjectReplication{
Status: existingStatus,
},
}
// validate rule after overlaying priority for pre-existing rule being disabled.
@ -211,13 +238,13 @@ func (c *Config) AddRule(opts Options) error {
}
for _, rule := range c.Rules {
if rule.Priority == newRule.Priority {
return fmt.Errorf("Priority must be unique. Replication configuration already has a rule with this priority")
return fmt.Errorf("priority must be unique. Replication configuration already has a rule with this priority")
}
if rule.Destination.Bucket != newRule.Destination.Bucket {
return fmt.Errorf("The destination bucket must be same for all rules")
return fmt.Errorf("the destination bucket must be same for all rules")
}
if rule.ID == newRule.ID {
return fmt.Errorf("A rule exists with this ID")
return fmt.Errorf("a rule exists with this ID")
}
}
@ -228,7 +255,7 @@ func (c *Config) AddRule(opts Options) error {
// EditRule modifies an existing rule in replication config
func (c *Config) EditRule(opts Options) error {
if opts.ID == "" {
return fmt.Errorf("Rule ID missing")
return fmt.Errorf("rule ID missing")
}
rIdx := -1
var newRule Rule
@ -240,7 +267,7 @@ func (c *Config) EditRule(opts Options) error {
}
}
if rIdx < 0 {
return fmt.Errorf("Rule with ID %s not found in replication configuration", opts.ID)
return fmt.Errorf("rule with ID %s not found in replication configuration", opts.ID)
}
prefixChg := opts.Prefix != newRule.Prefix()
if opts.IsTagSet || prefixChg {
@ -286,7 +313,7 @@ func (c *Config) EditRule(opts Options) error {
case "disable":
newRule.Status = Disabled
default:
return fmt.Errorf("Rule state should be either [enable|disable]")
return fmt.Errorf("rule state should be either [enable|disable]")
}
}
// set DeleteMarkerReplication rule status for edit option
@ -314,6 +341,27 @@ func (c *Config) EditRule(opts Options) error {
}
}
if opts.ReplicaSync != "" {
switch opts.ReplicaSync {
case "enable", "":
newRule.SourceSelectionCriteria.ReplicaModifications.Status = Enabled
case "disable":
newRule.SourceSelectionCriteria.ReplicaModifications.Status = Disabled
default:
return fmt.Errorf("replica metadata sync should be either [enable|disable]")
}
}
fmt.Println("opts.ExistingObjectReplicate>", opts.ExistingObjectReplicate)
if opts.ExistingObjectReplicate != "" {
switch opts.ExistingObjectReplicate {
case "enable":
newRule.ExistingObjectReplication.Status = Enabled
case "disable":
newRule.ExistingObjectReplication.Status = Disabled
default:
return fmt.Errorf("existingObjectsReplication state should be either [enable|disable]")
}
}
if opts.IsSCSet {
newRule.Destination.StorageClass = opts.StorageClass
}
@ -343,10 +391,10 @@ func (c *Config) EditRule(opts Options) error {
// ensure priority and destination bucket restrictions are not violated
for idx, rule := range c.Rules {
if rule.Priority == newRule.Priority && rIdx != idx {
return fmt.Errorf("Priority must be unique. Replication configuration already has a rule with this priority")
return fmt.Errorf("priority must be unique. Replication configuration already has a rule with this priority")
}
if rule.Destination.Bucket != newRule.Destination.Bucket {
return fmt.Errorf("The destination bucket must be same for all rules")
return fmt.Errorf("the destination bucket must be same for all rules")
}
}
@ -369,7 +417,7 @@ func (c *Config) RemoveRule(opts Options) error {
return fmt.Errorf("Rule with ID %s not found", opts.ID)
}
if len(newRules) == 0 {
return fmt.Errorf("Replication configuration should have at least one rule")
return fmt.Errorf("replication configuration should have at least one rule")
}
c.Rules = newRules
return nil
@ -378,15 +426,16 @@ func (c *Config) RemoveRule(opts Options) error {
// Rule - a rule for replication configuration.
type Rule struct {
XMLName xml.Name `xml:"Rule" json:"-"`
ID string `xml:"ID,omitempty"`
Status Status `xml:"Status"`
Priority int `xml:"Priority"`
DeleteMarkerReplication DeleteMarkerReplication `xml:"DeleteMarkerReplication"`
DeleteReplication DeleteReplication `xml:"DeleteReplication"`
Destination Destination `xml:"Destination"`
Filter Filter `xml:"Filter" json:"Filter"`
SourceSelectionCriteria SourceSelectionCriteria `xml:"SourceSelectionCriteria" json:"SourceSelectionCriteria"`
XMLName xml.Name `xml:"Rule" json:"-"`
ID string `xml:"ID,omitempty"`
Status Status `xml:"Status"`
Priority int `xml:"Priority"`
DeleteMarkerReplication DeleteMarkerReplication `xml:"DeleteMarkerReplication"`
DeleteReplication DeleteReplication `xml:"DeleteReplication"`
Destination Destination `xml:"Destination"`
Filter Filter `xml:"Filter" json:"Filter"`
SourceSelectionCriteria SourceSelectionCriteria `xml:"SourceSelectionCriteria" json:"SourceSelectionCriteria"`
ExistingObjectReplication ExistingObjectReplication `xml:"ExistingObjectReplication,omitempty" json:"ExistingObjectReplication,omitempty"`
}
// Validate validates the rule for correctness
@ -402,14 +451,13 @@ func (r Rule) Validate() error {
}
if r.Priority < 0 && r.Status == Enabled {
return fmt.Errorf("Priority must be set for the rule")
return fmt.Errorf("priority must be set for the rule")
}
if err := r.validateStatus(); err != nil {
return err
}
return nil
return r.ExistingObjectReplication.Validate()
}
// validateID - checks if ID is valid or not.
@ -525,11 +573,11 @@ func (tag Tag) IsEmpty() bool {
// Validate checks this tag.
func (tag Tag) Validate() error {
if len(tag.Key) == 0 || utf8.RuneCountInString(tag.Key) > 128 {
return fmt.Errorf("Invalid Tag Key")
return fmt.Errorf("invalid Tag Key")
}
if utf8.RuneCountInString(tag.Value) > 256 {
return fmt.Errorf("Invalid Tag Value")
return fmt.Errorf("invalid Tag Value")
}
return nil
}
@ -585,7 +633,7 @@ func (d DeleteReplication) IsEmpty() bool {
// ReplicaModifications specifies if replica modification sync is enabled
type ReplicaModifications struct {
Status Status `xml:"Status" json:"Status"`
Status Status `xml:"Status" json:"Status"` // should be set to "Enabled" by default
}
// SourceSelectionCriteria - specifies additional source selection criteria in ReplicationConfiguration.
@ -604,7 +652,45 @@ func (s SourceSelectionCriteria) Validate() error {
return nil
}
if !s.IsValid() {
return fmt.Errorf("Invalid ReplicaModification status")
return fmt.Errorf("invalid ReplicaModification status")
}
return nil
}
// ExistingObjectReplication - whether existing object replication is enabled
type ExistingObjectReplication struct {
Status Status `xml:"Status"` // should be set to "Disabled" by default
}
// IsEmpty returns true if DeleteMarkerReplication is not set
func (e ExistingObjectReplication) IsEmpty() bool {
return len(e.Status) == 0
}
// Validate validates whether the status is disabled.
func (e ExistingObjectReplication) Validate() error {
if e.IsEmpty() {
return nil
}
if e.Status != Disabled && e.Status != Enabled {
return fmt.Errorf("invalid ExistingObjectReplication status")
}
return nil
}
// Metrics represents inline replication metrics
// such as pending, failed and completed bytes in total for a bucket
type Metrics struct {
// Pending size in bytes
PendingSize uint64 `json:"pendingReplicationSize"`
// Completed size in bytes
ReplicatedSize uint64 `json:"completedReplicationSize"`
// Total Replica size in bytes
ReplicaSize uint64 `json:"replicaSize"`
// Failed size in bytes
FailedSize uint64 `json:"failedReplicationSize"`
// Total number of pending operations including metadata updates
PendingCount uint64 `json:"pendingReplicationCount"`
// Total number of failed operations including metadata updates
FailedCount uint64 `json:"failedReplicationCount"`
}