Fix hasSpaceFor in SNSD setup (#17630)

If drive is offline or filled we divide by 0.

Fixes #17629

Bonus: Reject when any valid disk exceeds minimum inode threshold.

.github/workflows/helm-lint.yml

@@ -0,0 +1,33 @@
+name: Helm Chart linting
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+permissions:
+  contents: read
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+
+      - name: Run helm lint
+        run: |
+          cd helm/minio
+          helm lint .

.gitignore

@@ -38,4 +38,6 @@ docs/debugging/s3-check-md5/s3-check-md5
 docs/debugging/hash-set/hash-set
 docs/debugging/healing-bin/healing-bin
 docs/debugging/inspect/inspect
+docs/debugging/pprofgoparser/pprofgoparser
 .bin/
+*.gz

cmd/admin-handlers-idp-config.go

@@ -37,7 +37,7 @@ import (
 	"github.com/minio/pkg/ldap"
 )
 
-func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
+func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
@@ -201,7 +201,7 @@ func (a adminAPIHandlers) AddIdentityProviderCfg(w http.ResponseWriter, r *http.
 	ctx := newContext(r, w, "AddIdentityProviderCfg")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	a.addOrUpdateIDPHandler(ctx, w, r, false)
+	addOrUpdateIDPHandler(ctx, w, r, false)
 }
 
 // UpdateIdentityProviderCfg: updates an existing IDP config for openid/ldap.
@@ -213,7 +213,7 @@ func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *ht
 	ctx := newContext(r, w, "UpdateIdentityProviderCfg")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	a.addOrUpdateIDPHandler(ctx, w, r, true)
+	addOrUpdateIDPHandler(ctx, w, r, true)
 }
 
 // ListIdentityProviderCfg:

cmd/admin-handlers-site-replication.go

@@ -549,12 +549,14 @@ func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http
 
 // SiteReplicationDevNull - everything goes to io.Discard
 // [POST] /minio/admin/v3/site-replication/devnull
-func (a *adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.Request) {
+func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationDevNull")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
 	globalSiteNetPerfRX.Connect()
 	defer globalSiteNetPerfRX.Disconnect()
 
 	connectTime := time.Now()
-	ctx := newContext(r, w, "SiteReplicationDevNull")
 	for {
 		n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
 		atomic.AddUint64(&globalSiteNetPerfRX.RX, uint64(n))
@@ -578,7 +580,10 @@ func (a *adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http
 
 // SiteReplicationNetPerf - everything goes to io.Discard
 // [POST] /minio/admin/v3/site-replication/netperf
-func (a *adminAPIHandlers) SiteReplicationNetPerf(w http.ResponseWriter, r *http.Request) {
+func (a adminAPIHandlers) SiteReplicationNetPerf(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationNetPerf")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
 	durationStr := r.Form.Get(peerRESTDuration)
 	duration, _ := time.ParseDuration(durationStr)
 	if duration < globalNetPerfMinDuration {

cmd/admin-handlers.go

@@ -297,15 +297,12 @@ type ServerProperties struct {
 	SQSARN       []string `json:"sqsARN"`
 }
 
-// ServerConnStats holds transferred bytes from/to the server
-type ServerConnStats struct {
-	TotalInputBytes  uint64 `json:"transferred"`
-	TotalOutputBytes uint64 `json:"received"`
-	Throughput       uint64 `json:"throughput,omitempty"`
-	S3InputBytes     uint64 `json:"transferredS3"`
-	S3OutputBytes    uint64 `json:"receivedS3"`
-	AdminInputBytes  uint64 `json:"transferredAdmin"`
-	AdminOutputBytes uint64 `json:"receivedAdmin"`
+// serverConnStats holds transferred bytes from/to the server
+type serverConnStats struct {
+	internodeInputBytes  uint64
+	internodeOutputBytes uint64
+	s3InputBytes         uint64
+	s3OutputBytes        uint64
 }
 
 // ServerHTTPAPIStats holds total number of HTTP operations from/to the server,
@@ -1245,11 +1242,6 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// SpeedtestHandler - Deprecated. See ObjectSpeedTestHandler
-func (a adminAPIHandlers) SpeedTestHandler(w http.ResponseWriter, r *http.Request) {
-	a.ObjectSpeedTestHandler(w, r)
-}
-
 // ObjectSpeedTestHandler - reports maximum speed of a cluster by performing PUT and
 // GET operations on the server, supports auto tuning by default by automatically
 // increasing concurrency and stopping when we have reached the limits on the
@@ -1419,6 +1411,7 @@ func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo,
 // NetSpeedtestHandler - reports maximum network throughput
 func (a adminAPIHandlers) NetSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "NetSpeedtestHandler")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 }
@@ -1548,6 +1541,7 @@ func extractTraceOptions(r *http.Request) (opts madmin.ServiceTraceOpts, err err
 // The handler sends http trace to the connected HTTP client.
 func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HTTPTrace")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "")

cmd/admin-router.go

@@ -283,7 +283,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 				Queries("paths", "{paths:.*}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ForceUnlockHandler)))
 		}
 
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedTestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedTestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedTestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(httpTraceHdrs(adminAPI.DriveSpeedtestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetperfHandler))

cmd/api-errors.go

@@ -138,6 +138,8 @@ const (
 	ErrReplicationDenyEditError
 	ErrRemoteTargetDenyAddError
 	ErrReplicationNoExistingObjects
+	ErrReplicationValidationError
+	ErrReplicationPermissionCheckError
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
 	ErrNoSuchUpload
@@ -1015,6 +1017,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Versioning must be 'Enabled' on the bucket to add a replication target",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrReplicationValidationError: {
+		Code:           "InvalidRequest",
+		Description:    "Replication validation failed on target",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrReplicationPermissionCheckError: {
+		Code:           "ReplicationPermissionCheck",
+		Description:    "X-Minio-Source-Replication-Check cannot be specified in request. Request cannot be completed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
 		Description:    "The specified object does not have a ObjectLock configuration",

cmd/api-router.go

@@ -464,6 +464,9 @@ func registerAPIRouter(router *mux.Router) {
 		// GetBucketReplicationMetrics
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "")
+		// ValidateBucketReplicationCreds
+		router.Methods(http.MethodGet).HandlerFunc(
+			collectAPIStats("checkbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.ValidateBucketReplicationCredsHandler))))).Queries("replication-check", "")
 
 		// Register rejected bucket APIs
 		for _, r := range rejectedBucketAPIs {

cmd/auth-handler.go

@@ -595,8 +595,8 @@ func isSupportedS3AuthType(aType authType) bool {
 	return ok
 }
 
-// setAuthHandler to validate authorization header for the incoming request.
-func setAuthHandler(h http.Handler) http.Handler {
+// setAuthMiddleware to validate authorization header for the incoming request.
+func setAuthMiddleware(h http.Handler) http.Handler {
 	// handler for validating incoming authorization headers.
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)

cmd/bitrot-streaming.go

@@ -167,7 +167,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 				if !IsErr(err, ignoredErrs...) {
 					logger.LogOnceIf(GlobalContext,
 						fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
-							b.disk, b.volume, b.filePath, err), "bitrot-read-file-stream"+b.volume+b.filePath)
+							b.disk, b.volume, b.filePath, err), "bitrot-read-file-stream-"+b.volume+"-"+b.filePath)
 				}
 			}
 		} else {

cmd/bootstrap-peer-server.go

@@ -153,14 +153,18 @@ func (b *bootstrapRESTServer) VerifyHandler(w http.ResponseWriter, r *http.Reque
 
 // registerBootstrapRESTHandlers - register bootstrap rest router.
 func registerBootstrapRESTHandlers(router *mux.Router) {
+	h := func(f http.HandlerFunc) http.HandlerFunc {
+		return collectInternodeStats(httpTraceHdrs(f))
+	}
+
 	server := &bootstrapRESTServer{}
 	subrouter := router.PathPrefix(bootstrapRESTPrefix).Subrouter()
 
 	subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodHealth).HandlerFunc(
-		httpTraceHdrs(server.HealthHandler))
+		h(server.HealthHandler))
 
 	subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodVerify).HandlerFunc(
-		httpTraceHdrs(server.VerifyHandler))
+		h(server.VerifyHandler))
 }
 
 // client to talk to bootstrap NEndpoints.

cmd/bucket-handlers.go

@@ -580,8 +580,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			}
 		}
 		if object.VersionID != "" && hasLockEnabled {
-			if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); apiErrCode != ErrNone {
-				apiErr := errorCodes.ToAPIErr(apiErrCode)
+			if err := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); err != nil {
+				apiErr := toAPIError(ctx, err)
 				deleteResults[index].errInfo = DeleteError{
 					Code:      apiErr.Code,
 					Message:   apiErr.Description,

cmd/bucket-object-lock.go

@@ -82,24 +82,24 @@ func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locke
 // For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
 // governance bypass headers are set and user has governance bypass permissions.
 // Objects in "Compliance" mode can be overwritten only if retention date is past.
-func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, oi ObjectInfo, gerr error) APIErrorCode {
+func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, oi ObjectInfo, gerr error) error {
 	if gerr != nil { // error from GetObjectInfo
 		if _, ok := gerr.(MethodNotAllowed); ok {
 			// This happens usually for a delete marker
 			if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 				// Delete marker should be present and valid.
-				return ErrNone
+				return nil
 			}
 		}
 		if isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr) {
-			return ErrNone
+			return nil
 		}
-		return toAPIErrorCode(ctx, gerr)
+		return gerr
 	}
 
 	lhold := objectlock.GetObjectLegalHoldMeta(oi.UserDefined)
 	if lhold.Status.Valid() && lhold.Status == objectlock.LegalHoldOn {
-		return ErrObjectLocked
+		return ObjectLocked{}
 	}
 
 	ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
@@ -115,13 +115,13 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 			t, err := objectlock.UTCNowNTP()
 			if err != nil {
 				logger.LogIf(ctx, err)
-				return ErrObjectLocked
+				return ObjectLocked{}
 			}
 
 			if !ret.RetainUntilDate.Before(t) {
-				return ErrObjectLocked
+				return ObjectLocked{}
 			}
-			return ErrNone
+			return nil
 		case objectlock.RetGovernance:
 			// In governance mode, users can't overwrite or delete an object
 			// version or alter its lock settings unless they have special
@@ -141,22 +141,22 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 				t, err := objectlock.UTCNowNTP()
 				if err != nil {
 					logger.LogIf(ctx, err)
-					return ErrObjectLocked
+					return ObjectLocked{}
 				}
 
 				if !ret.RetainUntilDate.Before(t) {
-					return ErrObjectLocked
+					return ObjectLocked{}
 				}
-				return ErrNone
+				return nil
 			}
 			// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
 			// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention, the operation will succeed.
 			if checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName) != ErrNone {
-				return ErrAccessDenied
+				return errAuthentication
 			}
 		}
 	}
-	return ErrNone
+	return nil
 }
 
 // enforceRetentionBypassForPut enforces whether an existing object under governance can be overwritten

cmd/bucket-quota.go

@@ -65,10 +65,10 @@ func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, error) {
 	v, err := sys.bucketStorageCache.Get()
 	if err != nil && v != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err))
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
 	}
 	if v == nil {
-		logger.LogIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"))
+		logger.LogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
 	}
 
 	var bui BucketUsageInfo

cmd/bucket-replication-handlers.go

@@ -18,13 +18,18 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"path"
 	"time"
 
+	"github.com/minio/minio-go/v7"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -466,3 +471,148 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 	// Write success response.
 	writeSuccessResponseJSON(w, data)
 }
+
+// ValidateBucketReplicationCredsHandler - validate replication credentials for a bucket.
+// ----------
+func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ValidateBucketReplicationCreds")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL)
+		return
+	}
+
+	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
+		return
+	}
+	replicationConfig, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationConfigurationNotFoundError, err), r.URL)
+		return
+	}
+
+	lockEnabled := false
+	lcfg, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
+	if err != nil {
+		if !errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucket}) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL)
+			return
+		}
+	}
+	if lcfg != nil {
+		lockEnabled = lcfg.Enabled()
+	}
+
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
+	if apiErr != noError {
+		writeErrorResponse(ctx, w, apiErr, r.URL)
+		return
+	}
+
+	// Validate the bucket replication config
+	if err = replicationConfig.Validate(bucket, sameTarget); err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL)
+		return
+	}
+	buf := bytes.Repeat([]byte("a"), 8)
+	for _, rule := range replicationConfig.Rules {
+		if rule.Status == replication.Disabled {
+			continue
+		}
+		clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rule.Destination.Bucket)
+		if clnt == nil {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetNotFoundError, fmt.Errorf("replication config with rule ID %s has a stale target", rule.ID)), r.URL)
+			return
+		}
+		if lockEnabled {
+			lock, _, _, _, err := clnt.GetObjectLockConfig(ctx, clnt.Bucket)
+			if err != nil {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL)
+				return
+			}
+			if lock != objectlock.Enabled {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationDestinationMissingLock, fmt.Errorf("target bucket %s is not object lock enabled", clnt.Bucket)), r.URL)
+				return
+			}
+		}
+		vcfg, err := clnt.GetBucketVersioning(ctx, clnt.Bucket)
+		if err != nil {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL)
+			return
+		}
+		if !vcfg.Enabled() {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetNotVersionedError, fmt.Errorf("target bucket %s is not versioned", clnt.Bucket)), r.URL)
+			return
+		}
+		if sameTarget && bucket == clnt.Bucket {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL)
+			return
+		}
+
+		reader := bytes.NewReader(buf)
+		// fake a PutObject and RemoveObject call to validate permissions
+		c := &minio.Core{Client: clnt.Client}
+		putOpts := minio.PutObjectOptions{
+			Internal: minio.AdvancedPutOptions{
+				SourceVersionID:          mustGetUUID(),
+				ReplicationStatus:        minio.ReplicationStatusReplica,
+				SourceMTime:              time.Now(),
+				ReplicationRequest:       true, // always set this to distinguish between `mc mirror` replication and serverside
+				ReplicationValidityCheck: true, // set this to validate the replication config
+			},
+		}
+		obj := path.Join(minioReservedBucket, globalLocalNodeNameHex, "deleteme")
+		ui, err := c.PutObject(ctx, clnt.Bucket, obj, reader, int64(len(buf)), "", "", putOpts)
+		if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateObject permissions missing for replication user: %w", err)), r.URL)
+			return
+		}
+
+		err = c.RemoveObject(ctx, clnt.Bucket, obj, minio.RemoveObjectOptions{
+			VersionID: ui.VersionID,
+			Internal: minio.AdvancedRemoveOptions{
+				ReplicationDeleteMarker:  true,
+				ReplicationMTime:         time.Now(),
+				ReplicationStatus:        minio.ReplicationStatusReplica,
+				ReplicationRequest:       true, // always set this to distinguish between `mc mirror` replication and serverside
+				ReplicationValidityCheck: true, // set this to validate the replication config
+			},
+		})
+		if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateDelete permissions missing for replication user: %w", err)), r.URL)
+			return
+		}
+		// fake a versioned delete - to ensure deny policies are not in place
+		err = c.RemoveObject(ctx, clnt.Bucket, obj, minio.RemoveObjectOptions{
+			VersionID: ui.VersionID,
+			Internal: minio.AdvancedRemoveOptions{
+				ReplicationDeleteMarker:  false,
+				ReplicationMTime:         time.Now(),
+				ReplicationStatus:        minio.ReplicationStatusReplica,
+				ReplicationRequest:       true, // always set this to distinguish between `mc mirror` replication and serverside
+				ReplicationValidityCheck: true, // set this to validate the replication config
+			},
+		})
+		if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateDelete/s3:DeleteObject permissions missing for replication user: %w", err)), r.URL)
+			return
+		}
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}

cmd/bucket-replication-utils.go

@@ -536,12 +536,18 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 	}
 }
 
+func (ri *ReplicateObjectInfo) getReplicationState() ReplicationState {
+	rs := ri.ObjectInfo.getReplicationState()
+	rs.ReplicateDecisionStr = ri.Dsc.String()
+	return rs
+}
+
 // vID here represents the versionID client specified in request - need to distinguish between delete marker and delete marker deletion
-func (o *ObjectInfo) getReplicationState(dsc string, vID string, heal bool) ReplicationState {
+func (o *ObjectInfo) getReplicationState() ReplicationState {
 	rs := ReplicationState{
 		ReplicationStatusInternal:  o.ReplicationStatusInternal,
 		VersionPurgeStatusInternal: o.VersionPurgeStatusInternal,
-		ReplicateDecisionStr:       dsc,
+		ReplicateDecisionStr:       o.replicationDecision,
 		Targets:                    make(map[string]replication.StatusType),
 		PurgeTargets:               make(map[string]VersionPurgeStatusType),
 		ResetStatusesMap:           make(map[string]string),

cmd/bucket-replication.go

@@ -605,7 +605,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		return
 	}
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete"+tgt.ARN)
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN)
 		sendEvent(eventArgs{
 			BucketName: dobj.Bucket,
 			Object: ObjectInfo{
@@ -634,13 +634,20 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 				IsReplicationReadyForDeleteMarker: true,
 			},
 		})
-		if isErrMethodNotAllowed(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
-			if dobj.VersionID == "" {
+		switch {
+		case isErrMethodNotAllowed(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)):
+			// delete marker already replicated
+			if dobj.VersionID == "" && rinfo.VersionPurgeStatus.Empty() {
 				rinfo.ReplicationStatus = replication.Completed
 				return
 			}
-		}
-		if !isErrObjectNotFound(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
+		case isErrObjectNotFound(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)):
+			// version being purged is already not found on target.
+			if !rinfo.VersionPurgeStatus.Empty() {
+				rinfo.VersionPurgeStatus = Complete
+				return
+			}
+		default:
 			// mark delete marker replication as failed if target cluster not ready to receive
 			// this request yet (object version not replicated yet)
 			if err != nil && !toi.ReplicationReady {
@@ -649,7 +656,6 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 			}
 		}
 	}
-
 	rmErr := tgt.RemoveObject(ctx, tgt.Bucket, dobj.ObjectName, minio.RemoveObjectOptions{
 		VersionID: versionID,
 		Internal: minio.AdvancedRemoveOptions{
@@ -970,7 +976,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
-		logger.LogOnceIf(ctx, err, "get-replication-config"+bucket)
+		logger.LogOnceIf(ctx, err, "get-replication-config-"+bucket)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1048,7 +1054,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		popts := ObjectOptions{
 			MTime:     objInfo.ModTime,
 			VersionID: objInfo.VersionID,
-			EvalMetadataFn: func(oi *ObjectInfo) error {
+			EvalMetadataFn: func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = newReplStatusInternal
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 				oi.UserDefined[xhttp.AmzBucketReplicationStatus] = string(rinfos.ReplicationStatus())
@@ -1060,7 +1066,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 				if objInfo.UserTags != "" {
 					oi.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
 				}
-				return nil
+				return dsc, nil
 			},
 		}
 
@@ -1121,7 +1127,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN), "replication-target-offline-obj"+tgt.ARN)
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN), "replication-target-offline-obj-"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1271,7 +1277,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN), "replication-target-offline-all"+tgt.ARN)
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN), "replication-target-offline-all-"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -2478,7 +2484,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 						ObjectName:            roi.Name,
 						DeleteMarkerVersionID: dmVersionID,
 						VersionID:             versionID,
-						ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
+						ReplicationState:      roi.getReplicationState(),
 						DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 						DeleteMarker:          roi.DeleteMarker,
 					},
@@ -2895,7 +2901,7 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 				ObjectName:            roi.Name,
 				DeleteMarkerVersionID: dmVersionID,
 				VersionID:             versionID,
-				ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
+				ReplicationState:      roi.getReplicationState(),
 				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 				DeleteMarker:          roi.DeleteMarker,
 			},

cmd/crossdomain-xml-handler.go

@@ -30,7 +30,7 @@ const crossDomainXMLEntity = "/crossdomain.xml"
 // When clients request content hosted on a particular source domain and that content make requests
 // directed towards a domain other than its own, the remote domain needs to host a cross-domain
 // policy file that grants access to the source domain, allowing the client to continue the transaction.
-func setCrossDomainPolicy(h http.Handler) http.Handler {
+func setCrossDomainPolicyMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Look for 'crossdomain.xml' in the incoming request.
 		if r.URL.Path == crossDomainXMLEntity {

cmd/crossdomain-xml-handler_test.go

@@ -29,7 +29,7 @@ import (
 func TestCrossXMLHandler(t *testing.T) {
 	// Server initialization.
 	router := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	handler := setCrossDomainPolicy(router)
+	handler := setCrossDomainPolicyMiddleware(router)
 	srv := httptest.NewServer(handler)
 
 	resp, err := http.Get(srv.URL + crossDomainXMLEntity)

cmd/data-scanner-metric.go

@@ -1,3 +1,20 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package cmd
 
 import (

cmd/data-scanner.go

@@ -206,7 +206,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 			go storeDataUsageInBackend(ctx, objAPI, results)
 			err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode)
 			logger.LogIf(ctx, err)
-			res := map[string]string{"cycle": fmt.Sprint(cycleInfo.current)}
+			res := map[string]string{"cycle": strconv.FormatUint(cycleInfo.current, 10)}
 			if err != nil {
 				res["error"] = err.Error()
 			}
@@ -813,11 +813,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			f.newCache.deleteRecursive(thisHash)
 			f.newCache.replaceHashed(thisHash, folder.parent, *flat)
 			total := map[string]string{
-				"objects": fmt.Sprint(flat.Objects),
-				"size":    fmt.Sprint(flat.Size),
+				"objects": strconv.FormatUint(flat.Objects, 10),
+				"size":    strconv.FormatInt(flat.Size, 10),
 			}
 			if flat.Versions > 0 {
-				total["versions"] = fmt.Sprint(flat.Versions)
+				total["versions"] = strconv.FormatUint(flat.Versions, 10)
 			}
 			stop(total)
 		}
@@ -944,7 +944,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 	defer globalScannerMetrics.timeILM(lcEvt.Action)()
 
 	switch lcEvt.Action {
-	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction, lifecycle.DeleteAllVersionsAction:
 		return applyLifecycleAction(lcEvt, lcEventSrc_Scanner, oi), 0
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
 		return applyLifecycleAction(lcEvt, lcEventSrc_Scanner, oi), size
@@ -1133,6 +1133,12 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr loc
 		return event
 	}
 
+	if obj.IsLatest && event.Action == lifecycle.DeleteAllVersionsAction {
+		if lr.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
+			return lifecycle.Event{Action: lifecycle.NoneAction}
+		}
+	}
+
 	switch event.Action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
 		// Defensive code, should never happen
@@ -1232,7 +1238,8 @@ func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) bool
 func applyLifecycleAction(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) (success bool) {
 	switch action := event.Action; action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction,
-		lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction,
+		lifecycle.DeleteAllVersionsAction:
 		success = applyExpiryRule(event, src, obj)
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
 		success = applyTransitionRule(event, src, obj)

cmd/erasure-object.go

@@ -1335,7 +1335,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		if errors.Is(err, errFileNotFound) {
 			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
 		}
-		logger.LogOnceIf(ctx, err, "erasure-object-rename"+bucket+object)
+		logger.LogOnceIf(ctx, err, "erasure-object-rename-"+bucket+"-"+object)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
@@ -1657,6 +1657,22 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 			return objInfo, gerr
 		}
 	}
+	if opts.EvalMetadataFn != nil {
+		dsc, err := opts.EvalMetadataFn(&goi, err)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		if dsc.ReplicateAny() {
+			opts.SetDeleteReplicationState(dsc, opts.VersionID)
+			goi.replicationDecision = opts.DeleteReplication.ReplicateDecisionStr
+		}
+	}
+
+	if opts.EvalRetentionBypassFn != nil {
+		if err := opts.EvalRetentionBypassFn(goi, gerr); err != nil {
+			return ObjectInfo{}, err
+		}
+	}
 
 	if opts.Expiration.Expire {
 		if gerr == nil {
@@ -1765,7 +1781,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		if err = er.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker); err != nil {
 			return objInfo, toObjectErr(err, bucket, object)
 		}
-		return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
+		oi := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
+		oi.replicationDecision = goi.replicationDecision
+		return oi, nil
 	}
 
 	// Delete the object version on all disks.
@@ -1855,7 +1873,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 
 	objInfo := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
 	if opts.EvalMetadataFn != nil {
-		if err := opts.EvalMetadataFn(&objInfo); err != nil {
+		if _, err := opts.EvalMetadataFn(&objInfo, err); err != nil {
 			return ObjectInfo{}, err
 		}
 	}

cmd/erasure-server-pool-decom.go

@@ -245,6 +245,7 @@ func (p *poolMeta) QueueBuckets(idx int, buckets []decomBucketInfo) {
 var (
 	errDecommissionAlreadyRunning = errors.New("decommission is already in progress")
 	errDecommissionComplete       = errors.New("decommission is complete, please remove the servers from command-line")
+	errDecommissionNotStarted     = errors.New("decommission is not in progress")
 )
 
 func (p *poolMeta) Decommission(idx int, pi poolSpaceInfo) error {
@@ -1183,7 +1184,11 @@ func (z *erasureServerPools) Status(ctx context.Context, idx int) (PoolStatus, e
 	poolInfo := z.poolMeta.Pools[idx]
 	if poolInfo.Decommission != nil {
 		poolInfo.Decommission.TotalSize = pi.Total
-		poolInfo.Decommission.CurrentSize = poolInfo.Decommission.StartSize + poolInfo.Decommission.BytesDone
+		if poolInfo.Decommission.Failed || poolInfo.Decommission.Canceled {
+			poolInfo.Decommission.CurrentSize = pi.Free
+		} else {
+			poolInfo.Decommission.CurrentSize = poolInfo.Decommission.StartSize + poolInfo.Decommission.BytesDone
+		}
 	} else {
 		poolInfo.Decommission = &PoolDecommissionInfo{
 			TotalSize:   pi.Total,
@@ -1219,15 +1224,21 @@ func (z *erasureServerPools) DecommissionCancel(ctx context.Context, idx int) (e
 	z.poolMetaMutex.Lock()
 	defer z.poolMetaMutex.Unlock()
 
+	fn := z.decommissionCancelers[idx]
+	if fn == nil {
+		// canceling a decommission before it started return an error.
+		return errDecommissionNotStarted
+	}
+
+	defer fn() // cancel any active thread.
+
 	if z.poolMeta.DecommissionCancel(idx) {
-		if fn := z.decommissionCancelers[idx]; fn != nil {
-			defer fn() // cancel any active thread.
-		}
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
 		globalNotificationSys.ReloadPoolMeta(ctx)
 	}
+
 	return nil
 }
 
@@ -1245,8 +1256,9 @@ func (z *erasureServerPools) DecommissionFailed(ctx context.Context, idx int) (e
 
 	if z.poolMeta.DecommissionFailed(idx) {
 		if fn := z.decommissionCancelers[idx]; fn != nil {
-			defer fn() // cancel any active thread.
-		}
+			defer fn()
+		} // cancel any active thread.
+
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -1269,8 +1281,9 @@ func (z *erasureServerPools) CompleteDecommission(ctx context.Context, idx int)
 
 	if z.poolMeta.DecommissionComplete(idx) {
 		if fn := z.decommissionCancelers[idx]; fn != nil {
-			defer fn() // cancel any active thread.
-		}
+			defer fn()
+		} // cancel any active thread.
+
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}

cmd/erasure-server-pool.go

@@ -1343,7 +1343,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
 		if !isErrBucketNotFound(err) {
-			logger.LogOnceIf(ctx, err, "erasure-list-objects-path"+bucket)
+			logger.LogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
 		}
 		return loi, err
 	}

cmd/erasure.go

@@ -176,7 +176,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 		g.Go(func() error {
 			diskEndpoint := endpoints[index].String()
 			if disks[index] == OfflineDisk {
-				logger.LogOnceIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]), "get-disks-info-offline"+diskEndpoint)
+				logger.LogOnceIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]), "get-disks-info-offline-"+diskEndpoint)
 				disksInfo[index] = madmin.Disk{
 					State:    diskErrToDriveState(errDiskNotFound),
 					Endpoint: diskEndpoint,

cmd/generic-handlers.go

@@ -100,7 +100,7 @@ func isHTTPHeaderSizeTooLarge(header http.Header) bool {
 }
 
 // Limits body and header to specific allowed maximum limits as per S3/MinIO API requirements.
-func setRequestLimitHandler(h http.Handler) http.Handler {
+func setRequestLimitMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
 
@@ -147,7 +147,7 @@ func guessIsBrowserReq(r *http.Request) bool {
 		globalBrowserEnabled && aType == authTypeAnonymous
 }
 
-func setBrowserRedirectHandler(h http.Handler) http.Handler {
+func setBrowserRedirectMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		read := r.Method == http.MethodGet || r.Method == http.MethodHead
 		// Re-direction is handled specifically for browser requests.
@@ -325,7 +325,7 @@ func hasMultipleAuth(r *http.Request) bool {
 
 // requestValidityHandler validates all the incoming paths for
 // any malicious requests.
-func setRequestValidityHandler(h http.Handler) http.Handler {
+func setRequestValidityMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
 
@@ -425,10 +425,10 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 	})
 }
 
-// setBucketForwardingHandler middleware forwards the path style requests
+// setBucketForwardingMiddleware middleware forwards the path style requests
 // on a bucket to the right bucket location, bucket to IP configuration
 // is obtained from centralized etcd configuration service.
-func setBucketForwardingHandler(h http.Handler) http.Handler {
+func setBucketForwardingMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if globalDNSConfig == nil || !globalBucketFederation ||
 			guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
@@ -493,9 +493,9 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 	})
 }
 
-// addCustomHeaders adds various HTTP(S) response headers.
+// addCustomHeadersMiddleware adds various HTTP(S) response headers.
 // Security Headers enable various security protections behaviors in the client's browser.
-func addCustomHeaders(h http.Handler) http.Handler {
+func addCustomHeadersMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		header := w.Header()
 		header.Set("X-XSS-Protection", "1; mode=block")                                // Prevents against XSS attacks
@@ -539,9 +539,9 @@ func setCriticalErrorHandler(h http.Handler) http.Handler {
 	})
 }
 
-// setUploadForwardingHandler middleware forwards multiparts requests
+// setUploadForwardingMiddleware middleware forwards multiparts requests
 // in a site replication setup to peer that initiated the upload
-func setUploadForwardingHandler(h http.Handler) http.Handler {
+func setUploadForwardingMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if !globalSiteReplicationSys.isEnabled() ||
 			guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||

cmd/generic-handlers_test.go

@@ -155,7 +155,7 @@ func TestSSETLSHandler(t *testing.T) {
 		r.Header = test.Header
 		r.URL = test.URL
 
-		h := setRequestValidityHandler(okHandler)
+		h := setRequestValidityMiddleware(okHandler)
 		h.ServeHTTP(w, r)
 
 		switch {

cmd/handler-utils.go

@@ -267,6 +267,20 @@ func trimAwsChunkedContentEncoding(contentEnc string) (trimmedContentEnc string)
 	return strings.Join(newEncs, ",")
 }
 
+func collectInternodeStats(f http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		f.ServeHTTP(w, r)
+
+		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
+		if !ok || tc == nil {
+			return
+		}
+
+		globalConnStats.incInternodeInputBytes(int64(tc.RequestRecorder.Size()))
+		globalConnStats.incInternodeOutputBytes(int64(tc.ResponseRecorder.Size()))
+	}
+}
+
 func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
@@ -297,21 +311,6 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 		}
 
 		if tc != nil {
-			if strings.HasPrefix(r.URL.Path, storageRESTPrefix) ||
-				strings.HasPrefix(r.URL.Path, peerRESTPrefix) ||
-				strings.HasPrefix(r.URL.Path, peerS3Prefix) ||
-				strings.HasPrefix(r.URL.Path, lockRESTPrefix) {
-				globalConnStats.incInputBytes(int64(tc.RequestRecorder.Size()))
-				globalConnStats.incOutputBytes(int64(tc.ResponseRecorder.Size()))
-				return
-			}
-
-			if strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
-				globalConnStats.incAdminInputBytes(int64(tc.RequestRecorder.Size()))
-				globalConnStats.incAdminOutputBytes(int64(tc.ResponseRecorder.Size()))
-				return
-			}
-
 			globalHTTPStats.updateStats(api, tc.ResponseRecorder)
 			globalConnStats.incS3InputBytes(int64(tc.RequestRecorder.Size()))
 			globalConnStats.incS3OutputBytes(int64(tc.ResponseRecorder.Size()))

cmd/http-stats.go

@@ -26,93 +26,69 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-// ConnStats - Network statistics
+// connStats - Network statistics
 // Count total input/output transferred bytes during
 // the server's life.
-type ConnStats struct {
-	totalInputBytes  uint64
-	totalOutputBytes uint64
-	s3InputBytes     uint64
-	s3OutputBytes    uint64
-	adminInputBytes  uint64
-	adminOutputBytes uint64
+type connStats struct {
+	internodeInputBytes  uint64
+	internodeOutputBytes uint64
+	s3InputBytes         uint64
+	s3OutputBytes        uint64
 }
 
 // Increase internode total input bytes
-func (s *ConnStats) incInputBytes(n int64) {
-	atomic.AddUint64(&s.totalInputBytes, uint64(n))
+func (s *connStats) incInternodeInputBytes(n int64) {
+	atomic.AddUint64(&s.internodeInputBytes, uint64(n))
 }
 
 // Increase internode total output bytes
-func (s *ConnStats) incOutputBytes(n int64) {
-	atomic.AddUint64(&s.totalOutputBytes, uint64(n))
+func (s *connStats) incInternodeOutputBytes(n int64) {
+	atomic.AddUint64(&s.internodeOutputBytes, uint64(n))
 }
 
 // Return internode total input bytes
-func (s *ConnStats) getTotalInputBytes() uint64 {
-	return atomic.LoadUint64(&s.totalInputBytes)
+func (s *connStats) getInternodeInputBytes() uint64 {
+	return atomic.LoadUint64(&s.internodeInputBytes)
 }
 
 // Return total output bytes
-func (s *ConnStats) getTotalOutputBytes() uint64 {
-	return atomic.LoadUint64(&s.totalOutputBytes)
+func (s *connStats) getInternodeOutputBytes() uint64 {
+	return atomic.LoadUint64(&s.internodeOutputBytes)
 }
 
 // Increase S3 total input bytes
-func (s *ConnStats) incS3InputBytes(n int64) {
+func (s *connStats) incS3InputBytes(n int64) {
 	atomic.AddUint64(&s.s3InputBytes, uint64(n))
 }
 
 // Increase S3 total output bytes
-func (s *ConnStats) incS3OutputBytes(n int64) {
+func (s *connStats) incS3OutputBytes(n int64) {
 	atomic.AddUint64(&s.s3OutputBytes, uint64(n))
 }
 
 // Return S3 total input bytes
-func (s *ConnStats) getS3InputBytes() uint64 {
+func (s *connStats) getS3InputBytes() uint64 {
 	return atomic.LoadUint64(&s.s3InputBytes)
 }
 
 // Return S3 total output bytes
-func (s *ConnStats) getS3OutputBytes() uint64 {
+func (s *connStats) getS3OutputBytes() uint64 {
 	return atomic.LoadUint64(&s.s3OutputBytes)
 }
 
-// Increase Admin total input bytes
-func (s *ConnStats) incAdminInputBytes(n int64) {
-	atomic.AddUint64(&s.adminInputBytes, uint64(n))
-}
-
-// Increase Admin total output bytes
-func (s *ConnStats) incAdminOutputBytes(n int64) {
-	atomic.AddUint64(&s.adminOutputBytes, uint64(n))
-}
-
-// Return Admin total input bytes
-func (s *ConnStats) getAdminInputBytes() uint64 {
-	return atomic.LoadUint64(&s.adminInputBytes)
-}
-
-// Return Admin total output bytes
-func (s *ConnStats) getAdminOutputBytes() uint64 {
-	return atomic.LoadUint64(&s.adminOutputBytes)
-}
-
 // Return connection stats (total input/output bytes and total s3 input/output bytes)
-func (s *ConnStats) toServerConnStats() ServerConnStats {
-	return ServerConnStats{
-		TotalInputBytes:  s.getTotalInputBytes(),  // Traffic internode received
-		TotalOutputBytes: s.getTotalOutputBytes(), // Traffic internode sent
-		S3InputBytes:     s.getS3InputBytes(),     // Traffic S3 received
-		S3OutputBytes:    s.getS3OutputBytes(),    // Traffic S3 sent
-		AdminInputBytes:  s.getAdminInputBytes(),  // Traffic admin calls received
-		AdminOutputBytes: s.getAdminOutputBytes(), // Traffic admin calls sent
+func (s *connStats) toServerConnStats() serverConnStats {
+	return serverConnStats{
+		internodeInputBytes:  s.getInternodeInputBytes(),  // Traffic internode received
+		internodeOutputBytes: s.getInternodeOutputBytes(), // Traffic internode sent
+		s3InputBytes:         s.getS3InputBytes(),         // Traffic S3 received
+		s3OutputBytes:        s.getS3OutputBytes(),        // Traffic S3 sent
 	}
 }
 
 // Prepare new ConnStats structure
-func newConnStats() *ConnStats {
-	return &ConnStats{}
+func newConnStats() *connStats {
+	return &connStats{}
 }
 
 type bucketS3RXTX struct {
@@ -260,28 +236,28 @@ func (s *bucketConnStats) incS3OutputBytes(bucket string, n int64) {
 	s.stats[bucket] = stats
 }
 
-// Return S3 total input bytes for input bucket
-func (s *bucketConnStats) getS3InputBytes(bucket string) uint64 {
-	s.RLock()
-	defer s.RUnlock()
-
-	stats := s.stats[bucket]
-	if stats == nil {
-		return 0
-	}
-	return stats.s3InputBytes
+type inOutBytes struct {
+	In  uint64
+	Out uint64
 }
 
-// Return S3 total output bytes
-func (s *bucketConnStats) getS3OutputBytes(bucket string) uint64 {
+// Return S3 total input bytes for input bucket
+func (s *bucketConnStats) getS3InOutBytes() map[string]inOutBytes {
 	s.RLock()
 	defer s.RUnlock()
 
-	stats := s.stats[bucket]
-	if stats == nil {
-		return 0
+	if len(s.stats) == 0 {
+		return nil
 	}
-	return stats.s3OutputBytes
+
+	bucketStats := make(map[string]inOutBytes, len(s.stats))
+	for k, v := range s.stats {
+		bucketStats[k] = inOutBytes{
+			In:  v.s3InputBytes,
+			Out: v.s3OutputBytes,
+		}
+	}
+	return bucketStats
 }
 
 // delete metrics once bucket is deleted.

cmd/http-tracer.go

@@ -65,7 +65,7 @@ func getOpName(name string) (op string) {
 
 // If trace is enabled, execute the request if it is traced by other handlers
 // otherwise, generate a trace event with request information but no response.
-func httpTracer(h http.Handler) http.Handler {
+func httpTracerMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Setup a http request response recorder - this is needed for
 		// http stats requests and audit if enabled.
@@ -77,7 +77,7 @@ func httpTracer(h http.Handler) http.Handler {
 
 		// Create tracing data structure and associate it to the request context
 		tc := mcontext.TraceCtxt{
-			AmzReqID:         r.Header.Get(xhttp.AmzRequestID),
+			AmzReqID:         w.Header().Get(xhttp.AmzRequestID),
 			RequestRecorder:  reqRecorder,
 			ResponseRecorder: respRecorder,
 		}

cmd/lock-rest-server.go

@@ -244,18 +244,22 @@ func lockMaintenance(ctx context.Context) {
 
 // registerLockRESTHandlers - register lock rest router.
 func registerLockRESTHandlers(router *mux.Router) {
+	h := func(f http.HandlerFunc) http.HandlerFunc {
+		return collectInternodeStats(httpTraceHdrs(f))
+	}
+
 	lockServer := &lockRESTServer{
 		ll: newLocker(),
 	}
 
 	subrouter := router.PathPrefix(lockRESTPrefix).Subrouter()
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodHealth).HandlerFunc(httpTraceHdrs(lockServer.HealthHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRefresh).HandlerFunc(httpTraceHdrs(lockServer.RefreshHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(lockServer.LockHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(lockServer.RLockHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(httpTraceHdrs(lockServer.RUnlockHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodForceUnlock).HandlerFunc(httpTraceAll(lockServer.ForceUnlockHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodHealth).HandlerFunc(h(lockServer.HealthHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRefresh).HandlerFunc(h(lockServer.RefreshHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(h(lockServer.LockHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(h(lockServer.RLockHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(h(lockServer.UnlockHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(h(lockServer.RUnlockHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodForceUnlock).HandlerFunc(h(lockServer.ForceUnlockHandler))
 
 	globalLockServer = lockServer.ll
 

cmd/metrics-v2.go

@@ -964,7 +964,7 @@ func getS3TTFBDistributionMD() MetricDescription {
 		Subsystem: timeSubsystem,
 		Name:      ttfbDistribution,
 		Help:      "Distribution of the time to first byte across API calls",
-		Type:      histogramMetric,
+		Type:      gaugeMetric,
 	}
 }
 
@@ -1990,6 +1990,67 @@ func getHTTPMetrics() *MetricsGroup {
 				VariableLabels: map[string]string{"api": api},
 			})
 		}
+
+		for bucket, inOut := range globalBucketConnStats.getS3InOutBytes() {
+			recvBytes := inOut.In
+			if recvBytes > 0 {
+				metrics = append(metrics, Metric{
+					Description:    getBucketTrafficReceivedBytes(),
+					Value:          float64(recvBytes),
+					VariableLabels: map[string]string{"bucket": bucket},
+				})
+			}
+			sentBytes := inOut.Out
+			if sentBytes > 0 {
+				metrics = append(metrics, Metric{
+					Description:    getBucketTrafficSentBytes(),
+					Value:          float64(sentBytes),
+					VariableLabels: map[string]string{"bucket": bucket},
+				})
+			}
+
+			httpStats := globalBucketHTTPStats.load(bucket)
+			for k, v := range httpStats.currentS3Requests.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsInFlightMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS3Requests.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsTotalMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS3Canceled.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsCanceledMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS34xxErrors.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3Requests4xxErrorsMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS35xxErrors.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3Requests5xxErrorsMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+		}
+
 		return
 	})
 	return mg
@@ -2018,20 +2079,20 @@ func getNetworkMetrics() *MetricsGroup {
 			})
 			metrics = append(metrics, Metric{
 				Description: getInterNodeSentBytesMD(),
-				Value:       float64(connStats.TotalOutputBytes),
+				Value:       float64(connStats.internodeOutputBytes),
 			})
 			metrics = append(metrics, Metric{
 				Description: getInterNodeReceivedBytesMD(),
-				Value:       float64(connStats.TotalInputBytes),
+				Value:       float64(connStats.internodeInputBytes),
 			})
 		}
 		metrics = append(metrics, Metric{
 			Description: getS3SentBytesMD(),
-			Value:       float64(connStats.S3OutputBytes),
+			Value:       float64(connStats.s3OutputBytes),
 		})
 		metrics = append(metrics, Metric{
 			Description: getS3ReceivedBytesMD(),
-			Value:       float64(connStats.S3InputBytes),
+			Value:       float64(connStats.s3InputBytes),
 		})
 		return
 	})
@@ -2104,65 +2165,6 @@ func getBucketUsageMetrics() *MetricsGroup {
 				})
 			}
 
-			recvBytes := globalBucketConnStats.getS3InputBytes(bucket)
-			if recvBytes > 0 {
-				metrics = append(metrics, Metric{
-					Description:    getBucketTrafficReceivedBytes(),
-					Value:          float64(recvBytes),
-					VariableLabels: map[string]string{"bucket": bucket},
-				})
-			}
-
-			sentBytes := globalBucketConnStats.getS3OutputBytes(bucket)
-			if sentBytes > 0 {
-				metrics = append(metrics, Metric{
-					Description:    getBucketTrafficSentBytes(),
-					Value:          float64(sentBytes),
-					VariableLabels: map[string]string{"bucket": bucket},
-				})
-			}
-
-			httpStats := globalBucketHTTPStats.load(bucket)
-			for k, v := range httpStats.currentS3Requests.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3RequestsInFlightMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS3Requests.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3RequestsTotalMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS3Canceled.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3RequestsCanceledMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS34xxErrors.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3Requests4xxErrorsMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS35xxErrors.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3Requests5xxErrorsMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
 			if stats.hasReplicationUsage() {
 				for arn, stat := range stats.Stats {
 					metrics = append(metrics, Metric{

cmd/metrics.go

@@ -322,7 +322,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total number of bytes sent to the other peer nodes by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
-		float64(connStats.TotalOutputBytes),
+		float64(connStats.internodeOutputBytes),
 	)
 
 	ch <- prometheus.MustNewConstMetric(
@@ -331,7 +331,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total number of internode bytes received by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
-		float64(connStats.TotalInputBytes),
+		float64(connStats.internodeInputBytes),
 	)
 
 	// Network Sent/Received Bytes (Outbound)
@@ -341,7 +341,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total number of s3 bytes sent by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
-		float64(connStats.S3OutputBytes),
+		float64(connStats.s3OutputBytes),
 	)
 
 	ch <- prometheus.MustNewConstMetric(
@@ -350,7 +350,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total number of s3 bytes received by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
-		float64(connStats.S3InputBytes),
+		float64(connStats.s3InputBytes),
 	)
 }
 

cmd/notification.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"sync"
@@ -87,6 +88,8 @@ func (g *NotificationGroup) Wait() []NotificationPeerErr {
 // The first call to return a non-nil error will be
 // collected in errs slice and returned by Wait().
 func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, addr xnet.Host) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
 	g.wg.Add(1)
 
 	go func() {
@@ -103,9 +106,9 @@ func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, a
 					ctx := logger.SetReqInfo(ctx, reqInfo)
 					logger.LogIf(ctx, err)
 				}
-				// Wait for one second and no need wait after last attempt.
+				// Wait for a minimum of 100ms and dynamically increase this based on number of attempts.
 				if i < g.retryCount-1 {
-					time.Sleep(1 * time.Second)
+					time.Sleep(100*time.Millisecond + time.Duration(r.Float64()*float64(time.Second)))
 				}
 				continue
 			}
@@ -804,7 +807,7 @@ func (sys *NotificationSys) addNodeErr(nodeInfo madmin.NodeInfo, peerClient *pee
 	addr := peerClient.host.String()
 	reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
 	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-	logger.LogOnceIf(ctx, err, "add-node-err"+addr)
+	logger.LogOnceIf(ctx, err, "add-node-err-"+addr)
 	nodeInfo.SetAddr(addr)
 	nodeInfo.SetError(err.Error())
 }

cmd/object-api-datatypes.go

@@ -185,6 +185,7 @@ type ObjectInfo struct {
 	VersionPurgeStatusInternal string
 	VersionPurgeStatus         VersionPurgeStatusType
 
+	replicationDecision string // internal representation of replication decision for use by DeleteObject handler
 	// The total count of all versions of this object
 	NumVersions int
 	//  The modtime of the successor object version if any

cmd/object-api-errors.go

@@ -735,3 +735,15 @@ func isErrInvalidRange(err error) bool {
 	_, ok := err.(InvalidRange)
 	return ok
 }
+
+// ReplicationPermissionCheck - Check if error type is ReplicationPermissionCheck.
+type ReplicationPermissionCheck struct{}
+
+func (e ReplicationPermissionCheck) Error() string {
+	return "Replication permission validation requests cannot be completed"
+}
+
+func isReplicationPermissionCheck(err error) bool {
+	_, ok := err.(ReplicationPermissionCheck)
+	return ok
+}

cmd/object-api-interface.go

@@ -35,8 +35,11 @@ import (
 // CheckPreconditionFn returns true if precondition check failed.
 type CheckPreconditionFn func(o ObjectInfo) bool
 
-// EvalMetadataFn validates input objInfo and returns an updated metadata
-type EvalMetadataFn func(o *ObjectInfo) error
+// EvalMetadataFn validates input objInfo and GetObjectInfo error and returns an updated metadata and replication decision if any
+type EvalMetadataFn func(o *ObjectInfo, gerr error) (ReplicateDecision, error)
+
+// EvalRetentionBypassFn validates input objInfo and GetObjectInfo error and returns an error if retention bypass is not allowed.
+type EvalRetentionBypassFn func(o ObjectInfo, gerr error) error
 
 // GetObjectInfoFn is the signature of GetObjectInfo function.
 type GetObjectInfoFn func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error)
@@ -100,7 +103,8 @@ type ObjectOptions struct {
 
 	InclFreeVersions bool
 
-	MetadataChg bool // is true if it is a metadata update operation.
+	MetadataChg           bool                  // is true if it is a metadata update operation.
+	EvalRetentionBypassFn EvalRetentionBypassFn // only set for enforcing retention bypass on DeleteObject.
 }
 
 // ExpirationOptions represents object options for object expiration at objectLayer.
@@ -182,6 +186,16 @@ func (o *ObjectOptions) PutReplicationState() (r ReplicationState) {
 	return
 }
 
+// SetEvalMetadataFn sets the metadata evaluation function
+func (o *ObjectOptions) SetEvalMetadataFn(f EvalMetadataFn) {
+	o.EvalMetadataFn = f
+}
+
+// SetEvalRetentionBypassFn sets the retention bypass function
+func (o *ObjectOptions) SetEvalRetentionBypassFn(f EvalRetentionBypassFn) {
+	o.EvalRetentionBypassFn = f
+}
+
 // ObjectLayer implements primitives for object API layer.
 type ObjectLayer interface {
 	// Locking operations on object.

cmd/object-api-utils.go

@@ -1166,7 +1166,7 @@ func hasSpaceFor(di []*DiskInfo, size int64) (bool, error) {
 	var total uint64
 	var nDisks int
 	for _, disk := range di {
-		if disk == nil || disk.Total == 0 || (disk.FreeInodes < diskMinInodes && disk.UsedInodes > 0) {
+		if disk == nil || disk.Total == 0 {
 			// Disk offline, no inodes or something else is wrong.
 			continue
 		}
@@ -1175,16 +1175,20 @@ func hasSpaceFor(di []*DiskInfo, size int64) (bool, error) {
 		available += disk.Total - disk.Used
 	}
 
-	if nDisks < len(di)/2 {
+	if nDisks < len(di)/2 || nDisks <= 0 {
 		return false, fmt.Errorf("not enough online disks to calculate the available space, expected (%d)/(%d)", (len(di)/2)+1, nDisks)
 	}
 
 	// Check we have enough on each disk, ignoring diskFillFraction.
 	perDisk := size / int64(nDisks)
 	for _, disk := range di {
-		if disk == nil || disk.Total == 0 || (disk.FreeInodes < diskMinInodes && disk.UsedInodes > 0) {
+		if disk == nil || disk.Total == 0 {
 			continue
 		}
+		if disk.FreeInodes < diskMinInodes && disk.UsedInodes > 0 {
+			// We have an inode count, but not enough inodes.
+			return false, nil
+		}
 		if int64(disk.Free) <= perDisk {
 			return false, nil
 		}

cmd/object-handlers.go

@@ -1707,6 +1707,12 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
+	if _, ok := r.Header[xhttp.MinIOSourceReplicationCheck]; ok {
+		// requests to just validate replication settings and permissions are not allowed to write data
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationPermissionCheckError), r.URL)
+		return
+	}
+
 	if err := enforceBucketQuotaHard(ctx, bucket, size); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -2297,6 +2303,11 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
+	if _, ok := r.Header[xhttp.MinIOSourceReplicationCheck]; ok {
+		// requests to just validate replication settings and permissions are not allowed to delete data
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationPermissionCheckError), r.URL)
+		return
+	}
 
 	replica := r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String()
 	if replica {
@@ -2326,33 +2337,29 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
-
 	os := newObjSweeper(bucket, object).WithVersion(opts.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended)
-	// Mutations of objects on versioning suspended buckets
-	// affect its null version. Through opts below we select
-	// the null version's remote object to delete if
-	// transitioned.
-	goiOpts := os.GetOpts()
-	goi, gerr := getObjectInfo(ctx, bucket, object, goiOpts)
-	if gerr == nil {
-		os.SetTransitionState(goi.TransitionedObject)
-	}
 
-	dsc := checkReplicateDelete(ctx, bucket, ObjectToDelete{
-		ObjectV: ObjectV{
-			ObjectName: object,
-			VersionID:  opts.VersionID,
-		},
-	}, goi, opts, gerr)
+	opts.SetEvalMetadataFn(func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
+		if replica { // no need to check replication on receiver
+			return dsc, nil
+		}
+		dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{
+			ObjectV: ObjectV{
+				ObjectName: object,
+				VersionID:  opts.VersionID,
+			},
+		}, *oi, opts, gerr)
+		// Mutations of objects on versioning suspended buckets
+		// affect its null version. Through opts below we select
+		// the null version's remote object to delete if
+		// transitioned.
+		if gerr == nil {
+			os.SetTransitionState(oi.TransitionedObject)
+		}
+		return dsc, nil
+	})
 
 	vID := opts.VersionID
-	if dsc.ReplicateAny() {
-		opts.SetDeleteReplicationState(dsc, opts.VersionID)
-	}
 	if replica {
 		opts.SetReplicaStatus(replication.Replica)
 		if opts.VersionPurgeStatus().Empty() {
@@ -2360,25 +2367,21 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			vID = ""
 		}
 	}
-
-	apiErr := ErrNone
-	if vID != "" {
-		apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
-			ObjectV: ObjectV{
-				ObjectName: object,
-				VersionID:  vID,
-			},
-		}, goi, gerr)
-		if apiErr != ErrNone && apiErr != ErrNoSuchKey {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL)
-			return
+	opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) (err error) {
+		err = nil
+		if vID != "" {
+			err := enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
+				ObjectV: ObjectV{
+					ObjectName: object,
+					VersionID:  vID,
+				},
+			}, goi, gerr)
+			if err != nil && !isErrObjectNotFound(err) {
+				return err
+			}
 		}
-	}
-
-	if apiErr == ErrNoSuchKey {
-		writeSuccessNoContent(w)
 		return
-	}
+	})
 
 	deleteObject := objectAPI.DeleteObject
 	if api.CacheAPI() != nil {
@@ -2393,6 +2396,12 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
+		if isErrObjectNotFound(err) {
+			writeSuccessNoContent(w)
+			return
+		}
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
 	}
 
 	if objInfo.Name == "" {
@@ -2419,7 +2428,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		Host:         handlers.GetSourceIP(r),
 	})
 
-	if dsc.ReplicateAny() {
+	if objInfo.ReplicationStatus == replication.Pending || objInfo.VersionPurgeStatus == Pending {
 		dmVersionID := ""
 		versionID := ""
 		if objInfo.DeleteMarker {
@@ -2434,7 +2443,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 				DeleteMarkerVersionID: dmVersionID,
 				DeleteMarkerMTime:     DeleteMarkerMTime{objInfo.ModTime},
 				DeleteMarker:          objInfo.DeleteMarker,
-				ReplicationState:      objInfo.getReplicationState(dsc.String(), opts.VersionID, false),
+				ReplicationState:      objInfo.getReplicationState(),
 			},
 			Bucket:    bucket,
 			EventType: ReplicateIncomingDelete,
@@ -2505,7 +2514,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 	popts := ObjectOptions{
 		MTime:     opts.MTime,
 		VersionID: opts.VersionID,
-		EvalMetadataFn: func(oi *ObjectInfo) error {
+		EvalMetadataFn: func(oi *ObjectInfo, gerr error) (ReplicateDecision, error) {
 			oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = strings.ToUpper(string(legalHold.Status))
 			oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
 
@@ -2514,7 +2523,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
 			}
-			return nil
+			return dsc, nil
 		},
 	}
 
@@ -2666,9 +2675,9 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 	popts := ObjectOptions{
 		MTime:     opts.MTime,
 		VersionID: opts.VersionID,
-		EvalMetadataFn: func(oi *ObjectInfo) error {
+		EvalMetadataFn: func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
 			if err := enforceRetentionBypassForPut(ctx, r, *oi, objRetention, cred, owner); err != nil {
-				return err
+				return dsc, err
 			}
 			if objRetention.Mode.Valid() {
 				oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(objRetention.Mode)
@@ -2678,12 +2687,12 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 				oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = ""
 			}
 			oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano)
-			dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(*oi, replication.MetadataReplicationType, opts))
+			dsc = mustReplicate(ctx, bucket, object, getMustReplicateOptions(*oi, replication.MetadataReplicationType, opts))
 			if dsc.ReplicateAny() {
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
 			}
-			return nil
+			return dsc, nil
 		},
 	}
 

cmd/peer-rest-server.go

@@ -1359,56 +1359,60 @@ func (s *peerRESTServer) NetSpeedTestHandler(w http.ResponseWriter, r *http.Requ
 
 // registerPeerRESTHandlers - register peer rest router.
 func registerPeerRESTHandlers(router *mux.Router) {
+	h := func(f http.HandlerFunc) http.HandlerFunc {
+		return collectInternodeStats(httpTraceHdrs(f))
+	}
+
 	server := &peerRESTServer{}
 	subrouter := router.PathPrefix(peerRESTPrefix).Subrouter()
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodHealth).HandlerFunc(httpTraceHdrs(server.HealthHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(httpTraceHdrs(server.GetLocksHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLocalStorageInfo).HandlerFunc(httpTraceHdrs(server.LocalStorageInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcInfo).HandlerFunc(httpTraceHdrs(server.GetProcInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemInfo).HandlerFunc(httpTraceHdrs(server.GetMemInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMetrics).HandlerFunc(httpTraceHdrs(server.GetMetricsHandler)).Queries(restQueries(peerRESTMetricsTypes)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysErrors).HandlerFunc(httpTraceHdrs(server.GetSysErrorsHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysServices).HandlerFunc(httpTraceHdrs(server.GetSysServicesHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysConfig).HandlerFunc(httpTraceHdrs(server.GetSysConfigHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfo).HandlerFunc(httpTraceHdrs(server.GetOSInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwInfo).HandlerFunc(httpTraceHdrs(server.GetPartitionsHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCPUInfo).HandlerFunc(httpTraceHdrs(server.GetCPUsHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetAllBucketStats).HandlerFunc(httpTraceHdrs(server.GetAllBucketStatsHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteBucketMetadata).HandlerFunc(httpTraceHdrs(server.DeleteBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadBucketMetadata).HandlerFunc(httpTraceHdrs(server.LoadBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetBucketStats).HandlerFunc(httpTraceHdrs(server.GetBucketStatsHandler)).Queries(restQueries(peerRESTBucket)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSignalService).HandlerFunc(httpTraceHdrs(server.SignalServiceHandler)).Queries(restQueries(peerRESTSignal)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadBinary).HandlerFunc(httpTraceHdrs(server.DownloadBinaryHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCommitBinary).HandlerFunc(httpTraceHdrs(server.CommitBinaryHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeletePolicy).HandlerFunc(httpTraceAll(server.DeletePolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicy).HandlerFunc(httpTraceAll(server.LoadPolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicyMapping).HandlerFunc(httpTraceAll(server.LoadPolicyMappingHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteUser).HandlerFunc(httpTraceAll(server.DeleteUserHandler)).Queries(restQueries(peerRESTUser)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteServiceAccount).HandlerFunc(httpTraceAll(server.DeleteServiceAccountHandler)).Queries(restQueries(peerRESTUser)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadUser).HandlerFunc(httpTraceAll(server.LoadUserHandler)).Queries(restQueries(peerRESTUser, peerRESTUserTemp)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadServiceAccount).HandlerFunc(httpTraceAll(server.LoadServiceAccountHandler)).Queries(restQueries(peerRESTUser)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadGroup).HandlerFunc(httpTraceAll(server.LoadGroupHandler)).Queries(restQueries(peerRESTGroup)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodHealth).HandlerFunc(h(server.HealthHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(h(server.GetLocksHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(h(server.ServerInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLocalStorageInfo).HandlerFunc(h(server.LocalStorageInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcInfo).HandlerFunc(h(server.GetProcInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemInfo).HandlerFunc(h(server.GetMemInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMetrics).HandlerFunc(h(server.GetMetricsHandler)).Queries(restQueries(peerRESTMetricsTypes)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysErrors).HandlerFunc(h(server.GetSysErrorsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysServices).HandlerFunc(h(server.GetSysServicesHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysConfig).HandlerFunc(h(server.GetSysConfigHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfo).HandlerFunc(h(server.GetOSInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwInfo).HandlerFunc(h(server.GetPartitionsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCPUInfo).HandlerFunc(h(server.GetCPUsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetAllBucketStats).HandlerFunc(h(server.GetAllBucketStatsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteBucketMetadata).HandlerFunc(h(server.DeleteBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadBucketMetadata).HandlerFunc(h(server.LoadBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetBucketStats).HandlerFunc(h(server.GetBucketStatsHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSignalService).HandlerFunc(h(server.SignalServiceHandler)).Queries(restQueries(peerRESTSignal)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadBinary).HandlerFunc(h(server.DownloadBinaryHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCommitBinary).HandlerFunc(h(server.CommitBinaryHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeletePolicy).HandlerFunc(h(server.DeletePolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicy).HandlerFunc(h(server.LoadPolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicyMapping).HandlerFunc(h(server.LoadPolicyMappingHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteUser).HandlerFunc(h(server.DeleteUserHandler)).Queries(restQueries(peerRESTUser)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteServiceAccount).HandlerFunc(h(server.DeleteServiceAccountHandler)).Queries(restQueries(peerRESTUser)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadUser).HandlerFunc(h(server.LoadUserHandler)).Queries(restQueries(peerRESTUser, peerRESTUserTemp)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadServiceAccount).HandlerFunc(h(server.LoadServiceAccountHandler)).Queries(restQueries(peerRESTUser)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadGroup).HandlerFunc(h(server.LoadGroupHandler)).Queries(restQueries(peerRESTGroup)...)
 
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStartProfiling).HandlerFunc(httpTraceAll(server.StartProfilingHandler)).Queries(restQueries(peerRESTProfiler)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadProfilingData).HandlerFunc(httpTraceHdrs(server.DownloadProfilingDataHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStartProfiling).HandlerFunc(h(server.StartProfilingHandler)).Queries(restQueries(peerRESTProfiler)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadProfilingData).HandlerFunc(h(server.DownloadProfilingDataHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodTrace).HandlerFunc(server.TraceHandler)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodListen).HandlerFunc(httpTraceHdrs(server.ListenHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodListen).HandlerFunc(h(server.ListenHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLog).HandlerFunc(server.ConsoleLogHandler)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocalDiskIDs).HandlerFunc(httpTraceHdrs(server.GetLocalDiskIDs))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetBandwidth).HandlerFunc(httpTraceHdrs(server.GetBandwidth))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetMetacacheListing).HandlerFunc(httpTraceHdrs(server.GetMetacacheListingHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodUpdateMetacacheListing).HandlerFunc(httpTraceHdrs(server.UpdateMetacacheListingHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetPeerMetrics).HandlerFunc(httpTraceHdrs(server.GetPeerMetrics))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedTest).HandlerFunc(httpTraceHdrs(server.SpeedTestHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveSpeedTest).HandlerFunc(httpTraceHdrs(server.DriveSpeedTestHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(httpTraceHdrs(server.NetSpeedTestHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDevNull).HandlerFunc(httpTraceHdrs(server.DevNull))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadSiteReplicationConfig).HandlerFunc(httpTraceHdrs(server.ReloadSiteReplicationConfigHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadPoolMeta).HandlerFunc(httpTraceHdrs(server.ReloadPoolMetaHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadRebalanceMeta).HandlerFunc(httpTraceHdrs(server.LoadRebalanceMetaHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStopRebalance).HandlerFunc(httpTraceHdrs(server.StopRebalanceHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLastDayTierStats).HandlerFunc(httpTraceHdrs(server.GetLastDayTierStatsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocalDiskIDs).HandlerFunc(h(server.GetLocalDiskIDs))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetBandwidth).HandlerFunc(h(server.GetBandwidth))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetMetacacheListing).HandlerFunc(h(server.GetMetacacheListingHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodUpdateMetacacheListing).HandlerFunc(h(server.UpdateMetacacheListingHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetPeerMetrics).HandlerFunc(h(server.GetPeerMetrics))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(h(server.LoadTransitionTierConfigHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedTest).HandlerFunc(h(server.SpeedTestHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveSpeedTest).HandlerFunc(h(server.DriveSpeedTestHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(h(server.NetSpeedTestHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDevNull).HandlerFunc(h(server.DevNull))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadSiteReplicationConfig).HandlerFunc(h(server.ReloadSiteReplicationConfigHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadPoolMeta).HandlerFunc(h(server.ReloadPoolMetaHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadRebalanceMeta).HandlerFunc(h(server.LoadRebalanceMetaHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStopRebalance).HandlerFunc(h(server.StopRebalanceHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLastDayTierStats).HandlerFunc(h(server.GetLastDayTierStatsHandler))
 }

cmd/peer-s3-server.go

@@ -333,9 +333,13 @@ func registerPeerS3Handlers(router *mux.Router) {
 	server := &peerS3Server{}
 	subrouter := router.PathPrefix(peerS3Prefix).Subrouter()
 
-	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodHealth).HandlerFunc(httpTraceHdrs(server.HealthHandler))
-	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodMakeBucket).HandlerFunc(httpTraceHdrs(server.MakeBucketHandler))
-	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodDeleteBucket).HandlerFunc(httpTraceHdrs(server.DeleteBucketHandler))
-	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodGetBucketInfo).HandlerFunc(httpTraceHdrs(server.GetBucketInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodListBuckets).HandlerFunc(httpTraceHdrs(server.ListBucketsHandler))
+	h := func(f http.HandlerFunc) http.HandlerFunc {
+		return collectInternodeStats(httpTraceHdrs(f))
+	}
+
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodHealth).HandlerFunc(h(server.HealthHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodMakeBucket).HandlerFunc(h(server.MakeBucketHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodDeleteBucket).HandlerFunc(h(server.DeleteBucketHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodGetBucketInfo).HandlerFunc(h(server.GetBucketInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodListBuckets).HandlerFunc(h(server.ListBucketsHandler))
 }

cmd/routers.go

@@ -41,32 +41,34 @@ func registerDistErasureRouters(router *mux.Router, endpointServerPools Endpoint
 	registerLockRESTHandlers(router)
 }
 
-// List of some generic handlers which are applied for all incoming requests.
-var globalHandlers = []mux.MiddlewareFunc{
-	// The generic tracer needs to be the first handler
-	// to catch all requests returned early by any other handler
-	httpTracer,
-	// Auth handler verifies incoming authorization headers and
-	// routes them accordingly. Client receives a HTTP error for
-	// invalid/unsupported signatures.
+// List of some generic middlewares which are applied for all incoming requests.
+var globalMiddlewares = []mux.MiddlewareFunc{
+	// set x-amz-request-id header and others
+	addCustomHeadersMiddleware,
+	// The generic tracer needs to be the first middleware to catch all requests
+	// returned early by any other middleware (but after the middleware that
+	// sets the amz request id).
+	httpTracerMiddleware,
+	// Auth middleware verifies incoming authorization headers and routes them
+	// accordingly. Client receives a HTTP error for invalid/unsupported
+	// signatures.
 	//
 	// Validates all incoming requests to have a valid date header.
-	setAuthHandler,
-	// Redirect some pre-defined browser request paths to a static location prefix.
-	setBrowserRedirectHandler,
-	// Adds 'crossdomain.xml' policy handler to serve legacy flash clients.
-	setCrossDomainPolicy,
+	setAuthMiddleware,
+	// Redirect some pre-defined browser request paths to a static location
+	// prefix.
+	setBrowserRedirectMiddleware,
+	// Adds 'crossdomain.xml' policy middleware to serve legacy flash clients.
+	setCrossDomainPolicyMiddleware,
 	// Limits all body and header sizes to a maximum fixed limit
-	setRequestLimitHandler,
+	setRequestLimitMiddleware,
 	// Validate all the incoming requests.
-	setRequestValidityHandler,
-	// set x-amz-request-id header.
-	addCustomHeaders,
-	// Add upload forwarding handler for site replication
-	setUploadForwardingHandler,
-	// Add bucket forwarding handler
-	setBucketForwardingHandler,
-	// Add new handlers here.
+	setRequestValidityMiddleware,
+	// Add upload forwarding middleware for site replication
+	setUploadForwardingMiddleware,
+	// Add bucket forwarding middleware
+	setBucketForwardingMiddleware,
+	// Add new middlewares here.
 }
 
 // configureServer handler returns final handler for the http server.
@@ -98,7 +100,7 @@ func configureServerHandler(endpointServerPools EndpointServerPools) (http.Handl
 	// Add API router
 	registerAPIRouter(router)
 
-	router.Use(globalHandlers...)
+	router.Use(globalMiddlewares...)
 
 	return router, nil
 }

cmd/s3-zip-handlers.go

@@ -519,10 +519,10 @@ func updateObjectMetadataWithZipInfo(ctx context.Context, objectAPI ObjectLayer,
 	popts := ObjectOptions{
 		MTime:     srcInfo.ModTime,
 		VersionID: srcInfo.VersionID,
-		EvalMetadataFn: func(oi *ObjectInfo) error {
+		EvalMetadataFn: func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
 			oi.UserDefined[archiveTypeMetadataKey] = at
 			oi.UserDefined[archiveInfoMetadataKey] = zipInfoStr
-			return nil
+			return dsc, nil
 		},
 	}
 

cmd/storage-rest-client.go

@@ -23,7 +23,6 @@ import (
 	"encoding/gob"
 	"encoding/hex"
 	"errors"
-	"fmt"
 	"io"
 	"net/http"
 	"net/url"
@@ -733,7 +732,7 @@ func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
-	values.Set(storageRESTGlob, fmt.Sprint(glob))
+	values.Set(storageRESTGlob, strconv.FormatBool(glob))
 	respBody, err := client.call(ctx, storageRESTMethodStatInfoFile, values, nil, -1)
 	if err != nil {
 		return stat, err

cmd/storage-rest-server.go

@@ -1349,6 +1349,10 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 	}
 	wg.Wait()
 
+	h := func(f http.HandlerFunc) http.HandlerFunc {
+		return collectInternodeStats(httpTraceHdrs(f))
+	}
+
 	for _, setDisks := range storageDisks {
 		for _, storage := range setDisks {
 			if storage == nil {
@@ -1362,39 +1366,39 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 
 			subrouter := router.PathPrefix(path.Join(storageRESTPrefix, endpoint.Path)).Subrouter()
 
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodHealth).HandlerFunc(httpTraceHdrs(server.HealthHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDiskInfo).HandlerFunc(httpTraceHdrs(server.DiskInfoHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodNSScanner).HandlerFunc(httpTraceHdrs(server.NSScannerHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVol).HandlerFunc(httpTraceHdrs(server.MakeVolHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVolBulk).HandlerFunc(httpTraceHdrs(server.MakeVolBulkHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatVol).HandlerFunc(httpTraceHdrs(server.StatVolHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVol).HandlerFunc(httpTraceHdrs(server.DeleteVolHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListVols).HandlerFunc(httpTraceHdrs(server.ListVolsHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodHealth).HandlerFunc(h(server.HealthHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDiskInfo).HandlerFunc(h(server.DiskInfoHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodNSScanner).HandlerFunc(h(server.NSScannerHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVol).HandlerFunc(h(server.MakeVolHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodMakeVolBulk).HandlerFunc(h(server.MakeVolBulkHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatVol).HandlerFunc(h(server.StatVolHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVol).HandlerFunc(h(server.DeleteVolHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListVols).HandlerFunc(h(server.ListVolsHandler))
 
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodAppendFile).HandlerFunc(httpTraceHdrs(server.AppendFileHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWriteAll).HandlerFunc(httpTraceHdrs(server.WriteAllHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWriteMetadata).HandlerFunc(httpTraceHdrs(server.WriteMetadataHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodUpdateMetadata).HandlerFunc(httpTraceHdrs(server.UpdateMetadataHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVersion).HandlerFunc(httpTraceHdrs(server.DeleteVersionHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadVersion).HandlerFunc(httpTraceHdrs(server.ReadVersionHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadXL).HandlerFunc(httpTraceHdrs(server.ReadXLHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodRenameData).HandlerFunc(httpTraceHdrs(server.RenameDataHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCreateFile).HandlerFunc(httpTraceHdrs(server.CreateFileHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCheckParts).HandlerFunc(httpTraceHdrs(server.CheckPartsHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadAll).HandlerFunc(httpTraceHdrs(server.ReadAllHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadFile).HandlerFunc(httpTraceHdrs(server.ReadFileHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadFileStream).HandlerFunc(httpTraceHdrs(server.ReadFileStreamHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListDir).HandlerFunc(httpTraceHdrs(server.ListDirHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodAppendFile).HandlerFunc(h(server.AppendFileHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWriteAll).HandlerFunc(h(server.WriteAllHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWriteMetadata).HandlerFunc(h(server.WriteMetadataHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodUpdateMetadata).HandlerFunc(h(server.UpdateMetadataHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVersion).HandlerFunc(h(server.DeleteVersionHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadVersion).HandlerFunc(h(server.ReadVersionHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadXL).HandlerFunc(h(server.ReadXLHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodRenameData).HandlerFunc(h(server.RenameDataHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCreateFile).HandlerFunc(h(server.CreateFileHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCheckParts).HandlerFunc(h(server.CheckPartsHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadAll).HandlerFunc(h(server.ReadAllHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadFile).HandlerFunc(h(server.ReadFileHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadFileStream).HandlerFunc(h(server.ReadFileStreamHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodListDir).HandlerFunc(h(server.ListDirHandler))
 
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVersions).HandlerFunc(httpTraceHdrs(server.DeleteVersionsHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteFile).HandlerFunc(httpTraceHdrs(server.DeleteFileHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVersions).HandlerFunc(h(server.DeleteVersionsHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteFile).HandlerFunc(h(server.DeleteFileHandler))
 
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodRenameFile).HandlerFunc(httpTraceHdrs(server.RenameFileHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodVerifyFile).HandlerFunc(httpTraceHdrs(server.VerifyFileHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(httpTraceHdrs(server.WalkDirHandler))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(httpTraceHdrs(server.StatInfoFile))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadMultiple).HandlerFunc(httpTraceHdrs(server.ReadMultiple))
-			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCleanAbandoned).HandlerFunc(httpTraceHdrs(server.CleanAbandonedDataHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodRenameFile).HandlerFunc(h(server.RenameFileHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodVerifyFile).HandlerFunc(h(server.VerifyFileHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(h(server.WalkDirHandler))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(h(server.StatInfoFile))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadMultiple).HandlerFunc(h(server.ReadMultiple))
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCleanAbandoned).HandlerFunc(h(server.CleanAbandonedDataHandler))
 		}
 	}
 }

cmd/test-utils_test.go

@@ -2070,11 +2070,11 @@ func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Hand
 	if len(apiFunctions) > 0 {
 		// Iterate the list of API functions requested for and register them in mux HTTP handler.
 		registerAPIFunctions(muxRouter, objLayer, apiFunctions...)
-		muxRouter.Use(globalHandlers...)
+		muxRouter.Use(globalMiddlewares...)
 		return muxRouter
 	}
 	registerAPIRouter(muxRouter)
-	muxRouter.Use(globalHandlers...)
+	muxRouter.Use(globalMiddlewares...)
 	return muxRouter
 }
 

cmd/utils.go

@@ -56,7 +56,6 @@ import (
 	ioutilx "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/message/audit"
-	"github.com/minio/minio/internal/mcontext"
 	"github.com/minio/minio/internal/rest"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/certs"
@@ -156,6 +155,8 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		err = InvalidUploadID{}
 	case "EntityTooSmall":
 		err = PartTooSmall{}
+	case "ReplicationPermissionCheck":
+		err = ReplicationPermissionCheck{}
 	}
 
 	if minioErr.StatusCode == http.StatusMethodNotAllowed {
@@ -855,14 +856,7 @@ func newContext(r *http.Request, w http.ResponseWriter, api string) context.Cont
 		VersionID:    strings.TrimSpace(r.Form.Get(xhttp.VersionID)),
 	}
 
-	ctx := context.WithValue(r.Context(),
-		mcontext.ContextTraceKey,
-		&mcontext.TraceCtxt{
-			AmzReqID: reqID,
-		},
-	)
-
-	return logger.SetReqInfo(ctx, reqInfo)
+	return logger.SetReqInfo(r.Context(), reqInfo)
 }
 
 // Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example)

cmd/xl-storage.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -29,6 +29,7 @@ import (
 	pathutil "path"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -504,17 +505,17 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 		doneSz := globalScannerMetrics.timeSize(scannerMetricReadMetadata)
 		buf, err := s.readMetadata(ctx, item.Path)
 		doneSz(len(buf))
-		res["metasize"] = fmt.Sprint(len(buf))
+		res["metasize"] = strconv.Itoa(len(buf))
 		if err != nil {
 			res["err"] = err.Error()
 			return sizeSummary{}, errSkipFile
 		}
-		defer metaDataPoolPut(buf)
 
 		// Remove filename which is the meta file.
 		item.transformMetaDir()
 
 		fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath())
+		metaDataPoolPut(buf)
 		if err != nil {
 			res["err"] = err.Error()
 			return sizeSummary{}, errSkipFile
@@ -563,7 +564,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 
 		// apply tier sweep action on free versions
 		if len(fivs.FreeVersions) > 0 {
-			res["free-versions"] = fmt.Sprint(len(fivs.FreeVersions))
+			res["free-versions"] = strconv.Itoa(len(fivs.FreeVersions))
 		}
 		for _, freeVersion := range fivs.FreeVersions {
 			oi := freeVersion.ToObjectInfo(item.bucket, item.objectPath(), versioned)
@@ -575,13 +576,13 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 		// These are rather expensive. Skip if nobody listens.
 		if globalTrace.NumSubscribers(madmin.TraceScanner) > 0 {
 			if sizeS.versions > 0 {
-				res["versions"] = fmt.Sprint()
+				res["versions"] = strconv.FormatUint(sizeS.versions, 10)
 			}
-			res["size"] = fmt.Sprint(sizeS.totalSize)
+			res["size"] = strconv.FormatInt(sizeS.totalSize, 10)
 			if len(sizeS.tiers) > 0 {
 				for name, tier := range sizeS.tiers {
-					res["size-"+name] = fmt.Sprint(tier.TotalSize)
-					res["versions-"+name] = fmt.Sprint(tier.NumVersions)
+					res["size-"+name] = strconv.FormatUint(tier.TotalSize, 10)
+					res["versions-"+name] = strconv.Itoa(tier.NumVersions)
 				}
 			}
 			if sizeS.failedCount > 0 {
@@ -591,7 +592,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 				res["repl-pending"] = fmt.Sprintf("%d versions, %d bytes", sizeS.pendingCount, sizeS.pendingSize)
 			}
 			for tgt, st := range sizeS.replTargetStats {
-				res["repl-size-"+tgt] = fmt.Sprint(st.replicatedSize)
+				res["repl-size-"+tgt] = strconv.FormatInt(st.replicatedSize, 10)
 				if st.failedCount > 0 {
 					res["repl-failed-"+tgt] = fmt.Sprintf("%d versions, %d bytes", st.failedCount, st.failedSize)
 				}
@@ -2167,7 +2168,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 			logger.LogOnceIf(ctx, fmt.Errorf("srcVolume: %s, srcPath: %s, dstVolume: %s:, dstPath: %s - error %v",
 				srcVolume, srcPath,
 				dstVolume, dstPath,
-				err), "xl-storage-rename-data"+srcVolume+dstVolume)
+				err), "xl-storage-rename-data-"+srcVolume+"-"+dstVolume)
 		}
 		if err == nil && s.globalSync {
 			globalSync()
@@ -2273,12 +2274,12 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 			xlMetaLegacy := &xlMetaV1Object{}
 			json := jsoniter.ConfigCompatibleWithStandardLibrary
 			if err := json.Unmarshal(dstBuf, xlMetaLegacy); err != nil {
-				logger.LogIf(ctx, err)
+				logger.LogOnceIf(ctx, err, "read-data-unmarshal-"+dstFilePath)
 				// Data appears corrupt. Drop data.
 			} else {
 				xlMetaLegacy.DataDir = legacyDataDir
 				if err = xlMeta.AddLegacy(xlMetaLegacy); err != nil {
-					logger.LogIf(ctx, err)
+					logger.LogOnceIf(ctx, err, "read-data-add-legacy-"+dstFilePath)
 				}
 				legacyPreserved = true
 			}

docs/bucket/replication/setup_2site_existing_replication.sh

@@ -84,7 +84,7 @@ remote_arn=$(./mc replicate ls sitea/bucket --json | jq -r .rule.Destination.Buc
 sleep 1
 
 ./mc replicate resync start sitea/bucket/ --remote-bucket "${remote_arn}"
-sleep 20s ## sleep for 20s idea is that we give 200ms per object.
+sleep 30s ## sleep for 30s idea is that we give 300ms per object.
 
 count=$(./mc replicate resync status sitea/bucket --remote-bucket "${remote_arn}" --json | jq .resyncInfo.target[].replicationCount)
 
@@ -99,7 +99,7 @@ if [ $ret -ne 0 ]; then
 fi
 
 if [ $count -ne 12 ]; then
-	echo "resync not complete after 10s unexpected failure"
+	echo "resync not complete after 30s - unexpected failure"
 	./mc diff sitea/bucket siteb/bucket
 	exit 1
 fi

docs/distributed/decom-compressed-sse-s3.sh

@@ -33,7 +33,7 @@ sleep 2
 ./mc admin policy create myminio/ lake ./docs/distributed/rw.json
 
 ./mc admin policy attach myminio/ rw --user=minio123
-./mc admin policy attach myminio/ lake,rw --user=minio12345
+./mc admin policy attach myminio/ lake --user=minio12345
 
 ./mc mb -l myminio/versioned
 

docs/distributed/decom-encrypted-sse-s3.sh

@@ -28,7 +28,7 @@ sleep 2
 ./mc admin policy create myminio/ lake ./docs/distributed/rw.json
 
 ./mc admin policy attach myminio/ rw --user=minio123
-./mc admin policy attach myminio/ lake,rw --user=minio12345
+./mc admin policy attach myminio/ lake --user=minio12345
 
 ./mc mb -l myminio/versioned
 

docs/distributed/decom-encrypted.sh

@@ -30,7 +30,7 @@ export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/"
 ./mc admin policy create myminio/ lake ./docs/distributed/rw.json
 
 ./mc admin policy attach myminio/ rw --user=minio123
-./mc admin policy attach myminio/ lake,rw --user=minio12345
+./mc admin policy attach myminio/ lake --user=minio12345
 
 ./mc mb -l myminio/versioned
 

docs/distributed/decom.sh

@@ -15,7 +15,7 @@ fi
 
 export CI=true
 
-(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) &
+(minio server /tmp/xl/{1...10}/disk{0...1} 2>&1 >/tmp/decom.log) &
 pid=$!
 
 sleep 2
@@ -29,7 +29,7 @@ export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/"
 ./mc admin policy create myminio/ lake ./docs/distributed/rw.json
 
 ./mc admin policy attach myminio/ rw --user=minio123
-./mc admin policy attach myminio/ lake,rw --user=minio12345
+./mc admin policy attach myminio/ lake --user=minio12345
 
 ./mc mb -l myminio/versioned
 

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2023-06-29T05-12-28Z
+  image: quay.io/minio/minio:RELEASE.2023-07-07T07-13-57Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 5.0.11
-appVersion: RELEASE.2023-06-19T19-52-50Z
+version: 5.0.13
+appVersion: RELEASE.2023-07-07T07-13-57Z
 keywords:
   - minio
   - storage

helm/minio/templates/post-job.yaml

@@ -85,27 +85,27 @@ spec:
       {{- if .Values.serviceAccount.create }}
       serviceAccountName: {{ .Values.serviceAccount.name }}
       {{- end }}
-      { { - if .Values.policies } }
+      {{- if .Values.policies }}
       initContainers:
         - name: minio-make-policy
           image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}"
-          { { - if .Values.makePolicyJob.securityContext.enabled } }
+          {{- if .Values.makePolicyJob.securityContext.enabled }}
           securityContext:
-            runAsUser: { { .Values.makePolicyJob.securityContext.runAsUser } }
-            runAsGroup: { { .Values.makePolicyJob.securityContext.runAsGroup } }
-          { { - end } }
-          imagePullPolicy: { { .Values.mcImage.pullPolicy } }
-          { { - if .Values.makePolicyJob.exitCommand } }
+            runAsUser: {{ .Values.makePolicyJob.securityContext.runAsUser }}
+            runAsGroup: {{ .Values.makePolicyJob.securityContext.runAsGroup }}
+          {{- end }}
+          imagePullPolicy: {{ .Values.mcImage.pullPolicy }}
+          {{- if .Values.makePolicyJob.exitCommand }}
           command: [ "/bin/sh", "-c" ]
           args: [ "/bin/sh /config/add-policy; EV=$?; {{ .Values.makePolicyJob.exitCommand }} && exit $EV" ]
-          { { - else } }
+          {{- else }}
           command: [ "/bin/sh", "/config/add-policy" ]
-          { { - end } }
+          {{- end }}
           env:
             - name: MINIO_ENDPOINT
-              value: { { template "minio.fullname" . } }
+              value: {{ template "minio.fullname" . }}
             - name: MINIO_PORT
-              value: { { .Values.service.port | quote } }
+              value: {{ .Values.service.port | quote }}
           volumeMounts:
             - name: etc-path
               mountPath: /etc/minio/mc
@@ -113,12 +113,12 @@ spec:
               mountPath: /tmp
             - name: minio-configuration
               mountPath: /config
-            { { - if .Values.tls.enabled } }
+            {{- if .Values.tls.enabled }}
             - name: cert-secret-volume-mc
-              mountPath: { { .Values.configPathmc } }certs
+              mountPath: {{ .Values.configPathmc }}certs
             {{- end }}
-          resources: { { - toYaml .Values.makePolicyJob.resources | nindent 12 } }
-      { { - end } }
+          resources: {{- toYaml .Values.makePolicyJob.resources | nindent 12 }}
+      {{- end }}
       containers:
         {{- if .Values.buckets }}
         - name: minio-make-bucket

helm/minio/values.yaml

@@ -14,7 +14,7 @@ clusterDomain: cluster.local
 ##
 image:
   repository: quay.io/minio/minio
-  tag: RELEASE.2023-06-19T19-52-50Z
+  tag: RELEASE.2023-07-07T07-13-57Z
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -25,7 +25,7 @@ imagePullSecrets: []
 ##
 mcImage:
   repository: quay.io/minio/mc
-  tag: RELEASE.2023-06-19T19-31-19Z
+  tag: RELEASE.2023-06-28T21-54-17Z
   pullPolicy: IfNotPresent
 
 ## minio mode, i.e. standalone or distributed

index.yaml

@@ -1,9 +1,53 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2023-07-07T07-13-57Z
+    created: "2023-07-09T00:12:52.083613719-07:00"
+    description: Multi-Cloud Object Storage
+    digest: 3c18f7381efe6d86497f952e6d5f59003ee5a009c54778ddea1ee8d3c7bed9c8
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-5.0.13.tgz
+    version: 5.0.13
+  - apiVersion: v1
+    appVersion: RELEASE.2023-07-07T07-13-57Z
+    created: "2023-07-09T00:12:52.082626304-07:00"
+    description: Multi-Cloud Object Storage
+    digest: 5318bc56c73a8f4539c3dd178f4d55c7f41bee4a25d7dc02ac6a5843eeee7976
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-5.0.12.tgz
+    version: 5.0.12
   - apiVersion: v1
     appVersion: RELEASE.2023-06-19T19-52-50Z
-    created: "2023-06-21T12:28:16.517289715-07:00"
+    created: "2023-07-09T00:12:52.081596401-07:00"
     description: Multi-Cloud Object Storage
     digest: cba44c8cddcda1fb5c082dce82004a39f53cc20677ab9698a6998f01efefd8db
     home: https://min.io
@@ -25,7 +69,7 @@ entries:
     version: 5.0.11
   - apiVersion: v1
     appVersion: RELEASE.2023-05-18T00-05-36Z
-    created: "2023-06-21T12:28:16.516319227-07:00"
+    created: "2023-07-09T00:12:52.07978903-07:00"
     description: Multi-Cloud Object Storage
     digest: a3d55b12f38a2049ddf3efe35b38b6dc4e59777452b72d18d5a82f3378deb9cd
     home: https://min.io
@@ -47,7 +91,7 @@ entries:
     version: 5.0.10
   - apiVersion: v1
     appVersion: RELEASE.2023-04-28T18-11-17Z
-    created: "2023-06-21T12:28:16.526226353-07:00"
+    created: "2023-07-09T00:12:52.092296696-07:00"
     description: Multi-Cloud Object Storage
     digest: cf98985e32675e4ce327304ea9ac61046a788b3d5190d6b501330f7803d41a11
     home: https://min.io
@@ -69,7 +113,7 @@ entries:
     version: 5.0.9
   - apiVersion: v1
     appVersion: RELEASE.2023-04-13T03-08-07Z
-    created: "2023-06-21T12:28:16.525003643-07:00"
+    created: "2023-07-09T00:12:52.091349939-07:00"
     description: Multi-Cloud Object Storage
     digest: 034d68f85799f6693836975797f85a91842cf2d003a6c4ff401bd4ea4c946af6
     home: https://min.io
@@ -91,7 +135,7 @@ entries:
     version: 5.0.8
   - apiVersion: v1
     appVersion: RELEASE.2023-02-10T18-48-39Z
-    created: "2023-06-21T12:28:16.523762296-07:00"
+    created: "2023-07-09T00:12:52.090389234-07:00"
     description: Multi-Cloud Object Storage
     digest: 3f935a310e1b5b873052629b66005c160356ca7b2bd394cb07b34dbaf9905e3f
     home: https://min.io
@@ -113,7 +157,7 @@ entries:
     version: 5.0.7
   - apiVersion: v1
     appVersion: RELEASE.2023-02-10T18-48-39Z
-    created: "2023-06-21T12:28:16.522807765-07:00"
+    created: "2023-07-09T00:12:52.089429881-07:00"
     description: Multi-Cloud Object Storage
     digest: 82ef858ce483c2d736444792986cb36bd0fb4fc90a80b97fe30d7b2f2034d24a
     home: https://min.io
@@ -135,7 +179,7 @@ entries:
     version: 5.0.6
   - apiVersion: v1
     appVersion: RELEASE.2023-01-31T02-24-19Z
-    created: "2023-06-21T12:28:16.5211354-07:00"
+    created: "2023-07-09T00:12:52.087989387-07:00"
     description: Multi-Cloud Object Storage
     digest: fefeea10e4e525e45f82fb80a03900d34605ec432dd92f56d94eaf4fb1b98c41
     home: https://min.io
@@ -157,7 +201,7 @@ entries:
     version: 5.0.5
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-06-21T12:28:16.520186913-07:00"
+    created: "2023-07-09T00:12:52.0865042-07:00"
     description: Multi-Cloud Object Storage
     digest: 6b305783c98b0b97ffab079ff4430094fd0ca6e98e82bb8153cb93033a1bf40f
     home: https://min.io
@@ -179,7 +223,7 @@ entries:
     version: 5.0.4
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-06-21T12:28:16.519243759-07:00"
+    created: "2023-07-09T00:12:52.08555416-07:00"
     description: Multi-Cloud Object Storage
     digest: bac89157c53b324aece263c294aa49f5c9b64f426b4b06c9bca3d72e77e244f2
     home: https://min.io
@@ -201,7 +245,7 @@ entries:
     version: 5.0.3
   - apiVersion: v1
     appVersion: RELEASE.2022-12-12T19-27-27Z
-    created: "2023-06-21T12:28:16.518297233-07:00"
+    created: "2023-07-09T00:12:52.084560152-07:00"
     description: Multi-Cloud Object Storage
     digest: 935ce4f09366231b11d414d626f887fa6fa6024dd30a42e81e810ca1438d5904
     home: https://min.io
@@ -223,7 +267,7 @@ entries:
     version: 5.0.2
   - apiVersion: v1
     appVersion: RELEASE.2022-11-11T03-44-20Z
-    created: "2023-06-21T12:28:16.515018082-07:00"
+    created: "2023-07-09T00:12:52.078824549-07:00"
     description: Multi-Cloud Object Storage
     digest: 3e952c5d737980b8ccdfb819021eafb4b4e8da226f764a1dc3de1ba63ceb1ffa
     home: https://min.io
@@ -245,7 +289,7 @@ entries:
     version: 5.0.1
   - apiVersion: v1
     appVersion: RELEASE.2022-10-24T18-35-07Z
-    created: "2023-06-21T12:28:16.51327371-07:00"
+    created: "2023-07-09T00:12:52.077895753-07:00"
     description: Multi-Cloud Object Storage
     digest: 6215c800d84fd4c40e4fb4142645fc1c6a039c251776a3cc8c11a24b9e3b59c7
     home: https://min.io
@@ -267,7 +311,7 @@ entries:
     version: 5.0.0
   - apiVersion: v1
     appVersion: RELEASE.2022-10-24T18-35-07Z
-    created: "2023-06-21T12:28:16.512327637-07:00"
+    created: "2023-07-09T00:12:52.076911932-07:00"
     description: Multi-Cloud Object Storage
     digest: 2d3d884490ea1127742f938bc9382844bae713caae08b3308f766f3c9000659a
     home: https://min.io
@@ -289,7 +333,7 @@ entries:
     version: 4.1.0
   - apiVersion: v1
     appVersion: RELEASE.2022-09-17T00-09-45Z
-    created: "2023-06-21T12:28:16.503109767-07:00"
+    created: "2023-07-09T00:12:52.068200433-07:00"
     description: Multi-Cloud Object Storage
     digest: 6f16f2dbfed91ab81a7fae60b6ea32f554365bd27bf5fda55b64a0fa264f4252
     home: https://min.io
@@ -311,7 +355,7 @@ entries:
     version: 4.0.15
   - apiVersion: v1
     appVersion: RELEASE.2022-09-01T23-53-36Z
-    created: "2023-06-21T12:28:16.502050092-07:00"
+    created: "2023-07-09T00:12:52.06625577-07:00"
     description: Multi-Cloud Object Storage
     digest: 35d89d8f49d53ea929466fb88ee26123431326033f1387e6b2d536a629c0a398
     home: https://min.io
@@ -333,7 +377,7 @@ entries:
     version: 4.0.14
   - apiVersion: v1
     appVersion: RELEASE.2022-08-22T23-53-06Z
-    created: "2023-06-21T12:28:16.50087561-07:00"
+    created: "2023-07-09T00:12:52.065251383-07:00"
     description: Multi-Cloud Object Storage
     digest: 5b86937ca88d9f6046141fdc2b1cc54760435ed92d289cd0a115fa7148781d4e
     home: https://min.io
@@ -355,7 +399,7 @@ entries:
     version: 4.0.13
   - apiVersion: v1
     appVersion: RELEASE.2022-08-13T21-54-44Z
-    created: "2023-06-21T12:28:16.499114109-07:00"
+    created: "2023-07-09T00:12:52.064278973-07:00"
     description: Multi-Cloud Object Storage
     digest: 2d9c227c0f46ea8bdef4d760c212156fd4c6623ddc5406779c569fe925527787
     home: https://min.io
@@ -377,7 +421,7 @@ entries:
     version: 4.0.12
   - apiVersion: v1
     appVersion: RELEASE.2022-08-05T23-27-09Z
-    created: "2023-06-21T12:28:16.498198525-07:00"
+    created: "2023-07-09T00:12:52.063368673-07:00"
     description: Multi-Cloud Object Storage
     digest: 6caaffcb636e040cd7e8bc4883a1674a673757f4781c32d53b5ec0f41fea3944
     home: https://min.io
@@ -399,7 +443,7 @@ entries:
     version: 4.0.11
   - apiVersion: v1
     appVersion: RELEASE.2022-08-02T23-59-16Z
-    created: "2023-06-21T12:28:16.497235138-07:00"
+    created: "2023-07-09T00:12:52.062416509-07:00"
     description: Multi-Cloud Object Storage
     digest: 841d87788fb094d6a7d8a91e91821fe1e847bc952e054c781fc93742d112e18a
     home: https://min.io
@@ -421,7 +465,7 @@ entries:
     version: 4.0.10
   - apiVersion: v1
     appVersion: RELEASE.2022-08-02T23-59-16Z
-    created: "2023-06-21T12:28:16.511372928-07:00"
+    created: "2023-07-09T00:12:52.075955122-07:00"
     description: Multi-Cloud Object Storage
     digest: 6f1a78382df3215deac07495a5e7de7009a1153b4cf6cb565630652a69aec4cf
     home: https://min.io
@@ -443,7 +487,7 @@ entries:
     version: 4.0.9
   - apiVersion: v1
     appVersion: RELEASE.2022-07-29T19-40-48Z
-    created: "2023-06-21T12:28:16.5105-07:00"
+    created: "2023-07-09T00:12:52.075062029-07:00"
     description: Multi-Cloud Object Storage
     digest: d11db37963636922cb778b6bc0ad2ca4724cb391ea7b785995ada52467d7dd83
     home: https://min.io
@@ -465,7 +509,7 @@ entries:
     version: 4.0.8
   - apiVersion: v1
     appVersion: RELEASE.2022-07-26T00-53-03Z
-    created: "2023-06-21T12:28:16.509600869-07:00"
+    created: "2023-07-09T00:12:52.073779859-07:00"
     description: Multi-Cloud Object Storage
     digest: ca775e08c84331bb5029d4d29867d30c16e2c62e897788eb432212a756e91e4e
     home: https://min.io
@@ -487,7 +531,7 @@ entries:
     version: 4.0.7
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-06-21T12:28:16.50852748-07:00"
+    created: "2023-07-09T00:12:52.07269585-07:00"
     description: Multi-Cloud Object Storage
     digest: 06542b8f3d149d5908b15de9a8d6f8cf304af0213830be56dc315785d14f9ccd
     home: https://min.io
@@ -509,7 +553,7 @@ entries:
     version: 4.0.6
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-06-21T12:28:16.506902052-07:00"
+    created: "2023-07-09T00:12:52.071826014-07:00"
     description: Multi-Cloud Object Storage
     digest: dd2676362f067454a496cdd293609d0c904b08f521625af49f95402a024ba1f5
     home: https://min.io
@@ -531,7 +575,7 @@ entries:
     version: 4.0.5
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-06-21T12:28:16.506035237-07:00"
+    created: "2023-07-09T00:12:52.070968807-07:00"
     description: Multi-Cloud Object Storage
     digest: bab9ef192d4eda4c572ad0ce0cf551736c847f582d1837d6833ee10543c23167
     home: https://min.io
@@ -553,7 +597,7 @@ entries:
     version: 4.0.4
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-06-21T12:28:16.505101634-07:00"
+    created: "2023-07-09T00:12:52.070118552-07:00"
     description: Multi-Cloud Object Storage
     digest: c770bb9841c76576e4e8573f78b0ec33e0d729504c9667e67ad62d48df5ed64c
     home: https://min.io
@@ -575,7 +619,7 @@ entries:
     version: 4.0.3
   - apiVersion: v1
     appVersion: RELEASE.2022-05-08T23-50-31Z
-    created: "2023-06-21T12:28:16.504061524-07:00"
+    created: "2023-07-09T00:12:52.069249607-07:00"
     description: Multi-Cloud Object Storage
     digest: 95835f4199d963e2a23a2493610b348e6f2ff8b71c1a648c4a3b84af9b7a83eb
     home: https://min.io
@@ -597,7 +641,7 @@ entries:
     version: 4.0.2
   - apiVersion: v1
     appVersion: RELEASE.2022-04-30T22-23-53Z
-    created: "2023-06-21T12:28:16.496136366-07:00"
+    created: "2023-07-09T00:12:52.061513072-07:00"
     description: Multi-Cloud Object Storage
     digest: 55a088c403b056e1f055a97426aa11759c3d6cbad38face170fe6cbbec7d568f
     home: https://min.io
@@ -619,7 +663,7 @@ entries:
     version: 4.0.1
   - apiVersion: v1
     appVersion: RELEASE.2022-04-26T01-20-24Z
-    created: "2023-06-21T12:28:16.495284446-07:00"
+    created: "2023-07-09T00:12:52.060505181-07:00"
     description: Multi-Cloud Object Storage
     digest: f541237e24336ec3f7f45ae0d523fef694e3a2f9ef648c5b11c15734db6ba2b2
     home: https://min.io
@@ -641,7 +685,7 @@ entries:
     version: 4.0.0
   - apiVersion: v1
     appVersion: RELEASE.2022-04-16T04-26-02Z
-    created: "2023-06-21T12:28:16.494416513-07:00"
+    created: "2023-07-09T00:12:52.058907841-07:00"
     description: Multi-Cloud Object Storage
     digest: edc0c3dd6d5246a06b74ba16bb4aff80a6d7225dc9aecf064fd89a8af371b9c1
     home: https://min.io
@@ -663,7 +707,7 @@ entries:
     version: 3.6.6
   - apiVersion: v1
     appVersion: RELEASE.2022-04-12T06-55-35Z
-    created: "2023-06-21T12:28:16.493496623-07:00"
+    created: "2023-07-09T00:12:52.058000086-07:00"
     description: Multi-Cloud Object Storage
     digest: 211e89f6b9eb0b9a3583abaa127be60e1f9717a098e6b2858cb9dc1cc50c1650
     home: https://min.io
@@ -685,7 +729,7 @@ entries:
     version: 3.6.5
   - apiVersion: v1
     appVersion: RELEASE.2022-04-09T15-09-52Z
-    created: "2023-06-21T12:28:16.491721844-07:00"
+    created: "2023-07-09T00:12:52.056972348-07:00"
     description: Multi-Cloud Object Storage
     digest: 534a879d73b370a18b554b93d0930e1c115419619c4ce4ec7dbaae632acacf06
     home: https://min.io
@@ -707,7 +751,7 @@ entries:
     version: 3.6.4
   - apiVersion: v1
     appVersion: RELEASE.2022-03-24T00-43-44Z
-    created: "2023-06-21T12:28:16.490862585-07:00"
+    created: "2023-07-09T00:12:52.056123789-07:00"
     description: Multi-Cloud Object Storage
     digest: 99508b20eb0083a567dcccaf9a6c237e09575ed1d70cd2e8333f89c472d13d75
     home: https://min.io
@@ -729,7 +773,7 @@ entries:
     version: 3.6.3
   - apiVersion: v1
     appVersion: RELEASE.2022-03-17T06-34-49Z
-    created: "2023-06-21T12:28:16.489997538-07:00"
+    created: "2023-07-09T00:12:52.055271276-07:00"
     description: Multi-Cloud Object Storage
     digest: b4cd25611ca322b1d23d23112fdfa6b068fd91eefe0b0663b88ff87ea4282495
     home: https://min.io
@@ -751,7 +795,7 @@ entries:
     version: 3.6.2
   - apiVersion: v1
     appVersion: RELEASE.2022-03-14T18-25-24Z
-    created: "2023-06-21T12:28:16.489050887-07:00"
+    created: "2023-07-09T00:12:52.054400832-07:00"
     description: Multi-Cloud Object Storage
     digest: d75b88162bfe54740a233bcecf87328bba2ae23d170bec3a35c828bc6fdc224c
     home: https://min.io
@@ -773,7 +817,7 @@ entries:
     version: 3.6.1
   - apiVersion: v1
     appVersion: RELEASE.2022-03-11T23-57-45Z
-    created: "2023-06-21T12:28:16.488128175-07:00"
+    created: "2023-07-09T00:12:52.053508093-07:00"
     description: Multi-Cloud Object Storage
     digest: 22e53a1184a21a679bc7d8b94e955777f3506340fc29da5ab0cb6d729bdbde8d
     home: https://min.io
@@ -795,7 +839,7 @@ entries:
     version: 3.6.0
   - apiVersion: v1
     appVersion: RELEASE.2022-03-03T21-21-16Z
-    created: "2023-06-21T12:28:16.487268803-07:00"
+    created: "2023-07-09T00:12:52.051823002-07:00"
     description: Multi-Cloud Object Storage
     digest: 6fda968d3fdfd60470c0055a4e1a3bd8e5aee9ad0af5ba2fb7b7b926fdc9e4a0
     home: https://min.io
@@ -817,7 +861,7 @@ entries:
     version: 3.5.9
   - apiVersion: v1
     appVersion: RELEASE.2022-02-26T02-54-46Z
-    created: "2023-06-21T12:28:16.486424696-07:00"
+    created: "2023-07-09T00:12:52.050984544-07:00"
     description: Multi-Cloud Object Storage
     digest: 8e015369048a3a82bbd53ad36696786f18561c6b25d14eee9e2c93a7336cef46
     home: https://min.io
@@ -839,7 +883,7 @@ entries:
     version: 3.5.8
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2023-06-21T12:28:16.485530859-07:00"
+    created: "2023-07-09T00:12:52.050137914-07:00"
     description: Multi-Cloud Object Storage
     digest: cb3543fe748e5f0d59b3ccf4ab9af8e10b731405ae445d1f5715e30013632373
     home: https://min.io
@@ -861,7 +905,7 @@ entries:
     version: 3.5.7
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2023-06-21T12:28:16.483954421-07:00"
+    created: "2023-07-09T00:12:52.049298332-07:00"
     description: Multi-Cloud Object Storage
     digest: f2e359fa5eefffc59abb3d14a8fa94b11ddeaa99f6cd8dd5f40f4e04121000d6
     home: https://min.io
@@ -883,7 +927,7 @@ entries:
     version: 3.5.6
   - apiVersion: v1
     appVersion: RELEASE.2022-02-16T00-35-27Z
-    created: "2023-06-21T12:28:16.483074029-07:00"
+    created: "2023-07-09T00:12:52.048393471-07:00"
     description: Multi-Cloud Object Storage
     digest: 529d56cca9d83a3d0e5672e63b6e87b5bcbe10a6b45f7a55ba998cceb32f9c81
     home: https://min.io
@@ -905,7 +949,7 @@ entries:
     version: 3.5.5
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2023-06-21T12:28:16.482232953-07:00"
+    created: "2023-07-09T00:12:52.047593644-07:00"
     description: Multi-Cloud Object Storage
     digest: 3d530598f8ece67bec5b7f990d206584893987c713502f9228e4ee24b5535414
     home: https://min.io
@@ -927,7 +971,7 @@ entries:
     version: 3.5.4
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2023-06-21T12:28:16.481377778-07:00"
+    created: "2023-07-09T00:12:52.046792295-07:00"
     description: Multi-Cloud Object Storage
     digest: 53937031348b29615f07fc4869b2d668391d8ba9084630a497abd7a7dea9dfb0
     home: https://min.io
@@ -949,7 +993,7 @@ entries:
     version: 3.5.3
   - apiVersion: v1
     appVersion: RELEASE.2022-02-07T08-17-33Z
-    created: "2023-06-21T12:28:16.480643181-07:00"
+    created: "2023-07-09T00:12:52.046012432-07:00"
     description: Multi-Cloud Object Storage
     digest: 68d643414ff0d565716c5715034fcbf1af262e041915a5c02eb51ec1a65c1ea0
     home: https://min.io
@@ -971,7 +1015,7 @@ entries:
     version: 3.5.2
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2023-06-21T12:28:16.479888214-07:00"
+    created: "2023-07-09T00:12:52.044649337-07:00"
     description: Multi-Cloud Object Storage
     digest: a3e855ed0f31233b989fffd775a29d6fbfa0590089010ff16783fd7f142ef6e7
     home: https://min.io
@@ -993,7 +1037,7 @@ entries:
     version: 3.5.1
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2023-06-21T12:28:16.479073935-07:00"
+    created: "2023-07-09T00:12:52.043927219-07:00"
     description: Multi-Cloud Object Storage
     digest: b1b0ae3c54b4260a698753e11d7781bb8ddc67b7e3fbf0af82796e4cd4ef92a3
     home: https://min.io
@@ -1015,7 +1059,7 @@ entries:
     version: 3.5.0
   - apiVersion: v1
     appVersion: RELEASE.2022-01-28T02-28-16Z
-    created: "2023-06-21T12:28:16.47526346-07:00"
+    created: "2023-07-09T00:12:52.043197969-07:00"
     description: Multi-Cloud Object Storage
     digest: fecf25d2d3fb208c6f894fed642a60780a570b7f6d0adddde846af7236dc80aa
     home: https://min.io
@@ -1037,7 +1081,7 @@ entries:
     version: 3.4.8
   - apiVersion: v1
     appVersion: RELEASE.2022-01-25T19-56-04Z
-    created: "2023-06-21T12:28:16.474511531-07:00"
+    created: "2023-07-09T00:12:52.042459778-07:00"
     description: Multi-Cloud Object Storage
     digest: c78008caa5ce98f64c887630f59d0cbd481cb3f19a7d4e9d3e81bf4e1e45cadc
     home: https://min.io
@@ -1059,7 +1103,7 @@ entries:
     version: 3.4.7
   - apiVersion: v1
     appVersion: RELEASE.2022-01-08T03-11-54Z
-    created: "2023-06-21T12:28:16.473750293-07:00"
+    created: "2023-07-09T00:12:52.041744623-07:00"
     description: Multi-Cloud Object Storage
     digest: 8f2e2691bf897f74ff094dd370ec56ba9d417e5e8926710c14c2ba346330238d
     home: https://min.io
@@ -1081,7 +1125,7 @@ entries:
     version: 3.4.6
   - apiVersion: v1
     appVersion: RELEASE.2022-01-04T07-41-07Z
-    created: "2023-06-21T12:28:16.472980985-07:00"
+    created: "2023-07-09T00:12:52.040983842-07:00"
     description: Multi-Cloud Object Storage
     digest: bacd140f0016fab35f516bde787da6449b3a960c071fad9e4b6563118033ac84
     home: https://min.io
@@ -1103,7 +1147,7 @@ entries:
     version: 3.4.5
   - apiVersion: v1
     appVersion: RELEASE.2021-12-29T06-49-06Z
-    created: "2023-06-21T12:28:16.472269597-07:00"
+    created: "2023-07-09T00:12:52.040253734-07:00"
     description: Multi-Cloud Object Storage
     digest: 48a453ea5ffeef25933904caefd9470bfb26224dfc2d1096bd0031467ba53007
     home: https://min.io
@@ -1125,7 +1169,7 @@ entries:
     version: 3.4.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2023-06-21T12:28:16.471541886-07:00"
+    created: "2023-07-09T00:12:52.039441814-07:00"
     description: Multi-Cloud Object Storage
     digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60
     home: https://min.io
@@ -1147,7 +1191,7 @@ entries:
     version: 3.4.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2023-06-21T12:28:16.470767853-07:00"
+    created: "2023-07-09T00:12:52.037915028-07:00"
     description: Multi-Cloud Object Storage
     digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff
     home: https://min.io
@@ -1169,7 +1213,7 @@ entries:
     version: 3.4.2
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-06-21T12:28:16.470048198-07:00"
+    created: "2023-07-09T00:12:52.037190929-07:00"
     description: Multi-Cloud Object Storage
     digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37
     home: https://min.io
@@ -1191,7 +1235,7 @@ entries:
     version: 3.4.1
   - apiVersion: v1
     appVersion: RELEASE.2021-12-18T04-42-33Z
-    created: "2023-06-21T12:28:16.469216986-07:00"
+    created: "2023-07-09T00:12:52.036385019-07:00"
     description: Multi-Cloud Object Storage
     digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453
     home: https://min.io
@@ -1213,7 +1257,7 @@ entries:
     version: 3.4.0
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-06-21T12:28:16.46784331-07:00"
+    created: "2023-07-09T00:12:52.035682883-07:00"
     description: Multi-Cloud Object Storage
     digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894
     home: https://min.io
@@ -1235,7 +1279,7 @@ entries:
     version: 3.3.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-06-21T12:28:16.467159883-07:00"
+    created: "2023-07-09T00:12:52.034982834-07:00"
     description: Multi-Cloud Object Storage
     digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08
     home: https://min.io
@@ -1257,7 +1301,7 @@ entries:
     version: 3.3.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2023-06-21T12:28:16.466460246-07:00"
+    created: "2023-07-09T00:12:52.034279304-07:00"
     description: Multi-Cloud Object Storage
     digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb
     home: https://min.io
@@ -1279,7 +1323,7 @@ entries:
     version: 3.3.2
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2023-06-21T12:28:16.465765846-07:00"
+    created: "2023-07-09T00:12:52.033504928-07:00"
     description: Multi-Cloud Object Storage
     digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4
     home: https://min.io
@@ -1301,7 +1345,7 @@ entries:
     version: 3.3.1
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2023-06-21T12:28:16.464966116-07:00"
+    created: "2023-07-09T00:12:52.032082552-07:00"
     description: Multi-Cloud Object Storage
     digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4
     home: https://min.io
@@ -1323,7 +1367,7 @@ entries:
     version: 3.3.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-13T00-23-17Z
-    created: "2023-06-21T12:28:16.464138605-07:00"
+    created: "2023-07-09T00:12:52.031187915-07:00"
     description: Multi-Cloud Object Storage
     digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054
     home: https://min.io
@@ -1345,7 +1389,7 @@ entries:
     version: 3.2.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-10T16-53-30Z
-    created: "2023-06-21T12:28:16.463263636-07:00"
+    created: "2023-07-09T00:12:52.030483162-07:00"
     description: Multi-Cloud Object Storage
     digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8
     home: https://min.io
@@ -1367,7 +1411,7 @@ entries:
     version: 3.1.9
   - apiVersion: v1
     appVersion: RELEASE.2021-10-06T23-36-31Z
-    created: "2023-06-21T12:28:16.461810713-07:00"
+    created: "2023-07-09T00:12:52.02979236-07:00"
     description: Multi-Cloud Object Storage
     digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159
     home: https://min.io
@@ -1389,7 +1433,7 @@ entries:
     version: 3.1.8
   - apiVersion: v1
     appVersion: RELEASE.2021-10-02T16-31-05Z
-    created: "2023-06-21T12:28:16.461037633-07:00"
+    created: "2023-07-09T00:12:52.029062485-07:00"
     description: Multi-Cloud Object Storage
     digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11
     home: https://min.io
@@ -1411,7 +1455,7 @@ entries:
     version: 3.1.7
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-06-21T12:28:16.460341082-07:00"
+    created: "2023-07-09T00:12:52.028178186-07:00"
     description: Multi-Cloud Object Storage
     digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310
     home: https://min.io
@@ -1433,7 +1477,7 @@ entries:
     version: 3.1.6
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-06-21T12:28:16.459362034-07:00"
+    created: "2023-07-09T00:12:52.027297471-07:00"
     description: Multi-Cloud Object Storage
     digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a
     home: https://min.io
@@ -1455,7 +1499,7 @@ entries:
     version: 3.1.5
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2023-06-21T12:28:16.458668001-07:00"
+    created: "2023-07-09T00:12:52.02646493-07:00"
     description: Multi-Cloud Object Storage
     digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0
     home: https://min.io
@@ -1477,7 +1521,7 @@ entries:
     version: 3.1.4
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-06-21T12:28:16.457956146-07:00"
+    created: "2023-07-09T00:12:52.025609511-07:00"
     description: Multi-Cloud Object Storage
     digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb
     home: https://min.io
@@ -1499,7 +1543,7 @@ entries:
     version: 3.1.3
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-06-21T12:28:16.457088113-07:00"
+    created: "2023-07-09T00:12:52.02463492-07:00"
     description: Multi-Cloud Object Storage
     digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd
     home: https://min.io
@@ -1521,7 +1565,7 @@ entries:
     version: 3.1.2
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-06-21T12:28:16.453750792-07:00"
+    created: "2023-07-09T00:12:52.016273999-07:00"
     description: Multi-Cloud Object Storage
     digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
     home: https://min.io
@@ -1543,7 +1587,7 @@ entries:
     version: 3.1.1
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2023-06-21T12:28:16.45296401-07:00"
+    created: "2023-07-09T00:12:52.015535262-07:00"
     description: Multi-Cloud Object Storage
     digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
     home: https://min.io
@@ -1565,7 +1609,7 @@ entries:
     version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2023-06-21T12:28:16.452145152-07:00"
+    created: "2023-07-09T00:12:52.014848017-07:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -1587,7 +1631,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-06-21T12:28:16.451338987-07:00"
+    created: "2023-07-09T00:12:52.01416606-07:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -1609,7 +1653,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-06-21T12:28:16.450532877-07:00"
+    created: "2023-07-09T00:12:52.013496497-07:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -1631,7 +1675,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2023-06-21T12:28:16.449686829-07:00"
+    created: "2023-07-09T00:12:52.012743494-07:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -1651,7 +1695,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-06-21T12:28:16.448619394-07:00"
+    created: "2023-07-09T00:12:52.011973175-07:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -1671,7 +1715,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-06-21T12:28:16.44773238-07:00"
+    created: "2023-07-09T00:12:52.011186882-07:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -1691,7 +1735,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2023-06-21T12:28:16.446225386-07:00"
+    created: "2023-07-09T00:12:52.010322776-07:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -1711,7 +1755,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-06-21T12:28:16.445495004-07:00"
+    created: "2023-07-09T00:12:52.00894929-07:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -1731,7 +1775,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-06-21T12:28:16.444648563-07:00"
+    created: "2023-07-09T00:12:52.008126414-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -1751,7 +1795,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2023-06-21T12:28:16.443847738-07:00"
+    created: "2023-07-09T00:12:52.007422376-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -1771,7 +1815,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2023-06-21T12:28:16.443024717-07:00"
+    created: "2023-07-09T00:12:52.006698484-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -1789,4 +1833,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2023-06-21T12:28:16.442126834-07:00"
+generated: "2023-07-09T00:12:52.00579794-07:00"

internal/bucket/lifecycle/lifecycle.go

@@ -92,7 +92,7 @@ func (a Action) Delete() bool {
 	if a.DeleteRestored() {
 		return true
 	}
-	return a == DeleteVersionAction || a == DeleteAction
+	return a == DeleteVersionAction || a == DeleteAction || a == DeleteAllVersionsAction
 }
 
 // Lifecycle - Configuration for bucket lifecycle.

internal/http/headers.go

@@ -199,6 +199,8 @@ const (
 	MinIOSourceProxyRequest = "X-Minio-Source-Proxy-Request"
 	// Header indicates that this request is a replication request to create a REPLICA
 	MinIOSourceReplicationRequest = "X-Minio-Source-Replication-Request"
+	// Header checks replication permissions without actually completing replication
+	MinIOSourceReplicationCheck = "X-Minio-Source-Replication-Check"
 	// Header indicates replication reset status.
 	MinIOReplicationResetStatus = "X-Minio-Replication-Reset-Status"
 	// Header indicating target cluster can receive delete marker replication requests because object has been replicated

internal/http/transports.go

@@ -1,6 +1,6 @@
 // Copyright (c) 2015-2022 MinIO, Inc.
 //
-// # This file is part of MinIO Object Storage stack
+// This file is part of MinIO Object Storage stack
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by

internal/rest/client.go

@@ -126,16 +126,14 @@ func removeEmptyPort(host string) string {
 }
 
 // Copied from http.NewRequest but implemented to ensure we re-use `url.URL` instance.
-func (c *Client) newRequest(ctx context.Context, u *url.URL, body io.Reader) (*http.Request, error) {
+func (c *Client) newRequest(ctx context.Context, u url.URL, body io.Reader) (*http.Request, error) {
 	rc, ok := body.(io.ReadCloser)
 	if !ok && body != nil {
 		rc = io.NopCloser(body)
 	}
-	u.Host = removeEmptyPort(u.Host)
-	// The host's colon:port should be normalized. See Issue 14836.
 	req := &http.Request{
 		Method:     http.MethodPost,
-		URL:        u,
+		URL:        &u,
 		Proto:      "HTTP/1.1",
 		ProtoMajor: 1,
 		ProtoMinor: 1,
@@ -288,22 +286,19 @@ func (c *Client) dumpHTTP(req *http.Request, resp *http.Response) {
 
 // Call - make a REST call with context.
 func (c *Client) Call(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (reply io.ReadCloser, err error) {
-	urlStr := c.url.String()
 	if !c.IsOnline() {
-		return nil, &NetworkError{c.LastError()}
-	}
-
-	u, err := url.Parse(urlStr)
-	if err != nil {
-		return nil, &NetworkError{Err: &url.Error{Op: method, URL: urlStr, Err: err}}
+		return nil, &NetworkError{Err: c.LastError()}
 	}
 
+	// Shallow copy. We don't modify the *UserInfo, if set.
+	// All other fields are copied.
+	u := *c.url
 	u.Path = path.Join(u.Path, method)
 	u.RawQuery = values.Encode()
 
 	req, err := c.newRequest(ctx, u, body)
 	if err != nil {
-		return nil, &NetworkError{err}
+		return nil, &NetworkError{Err: err}
 	}
 	if length > 0 {
 		req.ContentLength = length
@@ -379,14 +374,27 @@ func (c *Client) Close() {
 }
 
 // NewClient - returns new REST client.
-func NewClient(url *url.URL, tr http.RoundTripper, newAuthToken func(aud string) string) *Client {
+func NewClient(uu *url.URL, tr http.RoundTripper, newAuthToken func(aud string) string) *Client {
+	connected := int32(online)
+	urlStr := uu.String()
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		// Mark offline, with no reconnection attempts.
+		connected = int32(offline)
+		err = &url.Error{URL: urlStr, Err: err}
+	}
+	// The host's colon:port should be normalized. See Issue 14836.
+	u.Host = removeEmptyPort(u.Host)
+
 	// Transport is exactly same as Go default in https://golang.org/pkg/net/http/#RoundTripper
 	// except custom DialContext and TLSClientConfig.
 	return &Client{
 		httpClient:               &http.Client{Transport: tr},
-		url:                      url,
+		url:                      u,
+		lastErr:                  err,
+		lastErrTime:              time.Now(),
 		newAuthToken:             newAuthToken,
-		connected:                online,
+		connected:                connected,
 		lastConn:                 time.Now().UnixNano(),
 		MaxErrResponseSize:       4096,
 		HealthCheckReconnectUnit: 200 * time.Millisecond,