crypto: reduce retry delay when retrying KES requests (#10394)

This commit reduces the retry delay when retrying a request
to a KES server by:
 - reducing the max. jitter delay from 3s to 1.5s
 - skipping the random delay when there are more KES endpoints
   available.

If there are more KES endpoints we can directly retry to the request
by sending it to the next endpoint - as pointed out by @krishnasrinivas

.github/stale.yml

@@ -14,6 +14,7 @@ onlyLabels: []
 exemptLabels:
   - "security"
   - "pending discussion"
+  - "do not close"
 
 # Set to true to ignore issues in a project (defaults to false)
 exemptProjects: false

.github/workflows/go.yml

@@ -4,7 +4,6 @@ on:
   pull_request:
     branches:
     - master
-    - release
 
 jobs:
   build:
@@ -12,7 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.13.x]
+        go-version: [1.14.x, 1.15.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2
@@ -39,6 +38,9 @@ jobs:
           MINIO_CI_CD: 1
         run: |
           sudo apt-get install devscripts shellcheck
+          nancy_version=$(curl --retry 10 -Ls -o /dev/null -w "%{url_effective}" https://github.com/sonatype-nexus-community/nancy/releases/latest | sed "s/https:\/\/github.com\/sonatype-nexus-community\/nancy\/releases\/tag\///")
+          curl -L -o nancy https://github.com/sonatype-nexus-community/nancy/releases/download/${nancy_version}/nancy-linux.amd64-${nancy_version} && chmod +x nancy
+          go list -m all | ./nancy sleuth
           make
           diff -au <(gofmt -s -d cmd) <(printf "")
           diff -au <(gofmt -s -d pkg) <(printf "")

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13-alpine as builder
+FROM golang:1.14-alpine as builder
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 

Makefile

@@ -17,11 +17,12 @@ checks:
 getdeps:
 	@mkdir -p ${GOPATH}/bin
 	@which golangci-lint 1>/dev/null || (echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.27.0)
+	@which ruleguard 1>/dev/null || (echo "Installing ruleguard" && GO111MODULE=off go get github.com/quasilyte/go-ruleguard/...)
 
 crosscompile:
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
-verifiers: getdeps fmt lint
+verifiers: getdeps fmt lint ruleguard
 
 fmt:
 	@echo "Running $@ check"
@@ -33,6 +34,10 @@ lint:
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
 
+ruleguard:
+	@echo "Running $@ check"
+	@${GOPATH}/bin/ruleguard -rules ruleguard.rules.go github.com/minio/minio/...
+
 # Builds minio, runs the verifiers then runs the tests.
 check: test
 test: verifiers build

cmd/admin-bucket-handlers.go

@@ -26,7 +26,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/env"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
@@ -122,8 +121,8 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	writeSuccessResponseJSON(w, configData)
 }
 
-// SetBucketTargetHandler - sets a remote target for bucket
-func (a adminAPIHandlers) SetBucketTargetHandler(w http.ResponseWriter, r *http.Request) {
+// SetRemoteTargetHandler - sets a remote target for bucket
+func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetBucketTarget")
 
 	defer logger.AuditLog(w, r, "SetBucketTarget", mustGetClaimsFromToken(r))
@@ -174,6 +173,7 @@ func (a adminAPIHandlers) SetBucketTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL)
 		return
 	}
+	target.SourceBucket = bucket
 	target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target)
 	if target.Arn == "" {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
@@ -183,7 +183,7 @@ func (a adminAPIHandlers) SetBucketTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	targets, err := globalBucketTargetSys.ListTargets(ctx, bucket)
+	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -209,9 +209,9 @@ func (a adminAPIHandlers) SetBucketTargetHandler(w http.ResponseWriter, r *http.
 	writeSuccessResponseJSON(w, data)
 }
 
-// ListBucketTargetsHandler - lists remote target(s) for a bucket or gets a target
+// ListRemoteTargetsHandler - lists remote target(s) for a bucket or gets a target
 // for a particular ARN type
-func (a adminAPIHandlers) ListBucketTargetsHandler(w http.ResponseWriter, r *http.Request) {
+func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListBucketTargets")
 
 	defer logger.AuditLog(w, r, "ListBucketTargets", mustGetClaimsFromToken(r))
@@ -225,28 +225,13 @@ func (a adminAPIHandlers) ListBucketTargetsHandler(w http.ResponseWriter, r *htt
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
-
-	cfg, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	var (
-		targets []madmin.BucketTarget
-		tgt, ct madmin.BucketTarget
-		creds   auth.Credentials
-	)
-	if cfg != nil && !cfg.Empty() {
-		for idx, t := range cfg.Targets {
-			if string(t.Type) == arnType || arnType == "" {
-				ct = cfg.Targets[idx]
-				// remove secretKey from creds
-				creds.AccessKey = ct.Credentials.AccessKey
-				tgt = madmin.BucketTarget{Endpoint: ct.Endpoint, Secure: ct.Secure, TargetBucket: ct.TargetBucket, Credentials: &creds, Arn: ct.Arn, Type: ct.Type}
-				targets = append(targets, tgt)
-			}
+	if bucket != "" {
+		if _, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket); err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
 		}
 	}
+	targets := globalBucketTargetSys.ListTargets(ctx, bucket, arnType)
 	data, err := json.Marshal(targets)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -256,8 +241,8 @@ func (a adminAPIHandlers) ListBucketTargetsHandler(w http.ResponseWriter, r *htt
 	writeSuccessResponseJSON(w, data)
 }
 
-// RemoveBucketTargetHandler - removes a remote target for bucket with specified ARN
-func (a adminAPIHandlers) RemoveBucketTargetHandler(w http.ResponseWriter, r *http.Request) {
+// RemoveRemoteTargetHandler - removes a remote target for bucket with specified ARN
+func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "RemoveBucketTarget")
 
 	defer logger.AuditLog(w, r, "RemoveBucketTarget", mustGetClaimsFromToken(r))
@@ -275,11 +260,18 @@ func (a adminAPIHandlers) RemoveBucketTargetHandler(w http.ResponseWriter, r *ht
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	if err := globalBucketTargetSys.RemoveTarget(ctx, bucket, arn); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	targets, err := globalBucketTargetSys.ListTargets(ctx, bucket)
+	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return

cmd/admin-handlers.go

@@ -799,6 +799,59 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	keepConnLive(w, r, respCh)
 }
 
+func getAggregatedBackgroundHealState(ctx context.Context, failOnErr bool) (madmin.BgHealState, error) {
+	var bgHealStates []madmin.BgHealState
+
+	localHealState, ok := getLocalBackgroundHealStatus()
+	if !ok {
+		if failOnErr {
+			return madmin.BgHealState{}, errServerNotInitialized
+		}
+	}
+
+	// Get local heal status first
+	bgHealStates = append(bgHealStates, localHealState)
+
+	if globalIsDistErasure {
+		// Get heal status from other peers
+		peersHealStates, nerrs := globalNotificationSys.BackgroundHealStatus()
+		for _, nerr := range nerrs {
+			if nerr.Err != nil {
+				if failOnErr {
+					return madmin.BgHealState{}, nerr.Err
+				}
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+		bgHealStates = append(bgHealStates, peersHealStates...)
+	}
+
+	// Aggregate healing result
+	var aggregatedHealStateResult = madmin.BgHealState{
+		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
+		LastHealActivity:  bgHealStates[0].LastHealActivity,
+		NextHealRound:     bgHealStates[0].NextHealRound,
+		HealDisks:         bgHealStates[0].HealDisks,
+	}
+
+	bgHealStates = bgHealStates[1:]
+
+	for _, state := range bgHealStates {
+		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
+		aggregatedHealStateResult.HealDisks = append(aggregatedHealStateResult.HealDisks, state.HealDisks...)
+		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
+			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
+			// The node which has the last heal activity means its
+			// is the node that is orchestrating self healing operations,
+			// which also means it is the same node which decides when
+			// the next self healing operation will be done.
+			aggregatedHealStateResult.NextHealRound = state.NextHealRound
+		}
+	}
+
+	return aggregatedHealStateResult, nil
+}
+
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "HealBackgroundStatus")
 
@@ -815,39 +868,8 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 		return
 	}
 
-	var bgHealStates []madmin.BgHealState
-
-	// Get local heal status first
-	bgHealStates = append(bgHealStates, getLocalBackgroundHealStatus())
-
-	if globalIsDistErasure {
-		// Get heal status from other peers
-		peersHealStates := globalNotificationSys.BackgroundHealStatus()
-		bgHealStates = append(bgHealStates, peersHealStates...)
-	}
-
-	// Aggregate healing result
-	var aggregatedHealStateResult = madmin.BgHealState{
-		ScannedItemsCount: bgHealStates[0].ScannedItemsCount,
-		LastHealActivity:  bgHealStates[0].LastHealActivity,
-		NextHealRound:     bgHealStates[0].NextHealRound,
-	}
-
-	bgHealStates = bgHealStates[1:]
-
-	for _, state := range bgHealStates {
-		aggregatedHealStateResult.ScannedItemsCount += state.ScannedItemsCount
-		if !state.LastHealActivity.IsZero() && aggregatedHealStateResult.LastHealActivity.Before(state.LastHealActivity) {
-			aggregatedHealStateResult.LastHealActivity = state.LastHealActivity
-			// The node which has the last heal activity means its
-			// is the node that is orchestrating self healing operations,
-			// which also means it is the same node which decides when
-			// the next self healing operation will be done.
-			aggregatedHealStateResult.NextHealRound = state.NextHealRound
-		}
-	}
-
-	if err := json.NewEncoder(w).Encode(aggregatedHealStateResult); err != nil {
+	aggregateHealStateResult, _ := getAggregatedBackgroundHealState(r.Context(), false)
+	if err := json.NewEncoder(w).Encode(aggregateHealStateResult); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -1063,7 +1085,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	// Avoid reusing tcp connection if read timeout is hit
 	// This is needed to make r.Context().Done() work as
 	// expected in case of read timeout
-	w.Header().Add("Connection", "close")
+	w.Header().Set("Connection", "close")
 
 	setEventStreamHeaders(w)
 
@@ -1248,10 +1270,9 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	}
 
 	deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
-
 	defer cancel()
 
-	nsLock := objectAPI.NewNSLock(deadlinedCtx, minioMetaBucket, "obd-in-progress")
+	nsLock := objectAPI.NewNSLock(ctx, minioMetaBucket, "obd-in-progress")
 	if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
 		errResp(err)
 		return
@@ -1478,6 +1499,9 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 	// add all the disks local to this server.
 	for _, disk := range storageInfo.Disks {
+		if disk.DrivePath == "" && disk.Endpoint == "" {
+			continue
+		}
 		if disk.Endpoint == disk.DrivePath {
 			servers[len(servers)-1].Disks = append(servers[len(servers)-1].Disks, disk)
 		}
@@ -1558,12 +1582,12 @@ func fetchVaultStatus(cfg config.Config) madmin.Vault {
 	keyID := GlobalKMS.DefaultKeyID()
 	kmsInfo := GlobalKMS.Info()
 
-	if kmsInfo.Endpoint == "" {
+	if len(kmsInfo.Endpoints) == 0 {
 		vault.Status = "KMS configured using master key"
 		return vault
 	}
 
-	if err := checkConnection(kmsInfo.Endpoint, 15*time.Second); err != nil {
+	if err := checkConnection(kmsInfo.Endpoints[0], 15*time.Second); err != nil {
 		vault.Status = "offline"
 	} else {
 		vault.Status = "online"

cmd/admin-heal-ops.go

@@ -88,7 +88,8 @@ type allHealState struct {
 	sync.Mutex
 
 	// map of heal path to heal sequence
-	healSeqMap map[string]*healSequence
+	healSeqMap     map[string]*healSequence
+	healLocalDisks []Endpoints
 }
 
 // newHealState - initialize global heal state management
@@ -102,6 +103,22 @@ func newHealState() *allHealState {
 	return healState
 }
 
+func (ahs *allHealState) getHealLocalDisks() []Endpoints {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	healLocalDisks := make([]Endpoints, len(ahs.healLocalDisks))
+	copy(healLocalDisks, ahs.healLocalDisks)
+	return healLocalDisks
+}
+
+func (ahs *allHealState) updateHealLocalDisks(healLocalDisks []Endpoints) {
+	ahs.Lock()
+	defer ahs.Unlock()
+
+	ahs.healLocalDisks = healLocalDisks
+}
+
 func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) {
 	// Launch clean-up routine to remove this heal sequence (after
 	// it ends) from the global state after timeout has elapsed.
@@ -485,9 +502,12 @@ func (h *healSequence) isQuitting() bool {
 // check if the heal sequence has ended
 func (h *healSequence) hasEnded() bool {
 	h.mutex.RLock()
-	ended := len(h.currentStatus.Items) == 0 || h.currentStatus.Summary == healStoppedStatus || h.currentStatus.Summary == healFinishedStatus
-	h.mutex.RUnlock()
-	return ended
+	defer h.mutex.RUnlock()
+	// background heal never ends
+	if h.clientToken == bgHealingUUID {
+		return false
+	}
+	return len(h.currentStatus.Items) == 0 || h.currentStatus.Summary == healStoppedStatus || h.currentStatus.Summary == healFinishedStatus
 }
 
 // stops the heal sequence - safe to call multiple times.
@@ -627,6 +647,12 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	if source.opts != nil {
 		task.opts = *source.opts
 	}
+
+	h.mutex.Lock()
+	h.scannedItemsMap[healType]++
+	h.lastHealActivity = UTCNow()
+	h.mutex.Unlock()
+
 	globalBackgroundHealRoutine.queueHealTask(task)
 
 	select {
@@ -634,9 +660,11 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		if !h.reportProgress {
 			// Object might have been deleted, by the time heal
 			// was attempted, we should ignore this object and
-			// return success.
+			// return the error and not calculate this object
+			// as part of the metrics.
 			if isErrObjectNotFound(res.err) || isErrVersionNotFound(res.err) {
-				return nil
+				// Return the error so that caller can handle it.
+				return res.err
 			}
 
 			h.mutex.Lock()
@@ -700,14 +728,13 @@ func (h *healSequence) healItemsFromSourceCh() error {
 			if err := h.queueHealTask(source, itemType); err != nil {
 				switch err.(type) {
 				case ObjectExistsAsDirectory:
+				case ObjectNotFound:
+				case VersionNotFound:
 				default:
 					logger.LogIf(h.ctx, fmt.Errorf("Heal attempt failed for %s: %w",
 						pathJoin(source.bucket, source.object), err))
 				}
 			}
-
-			h.scannedItemsMap[itemType]++
-			h.lastHealActivity = UTCNow()
 		case <-h.ctx.Done():
 			return nil
 		}
@@ -773,17 +800,17 @@ func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 				return errHealStopSignalled
 			}
 
-			herr := h.queueHealTask(healSource{
+			err := h.queueHealTask(healSource{
 				bucket:    bucket,
 				object:    object,
 				versionID: versionID,
 			}, madmin.HealItemBucketMetadata)
 			// Object might have been deleted, by the time heal
 			// was attempted we ignore this object an move on.
-			if isErrObjectNotFound(herr) || isErrVersionNotFound(herr) {
+			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 				return nil
 			}
-			return herr
+			return err
 		})
 	}
 }
@@ -844,7 +871,9 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	}
 
 	if err := h.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket); err != nil {
-		return err
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return err
+		}
 	}
 
 	if bucketsOnly {
@@ -855,9 +884,12 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		if h.object != "" {
 			// Check if an object named as the objPrefix exists,
 			// and if so heal it.
-			_, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
+			oi, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
 			if err == nil {
-				if err = h.healObject(bucket, h.object, ""); err != nil {
+				if err = h.healObject(bucket, h.object, oi.VersionID); err != nil {
+					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+						return nil
+					}
 					return err
 				}
 			}
@@ -867,7 +899,11 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	}
 
 	if err := objectAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
-		return errFnHealFromAPIErr(h.ctx, err)
+		// Object might have been deleted, by the time heal
+		// was attempted we ignore this object an move on.
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			return errFnHealFromAPIErr(h.ctx, err)
+		}
 	}
 	return nil
 }
@@ -884,9 +920,10 @@ func (h *healSequence) healObject(bucket, object, versionID string) error {
 		return errHealStopSignalled
 	}
 
-	return h.queueHealTask(healSource{
+	err := h.queueHealTask(healSource{
 		bucket:    bucket,
 		object:    object,
 		versionID: versionID,
 	}, madmin.HealItemObject)
+	return err
 }

cmd/admin-router.go

@@ -183,14 +183,14 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 			}
 			// Bucket replication operations
 			// GetBucketTargetHandler
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-bucket-targets").HandlerFunc(
-				httpTraceHdrs(adminAPI.ListBucketTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
-			// SetBucketTargetHandler
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.SetBucketTargetHandler)).Queries("bucket", "{bucket:.*}")
-			// SetBucketTargetHandler
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-bucket-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.RemoveBucketTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
+				httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+			// SetRemoteTargetHandler
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
+				httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
+			// SetRemoteTargetHandler
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
+				httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
 		}
 
 		// -- Top APIs --

cmd/api-errors.go

@@ -115,6 +115,7 @@ const (
 	ErrBucketRemoteArnInvalid
 	ErrBucketRemoteRemoveDisallowed
 	ErrReplicationTargetNotVersionedError
+	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
 	ErrBucketReplicationDisabledError
@@ -662,20 +663,9 @@ var errorCodes = errorCodeMap{
 		Description:    "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
-	// right Description:    "Error parsing the X-Amz-Credential parameter; incorrect date format \"%s\". This date in the credential must be in the format \"yyyyMMdd\".",
-	// Need changes to make sure variable messages can be constructed.
 	ErrMalformedCredentialDate: {
 		Code:           "AuthorizationQueryParametersError",
-		Description:    "Error parsing the X-Amz-Credential parameter; incorrect date format \"%s\". This date in the credential must be in the format \"yyyyMMdd\".",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
-	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
-	// right Description:    "Error parsing the X-Amz-Credential parameter; the region 'us-east-' is wrong; expecting 'us-east-1'".
-	// Need changes to make sure variable messages can be constructed.
-	ErrMalformedCredentialRegion: {
-		Code:           "AuthorizationQueryParametersError",
-		Description:    "Error parsing the X-Amz-Credential parameter; the region is wrong;",
+		Description:    "Error parsing the X-Amz-Credential parameter; incorrect date format. This date in the credential must be in the format \"yyyyMMdd\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidRegion: {
@@ -683,9 +673,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Region does not match.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
-	// right Description:   "Error parsing the X-Amz-Credential parameter; incorrect service \"s4\". This endpoint belongs to \"s3\".".
-	// Need changes to make sure variable messages can be constructed.
 	ErrInvalidServiceS3: {
 		Code:           "AuthorizationParametersError",
 		Description:    "Error parsing the Credential/X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".",
@@ -696,9 +683,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Error parsing the Credential parameter; incorrect service. This endpoint belongs to \"sts\".",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	// FIXME: Should contain the invalid param set as seen in https://github.com/minio/minio/issues/2385.
-	// Description:   "Error parsing the X-Amz-Credential parameter; incorrect terminal "aws4_reque". This endpoint uses "aws4_request".
-	// Need changes to make sure variable messages can be constructed.
 	ErrInvalidRequestVersion: {
 		Code:           "AuthorizationQueryParametersError",
 		Description:    "Error parsing the X-Amz-Credential parameter; incorrect terminal. This endpoint uses \"aws4_request\".",
@@ -769,8 +753,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Your key is too long",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-
-	// FIXME: Actual XML error response also contains the header which missed in list of signed header parameters.
 	ErrUnsignedHeaders: {
 		Code:           "AccessDenied",
 		Description:    "There were headers present in the request which were not signed",
@@ -871,6 +853,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The replication target does not have versioning enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrReplicationSourceNotVersionedError: {
+		Code:           "ReplicationSourceNotVersionedError",
+		Description:    "The replication source does not have versioning enabled",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrReplicationNeedsVersioningError: {
 		Code:           "InvalidRequest",
 		Description:    "Versioning must be 'Enabled' on the bucket to apply a replication configuration",
@@ -1929,6 +1916,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrBucketRemoteRemoveDisallowed
 	case BucketReplicationTargetNotVersioned:
 		apiErr = ErrReplicationTargetNotVersionedError
+	case BucketReplicationSourceNotVersioned:
+		apiErr = ErrReplicationSourceNotVersionedError
 	case BucketQuotaExceeded:
 		apiErr = ErrAdminBucketQuotaExceeded
 	case *event.ErrInvalidEventName:

cmd/api-headers.go

@@ -128,6 +128,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 			// values to client.
 			continue
 		}
+
+		// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+			continue
+		}
 		var isSet bool
 		for _, userMetadataPrefix := range userMetadataKeyPrefixes {
 			if !strings.HasPrefix(k, userMetadataPrefix) {

cmd/api-response.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/base64"
 	"encoding/xml"
+	"fmt"
 	"net/http"
 	"net/url"
 	"path"
@@ -35,7 +36,7 @@ import (
 const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
 	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 10000                      // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
 	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
 	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
 	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
@@ -564,6 +565,10 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 					// values to client.
 					continue
 				}
+				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+				if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+					continue
+				}
 				content.UserMetadata[k] = v
 			}
 		}
@@ -759,6 +764,10 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set(xhttp.RetryAfter, "120")
+	case "InvalidRegion":
+		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalServerRegion)
+	case "AuthorizationHeaderMalformed":
+		err.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalServerRegion)
 	case "AccessDenied":
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
@@ -817,38 +826,3 @@ func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, er
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }
-
-// writeCustomErrorResponseXML - similar to writeErrorResponse,
-// but accepts the error message directly (this allows messages to be
-// dynamically generated.)
-func writeCustomErrorResponseXML(ctx context.Context, w http.ResponseWriter, err APIError, errBody string, reqURL *url.URL, browser bool) {
-
-	switch err.Code {
-	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
-		// Set retry-after header to indicate user-agents to retry request after 120secs.
-		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-		w.Header().Set(xhttp.RetryAfter, "120")
-	case "AccessDenied":
-		// The request is from browser and also if browser
-		// is enabled we need to redirect.
-		if browser && globalBrowserEnabled {
-			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
-			w.WriteHeader(http.StatusTemporaryRedirect)
-			return
-		}
-	}
-
-	reqInfo := logger.GetReqInfo(ctx)
-	errorResponse := APIErrorResponse{
-		Code:       err.Code,
-		Message:    errBody,
-		Resource:   reqURL.Path,
-		BucketName: reqInfo.BucketName,
-		Key:        reqInfo.ObjectName,
-		RequestID:  w.Header().Get(xhttp.AmzRequestID),
-		HostID:     globalDeploymentID,
-	}
-
-	encodedErrorResponse := encodeResponse(errorResponse)
-	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
-}

cmd/background-heal-ops.go

@@ -55,19 +55,20 @@ func (h *healRoutine) queueHealTask(task healTask) {
 	h.tasks <- task
 }
 
-func waitForLowHTTPReq(tolerance int32) {
+func waitForLowHTTPReq(tolerance int32, maxWait time.Duration) {
+	const wait = 10 * time.Millisecond
+	waitCount := maxWait / wait
+
 	// Bucket notification and http trace are not costly, it is okay to ignore them
 	// while counting the number of concurrent connections
 	tolerance += int32(globalHTTPListen.NumSubscribers() + globalHTTPTrace.NumSubscribers())
 
 	if httpServer := newHTTPServerFn(); httpServer != nil {
-		// Wait at max 10 minute for an inprogress request before proceeding to heal
-		waitCount := 600
 		// Any requests in progress, delay the heal.
 		for (httpServer.GetRequestCount() >= tolerance) &&
 			waitCount > 0 {
 			waitCount--
-			time.Sleep(1 * time.Second)
+			time.Sleep(wait)
 		}
 	}
 }
@@ -82,7 +83,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 			}
 
 			// Wait and proceed if there are active requests
-			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()))
+			waitForLowHTTPReq(int32(globalEndpoints.NEndpoints()), time.Second)
 
 			var res madmin.HealResultItem
 			var err error
@@ -100,6 +101,7 @@ func (h *healRoutine) run(ctx context.Context, objAPI ObjectLayer) {
 				ObjectPathUpdated(path.Join(task.bucket, task.object))
 			}
 			task.responseCh <- healResult{result: res, err: err}
+
 		case <-h.doneCh:
 			return
 		case <-ctx.Done():
@@ -116,24 +118,6 @@ func newHealRoutine() *healRoutine {
 
 }
 
-func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
-	// Run the background healer
-	globalBackgroundHealRoutine = newHealRoutine()
-	go globalBackgroundHealRoutine.run(ctx, objAPI)
-
-	nh := newBgHealSequence()
-	// Heal any disk format and metadata early, if possible.
-	if err := nh.healDiskMeta(); err != nil {
-		if newObjectLayerFn() != nil {
-			// log only in situations, when object layer
-			// has fully initialized.
-			logger.LogIf(nh.ctx, err)
-		}
-	}
-
-	globalBackgroundHealState.LaunchNewHealSequence(nh)
-}
-
 // healDiskFormat - heals format.json, return value indicates if a
 // failure error occurred.
 func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpts) (madmin.HealResultItem, error) {

cmd/background-newdisks-heal-ops.go

@@ -18,6 +18,8 @@ package cmd
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -26,19 +28,23 @@ import (
 
 const defaultMonitorNewDiskInterval = time.Minute * 3
 
-func initLocalDisksAutoHeal(ctx context.Context, objAPI ObjectLayer) {
-	go monitorLocalDisksAndHeal(ctx, objAPI)
-}
-
-// monitorLocalDisksAndHeal - ensures that detected new disks are healed
-//  1. Only the concerned erasure set will be listed and healed
-//  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
+func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	z, ok := objAPI.(*erasureZones)
 	if !ok {
 		return
 	}
 
+	initBackgroundHealing(ctx, objAPI) // start quick background healing
+
+	localDisksInZoneHeal := getLocalDisksToHeal(objAPI)
+	globalBackgroundHealState.updateHealLocalDisks(localDisksInZoneHeal)
+
+	drivesToHeal := getDrivesToHealCount(localDisksInZoneHeal)
+	if drivesToHeal != 0 {
+		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content...",
+			drivesToHeal, defaultMonitorNewDiskInterval))
+	}
+
 	var bgSeq *healSequence
 	var found bool
 
@@ -50,53 +56,99 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 		time.Sleep(time.Second)
 	}
 
+	if drivesToHeal != 0 {
+		// Heal any disk format and metadata early, if possible.
+		if err := bgSeq.healDiskMeta(); err != nil {
+			if newObjectLayerFn() != nil {
+				// log only in situations, when object layer
+				// has fully initialized.
+				logger.LogIf(bgSeq.ctx, err)
+			}
+		}
+	}
+
+	go monitorLocalDisksAndHeal(ctx, z, drivesToHeal, localDisksInZoneHeal, bgSeq)
+}
+
+func getLocalDisksToHeal(objAPI ObjectLayer) []Endpoints {
+	z, ok := objAPI.(*erasureZones)
+	if !ok {
+		return nil
+	}
+
+	// Attempt a heal as the server starts-up first.
+	localDisksInZoneHeal := make([]Endpoints, len(z.zones))
+	for i, ep := range globalEndpoints {
+		localDisksToHeal := Endpoints{}
+		for _, endpoint := range ep.Endpoints {
+			if !endpoint.IsLocal {
+				continue
+			}
+			// Try to connect to the current endpoint
+			// and reformat if the current disk is not formatted
+			_, _, err := connectEndpoint(endpoint)
+			if errors.Is(err, errUnformattedDisk) {
+				localDisksToHeal = append(localDisksToHeal, endpoint)
+			}
+		}
+		if len(localDisksToHeal) == 0 {
+			continue
+		}
+		localDisksInZoneHeal[i] = localDisksToHeal
+	}
+	return localDisksInZoneHeal
+
+}
+
+func getDrivesToHealCount(localDisksInZoneHeal []Endpoints) int {
+	var drivesToHeal int
+	for _, eps := range localDisksInZoneHeal {
+		for range eps {
+			drivesToHeal++
+		}
+	}
+	return drivesToHeal
+}
+
+func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
+	// Run the background healer
+	globalBackgroundHealRoutine = newHealRoutine()
+	go globalBackgroundHealRoutine.run(ctx, objAPI)
+
+	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence())
+}
+
+// monitorLocalDisksAndHeal - ensures that detected new disks are healed
+//  1. Only the concerned erasure set will be listed and healed
+//  2. Only the node hosting the disk is responsible to perform the heal
+func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, drivesToHeal int, localDisksInZoneHeal []Endpoints, bgSeq *healSequence) {
 	// Perform automatic disk healing when a disk is replaced locally.
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-time.After(defaultMonitorNewDiskInterval):
-			// Attempt a heal as the server starts-up first.
-			localDisksInZoneHeal := make([]Endpoints, len(z.zones))
-			var healNewDisks bool
-			for i, ep := range globalEndpoints {
-				localDisksToHeal := Endpoints{}
-				for _, endpoint := range ep.Endpoints {
-					if !endpoint.IsLocal {
-						continue
-					}
-					// Try to connect to the current endpoint
-					// and reformat if the current disk is not formatted
-					_, _, err := connectEndpoint(endpoint)
-					if err == errUnformattedDisk {
-						localDisksToHeal = append(localDisksToHeal, endpoint)
-					}
-				}
-				if len(localDisksToHeal) == 0 {
+			// heal only if new disks found.
+			if drivesToHeal == 0 {
+				localDisksInZoneHeal = getLocalDisksToHeal(z)
+				drivesToHeal = getDrivesToHealCount(localDisksInZoneHeal)
+				if drivesToHeal == 0 {
+					// No drives to heal.
+					globalBackgroundHealState.updateHealLocalDisks(nil)
 					continue
 				}
-				localDisksInZoneHeal[i] = localDisksToHeal
-				healNewDisks = true
+				globalBackgroundHealState.updateHealLocalDisks(localDisksInZoneHeal)
+
+				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
+					drivesToHeal))
+
+				// Reformat disks
+				bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
+
+				// Ensure that reformatting disks is finished
+				bgSeq.sourceCh <- healSource{bucket: nopHeal}
 			}
 
-			// Reformat disks only if needed.
-			if !healNewDisks {
-				continue
-			}
-
-			logger.Info("New unformatted drives detected attempting to heal...")
-			for i, disks := range localDisksInZoneHeal {
-				for _, disk := range disks {
-					logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
-				}
-			}
-
-			// Reformat disks
-			bgSeq.sourceCh <- healSource{bucket: SlashSeparator}
-
-			// Ensure that reformatting disks is finished
-			bgSeq.sourceCh <- healSource{bucket: nopHeal}
-
 			var erasureSetInZoneToHeal = make([][]int, len(localDisksInZoneHeal))
 			// Compute the list of erasure set to heal
 			for i, localDisksToHeal := range localDisksInZoneHeal {
@@ -108,6 +160,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 						printEndpointError(endpoint, err, true)
 						continue
 					}
+
 					// Calculate the set index where the current endpoint belongs
 					setIndex, _, err := findDiskIndex(z.zones[i].format, format)
 					if err != nil {
@@ -120,13 +173,25 @@ func monitorLocalDisksAndHeal(ctx context.Context, objAPI ObjectLayer) {
 				erasureSetInZoneToHeal[i] = erasureSetToHeal
 			}
 
+			logger.Info("New unformatted drives detected attempting to heal the content...")
+			for i, disks := range localDisksInZoneHeal {
+				for _, disk := range disks {
+					logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
+				}
+			}
+
 			// Heal all erasure sets that need
 			for i, erasureSetToHeal := range erasureSetInZoneToHeal {
 				for _, setIndex := range erasureSetToHeal {
-					err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex], z.zones[i].drivesPerSet)
+					err := healErasureSet(ctx, setIndex, z.zones[i].sets[setIndex], z.zones[i].setDriveCount)
 					if err != nil {
 						logger.LogIf(ctx, err)
 					}
+
+					// Only upon success reduce the counter
+					if err == nil {
+						drivesToHeal--
+					}
 				}
 			}
 		}

cmd/bitrot-streaming.go

@@ -139,8 +139,8 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 	b.h.Write(buf)
 
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		err := &errHashMismatch{fmt.Sprintf("hashes do not match expected %s, got %s",
-			hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
+		err := &errHashMismatch{fmt.Sprintf("Disk: %s - content hash does not match - expected %s, got %s",
+			b.disk, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil)))}
 		logger.LogIf(GlobalContext, err)
 		return 0, err
 	}

cmd/bitrot-whole.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"fmt"
 	"hash"
 	"io"
 
@@ -35,12 +36,12 @@ type wholeBitrotWriter struct {
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
 	err := b.disk.AppendFile(b.volume, b.filePath, p)
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
+		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	_, err = b.Hash.Write(p)
 	if err != nil {
-		logger.LogIf(GlobalContext, err)
+		logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	return len(p), nil
@@ -69,9 +70,7 @@ func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error)
 	if b.buf == nil {
 		b.buf = make([]byte, b.tillOffset-offset)
 		if _, err := b.disk.ReadFile(b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			ctx := GlobalContext
-			logger.GetReqInfo(ctx).AppendTags("disk", b.disk.String())
-			logger.LogIf(ctx, err)
+			logger.LogIf(GlobalContext, fmt.Errorf("Disk: %s returned %w", b.disk, err))
 			return 0, err
 		}
 	}

cmd/bitrot_test.go

@@ -34,7 +34,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 	volume := "testvol"
 	filePath := "testfile"
 
-	disk, err := newXLStorage(tmpDir, "")
+	disk, err := newLocalXLStorage(tmpDir)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/bucket-handlers.go

@@ -449,14 +449,15 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	deleteList := toNames(objectsToDelete)
 	dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{
-		Versioned: globalBucketVersioningSys.Enabled(bucket),
+		Versioned:        globalBucketVersioningSys.Enabled(bucket),
+		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
 	})
 
 	deletedObjects := make([]DeletedObject, len(deleteObjects.Objects))
 	for i := range errs {
 		dindex := objectsToDelete[deleteList[i]]
 		apiErr := toAPIError(ctx, errs[i])
-		if apiErr.Code == "" || apiErr.Code == "NoSuchKey" {
+		if apiErr.Code == "" || apiErr.Code == "NoSuchKey" || apiErr.Code == "InvalidArgument" {
 			deletedObjects[dindex] = dObjects[i]
 			continue
 		}
@@ -762,7 +763,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 		// Make sure formValues adhere to policy restrictions.
 		if err = checkPostPolicy(formValues, postPolicyForm); err != nil {
-			writeCustomErrorResponseXML(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), err.Error(), r.URL, guessIsBrowserReq(r))
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrAccessDenied, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
 
@@ -804,7 +805,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	if _, err = globalBucketSSEConfigSys.Get(bucket); err == nil || globalAutoEncryption {
 		// This request header needs to be set prior to setting ObjectOptions
 		if !crypto.SSEC.IsRequested(r.Header) {
-			r.Header.Add(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
+			r.Header.Set(crypto.SSEHeader, crypto.SSEAlgorithmAES256)
 		}
 	}
 

cmd/bucket-listobjects-handlers.go

@@ -24,12 +24,36 @@ import (
 	"strings"
 
 	"github.com/gorilla/mux"
-	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/minio/cmd/logger"
 
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
+func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
+	inParallel := func(objects []ObjectInfo) {
+		g := errgroup.WithNErrs(len(objects))
+		for index := range objects {
+			index := index
+			g.Go(func() error {
+				objects[index].ETag = objects[index].GetActualETag(nil)
+				objects[index].Size, _ = objects[index].GetActualSize()
+				return nil
+			}, index)
+		}
+		g.Wait()
+	}
+	const maxConcurrent = 500
+	for {
+		if len(objects) < maxConcurrent {
+			inParallel(objects)
+			return
+		}
+		inParallel(objects[:maxConcurrent])
+		objects = objects[maxConcurrent:]
+	}
+}
+
 // Validate all the ListObjects query arguments, returns an APIErrorCode
 // if one of the args do not meet the required conditions.
 // Special conditions required by MinIO server are as below
@@ -89,6 +113,10 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
+	if proxyRequestByBucket(ctx, w, r, bucket) {
+		return
+	}
+
 	listObjectVersions := objectAPI.ListObjectVersions
 
 	// Inititate a list object versions operation based on the input params.
@@ -100,16 +128,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	for i := range listObjectVersionsInfo.Objects {
-		if crypto.IsEncrypted(listObjectVersionsInfo.Objects[i].UserDefined) {
-			listObjectVersionsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectVersionsInfo.Objects[i], false)
-		}
-		listObjectVersionsInfo.Objects[i].Size, err = listObjectVersionsInfo.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectVersionsInfo.Objects)
 
 	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
 
@@ -178,16 +197,7 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	for i := range listObjectsV2Info.Objects {
-		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
-			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-		}
-		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
 
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
@@ -264,16 +274,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	for i := range listObjectsV2Info.Objects {
-		if crypto.IsEncrypted(listObjectsV2Info.Objects[i].UserDefined) {
-			listObjectsV2Info.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsV2Info.Objects[i], false)
-		}
-		listObjectsV2Info.Objects[i].Size, err = listObjectsV2Info.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
 
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
@@ -396,16 +397,7 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	for i := range listObjectsInfo.Objects {
-		if crypto.IsEncrypted(listObjectsInfo.Objects[i].UserDefined) {
-			listObjectsInfo.Objects[i].ETag = getDecryptedETag(r.Header, listObjectsInfo.Objects[i], false)
-		}
-		listObjectsInfo.Objects[i].Size, err = listObjectsInfo.Objects[i].GetActualSize()
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-			return
-		}
-	}
+	concurrentDecryptETag(ctx, listObjectsInfo.Objects)
 
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
 

cmd/bucket-metadata-sys.go

@@ -24,6 +24,7 @@ import (
 	"sync"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/cmd/logger"
 	bucketsse "github.com/minio/minio/pkg/bucket/encryption"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
@@ -80,7 +81,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		// This code is needed only for gateway implementations.
 		switch configFile {
 		case bucketSSEConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -89,7 +90,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 				return meta.Save(GlobalContext, objAPI)
 			}
 		case bucketLifecycleConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -98,7 +99,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 				return meta.Save(GlobalContext, objAPI)
 			}
 		case bucketTaggingConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -107,7 +108,7 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 				return meta.Save(GlobalContext, objAPI)
 			}
 		case bucketNotificationConfig:
-			if globalGatewayName == "nas" {
+			if globalGatewayName == NASBackendGateway {
 				meta, err := loadBucketMetadata(GlobalContext, objAPI, bucket)
 				if err != nil {
 					return err
@@ -148,15 +149,27 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 		meta.EncryptionConfigXML = configData
 	case bucketTaggingConfig:
 		meta.TaggingConfigXML = configData
-	case objectLockConfig:
-		meta.ObjectLockConfigXML = configData
-	case bucketVersioningConfig:
-		meta.VersioningConfigXML = configData
 	case bucketQuotaConfigFile:
 		meta.QuotaConfigJSON = configData
+	case objectLockConfig:
+		if !globalIsErasure && !globalIsDistErasure {
+			return NotImplemented{}
+		}
+		meta.ObjectLockConfigXML = configData
+	case bucketVersioningConfig:
+		if !globalIsErasure && !globalIsDistErasure {
+			return NotImplemented{}
+		}
+		meta.VersioningConfigXML = configData
 	case bucketReplicationConfig:
+		if !globalIsErasure && !globalIsDistErasure {
+			return NotImplemented{}
+		}
 		meta.ReplicationConfigXML = configData
 	case bucketTargetsFile:
+		if !globalIsErasure && !globalIsDistErasure {
+			return NotImplemented{}
+		}
 		meta.BucketTargetsConfigJSON = configData
 	default:
 		return fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
@@ -176,6 +189,13 @@ func (sys *BucketMetadataSys) Update(bucket string, configFile string, configDat
 // If no metadata exists errConfigNotFound is returned and a new metadata is returned.
 // Only a shallow copy is returned, so referenced data should not be modified,
 // but can be replaced atomically.
+//
+// This function should only be used with
+// - GetBucketInfo
+// - ListBuckets
+// - ListBucketsHeal (only in case of erasure coding mode)
+// For all other bucket specific metadata, use the relevant
+// calls implemented specifically for each of those features.
 func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
 	if globalIsGateway || bucket == minioMetaBucket {
 		return newBucketMetadata(bucket), errConfigNotFound
@@ -253,7 +273,7 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 // GetNotificationConfig returns configured notification config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
-	if globalIsGateway && globalGatewayName == "nas" {
+	if globalIsGateway && globalGatewayName == NASBackendGateway {
 		// Only needed in case of NAS gateway.
 		objAPI := newObjectLayerWithoutSafeModeFn()
 		if objAPI == nil {
@@ -397,8 +417,9 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 		return nil
 	}
 
-	// Load PolicySys once during boot.
-	return sys.load(ctx, buckets, objAPI)
+	// Load bucket metadata sys in background
+	go logger.LogIf(ctx, sys.load(ctx, buckets, objAPI))
+	return nil
 }
 
 // concurrently load bucket metadata to speed up loading bucket metadata.

cmd/bucket-policy.go

@@ -68,6 +68,12 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	principalType := "Anonymous"
 	if username != "" {
 		principalType = "User"
+		if len(claims) > 0 {
+			principalType = "AssumedRole"
+		}
+		if username == globalActiveCred.AccessKey {
+			principalType = "Account"
+		}
 	}
 
 	vid := r.URL.Query().Get("versionId")
@@ -143,7 +149,12 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	for k, v := range claims {
 		vStr, ok := v.(string)
 		if ok {
-			args[k] = []string{vStr}
+			// Special case for AD/LDAP STS users
+			if k == ldapUser {
+				args["user"] = []string{vStr}
+			} else {
+				args[k] = []string{vStr}
+			}
 		}
 	}
 

cmd/bucket-replication.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"net/http"
+	"strings"
 	"time"
 
 	miniogo "github.com/minio/minio-go/v7"
@@ -49,7 +50,7 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
 // validateReplicationDestination returns error if replication destination bucket missing or not configured
 // It also returns true if replication destination is same as this server.
 func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config) (bool, error) {
-	clnt := globalBucketTargetSys.GetReplicationTargetClient(ctx, rCfg.ReplicationArn)
+	clnt := globalBucketTargetSys.GetReplicationTargetClient(ctx, rCfg.RoleArn)
 	if clnt == nil {
 		return false, BucketRemoteTargetNotFound{Bucket: bucket}
 	}
@@ -65,7 +66,7 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 		}
 	}
 	// validate replication ARN against target endpoint
-	c, ok := globalBucketTargetSys.arnRemotesMap[rCfg.ReplicationArn]
+	c, ok := globalBucketTargetSys.arnRemotesMap[rCfg.RoleArn]
 	if ok {
 		if c.EndpointURL().String() == clnt.EndpointURL().String() {
 			sameTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
@@ -75,8 +76,23 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 	return false, BucketRemoteTargetNotFound{Bucket: bucket}
 }
 
+func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) bool {
+	if permErr != ErrNone {
+		return false
+	}
+	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
+}
+
 // mustReplicate returns true if object meets replication criteria.
 func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
+	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
+		return false
+	}
+	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
+}
+
+// mustReplicater returns true if object meets replication criteria.
+func mustReplicater(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
 	if globalIsGateway {
 		return false
 	}
@@ -86,9 +102,6 @@ func mustReplicate(ctx context.Context, r *http.Request, bucket, object string,
 	if replication.StatusType(replStatus) == replication.Replica {
 		return false
 	}
-	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
-		return false
-	}
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
 		return false
@@ -110,22 +123,29 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 		if k == xhttp.AmzBucketReplicationStatus {
 			continue
 		}
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+			continue
+		}
 		meta[k] = v
 	}
-
 	tag, err := tags.ParseObjectTags(objInfo.UserTags)
 	if err != nil {
 		return
 	}
+	sc := dest.StorageClass
+	if sc == "" {
+		sc = objInfo.StorageClass
+	}
 	putOpts = miniogo.PutObjectOptions{
 		UserMetadata:         meta,
 		UserTags:             tag.ToMap(),
 		ContentType:          objInfo.ContentType,
 		ContentEncoding:      objInfo.ContentEncoding,
-		StorageClass:         dest.StorageClass,
+		StorageClass:         sc,
 		ReplicationVersionID: objInfo.VersionID,
 		ReplicationStatus:    miniogo.ReplicationStatusReplica,
 		ReplicationMTime:     objInfo.ModTime,
+		ReplicationETag:      objInfo.ETag,
 	}
 	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
 		rmode := miniogo.RetentionMode(mode)
@@ -155,7 +175,7 @@ func replicateObject(ctx context.Context, bucket, object, versionID string, obje
 		logger.LogIf(ctx, err)
 		return
 	}
-	tgt := globalBucketTargetSys.GetReplicationTargetClient(ctx, cfg.ReplicationArn)
+	tgt := globalBucketTargetSys.GetReplicationTargetClient(ctx, cfg.RoleArn)
 	if tgt == nil {
 		return
 	}
@@ -215,3 +235,26 @@ func replicateObject(ctx context.Context, bucket, object, versionID string, obje
 		logger.LogIf(ctx, err)
 	}
 }
+
+// filterReplicationStatusMetadata filters replication status metadata for COPY
+func filterReplicationStatusMetadata(metadata map[string]string) map[string]string {
+	// Copy on write
+	dst := metadata
+	var copied bool
+	delKey := func(key string) {
+		if _, ok := metadata[key]; !ok {
+			return
+		}
+		if !copied {
+			dst = make(map[string]string, len(metadata))
+			for k, v := range metadata {
+				dst[k] = v
+			}
+			copied = true
+		}
+		delete(dst, key)
+	}
+
+	delKey(xhttp.AmzBucketReplicationStatus)
+	return dst
+}

cmd/bucket-targets.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"sync"
 
@@ -36,16 +35,42 @@ type BucketTargetSys struct {
 	clientsCache  map[string]*miniogo.Core
 }
 
-// ListTargets - gets list of bucket targets for this bucket.
-func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) {
-	if globalIsGateway {
-		return nil, nil
+// ListTargets lists bucket targets across tenant or for individual bucket, and returns
+// results filtered by arnType
+func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType string) (targets []madmin.BucketTarget) {
+	if bucket != "" {
+		if ts, err := sys.ListBucketTargets(ctx, bucket); err == nil {
+			for _, t := range ts.Targets {
+				if string(t.Type) == arnType || arnType == "" {
+					targets = append(targets, t.Clone())
+				}
+			}
+		}
+		return targets
 	}
+	sys.RLock()
+	defer sys.RUnlock()
+	for _, tgts := range sys.targetsMap {
+		for _, t := range tgts {
+			if string(t.Type) == arnType || arnType == "" {
+				targets = append(targets, t.Clone())
+			}
+		}
+	}
+	return
+}
+
+// ListBucketTargets - gets list of bucket targets for this bucket.
+func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) {
+
+	sys.RLock()
+	defer sys.RUnlock()
+
 	tgts, ok := sys.targetsMap[bucket]
 	if ok {
 		return &madmin.BucketTargets{Targets: tgts}, nil
 	}
-	return nil, fmt.Errorf("No remote targets exist for bucket %s", bucket)
+	return nil, BucketRemoteTargetNotFound{Bucket: bucket}
 }
 
 // SetTarget - sets a new minio-go client target for this bucket.
@@ -60,14 +85,19 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	if err != nil {
 		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
 	}
-
-	if tgt.Type == madmin.ReplicationArn {
+	if tgt.Type == madmin.ReplicationService {
+		if !globalBucketVersioningSys.Enabled(bucket) {
+			return BucketReplicationSourceNotVersioned{Bucket: bucket}
+		}
 		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
-		if err != nil || vcfg.Status != string(versioning.Enabled) {
+		if err != nil {
 			if isErrBucketNotFound(err) {
 				return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
 			}
-			return BucketReplicationTargetNotVersioned{Bucket: tgt.TargetBucket}
+			if vcfg.Status != string(versioning.Enabled) {
+				return BucketReplicationTargetNotVersioned{Bucket: tgt.TargetBucket}
+			}
+			return err
 		}
 	}
 
@@ -112,10 +142,10 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	if err != nil {
 		return BucketRemoteArnInvalid{Bucket: bucket}
 	}
-	if arn.Type == madmin.ReplicationArn {
+	if arn.Type == madmin.ReplicationService {
 		// reject removal of remote target if replication configuration is present
 		rcfg, err := getReplicationConfig(ctx, bucket)
-		if err == nil && rcfg.ReplicationArn == arnStr {
+		if err == nil && rcfg.RoleArn == arnStr {
 			if _, ok := sys.arnRemotesMap[arnStr]; ok {
 				return BucketRemoteRemoveDisallowed{Bucket: bucket}
 			}
@@ -125,11 +155,17 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	sys.Lock()
 	defer sys.Unlock()
 	targets := make([]madmin.BucketTarget, 0)
+	found := false
 	tgts := sys.targetsMap[bucket]
 	for _, tgt := range tgts {
 		if tgt.Arn != arnStr {
 			targets = append(targets, tgt)
+			continue
 		}
+		found = true
+	}
+	if !found {
+		return BucketRemoteTargetNotFound{Bucket: bucket}
 	}
 	sys.targetsMap[bucket] = targets
 	delete(sys.arnRemotesMap, arnStr)
@@ -163,11 +199,45 @@ func (sys *BucketTargetSys) Init(ctx context.Context, buckets []BucketInfo, objA
 		return nil
 	}
 
-	// Load bucket targets once during boot.
-	sys.load(ctx, buckets, objAPI)
+	// Load bucket targets once during boot in background.
+	go sys.load(ctx, buckets, objAPI)
 	return nil
 }
 
+// UpdateTarget updates target to reflect metadata updates
+func (sys *BucketTargetSys) UpdateTarget(bucket string, cfg *madmin.BucketTargets) {
+	if sys == nil {
+		return
+	}
+	sys.Lock()
+	defer sys.Unlock()
+	if cfg == nil || cfg.Empty() {
+		// remove target and arn association
+		if tgts, ok := sys.targetsMap[bucket]; ok {
+			for _, t := range tgts {
+				delete(sys.arnRemotesMap, t.Arn)
+			}
+		}
+		delete(sys.targetsMap, bucket)
+		return
+	}
+
+	if len(cfg.Targets) > 0 {
+		sys.targetsMap[bucket] = cfg.Targets
+	}
+	for _, tgt := range cfg.Targets {
+		tgtClient, err := sys.getRemoteTargetClient(&tgt)
+		if err != nil {
+			continue
+		}
+		sys.arnRemotesMap[tgt.Arn] = tgtClient
+		if _, ok := sys.clientsCache[tgtClient.EndpointURL().String()]; !ok {
+			sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
+		}
+	}
+	sys.targetsMap[bucket] = cfg.Targets
+}
+
 // create minio-go clients for buckets having remote targets
 func (sys *BucketTargetSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
 	for _, bucket := range buckets {
@@ -229,7 +299,7 @@ func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTar
 			return tgt.Arn
 		}
 	}
-	if !madmin.ArnType(target.Type).IsValid() {
+	if !madmin.ServiceType(target.Type).IsValid() {
 		return ""
 	}
 	arn := madmin.ARN{

cmd/config-current.go

@@ -455,28 +455,32 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 	for _, l := range loggerCfg.HTTP {
 		if l.Enabled {
 			// Enable http logging
-			logger.AddTarget(
+			if err = logger.AddTarget(
 				http.New(http.WithEndpoint(l.Endpoint),
 					http.WithAuthToken(l.AuthToken),
 					http.WithUserAgent(loggerUserAgent),
 					http.WithLogKind(string(logger.All)),
 					http.WithTransport(NewGatewayHTTPTransport()),
 				),
-			)
+			); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize console HTTP target: %w", err))
+			}
 		}
 	}
 
 	for _, l := range loggerCfg.Audit {
 		if l.Enabled {
 			// Enable http audit logging
-			logger.AddAuditTarget(
+			if err = logger.AddAuditTarget(
 				http.New(http.WithEndpoint(l.Endpoint),
 					http.WithAuthToken(l.AuthToken),
 					http.WithUserAgent(loggerUserAgent),
 					http.WithLogKind(string(logger.All)),
 					http.WithTransport(NewGatewayHTTPTransport()),
 				),
-			)
+			); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit HTTP target: %w", err))
+			}
 		}
 	}
 

cmd/config/certs.go

@@ -24,8 +24,6 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"io/ioutil"
-	"os"
-	"path"
 
 	"github.com/minio/minio/pkg/env"
 )
@@ -69,38 +67,6 @@ func ParsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 	return x509Certs, nil
 }
 
-// GetRootCAs - returns all the root CAs into certPool
-// at the input certsCADir
-func GetRootCAs(certsCAsDir string) (*x509.CertPool, error) {
-	rootCAs, _ := x509.SystemCertPool()
-	if rootCAs == nil {
-		// In some systems (like Windows) system cert pool is
-		// not supported or no certificates are present on the
-		// system - so we create a new cert pool.
-		rootCAs = x509.NewCertPool()
-	}
-
-	fis, err := ioutil.ReadDir(certsCAsDir)
-	if err != nil {
-		if os.IsNotExist(err) || os.IsPermission(err) {
-			// Return success if CA's directory is missing or permission denied.
-			err = nil
-		}
-		return rootCAs, err
-	}
-
-	// Load all custom CA files.
-	for _, fi := range fis {
-		caCert, err := ioutil.ReadFile(path.Join(certsCAsDir, fi.Name()))
-		if err != nil {
-			// ignore files which are not readable.
-			continue
-		}
-		rootCAs.AppendCertsFromPEM(caCert)
-	}
-	return rootCAs, nil
-}
-
 // LoadX509KeyPair - load an X509 key pair (private key , certificate)
 // from the provided paths. The private key may be encrypted and is
 // decrypted using the ENV_VAR: MINIO_CERT_PASSWD.

cmd/config/certs_test.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path/filepath"
 	"runtime"
 	"testing"
 )
@@ -194,60 +193,6 @@ M9ofSEt/bdRD
 	}
 }
 
-func TestGetRootCAs(t *testing.T) {
-	emptydir, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", emptydir)
-	}
-	defer os.RemoveAll(emptydir)
-
-	dir1, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", dir1)
-	}
-	defer os.RemoveAll(dir1)
-	if err = os.Mkdir(filepath.Join(dir1, "empty-dir"), 0755); err != nil {
-		t.Fatalf("Unable create empty dir. %v", err)
-	}
-
-	dir2, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", dir2)
-	}
-	defer os.RemoveAll(dir2)
-	if err = ioutil.WriteFile(filepath.Join(dir2, "empty-file"), []byte{}, 0644); err != nil {
-		t.Fatalf("Unable create test file. %v", err)
-	}
-
-	testCases := []struct {
-		certCAsDir  string
-		expectedErr error
-	}{
-		// ignores non-existent directories.
-		{"nonexistent-dir", nil},
-		// Ignores directories.
-		{dir1, nil},
-		// Ignore empty directory.
-		{emptydir, nil},
-		// Loads the cert properly.
-		{dir2, nil},
-	}
-
-	for _, testCase := range testCases {
-		_, err := GetRootCAs(testCase.certCAsDir)
-
-		if testCase.expectedErr == nil {
-			if err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
-			}
-		} else if err == nil {
-			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else if testCase.expectedErr.Error() != err.Error() {
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
-		}
-	}
-}
-
 func TestLoadX509KeyPair(t *testing.T) {
 	for i, testCase := range loadX509KeyPairTests {
 		privateKey, err := createTempFile("private.key", testCase.privateKey)

cmd/config/constants.go

@@ -31,11 +31,12 @@ const (
 	EnvDomain       = "MINIO_DOMAIN"
 	EnvRegionName   = "MINIO_REGION_NAME"
 	EnvPublicIPs    = "MINIO_PUBLIC_IPS"
-	EnvEndpoints    = "MINIO_ENDPOINTS"
 	EnvFSOSync      = "MINIO_FS_OSYNC"
+	EnvArgs         = "MINIO_ARGS"
 
 	EnvUpdate = "MINIO_UPDATE"
 
-	EnvWorm   = "MINIO_WORM"   // legacy
-	EnvRegion = "MINIO_REGION" // legacy
+	EnvEndpoints = "MINIO_ENDPOINTS" // legacy
+	EnvWorm      = "MINIO_WORM"      // legacy
+	EnvRegion    = "MINIO_REGION"    // legacy
 )

cmd/config/identity/openid/jwt.go

@@ -25,6 +25,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	jwtgo "github.com/dgrijalva/jwt-go"
@@ -49,10 +50,14 @@ type Config struct {
 	publicKeys   map[string]crypto.PublicKey
 	transport    *http.Transport
 	closeRespFn  func(io.ReadCloser)
+	mutex        *sync.Mutex
 }
 
 // PopulatePublicKey - populates a new publickey from the JWKS URL.
 func (r *Config) PopulatePublicKey() error {
+	r.mutex.Lock()
+	defer r.mutex.Unlock()
+
 	if r.JWKS.URL == nil || r.JWKS.URL.String() == "" {
 		return nil
 	}
@@ -185,7 +190,15 @@ func (p *JWT) Validate(token, dsecs string) (map[string]interface{}, error) {
 	var claims jwtgo.MapClaims
 	jwtToken, err := jp.ParseWithClaims(token, &claims, keyFuncCallback)
 	if err != nil {
-		return nil, err
+		// Re-populate the public key in-case the JWKS
+		// pubkeys are refreshed
+		if err = p.PopulatePublicKey(); err != nil {
+			return nil, err
+		}
+		jwtToken, err = jwtgo.ParseWithClaims(token, &claims, keyFuncCallback)
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	if !jwtToken.Valid {
@@ -317,6 +330,7 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 		ClientID:    env.Get(EnvIdentityOpenIDClientID, kvs.Get(ClientID)),
 		transport:   transport,
 		closeRespFn: closeRespFn,
+		mutex:       &sync.Mutex{}, // allocate for copying
 	}
 
 	configURL := env.Get(EnvIdentityOpenIDURL, kvs.Get(ConfigURL))

cmd/config/identity/openid/jwt_test.go

@@ -20,6 +20,7 @@ import (
 	"crypto"
 	"encoding/json"
 	"net/url"
+	"sync"
 	"testing"
 	"time"
 
@@ -89,6 +90,7 @@ func TestJWTAzureFail(t *testing.T) {
 	}
 
 	cfg := Config{}
+	cfg.mutex = &sync.Mutex{}
 	cfg.JWKS.URL = u1
 	cfg.publicKeys = keys
 	jwt := NewJWT(cfg)
@@ -136,6 +138,7 @@ func TestJWT(t *testing.T) {
 	}
 
 	cfg := Config{}
+	cfg.mutex = &sync.Mutex{}
 	cfg.JWKS.URL = u1
 	cfg.publicKeys = keys
 	jwt := NewJWT(cfg)

cmd/config/notify/help.go

@@ -559,6 +559,18 @@ var (
 			Optional:    true,
 			Type:        "number",
 		},
+		config.HelpKV{
+			Key:         target.ElasticUsername,
+			Description: "username for Elasticsearch basic-auth",
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         target.ElasticPassword,
+			Description: "password for Elasticsearch basic-auth",
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

cmd/config/notify/legacy.go

@@ -201,6 +201,14 @@ func SetNotifyES(s config.Config, esName string, cfg target.ElasticsearchArgs) e
 			Key:   target.ElasticQueueLimit,
 			Value: strconv.Itoa(int(cfg.QueueLimit)),
 		},
+		config.KV{
+			Key:   target.ElasticUsername,
+			Value: cfg.Username,
+		},
+		config.KV{
+			Key:   target.ElasticPassword,
+			Value: cfg.Password,
+		},
 	}
 
 	return nil

cmd/config/notify/parse.go

@@ -118,7 +118,7 @@ func FetchRegisteredTargets(cfg config.Config, doneCh <-chan struct{}, transport
 		return nil, err
 	}
 
-	esTargets, err := GetNotifyES(cfg[config.NotifyESSubSys])
+	esTargets, err := GetNotifyES(cfg[config.NotifyESSubSys], transport)
 	if err != nil {
 		return nil, err
 	}
@@ -1532,11 +1532,19 @@ var (
 			Key:   target.ElasticQueueLimit,
 			Value: "0",
 		},
+		config.KV{
+			Key:   target.ElasticUsername,
+			Value: "",
+		},
+		config.KV{
+			Key:   target.ElasticPassword,
+			Value: "",
+		},
 	}
 )
 
 // GetNotifyES - returns a map of registered notification 'elasticsearch' targets
-func GetNotifyES(esKVS map[string]config.KVS) (map[string]target.ElasticsearchArgs, error) {
+func GetNotifyES(esKVS map[string]config.KVS, transport *http.Transport) (map[string]target.ElasticsearchArgs, error) {
 	esTargets := make(map[string]target.ElasticsearchArgs)
 	for k, kv := range mergeTargets(esKVS, target.EnvElasticEnable, DefaultESKVS) {
 		enableEnv := target.EnvElasticEnable
@@ -1586,6 +1594,16 @@ func GetNotifyES(esKVS map[string]config.KVS) (map[string]target.ElasticsearchAr
 			queueDirEnv = queueDirEnv + config.Default + k
 		}
 
+		usernameEnv := target.EnvElasticUsername
+		if k != config.Default {
+			usernameEnv = usernameEnv + config.Default + k
+		}
+
+		passwordEnv := target.EnvElasticPassword
+		if k != config.Default {
+			passwordEnv = passwordEnv + config.Default + k
+		}
+
 		esArgs := target.ElasticsearchArgs{
 			Enable:     enabled,
 			Format:     env.Get(formatEnv, kv.Get(target.ElasticFormat)),
@@ -1593,6 +1611,9 @@ func GetNotifyES(esKVS map[string]config.KVS) (map[string]target.ElasticsearchAr
 			Index:      env.Get(indexEnv, kv.Get(target.ElasticIndex)),
 			QueueDir:   env.Get(queueDirEnv, kv.Get(target.ElasticQueueDir)),
 			QueueLimit: uint64(queueLimit),
+			Transport:  transport,
+			Username:   env.Get(usernameEnv, kv.Get(target.ElasticUsername)),
+			Password:   env.Get(passwordEnv, kv.Get(target.ElasticPassword)),
 		}
 		if err = esArgs.Validate(); err != nil {
 			return nil, err

cmd/config/storageclass/storage-class.go

@@ -156,7 +156,7 @@ func parseStorageClass(storageClassEnv string) (sc StorageClass, err error) {
 }
 
 // Validates the parity disks.
-func validateParity(ssParity, rrsParity, drivesPerSet int) (err error) {
+func validateParity(ssParity, rrsParity, setDriveCount int) (err error) {
 	if ssParity == 0 && rrsParity == 0 {
 		return nil
 	}
@@ -174,12 +174,12 @@ func validateParity(ssParity, rrsParity, drivesPerSet int) (err error) {
 		return fmt.Errorf("Reduced redundancy storage class parity %d should be greater than or equal to %d", rrsParity, minParityDisks)
 	}
 
-	if ssParity > drivesPerSet/2 {
-		return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, drivesPerSet/2)
+	if ssParity > setDriveCount/2 {
+		return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, setDriveCount/2)
 	}
 
-	if rrsParity > drivesPerSet/2 {
-		return fmt.Errorf("Reduced redundancy storage class parity %d should be less than  or equal to %d", rrsParity, drivesPerSet/2)
+	if rrsParity > setDriveCount/2 {
+		return fmt.Errorf("Reduced redundancy storage class parity %d should be less than  or equal to %d", rrsParity, setDriveCount/2)
 	}
 
 	if ssParity > 0 && rrsParity > 0 {
@@ -220,13 +220,13 @@ func Enabled(kvs config.KVS) bool {
 }
 
 // LookupConfig - lookup storage class config and override with valid environment settings if any.
-func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
+func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 	cfg = Config{}
-	cfg.Standard.Parity = drivesPerSet / 2
+	cfg.Standard.Parity = setDriveCount / 2
 	cfg.RRS.Parity = defaultRRSParity
 
 	if err = config.CheckValidKeys(config.StorageClassSubSys, kvs, DefaultKVS); err != nil {
-		return cfg, err
+		return Config{}, err
 	}
 
 	ssc := env.Get(StandardEnv, kvs.Get(ClassStandard))
@@ -235,17 +235,17 @@ func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
 	if ssc != "" {
 		cfg.Standard, err = parseStorageClass(ssc)
 		if err != nil {
-			return cfg, err
+			return Config{}, err
 		}
 	}
 	if cfg.Standard.Parity == 0 {
-		cfg.Standard.Parity = drivesPerSet / 2
+		cfg.Standard.Parity = setDriveCount / 2
 	}
 
 	if rrsc != "" {
 		cfg.RRS, err = parseStorageClass(rrsc)
 		if err != nil {
-			return cfg, err
+			return Config{}, err
 		}
 	}
 	if cfg.RRS.Parity == 0 {
@@ -254,8 +254,8 @@ func LookupConfig(kvs config.KVS, drivesPerSet int) (cfg Config, err error) {
 
 	// Validation is done after parsing both the storage classes. This is needed because we need one
 	// storage class value to deduce the correct value of the other storage class.
-	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, drivesPerSet); err != nil {
-		return cfg, err
+	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, setDriveCount); err != nil {
+		return Config{}, err
 	}
 
 	return cfg, nil

cmd/config/storageclass/storage-class_test.go

@@ -69,10 +69,10 @@ func TestParseStorageClass(t *testing.T) {
 
 func TestValidateParity(t *testing.T) {
 	tests := []struct {
-		rrsParity    int
-		ssParity     int
-		success      bool
-		drivesPerSet int
+		rrsParity     int
+		ssParity      int
+		success       bool
+		setDriveCount int
 	}{
 		{2, 4, true, 16},
 		{3, 3, true, 16},
@@ -85,7 +85,7 @@ func TestValidateParity(t *testing.T) {
 		{9, 2, false, 16},
 	}
 	for i, tt := range tests {
-		err := validateParity(tt.ssParity, tt.rrsParity, tt.drivesPerSet)
+		err := validateParity(tt.ssParity, tt.rrsParity, tt.setDriveCount)
 		if err != nil && tt.success {
 			t.Errorf("Test %d, Expected success, got %s", i+1, err)
 		}

cmd/consolelogger.go

@@ -117,6 +117,11 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan
 	sys.pubsub.Subscribe(subCh, doneCh, filter)
 }
 
+// Validate if HTTPConsoleLoggerSys is valid, always returns nil right now
+func (sys *HTTPConsoleLoggerSys) Validate() error {
+	return nil
+}
+
 // Send log message 'e' to console and publish to console
 // log pubsub system
 func (sys *HTTPConsoleLoggerSys) Send(e interface{}, logKind string) error {

cmd/crypto/config.go

@@ -16,11 +16,14 @@ package crypto
 
 import (
 	"errors"
+	"math/rand"
 	"net/http"
 	"reflect"
 	"strconv"
+	"strings"
 
 	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/pkg/ellipses"
 	"github.com/minio/minio/pkg/env"
 	xnet "github.com/minio/minio/pkg/net"
 )
@@ -167,7 +170,8 @@ const (
 
 const (
 	// EnvKMSKesEndpoint is the environment variable used to specify
-	// the kes server HTTPS endpoint.
+	// one or multiple KES server HTTPS endpoints. The individual
+	// endpoints should be separated by ','.
 	EnvKMSKesEndpoint = "MINIO_KMS_KES_ENDPOINT"
 
 	// EnvKMSKesKeyFile is the environment variable used to specify
@@ -216,16 +220,36 @@ func LookupKesConfig(kvs config.KVS) (KesConfig, error) {
 	kesCfg := KesConfig{}
 
 	endpointStr := env.Get(EnvKMSKesEndpoint, kvs.Get(KMSKesEndpoint))
-	if endpointStr != "" {
-		// Lookup kes configuration & overwrite config entry if ENV var is present
-		endpoint, err := xnet.ParseHTTPURL(endpointStr)
+	var endpoints []string
+	for _, endpoint := range strings.Split(endpointStr, ",") {
+		if strings.TrimSpace(endpoint) == "" {
+			continue
+		}
+		if !ellipses.HasEllipses(endpoint) {
+			endpoints = append(endpoints, endpoint)
+			continue
+		}
+		pattern, err := ellipses.FindEllipsesPatterns(endpoint)
 		if err != nil {
 			return kesCfg, err
 		}
-		endpointStr = endpoint.String()
+		for _, p := range pattern {
+			endpoints = append(endpoints, p.Expand()...)
+		}
+	}
+	if len(endpoints) == 0 {
+		return kesCfg, nil
 	}
 
-	kesCfg.Endpoint = endpointStr
+	randNum := rand.Intn(len(endpoints) + 1) // We add 1 b/c len(endpoints) may be 0: See: rand.Intn docs
+	kesCfg.Endpoint = make([]string, len(endpoints))
+	for i, endpoint := range endpoints {
+		endpoint, err := xnet.ParseHTTPURL(endpoint)
+		if err != nil {
+			return kesCfg, err
+		}
+		kesCfg.Endpoint[(randNum+i)%len(endpoints)] = endpoint.String()
+	}
 	kesCfg.KeyFile = env.Get(EnvKMSKesKeyFile, kvs.Get(KMSKesKeyFile))
 	kesCfg.CertFile = env.Get(EnvKMSKesCertFile, kvs.Get(KMSKesCertFile))
 	kesCfg.CAPath = env.Get(EnvKMSKesCAPath, kvs.Get(KMSKesCAPath))

cmd/crypto/header.go

@@ -18,9 +18,11 @@ import (
 	"bytes"
 	"crypto/md5"
 	"encoding/base64"
-	"encoding/json"
 	"net/http"
 	"strings"
+
+	jsoniter "github.com/json-iterator/go"
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 // SSEHeader is the general AWS SSE HTTP header key.
@@ -81,6 +83,8 @@ const (
 func RemoveSensitiveHeaders(h http.Header) {
 	h.Del(SSECKey)
 	h.Del(SSECopyKey)
+	h.Del(xhttp.AmzMetaUnencryptedContentLength)
+	h.Del(xhttp.AmzMetaUnencryptedContentMD5)
 }
 
 // IsRequested returns true if the HTTP headers indicates
@@ -144,6 +148,7 @@ func (s3KMS) ParseHTTP(h http.Header) (string, interface{}, error) {
 	contextStr, ok := h[SSEKmsContext]
 	if ok {
 		var context map[string]interface{}
+		var json = jsoniter.ConfigCompatibleWithStandardLibrary
 		if err := json.Unmarshal([]byte(contextStr[0]), &context); err != nil {
 			return "", nil, err
 		}

cmd/crypto/header_test.go

@@ -457,6 +457,16 @@ var removeSensitiveHeadersTests = []struct {
 			"X-Amz-Meta-Test-1": []string{"Test-1"},
 		},
 	},
+	{ // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		Header: http.Header{
+			"X-Amz-Meta-X-Amz-Unencrypted-Content-Md5":    []string{"value"},
+			"X-Amz-Meta-X-Amz-Unencrypted-Content-Length": []string{"value"},
+			"X-Amz-Meta-Test-1":                           []string{"Test-1"},
+		},
+		ExpectedHeader: http.Header{
+			"X-Amz-Meta-Test-1": []string{"Test-1"},
+		},
+	},
 }
 
 func TestRemoveSensitiveHeaders(t *testing.T) {

cmd/crypto/kes.go

@@ -16,9 +16,9 @@ package crypto
 
 import (
 	"bytes"
+	"context"
 	"crypto/tls"
 	"crypto/x509"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -30,10 +30,13 @@ import (
 	"strings"
 	"time"
 
+	jsoniter "github.com/json-iterator/go"
 	xhttp "github.com/minio/minio/cmd/http"
 	xnet "github.com/minio/minio/pkg/net"
 )
 
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
+
 // ErrKESKeyExists is the error returned a KES server
 // when a master key does exist.
 var ErrKESKeyExists = NewKESError(http.StatusBadRequest, "key does already exist")
@@ -43,8 +46,8 @@ var ErrKESKeyExists = NewKESError(http.StatusBadRequest, "key does already exist
 type KesConfig struct {
 	Enabled bool
 
-	// The kes server endpoint.
-	Endpoint string
+	// The KES server endpoints.
+	Endpoint []string
 
 	// The path to the TLS private key used
 	// by MinIO to authenticate to the kes
@@ -83,7 +86,7 @@ type KesConfig struct {
 // Verify verifies if the kes configuration is correct
 func (k KesConfig) Verify() (err error) {
 	switch {
-	case k.Endpoint == "":
+	case len(k.Endpoint) == 0:
 		err = Errorf("crypto: missing kes endpoint")
 	case k.CertFile == "":
 		err = Errorf("crypto: missing cert file")
@@ -98,7 +101,7 @@ func (k KesConfig) Verify() (err error) {
 type kesService struct {
 	client *kesClient
 
-	endpoint     string
+	endpoints    []string
 	defaultKeyID string
 }
 
@@ -113,23 +116,37 @@ func NewKes(cfg KesConfig) (KMS, error) {
 	if err != nil {
 		return nil, err
 	}
-	certPool, err := loadCACertificates(cfg.CAPath)
-	if err != nil {
-		return nil, err
+	if cfg.Transport.TLSClientConfig != nil {
+		if err = loadCACertificates(cfg.CAPath,
+			cfg.Transport.TLSClientConfig.RootCAs); err != nil {
+			return nil, err
+		}
+	} else {
+		rootCAs, _ := x509.SystemCertPool()
+		if rootCAs == nil {
+			// In some systems (like Windows) system cert pool is
+			// not supported or no certificates are present on the
+			// system - so we create a new cert pool.
+			rootCAs = x509.NewCertPool()
+		}
+		if err = loadCACertificates(cfg.CAPath, rootCAs); err != nil {
+			return nil, err
+		}
+		cfg.Transport.TLSClientConfig = &tls.Config{
+			RootCAs: rootCAs,
+		}
 	}
-	cfg.Transport.TLSClientConfig = &tls.Config{
-		Certificates: []tls.Certificate{cert},
-		RootCAs:      certPool,
-	}
-	cfg.Transport.ForceAttemptHTTP2 = true
+	cfg.Transport.TLSClientConfig.Certificates = []tls.Certificate{cert}
+	cfg.Transport.TLSClientConfig.NextProtos = []string{"h2"}
+
 	return &kesService{
 		client: &kesClient{
-			addr: cfg.Endpoint,
+			endpoints: cfg.Endpoint,
 			httpClient: http.Client{
 				Transport: cfg.Transport,
 			},
 		},
-		endpoint:     cfg.Endpoint,
+		endpoints:    cfg.Endpoint,
 		defaultKeyID: cfg.DefaultKeyID,
 	}, nil
 }
@@ -146,9 +163,9 @@ func (kes *kesService) DefaultKeyID() string {
 // method.
 func (kes *kesService) Info() KMSInfo {
 	return KMSInfo{
-		Endpoint: kes.endpoint,
-		Name:     kes.DefaultKeyID(),
-		AuthType: "TLS",
+		Endpoints: kes.endpoints,
+		Name:      kes.DefaultKeyID(),
+		AuthType:  "TLS",
 	}
 }
 
@@ -204,7 +221,7 @@ func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 //   • GenerateDataKey (API: /v1/key/generate/)
 //   • DecryptDataKey  (API: /v1/key/decrypt/)
 type kesClient struct {
-	addr       string
+	endpoints  []string
 	httpClient http.Client
 }
 
@@ -215,8 +232,8 @@ type kesClient struct {
 // application does not have the cryptographic key at
 // any point in time.
 func (c *kesClient) CreateKey(name string) error {
-	url := fmt.Sprintf("%s/v1/key/create/%s", c.addr, url.PathEscape(name))
-	_, err := c.postRetry(url, nil, 0) // No request body and no response expected
+	path := fmt.Sprintf("/v1/key/create/%s", url.PathEscape(name))
+	_, err := c.postRetry(path, nil, 0) // No request body and no response expected
 	if err != nil {
 		return err
 	}
@@ -248,8 +265,8 @@ func (c *kesClient) GenerateDataKey(name string, context []byte) ([]byte, []byte
 	}
 
 	const limit = 1 << 20 // A plaintext/ciphertext key pair will never be larger than 1 MB
-	url := fmt.Sprintf("%s/v1/key/generate/%s", c.addr, url.PathEscape(name))
-	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
+	path := fmt.Sprintf("/v1/key/generate/%s", url.PathEscape(name))
+	resp, err := c.postRetry(path, bytes.NewReader(body), limit)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -285,8 +302,8 @@ func (c *kesClient) DecryptDataKey(name string, ciphertext, context []byte) ([]b
 	}
 
 	const limit = 1 << 20 // A data key will never be larger than 1 MiB
-	url := fmt.Sprintf("%s/v1/key/decrypt/%s", c.addr, url.PathEscape(name))
-	resp, err := c.postRetry(url, bytes.NewReader(body), limit)
+	path := fmt.Sprintf("/v1/key/decrypt/%s", url.PathEscape(name))
+	resp, err := c.postRetry(path, bytes.NewReader(body), limit)
 	if err != nil {
 		return nil, err
 	}
@@ -357,7 +374,16 @@ func parseErrorResponse(resp *http.Response) error {
 }
 
 func (c *kesClient) post(url string, body io.Reader, limit int64) (io.Reader, error) {
-	resp, err := c.httpClient.Post(url, "application/json", body)
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, body)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err := c.httpClient.Do(req)
 	if err != nil {
 		return nil, err
 	}
@@ -376,26 +402,34 @@ func (c *kesClient) post(url string, body io.Reader, limit int64) (io.Reader, er
 	return &respBody, nil
 }
 
-func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (io.Reader, error) {
+func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.Reader, error) {
+	retryMax := 1 + len(c.endpoints)
 	for i := 0; ; i++ {
 		if body != nil {
 			body.Seek(0, io.SeekStart) // seek to the beginning of the body.
 		}
-		response, err := c.post(url, body, limit)
+
+		response, err := c.post(c.endpoints[i%len(c.endpoints)]+path, body, limit)
 		if err == nil {
 			return response, nil
 		}
 
-		if !xnet.IsNetworkOrHostDown(err) && !errors.Is(err, io.EOF) && !errors.Is(err, io.ErrUnexpectedEOF) {
+		// If the error is not temp. / retryable => fail the request immediately.
+		if !xnet.IsNetworkOrHostDown(err) &&
+			!errors.Is(err, io.EOF) &&
+			!errors.Is(err, io.ErrUnexpectedEOF) &&
+			!errors.Is(err, context.DeadlineExceeded) {
 			return nil, err
 		}
-
-		// retriable network errors.
-		remain := retryMax - i
-		if remain <= 0 {
+		if remain := retryMax - i; remain <= 0 { // Fail if we exceeded our retry limit.
 			return response, err
 		}
 
+		// If there are more KES instances then skip waiting and
+		// try the next endpoint directly.
+		if i < len(c.endpoints) {
+			continue
+		}
 		<-time.After(LinearJitterBackoff(retryWaitMin, retryWaitMax, i))
 	}
 }
@@ -413,24 +447,17 @@ func (c *kesClient) postRetry(url string, body io.ReadSeeker, limit int64) (io.R
 // file as PEM-encoded certificate and add it to
 // the CertPool. If a file is not a PEM certificate
 // it will be ignored.
-func loadCACertificates(path string) (*x509.CertPool, error) {
-	rootCAs, _ := x509.SystemCertPool()
-	if rootCAs == nil {
-		// In some systems (like Windows) system cert pool is
-		// not supported or no certificates are present on the
-		// system - so we create a new cert pool.
-		rootCAs = x509.NewCertPool()
-	}
+func loadCACertificates(path string, rootCAs *x509.CertPool) error {
 	if path == "" {
-		return rootCAs, nil
+		return nil
 	}
 
 	stat, err := os.Stat(path)
 	if err != nil {
 		if os.IsNotExist(err) || os.IsPermission(err) {
-			return rootCAs, nil
+			return nil
 		}
-		return nil, Errorf("crypto: cannot open '%s': %v", path, err)
+		return Errorf("crypto: cannot open '%s': %v", path, err)
 	}
 
 	// If path is a file, parse as PEM-encoded certifcate
@@ -439,12 +466,12 @@ func loadCACertificates(path string) (*x509.CertPool, error) {
 	if !stat.IsDir() {
 		cert, err := ioutil.ReadFile(path)
 		if err != nil {
-			return nil, err
+			return err
 		}
 		if !rootCAs.AppendCertsFromPEM(cert) {
-			return nil, Errorf("crypto: '%s' is not a valid PEM-encoded certificate", path)
+			return Errorf("crypto: '%s' is not a valid PEM-encoded certificate", path)
 		}
-		return rootCAs, nil
+		return nil
 	}
 
 	// If path is a directory then try
@@ -454,7 +481,7 @@ func loadCACertificates(path string) (*x509.CertPool, error) {
 	// we ignore it.
 	files, err := ioutil.ReadDir(path)
 	if err != nil {
-		return nil, err
+		return err
 	}
 	for _, file := range files {
 		cert, err := ioutil.ReadFile(filepath.Join(path, file.Name()))
@@ -463,6 +490,6 @@ func loadCACertificates(path string) (*x509.CertPool, error) {
 		}
 		rootCAs.AppendCertsFromPEM(cert) // ignore files which are not PEM certtificates
 	}
-	return rootCAs, nil
+	return nil
 
 }

cmd/crypto/kms.go

@@ -109,9 +109,9 @@ type masterKeyKMS struct {
 // KMSInfo contains some describing information about
 // the KMS.
 type KMSInfo struct {
-	Endpoint string
-	Name     string
-	AuthType string
+	Endpoints []string
+	Name      string
+	AuthType  string
 }
 
 // NewMasterKey returns a basic KMS implementation from a single 256 bit master key.
@@ -147,9 +147,9 @@ func (kms *masterKeyKMS) GenerateKey(keyID string, ctx Context) (key [32]byte, s
 // KMS is configured directly using master key
 func (kms *masterKeyKMS) Info() (info KMSInfo) {
 	return KMSInfo{
-		Endpoint: "",
-		Name:     "",
-		AuthType: "master-key",
+		Endpoints: []string{},
+		Name:      "",
+		AuthType:  "master-key",
 	}
 }
 

cmd/crypto/metadata.go

@@ -19,6 +19,7 @@ import (
 	"encoding/base64"
 	"errors"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -38,6 +39,8 @@ func IsMultiPart(metadata map[string]string) bool {
 func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
 	delete(metadata, SSECKey)
 	delete(metadata, SSECopyKey)
+	delete(metadata, xhttp.AmzMetaUnencryptedContentLength)
+	delete(metadata, xhttp.AmzMetaUnencryptedContentMD5)
 }
 
 // RemoveSSEHeaders removes all crypto-specific SSE
@@ -62,6 +65,17 @@ func RemoveInternalEntries(metadata map[string]string) {
 	delete(metadata, S3KMSSealedKey)
 }
 
+// IsSourceEncrypted returns true if the source is encrypted
+func IsSourceEncrypted(metadata map[string]string) bool {
+	if _, ok := metadata[SSECAlgorithm]; ok {
+		return true
+	}
+	if _, ok := metadata[SSEHeader]; ok {
+		return true
+	}
+	return false
+}
+
 // IsEncrypted returns true if the object metadata indicates
 // that it was uploaded using some form of server-side-encryption.
 //

cmd/crypto/retry.go

@@ -21,9 +21,8 @@ import (
 
 // default retry configuration
 const (
-	retryWaitMin = 500 * time.Millisecond // minimum retry limit.
-	retryWaitMax = 3 * time.Second        // 3 secs worth of max retry.
-	retryMax     = 2
+	retryWaitMin = 100 * time.Millisecond  // minimum retry limit.
+	retryWaitMax = 1500 * time.Millisecond // 1.5 secs worth of max retry.
 )
 
 // LinearJitterBackoff provides the time.Duration for a caller to

cmd/crypto/vault.go

@@ -199,13 +199,13 @@ func (v *vaultService) DefaultKeyID() string {
 }
 
 // Info returns some information about the Vault,
-// configuration - like the endpoint or authentication
+// configuration - like the endpoints or authentication
 // method.
 func (v *vaultService) Info() KMSInfo {
 	return KMSInfo{
-		Endpoint: v.config.Endpoint,
-		Name:     v.DefaultKeyID(),
-		AuthType: v.config.Auth.Type,
+		Endpoints: []string{v.config.Endpoint},
+		Name:      v.DefaultKeyID(),
+		AuthType:  v.config.Auth.Type,
 	}
 }
 

cmd/data-crawler.go

@@ -27,6 +27,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio/pkg/madmin"
+
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
@@ -44,6 +46,7 @@ const (
 	dataCrawlStartDelay      = 5 * time.Minute  // Time to wait on startup and between cycles.
 	dataUsageUpdateDirCycles = 16               // Visit all folders every n cycles.
 
+	healDeleteDangling = true
 )
 
 // initDataCrawler will start the crawler unless disabled.
@@ -87,12 +90,6 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 			if err == nil {
 				// Store new cycle...
 				nextBloomCycle++
-				if nextBloomCycle%dataUpdateTrackerResetEvery == 0 {
-					if intDataUpdateTracker.debug {
-						logger.Info(color.Green("runDataCrawler:") + " Resetting bloom filter for next runs.")
-					}
-					nextBloomCycle++
-				}
 				var tmp [8]byte
 				binary.LittleEndian.PutUint64(tmp[:], nextBloomCycle)
 				r, err := hash.NewReader(bytes.NewReader(tmp[:]), int64(len(tmp)), "", "", int64(len(tmp)), false)
@@ -111,8 +108,9 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 }
 
 type cachedFolder struct {
-	name   string
-	parent *dataUsageHash
+	name              string
+	parent            *dataUsageHash
+	objectHealProbDiv uint32
 }
 
 type folderScanner struct {
@@ -125,6 +123,8 @@ type folderScanner struct {
 
 	dataUsageCrawlMult  float64
 	dataUsageCrawlDebug bool
+	healFolderInclude   uint32 // Include a clean folder one in n cycles.
+	healObjectSelect    uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 
 	newFolders      []cachedFolder
 	existingFolders []cachedFolder
@@ -167,8 +167,17 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		existingFolders:     nil,
 		dataUsageCrawlMult:  delayMult,
 		dataUsageCrawlDebug: intDataUpdateTracker.debug,
+		healFolderInclude:   0,
+		healObjectSelect:    0,
 	}
 
+	// Enable healing in XL mode.
+	if globalIsErasure {
+		// Include a clean folder one in n cycles.
+		s.healFolderInclude = 32
+		// Do a heal check on an object once every n cycles. Must divide into healFolderInclude
+		s.healObjectSelect = 512
+	}
 	if len(cache.Info.BloomFilter) > 0 {
 		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
 		_, err := s.withFilter.ReadFrom(bytes.NewBuffer(cache.Info.BloomFilter))
@@ -189,7 +198,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 	}
 
 	// Always scan flattenLevels deep. Cache root is level 0.
-	todo := []cachedFolder{{name: cache.Info.Name}}
+	todo := []cachedFolder{{name: cache.Info.Name, objectHealProbDiv: 1}}
 	for i := 0; i < flattenLevels; i++ {
 		if s.dataUsageCrawlDebug {
 			logger.Info(logPrefix+"Level %v, scanning %v directories."+logSuffix, i, len(todo))
@@ -218,7 +227,7 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 			return s.newCache, ctx.Err()
 		default:
 		}
-		du, err := s.deepScanFolder(ctx, folder.name)
+		du, err := s.deepScanFolder(ctx, folder)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			continue
@@ -249,26 +258,38 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		}
 		h := hashPath(folder.name)
 		if !h.mod(s.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
-			s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
-			continue
+			if !h.mod(s.oldCache.Info.NextCycle, s.healFolderInclude/folder.objectHealProbDiv) {
+				s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
+				continue
+			} else {
+				folder.objectHealProbDiv = s.healFolderInclude
+			}
+			folder.objectHealProbDiv = dataUsageUpdateDirCycles
 		}
-
 		if s.withFilter != nil {
 			_, prefix := path2BucketObjectWithBasePath(basePath, folder.name)
 			if s.oldCache.Info.lifeCycle == nil || !s.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
 				// If folder isn't in filter, skip it completely.
 				if !s.withFilter.containsDir(folder.name) {
-					if s.dataUsageCrawlDebug {
-						logger.Info(logPrefix+"Skipping non-updated folder: %v"+logSuffix, folder)
+					if !h.mod(s.oldCache.Info.NextCycle, s.healFolderInclude/folder.objectHealProbDiv) {
+						if s.dataUsageCrawlDebug {
+							logger.Info(logPrefix+"Skipping non-updated folder: %v"+logSuffix, folder)
+						}
+						s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
+						continue
+					} else {
+						if s.dataUsageCrawlDebug {
+							logger.Info(logPrefix+"Adding non-updated folder to heal check: %v"+logSuffix, folder.name)
+						}
+						// Update probability of including objects
+						folder.objectHealProbDiv = s.healFolderInclude
 					}
-					s.newCache.replaceHashed(h, folder.parent, s.oldCache.Cache[h.Key()])
-					continue
 				}
 			}
 		}
 
 		// Update on this cycle...
-		du, err := s.deepScanFolder(ctx, folder.name)
+		du, err := s.deepScanFolder(ctx, folder)
 		if err != nil {
 			logger.LogIf(ctx, err)
 			continue
@@ -301,6 +322,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 		default:
 		}
 		thisHash := hashPath(folder.name)
+		existing := f.oldCache.findChildrenCopy(thisHash)
 
 		// If there are lifecycle rules for the prefix, remove the filter.
 		filter := f.withFilter
@@ -309,7 +331,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, true) {
 				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("data-usage:")+" Prefix %q has active rules", prefix)
+					logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
 				}
 				activeLifeCycle = f.oldCache.Info.lifeCycle
 				filter = nil
@@ -318,11 +340,19 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 		if _, ok := f.oldCache.Cache[thisHash.Key()]; filter != nil && ok {
 			// If folder isn't in filter and we have data, skip it completely.
 			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
-				f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
-				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("data-usage:")+" Skipping non-updated folder: %v", folder.name)
+				if !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
+					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
+					if f.dataUsageCrawlDebug {
+						logger.Info(color.Green("folder-scanner:")+" Skipping non-updated folder: %v", folder.name)
+					}
+					continue
+				} else {
+					if f.dataUsageCrawlDebug {
+						logger.Info(color.Green("folder-scanner:")+" Adding non-updated folder to heal check: %v", folder.name)
+					}
+					// If probability was already crawlerHealFolderInclude, keep it.
+					folder.objectHealProbDiv = f.healFolderInclude
 				}
-				continue
 			}
 		}
 		f.waitForLowActiveIO()
@@ -336,14 +366,14 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			bucket, prefix := path2BucketObjectWithBasePath(f.root, entName)
 			if bucket == "" {
 				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("data-usage:")+" no bucket (%s,%s)", f.root, entName)
+					logger.Info(color.Green("folder-scanner:")+" no bucket (%s,%s)", f.root, entName)
 				}
 				return nil
 			}
 
 			if isReservedOrInvalidBucket(bucket, false) {
 				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("data-usage:")+" invalid bucket: %v, entry: %v", bucket, entName)
+					logger.Info(color.Green("folder-scanner:")+" invalid bucket: %v, entry: %v", bucket, entName)
 				}
 				return nil
 			}
@@ -359,7 +389,8 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				_, exists := f.oldCache.Cache[h.Key()]
 				cache.addChildString(entName)
 
-				this := cachedFolder{name: entName, parent: &thisHash}
+				this := cachedFolder{name: entName, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv}
+				delete(existing, h.Key())
 				cache.addChild(h)
 				if final {
 					if exists {
@@ -385,11 +416,12 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				objectName: path.Base(entName),
 				debug:      f.dataUsageCrawlDebug,
 				lifeCycle:  activeLifeCycle,
+				heal:       thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv),
 			}
 			size, err := f.getSize(item)
 
 			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
-			if err == errSkipFile || err == errFileNotFound {
+			if err == errSkipFile {
 				return nil
 			}
 			logger.LogIf(ctx, err)
@@ -402,19 +434,78 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 		if err != nil {
 			return nil, err
 		}
+
+		if f.healObjectSelect == 0 {
+			// If we are not scanning, return now.
+			f.newCache.replaceHashed(thisHash, folder.parent, cache)
+			continue
+		}
+
+		objAPI := newObjectLayerWithoutSafeModeFn()
+		if objAPI == nil {
+			continue
+		}
+
+		bgSeq, found := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
+		if !found {
+			continue
+		}
+
+		// Whatever remains in 'existing' are folders at this level
+		// that existed in the previous run but wasn't found now.
+		//
+		// This may be because of 2 reasons:
+		//
+		// 1) The folder/object was deleted.
+		// 2) We come from another disk and this disk missed the write.
+		//
+		// We therefore perform a heal check.
+		// If that doesn't bring it back we remove the folder and assume it was deleted.
+		// This means that the next run will not look for it.
+		for k := range existing {
+			f.waitForLowActiveIO()
+			bucket, prefix := path2BucketObject(k)
+			if f.dataUsageCrawlDebug {
+				logger.Info(color.Green("folder-scanner:")+" checking disappeared folder: %v/%v", bucket, prefix)
+			}
+
+			err = objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{Recursive: true, Remove: healDeleteDangling},
+				func(bucket, object, versionID string) error {
+					return bgSeq.queueHealTask(healSource{
+						bucket:    bucket,
+						object:    object,
+						versionID: versionID,
+					}, madmin.HealItemObject)
+				})
+
+			if f.dataUsageCrawlDebug && err != nil {
+				logger.Info(color.Green("healObjects:")+" checking returned value %v", err)
+			}
+
+			// Add unless healing returned an error.
+			if err == nil {
+				this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv}
+				cache.addChild(hashPath(k))
+				if final {
+					f.existingFolders = append(f.existingFolders, this)
+				} else {
+					nextFolders = append(nextFolders, this)
+				}
+			}
+		}
 		f.newCache.replaceHashed(thisHash, folder.parent, cache)
 	}
 	return nextFolders, nil
 }
 
 // deepScanFolder will deep scan a folder and return the size if no error occurs.
-func (f *folderScanner) deepScanFolder(ctx context.Context, folder string) (*dataUsageEntry, error) {
+func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder) (*dataUsageEntry, error) {
 	var cache dataUsageEntry
 
 	done := ctx.Done()
 
 	var addDir func(entName string, typ os.FileMode) error
-	var dirStack = []string{f.root, folder}
+	var dirStack = []string{f.root, folder.name}
 
 	addDir = func(entName string, typ os.FileMode) error {
 		select {
@@ -445,7 +536,7 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder string) (*dat
 		if f.oldCache.Info.lifeCycle != nil {
 			if f.oldCache.Info.lifeCycle.HasActiveRules(prefix, false) {
 				if f.dataUsageCrawlDebug {
-					logger.Info(color.Green("data-usage:")+" Prefix %q has active rules", prefix)
+					logger.Info(color.Green("folder-scanner:")+" Prefix %q has active rules", prefix)
 				}
 				activeLifeCycle = f.oldCache.Info.lifeCycle
 			}
@@ -460,6 +551,7 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder string) (*dat
 				objectName: path.Base(entName),
 				debug:      f.dataUsageCrawlDebug,
 				lifeCycle:  activeLifeCycle,
+				heal:       hashPath(path.Join(prefix, entName)).mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv),
 			})
 
 		// Don't sleep for really small amount of time
@@ -490,6 +582,7 @@ type crawlItem struct {
 	prefix     string // Only the prefix if any, does not have final object name.
 	objectName string // Only the object name without prefixes.
 	lifeCycle  *lifecycle.Lifecycle
+	heal       bool // Has the object been selected for heal check?
 	debug      bool
 }
 
@@ -522,6 +615,20 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 	if i.debug {
 		logger.LogIf(ctx, err)
 	}
+	if i.heal {
+		if i.debug {
+			logger.Info(color.Green("applyActions:")+" heal checking: %v/%v v%s", i.bucket, i.objectPath(), meta.oi.VersionID)
+		}
+		res, err := o.HealObject(ctx, i.bucket, i.objectPath(), meta.oi.VersionID, madmin.HealOpts{Remove: healDeleteDangling})
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
+			return 0
+		}
+		if !errors.Is(err, NotImplemented{}) {
+			logger.LogIf(ctx, err)
+			return 0
+		}
+		size = res.ObjectSize
+	}
 	if i.lifeCycle == nil {
 		return size
 	}

cmd/data-update-tracker.go

@@ -48,9 +48,6 @@ const (
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
 	dataUpdateTrackerVersion      = 2
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
-
-	// Reset bloom filters every n cycle
-	dataUpdateTrackerResetEvery = 1000
 )
 
 var (
@@ -181,6 +178,8 @@ func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
 	}
 	d.load(ctx, drives...)
 	go d.startCollector(ctx)
+	// startSaver will unlock.
+	d.mu.Lock()
 	go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives)
 }
 
@@ -214,17 +213,17 @@ func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
 }
 
 // startSaver will start a saver that will write d to all supplied drives at specific intervals.
+// 'd' must be write locked when started and will be unlocked.
 // The saver will save and exit when supplied context is closed.
 func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives []string) {
-	t := time.NewTicker(interval)
-	defer t.Stop()
-	var buf bytes.Buffer
-	d.mu.Lock()
 	saveNow := d.save
 	exited := make(chan struct{})
 	d.saveExited = exited
 	d.mu.Unlock()
+	t := time.NewTicker(interval)
+	defer t.Stop()
 	defer close(exited)
+	var buf bytes.Buffer
 	for {
 		var exit bool
 		select {

cmd/data-update-tracker_test.go

@@ -42,6 +42,10 @@ type testingLogger struct {
 	t  testLoggerI
 }
 
+func (t *testingLogger) Validate() error {
+	return nil
+}
+
 func (t *testingLogger) Send(entry interface{}, errKind string) error {
 	t.mu.Lock()
 	defer t.mu.Unlock()

cmd/data-usage-cache.go

@@ -85,7 +85,12 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 }
 
 // mod returns true if the hash mod cycles == cycle.
+// If cycles is 0 false is always returned.
+// If cycles is 1 true is always returned (as expected).
 func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool {
+	if cycles <= 1 {
+		return cycles == 1
+	}
 	return uint32(xxhash.Sum64String(string(h)))%cycles == cycle%cycles
 }
 
@@ -117,6 +122,16 @@ func (d *dataUsageCache) find(path string) *dataUsageEntry {
 	return &due
 }
 
+// findChildrenCopy returns a copy of the children of the supplied hash.
+func (d *dataUsageCache) findChildrenCopy(h dataUsageHash) dataUsageHashMap {
+	ch := d.Cache[h.String()].Children
+	res := make(dataUsageHashMap, len(ch))
+	for k := range ch {
+		res[k] = struct{}{}
+	}
+	return res
+}
+
 // Returns nil if not found.
 func (d *dataUsageCache) subCache(path string) dataUsageCache {
 	dst := dataUsageCache{Info: dataUsageCacheInfo{
@@ -328,7 +343,7 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 		flat := d.flatten(*e)
 		dst[bucket.Name] = BucketUsageInfo{
 			Size:                 uint64(flat.Size),
-			ObjectsCount:         uint64(flat.Objects),
+			ObjectsCount:         flat.Objects,
 			ObjectSizesHistogram: flat.ObjSizes.toMap(),
 		}
 	}
@@ -345,7 +360,7 @@ func (d *dataUsageCache) bucketUsageInfo(bucket string) BucketUsageInfo {
 	flat := d.flatten(*e)
 	return BucketUsageInfo{
 		Size:                 uint64(flat.Size),
-		ObjectsCount:         uint64(flat.Objects),
+		ObjectsCount:         flat.Objects,
 		ObjectSizesHistogram: flat.ObjSizes.toMap(),
 	}
 }

cmd/data-usage_test.go

@@ -360,6 +360,13 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
+	if got.root() == nil {
+		t.Log("cached folders:")
+		for folder := range got.Cache {
+			t.Log("folder:", folder)
+		}
+		t.Fatal("got nil root.")
+	}
 
 	// Test dirs
 	var want = []struct {

cmd/disk-cache-backend.go

@@ -46,7 +46,7 @@ const (
 	cacheMetaJSONFile = "cache.json"
 	cacheDataFile     = "part.1"
 	cacheMetaVersion  = "1.0.0"
-	cacheExpiryDays   = time.Duration(90 * time.Hour * 24) // defaults to 90 days
+	cacheExpiryDays   = 90 * time.Hour * 24 // defaults to 90 days
 	// SSECacheEncrypted is the metadata key indicating that the object
 	// is a cache entry encrypted with cache KMS master key in globalCacheKMS.
 	SSECacheEncrypted = "X-Minio-Internal-Encrypted-Cache"
@@ -417,7 +417,7 @@ func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectI
 func (c *diskCache) statCachedMeta(ctx context.Context, cacheObjPath string) (meta *cacheMeta, partial bool, numHits int, err error) {
 
 	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err = cLock.GetRLock(globalObjectTimeout); err != nil {
+	if err = cLock.GetRLock(globalOperationTimeout); err != nil {
 		return
 	}
 
@@ -499,7 +499,7 @@ func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (meta *c
 func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
 	cachedPath := getCacheSHADir(c.dir, bucket, object)
 	cLock := c.NewNSLockFn(ctx, cachedPath)
-	if err := cLock.GetLock(globalObjectTimeout); err != nil {
+	if err := cLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}
 	defer cLock.Unlock()
@@ -665,7 +665,7 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 	}
 	cachePath := getCacheSHADir(c.dir, bucket, object)
 	cLock := c.NewNSLockFn(ctx, cachePath)
-	if err := cLock.GetLock(globalObjectTimeout); err != nil {
+	if err := cLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}
 	defer cLock.Unlock()
@@ -871,7 +871,7 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, numHits int, err error) {
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
 	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err := cLock.GetRLock(globalObjectTimeout); err != nil {
+	if err := cLock.GetRLock(globalOperationTimeout); err != nil {
 		return nil, numHits, err
 	}
 
@@ -935,7 +935,7 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 // Deletes the cached object
 func (c *diskCache) delete(ctx context.Context, cacheObjPath string) (err error) {
 	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err := cLock.GetLock(globalObjectTimeout); err != nil {
+	if err := cLock.GetLock(globalOperationTimeout); err != nil {
 		return err
 	}
 	defer cLock.Unlock()

cmd/disk-cache-utils.go

@@ -102,7 +102,7 @@ func (c *cacheControl) isStale(modTime time.Time) bool {
 func cacheControlOpts(o ObjectInfo) *cacheControl {
 	c := cacheControl{}
 	m := o.UserDefined
-	if o.Expires != timeSentinel {
+	if !o.Expires.Equal(timeSentinel) {
 		c.expiry = o.Expires
 	}
 

cmd/disk-cache-utils_test.go

@@ -45,7 +45,7 @@ func TestGetCacheControlOpts(t *testing.T) {
 		t.Run("", func(t *testing.T) {
 			m := make(map[string]string)
 			m["cache-control"] = testCase.cacheControlHeaderVal
-			if testCase.expiryHeaderVal != timeSentinel {
+			if !testCase.expiryHeaderVal.Equal(timeSentinel) {
 				m["expires"] = testCase.expiryHeaderVal.String()
 			}
 			c := cacheControlOpts(ObjectInfo{UserDefined: m, Expires: testCase.expiryHeaderVal})

cmd/disk-cache.go

@@ -172,7 +172,7 @@ func getMetadata(objInfo ObjectInfo) map[string]string {
 	if objInfo.ContentEncoding != "" {
 		metadata["content-encoding"] = objInfo.ContentEncoding
 	}
-	if objInfo.Expires != timeSentinel {
+	if !objInfo.Expires.Equal(timeSentinel) {
 		metadata["expires"] = objInfo.Expires.Format(http.TimeFormat)
 	}
 	for k, v := range objInfo.UserDefined {
@@ -317,16 +317,18 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 				rs = nil
 			}
 			// fill cache in the background for range GET requests
-			bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+			bReader, bErr := c.GetObjectNInfoFn(GlobalContext, bucket, object, rs, h, lockType, opts)
 			if bErr != nil {
 				return
 			}
 			defer bReader.Close()
-			oi, _, _, err := dcache.statRange(ctx, bucket, object, rs)
+			oi, _, _, err := dcache.statRange(GlobalContext, bucket, object, rs)
 			// avoid cache overwrite if another background routine filled cache
 			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
 				// use a new context to avoid locker prematurely timing out operation when the GetObjectNInfo returns.
-				dcache.Put(context.Background(), bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+				dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, rs, ObjectOptions{
+					UserDefined: getMetadata(bReader.ObjInfo),
+				}, false)
 				return
 			}
 		}()
@@ -337,7 +339,11 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	pipeReader, pipeWriter := io.Pipe()
 	teeReader := io.TeeReader(bkReader, pipeWriter)
 	go func() {
-		putErr := dcache.Put(ctx, bucket, object, io.LimitReader(pipeReader, bkReader.ObjInfo.Size), bkReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bkReader.ObjInfo)}, false)
+		putErr := dcache.Put(ctx, bucket, object,
+			io.LimitReader(pipeReader, bkReader.ObjInfo.Size),
+			bkReader.ObjInfo.Size, nil, ObjectOptions{
+				UserDefined: getMetadata(bkReader.ObjInfo),
+			}, false)
 		// close the write end of the pipe, so the error gets
 		// propagated to getObjReader
 		pipeWriter.CloseWithError(putErr)
@@ -634,15 +640,15 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 	if err == nil {
 		go func() {
 			// fill cache in the background
-			bReader, bErr := c.GetObjectNInfoFn(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
+			bReader, bErr := c.GetObjectNInfoFn(GlobalContext, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
 			if bErr != nil {
 				return
 			}
 			defer bReader.Close()
-			oi, _, err := dcache.Stat(ctx, bucket, object)
+			oi, _, err := dcache.Stat(GlobalContext, bucket, object)
 			// avoid cache overwrite if another background routine filled cache
 			if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-				dcache.Put(ctx, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
+				dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false)
 			}
 		}()
 	}

cmd/disk-cache_test.go

@@ -27,7 +27,7 @@ func TestCacheMetadataObjInfo(t *testing.T) {
 	if objInfo.Size != 0 {
 		t.Fatal("Unexpected object info value for Size", objInfo.Size)
 	}
-	if objInfo.ModTime != timeSentinel {
+	if !objInfo.ModTime.Equal(timeSentinel) {
 		t.Fatal("Unexpected object info value for ModTime ", objInfo.ModTime)
 	}
 	if objInfo.IsDir {

cmd/dynamic-timeouts.go

@@ -26,7 +26,7 @@ const (
 	dynamicTimeoutIncreaseThresholdPct = 0.33 // Upper threshold for failures in order to increase timeout
 	dynamicTimeoutDecreaseThresholdPct = 0.10 // Lower threshold for failures in order to decrease timeout
 	dynamicTimeoutLogSize              = 16
-	maxDuration                        = time.Duration(1<<63 - 1)
+	maxDuration                        = 1<<63 - 1
 )
 
 // timeouts that are dynamically adapted based on actual usage results
@@ -110,7 +110,7 @@ func (dt *dynamicTimeout) adjust(entries [dynamicTimeoutLogSize]time.Duration) {
 		// We are hitting the timeout relatively few times, so decrease the timeout
 		average = average * 125 / 100 // Add buffer of 25% on top of average
 
-		timeout := (atomic.LoadInt64(&dt.timeout) + int64(average)) / 2 // Middle between current timeout and average success
+		timeout := (atomic.LoadInt64(&dt.timeout) + average) / 2 // Middle between current timeout and average success
 		if timeout < dt.minimum {
 			timeout = dt.minimum
 		}

cmd/encryption-v1.go

@@ -605,12 +605,14 @@ func getDecryptedETag(headers http.Header, objInfo ObjectInfo, copySource bool)
 	if crypto.IsMultiPart(objInfo.UserDefined) {
 		return objInfo.ETag
 	}
+
 	if crypto.SSECopy.IsRequested(headers) {
 		key, err = crypto.SSECopy.ParseHTTP(headers)
 		if err != nil {
 			return objInfo.ETag
 		}
 	}
+
 	// As per AWS S3 Spec, ETag for SSE-C encrypted objects need not be MD5Sum of the data.
 	// Since server side copy with same source and dest just replaces the ETag, we save
 	// encrypted content MD5Sum as ETag for both SSE-C and SSE-S3, we standardize the ETag
@@ -664,7 +666,7 @@ func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, sk
 
 	if rs == nil {
 		// No range, so offsets refer to the whole object.
-		return 0, int64(o.Size), 0, 0, 0, nil
+		return 0, o.Size, 0, 0, 0, nil
 	}
 
 	// Assemble slice of (decrypted) part sizes in `sizes`

cmd/encryption-v1_test.go

@@ -373,7 +373,7 @@ func TestGetDecryptedRange(t *testing.T) {
 			sum := int64(0)
 			for i, s := range sizes {
 				r[i].Number = i
-				r[i].Size = int64(getEncSize(s))
+				r[i].Size = getEncSize(s)
 				sum += r[i].Size
 			}
 			return ObjectInfo{

cmd/endpoint.go

@@ -193,7 +193,7 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 }
 
 // ZoneEndpoints represent endpoints in a given zone
-// along with its setCount and drivesPerSet.
+// along with its setCount and setDriveCount.
 type ZoneEndpoints struct {
 	SetCount     int
 	DrivesPerSet int

cmd/erasure-bucket.go

@@ -133,8 +133,7 @@ func (er erasureObjects) listBuckets(ctx context.Context) (bucketsInfo []BucketI
 		if err == nil {
 			// NOTE: The assumption here is that volumes across all disks in
 			// readQuorum have consistent view i.e they all have same number
-			// of buckets. This is essentially not verified since healing
-			// should take care of this.
+			// of buckets.
 			var bucketsInfo []BucketInfo
 			for _, volInfo := range volsInfo {
 				if isReservedOrInvalidBucket(volInfo.Name, true) {

cmd/erasure-coding.go

@@ -125,7 +125,7 @@ func (e *Erasure) ShardFileSize(totalLength int64) int64 {
 		return -1
 	}
 	numShards := totalLength / e.blockSize
-	lastBlockSize := totalLength % int64(e.blockSize)
+	lastBlockSize := totalLength % e.blockSize
 	lastShardSize := ceilFrac(lastBlockSize, int64(e.dataBlocks))
 	return numShards*e.ShardSize() + lastShardSize
 }
@@ -134,7 +134,7 @@ func (e *Erasure) ShardFileSize(totalLength int64) int64 {
 func (e *Erasure) ShardFileOffset(startOffset, length, totalLength int64) int64 {
 	shardSize := e.ShardSize()
 	shardFileSize := e.ShardFileSize(totalLength)
-	endShard := (startOffset + int64(length)) / e.blockSize
+	endShard := (startOffset + length) / e.blockSize
 	tillOffset := endShard*shardSize + shardSize
 	if tillOffset > shardFileSize {
 		tillOffset = shardFileSize

cmd/erasure-common.go

@@ -23,6 +23,17 @@ import (
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
+func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
+	disks := er.getDisks()
+	// Based on the random shuffling return back randomized disks.
+	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+		if disks[i-1] != nil && disks[i-1].IsLocal() {
+			newDisks = append(newDisks, disks[i-1])
+		}
+	}
+	return newDisks
+}
+
 // getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
 func (er erasureObjects) getLoadBalancedDisks() (newDisks []StorageAPI) {
 	disks := er.getDisks()

cmd/erasure-healing-common_test.go

@@ -83,7 +83,7 @@ func TestCommonTime(t *testing.T) {
 	for i, testCase := range testCases {
 		// Obtain a common mod time from modTimes slice.
 		ctime, _ := commonTime(testCase.times)
-		if testCase.time != ctime {
+		if !testCase.time.Equal(ctime) {
 			t.Fatalf("Test case %d, expect to pass but failed. Wanted modTime: %s, got modTime: %s\n", i+1, testCase.time, ctime)
 		}
 	}

cmd/erasure-multipart.go

@@ -24,6 +24,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/minio/minio-go/v7/pkg/set"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -69,6 +70,57 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 	g.Wait()
 }
 
+// Clean-up the old multipart uploads. Should be run in a Go routine.
+func (er erasureObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
+	ticker := time.NewTicker(cleanupInterval)
+	defer ticker.Stop()
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			var disk StorageAPI
+			// run multiple cleanup's local to this server.
+			for _, d := range er.getLoadBalancedLocalDisks() {
+				if d != nil {
+					disk = d
+					break
+				}
+			}
+			if disk == nil {
+				continue
+			}
+			er.cleanupStaleMultipartUploadsOnDisk(ctx, disk, expiry)
+		}
+	}
+}
+
+// Remove the old multipart uploads on the given disk.
+func (er erasureObjects) cleanupStaleMultipartUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
+	now := time.Now()
+	shaDirs, err := disk.ListDir(minioMetaMultipartBucket, "", -1)
+	if err != nil {
+		return
+	}
+	for _, shaDir := range shaDirs {
+		uploadIDDirs, err := disk.ListDir(minioMetaMultipartBucket, shaDir, -1)
+		if err != nil {
+			continue
+		}
+		for _, uploadIDDir := range uploadIDDirs {
+			uploadIDPath := pathJoin(shaDir, uploadIDDir)
+			fi, err := disk.ReadVersion(minioMetaMultipartBucket, uploadIDPath, "")
+			if err != nil {
+				continue
+			}
+			if now.Sub(fi.ModTime) > expiry {
+				er.deleteObject(ctx, minioMetaMultipartBucket, uploadIDPath, fi.Erasure.DataBlocks+1)
+			}
+		}
+	}
+}
+
 // ListMultipartUploads - lists all the pending multipart
 // uploads for a particular object in a bucket.
 //
@@ -702,6 +754,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 
 	// Save successfully calculated md5sum.
 	fi.Metadata["etag"] = s3MD5
+	if opts.UserDefined["etag"] != "" { // preserve ETag if set
+		fi.Metadata["etag"] = opts.UserDefined["etag"]
+	}
 
 	// Save the consolidated actual size.
 	fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)

cmd/erasure-object.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -710,8 +711,9 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			Hash:       bitrotWriterSum(w),
 		})
 	}
-
-	opts.UserDefined["etag"] = r.MD5CurrentHexString()
+	if opts.UserDefined["etag"] == "" {
+		opts.UserDefined["etag"] = r.MD5CurrentHexString()
+	}
 
 	// Guess content-type from the extension if possible.
 	if opts.UserDefined["content-type"] == "" {
@@ -859,14 +861,22 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 	versions := make([]FileInfo, len(objects))
 	for i := range objects {
 		if objects[i].VersionID == "" {
-			if opts.Versioned && !HasSuffix(objects[i].ObjectName, SlashSeparator) {
-				versions[i] = FileInfo{
-					Name:      objects[i].ObjectName,
-					VersionID: mustGetUUID(),
-					ModTime:   UTCNow(),
-					Deleted:   true, // delete marker
+			if opts.Versioned || opts.VersionSuspended {
+				if !HasSuffix(objects[i].ObjectName, SlashSeparator) {
+					fi := FileInfo{
+						Name:    objects[i].ObjectName,
+						ModTime: UTCNow(),
+						Deleted: true, // delete marker
+					}
+					if opts.Versioned {
+						fi.VersionID = mustGetUUID()
+					}
+					// versioning suspended means we add `null`
+					// version as delete marker
+
+					versions[i] = fi
+					continue
 				}
-				continue
 			}
 		}
 		versions[i] = FileInfo{
@@ -906,9 +916,11 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		for i := range delObjErrs {
 			// delObjErrs[i] is not nil when disks[i] is also not nil
 			if delObjErrs[i] != nil {
-				if delObjErrs[i][objIndex] != errFileNotFound {
-					diskErrs[i] = delObjErrs[i][objIndex]
+				if errors.Is(delObjErrs[i][objIndex], errFileNotFound) ||
+					errors.Is(delObjErrs[i][objIndex], errFileVersionNotFound) {
+					continue
 				}
+				diskErrs[i] = delObjErrs[i][objIndex]
 			}
 		}
 		errs[objIndex] = reduceWriteQuorumErrs(ctx, diskErrs, objectOpIgnoredErrs, writeQuorums[objIndex])
@@ -960,18 +972,27 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	writeQuorum := len(storageDisks)/2 + 1
 
 	if opts.VersionID == "" {
-		if opts.Versioned && !HasSuffix(object, SlashSeparator) {
-			fi := FileInfo{
-				Name:      object,
-				VersionID: mustGetUUID(),
-				Deleted:   true,
-				ModTime:   UTCNow(),
+		if opts.Versioned || opts.VersionSuspended {
+			if !HasSuffix(object, SlashSeparator) {
+				fi := FileInfo{
+					Name:    object,
+					Deleted: true,
+					ModTime: UTCNow(),
+				}
+
+				if opts.Versioned {
+					fi.VersionID = mustGetUUID()
+				}
+
+				// versioning suspended means we add `null`
+				// version as delete marker
+
+				// Add delete marker, since we don't have any version specified explicitly.
+				if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi); err != nil {
+					return objInfo, toObjectErr(err, bucket, object)
+				}
+				return fi.ToObjectInfo(bucket, object), nil
 			}
-			// Add delete marker, since we don't have any version specified explicitly.
-			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi); err != nil {
-				return objInfo, toObjectErr(err, bucket, object)
-			}
-			return fi.ToObjectInfo(bucket, object), nil
 		}
 	}
 

cmd/erasure-sets.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"hash/crc32"
 	"io"
@@ -74,7 +75,8 @@ type erasureSets struct {
 	endpointStrings []string
 
 	// Total number of sets and the number of disks per set.
-	setCount, drivesPerSet int
+	setCount, setDriveCount int
+	listTolerancePerSet     int
 
 	disksConnectEvent chan diskConnectInfo
 
@@ -111,7 +113,7 @@ func (s *erasureSets) getDiskMap() map[string]StorageAPI {
 	defer s.erasureDisksMu.RUnlock()
 
 	for i := 0; i < s.setCount; i++ {
-		for j := 0; j < s.drivesPerSet; j++ {
+		for j := 0; j < s.setDriveCount; j++ {
 			disk := s.erasureDisks[i][j]
 			if disk == nil {
 				continue
@@ -137,7 +139,14 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 	if err != nil {
 		// Close the internal connection to avoid connection leaks.
 		disk.Close()
-		return nil, nil, err
+		if errors.Is(err, errUnformattedDisk) {
+			info, derr := disk.DiskInfo()
+			if derr != nil && info.RootDisk {
+				return nil, nil, fmt.Errorf("Disk: %s returned %w but its a root disk refusing to use it",
+					disk, derr) // make sure to '%w' to wrap the error
+			}
+		}
+		return nil, nil, fmt.Errorf("Disk: %s returned %w", disk, err) // make sure to '%w' to wrap the error
 	}
 
 	return disk, format, nil
@@ -220,7 +229,7 @@ func (s *erasureSets) connectDisks() {
 				s.erasureDisks[setIndex][diskIndex].Close()
 			}
 			s.erasureDisks[setIndex][diskIndex] = disk
-			s.endpointStrings[setIndex*s.drivesPerSet+diskIndex] = disk.String()
+			s.endpointStrings[setIndex*s.setDriveCount+diskIndex] = disk.String()
 			s.erasureDisksMu.Unlock()
 			go func(setIndex int) {
 				// Send a new disk connect event with a timeout
@@ -252,7 +261,7 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 
 func (s *erasureSets) GetLockers(setIndex int) func() []dsync.NetLocker {
 	return func() []dsync.NetLocker {
-		lockers := make([]dsync.NetLocker, s.drivesPerSet)
+		lockers := make([]dsync.NetLocker, s.setDriveCount)
 		copy(lockers, s.erasureLockers[setIndex])
 		return lockers
 	}
@@ -263,9 +272,9 @@ func (s *erasureSets) GetEndpoints(setIndex int) func() []string {
 		s.erasureDisksMu.RLock()
 		defer s.erasureDisksMu.RUnlock()
 
-		eps := make([]string, s.drivesPerSet)
-		for i := 0; i < s.drivesPerSet; i++ {
-			eps[i] = s.endpointStrings[setIndex*s.drivesPerSet+i]
+		eps := make([]string, s.setDriveCount)
+		for i := 0; i < s.setDriveCount; i++ {
+			eps[i] = s.endpointStrings[setIndex*s.setDriveCount+i]
 		}
 		return eps
 	}
@@ -276,7 +285,7 @@ func (s *erasureSets) GetDisks(setIndex int) func() []StorageAPI {
 	return func() []StorageAPI {
 		s.erasureDisksMu.RLock()
 		defer s.erasureDisksMu.RUnlock()
-		disks := make([]StorageAPI, s.drivesPerSet)
+		disks := make([]StorageAPI, s.setDriveCount)
 		copy(disks, s.erasureDisks[setIndex])
 		return disks
 	}
@@ -287,46 +296,47 @@ const defaultMonitorConnectEndpointInterval = time.Second * 10 // Set to 10 secs
 // Initialize new set of erasure coded sets.
 func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []StorageAPI, format *formatErasureV3) (*erasureSets, error) {
 	setCount := len(format.Erasure.Sets)
-	drivesPerSet := len(format.Erasure.Sets[0])
+	setDriveCount := len(format.Erasure.Sets[0])
 
 	endpointStrings := make([]string, len(endpoints))
 	// Initialize the erasure sets instance.
 	s := &erasureSets{
-		sets:               make([]*erasureObjects, setCount),
-		erasureDisks:       make([][]StorageAPI, setCount),
-		erasureLockers:     make([][]dsync.NetLocker, setCount),
-		endpoints:          endpoints,
-		endpointStrings:    endpointStrings,
-		setCount:           setCount,
-		drivesPerSet:       drivesPerSet,
-		format:             format,
-		disksConnectEvent:  make(chan diskConnectInfo),
-		disksConnectDoneCh: make(chan struct{}),
-		distributionAlgo:   format.Erasure.DistributionAlgo,
-		deploymentID:       uuid.MustParse(format.ID),
-		pool:               NewMergeWalkPool(globalMergeLookupTimeout),
-		poolSplunk:         NewMergeWalkPool(globalMergeLookupTimeout),
-		poolVersions:       NewMergeWalkVersionsPool(globalMergeLookupTimeout),
-		mrfOperations:      make(map[healSource]int),
+		sets:                make([]*erasureObjects, setCount),
+		erasureDisks:        make([][]StorageAPI, setCount),
+		erasureLockers:      make([][]dsync.NetLocker, setCount),
+		endpoints:           endpoints,
+		endpointStrings:     endpointStrings,
+		setCount:            setCount,
+		setDriveCount:       setDriveCount,
+		listTolerancePerSet: setDriveCount / 2,
+		format:              format,
+		disksConnectEvent:   make(chan diskConnectInfo),
+		disksConnectDoneCh:  make(chan struct{}),
+		distributionAlgo:    format.Erasure.DistributionAlgo,
+		deploymentID:        uuid.MustParse(format.ID),
+		pool:                NewMergeWalkPool(globalMergeLookupTimeout),
+		poolSplunk:          NewMergeWalkPool(globalMergeLookupTimeout),
+		poolVersions:        NewMergeWalkVersionsPool(globalMergeLookupTimeout),
+		mrfOperations:       make(map[healSource]int),
 	}
 
 	mutex := newNSLock(globalIsDistErasure)
 
 	// Initialize byte pool once for all sets, bpool size is set to
-	// setCount * drivesPerSet with each memory upto blockSizeV1.
-	bp := bpool.NewBytePoolCap(setCount*drivesPerSet, blockSizeV1, blockSizeV1*2)
+	// setCount * setDriveCount with each memory upto blockSizeV1.
+	bp := bpool.NewBytePoolCap(setCount*setDriveCount, blockSizeV1, blockSizeV1*2)
 
 	for i := 0; i < setCount; i++ {
-		s.erasureDisks[i] = make([]StorageAPI, drivesPerSet)
-		s.erasureLockers[i] = make([]dsync.NetLocker, drivesPerSet)
+		s.erasureDisks[i] = make([]StorageAPI, setDriveCount)
+		s.erasureLockers[i] = make([]dsync.NetLocker, setDriveCount)
 	}
 
 	for i := 0; i < setCount; i++ {
-		for j := 0; j < drivesPerSet; j++ {
-			endpoint := endpoints[i*drivesPerSet+j]
+		for j := 0; j < setDriveCount; j++ {
+			endpoint := endpoints[i*setDriveCount+j]
 			// Rely on endpoints list to initialize, init lockers and available disks.
 			s.erasureLockers[i][j] = newLockAPI(endpoint)
-			disk := storageDisks[i*drivesPerSet+j]
+			disk := storageDisks[i*setDriveCount+j]
 			if disk == nil {
 				continue
 			}
@@ -340,7 +350,7 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 				disk.Close()
 				continue
 			}
-			s.endpointStrings[m*drivesPerSet+n] = disk.String()
+			s.endpointStrings[m*setDriveCount+n] = disk.String()
 			s.erasureDisks[m][n] = disk
 		}
 
@@ -353,6 +363,9 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 			bp:           bp,
 			mrfOpCh:      make(chan partialOperation, 10000),
 		}
+
+		go s.sets[i].cleanupStaleMultipartUploads(ctx,
+			GlobalMultipartCleanupInterval, GlobalMultipartExpiry)
 	}
 
 	// Start the disk monitoring and connect routine.
@@ -373,7 +386,7 @@ func (s *erasureSets) NewNSLock(ctx context.Context, bucket string, objects ...s
 
 // SetDriveCount returns the current drives per set.
 func (s *erasureSets) SetDriveCount() int {
-	return s.drivesPerSet
+	return s.setDriveCount
 }
 
 // StorageUsageInfo - combines output of StorageInfo across all erasure coded object sets.
@@ -447,13 +460,13 @@ func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo,
 
 	scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD)
 	if scParity == 0 {
-		scParity = s.drivesPerSet / 2
+		scParity = s.setDriveCount / 2
 	}
-	storageInfo.Backend.StandardSCData = s.drivesPerSet - scParity
+	storageInfo.Backend.StandardSCData = s.setDriveCount - scParity
 	storageInfo.Backend.StandardSCParity = scParity
 
 	rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
-	storageInfo.Backend.RRSCData = s.drivesPerSet - rrSCParity
+	storageInfo.Backend.RRSCData = s.setDriveCount - rrSCParity
 	storageInfo.Backend.RRSCParity = rrSCParity
 
 	if local {
@@ -827,8 +840,8 @@ func (f *FileInfoCh) Push(fi FileInfo) {
 // if the caller wishes to list N entries to call lexicallySortedEntry
 // N times until this boolean is 'false'.
 func lexicallySortedEntry(entryChs []FileInfoCh, entries []FileInfo, entriesValid []bool) (FileInfo, int, bool) {
-	for i := range entryChs {
-		entries[i], entriesValid[i] = entryChs[i].Pop()
+	for j := range entryChs {
+		entries[j], entriesValid[j] = entryChs[j].Pop()
 	}
 
 	var isTruncated = false
@@ -891,8 +904,8 @@ func lexicallySortedEntry(entryChs []FileInfoCh, entries []FileInfo, entriesVali
 // if the caller wishes to list N entries to call lexicallySortedEntry
 // N times until this boolean is 'false'.
 func lexicallySortedEntryVersions(entryChs []FileInfoVersionsCh, entries []FileInfoVersions, entriesValid []bool) (FileInfoVersions, int, bool) {
-	for i := range entryChs {
-		entries[i], entriesValid[i] = entryChs[i].Pop()
+	for j := range entryChs {
+		entries[j], entriesValid[j] = entryChs[j].Pop()
 	}
 
 	var isTruncated = false
@@ -1205,7 +1218,7 @@ func (s *erasureSets) ReloadFormat(ctx context.Context, dryRun bool) (err error)
 	}(storageDisks)
 
 	formats, _ := loadFormatErasureAll(storageDisks, false)
-	if err = checkFormatErasureValues(formats, s.drivesPerSet); err != nil {
+	if err = checkFormatErasureValues(formats, s.setDriveCount); err != nil {
 		return err
 	}
 
@@ -1245,7 +1258,7 @@ func (s *erasureSets) ReloadFormat(ctx context.Context, dryRun bool) (err error)
 			s.erasureDisks[m][n].Close()
 		}
 
-		s.endpointStrings[m*s.drivesPerSet+n] = disk.String()
+		s.endpointStrings[m*s.setDriveCount+n] = disk.String()
 		s.erasureDisks[m][n] = disk
 	}
 	s.erasureDisksMu.Unlock()
@@ -1274,19 +1287,20 @@ func isTestSetup(infos []DiskInfo, errs []error) bool {
 	return rootDiskCount == len(infos)
 }
 
-func getHealDiskInfos(storageDisks []StorageAPI) ([]DiskInfo, []error) {
+func getHealDiskInfos(storageDisks []StorageAPI, errs []error) ([]DiskInfo, []error) {
 	infos := make([]DiskInfo, len(storageDisks))
 	g := errgroup.WithNErrs(len(storageDisks))
 	for index := range storageDisks {
 		index := index
 		g.Go(func() error {
-			var err error
-			if storageDisks[index] != nil {
-				infos[index], err = storageDisks[index].DiskInfo()
-			} else {
-				// Disk not found.
-				err = errDiskNotFound
+			if errs[index] != nil && errs[index] != errUnformattedDisk {
+				return errs[index]
 			}
+			if storageDisks[index] == nil {
+				return errDiskNotFound
+			}
+			var err error
+			infos[index], err = storageDisks[index].DiskInfo()
 			return err
 		}, index)
 	}
@@ -1294,19 +1308,18 @@ func getHealDiskInfos(storageDisks []StorageAPI) ([]DiskInfo, []error) {
 }
 
 // Mark root disks as down so as not to heal them.
-func markRootDisksAsDown(storageDisks []StorageAPI) {
-	infos, errs := getHealDiskInfos(storageDisks)
-	if isTestSetup(infos, errs) {
-		// Allow healing of disks for test setups to help with testing.
-		return
-	}
-	for i := range storageDisks {
-		if infos[i].RootDisk {
-			// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
-			// defective drive we should not heal a path on the root disk.
-			logger.Info("Disk `%s` is a root disk. Please ensure the disk is mounted properly, refusing to use root disk.",
-				storageDisks[i].String())
-			storageDisks[i] = nil
+func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
+	var infos []DiskInfo
+	infos, errs = getHealDiskInfos(storageDisks, errs)
+	if !isTestSetup(infos, errs) {
+		for i := range storageDisks {
+			if storageDisks[i] != nil && infos[i].RootDisk {
+				// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
+				// defective drive we should not heal a path on the root disk.
+				logger.Info("Disk `%s` is a root disk. Please ensure the disk is mounted properly, refusing to use root disk.",
+					storageDisks[i].String())
+				storageDisks[i] = nil
+			}
 		}
 	}
 }
@@ -1326,18 +1339,19 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 		}
 	}(storageDisks)
 
-	markRootDisksAsDown(storageDisks)
-
 	formats, sErrs := loadFormatErasureAll(storageDisks, true)
-	if err = checkFormatErasureValues(formats, s.drivesPerSet); err != nil {
+	if err = checkFormatErasureValues(formats, s.setDriveCount); err != nil {
 		return madmin.HealResultItem{}, err
 	}
 
+	// Mark all root disks down
+	markRootDisksAsDown(storageDisks, sErrs)
+
 	// Prepare heal-result
 	res = madmin.HealResultItem{
 		Type:      madmin.HealItemMetadata,
 		Detail:    "disk-format",
-		DiskCount: s.setCount * s.drivesPerSet,
+		DiskCount: s.setCount * s.setDriveCount,
 		SetCount:  s.setCount,
 	}
 
@@ -1368,7 +1382,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 	markUUIDsOffline(refFormat, formats)
 
 	// Initialize a new set of set formats which will be written to disk.
-	newFormatSets := newHealFormatSets(refFormat, s.setCount, s.drivesPerSet, formats, sErrs)
+	newFormatSets := newHealFormatSets(refFormat, s.setCount, s.setDriveCount, formats, sErrs)
 
 	// Look for all offline/unformatted disks in our reference format,
 	// such that we can fill them up with new UUIDs, this looping also
@@ -1385,7 +1399,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 						newFormatSets[i][l].Erasure.This = mustGetUUID()
 						refFormat.Erasure.Sets[i][j] = newFormatSets[i][l].Erasure.This
 						for m, v := range res.After.Drives {
-							if v.Endpoint == s.endpoints.GetString(i*s.drivesPerSet+l) {
+							if v.Endpoint == s.endpoints.GetString(i*s.setDriveCount+l) {
 								res.After.Drives[m].UUID = newFormatSets[i][l].Erasure.This
 								res.After.Drives[m].State = madmin.DriveStateOk
 							}
@@ -1398,14 +1412,14 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 	}
 
 	if !dryRun {
-		var tmpNewFormats = make([]*formatErasureV3, s.setCount*s.drivesPerSet)
+		var tmpNewFormats = make([]*formatErasureV3, s.setCount*s.setDriveCount)
 		for i := range newFormatSets {
 			for j := range newFormatSets[i] {
 				if newFormatSets[i][j] == nil {
 					continue
 				}
-				tmpNewFormats[i*s.drivesPerSet+j] = newFormatSets[i][j]
-				tmpNewFormats[i*s.drivesPerSet+j].Erasure.Sets = refFormat.Erasure.Sets
+				tmpNewFormats[i*s.setDriveCount+j] = newFormatSets[i][j]
+				tmpNewFormats[i*s.setDriveCount+j].Erasure.Sets = refFormat.Erasure.Sets
 			}
 		}
 
@@ -1414,6 +1428,11 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			return madmin.HealResultItem{}, err
 		}
 
+		refFormat, err = getFormatErasureInQuorum(tmpNewFormats)
+		if err != nil {
+			return madmin.HealResultItem{}, err
+		}
+
 		// kill the monitoring loop such that we stop writing
 		// to indicate that we will re-initialize everything
 		// with new format.
@@ -1445,7 +1464,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 				s.erasureDisks[m][n].Close()
 			}
 
-			s.endpointStrings[m*s.drivesPerSet+n] = disk.String()
+			s.endpointStrings[m*s.setDriveCount+n] = disk.String()
 			s.erasureDisks[m][n] = disk
 		}
 		s.erasureDisksMu.Unlock()
@@ -1463,7 +1482,7 @@ func (s *erasureSets) HealBucket(ctx context.Context, bucket string, dryRun, rem
 	result = madmin.HealResultItem{
 		Type:      madmin.HealItemBucket,
 		Bucket:    bucket,
-		DiskCount: s.setCount * s.drivesPerSet,
+		DiskCount: s.setCount * s.setDriveCount,
 		SetCount:  s.setCount,
 	}
 
@@ -1479,7 +1498,7 @@ func (s *erasureSets) HealBucket(ctx context.Context, bucket string, dryRun, rem
 
 	// Check if we had quorum to write, if not return an appropriate error.
 	_, afterDriveOnline := result.GetOnlineCounts()
-	if afterDriveOnline < ((s.setCount*s.drivesPerSet)/2)+1 {
+	if afterDriveOnline < ((s.setCount*s.setDriveCount)/2)+1 {
 		return result, toObjectErr(errErasureWriteQuorum, bucket)
 	}
 
@@ -1535,7 +1554,7 @@ func (s *erasureSets) Walk(ctx context.Context, bucket, prefix string, results c
 					return
 				}
 
-				if quorumCount >= s.drivesPerSet/2 {
+				if quorumCount >= s.setDriveCount/2 {
 					// Read quorum exists proceed
 					for _, version := range entry.Versions {
 						results <- version.ToObjectInfo(bucket, version.Name)
@@ -1562,7 +1581,7 @@ func (s *erasureSets) Walk(ctx context.Context, bucket, prefix string, results c
 				return
 			}
 
-			if quorumCount >= s.drivesPerSet/2 {
+			if quorumCount >= s.setDriveCount/2 {
 				// Read quorum exists proceed
 				results <- entry.ToObjectInfo(bucket, entry.Name)
 			}
@@ -1589,15 +1608,15 @@ func (s *erasureSets) HealObjects(ctx context.Context, bucket, prefix string, op
 			break
 		}
 
-		if quorumCount == s.drivesPerSet && opts.ScanMode == madmin.HealNormalScan {
+		if quorumCount == s.setDriveCount && opts.ScanMode == madmin.HealNormalScan {
 			// Skip good entries.
 			continue
 		}
 
-		// Wait and proceed if there are active requests
-		waitForLowHTTPReq(int32(s.drivesPerSet))
-
 		for _, version := range entry.Versions {
+			// Wait and proceed if there are active requests
+			waitForLowHTTPReq(int32(s.setDriveCount), time.Second)
+
 			if err := healObject(bucket, version.Name, version.VersionID); err != nil {
 				return toObjectErr(err, bucket, version.Name)
 			}

cmd/erasure-zones.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -79,7 +80,7 @@ func newErasureZones(ctx context.Context, endpointZones EndpointZones) (ObjectLa
 		}
 	}
 
-	go intDataUpdateTracker.start(GlobalContext, localDrives...)
+	go intDataUpdateTracker.start(ctx, localDrives...)
 	return z, nil
 }
 
@@ -370,8 +371,10 @@ func (z *erasureZones) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter
 			case v := <-updateCloser:
 				update()
 				// Enforce quotas when all is done.
-				for _, b := range allBuckets {
-					enforceFIFOQuotaBucket(ctx, z, b.Name, allMerged.bucketUsageInfo(b.Name))
+				if firstErr == nil {
+					for _, b := range allBuckets {
+						enforceFIFOQuotaBucket(ctx, z, b.Name, allMerged.bucketUsageInfo(b.Name))
+					}
 				}
 				close(v)
 				return
@@ -387,6 +390,9 @@ func (z *erasureZones) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilter
 	case updateCloser <- ch:
 		<-ch
 	case <-ctx.Done():
+		if firstErr == nil {
+			firstErr = ctx.Err()
+		}
 	}
 	return firstErr
 }
@@ -457,12 +463,12 @@ func (z *erasureZones) GetObjectNInfo(ctx context.Context, bucket, object string
 		lock := z.NewNSLock(ctx, bucket, object)
 		switch lockType {
 		case writeLock:
-			if err = lock.GetLock(globalObjectTimeout); err != nil {
+			if err = lock.GetLock(globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.Unlock
 		case readLock:
-			if err = lock.GetRLock(globalObjectTimeout); err != nil {
+			if err = lock.GetRLock(globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.RUnlock
@@ -491,7 +497,7 @@ func (z *erasureZones) GetObjectNInfo(ctx context.Context, bucket, object string
 func (z *erasureZones) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
 	// Lock the object before reading.
 	lk := z.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalObjectTimeout); err != nil {
+	if err := lk.GetRLock(globalOperationTimeout); err != nil {
 		return err
 	}
 	defer lk.RUnlock()
@@ -515,7 +521,7 @@ func (z *erasureZones) GetObject(ctx context.Context, bucket, object string, sta
 func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	// Lock the object before reading.
 	lk := z.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalObjectTimeout); err != nil {
+	if err := lk.GetRLock(globalOperationTimeout); err != nil {
 		return ObjectInfo{}, err
 	}
 	defer lk.RUnlock()
@@ -543,7 +549,7 @@ func (z *erasureZones) GetObjectInfo(ctx context.Context, bucket, object string,
 func (z *erasureZones) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) {
 	// Lock the object.
 	lk := z.NewNSLock(ctx, bucket, object)
-	if err := lk.GetLock(globalObjectTimeout); err != nil {
+	if err := lk.GetLock(globalOperationTimeout); err != nil {
 		return ObjectInfo{}, err
 	}
 	defer lk.Unlock()
@@ -624,7 +630,7 @@ func (z *erasureZones) CopyObject(ctx context.Context, srcBucket, srcObject, dst
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 	if !cpSrcDstSame {
 		lk := z.NewNSLock(ctx, dstBucket, dstObject)
-		if err := lk.GetLock(globalObjectTimeout); err != nil {
+		if err := lk.GetLock(globalOperationTimeout); err != nil {
 			return objInfo, err
 		}
 		defer lk.Unlock()
@@ -684,15 +690,15 @@ func (z *erasureZones) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 func (z *erasureZones) listObjectsNonSlash(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
 
 	var zonesEntryChs [][]FileInfoCh
-	var zonesDrivesPerSet []int
+	var zonesListTolerancePerSet []int
 
 	endWalkCh := make(chan struct{})
 	defer close(endWalkCh)
 
 	for _, zone := range z.zones {
 		zonesEntryChs = append(zonesEntryChs,
-			zone.startMergeWalksN(ctx, bucket, prefix, "", true, endWalkCh, zone.drivesPerSet))
-		zonesDrivesPerSet = append(zonesDrivesPerSet, zone.drivesPerSet)
+			zone.startMergeWalksN(ctx, bucket, prefix, "", true, endWalkCh, zone.listTolerancePerSet))
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
 	}
 
 	var objInfos []ObjectInfo
@@ -717,7 +723,7 @@ func (z *erasureZones) listObjectsNonSlash(ctx context.Context, bucket, prefix,
 			break
 		}
 
-		if quorumCount < zonesDrivesPerSet[zoneIndex]/2 {
+		if quorumCount < zonesListTolerancePerSet[zoneIndex] {
 			// Skip entries which are not found on upto ndisks/2.
 			continue
 		}
@@ -804,20 +810,20 @@ func (z *erasureZones) listObjectsSplunk(ctx context.Context, bucket, prefix, ma
 
 	var zonesEntryChs [][]FileInfoCh
 	var zonesEndWalkCh []chan struct{}
-	var drivesPerSets []int
+	var zonesListTolerancePerSet []int
 
 	for _, zone := range z.zones {
 		entryChs, endWalkCh := zone.poolSplunk.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
 			endWalkCh = make(chan struct{})
-			entryChs = zone.startSplunkMergeWalksN(ctx, bucket, prefix, marker, endWalkCh, zone.drivesPerSet)
+			entryChs = zone.startSplunkMergeWalksN(ctx, bucket, prefix, marker, endWalkCh, zone.listTolerancePerSet)
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		drivesPerSets = append(drivesPerSets, zone.drivesPerSet)
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
 	}
 
-	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, drivesPerSets)
+	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)
 	if len(entries.Files) == 0 {
 		return loi, nil
 	}
@@ -896,20 +902,20 @@ func (z *erasureZones) listObjects(ctx context.Context, bucket, prefix, marker,
 
 	var zonesEntryChs [][]FileInfoCh
 	var zonesEndWalkCh []chan struct{}
-	var drivesPerSets []int
+	var zonesListTolerancePerSet []int
 
 	for _, zone := range z.zones {
 		entryChs, endWalkCh := zone.pool.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
 			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.drivesPerSet)
+			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet)
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		drivesPerSets = append(drivesPerSets, zone.drivesPerSet)
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
 	}
 
-	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, drivesPerSets)
+	entries := mergeZonesEntriesCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)
 	if len(entries.Files) == 0 {
 		return loi, nil
 	}
@@ -945,18 +951,9 @@ func (z *erasureZones) listObjects(ctx context.Context, bucket, prefix, marker,
 // N times until this boolean is 'false'.
 func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zoneEntriesValid [][]bool) (FileInfo, int, int, bool) {
 	for i, entryChs := range zoneEntryChs {
-		i := i
-		var wg sync.WaitGroup
 		for j := range entryChs {
-			j := j
-			wg.Add(1)
-			// Pop() entries in parallel for large drive setups.
-			go func() {
-				defer wg.Done()
-				zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
-			}()
+			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
 		}
-		wg.Wait()
 	}
 
 	var isTruncated = false
@@ -1034,18 +1031,9 @@ func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileI
 // N times until this boolean is 'false'.
 func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneEntries [][]FileInfoVersions, zoneEntriesValid [][]bool) (FileInfoVersions, int, int, bool) {
 	for i, entryChs := range zoneEntryChs {
-		i := i
-		var wg sync.WaitGroup
 		for j := range entryChs {
-			j := j
-			wg.Add(1)
-			// Pop() entries in parallel for large drive setups.
-			go func() {
-				defer wg.Done()
-				zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
-			}()
+			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
 		}
-		wg.Wait()
 	}
 
 	var isTruncated = false
@@ -1113,7 +1101,7 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 }
 
 // mergeZonesEntriesVersionsCh - merges FileInfoVersions channel to entries upto maxKeys.
-func mergeZonesEntriesVersionsCh(zonesEntryChs [][]FileInfoVersionsCh, maxKeys int, drivesPerSets []int) (entries FilesInfoVersions) {
+func mergeZonesEntriesVersionsCh(zonesEntryChs [][]FileInfoVersionsCh, maxKeys int, zonesListTolerancePerSet []int) (entries FilesInfoVersions) {
 	var i = 0
 	var zonesEntriesInfos [][]FileInfoVersions
 	var zonesEntriesValid [][]bool
@@ -1128,8 +1116,8 @@ func mergeZonesEntriesVersionsCh(zonesEntryChs [][]FileInfoVersionsCh, maxKeys i
 			break
 		}
 
-		if quorumCount < drivesPerSets[zoneIndex]/2 {
-			// Skip entries which are not found on upto ndisks/2.
+		if quorumCount < zonesListTolerancePerSet[zoneIndex] {
+			// Skip entries which are not found upto the expected tolerance
 			continue
 		}
 
@@ -1144,7 +1132,7 @@ func mergeZonesEntriesVersionsCh(zonesEntryChs [][]FileInfoVersionsCh, maxKeys i
 }
 
 // mergeZonesEntriesCh - merges FileInfo channel to entries upto maxKeys.
-func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, drivesPerSets []int) (entries FilesInfo) {
+func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, zonesListTolerancePerSet []int) (entries FilesInfo) {
 	var i = 0
 	var zonesEntriesInfos [][]FileInfo
 	var zonesEntriesValid [][]bool
@@ -1159,8 +1147,8 @@ func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, drivesPerSet
 			break
 		}
 
-		if quorumCount < drivesPerSets[zoneIndex]/2 {
-			// Skip entries which are not found on upto ndisks/2.
+		if quorumCount < zonesListTolerancePerSet[zoneIndex] {
+			// Skip entries which are not found upto configured tolerance.
 			continue
 		}
 
@@ -1176,18 +1164,9 @@ func mergeZonesEntriesCh(zonesEntryChs [][]FileInfoCh, maxKeys int, drivesPerSet
 
 func isTruncatedZones(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zoneEntriesValid [][]bool) bool {
 	for i, entryChs := range zoneEntryChs {
-		i := i
-		var wg sync.WaitGroup
 		for j := range entryChs {
-			j := j
-			wg.Add(1)
-			// Pop() entries in parallel for large drive setups.
-			go func() {
-				defer wg.Done()
-				zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
-			}()
+			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
 		}
-		wg.Wait()
 	}
 
 	var isTruncated = false
@@ -1208,24 +1187,16 @@ func isTruncatedZones(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zon
 				zoneEntryChs[i][j].Push(zoneEntries[i][j])
 			}
 		}
+
 	}
 	return isTruncated
 }
 
 func isTruncatedZonesVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneEntries [][]FileInfoVersions, zoneEntriesValid [][]bool) bool {
 	for i, entryChs := range zoneEntryChs {
-		i := i
-		var wg sync.WaitGroup
 		for j := range entryChs {
-			j := j
-			wg.Add(1)
-			// Pop() entries in parallel for large drive setups.
-			go func() {
-				defer wg.Done()
-				zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
-			}()
+			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
 		}
-		wg.Wait()
 	}
 
 	var isTruncated = false
@@ -1301,19 +1272,19 @@ func (z *erasureZones) listObjectVersions(ctx context.Context, bucket, prefix, m
 
 	var zonesEntryChs [][]FileInfoVersionsCh
 	var zonesEndWalkCh []chan struct{}
-	var drivesPerSets []int
+	var zonesListTolerancePerSet []int
 	for _, zone := range z.zones {
 		entryChs, endWalkCh := zone.poolVersions.Release(listParams{bucket, recursive, marker, prefix})
 		if entryChs == nil {
 			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.drivesPerSet)
+			entryChs = zone.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet)
 		}
 		zonesEntryChs = append(zonesEntryChs, entryChs)
 		zonesEndWalkCh = append(zonesEndWalkCh, endWalkCh)
-		drivesPerSets = append(drivesPerSets, zone.drivesPerSet)
+		zonesListTolerancePerSet = append(zonesListTolerancePerSet, zone.listTolerancePerSet)
 	}
 
-	entries := mergeZonesEntriesVersionsCh(zonesEntryChs, maxKeys, drivesPerSets)
+	entries := mergeZonesEntriesVersionsCh(zonesEntryChs, maxKeys, zonesListTolerancePerSet)
 	if len(entries.FilesVersions) == 0 {
 		return loi, nil
 	}
@@ -1731,7 +1702,7 @@ func (z *erasureZones) ListBuckets(ctx context.Context) (buckets []BucketInfo, e
 func (z *erasureZones) ReloadFormat(ctx context.Context, dryRun bool) error {
 	// Acquire lock on format.json
 	formatLock := z.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
-	if err := formatLock.GetRLock(globalHealingTimeout); err != nil {
+	if err := formatLock.GetRLock(globalOperationTimeout); err != nil {
 		return err
 	}
 	defer formatLock.RUnlock()
@@ -1747,7 +1718,7 @@ func (z *erasureZones) ReloadFormat(ctx context.Context, dryRun bool) error {
 func (z *erasureZones) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
 	// Acquire lock on format.json
 	formatLock := z.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
-	if err := formatLock.GetLock(globalHealingTimeout); err != nil {
+	if err := formatLock.GetLock(globalOperationTimeout); err != nil {
 		return madmin.HealResultItem{}, err
 	}
 	defer formatLock.Unlock()
@@ -1824,7 +1795,7 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 
 		var zoneDrivesPerSet []int
 		for _, zone := range z.zones {
-			zoneDrivesPerSet = append(zoneDrivesPerSet, zone.drivesPerSet)
+			zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount)
 		}
 
 		var zonesEntriesInfos [][]FileInfoVersions
@@ -1865,7 +1836,7 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 
 	var zoneDrivesPerSet []int
 	for _, zone := range z.zones {
-		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.drivesPerSet)
+		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount)
 	}
 
 	var zonesEntriesInfos [][]FileInfo
@@ -1897,7 +1868,7 @@ func (z *erasureZones) Walk(ctx context.Context, bucket, prefix string, results
 }
 
 // HealObjectFn closure function heals the object.
-type HealObjectFn func(string, string, string) error
+type HealObjectFn func(bucket, object, versionID string) error
 
 func (z *erasureZones) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObject HealObjectFn) error {
 	var zonesEntryChs [][]FileInfoVersionsCh
@@ -1912,7 +1883,7 @@ func (z *erasureZones) HealObjects(ctx context.Context, bucket, prefix string, o
 
 	var zoneDrivesPerSet []int
 	for _, zone := range z.zones {
-		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.drivesPerSet)
+		zoneDrivesPerSet = append(zoneDrivesPerSet, zone.setDriveCount)
 	}
 
 	var zonesEntriesInfos [][]FileInfoVersions
@@ -1922,42 +1893,51 @@ func (z *erasureZones) HealObjects(ctx context.Context, bucket, prefix string, o
 		zonesEntriesValid = append(zonesEntriesValid, make([]bool, len(entryChs)))
 	}
 
+	// If listing did not return any entries upon first attempt, we
+	// return `ObjectNotFound`, to indicate the caller for any
+	// actions they may want to take as if `prefix` is missing.
+	err := toObjectErr(errFileNotFound, bucket, prefix)
 	for {
 		entry, quorumCount, zoneIndex, ok := lexicallySortedEntryZoneVersions(zonesEntryChs, zonesEntriesInfos, zonesEntriesValid)
 		if !ok {
 			break
 		}
+		// Indicate that first attempt was a success and subsequent loop
+		// knows that its not our first attempt at 'prefix'
+		err = nil
 
+		if zoneIndex >= len(zoneDrivesPerSet) || zoneIndex < 0 {
+			return fmt.Errorf("invalid zone index returned: %d", zoneIndex)
+		}
 		if quorumCount == zoneDrivesPerSet[zoneIndex] && opts.ScanMode == madmin.HealNormalScan {
 			// Skip good entries.
 			continue
 		}
 
-		// Wait and proceed if there are active requests
-		waitForLowHTTPReq(int32(zoneDrivesPerSet[zoneIndex]))
-
 		for _, version := range entry.Versions {
+			// Wait and proceed if there are active requests
+			waitForLowHTTPReq(int32(zoneDrivesPerSet[zoneIndex]), time.Second)
 			if err := healObject(bucket, version.Name, version.VersionID); err != nil {
 				return toObjectErr(err, bucket, version.Name)
 			}
 		}
 	}
 
-	return nil
+	return err
 }
 
 func (z *erasureZones) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	lk := z.NewNSLock(ctx, bucket, object)
 	if bucket == minioMetaBucket {
 		// For .minio.sys bucket heals we should hold write locks.
-		if err := lk.GetLock(globalHealingTimeout); err != nil {
+		if err := lk.GetLock(globalOperationTimeout); err != nil {
 			return madmin.HealResultItem{}, err
 		}
 		defer lk.Unlock()
 	} else {
 		// Lock the object before healing. Use read lock since healing
 		// will only regenerate parts & xl.meta of outdated disks.
-		if err := lk.GetRLock(globalHealingTimeout); err != nil {
+		if err := lk.GetRLock(globalOperationTimeout); err != nil {
 			return madmin.HealResultItem{}, err
 		}
 		defer lk.RUnlock()
@@ -2032,6 +2012,7 @@ type HealthOptions struct {
 // was queried
 type HealthResult struct {
 	Healthy       bool
+	HealingDrives int
 	ZoneID, SetID int
 	WriteQuorum   int
 }
@@ -2062,9 +2043,11 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 		}
 	}
 
+	reqInfo := (&logger.ReqInfo{}).AppendTags("maintenance", strconv.FormatBool(opts.Maintenance))
+
 	for zoneIdx := range erasureSetUpCount {
 		parityDrives := globalStorageClass.GetParityForSC(storageclass.STANDARD)
-		diskCount := z.zones[zoneIdx].drivesPerSet
+		diskCount := z.zones[zoneIdx].setDriveCount
 		if parityDrives == 0 {
 			parityDrives = getDefaultParityBlocks(diskCount)
 		}
@@ -2075,8 +2058,9 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 		}
 		for setIdx := range erasureSetUpCount[zoneIdx] {
 			if erasureSetUpCount[zoneIdx][setIdx] < writeQuorum {
-				logger.LogIf(ctx, fmt.Errorf("Write quorum lost on zone: %d, set: %d, expected write quorum: %d",
-					zoneIdx, setIdx, writeQuorum))
+				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
+					fmt.Errorf("Write quorum may be lost on zone: %d, set: %d, expected write quorum: %d",
+						zoneIdx, setIdx, writeQuorum))
 				return HealthResult{
 					Healthy:     false,
 					ZoneID:      zoneIdx,
@@ -2086,8 +2070,35 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 			}
 		}
 	}
+
+	// when maintenance is not specified we don't have
+	// to look at the healing side of the code.
+	if !opts.Maintenance {
+		return HealthResult{
+			Healthy: true,
+		}
+	}
+
+	// check if local disks are being healed, if they are being healed
+	// we need to tell healthy status as 'false' so that this server
+	// is not taken down for maintenance
+	aggHealStateResult, err := getAggregatedBackgroundHealState(ctx, true)
+	if err != nil {
+		logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Unable to verify global heal status: %w", err))
+		return HealthResult{
+			Healthy: false,
+		}
+	}
+
+	if len(aggHealStateResult.HealDisks) > 0 {
+		logger.LogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Total drives to be healed %d", len(aggHealStateResult.HealDisks)))
+	}
+
+	healthy := len(aggHealStateResult.HealDisks) == 0
+
 	return HealthResult{
-		Healthy: true,
+		Healthy:       healthy,
+		HealingDrives: len(aggHealStateResult.HealDisks),
 	}
 }
 

cmd/erasure.go

@@ -146,6 +146,11 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 					ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 					logger.LogIf(ctx, err)
 				}
+				disksInfo[index] = madmin.Disk{
+					State:    diskErrToDriveState(err),
+					Endpoint: endpoints[index],
+				}
+				return err
 			}
 			di := madmin.Disk{
 				Endpoint:       endpoints[index],
@@ -154,6 +159,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 				UsedSpace:      info.Used,
 				AvailableSpace: info.Free,
 				UUID:           info.ID,
+				RootDisk:       info.RootDisk,
 				State:          diskErrToDriveState(err),
 			}
 			if info.Total > 0 {
@@ -167,7 +173,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 	errs = g.Wait()
 	// Wait for the routines.
 	for i, diskInfoErr := range errs {
-		ep := endpoints[i]
+		ep := disksInfo[i].Endpoint
 		if diskInfoErr != nil {
 			offlineDisks[ep]++
 			continue
@@ -175,7 +181,27 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 		onlineDisks[ep]++
 	}
 
-	// Success.
+	rootDiskCount := 0
+	for _, di := range disksInfo {
+		if di.RootDisk {
+			rootDiskCount++
+		}
+	}
+
+	if len(disksInfo) == rootDiskCount {
+		// Success.
+		return disksInfo, errs, onlineDisks, offlineDisks
+	}
+
+	// Root disk should be considered offline
+	for i := range disksInfo {
+		ep := disksInfo[i].Endpoint
+		if disksInfo[i].RootDisk {
+			offlineDisks[ep]++
+			onlineDisks[ep]--
+		}
+	}
+
 	return disksInfo, errs, onlineDisks, offlineDisks
 }
 
@@ -235,21 +261,45 @@ func (er erasureObjects) CrawlAndGetDataUsage(ctx context.Context, bf *bloomFilt
 // CrawlAndGetDataUsage will start crawling buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
 func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error {
-	var disks []StorageAPI
+	if len(buckets) == 0 {
+		return nil
+	}
 
+	// Collect any disk healing.
+	healing, err := getAggregatedBackgroundHealState(ctx, true)
+	if err != nil {
+		return err
+	}
+	healDisks := make(map[string]struct{}, len(healing.HealDisks))
+	for _, disk := range healing.HealDisks {
+		healDisks[disk] = struct{}{}
+	}
+
+	// Collect disks we can use.
+	var disks []StorageAPI
 	for _, d := range er.getLoadBalancedDisks() {
 		if d == nil || !d.IsOnline() {
 			continue
 		}
+		di, err := d.DiskInfo()
+		if err != nil {
+			logger.LogIf(ctx, err)
+			continue
+		}
+		if _, ok := healDisks[di.Endpoint]; ok {
+			logger.Info(color.Green("data-crawl:")+" Disk %q is Healing, skipping disk.", di.Endpoint)
+			continue
+		}
 		disks = append(disks, d)
 	}
-	if len(disks) == 0 || len(buckets) == 0 {
+	if len(disks) == 0 {
+		logger.Info(color.Green("data-crawl:") + " No disks found, skipping crawl")
 		return nil
 	}
 
 	// Load bucket totals
 	oldCache := dataUsageCache{}
-	err := oldCache.load(ctx, er, dataUsageCacheName)
+	err = oldCache.load(ctx, er, dataUsageCacheName)
 	if err != nil {
 		return err
 	}
@@ -273,20 +323,12 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 		}
 	}
 
-	// Add existing buckets if changes or lifecycles.
+	// Add existing buckets.
 	for _, b := range buckets {
 		e := oldCache.find(b.Name)
 		if e != nil {
 			cache.replace(b.Name, dataUsageRoot, *e)
-			lc, err := globalLifecycleSys.Get(b.Name)
-			activeLC := err == nil && lc.HasActiveRules("", true)
-			if activeLC || bf == nil || bf.containsDir(b.Name) {
-				bucketCh <- b
-			} else {
-				if intDataUpdateTracker.debug {
-					logger.Info(color.Green("crawlAndGetDataUsage:")+" Skipping bucket %v, not updated", b.Name)
-				}
-			}
+			bucketCh <- b
 		}
 	}
 

cmd/format-disk-cache.go

@@ -365,7 +365,7 @@ func migrateCacheData(ctx context.Context, c *diskCache, bucket, object, oldfile
 		}
 		actualSize, _ = sio.EncryptedSize(uint64(st.Size()))
 	}
-	_, err = c.bitrotWriteToCache(destDir, cacheDataFile, reader, uint64(actualSize))
+	_, err = c.bitrotWriteToCache(destDir, cacheDataFile, reader, actualSize)
 	return err
 }
 

cmd/format-erasure.go

@@ -335,11 +335,13 @@ func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasur
 	return formats, g.Wait()
 }
 
-func saveFormatErasure(disk StorageAPI, format interface{}, diskID string) error {
-	if format == nil || disk == nil {
+func saveFormatErasure(disk StorageAPI, format *formatErasureV3) error {
+	if disk == nil || format == nil {
 		return errDiskNotFound
 	}
 
+	diskID := format.Erasure.This
+
 	if err := makeFormatErasureMetaVolumes(disk); err != nil {
 		return err
 	}
@@ -437,7 +439,7 @@ func checkFormatErasureValue(formatErasure *formatErasureV3) error {
 }
 
 // Check all format values.
-func checkFormatErasureValues(formats []*formatErasureV3, drivesPerSet int) error {
+func checkFormatErasureValues(formats []*formatErasureV3, setDriveCount int) error {
 	for i, formatErasure := range formats {
 		if formatErasure == nil {
 			continue
@@ -452,8 +454,8 @@ func checkFormatErasureValues(formats []*formatErasureV3, drivesPerSet int) erro
 		// Only if custom erasure drive count is set,
 		// we should fail here other proceed to honor what
 		// is present on the disk.
-		if globalCustomErasureDriveCount && len(formatErasure.Erasure.Sets[0]) != drivesPerSet {
-			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", humanize.Ordinal(i+1), len(formatErasure.Erasure.Sets[0]), drivesPerSet)
+		if globalCustomErasureDriveCount && len(formatErasure.Erasure.Sets[0]) != setDriveCount {
+			return fmt.Errorf("%s disk is already formatted with %d drives per erasure set. This cannot be changed to %d, please revert your MINIO_ERASURE_SET_DRIVE_COUNT setting", humanize.Ordinal(i+1), len(formatErasure.Erasure.Sets[0]), setDriveCount)
 		}
 	}
 	return nil
@@ -549,7 +551,7 @@ func formatErasureFixLocalDeploymentID(endpoints Endpoints, storageDisks []Stora
 					return nil
 				}
 				format.ID = refFormat.ID
-				if err := saveFormatErasure(storageDisks[index], format, format.Erasure.This); err != nil {
+				if err := saveFormatErasure(storageDisks[index], format); err != nil {
 					logger.LogIf(GlobalContext, err)
 					return fmt.Errorf("Unable to save format.json, %w", err)
 				}
@@ -695,7 +697,7 @@ func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, format
 			if formats[index] == nil {
 				return errDiskNotFound
 			}
-			return saveFormatErasure(storageDisks[index], formats[index], formats[index].Erasure.This)
+			return saveFormatErasure(storageDisks[index], formats[index])
 		}, index)
 	}
 
@@ -722,13 +724,9 @@ func initStorageDisksWithErrors(endpoints Endpoints) ([]StorageAPI, []error) {
 	g := errgroup.WithNErrs(len(endpoints))
 	for index := range endpoints {
 		index := index
-		g.Go(func() error {
-			storageDisk, err := newStorageAPI(endpoints[index])
-			if err != nil {
-				return err
-			}
-			storageDisks[index] = storageDisk
-			return nil
+		g.Go(func() (err error) {
+			storageDisks[index], err = newStorageAPI(endpoints[index])
+			return err
 		}, index)
 	}
 	return storageDisks, g.Wait()
@@ -773,7 +771,7 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats
 			}
 			if formats[i].Erasure.This == "" {
 				formats[i].Erasure.This = formats[i].Erasure.Sets[0][i]
-				if err := saveFormatErasure(storageDisks[i], formats[i], formats[i].Erasure.This); err != nil {
+				if err := saveFormatErasure(storageDisks[i], formats[i]); err != nil {
 					return err
 				}
 			}
@@ -790,22 +788,22 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats
 }
 
 // initFormatErasure - save Erasure format configuration on all disks.
-func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, drivesPerSet int, deploymentID string) (*formatErasureV3, error) {
-	format := newFormatErasureV3(setCount, drivesPerSet)
+func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, setDriveCount int, deploymentID string, sErrs []error) (*formatErasureV3, error) {
+	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, len(storageDisks))
-	wantAtMost := ecDrivesNoConfig(drivesPerSet)
+	wantAtMost := ecDrivesNoConfig(setDriveCount)
 
 	for i := 0; i < setCount; i++ {
-		hostCount := make(map[string]int, drivesPerSet)
-		for j := 0; j < drivesPerSet; j++ {
-			disk := storageDisks[i*drivesPerSet+j]
+		hostCount := make(map[string]int, setDriveCount)
+		for j := 0; j < setDriveCount; j++ {
+			disk := storageDisks[i*setDriveCount+j]
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
 			if deploymentID != "" {
 				newFormat.ID = deploymentID
 			}
 			hostCount[disk.Hostname()]++
-			formats[i*drivesPerSet+j] = newFormat
+			formats[i*setDriveCount+j] = newFormat
 		}
 		if len(hostCount) > 0 {
 			var once sync.Once
@@ -819,8 +817,8 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 							return
 						}
 						logger.Info(" * Set %v:", i+1)
-						for j := 0; j < drivesPerSet; j++ {
-							disk := storageDisks[i*drivesPerSet+j]
+						for j := 0; j < setDriveCount; j++ {
+							disk := storageDisks[i*setDriveCount+j]
 							logger.Info("   - Drive: %s", disk.String())
 						}
 					})
@@ -831,6 +829,9 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 		}
 	}
 
+	// Mark all root disks down
+	markRootDisksAsDown(storageDisks, sErrs)
+
 	// Save formats `format.json` across all disks.
 	if err := saveFormatErasureAll(ctx, storageDisks, formats); err != nil {
 		return nil, err
@@ -841,15 +842,15 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 
 // ecDrivesNoConfig returns the erasure coded drives in a set if no config has been set.
 // It will attempt to read it from env variable and fall back to drives/2.
-func ecDrivesNoConfig(drivesPerSet int) int {
+func ecDrivesNoConfig(setDriveCount int) int {
 	ecDrives := globalStorageClass.GetParityForSC(storageclass.STANDARD)
 	if ecDrives == 0 {
-		cfg, err := storageclass.LookupConfig(nil, drivesPerSet)
+		cfg, err := storageclass.LookupConfig(nil, setDriveCount)
 		if err == nil {
 			ecDrives = cfg.Standard.Parity
 		}
 		if ecDrives == 0 {
-			ecDrives = drivesPerSet / 2
+			ecDrives = setDriveCount / 2
 		}
 	}
 	return ecDrives
@@ -919,14 +920,14 @@ func markUUIDsOffline(refFormat *formatErasureV3, formats []*formatErasureV3) {
 }
 
 // Initialize a new set of set formats which will be written to all disks.
-func newHealFormatSets(refFormat *formatErasureV3, setCount, drivesPerSet int, formats []*formatErasureV3, errs []error) [][]*formatErasureV3 {
+func newHealFormatSets(refFormat *formatErasureV3, setCount, setDriveCount int, formats []*formatErasureV3, errs []error) [][]*formatErasureV3 {
 	newFormats := make([][]*formatErasureV3, setCount)
 	for i := range refFormat.Erasure.Sets {
-		newFormats[i] = make([]*formatErasureV3, drivesPerSet)
+		newFormats[i] = make([]*formatErasureV3, setDriveCount)
 	}
 	for i := range refFormat.Erasure.Sets {
 		for j := range refFormat.Erasure.Sets[i] {
-			if errs[i*drivesPerSet+j] == errUnformattedDisk || errs[i*drivesPerSet+j] == nil {
+			if errs[i*setDriveCount+j] == errUnformattedDisk || errs[i*setDriveCount+j] == nil {
 				newFormats[i][j] = &formatErasureV3{}
 				newFormats[i][j].Version = refFormat.Version
 				newFormats[i][j].ID = refFormat.ID
@@ -934,13 +935,13 @@ func newHealFormatSets(refFormat *formatErasureV3, setCount, drivesPerSet int, f
 				newFormats[i][j].Erasure.Version = refFormat.Erasure.Version
 				newFormats[i][j].Erasure.DistributionAlgo = refFormat.Erasure.DistributionAlgo
 			}
-			if errs[i*drivesPerSet+j] == errUnformattedDisk {
+			if errs[i*setDriveCount+j] == errUnformattedDisk {
 				newFormats[i][j].Erasure.This = ""
 				newFormats[i][j].Erasure.Sets = nil
 				continue
 			}
-			if errs[i*drivesPerSet+j] == nil {
-				newFormats[i][j].Erasure.This = formats[i*drivesPerSet+j].Erasure.This
+			if errs[i*setDriveCount+j] == nil {
+				newFormats[i][j].Erasure.This = formats[i*setDriveCount+j].Erasure.This
 				newFormats[i][j].Erasure.Sets = nil
 			}
 		}

cmd/format-erasure_test.go

@@ -324,16 +324,16 @@ func TestCheckFormatErasureValue(t *testing.T) {
 // Tests getFormatErasureInQuorum()
 func TestGetFormatErasureInQuorumCheck(t *testing.T) {
 	setCount := 2
-	drivesPerSet := 16
+	setDriveCount := 16
 
-	format := newFormatErasureV3(setCount, drivesPerSet)
+	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, 32)
 
 	for i := 0; i < setCount; i++ {
-		for j := 0; j < drivesPerSet; j++ {
+		for j := 0; j < setDriveCount; j++ {
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
-			formats[i*drivesPerSet+j] = newFormat
+			formats[i*setDriveCount+j] = newFormat
 		}
 	}
 
@@ -390,16 +390,16 @@ func TestGetFormatErasureInQuorumCheck(t *testing.T) {
 // Tests formatErasureGetDeploymentID()
 func TestGetErasureID(t *testing.T) {
 	setCount := 2
-	drivesPerSet := 8
+	setDriveCount := 8
 
-	format := newFormatErasureV3(setCount, drivesPerSet)
+	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, 16)
 
 	for i := 0; i < setCount; i++ {
-		for j := 0; j < drivesPerSet; j++ {
+		for j := 0; j < setDriveCount; j++ {
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
-			formats[i*drivesPerSet+j] = newFormat
+			formats[i*setDriveCount+j] = newFormat
 		}
 	}
 
@@ -445,17 +445,17 @@ func TestGetErasureID(t *testing.T) {
 // Initialize new format sets.
 func TestNewFormatSets(t *testing.T) {
 	setCount := 2
-	drivesPerSet := 16
+	setDriveCount := 16
 
-	format := newFormatErasureV3(setCount, drivesPerSet)
+	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, 32)
 	errs := make([]error, 32)
 
 	for i := 0; i < setCount; i++ {
-		for j := 0; j < drivesPerSet; j++ {
+		for j := 0; j < setDriveCount; j++ {
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
-			formats[i*drivesPerSet+j] = newFormat
+			formats[i*setDriveCount+j] = newFormat
 		}
 	}
 
@@ -467,7 +467,7 @@ func TestNewFormatSets(t *testing.T) {
 	// 16th disk is unformatted.
 	errs[15] = errUnformattedDisk
 
-	newFormats := newHealFormatSets(quorumFormat, setCount, drivesPerSet, formats, errs)
+	newFormats := newHealFormatSets(quorumFormat, setCount, setDriveCount, formats, errs)
 	if newFormats == nil {
 		t.Fatal("Unexpected failure")
 	}

cmd/fs-v1-helpers.go

@@ -41,9 +41,8 @@ func fsRemoveFile(ctx context.Context, filePath string) (err error) {
 		return err
 	}
 
-	if err = os.Remove((filePath)); err != nil {
-		err = osErrToFileErr(err)
-		if err != errFileNotFound {
+	if err = os.Remove(filePath); err != nil {
+		if err = osErrToFileErr(err); err != errFileNotFound {
 			logger.LogIf(ctx, err)
 		}
 	}

cmd/fs-v1-multipart.go

@@ -319,22 +319,22 @@ func (fs *FSObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID
 
 	tmpPartPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, uploadID+"."+mustGetUUID()+"."+strconv.Itoa(partID))
 	bytesWritten, err := fsCreateFile(ctx, tmpPartPath, data, buf, data.Size())
+
+	// Delete temporary part in case of failure. If
+	// PutObjectPart succeeds then there would be nothing to
+	// delete in which case we just ignore the error.
+	defer fsRemoveFile(ctx, tmpPartPath)
+
 	if err != nil {
-		fsRemoveFile(ctx, tmpPartPath)
 		return pi, toObjectErr(err, minioMetaTmpBucket, tmpPartPath)
 	}
 
 	// Should return IncompleteBody{} error when reader has fewer
 	// bytes than specified in request header.
 	if bytesWritten < data.Size() {
-		fsRemoveFile(ctx, tmpPartPath)
 		return pi, IncompleteBody{}
 	}
 
-	// Delete temporary part in case of failure. If
-	// PutObjectPart succeeds then there would be nothing to
-	// delete in which case we just ignore the error.
-	defer fsRemoveFile(ctx, tmpPartPath)
 	etag := r.MD5CurrentHexString()
 
 	if etag == "" {
@@ -707,7 +707,7 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 
 	// Hold write lock on the object.
 	destLock := fs.NewNSLock(ctx, bucket, object)
-	if err = destLock.GetLock(globalObjectTimeout); err != nil {
+	if err = destLock.GetLock(globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer destLock.Unlock()
@@ -863,8 +863,7 @@ func (fs *FSObjects) cleanupStaleMultipartUploads(ctx context.Context, cleanupIn
 						fs.appendFileMapMu.Lock()
 						bgAppend, ok := fs.appendFileMap[uploadID]
 						if ok {
-							err := fsRemoveFile(ctx, bgAppend.filePath)
-							logger.LogIf(ctx, err)
+							_ = fsRemoveFile(ctx, bgAppend.filePath)
 							delete(fs.appendFileMap, uploadID)
 						}
 						fs.appendFileMapMu.Unlock()

cmd/fs-v1.go

@@ -179,7 +179,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	fs.fsFormatRlk = rlk
 
 	go fs.cleanupStaleMultipartUploads(ctx, GlobalMultipartCleanupInterval, GlobalMultipartExpiry)
-	go intDataUpdateTracker.start(GlobalContext, fsPath)
+	go intDataUpdateTracker.start(ctx, fsPath)
 
 	// Return successfully initialized object layer.
 	return fs, nil
@@ -352,6 +352,8 @@ func (fs *FSObjects) crawlBucket(ctx context.Context, bucket string, cache dataU
 		if fiErr != nil {
 			return 0, errSkipFile
 		}
+		// We cannot heal in FS mode.
+		item.heal = false
 
 		oi := fsMeta.ToObjectInfo(bucket, object, fi)
 		sz := item.applyActions(ctx, fs, actionMeta{oi: oi})
@@ -601,7 +603,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 
 	if !cpSrcDstSame {
 		objectDWLock := fs.NewNSLock(ctx, dstBucket, dstObject)
-		if err := objectDWLock.GetLock(globalObjectTimeout); err != nil {
+		if err := objectDWLock.GetLock(globalOperationTimeout); err != nil {
 			return oi, err
 		}
 		defer objectDWLock.Unlock()
@@ -691,12 +693,12 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		lock := fs.NewNSLock(ctx, bucket, object)
 		switch lockType {
 		case writeLock:
-			if err = lock.GetLock(globalObjectTimeout); err != nil {
+			if err = lock.GetLock(globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.Unlock
 		case readLock:
-			if err = lock.GetRLock(globalObjectTimeout); err != nil {
+			if err = lock.GetRLock(globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.RUnlock
@@ -709,7 +711,7 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 		nsUnlocker()
 		return nil, toObjectErr(err, bucket, object)
 	}
-	// For a directory, we need to send an reader that returns no bytes.
+	// For a directory, we need to return a reader that returns no bytes.
 	if HasSuffix(object, SlashSeparator) {
 		// The lock taken above is released when
 		// objReader.Close() is called by the caller.
@@ -782,7 +784,7 @@ func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offse
 
 	// Lock the object before reading.
 	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalObjectTimeout); err != nil {
+	if err := lk.GetRLock(globalOperationTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return err
 	}
@@ -906,8 +908,62 @@ func (fs *FSObjects) defaultFsJSON(object string) fsMetaV1 {
 	return fsMeta
 }
 
+func (fs *FSObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
+	fsMeta := fsMetaV1{}
+	if HasSuffix(object, SlashSeparator) {
+		fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object))
+		if err != nil {
+			return oi, err
+		}
+		return fsMeta.ToObjectInfo(bucket, object, fi), nil
+	}
+
+	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
+	// Read `fs.json` to perhaps contend with
+	// parallel Put() operations.
+
+	rc, _, err := fsOpenFile(ctx, fsMetaPath, 0)
+	if err == nil {
+		fsMetaBuf, rerr := ioutil.ReadAll(rc)
+		rc.Close()
+		if rerr == nil {
+			var json = jsoniter.ConfigCompatibleWithStandardLibrary
+			if rerr = json.Unmarshal(fsMetaBuf, &fsMeta); rerr != nil {
+				// For any error to read fsMeta, set default ETag and proceed.
+				fsMeta = fs.defaultFsJSON(object)
+			}
+		} else {
+			// For any error to read fsMeta, set default ETag and proceed.
+			fsMeta = fs.defaultFsJSON(object)
+		}
+	}
+
+	// Return a default etag and content-type based on the object's extension.
+	if err == errFileNotFound {
+		fsMeta = fs.defaultFsJSON(object)
+	}
+
+	// Ignore if `fs.json` is not available, this is true for pre-existing data.
+	if err != nil && err != errFileNotFound {
+		logger.LogIf(ctx, err)
+		return oi, err
+	}
+
+	// Stat the file to get file size.
+	fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
+	if err != nil {
+		return oi, err
+	}
+
+	return fsMeta.ToObjectInfo(bucket, object, fi), nil
+}
+
 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
 func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
+	if strings.HasSuffix(object, SlashSeparator) && !fs.isObjectDir(bucket, object) {
+		return oi, errFileNotFound
+	}
+
 	fsMeta := fsMetaV1{}
 	if HasSuffix(object, SlashSeparator) {
 		fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object))
@@ -956,7 +1012,7 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
 	// Lock the object before reading.
 	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalObjectTimeout); err != nil {
+	if err := lk.GetRLock(globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer lk.RUnlock()
@@ -994,7 +1050,7 @@ func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
 	oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
 	if err == errCorruptedFormat || err == io.EOF {
 		lk := fs.NewNSLock(ctx, bucket, object)
-		if err = lk.GetLock(globalObjectTimeout); err != nil {
+		if err = lk.GetLock(globalOperationTimeout); err != nil {
 			return oi, toObjectErr(err, bucket, object)
 		}
 
@@ -1045,7 +1101,7 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 
 	// Lock the object.
 	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetLock(globalObjectTimeout); err != nil {
+	if err := lk.GetLock(globalOperationTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return objInfo, err
 	}
@@ -1144,8 +1200,13 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	buf := make([]byte, int(bufSize))
 	fsTmpObjPath := pathJoin(fs.fsPath, minioMetaTmpBucket, fs.fsUUID, tempObj)
 	bytesWritten, err := fsCreateFile(ctx, fsTmpObjPath, data, buf, data.Size())
+
+	// Delete the temporary object in the case of a
+	// failure. If PutObject succeeds, then there would be
+	// nothing to delete.
+	defer fsRemoveFile(ctx, fsTmpObjPath)
+
 	if err != nil {
-		fsRemoveFile(ctx, fsTmpObjPath)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 	fsMeta.Meta["etag"] = r.MD5CurrentHexString()
@@ -1153,15 +1214,9 @@ func (fs *FSObjects) putObject(ctx context.Context, bucket string, object string
 	// Should return IncompleteBody{} error when reader has fewer
 	// bytes than specified in request header.
 	if bytesWritten < data.Size() {
-		fsRemoveFile(ctx, fsTmpObjPath)
 		return ObjectInfo{}, IncompleteBody{}
 	}
 
-	// Delete the temporary object in the case of a
-	// failure. If PutObject succeeds, then there would be
-	// nothing to delete.
-	defer fsRemoveFile(ctx, fsTmpObjPath)
-
 	// Entire object was written to the temp location, now it's safe to rename it to the actual location.
 	fsNSObjPath := pathJoin(fs.fsPath, bucket, object)
 	if err = fsRenameFile(ctx, fsTmpObjPath, fsNSObjPath); err != nil {
@@ -1278,23 +1333,31 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string, op
 	return ObjectInfo{Bucket: bucket, Name: object}, nil
 }
 
+func (fs *FSObjects) isLeafDir(bucket string, leafPath string) bool {
+	return fs.isObjectDir(bucket, leafPath)
+}
+
+func (fs *FSObjects) isLeaf(bucket string, leafPath string) bool {
+	return !strings.HasSuffix(leafPath, slashSeparator)
+}
+
 // Returns function "listDir" of the type listDirFunc.
 // isLeaf - is used by listDir function to check if an entry
 // is a leaf or non-leaf entry.
 func (fs *FSObjects) listDirFactory() ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
 		var err error
 		entries, err = readDir(pathJoin(fs.fsPath, bucket, prefixDir))
 		if err != nil && err != errFileNotFound {
 			logger.LogIf(GlobalContext, err)
-			return false, nil
+			return false, nil, false
 		}
 		if len(entries) == 0 {
-			return true, nil
+			return true, nil, false
 		}
-		sort.Strings(entries)
-		return false, filterMatchingPrefix(entries, prefixEntry)
+		entries, delayIsLeaf = filterListEntries(bucket, prefixDir, entries, prefixEntry, fs.isLeaf)
+		return false, entries, delayIsLeaf
 	}
 
 	// Return list factory instance.
@@ -1391,14 +1454,13 @@ func (fs *FSObjects) ListObjectVersions(ctx context.Context, bucket, prefix, mar
 // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
 // state for future re-entrant list requests.
 func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-
 	atomic.AddInt64(&fs.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt64(&fs.activeIOCount, -1)
 	}()
 
 	return listObjects(ctx, fs, bucket, prefix, marker, delimiter, maxKeys, fs.listPool,
-		fs.listDirFactory(), fs.getObjectInfo, fs.getObjectInfo)
+		fs.listDirFactory(), fs.isLeaf, fs.isLeafDir, fs.getObjectInfoNoFSLock, fs.getObjectInfoNoFSLock)
 }
 
 // GetObjectTags - get object tags from an existing object
@@ -1495,7 +1557,7 @@ func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun, remo
 // error walker returns error. Optionally if context.Done() is received
 // then Walk() stops the walker.
 func (fs *FSObjects) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo, opts ObjectOptions) error {
-	return fsWalk(ctx, fs, bucket, prefix, fs.listDirFactory(), results, fs.getObjectInfo, fs.getObjectInfo)
+	return fsWalk(ctx, fs, bucket, prefix, fs.listDirFactory(), fs.isLeaf, fs.isLeafDir, results, fs.getObjectInfoNoFSLock, fs.getObjectInfoNoFSLock)
 }
 
 // HealObjects - no-op for fs. Valid only for Erasure.

cmd/gateway-common.go

@@ -48,8 +48,8 @@ var (
 	// ListObjects function alias.
 	ListObjects = listObjects
 
-	// FilterMatchingPrefix function alias.
-	FilterMatchingPrefix = filterMatchingPrefix
+	// FilterListEntries function alias.
+	FilterListEntries = filterListEntries
 
 	// IsStringEqual is string equal.
 	IsStringEqual = isStringEqual

cmd/gateway-interface.go

@@ -21,7 +21,14 @@ import (
 )
 
 // GatewayMinioSysTmp prefix is used in Azure/GCS gateway for save metadata sent by Initialize Multipart Upload API.
-const GatewayMinioSysTmp = "minio.sys.tmp/"
+const (
+	GatewayMinioSysTmp  = "minio.sys.tmp/"
+	AzureBackendGateway = "azure"
+	GCSBackendGateway   = "gcs"
+	HDFSBackendGateway  = "hdfs"
+	NASBackendGateway   = "nas"
+	S3BackendGateway    = "s3"
+)
 
 // Gateway represents a gateway backend.
 type Gateway interface {

cmd/gateway-main.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/cli"
-	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/certs"
@@ -172,6 +171,23 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Handle common command args.
 	handleCommonCmdArgs(ctx)
 
+	// Check and load TLS certificates.
+	var err error
+	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
+	logger.FatalIf(err, "Invalid TLS certificate file")
+
+	// Check and load Root CAs.
+	globalRootCAs, err = certs.GetRootCAs(globalCertsCADir.Get())
+	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
+
+	// Add the global public crts as part of global root CAs
+	for _, publicCrt := range globalPublicCerts {
+		globalRootCAs.AddCert(publicCrt)
+	}
+
+	// Register root CAs for remote ENVs
+	env.RegisterGlobalCAs(globalRootCAs)
+
 	// Initialize all help
 	initHelp()
 
@@ -184,15 +200,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// To avoid this error situation we check for port availability.
 	logger.FatalIf(checkPortAvailability(globalMinioHost, globalMinioPort), "Unable to start the gateway")
 
-	// Check and load TLS certificates.
-	var err error
-	globalPublicCerts, globalTLSCerts, globalIsSSL, err = getTLSConfig()
-	logger.FatalIf(err, "Invalid TLS certificate file")
-
-	// Check and load Root CAs.
-	globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
-	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
-
 	globalMinioEndpoint = func() string {
 		host := globalMinioHost
 		if host == "" {
@@ -255,8 +262,8 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	}
 
 	// Currently only NAS and S3 gateway support encryption headers.
-	encryptionEnabled := gatewayName == "s3" || gatewayName == "nas"
-	allowSSEKMS := gatewayName == "s3" // Only S3 can support SSE-KMS (as pass-through)
+	encryptionEnabled := gatewayName == S3BackendGateway || gatewayName == NASBackendGateway
+	allowSSEKMS := gatewayName == S3BackendGateway // Only S3 can support SSE-KMS (as pass-through)
 
 	// Add API router.
 	registerAPIRouter(router, encryptionEnabled, allowSSEKMS)
@@ -303,12 +310,12 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Calls all New() for all sub-systems.
 	newAllSubsystems()
 
-	if gatewayName == "nas" {
+	if gatewayName == NASBackendGateway {
 		buckets, err := newObject.ListBuckets(GlobalContext)
 		if err != nil {
 			logger.Fatal(err, "Unable to list buckets")
 		}
-		logger.FatalIf(globalNotificationSys.Init(buckets, newObject), "Unable to initialize notification system")
+		logger.FatalIf(globalNotificationSys.Init(GlobalContext, buckets, newObject), "Unable to initialize notification system")
 	}
 
 	if globalEtcdClient != nil {

cmd/gateway/azure/gateway-azure.go

@@ -73,7 +73,6 @@ const (
 	azureBlockSize             = 100 * humanize.MiByte
 	azureS3MinPartSize         = 5 * humanize.MiByte
 	metadataObjectNameTemplate = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x/azure.json"
-	azureBackend               = "azure"
 	azureMarkerPrefix          = "{minio}"
 	metadataPartNamePrefix     = minio.GatewayMinioSysTmp + "multipart/v1/%s.%x"
 	maxPartsCount              = 10000
@@ -113,7 +112,7 @@ EXAMPLES:
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
-		Name:               azureBackend,
+		Name:               minio.AzureBackendGateway,
 		Usage:              "Microsoft Azure Blob Storage",
 		Action:             azureGatewayMain,
 		CustomHelpTemplate: azureGatewayTemplate,
@@ -170,7 +169,7 @@ type Azure struct {
 
 // Name implements Gateway interface.
 func (g *Azure) Name() string {
-	return azureBackend
+	return minio.AzureBackendGateway
 }
 
 // NewGatewayLayer initializes azure blob storage client and returns AzureObjects.
@@ -530,7 +529,7 @@ func checkAzureUploadID(ctx context.Context, uploadID string) (err error) {
 func parseAzurePart(metaPartFileName, prefix string) (partID int, err error) {
 	partStr := strings.TrimPrefix(metaPartFileName, prefix+minio.SlashSeparator)
 	if partID, err = strconv.Atoi(partStr); err != nil || partID <= 0 {
-		err = fmt.Errorf("invalid part number in block id '%s'", string(partID))
+		err = fmt.Errorf("invalid part number in block id '%d'", partID)
 		return
 	}
 	return

cmd/gateway/gcs/gateway-gcs.go

@@ -91,8 +91,6 @@ const (
 
 	// Project ID key in credentials.json
 	gcsProjectIDKey = "project_id"
-
-	gcsBackend = "gcs"
 )
 
 func init() {
@@ -132,7 +130,7 @@ EXAMPLES:
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
-		Name:               gcsBackend,
+		Name:               minio.GCSBackendGateway,
 		Usage:              "Google Cloud Storage",
 		Action:             gcsGatewayMain,
 		CustomHelpTemplate: gcsGatewayTemplate,
@@ -145,13 +143,13 @@ func gcsGatewayMain(ctx *cli.Context) {
 	projectID := ctx.Args().First()
 	if projectID == "" && os.Getenv("GOOGLE_APPLICATION_CREDENTIALS") == "" {
 		logger.LogIf(minio.GlobalContext, errGCSProjectIDNotFound, logger.Application)
-		cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
+		cli.ShowCommandHelpAndExit(ctx, minio.GCSBackendGateway, 1)
 	}
 	if projectID != "" && !isValidGCSProjectIDFormat(projectID) {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("projectID", ctx.Args().First())
 		contxt := logger.SetReqInfo(minio.GlobalContext, reqInfo)
 		logger.LogIf(contxt, errGCSInvalidProjectID, logger.Application)
-		cli.ShowCommandHelpAndExit(ctx, "gcs", 1)
+		cli.ShowCommandHelpAndExit(ctx, minio.GCSBackendGateway, 1)
 	}
 
 	minio.StartGateway(ctx, &GCS{projectID})
@@ -164,7 +162,7 @@ type GCS struct {
 
 // Name returns the name of gcs ObjectLayer.
 func (g *GCS) Name() string {
-	return gcsBackend
+	return minio.GCSBackendGateway
 }
 
 // NewGatewayLayer returns gcs ObjectLayer.

cmd/gateway/gcs/gateway-gcs_test.go

@@ -478,7 +478,7 @@ func TestGCSAttrsToObjectInfo(t *testing.T) {
 	if objInfo.Bucket != attrs.Bucket {
 		t.Fatalf("Test failed with Bucket mistmatch, expected %s, got %s", attrs.Bucket, objInfo.Bucket)
 	}
-	if objInfo.ModTime != attrs.Updated {
+	if !objInfo.ModTime.Equal(attrs.Updated) {
 		t.Fatalf("Test failed with ModTime mistmatch, expected %s, got %s", attrs.Updated, objInfo.ModTime)
 	}
 	if objInfo.Size != attrs.Size {

cmd/gateway/hdfs/gateway-hdfs.go

@@ -47,8 +47,6 @@ import (
 )
 
 const (
-	hdfsBackend = "hdfs"
-
 	hdfsSeparator = minio.SlashSeparator
 )
 
@@ -84,7 +82,7 @@ EXAMPLES:
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
-		Name:               hdfsBackend,
+		Name:               minio.HDFSBackendGateway,
 		Usage:              "Hadoop Distributed File System (HDFS)",
 		Action:             hdfsGatewayMain,
 		CustomHelpTemplate: hdfsGatewayTemplate,
@@ -96,7 +94,7 @@ EXAMPLES:
 func hdfsGatewayMain(ctx *cli.Context) {
 	// Validate gateway arguments.
 	if ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, hdfsBackend, 1)
+		cli.ShowCommandHelpAndExit(ctx, minio.HDFSBackendGateway, 1)
 	}
 
 	minio.StartGateway(ctx, &HDFS{args: ctx.Args()})
@@ -109,7 +107,7 @@ type HDFS struct {
 
 // Name implements Gateway interface.
 func (g *HDFS) Name() string {
-	return hdfsBackend
+	return minio.HDFSBackendGateway
 }
 
 func getKerberosClient() (*krb.Client, error) {
@@ -348,9 +346,17 @@ func (n *hdfsObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketIn
 	return buckets, nil
 }
 
+func (n *hdfsObjects) isLeafDir(bucket, leafPath string) bool {
+	return n.isObjectDir(context.Background(), bucket, leafPath)
+}
+
+func (n *hdfsObjects) isLeaf(bucket, leafPath string) bool {
+	return !strings.HasSuffix(leafPath, hdfsSeparator)
+}
+
 func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 	// listDir - lists all the entries at a given prefix and given entry in the prefix.
-	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string) {
+	listDir := func(bucket, prefixDir, prefixEntry string) (emptyDir bool, entries []string, delayIsLeaf bool) {
 		f, err := n.clnt.Open(n.hdfsPathJoin(bucket, prefixDir))
 		if err != nil {
 			if os.IsNotExist(err) {
@@ -366,7 +372,7 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 			return
 		}
 		if len(fis) == 0 {
-			return true, nil
+			return true, nil, false
 		}
 		for _, fi := range fis {
 			if fi.IsDir() {
@@ -375,7 +381,8 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 				entries = append(entries, fi.Name())
 			}
 		}
-		return false, minio.FilterMatchingPrefix(entries, prefixEntry)
+		entries, delayIsLeaf = minio.FilterListEntries(bucket, prefixDir, entries, prefixEntry, n.isLeaf)
+		return false, entries, delayIsLeaf
 	}
 
 	// Return list factory instance.
@@ -385,12 +392,27 @@ func (n *hdfsObjects) listDirFactory() minio.ListDirFunc {
 // ListObjects lists all blobs in HDFS bucket filtered by prefix.
 func (n *hdfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, err error) {
 	fileInfos := make(map[string]os.FileInfo)
-	directoryPath := n.hdfsPathJoin(bucket, prefix)
+	targetPath := n.hdfsPathJoin(bucket, prefix)
 
-	if err = n.populateDirectoryListing(directoryPath, fileInfos); err != nil {
+	var targetFileInfo os.FileInfo
+
+	if targetFileInfo, err = n.populateDirectoryListing(targetPath, fileInfos); err != nil {
 		return loi, hdfsToObjectErr(ctx, err, bucket)
 	}
 
+	// If the user is trying to list a single file, bypass the entire directory-walking code below
+	// and just return the single file's information.
+	if !targetFileInfo.IsDir() {
+		return minio.ListObjectsInfo{
+			IsTruncated: false,
+			NextMarker:  "",
+			Objects: []minio.ObjectInfo{
+				fileInfoToObjectInfo(bucket, prefix, targetFileInfo),
+			},
+			Prefixes: []string{},
+		}, nil
+	}
+
 	getObjectInfo := func(ctx context.Context, bucket, entry string) (minio.ObjectInfo, error) {
 		filePath := path.Clean(n.hdfsPathJoin(bucket, entry))
 		fi, ok := fileInfos[filePath]
@@ -400,7 +422,7 @@ func (n *hdfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, d
 		if !ok {
 			parentPath := path.Dir(filePath)
 
-			if err := n.populateDirectoryListing(parentPath, fileInfos); err != nil {
+			if _, err := n.populateDirectoryListing(parentPath, fileInfos); err != nil {
 				return minio.ObjectInfo{}, hdfsToObjectErr(ctx, err, bucket)
 			}
 
@@ -412,40 +434,48 @@ func (n *hdfsObjects) ListObjects(ctx context.Context, bucket, prefix, marker, d
 			}
 		}
 
-		objectInfo := minio.ObjectInfo{
-			Bucket:  bucket,
-			Name:    entry,
-			ModTime: fi.ModTime(),
-			Size:    fi.Size(),
-			IsDir:   fi.IsDir(),
-			AccTime: fi.(*hdfs.FileInfo).AccessTime(),
-		}
+		objectInfo := fileInfoToObjectInfo(bucket, entry, fi)
 
 		delete(fileInfos, filePath)
 
 		return objectInfo, nil
 	}
 
-	return minio.ListObjects(ctx, n, bucket, prefix, marker, delimiter, maxKeys, n.listPool, n.listDirFactory(), getObjectInfo, getObjectInfo)
+	return minio.ListObjects(ctx, n, bucket, prefix, marker, delimiter, maxKeys, n.listPool, n.listDirFactory(), n.isLeaf, n.isLeafDir, getObjectInfo, getObjectInfo)
+}
+
+func fileInfoToObjectInfo(bucket string, entry string, fi os.FileInfo) minio.ObjectInfo {
+	return minio.ObjectInfo{
+		Bucket:  bucket,
+		Name:    entry,
+		ModTime: fi.ModTime(),
+		Size:    fi.Size(),
+		IsDir:   fi.IsDir(),
+		AccTime: fi.(*hdfs.FileInfo).AccessTime(),
+	}
 }
 
 // Lists a path's direct, first-level entries and populates them in the `fileInfos` cache which maps
 // a path entry to an `os.FileInfo`. It also saves the listed path's `os.FileInfo` in the cache.
-func (n *hdfsObjects) populateDirectoryListing(filePath string, fileInfos map[string]os.FileInfo) error {
+func (n *hdfsObjects) populateDirectoryListing(filePath string, fileInfos map[string]os.FileInfo) (os.FileInfo, error) {
 	dirReader, err := n.clnt.Open(filePath)
 
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	dirStat := dirReader.Stat()
 	key := path.Clean(filePath)
 
+	if !dirStat.IsDir() {
+		return dirStat, nil
+	}
+
 	fileInfos[key] = dirStat
 	infos, err := dirReader.Readdir(0)
 
 	if err != nil {
-		return err
+		return nil, err
 	}
 
 	for _, fileInfo := range infos {
@@ -453,7 +483,7 @@ func (n *hdfsObjects) populateDirectoryListing(filePath string, fileInfos map[st
 		fileInfos[filePath] = fileInfo
 	}
 
-	return nil
+	return dirStat, nil
 }
 
 // deleteObject deletes a file path if its empty. If it's successfully deleted,

cmd/gateway/nas/gateway-nas.go

@@ -24,10 +24,6 @@ import (
 	"github.com/minio/minio/pkg/auth"
 )
 
-const (
-	nasBackend = "nas"
-)
-
 func init() {
 	const nasGatewayTemplate = `NAME:
   {{.HelpName}} - {{.Usage}}
@@ -61,7 +57,7 @@ EXAMPLES:
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
-		Name:               nasBackend,
+		Name:               minio.NASBackendGateway,
 		Usage:              "Network-attached storage (NAS)",
 		Action:             nasGatewayMain,
 		CustomHelpTemplate: nasGatewayTemplate,
@@ -73,7 +69,7 @@ EXAMPLES:
 func nasGatewayMain(ctx *cli.Context) {
 	// Validate gateway arguments.
 	if !ctx.Args().Present() || ctx.Args().First() == "help" {
-		cli.ShowCommandHelpAndExit(ctx, nasBackend, 1)
+		cli.ShowCommandHelpAndExit(ctx, minio.NASBackendGateway, 1)
 	}
 
 	minio.StartGateway(ctx, &NAS{ctx.Args().First()})
@@ -86,7 +82,7 @@ type NAS struct {
 
 // Name implements Gateway interface.
 func (g *NAS) Name() string {
-	return nasBackend
+	return minio.NASBackendGateway
 }
 
 // NewGatewayLayer returns nas gatewaylayer.

cmd/gateway/s3/gateway-s3.go

@@ -40,10 +40,6 @@ import (
 	"github.com/minio/minio/pkg/bucket/policy"
 )
 
-const (
-	s3Backend = "s3"
-)
-
 func init() {
 	const s3GatewayTemplate = `NAME:
   {{.HelpName}} - {{.Usage}}
@@ -76,7 +72,7 @@ EXAMPLES:
 `
 
 	minio.RegisterGatewayCommand(cli.Command{
-		Name:               s3Backend,
+		Name:               minio.S3BackendGateway,
 		Usage:              "Amazon Simple Storage Service (S3)",
 		Action:             s3GatewayMain,
 		CustomHelpTemplate: s3GatewayTemplate,
@@ -109,7 +105,7 @@ type S3 struct {
 
 // Name implements Gateway interface.
 func (g *S3) Name() string {
-	return s3Backend
+	return minio.S3BackendGateway
 }
 
 const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569"

cmd/generic-handlers_test.go

@@ -155,12 +155,12 @@ var isHTTPHeaderSizeTooLargeTests = []struct {
 func generateHeader(size, usersize int) http.Header {
 	header := http.Header{}
 	for i := 0; i < size; i++ {
-		header.Add(strconv.Itoa(i), "")
+		header.Set(strconv.Itoa(i), "")
 	}
 	userlength := 0
 	for i := 0; userlength < usersize; i++ {
 		userlength += len(userMetadataKeyPrefixes[0] + strconv.Itoa(i))
-		header.Add(userMetadataKeyPrefixes[0]+strconv.Itoa(i), "")
+		header.Set(userMetadataKeyPrefixes[0]+strconv.Itoa(i), "")
 	}
 	return header
 }

cmd/global-heal.go

@@ -29,8 +29,6 @@ const (
 	// sleep for an hour after a lock timeout
 	// before retrying to acquire lock again.
 	leaderLockTimeoutSleepInterval = time.Hour
-	// heal entire namespace once in 30 days
-	healInterval = 30 * 24 * time.Hour
 )
 
 var leaderLockTimeout = newDynamicTimeout(30*time.Second, time.Minute)
@@ -53,7 +51,9 @@ func newBgHealSequence() *healSequence {
 		respCh:      make(chan healResult),
 		startTime:   UTCNow(),
 		clientToken: bgHealingUUID,
-		settings:    hs,
+		// run-background heal with reserved bucket
+		bucket:   minioReservedBucket,
+		settings: hs,
 		currentStatus: healSequenceStatus{
 			Summary:      healNotStartedStatus,
 			HealSettings: hs,
@@ -67,21 +67,29 @@ func newBgHealSequence() *healSequence {
 	}
 }
 
-func getLocalBackgroundHealStatus() madmin.BgHealState {
+func getLocalBackgroundHealStatus() (madmin.BgHealState, bool) {
 	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
 	if !ok {
-		return madmin.BgHealState{}
+		return madmin.BgHealState{}, false
+	}
+
+	var healDisks []string
+	for _, eps := range globalBackgroundHealState.getHealLocalDisks() {
+		for _, ep := range eps {
+			healDisks = append(healDisks, ep.String())
+		}
 	}
 
 	return madmin.BgHealState{
 		ScannedItemsCount: bgSeq.getScannedItemsCount(),
 		LastHealActivity:  bgSeq.lastHealActivity,
-		NextHealRound:     UTCNow().Add(durationToNextHealRound(bgSeq.lastHealActivity)),
-	}
+		HealDisks:         healDisks,
+		NextHealRound:     UTCNow(),
+	}, true
 }
 
 // healErasureSet lists and heals all objects in a specific erasure set
-func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, drivesPerSet int) error {
+func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, setDriveCount int) error {
 	buckets, err := xlObj.ListBuckets(ctx)
 	if err != nil {
 		return err
@@ -143,7 +151,7 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, dr
 				break
 			}
 
-			if quorumCount == drivesPerSet {
+			if quorumCount == setDriveCount {
 				// Skip good entries.
 				continue
 			}
@@ -164,77 +172,13 @@ func healErasureSet(ctx context.Context, setIndex int, xlObj *erasureObjects, dr
 // deepHealObject heals given object path in deep to fix bitrot.
 func deepHealObject(bucket, object, versionID string) {
 	// Get background heal sequence to send elements to heal
-	bgSeq, _ := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
-
-	bgSeq.sourceCh <- healSource{
-		bucket:    bucket,
-		object:    object,
-		versionID: versionID,
-		opts:      &madmin.HealOpts{ScanMode: madmin.HealDeepScan},
-	}
-}
-
-// Returns the duration to the next background healing round
-func durationToNextHealRound(lastHeal time.Time) time.Duration {
-	if lastHeal.IsZero() {
-		lastHeal = globalBootTime
-	}
-
-	d := lastHeal.Add(healInterval).Sub(UTCNow())
-	if d < 0 {
-		return time.Second
-	}
-	return d
-}
-
-// Healing leader will take the charge of healing all erasure sets
-func execLeaderTasks(ctx context.Context, z *erasureZones) {
-	// So that we don't heal immediately, but after one month.
-	lastScanTime := UTCNow()
-	// Get background heal sequence to send elements to heal
-	var bgSeq *healSequence
-	var ok bool
-	for {
-		bgSeq, ok = globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
-		if ok {
-			break
-		}
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.After(time.Second):
-			continue
-		}
-	}
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-time.NewTimer(durationToNextHealRound(lastScanTime)).C:
-			bgSeq.resetHealStatusCounters()
-			for _, zone := range z.zones {
-				// Heal set by set
-				for i, set := range zone.sets {
-					if err := healErasureSet(ctx, i, set, zone.drivesPerSet); err != nil {
-						logger.LogIf(ctx, err)
-						continue
-					}
-				}
-			}
-			lastScanTime = UTCNow()
+	bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
+	if ok {
+		bgSeq.sourceCh <- healSource{
+			bucket:    bucket,
+			object:    object,
+			versionID: versionID,
+			opts:      &madmin.HealOpts{ScanMode: madmin.HealDeepScan},
 		}
 	}
 }
-
-func startGlobalHeal(ctx context.Context, objAPI ObjectLayer) {
-	zones, ok := objAPI.(*erasureZones)
-	if !ok {
-		return
-	}
-
-	execLeaderTasks(ctx, zones)
-}
-
-func initGlobalHeal(ctx context.Context, objAPI ObjectLayer) {
-	go startGlobalHeal(ctx, objAPI)
-}

cmd/globals.go

@@ -208,9 +208,7 @@ var (
 	globalDomainNames []string      // Root domains for virtual host style requests
 	globalDomainIPs   set.StringSet // Root domain IP address(s) for a distributed MinIO deployment
 
-	globalObjectTimeout    = newDynamicTimeout( /*1*/ 10*time.Minute /*10*/, 600*time.Second) // timeout for Object API related ops
-	globalOperationTimeout = newDynamicTimeout(10*time.Minute /*30*/, 600*time.Second)        // default timeout for general ops
-	globalHealingTimeout   = newDynamicTimeout(30*time.Minute /*1*/, 30*time.Minute)          // timeout for healing related ops
+	globalOperationTimeout = newDynamicTimeout(10*time.Minute, 5*time.Minute) // default timeout for general ops
 
 	globalBucketObjectLockSys *BucketObjectLockSys
 	globalBucketQuotaSys      *BucketQuotaSys

cmd/handler-utils.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -131,6 +132,13 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]
 		metadata[strings.ToLower(xhttp.ContentType)] = "application/octet-stream"
 	}
 
+	// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+	for k := range metadata {
+		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+			delete(metadata, k)
+		}
+	}
+
 	if contentEncoding, ok := metadata[strings.ToLower(xhttp.ContentEncoding)]; ok {
 		contentEncoding = trimAwsChunkedContentEncoding(contentEncoding)
 		if contentEncoding != "" {
@@ -480,15 +488,20 @@ func getHostName(r *http.Request) (hostName string) {
 func proxyRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, ep ProxyEndpoint) (success bool) {
 	success = true
 
+	// Make sure we remove any existing headers before
+	// proxying the request to another node.
+	for k := range w.Header() {
+		w.Header().Del(k)
+	}
+
 	f := handlers.NewForwarder(&handlers.Forwarder{
 		PassHost:     true,
 		RoundTripper: ep.Transport,
 		ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
 			success = false
-			w.WriteHeader(http.StatusBadGateway)
-		},
-		Logger: func(err error) {
-			logger.LogIf(GlobalContext, err)
+			if err != nil && !errors.Is(err, context.Canceled) {
+				logger.LogIf(GlobalContext, err)
+			}
 		},
 	})
 

cmd/healthcheck-handler.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"net/http"
+	"strconv"
 )
 
 // ClusterCheckHandler returns if the server is ready for requests.
@@ -38,6 +39,8 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 	opts := HealthOptions{Maintenance: r.URL.Query().Get("maintenance") == "true"}
 	result := objLayer.Health(ctx, opts)
 	if !result.Healthy {
+		// return how many drives are being healed if any
+		w.Header().Set("X-Minio-Healing-Drives", strconv.Itoa(result.HealingDrives))
 		// As a maintenance call we are purposefully asked to be taken
 		// down, this is for orchestrators to know if we can safely
 		// take this server down, return appropriate error.

cmd/http-tracer.go

@@ -104,9 +104,8 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 	reqHeaders.Set("Host", r.Host)
 	if len(r.TransferEncoding) == 0 {
 		reqHeaders.Set("Content-Length", strconv.Itoa(int(r.ContentLength)))
-	}
-	for _, enc := range r.TransferEncoding {
-		reqHeaders.Add("Transfer-Encoding", enc)
+	} else {
+		reqHeaders.Set("Transfer-Encoding", strings.Join(r.TransferEncoding, ","))
 	}
 
 	var reqBodyRecorder *recordRequest
@@ -129,6 +128,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 
 	rq := trace.RequestInfo{
 		Time:     time.Now().UTC(),
+		Proto:    r.Proto,
 		Method:   r.Method,
 		Path:     r.URL.Path,
 		RawQuery: r.URL.RawQuery,

cmd/http/dial_linux.go

@@ -36,22 +36,22 @@ func setInternalTCPParameters(c syscall.RawConn) error {
 		// TCPFastOpenConnect sets the underlying socket to use
 		// the TCP fast open connect. This feature is supported
 		// since Linux 4.11.
-		_ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_FASTOPEN_CONNECT, 1)
+		_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_FASTOPEN_CONNECT, 1)
 
 		// The time (in seconds) the connection needs to remain idle before
 		// TCP starts sending keepalive probes, set this to 5 secs
 		// system defaults to 7200 secs!!!
-		_ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, 5)
+		_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, 5)
 
 		// Number of probes.
 		// ~ cat /proc/sys/net/ipv4/tcp_keepalive_probes (defaults to 9, we reduce it to 5)
 		// 9
-		_ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, 5)
+		_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, 5)
 
 		// Wait time after successful probe in seconds.
 		// ~ cat /proc/sys/net/ipv4/tcp_keepalive_intvl (defaults to 75 secs, we reduce it to 3 secs)
 		// 75
-		_ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 3)
+		_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 3)
 	})
 }
 
@@ -85,7 +85,7 @@ func NewCustomDialContext(dialTimeout time.Duration) DialContext {
 					// TCPFastOpenConnect sets the underlying socket to use
 					// the TCP fast open connect. This feature is supported
 					// since Linux 4.11.
-					_ = syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_FASTOPEN_CONNECT, 1)
+					_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_FASTOPEN_CONNECT, 1)
 				})
 			},
 		}

cmd/http/headers.go

@@ -102,6 +102,9 @@ const (
 	AmzSecurityToken        = "X-Amz-Security-Token"
 	AmzDecodedContentLength = "X-Amz-Decoded-Content-Length"
 
+	AmzMetaUnencryptedContentLength = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length"
+	AmzMetaUnencryptedContentMD5    = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5"
+
 	// Signature v2 related constants
 	AmzSignatureV2 = "Signature"
 	AmzAccessKeyID = "AWSAccessKeyId"
@@ -120,6 +123,9 @@ const (
 
 	// Header indicates if the mtime should be preserved by client
 	MinIOSourceMTime = "x-minio-source-mtime"
+
+	// Header indicates if the etag should be preserved by client
+	MinIOSourceETag = "x-minio-source-etag"
 )
 
 // Common http query params S3 API

cmd/http/listener_test.go

@@ -186,6 +186,10 @@ func TestHTTPListenerStartClose(t *testing.T) {
 			testCase.serverAddrs,
 		)
 		if err != nil {
+			if strings.Contains(err.Error(), "The requested address is not valid in its context") {
+				// Ignore if IP is unbindable.
+				continue
+			}
 			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
 		}
 
@@ -225,6 +229,10 @@ func TestHTTPListenerAddr(t *testing.T) {
 			testCase.serverAddrs,
 		)
 		if err != nil {
+			if strings.Contains(err.Error(), "The requested address is not valid in its context") {
+				// Ignore if IP is unbindable.
+				continue
+			}
 			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
 		}
 
@@ -261,6 +269,10 @@ func TestHTTPListenerAddrs(t *testing.T) {
 			testCase.serverAddrs,
 		)
 		if err != nil {
+			if strings.Contains(err.Error(), "The requested address is not valid in its context") {
+				// Ignore if IP is unbindable.
+				continue
+			}
 			t.Fatalf("Test %d: error: expected = <nil>, got = %v", i+1, err)
 		}
 

cmd/http/server.go

@@ -191,14 +191,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 			// TLS hardening
 			PreferServerCipherSuites: true,
 			MinVersion:               tls.VersionTLS12,
-			// Do not edit the next line, protos priority is kept
-			// on purpose in this manner for HTTP 2.0, we would
-			// still like HTTP 2.0 clients to negotiate connection
-			// to server if needed but by default HTTP 1.1 is
-			// expected. We need to change this in future
-			// when we wish to go back to HTTP 2.0 as default
-			// priority for HTTP protocol negotiation.
-			NextProtos: []string{"http/1.1", "h2"},
+			NextProtos:               []string{"h2", "http/1.1"},
 		}
 		tlsConfig.GetCertificate = getCert
 	}

cmd/iam-object-store.go

@@ -151,7 +151,7 @@ func (iamOS *IAMObjectStore) migrateUsersConfigToV1(ctx context.Context, isSTS b
 		cred.AccessKey = user
 		u := newUserIdentity(cred)
 		if err := iamOS.saveIAMConfig(u, identityPath); err != nil {
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(ctx, err)
 			return err
 		}
 
@@ -216,7 +216,7 @@ func (iamOS *IAMObjectStore) saveIAMConfig(item interface{}, path string) error
 			return err
 		}
 	}
-	return saveConfig(context.Background(), iamOS.objAPI, path, data)
+	return saveConfig(GlobalContext, iamOS.objAPI, path, data)
 }
 
 func (iamOS *IAMObjectStore) loadIAMConfig(item interface{}, path string) error {
@@ -586,7 +586,7 @@ func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix stri
 
 		marker := ""
 		for {
-			lo, err := objAPI.ListObjects(context.Background(),
+			lo, err := objAPI.ListObjects(ctx,
 				minioMetaBucket, pathPrefix, marker, SlashSeparator, maxObjectList)
 			if err != nil {
 				select {

cmd/iam.go

@@ -449,10 +449,13 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	rquorum := InsufficientReadQuorum{}
 	wquorum := InsufficientWriteQuorum{}
 
+	// allocate dynamic timeout once before the loop
+	iamLockTimeout := newDynamicTimeout(3*time.Second, 5*time.Second)
+
 	for range retry.NewTimerWithJitter(retryCtx, time.Second, 5*time.Second, retry.MaxJitter) {
 		// let one of the server acquire the lock, if not let them timeout.
 		// which shall be retried again by this loop.
-		if err := txnLk.GetLock(newDynamicTimeout(3*time.Second, 5*time.Second)); err != nil {
+		if err := txnLk.GetLock(iamLockTimeout); err != nil {
 			logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. trying to acquire lock")
 			continue
 		}
@@ -1640,7 +1643,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) b
 	subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
 	if err != nil {
 		// Log any error in input session policy config.
-		logger.LogIf(context.Background(), err)
+		logger.LogIf(GlobalContext, err)
 		return false
 	}
 
@@ -1783,7 +1786,7 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args) bool {
 		subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
 		if err != nil {
 			// Log any error in input session policy config.
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(GlobalContext, err)
 			return false
 		}
 
@@ -1807,7 +1810,7 @@ func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
 	if globalPolicyOPA != nil {
 		ok, err := globalPolicyOPA.IsAllowed(args)
 		if err != nil {
-			logger.LogIf(context.Background(), err)
+			logger.LogIf(GlobalContext, err)
 		}
 		return ok
 	}

cmd/listen-notification-handlers.go

@@ -153,8 +153,8 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 	for {
 		select {
 		case evI := <-listenCh:
-			ev := evI.(event.Event)
-			if len(string(ev.EventName)) > 0 {
+			ev, ok := evI.(event.Event)
+			if ok {
 				if err := enc.Encode(struct{ Records []event.Event }{[]event.Event{ev}}); err != nil {
 					return
 				}

cmd/lock-rest-client.go

@@ -149,7 +149,6 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
 		tlsConfig = &tls.Config{
 			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
-			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 

cmd/lock-rest-server.go

@@ -355,7 +355,7 @@ func registerLockRESTHandlers(router *mux.Router, endpointZones EndpointZones) {
 			}
 
 			subrouter := router.PathPrefix(path.Join(lockRESTPrefix, endpoint.Path)).Subrouter()
-			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodHealth).HandlerFunc(httpTraceHdrs(lockServer.HealthHandler)).Queries(queries...)
+			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodHealth).HandlerFunc(httpTraceHdrs(lockServer.HealthHandler))
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(lockServer.LockHandler)).Queries(queries...)
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(lockServer.RLockHandler)).Queries(queries...)
 			subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler)).Queries(queries...)

cmd/logger/audit.go

@@ -125,15 +125,6 @@ func (lrw *ResponseWriter) Size() int {
 	return lrw.bytesWritten
 }
 
-// AuditTargets is the list of enabled audit loggers
-var AuditTargets = []Target{}
-
-// AddAuditTarget adds a new audit logger target to the
-// list of enabled loggers
-func AddAuditTarget(t Target) {
-	AuditTargets = append(AuditTargets, t)
-}
-
 // AuditLog - logs audit logs to all audit targets.
 func AuditLog(w http.ResponseWriter, r *http.Request, api string, reqClaims map[string]interface{}, filterKeys ...string) {
 	// Fast exit if there is not audit target configured

cmd/logger/target/console/console.go

@@ -32,6 +32,11 @@ import (
 // in plain or json format to the standard output.
 type Target struct{}
 
+// Validate - validate if the tty can be written to
+func (c *Target) Validate() error {
+	return nil
+}
+
 // Send log message 'e' to console
 func (c *Target) Send(e interface{}, logKind string) error {
 	entry, ok := e.(log.Entry)

cmd/logger/target/http/http.go

@@ -18,12 +18,16 @@ package http
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"strings"
+	"time"
 
 	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
 )
 
 // Target implements logger.Target and sends the json
@@ -45,6 +49,47 @@ type Target struct {
 	client    http.Client
 }
 
+// Validate validate the http target
+func (h *Target) Validate() error {
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, h.endpoint, strings.NewReader(`{}`))
+	if err != nil {
+		return err
+	}
+
+	req.Header.Set(xhttp.ContentType, "application/json")
+
+	// Set user-agent to indicate MinIO release
+	// version to the configured log endpoint
+	req.Header.Set("User-Agent", h.userAgent)
+
+	if h.authToken != "" {
+		req.Header.Set("Authorization", h.authToken)
+	}
+
+	resp, err := h.client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	// Drain any response.
+	xhttp.DrainBody(resp.Body)
+
+	if resp.StatusCode != http.StatusOK {
+		switch resp.StatusCode {
+		case http.StatusForbidden:
+			return fmt.Errorf("%s returned '%s', please check if your auth token is correctly set",
+				h.endpoint, resp.Status)
+		}
+		return fmt.Errorf("%s returned '%s', please check your endpoint configuration",
+			h.endpoint, resp.Status)
+	}
+
+	return nil
+}
+
 func (h *Target) startHTTPLogger() {
 	// Create a routine which sends json logs received
 	// from an internal channel.
@@ -55,8 +100,11 @@ func (h *Target) startHTTPLogger() {
 				continue
 			}
 
-			req, err := http.NewRequest(http.MethodPost, h.endpoint, bytes.NewReader(logJSON))
+			ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+			req, err := http.NewRequestWithContext(ctx, http.MethodPost,
+				h.endpoint, bytes.NewReader(logJSON))
 			if err != nil {
+				cancel()
 				continue
 			}
 			req.Header.Set(xhttp.ContentType, "application/json")
@@ -70,13 +118,26 @@ func (h *Target) startHTTPLogger() {
 			}
 
 			resp, err := h.client.Do(req)
+			cancel()
 			if err != nil {
-				h.client.CloseIdleConnections()
+				logger.LogIf(ctx, fmt.Errorf("%s returned '%w', please check your endpoint configuration\n",
+					h.endpoint, err))
 				continue
 			}
 
 			// Drain any response.
 			xhttp.DrainBody(resp.Body)
+
+			if resp.StatusCode != http.StatusOK {
+				switch resp.StatusCode {
+				case http.StatusForbidden:
+					logger.LogIf(ctx, fmt.Errorf("%s returned '%s', please check if your auth token is correctly set",
+						h.endpoint, resp.Status))
+				default:
+					logger.LogIf(ctx, fmt.Errorf("%s returned '%s', please check your endpoint configuration",
+						h.endpoint, resp.Status))
+				}
+			}
 		}
 	}()
 }

cmd/logger/targets.go

@@ -20,14 +20,33 @@ package logger
 // a single log entry and Send it to the log target
 //   e.g. Send the log to a http server
 type Target interface {
+	Validate() error
 	Send(entry interface{}, errKind string) error
 }
 
 // Targets is the set of enabled loggers
 var Targets = []Target{}
 
+// AuditTargets is the list of enabled audit loggers
+var AuditTargets = []Target{}
+
+// AddAuditTarget adds a new audit logger target to the
+// list of enabled loggers
+func AddAuditTarget(t Target) error {
+	if err := t.Validate(); err != nil {
+		return err
+	}
+
+	AuditTargets = append(AuditTargets, t)
+	return nil
+}
+
 // AddTarget adds a new logger target to the
 // list of enabled loggers
-func AddTarget(t Target) {
+func AddTarget(t Target) error {
+	if err := t.Validate(); err != nil {
+		return err
+	}
 	Targets = append(Targets, t)
+	return nil
 }