pass the necessary query params for remote NSSCanner (#14719)

fixes a regression from #14464

.github/workflows/go-cross.yml

@@ -11,6 +11,9 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.head_ref }}
   cancel-in-progress: true
 
+permissions:
+  contents: read
+
 jobs:
   build:
     name: Build Tests with Go ${{ matrix.go-version }} on ${{ matrix.os }}
@@ -20,10 +23,11 @@ jobs:
         go-version: [1.17.x]
         os: [ubuntu-latest]
     steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
+      - uses: actions/setup-go@bfdd3570ce990073878bf10f6b2d79082de49492 # v2
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - name: Build on ${{ matrix.os }}
         if: matrix.os == 'ubuntu-latest'
         env:

.github/workflows/go-healing.yml

@@ -21,9 +21,10 @@ jobs:
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - uses: actions/cache@v2
         with:
           path: |
@@ -37,10 +38,7 @@ jobs:
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
-          MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
-          MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
-          MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
-          MINIO_KMS_KES_KEY_NAME: "my-minio-key"
+          MINIO_KMS_SECRET_KEY: "my-minio-key:oyArl7zlPECEduNbB1KXgdzDn2Bdpvvw0l8VO51HQnY="
           MINIO_KMS_AUTO_ENCRYPTION: on
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0

.github/workflows/go-lint.yml

@@ -21,9 +21,10 @@ jobs:
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - uses: actions/cache@v2
         if: matrix.os == 'ubuntu-latest'
         with:

.github/workflows/go.yml

@@ -21,9 +21,10 @@ jobs:
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - uses: actions/cache@v2
         with:
           path: |

.github/workflows/iam-integrations.yaml

@@ -65,9 +65,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - uses: actions/cache@v2
         with:
           path: |

.github/workflows/replication.yaml

@@ -22,9 +22,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
+          check-latest: true
       - uses: actions/cache@v2
         with:
           path: |

.github/workflows/upgrade-ci-cd.yaml

@@ -22,10 +22,10 @@ jobs:
 
     steps:
       - uses: actions/checkout@v1
-      - uses: actions/setup-go@v2
+      - uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go-version }}
-
+          check-latest: true
       - name: Start upgrade tests
         run: |
           make test-upgrade

cmd/admin-handlers-pools.go

@@ -58,6 +58,16 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
+	if ep := globalEndpoints[idx].Endpoints[0]; !ep.IsLocal {
+		for nodeIdx, proxyEp := range globalProxyEndpoints {
+			if proxyEp.Endpoint.Host == ep.Host {
+				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
+					return
+				}
+			}
+		}
+	}
+
 	if err := pools.Decommission(r.Context(), idx); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -96,6 +106,16 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	if ep := globalEndpoints[idx].Endpoints[0]; !ep.IsLocal {
+		for nodeIdx, proxyEp := range globalProxyEndpoints {
+			if proxyEp.Endpoint.Host == ep.Host {
+				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
+					return
+				}
+			}
+		}
+	}
+
 	if err := pools.DecommissionCancel(ctx, idx); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/admin-handlers.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2022 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -18,14 +18,16 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	crand "crypto/rand"
 	"crypto/subtle"
-	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"hash/crc32"
 	"io"
+	"io/ioutil"
 	"math/rand"
 	"net/http"
 	"net/url"
@@ -499,7 +501,7 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 }
 
 // StartProfilingResult contains the status of the starting
-// profiling action in a given server
+// profiling action in a given server - deprecated API
 type StartProfilingResult struct {
 	NodeName string `json:"nodeName"`
 	Success  bool   `json:"success"`
@@ -593,6 +595,85 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 	writeSuccessResponseJSON(w, startProfilingResultInBytes)
 }
 
+// ProfileHandler - POST /minio/admin/v3/profile/?profilerType={profilerType}
+// ----------
+// Enable server profiling
+func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "Profile")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	if globalNotificationSys == nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	profileStr := r.Form.Get("profilerType")
+	profiles := strings.Split(profileStr, ",")
+	duration := time.Minute
+	if dstr := r.Form.Get("duration"); dstr != "" {
+		var err error
+		duration, err = time.ParseDuration(dstr)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+			return
+		}
+	}
+	// read request body
+	io.CopyN(ioutil.Discard, r.Body, 1)
+
+	globalProfilerMu.Lock()
+
+	if globalProfiler == nil {
+		globalProfiler = make(map[string]minioProfiler, 10)
+	}
+
+	// Stop profiler of all types if already running
+	for k, v := range globalProfiler {
+		v.Stop()
+		delete(globalProfiler, k)
+	}
+
+	// Start profiling on remote servers.
+	for _, profiler := range profiles {
+		globalNotificationSys.StartProfiling(profiler)
+
+		// Start profiling locally as well.
+		prof, err := startProfiler(profiler)
+		if err == nil {
+			globalProfiler[profiler] = prof
+		}
+	}
+	globalProfilerMu.Unlock()
+
+	timer := time.NewTimer(duration)
+	for {
+		select {
+		case <-ctx.Done():
+			if !timer.Stop() {
+				<-timer.C
+			}
+			for k, v := range globalProfiler {
+				v.Stop()
+				delete(globalProfiler, k)
+			}
+			return
+		case <-timer.C:
+			if !globalNotificationSys.DownloadProfilingData(ctx, w) {
+				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminProfilerNotEnabled), r.URL)
+				return
+			}
+			return
+		}
+	}
+}
+
 // dummyFileInfo represents a dummy representation of a profile data file
 // present only in memory, it helps to generate the zip stream.
 type dummyFileInfo struct {
@@ -613,7 +694,7 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 
 // DownloadProfilingHandler - POST /minio/admin/v3/profiling/download
 // ----------
-// Download profiling information of all nodes in a zip format
+// Download profiling information of all nodes in a zip format - deprecated API
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DownloadProfiling")
 
@@ -935,6 +1016,50 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 }
 
+// NetperfHandler - perform mesh style network throughput test
+func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "NetperfHandler")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if !globalIsDistErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "netperf")
+	lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
+		return
+	}
+	defer nsLock.Unlock(lkctx.Cancel)
+
+	durationStr := r.Form.Get(peerRESTDuration)
+	duration, err := time.ParseDuration(durationStr)
+	if err != nil {
+		duration = globalNetPerfMinDuration
+	}
+
+	if duration < globalNetPerfMinDuration {
+		// We need sample size of minimum 10 secs.
+		duration = globalNetPerfMinDuration
+	}
+
+	duration = duration.Round(time.Second)
+
+	results := globalNotificationSys.Netperf(ctx, duration)
+	enc := json.NewEncoder(w)
+	if err := enc.Encode(madmin.NetperfResult{NodeResults: results}); err != nil {
+		return
+	}
+}
+
 // SpeedtestHandler - Deprecated. See ObjectSpeedtestHandler
 func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	a.ObjectSpeedtestHandler(w, r)
@@ -959,6 +1084,8 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
+	var bucketExists bool
+
 	sizeStr := r.Form.Get(peerRESTSize)
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
@@ -1009,13 +1136,25 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 		autotune = false
 	}
 
+	err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{})
+	if err != nil {
+		if _, ok := err.(BucketExists); !ok {
+			// Only BucketExists error can be ignored.
+			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+		bucketExists = true
+	}
+
 	deleteBucket := func() {
-		objectAPI.DeleteBucket(context.Background(), pathJoin(minioMetaBucket, "speedtest"), DeleteBucketOptions{
+		objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
 			Force:      true,
 			NoRecreate: true,
 		})
 	}
-	defer deleteBucket()
+	if !bucketExists {
+		defer deleteBucket()
+	}
 
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
@@ -1618,7 +1757,14 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	query := r.Form
-	healthInfo := madmin.HealthInfo{Version: madmin.HealthInfoVersion}
+	healthInfo := madmin.HealthInfo{
+		Version: madmin.HealthInfoVersion,
+		Minio: madmin.MinioHealthInfo{
+			Info: madmin.MinioInfo{
+				DeploymentID: globalDeploymentID,
+			},
+		},
+	}
 	healthInfoCh := make(chan madmin.HealthInfo)
 
 	enc := json.NewEncoder(w)
@@ -1640,7 +1786,10 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		logger.LogIf(ctx, enc.Encode(healthInfo))
 	}
 
-	deadline := 1 * time.Hour
+	deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics.
+	if query.Get("perfnet") != "" || query.Get("perfdrive") != "" {
+		deadline = 1 * time.Hour
+	}
 	if dstr := r.Form.Get("deadline"); dstr != "" {
 		var err error
 		deadline, err = time.ParseDuration(dstr)
@@ -1951,7 +2100,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	go func() {
 		defer close(healthInfoCh)
 
-		partialWrite(healthInfo) // Write first message with only version populated
+		partialWrite(healthInfo) // Write first message with only version and deployment id populated
 		getAndWriteCPUs()
 		getAndWritePartitions()
 		getAndWriteOSInfo()
@@ -2287,19 +2436,9 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
 	defer cancel()
 
-	client := &http.Client{Transport: &http.Transport{
-		Proxy:                 http.ProxyFromEnvironment,
-		DialContext:           xhttp.NewCustomDialContext(timeout),
-		ResponseHeaderTimeout: 5 * time.Second,
-		TLSHandshakeTimeout:   5 * time.Second,
-		ExpectContinueTimeout: 5 * time.Second,
-		TLSClientConfig:       &tls.Config{RootCAs: globalRootCAs},
-		// Go net/http automatically unzip if content-type is
-		// gzip disable this feature, as we are always interested
-		// in raw stream.
-		DisableCompression: true,
-	}}
-	defer client.CloseIdleConnections()
+	client := &http.Client{
+		Transport: globalProxyTransport,
+	}
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodHead, endpointStr, nil)
 	if err != nil {
@@ -2392,8 +2531,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	// of profiling data of all nodes
 	zipWriter := zip.NewWriter(encw)
 	defer zipWriter.Close()
-
-	err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, si StatInfo) error {
+	rawDataFn := func(r io.Reader, host, disk, filename string, si StatInfo) error {
 		// Prefix host+disk
 		filename = path.Join(host, disk, filename)
 		if si.Dir {
@@ -2404,6 +2542,10 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 			// Not, set it to default.
 			si.Mode = 0o600
 		}
+		if si.ModTime.IsZero() {
+			// Set time to now.
+			si.ModTime = time.Now()
+		}
 		header, zerr := zip.FileInfoHeader(dummyFileInfo{
 			name:    filename,
 			size:    si.Size,
@@ -2426,8 +2568,33 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 			logger.LogIf(ctx, err)
 		}
 		return nil
-	})
-	logger.LogIf(ctx, err)
+	}
+	err = o.GetRawData(ctx, volume, file, rawDataFn)
+	if !errors.Is(err, errFileNotFound) {
+		logger.LogIf(ctx, err)
+	}
+
+	// save the format.json as part of inspect by default
+	if volume != minioMetaBucket && file != formatConfigFile {
+		err = o.GetRawData(ctx, minioMetaBucket, formatConfigFile, rawDataFn)
+	}
+	if !errors.Is(err, errFileNotFound) {
+		logger.LogIf(ctx, err)
+	}
+	// save args passed to inspect command
+	inspectArgs := []string{fmt.Sprintf(" Inspect path: %s%s%s\n", volume, slashSeparator, file)}
+	cmdLine := []string{"Server command line args: "}
+	for _, pool := range globalEndpoints {
+		cmdLine = append(cmdLine, pool.CmdLine)
+	}
+	cmdLine = append(cmdLine, "\n")
+	inspectArgs = append(inspectArgs, cmdLine...)
+	inspectArgsBytes := []byte(strings.Join(inspectArgs, " "))
+	if err = rawDataFn(bytes.NewReader(inspectArgsBytes), "", "", "inspect-input.txt", StatInfo{
+		Size: int64(len(inspectArgsBytes)),
+	}); err != nil {
+		logger.LogIf(ctx, err)
+	}
 }
 
 func createHostAnonymizerForFSMode() map[string]string {

cmd/admin-heal-ops.go

@@ -700,8 +700,9 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	}
 	if source.opts != nil {
 		task.opts = *source.opts
+	} else {
+		task.opts.ScanMode = madmin.HealNormalScan
 	}
-	task.opts.ScanMode = globalHealConfig.ScanMode()
 
 	h.mutex.Lock()
 	h.scannedItemsMap[healType]++

cmd/admin-router.go

@@ -84,10 +84,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(gz(httpTraceAll(adminAPI.CancelDecommission))).Queries("pool", "{pool:.*}")
 		}
 
-		// Profiling operations
+		// Profiling operations - deprecated API
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(gz(httpTraceAll(adminAPI.StartProfilingHandler))).
 			Queries("profilerType", "{profilerType:.*}")
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(gz(httpTraceAll(adminAPI.DownloadProfilingHandler)))
+		// Profiling operations
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/profile").HandlerFunc(gz(httpTraceAll(adminAPI.ProfileHandler)))
 
 		// Config KV operations.
 		if enableConfigOps {
@@ -224,7 +226,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedtestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedtestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(httpTraceHdrs(adminAPI.DriveSpeedtestHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetSpeedtestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetperfHandler))
 
 		// HTTP Trace
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(gz(http.HandlerFunc(adminAPI.TraceHandler)))

cmd/admin-server-info.go

@@ -50,7 +50,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 			}
 			_, present := network[nodeName]
 			if !present {
-				if err := isServerResolvable(endpoint, 2*time.Second); err == nil {
+				if err := isServerResolvable(endpoint, 5*time.Second); err == nil {
 					network[nodeName] = string(madmin.ItemOnline)
 				} else {
 					network[nodeName] = string(madmin.ItemOffline)

cmd/auth-handler.go

@@ -197,13 +197,7 @@ func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
 	return claims
 }
 
-// Fetch claims in the security token returned by the client.
-func getClaimsFromToken(token string) (map[string]interface{}, error) {
-	if token == "" {
-		claims := xjwt.NewMapClaims()
-		return claims.Map(), nil
-	}
-
+func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{}, error) {
 	// JWT token for x-amz-security-token is signed with admin
 	// secret key, temporary credentials become invalid if
 	// server admin credentials change. This is done to ensure
@@ -212,9 +206,15 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 	// hijacking the policies. We need to make sure that this is
 	// based an admin credential such that token cannot be decoded
 	// on the client side and is treated like an opaque value.
-	claims, err := auth.ExtractClaims(token, globalActiveCred.SecretKey)
+	claims, err := auth.ExtractClaims(token, secret)
 	if err != nil {
-		return nil, errAuthentication
+		if subtle.ConstantTimeCompare([]byte(secret), []byte(globalActiveCred.SecretKey)) == 1 {
+			return nil, errAuthentication
+		}
+		claims, err = auth.ExtractClaims(token, globalActiveCred.SecretKey)
+		if err != nil {
+			return nil, errAuthentication
+		}
 	}
 
 	// If OPA is set, return without any further checks.
@@ -241,23 +241,50 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 	return claims.Map(), nil
 }
 
+// Fetch claims in the security token returned by the client.
+func getClaimsFromToken(token string) (map[string]interface{}, error) {
+	return getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
+}
+
 // Fetch claims in the security token returned by the client and validate the token.
 func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
 	token := getSessionToken(r)
 	if token != "" && cred.AccessKey == "" {
+		// x-amz-security-token is not allowed for anonymous access.
 		return nil, ErrNoAccessKey
 	}
-	if cred.IsServiceAccount() && token == "" {
-		token = cred.SessionToken
-	}
-	if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+
+	if token == "" && cred.IsTemp() {
+		// Temporary credentials should always have x-amz-security-token
 		return nil, ErrInvalidToken
 	}
-	claims, err := getClaimsFromToken(token)
-	if err != nil {
-		return nil, toAPIErrorCode(r.Context(), err)
+
+	if token != "" && !cred.IsTemp() {
+		// x-amz-security-token should not present for static credentials.
+		return nil, ErrInvalidToken
 	}
-	return claims, ErrNone
+
+	if cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+		// validate token for temporary credentials only.
+		return nil, ErrInvalidToken
+	}
+
+	secret := globalActiveCred.SecretKey
+	if cred.IsServiceAccount() {
+		token = cred.SessionToken
+		secret = cred.SecretKey
+	}
+
+	if token != "" {
+		claims, err := getClaimsFromTokenWithSecret(token, secret)
+		if err != nil {
+			return nil, toAPIErrorCode(r.Context(), err)
+		}
+		return claims, ErrNone
+	}
+
+	claims := xjwt.NewMapClaims()
+	return claims.Map(), ErrNone
 }
 
 // Check request auth type verifies the incoming http request

cmd/background-newdisks-heal-ops.go

@@ -277,14 +277,6 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 		}
 	}
 
-	if err := bgSeq.healDiskMeta(objAPI); err != nil {
-		if newObjectLayerFn() != nil {
-			// log only in situations, when object layer
-			// has fully initialized.
-			logger.LogIf(bgSeq.ctx, err)
-		}
-	}
-
 	go monitorLocalDisksAndHeal(ctx, z, bgSeq)
 }
 

cmd/bucket-encryption-handlers.go

@@ -206,6 +206,15 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
+	// Call site replication hook.
+	//
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:      madmin.SRBucketMetaTypeSSEConfig,
+		Bucket:    bucket,
+		SSEConfig: nil,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 	writeSuccessNoContent(w)
 }

cmd/bucket-handlers.go

@@ -1270,6 +1270,17 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 				return
 			}
+			rcfg, err := getReplicationConfig(ctx, bucket)
+			switch {
+			case err != nil:
+				if _, ok := err.(BucketReplicationConfigNotFound); !ok {
+					writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+					return
+				}
+			case rcfg.HasActiveRules("", true):
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+				return
+			}
 		}
 	}
 

cmd/bucket-listobjects-handlers.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -26,26 +27,9 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/logger"
 
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/bucket/policy"
 )
 
-func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
-	g := errgroup.WithNErrs(len(objects)).WithConcurrency(500)
-	for index := range objects {
-		index := index
-		g.Go(func() error {
-			size, err := objects[index].GetActualSize()
-			if err == nil {
-				objects[index].Size = size
-			}
-			objects[index].ETag = objects[index].GetActualETag(nil)
-			return nil
-		}, index)
-	}
-	g.Wait()
-}
-
 // Validate all the ListObjects query arguments, returns an APIErrorCode
 // if one of the args do not meet the required conditions.
 // Special conditions required by MinIO server are as below
@@ -116,7 +100,11 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	concurrentDecryptETag(ctx, listObjectVersionsInfo.Objects)
+	if err = DecryptETags(ctx, GlobalKMS, listObjectVersionsInfo.Objects); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 
 	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
 
@@ -178,7 +166,11 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
+	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
@@ -253,7 +245,11 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
+	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 
 	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
@@ -350,7 +346,11 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	concurrentDecryptETag(ctx, listObjectsInfo.Objects)
+	if err = DecryptETags(ctx, GlobalKMS, listObjectsInfo.Objects); err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Failed to decrypt ETag: %v", err)) // TODO(aead): Remove once we are confident that decryption does not fail accidentially
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
 

cmd/bucket-replication-utils.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
@@ -27,6 +28,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
 )
 
 //go:generate msgp -file=$GOFILE
@@ -699,3 +701,26 @@ func newBucketResyncStatus(bucket string) BucketReplicationResyncStatus {
 		Version:    resyncMetaVersion,
 	}
 }
+
+var contentRangeRegexp = regexp.MustCompile(`bytes ([0-9]+)-([0-9]+)/([0-9]+|\\*)`)
+
+// parse size from content-range header
+func parseSizeFromContentRange(h http.Header) (sz int64, err error) {
+	cr := h.Get(xhttp.ContentRange)
+	if cr == "" {
+		return sz, fmt.Errorf("Content-Range not set")
+	}
+	parts := contentRangeRegexp.FindStringSubmatch(cr)
+	if len(parts) != 4 {
+		return sz, fmt.Errorf("invalid Content-Range header %s", cr)
+	}
+	if parts[3] == "*" {
+		return -1, nil
+	}
+	var usz uint64
+	usz, err = strconv.ParseUint(parts[3], 10, 64)
+	if err != nil {
+		return sz, err
+	}
+	return int64(usz), nil
+}

cmd/bucket-replication.go

@@ -425,7 +425,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		wg.Add(1)
 		go func(index int, tgt *TargetClient) {
 			defer wg.Done()
-			rinfo := replicateDeleteToTarget(ctx, dobj, objectAPI, tgt)
+			rinfo := replicateDeleteToTarget(ctx, dobj, tgt)
 			rinfos.Targets[index] = rinfo
 		}(idx, tgt)
 	}
@@ -482,7 +482,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	}
 }
 
-func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer, tgt *TargetClient) (rinfo replicatedTargetInfo) {
+func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
 		versionID = dobj.VersionID
@@ -1531,16 +1531,21 @@ func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
 	go globalReplicationStats.loadInitialReplicationMetrics(ctx)
 }
 
+type proxyResult struct {
+	Proxy bool
+	Err   error
+}
+
 // get Reader from replication target if active-active replication is in place and
 // this node returns a 404
-func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (gr *GetObjectReader, proxy bool) {
-	tgt, oi, proxy := proxyHeadToRepTarget(ctx, bucket, object, opts, proxyTargets)
-	if !proxy {
-		return nil, false
+func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (gr *GetObjectReader, proxy proxyResult, err error) {
+	tgt, oi, proxy := proxyHeadToRepTarget(ctx, bucket, object, rs, opts, proxyTargets)
+	if !proxy.Proxy {
+		return nil, proxy, nil
 	}
-	fn, off, length, err := NewGetObjectReader(rs, oi, opts)
+	fn, _, _, err := NewGetObjectReader(nil, oi, opts)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	gopts := miniogo.GetObjectOptions{
 		VersionID:            opts.VersionID,
@@ -1548,30 +1553,40 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 		Internal: miniogo.AdvancedGetOptions{
 			ReplicationProxyRequest: "true",
 		},
+		PartNumber: opts.PartNumber,
 	}
 	// get correct offsets for encrypted object
-	if off >= 0 && length >= 0 {
-		if err := gopts.SetRange(off, off+length-1); err != nil {
-			return nil, false
+	if rs != nil {
+		h, err := rs.ToHeader()
+		if err != nil {
+			return nil, proxy, err
 		}
+		gopts.Set(xhttp.Range, h)
 	}
 	// Make sure to match ETag when proxying.
 	if err = gopts.SetMatchETag(oi.ETag); err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	c := miniogo.Core{Client: tgt.Client}
-	obj, _, _, err := c.GetObject(ctx, bucket, object, gopts)
+	obj, _, h, err := c.GetObject(ctx, bucket, object, gopts)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	closeReader := func() { obj.Close() }
-
 	reader, err := fn(obj, h, closeReader)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	reader.ObjInfo = oi.Clone()
-	return reader, true
+	if rs != nil {
+		contentSize, err := parseSizeFromContentRange(h)
+		if err != nil {
+			return nil, proxy, err
+		}
+		reader.ObjInfo.Size = contentSize
+	}
+
+	return reader, proxyResult{Proxy: true}, nil
 }
 
 func getproxyTargets(ctx context.Context, bucket, object string, opts ObjectOptions) (tgts *madmin.BucketTargets) {
@@ -1590,11 +1605,11 @@ func getproxyTargets(ctx context.Context, bucket, object string, opts ObjectOpti
 	return tgts
 }
 
-func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgt *TargetClient, oi ObjectInfo, proxy bool) {
+func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgt *TargetClient, oi ObjectInfo, proxy proxyResult) {
 	// this option is set when active-active replication is in place between site A -> B,
 	// and site B does not have the object yet.
 	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
-		return nil, oi, false
+		return nil, oi, proxy
 	}
 	for _, t := range proxyTargets.Targets {
 		tgt = globalBucketTargetSys.GetRemoteTargetClient(ctx, t.Arn)
@@ -1612,9 +1627,22 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 			Internal: miniogo.AdvancedGetOptions{
 				ReplicationProxyRequest: "true",
 			},
+			PartNumber: opts.PartNumber,
 		}
+		if rs != nil {
+			h, err := rs.ToHeader()
+			if err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
+				continue
+			}
+			gopts.Set(xhttp.Range, h)
+		}
+
 		objInfo, err := tgt.StatObject(ctx, t.TargetBucket, object, gopts)
 		if err != nil {
+			if isErrInvalidRange(ErrorRespToObjectError(err, bucket, object)) {
+				return nil, oi, proxyResult{Err: err}
+			}
 			continue
 		}
 
@@ -1645,15 +1673,15 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 		if ok {
 			oi.ContentEncoding = ce
 		}
-		return tgt, oi, true
+		return tgt, oi, proxyResult{Proxy: true}
 	}
-	return nil, oi, false
+	return nil, oi, proxy
 }
 
 // get object info from replication target if active-active replication is in place and
 // this node returns a 404
-func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (oi ObjectInfo, proxy bool) {
-	_, oi, proxy = proxyHeadToRepTarget(ctx, bucket, object, opts, proxyTargets)
+func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (oi ObjectInfo, proxy proxyResult) {
+	_, oi, proxy = proxyHeadToRepTarget(ctx, bucket, object, rs, opts, proxyTargets)
 	return oi, proxy
 }
 

cmd/common-main.go

@@ -24,6 +24,7 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/gob"
+	"encoding/pem"
 	"errors"
 	"fmt"
 	"io/ioutil"
@@ -100,12 +101,16 @@ func init() {
 		PersistOnFailure: false,
 	}
 
+	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != ""
+
+	containers := IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile()
+
 	// Call to refresh will refresh names in cache. If you pass true, it will also
 	// remove cached names not looked up since the last call to Refresh. It is a good idea
 	// to call this method on a regular interval.
 	go func() {
 		var t *time.Ticker
-		if IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile() {
+		if containers {
 			t = time.NewTicker(1 * time.Minute)
 		} else {
 			t = time.NewTicker(10 * time.Minute)
@@ -482,6 +487,13 @@ func parsEnvEntry(envEntry string) (envKV, error) {
 	key := envTokens[0]
 	val := envTokens[1]
 
+	if strings.HasPrefix(key, "#") {
+		// Skip commented lines
+		return envKV{
+			Skip: true,
+		}, nil
+	}
+
 	// Remove quotes from the value if found
 	if len(val) >= 2 {
 		quote := val[0]
@@ -664,8 +676,6 @@ func handleCommonEnvVars() {
 		globalRootDiskThreshold = size
 	}
 
-	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != ""
-
 	domains := env.Get(config.EnvDomain, "")
 	if len(domains) != 0 {
 		for _, domainName := range strings.Split(domains, config.ValueSeparator) {
@@ -796,7 +806,29 @@ func handleCommonEnvVars() {
 				endpoints = append(endpoints, strings.Join(lbls, ""))
 			}
 		}
-		certificate, err := tls.LoadX509KeyPair(env.Get(config.EnvKESClientCert, ""), env.Get(config.EnvKESClientKey, ""))
+		// Manually load the certificate and private key into memory.
+		// We need to check whether the private key is encrypted, and
+		// if so, decrypt it using the user-provided password.
+		certBytes, err := os.ReadFile(env.Get(config.EnvKESClientCert, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
+		}
+		keyBytes, err := os.ReadFile(env.Get(config.EnvKESClientKey, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to load KES client private key as specified by the shell environment")
+		}
+		privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
+		if len(rest) != 0 {
+			logger.Fatal(errors.New("private key contains additional data"), "Unable to load KES client private key as specified by the shell environment")
+		}
+		if x509.IsEncryptedPEMBlock(privateKeyPEM) {
+			keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
+			if err != nil {
+				logger.Fatal(err, "Unable to decrypt KES client private key as specified by the shell environment")
+			}
+			keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
+		}
+		certificate, err := tls.X509KeyPair(certBytes, keyBytes)
 		if err != nil {
 			logger.Fatal(err, "Unable to load KES client certificate as specified by the shell environment")
 		}
@@ -903,6 +935,9 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 	}
 	secureConn = true
 
+	// Certs might be symlinks, reload them every 10 seconds.
+	manager.UpdateReloadDuration(10 * time.Second)
+
 	// syscall.SIGHUP to reload the certs.
 	manager.ReloadOnSignal(syscall.SIGHUP)
 

cmd/common-main_test.go

@@ -134,6 +134,24 @@ export MINIO_ROOT_PASSWORD=minio123`,
 			true,
 			nil,
 		},
+		{
+			`
+# MINIO_ROOT_USER=minioadmin
+# MINIO_ROOT_PASSWORD=minioadmin
+MINIO_ROOT_USER=minio
+MINIO_ROOT_PASSWORD=minio123`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+				{
+					Key:   "MINIO_ROOT_PASSWORD",
+					Value: "minio123",
+				},
+			},
+		},
 	}
 	for _, testCase := range testCases {
 		testCase := testCase
@@ -153,6 +171,11 @@ export MINIO_ROOT_PASSWORD=minio123`,
 			if err == nil && testCase.expectedErr {
 				t.Error(errors.New("expected error, found success"))
 			}
+
+			if len(ekvs) != len(testCase.expectedEkvs) {
+				t.Errorf("expected %v keys, got %v keys", len(testCase.expectedEkvs), len(ekvs))
+			}
+
 			if !reflect.DeepEqual(ekvs, testCase.expectedEkvs) {
 				t.Errorf("expected %v, got %v", testCase.expectedEkvs, ekvs)
 			}

cmd/data-scanner.go

@@ -21,6 +21,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io/fs"
@@ -103,6 +104,63 @@ func (s *safeDuration) Get() time.Duration {
 	return s.t
 }
 
+func getCycleScanMode(currentCycle, bitrotStartCycle uint64, bitrotStartTime time.Time) madmin.HealScanMode {
+	bitrotCycle := globalHealConfig.BitrotScanCycle()
+	switch bitrotCycle {
+	case -1:
+		return madmin.HealNormalScan
+	case 0:
+		return madmin.HealDeepScan
+	}
+
+	if currentCycle-bitrotStartCycle < healObjectSelectProb {
+		return madmin.HealDeepScan
+	}
+
+	if time.Since(bitrotStartTime) > bitrotCycle {
+		return madmin.HealDeepScan
+	}
+
+	return madmin.HealNormalScan
+}
+
+type backgroundHealInfo struct {
+	BitrotStartTime  time.Time           `json:"bitrotStartTime"`
+	BitrotStartCycle uint64              `json:"bitrotStartCycle"`
+	CurrentScanMode  madmin.HealScanMode `json:"currentScanMode"`
+}
+
+func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundHealInfo {
+	// Get last healing information
+	buf, err := readConfig(ctx, objAPI, backgroundHealInfoPath)
+	if err != nil {
+		if !errors.Is(err, errConfigNotFound) {
+			logger.LogIf(ctx, err)
+		}
+		return backgroundHealInfo{}
+	}
+	var info backgroundHealInfo
+	err = json.Unmarshal(buf, &info)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return backgroundHealInfo{}
+	}
+	return info
+}
+
+func saveBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer, info backgroundHealInfo) {
+	b, err := json.Marshal(info)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+	// Get last healing information
+	err = saveConfig(ctx, objAPI, backgroundHealInfoPath, b)
+	if err != nil {
+		logger.LogIf(ctx, err)
+	}
+}
+
 // runDataScanner will start a data scanner.
 // The function will block until the context is canceled.
 // There should only ever be one scanner running per cluster.
@@ -145,12 +203,24 @@ func runDataScanner(pctx context.Context, objAPI ObjectLayer) {
 				console.Debugln("starting scanner cycle")
 			}
 
+			bgHealInfo := readBackgroundHealInfo(ctx, objAPI)
+			scanMode := getCycleScanMode(nextBloomCycle, bgHealInfo.BitrotStartCycle, bgHealInfo.BitrotStartTime)
+			if bgHealInfo.CurrentScanMode != scanMode {
+				newHealInfo := bgHealInfo
+				newHealInfo.CurrentScanMode = scanMode
+				if scanMode == madmin.HealDeepScan {
+					newHealInfo.BitrotStartTime = time.Now().UTC()
+					newHealInfo.BitrotStartCycle = nextBloomCycle
+				}
+				saveBackgroundHealInfo(ctx, objAPI, newHealInfo)
+			}
+
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
 			bf, err := globalNotificationSys.updateBloomFilter(ctx, nextBloomCycle)
 			logger.LogIf(ctx, err)
-			err = objAPI.NSScanner(ctx, bf, results, uint32(nextBloomCycle))
+			err = objAPI.NSScanner(ctx, bf, results, uint32(nextBloomCycle), scanMode)
 			logger.LogIf(ctx, err)
 			if err == nil {
 				// Store new cycle...
@@ -182,6 +252,7 @@ type folderScanner struct {
 	dataUsageScannerDebug bool
 	healFolderInclude     uint32 // Include a clean folder one in n cycles.
 	healObjectSelect      uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
+	scanMode              madmin.HealScanMode
 
 	disks       []StorageAPI
 	disksQuorum int
@@ -250,7 +321,7 @@ var globalScannerStats scannerStats
 // The returned cache will always be valid, but may not be updated from the existing.
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
-func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn) (dataUsageCache, error) {
+func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
 	t := UTCNow()
 
 	logPrefix := color.Green("data-usage: ")
@@ -279,6 +350,7 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		dataUsageScannerDebug: intDataUpdateTracker.debug,
 		healFolderInclude:     0,
 		healObjectSelect:      0,
+		scanMode:              scanMode,
 		updates:               cache.Info.updates,
 	}
 
@@ -403,7 +475,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		if filter != nil && ok && existing.Compacted {
 			// If folder isn't in filter and we have data, skip it completely.
 			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
-				if f.healObjectSelect == 0 || !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
+				if f.healObjectSelect == 0 || !thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
 					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
 					f.updateCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
 					if f.dataUsageScannerDebug {
@@ -482,12 +554,15 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				debug:       f.dataUsageScannerDebug,
 				lifeCycle:   activeLifeCycle,
 				replication: replicationCfg,
-				heal:        thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
+
+			item.heal.enabled = thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure
+			item.heal.bitrot = f.scanMode == madmin.HealDeepScan
+
 			// if the drive belongs to an erasure set
 			// that is already being healed, skip the
 			// healing attempt on this drive.
-			item.heal = item.heal && f.healObjectSelect > 0
+			item.heal.enabled = item.heal.enabled && f.healObjectSelect > 0
 
 			sz, err := f.getSize(item)
 			if err != nil {
@@ -609,13 +684,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// and the entry itself is compacted.
 			if !into.Compacted && f.oldCache.isCompacted(h) {
 				if !h.mod(f.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
-					if f.healObjectSelect == 0 || !h.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
+					if f.healObjectSelect == 0 || !h.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
 						// Transfer and add as child...
 						f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
 						into.addChild(h)
 						continue
 					}
-					folder.objectHealProbDiv = dataUsageUpdateDirCycles
+					folder.objectHealProbDiv = f.healFolderInclude
 				}
 			}
 			scanFolder(folder)
@@ -821,8 +896,11 @@ type scannerItem struct {
 	replication replicationConfig
 	lifeCycle   *lifecycle.Lifecycle
 	Typ         fs.FileMode
-	heal        bool // Has the object been selected for heal check?
-	debug       bool
+	heal        struct {
+		enabled bool
+		bitrot  bool
+	} // Has the object been selected for heal check?
+	debug bool
 }
 
 type sizeSummary struct {
@@ -874,9 +952,13 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi Object
 			console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v\n", i.bucket, i.objectPath())
 		}
 	}
+	scanMode := madmin.HealNormalScan
+	if i.heal.bitrot {
+		scanMode = madmin.HealDeepScan
+	}
 	healOpts := madmin.HealOpts{
 		Remove:   healDeleteDangling,
-		ScanMode: globalHealConfig.ScanMode(),
+		ScanMode: scanMode,
 	}
 	res, err := o.HealObject(ctx, i.bucket, i.objectPath(), oi.VersionID, healOpts)
 	if err != nil && !errors.Is(err, NotImplemented{}) {
@@ -1040,7 +1122,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 	// from the current deployment, which means we don't have to call healing
 	// routine even if we are asked to do via heal flag.
 	if !applied {
-		if i.heal {
+		if i.heal.enabled {
 			size = i.applyHealing(ctx, o, oi)
 		}
 		// replicate only if lifecycle rules are not applied.

cmd/data-usage-cache.go

@@ -369,6 +369,17 @@ func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool {
 	return uint32(xxhash.Sum64String(string(h)))%cycles == cycle%cycles
 }
 
+// modAlt returns true if the hash mod cycles == cycle.
+// This is out of sync with mod.
+// If cycles is 0 false is always returned.
+// If cycles is 1 true is always returned (as expected).
+func (h dataUsageHash) modAlt(cycle uint32, cycles uint32) bool {
+	if cycles <= 1 {
+		return cycles == 1
+	}
+	return uint32(xxhash.Sum64String(string(h))>>32)%(cycles) == cycle%cycles
+}
+
 // addChild will add a child based on its hash.
 // If it already exists it will not be added again.
 func (e *dataUsageEntry) addChild(hash dataUsageHash) {

cmd/data-usage-utils.go

@@ -92,32 +92,37 @@ type DataUsageInfo struct {
 }
 
 func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
-	if globalTierConfigMgr.Empty() {
+	if dui.TierStats == nil {
 		return nil
 	}
 
-	ts := make(map[string]madmin.TierStats)
-	// Add configured remote tiers
-	for tier := range globalTierConfigMgr.Tiers {
-		ts[tier] = madmin.TierStats{}
+	cfgs := globalTierConfigMgr.ListTiers()
+	if len(cfgs) == 0 {
+		return nil
 	}
+
+	ts := make(map[string]madmin.TierStats, len(cfgs)+1)
+	infos := make([]madmin.TierInfo, 0, len(ts))
+
 	// Add STANDARD (hot-tier)
 	ts[minioHotTier] = madmin.TierStats{}
+	infos = append(infos, madmin.TierInfo{
+		Name: minioHotTier,
+		Type: "internal",
+	})
+	// Add configured remote tiers
+	for _, cfg := range cfgs {
+		ts[cfg.Name] = madmin.TierStats{}
+		infos = append(infos, madmin.TierInfo{
+			Name: cfg.Name,
+			Type: cfg.Type.String(),
+		})
+	}
 
 	ts = dui.TierStats.adminStats(ts)
-	infos := make([]madmin.TierInfo, 0, len(ts))
-	for tier, st := range ts {
-		var tierType string
-		if tier == minioHotTier {
-			tierType = "internal"
-		} else {
-			tierType = globalTierConfigMgr.Tiers[tier].Type.String()
-		}
-		infos = append(infos, madmin.TierInfo{
-			Name:  tier,
-			Type:  tierType,
-			Stats: st,
-		})
+	for i := range infos {
+		info := infos[i]
+		infos[i].Stats = ts[info.Name]
 	}
 
 	sort.Slice(infos, func(i, j int) bool {
@@ -131,3 +136,31 @@ func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
 	})
 	return infos
 }
+
+func (dui DataUsageInfo) tierMetrics() (metrics []Metric) {
+	if dui.TierStats == nil {
+		return nil
+	}
+	// e.g minio_cluster_ilm_transitioned_bytes{tier="S3TIER-1"}=136314880
+	//     minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1
+	//     minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3
+	for tier, st := range dui.TierStats.Tiers {
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedBytesMD(),
+			Value:          float64(st.TotalSize),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedObjectsMD(),
+			Value:          float64(st.NumObjects),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedVersionsMD(),
+			Value:          float64(st.NumVersions),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+	}
+
+	return metrics
+}

cmd/data-usage.go

@@ -35,6 +35,8 @@ const (
 	dataUsageBloomName     = ".bloomcycle.bin"
 	dataUsageBloomNamePath = bucketMetaPrefix + SlashSeparator + dataUsageBloomName
 
+	backgroundHealInfoPath = bucketMetaPrefix + SlashSeparator + ".background-heal.json"
+
 	dataUsageCacheName = ".usage-cache.bin"
 )
 

cmd/data-usage_test.go

@@ -67,7 +67,7 @@ func TestDataUsageUpdate(t *testing.T) {
 		return
 	}
 
-	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
+	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -178,7 +178,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize)
+		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize, 0)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -289,7 +289,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		}
 		return
 	}
-	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize)
+	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -423,7 +423,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize)
+		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize, 0)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -575,7 +575,7 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		}
 		return
 	}
-	want, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
+	want, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/disk-cache-utils.go

@@ -19,19 +19,17 @@ package cmd
 
 import (
 	"container/list"
-	"encoding/hex"
 	"errors"
 	"fmt"
 	"io"
 	"math"
 	"os"
-	"path"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/minio/minio/internal/crypto"
-	"github.com/minio/minio/internal/kms"
+	"github.com/minio/minio/internal/etag"
 )
 
 // CacheStatusType - whether the request was served from cache.
@@ -234,77 +232,85 @@ func isCacheEncrypted(meta map[string]string) bool {
 
 // decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
 func decryptCacheObjectETag(info *ObjectInfo) error {
-	// Directories are never encrypted.
 	if info.IsDir {
-		return nil
+		return nil // Directories are never encrypted.
 	}
-	encrypted := crypto.S3.IsEncrypted(info.UserDefined) && isCacheEncrypted(info.UserDefined)
 
-	switch {
-	case encrypted:
-		if globalCacheKMS == nil {
-			return errKMSNotConfigured
+	// Depending on the SSE type we handle ETags slightly
+	// differently. ETags encrypted with SSE-S3 must be
+	// decrypted first, since the client expects that
+	// a single-part SSE-S3 ETag is equal to the content MD5.
+	//
+	// For all other SSE types, the ETag is not the content MD5.
+	// Therefore, we don't decrypt but only format it.
+	switch kind, ok := crypto.IsEncrypted(info.UserDefined); {
+	case ok && kind == crypto.S3 && isCacheEncrypted(info.UserDefined):
+		ETag, err := etag.Parse(info.ETag)
+		if err != nil {
+			return err
 		}
-		if len(info.Parts) > 0 { // multipart ETag is not encrypted since it is not md5sum
+		if !ETag.IsEncrypted() {
+			info.ETag = ETag.Format().String()
 			return nil
 		}
-		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(info.UserDefined)
-		if err != nil {
-			return err
-		}
-		extKey, err := globalCacheKMS.DecryptKey(keyID, kmsKey, kms.Context{info.Bucket: path.Join(info.Bucket, info.Name)})
-		if err != nil {
-			return err
-		}
-		var objectKey crypto.ObjectKey
-		if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), info.Bucket, info.Name); err != nil {
-			return err
-		}
-		etagStr := tryDecryptETag(objectKey[:], info.ETag, false)
-		// backend ETag was hex encoded before encrypting, so hex decode to get actual ETag
-		etag, err := hex.DecodeString(etagStr)
-		if err != nil {
-			return err
-		}
-		info.ETag = string(etag)
-		return nil
-	}
 
+		key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, info.UserDefined, info.Bucket, info.Name)
+		if err != nil {
+			return err
+		}
+		ETag, err = etag.Decrypt(key[:], ETag)
+		if err != nil {
+			return err
+		}
+		info.ETag = ETag.Format().String()
+	case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(info.UserDefined):
+		ETag, err := etag.Parse(info.ETag)
+		if err != nil {
+			return err
+		}
+		info.ETag = ETag.Format().String()
+	}
 	return nil
 }
 
 // decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
 func decryptCachePartETags(c *cacheMeta) ([]string, error) {
-	var partETags []string
-	encrypted := crypto.S3.IsEncrypted(c.Meta) && isCacheEncrypted(c.Meta)
-
-	switch {
-	case encrypted:
-		if globalCacheKMS == nil {
-			return partETags, errKMSNotConfigured
-		}
-		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(c.Meta)
+	// Depending on the SSE type we handle ETags slightly
+	// differently. ETags encrypted with SSE-S3 must be
+	// decrypted first, since the client expects that
+	// a single-part SSE-S3 ETag is equal to the content MD5.
+	//
+	// For all other SSE types, the ETag is not the content MD5.
+	// Therefore, we don't decrypt but only format it.
+	switch kind, ok := crypto.IsEncrypted(c.Meta); {
+	case ok && kind == crypto.S3 && isCacheEncrypted(c.Meta):
+		key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, c.Meta, c.Bucket, c.Object)
 		if err != nil {
-			return partETags, err
-		}
-		extKey, err := globalCacheKMS.DecryptKey(keyID, kmsKey, kms.Context{c.Bucket: path.Join(c.Bucket, c.Object)})
-		if err != nil {
-			return partETags, err
-		}
-		var objectKey crypto.ObjectKey
-		if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), c.Bucket, c.Object); err != nil {
-			return partETags, err
+			return nil, err
 		}
+		etags := make([]string, 0, len(c.PartETags))
 		for i := range c.PartETags {
-			etagStr := tryDecryptETag(objectKey[:], c.PartETags[i], false)
-			// backend ETag was hex encoded before encrypting, so hex decode to get actual ETag
-			etag, err := hex.DecodeString(etagStr)
+			ETag, err := etag.Parse(c.PartETags[i])
 			if err != nil {
-				return []string{}, err
+				return nil, err
 			}
-			partETags = append(partETags, string(etag))
+			ETag, err = etag.Decrypt(key[:], ETag)
+			if err != nil {
+				return nil, err
+			}
+			etags = append(etags, ETag.Format().String())
 		}
-		return partETags, nil
+		return etags, nil
+	case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(c.Meta):
+		etags := make([]string, 0, len(c.PartETags))
+		for i := range c.PartETags {
+			ETag, err := etag.Parse(c.PartETags[i])
+			if err != nil {
+				return nil, err
+			}
+			etags = append(etags, ETag.Format().String())
+		}
+		return etags, nil
 	default:
 		return c.PartETags, nil
 	}

cmd/disk-cache.go

@@ -31,6 +31,7 @@ import (
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/cache"
+	"github.com/minio/minio/internal/disk"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -594,12 +595,23 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 	if err != nil {
 		return nil, false, err
 	}
+	var warningMsg string
 	for i, dir := range config.Drives {
 		// skip diskCache creation for cache drives missing a format.json
 		if formats[i] == nil {
 			caches = append(caches, nil)
 			continue
 		}
+		if !globalIsCICD && len(warningMsg) == 0 {
+			rootDsk, err := disk.IsRootDisk(dir, "/")
+			if err != nil {
+				warningMsg = fmt.Sprintf("Invalid cache dir %s err : %s", dir, err.Error())
+			}
+			if rootDsk {
+				warningMsg = fmt.Sprintf("cache dir cannot be part of root disk: %s", dir)
+			}
+		}
+
 		if err := checkAtimeSupport(dir); err != nil {
 			return nil, false, fmt.Errorf("Atime support required for disk caching, atime check failed with %w", err)
 		}
@@ -610,6 +622,9 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 		}
 		caches = append(caches, cache)
 	}
+	if warningMsg != "" {
+		logger.Info(color.Yellow(fmt.Sprintf("WARNING: Usage of root disk for disk caching is deprecated: %s", warningMsg)))
+	}
 	return caches, migrating, nil
 }
 

cmd/encryption-v1.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bufio"
+	"context"
 	"crypto/hmac"
 	"crypto/rand"
 	"crypto/sha256"
@@ -35,6 +36,7 @@ import (
 
 	"github.com/minio/kes"
 	"github.com/minio/minio/internal/crypto"
+	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/fips"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
@@ -71,18 +73,132 @@ const (
 )
 
 // KMSKeyID returns in AWS compatible KMS KeyID() format.
-func (o ObjectInfo) KMSKeyID() string {
-	if len(o.UserDefined) == 0 {
+func (o *ObjectInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDefined) }
+
+// KMSKeyID returns in AWS compatible KMS KeyID() format.
+func (o *MultipartInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDefined) }
+
+// kmsKeyIDFromMetadata returns any AWS S3 KMS key ID in the
+// metadata, if any. It returns an empty ID if no key ID is
+// present.
+func kmsKeyIDFromMetadata(metadata map[string]string) string {
+	const ARNPrefix = "arn:aws:kms:"
+	if len(metadata) == 0 {
 		return ""
 	}
-	kmsID, ok := o.UserDefined[crypto.MetaKeyID]
+	kmsID, ok := metadata[crypto.MetaKeyID]
 	if !ok {
 		return ""
 	}
-	if strings.HasPrefix(kmsID, "arn:aws:kms:") {
+	if strings.HasPrefix(kmsID, ARNPrefix) {
 		return kmsID
 	}
-	return "arn:aws:kms:" + kmsID
+	return ARNPrefix + kmsID
+}
+
+// DecryptETags decryptes the ETag of all ObjectInfos using the KMS.
+//
+// It adjusts the size of all encrypted objects since encrypted
+// objects are slightly larger due to encryption overhead.
+// Further, it decrypts all single-part SSE-S3 encrypted objects
+// and formats ETags of SSE-C / SSE-KMS encrypted objects to
+// be AWS S3 compliant.
+//
+// DecryptETags uses a KMS bulk decryption API, if available, which
+// is more efficient than decrypting ETags sequentually.
+func DecryptETags(ctx context.Context, KMS kms.KMS, objects []ObjectInfo) error {
+	const BatchSize = 250 // We process the objects in batches - 250 is a reasonable default.
+	var (
+		metadata = make([]map[string]string, 0, BatchSize)
+		buckets  = make([]string, 0, BatchSize)
+		names    = make([]string, 0, BatchSize)
+	)
+	for len(objects) > 0 {
+		N := BatchSize
+		if len(objects) < BatchSize {
+			N = len(objects)
+		}
+		batch := objects[:N]
+
+		// We have to decrypt only ETags of SSE-S3 single-part
+		// objects.
+		// Therefore, we remember which objects (there index)
+		// in the current batch are single-part SSE-S3 objects.
+		metadata = metadata[:0:N]
+		buckets = buckets[:0:N]
+		names = names[:0:N]
+		SSES3SinglePartObjects := make(map[int]bool)
+		for i, object := range batch {
+			if kind, ok := crypto.IsEncrypted(object.UserDefined); ok && kind == crypto.S3 && !crypto.IsMultiPart(object.UserDefined) {
+				SSES3SinglePartObjects[i] = true
+
+				metadata = append(metadata, object.UserDefined)
+				buckets = append(buckets, object.Bucket)
+				names = append(names, object.Name)
+			}
+		}
+
+		// If there are no SSE-S3 single-part objects
+		// we can skip the decryption process. However,
+		// we still have to adjust the size and ETag
+		// of SSE-C and SSE-KMS objects.
+		if len(SSES3SinglePartObjects) == 0 {
+			for i := range batch {
+				size, err := batch[i].GetActualSize()
+				if err != nil {
+					return err
+				}
+				batch[i].Size = size
+
+				if _, ok := crypto.IsEncrypted(batch[i].UserDefined); ok {
+					ETag, err := etag.Parse(batch[i].ETag)
+					if err != nil {
+						return err
+					}
+					batch[i].ETag = ETag.Format().String()
+				}
+			}
+			objects = objects[N:]
+			continue
+		}
+
+		// There is at least one SSE-S3 single-part object.
+		// For all SSE-S3 single-part objects we have to
+		// fetch their decryption keys. We do this using
+		// a Bulk-Decryption API call, if available.
+		keys, err := crypto.S3.UnsealObjectKeys(ctx, KMS, metadata, buckets, names)
+		if err != nil {
+			return err
+		}
+
+		// Now, we have to decrypt the ETags of SSE-S3 single-part
+		// objects and adjust the size and ETags of all encrypted
+		// objects.
+		for i := range batch {
+			size, err := batch[i].GetActualSize()
+			if err != nil {
+				return err
+			}
+			batch[i].Size = size
+
+			if _, ok := crypto.IsEncrypted(batch[i].UserDefined); ok {
+				ETag, err := etag.Parse(batch[i].ETag)
+				if err != nil {
+					return err
+				}
+				if SSES3SinglePartObjects[i] && ETag.IsEncrypted() {
+					ETag, err = etag.Decrypt(keys[0][:], ETag)
+					if err != nil {
+						return err
+					}
+					keys = keys[1:]
+				}
+				batch[i].ETag = ETag.Format().String()
+			}
+		}
+		objects = objects[N:]
+	}
+	return nil
 }
 
 // isMultipart returns true if the current object is

cmd/erasure-bucket.go

@@ -39,8 +39,10 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
 	defer NSUpdated(bucket, slashSeparator)
 
 	// Verify if bucket is valid.
-	if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
-		return BucketNameInvalid{Bucket: bucket}
+	if !isMinioMetaBucketName(bucket) {
+		if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
+			return BucketNameInvalid{Bucket: bucket}
+		}
 	}
 
 	storageDisks := er.getDisks()

cmd/erasure-healing-common.go

@@ -83,7 +83,10 @@ func commonTime(modTimes []time.Time) (modTime time.Time) {
 }
 
 // Beginning of unix time is treated as sentinel value here.
-var timeSentinel = time.Unix(0, 0).UTC()
+var (
+	timeSentinel     = time.Unix(0, 0).UTC()
+	timeSentinel1970 = time.Unix(0, 1).UTC() // 1970 used for special cases when xlmeta.version == 0
+)
 
 // Boot modTimes up to disk count, setting the value to time sentinel.
 func bootModtimes(diskCount int) []time.Time {

cmd/erasure-healing.go

@@ -931,5 +931,12 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	}
 
 	// Heal the object.
-	return er.healObject(healCtx, bucket, object, versionID, opts)
+	hr, err = er.healObject(healCtx, bucket, object, versionID, opts)
+	if errors.Is(err, errFileCorrupt) && opts.ScanMode != madmin.HealDeepScan {
+		// Instead of returning an error when a bitrot error is detected
+		// during a normal heal scan, heal again with bitrot flag enabled.
+		opts.ScanMode = madmin.HealDeepScan
+		hr, err = er.healObject(healCtx, bucket, object, versionID, opts)
+	}
+	return hr, err
 }

cmd/erasure-healing_test.go

@@ -382,6 +382,13 @@ func TestHealingDanglingObject(t *testing.T) {
 		copy(disks, newDisks)
 		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock()
 	}
+	getDisk := func(n int) StorageAPI {
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Lock()
+		disk := disks[n]
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock()
+		return disk
+	}
+
 	// Remove 4 disks.
 	setDisks(nil, nil, nil, nil)
 
@@ -440,8 +447,8 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	setDisks(orgDisks[:4]...)
-
-	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk := getDisk(0)
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -458,7 +465,8 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -488,7 +496,8 @@ func TestHealingDanglingObject(t *testing.T) {
 
 	setDisks(orgDisks[:4]...)
 
-	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -505,7 +514,8 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -807,7 +817,140 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	}
 }
 
-func TestHealObjectCorrupted(t *testing.T) {
+func TestHealObjectCorruptedXLMeta(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resetGlobalHealState()
+	defer resetGlobalHealState()
+
+	nDisks := 16
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	// Everything is fine, should return nil
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+	data := bytes.Repeat([]byte("a"), 5*1024*1024)
+	var opts ObjectOptions
+
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	// Create an object with multiple parts uploaded in decreasing
+	// part number.
+	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	if err != nil {
+		t.Fatalf("Failed to create a multipart upload - %v", err)
+	}
+
+	var uploadedParts []CompletePart
+	for _, partID := range []int{2, 1} {
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		if err1 != nil {
+			t.Fatalf("Failed to upload a part - %v", err1)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Failed to complete multipart upload - %v", err)
+	}
+
+	z := objLayer.(*erasureServerPools)
+	er := z.serverPools[0].sets[0]
+	erasureDisks := er.getDisks()
+	firstDisk := erasureDisks[0]
+
+	// Test 1: Remove the object backend files from the first disk.
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	if err != nil {
+		t.Fatalf("Failed to delete a file - %v", err)
+	}
+
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+	if err != nil {
+		t.Fatalf("Failed to heal object - %v", err)
+	}
+
+	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
+		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+	}
+
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	nfi1, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	if !reflect.DeepEqual(fi, nfi1) {
+		t.Fatalf("FileInfo not equal after healing")
+	}
+
+	// Test 2: Test with a corrupted xl.meta
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), []byte("abcd"))
+	if err != nil {
+		t.Errorf("Failure during creating part.1 - %v", err)
+	}
+
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+	if err != nil {
+		t.Errorf("Expected nil but received %v", err)
+	}
+
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	nfi2, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	if !reflect.DeepEqual(fi, nfi2) {
+		t.Fatalf("FileInfo not equal after healing")
+	}
+
+	// Test 3: checks if HealObject returns an error when xl.meta is not found
+	// in more than read quorum number of disks, to create a corrupted situation.
+	for i := 0; i <= nfi2.Erasure.DataBlocks; i++ {
+		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	}
+
+	// Try healing now, expect to receive errFileNotFound.
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	if err != nil {
+		if _, ok := err.(ObjectNotFound); !ok {
+			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+		}
+	}
+
+	// since majority of xl.meta's are not available, object should be successfully deleted.
+	_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if _, ok := err.(ObjectNotFound); !ok {
+		t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+	}
+}
+
+func TestHealObjectCorruptedParts(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -867,7 +1010,26 @@ func TestHealObjectCorrupted(t *testing.T) {
 	er := z.serverPools[0].sets[0]
 	erasureDisks := er.getDisks()
 	firstDisk := erasureDisks[0]
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	secondDisk := erasureDisks[1]
+
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	part1Disk1Origin, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	if err != nil {
+		t.Fatalf("Failed to read a file - %v", err)
+	}
+
+	part1Disk2Origin, err := secondDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	if err != nil {
+		t.Fatalf("Failed to read a file - %v", err)
+	}
+
+	// Test 1, remove part.1
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -877,85 +1039,67 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	part1Replaced, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
-		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+	if !reflect.DeepEqual(part1Disk1Origin, part1Replaced) {
+		t.Fatalf("part.1 not healed correctly")
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	// Test 2, Corrupt part.1
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte("foobytes"))
 	if err != nil {
-		t.Errorf("Failure during deleting part.1 - %v", err)
+		t.Fatalf("Failed to write a file - %v", err)
 	}
 
-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
-		t.Errorf("Failure during creating part.1 - %v", err)
+		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	part1Replaced, err = firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Errorf("Expected nil but received %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if !reflect.DeepEqual(part1Disk1Origin, part1Replaced) {
+		t.Fatalf("part.1 not healed correctly")
+	}
+
+	// Test 3, Corrupt one part and remove data in another disk
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte("foobytes"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to write a file - %v", err)
 	}
 
-	if !reflect.DeepEqual(fi, nfi) {
-		t.Fatalf("FileInfo not equal after healing")
-	}
-
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	err = secondDisk.Delete(context.Background(), bucket, object, true)
 	if err != nil {
-		t.Errorf("Failure during deleting part.1 - %v", err)
+		t.Fatalf("Failed to delete a file - %v", err)
 	}
 
-	bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
-		t.Errorf("Failure during creating part.1 - %v", err)
+		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	partReconstructed, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Errorf("Expected nil but received %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+	if !reflect.DeepEqual(part1Disk1Origin, partReconstructed) {
+		t.Fatalf("part.1 not healed correctly")
+	}
+
+	partReconstructed, err = secondDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	if !reflect.DeepEqual(fi, nfi) {
-		t.Fatalf("FileInfo not equal after healing")
-	}
-
-	// Test 4: checks if HealObject returns an error when xl.meta is not found
-	// in more than read quorum number of disks, to create a corrupted situation.
-	for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
-		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
-	}
-
-	// Try healing now, expect to receive errFileNotFound.
-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
-	if err != nil {
-		if _, ok := err.(ObjectNotFound); !ok {
-			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
-		}
-	}
-
-	// since majority of xl.meta's are not available, object should be successfully deleted.
-	_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
-	if _, ok := err.(ObjectNotFound); !ok {
-		t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+	if !reflect.DeepEqual(part1Disk2Origin, partReconstructed) {
+		t.Fatalf("part.1 not healed correctly")
 	}
 }
 
@@ -1017,7 +1161,8 @@ func TestHealObjectErasure(t *testing.T) {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	// Delete the whole object folder
+	err = firstDisk.Delete(context.Background(), bucket, object, true)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}

cmd/erasure-object.go

@@ -493,11 +493,6 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 		return objInfo, toObjectErr(err, bucket, object)
 	}
 	objInfo = fi.ToObjectInfo(bucket, object)
-	if opts.VersionID != "" && !fi.VersionPurgeStatus().Empty() {
-		// Make sure to return object info to provide extra information.
-		return objInfo, toObjectErr(errMethodNotAllowed, bucket, object)
-	}
-
 	if fi.Deleted {
 		if opts.VersionID == "" || opts.DeleteMarker {
 			return objInfo, toObjectErr(errFileNotFound, bucket, object)

cmd/erasure-object_test.go

@@ -128,6 +128,86 @@ func TestErasureDeleteObjectBasic(t *testing.T) {
 	removeRoots(fsDirs)
 }
 
+func TestDeleteObjectsVersioned(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	obj, fsDirs, err := prepareErasure(ctx, 16)
+	if err != nil {
+		t.Fatal("Unable to initialize 'Erasure' object layer.", err)
+	}
+	// Remove all dirs.
+	for _, dir := range fsDirs {
+		defer os.RemoveAll(dir)
+	}
+
+	type testCaseType struct {
+		bucket string
+		object string
+	}
+
+	bucketName := "bucket"
+	testCases := []testCaseType{
+		{bucketName, "dir/obj1"},
+		{bucketName, "dir/obj1"},
+	}
+
+	err = obj.MakeBucketWithLocation(ctx, bucketName, BucketOptions{
+		VersioningEnabled: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	names := make([]ObjectToDelete, len(testCases))
+	for i, testCase := range testCases {
+		objInfo, err := obj.PutObject(ctx, testCase.bucket, testCase.object,
+			mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{
+				Versioned: true,
+			})
+		if err != nil {
+			t.Fatalf("Erasure Object upload failed: <ERROR> %s", err)
+		}
+		names[i] = ObjectToDelete{
+			ObjectV: ObjectV{
+				ObjectName: objInfo.Name,
+				VersionID:  objInfo.VersionID,
+			},
+		}
+
+	}
+	names = append(names, ObjectToDelete{
+		ObjectV: ObjectV{
+			ObjectName: "dir/obj1",
+			VersionID:  mustGetUUID(), // add a non-existent UUID.
+		},
+	})
+
+	_, delErrs := obj.DeleteObjects(ctx, bucketName, names, ObjectOptions{
+		Versioned: true,
+	})
+	for i := range delErrs {
+		if delErrs[i] != nil {
+			t.Errorf("Failed to remove object `%v` with the error: `%v`", names[i], delErrs[i])
+		}
+	}
+
+	for i, test := range testCases {
+		_, statErr := obj.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{
+			VersionID: names[i].ObjectV.VersionID,
+		})
+		switch statErr.(type) {
+		case VersionNotFound:
+		default:
+			t.Fatalf("Object %s is not removed", test.bucket+SlashSeparator+test.object)
+		}
+	}
+
+	if _, err = ioutil.ReadFile(pathJoin(fsDirs[0], bucketName, "dir/obj1", "xl.meta")); err == nil {
+		t.Fatalf("xl.meta still present after removal")
+	}
+}
+
 func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

cmd/erasure-server-pool-decom.go

@@ -24,12 +24,15 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
+	"strconv"
+	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
+	"github.com/minio/pkg/env"
 )
 
 // PoolDecommissionInfo currently decommissioning information
@@ -141,7 +144,7 @@ func (p *poolMeta) returnResumablePools(n int) []PoolStatus {
 }
 
 func (p *poolMeta) DecommissionComplete(idx int) bool {
-	if p.Pools[idx].Decommission != nil {
+	if p.Pools[idx].Decommission != nil && !p.Pools[idx].Decommission.Complete {
 		p.Pools[idx].LastUpdate = UTCNow()
 		p.Pools[idx].Decommission.Complete = true
 		p.Pools[idx].Decommission.Failed = false
@@ -152,7 +155,7 @@ func (p *poolMeta) DecommissionComplete(idx int) bool {
 }
 
 func (p *poolMeta) DecommissionFailed(idx int) bool {
-	if p.Pools[idx].Decommission != nil {
+	if p.Pools[idx].Decommission != nil && !p.Pools[idx].Decommission.Failed {
 		p.Pools[idx].LastUpdate = UTCNow()
 		p.Pools[idx].Decommission.StartTime = time.Time{}
 		p.Pools[idx].Decommission.Complete = false
@@ -164,7 +167,7 @@ func (p *poolMeta) DecommissionFailed(idx int) bool {
 }
 
 func (p *poolMeta) DecommissionCancel(idx int) bool {
-	if p.Pools[idx].Decommission != nil {
+	if p.Pools[idx].Decommission != nil && !p.Pools[idx].Decommission.Canceled {
 		p.Pools[idx].LastUpdate = UTCNow()
 		p.Pools[idx].Decommission.StartTime = time.Time{}
 		p.Pools[idx].Decommission.Complete = false
@@ -466,21 +469,32 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 
 	// if no update is needed return right away.
 	if !update {
+		z.poolMeta = meta
+
 		// We are only supporting single pool decommission at this time
 		// so it makes sense to only resume single pools at any given
 		// time, in future meta.returnResumablePools() might take
 		// '-1' as argument to decommission multiple pools at a time
 		// but this is not a priority at the moment.
 		for _, pool := range meta.returnResumablePools(1) {
-			err := z.Decommission(ctx, pool.ID)
-			switch err {
-			case errDecommissionAlreadyRunning:
-				fallthrough
-			case nil:
-				go z.doDecommissionInRoutine(ctx, pool.ID)
+			idx := globalEndpoints.GetPoolIdx(pool.CmdLine)
+			if idx == -1 {
+				return fmt.Errorf("unexpected state present for decommission status pool(%s) not found", pool.CmdLine)
+			}
+			if globalEndpoints[idx].Endpoints[0].IsLocal {
+				go func(pool PoolStatus) {
+					switch err := z.Decommission(ctx, pool.ID); err {
+					case nil:
+						// we already started decommission
+					case errDecommissionAlreadyRunning:
+						// A previous decommission running found restart it.
+						z.doDecommissionInRoutine(ctx, idx)
+					default:
+						logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pool, err))
+					}
+				}(pool)
 			}
 		}
-		z.poolMeta = meta
 
 		return nil
 	}
@@ -567,15 +581,16 @@ func (v versionsSorter) reverse() {
 }
 
 func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bName string) error {
-	var forwardTo string
-	// If we resume to the same bucket, forward to last known item.
-	rbucket, robject := z.poolMeta.ResumeBucketObject(idx)
-	if rbucket != "" && rbucket == bName {
-		forwardTo = robject
-	}
+	var wg sync.WaitGroup
+	wStr := env.Get("_MINIO_DECOMMISSION_WORKERS", strconv.Itoa(len(pool.sets)))
+	workerSize, _ := strconv.Atoi(wStr)
+
+	parallelWorkers := make(chan struct{}, workerSize)
 
 	versioned := globalBucketVersioningSys.Enabled(bName)
 	for _, set := range pool.sets {
+		parallelWorkers <- struct{}{}
+		set := set
 		disks := set.getOnlineDisks()
 		if len(disks) == 0 {
 			logger.LogIf(GlobalContext, fmt.Errorf("no online disks found for set with endpoints %s",
@@ -593,7 +608,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				return
 			}
 
-			// We need a reversed order for Decommissioning,
+			// We need a reversed order for decommissioning,
 			// to create the appropriate stack.
 			versionsSorter(fivs.Versions).reverse()
 
@@ -687,26 +702,30 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			bucket:    bName,
 		}
 
-		if err := listPathRaw(ctx, listPathRawOptions{
-			disks:          disks,
-			bucket:         bName,
-			recursive:      true,
-			forwardTo:      forwardTo,
-			minDisks:       len(disks) / 2, // to capture all quorum ratios
-			reportNotFound: false,
-			agreed:         decommissionEntry,
-			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
-				entry, ok := entries.resolve(&resolver)
-				if ok {
-					decommissionEntry(*entry)
-				}
-			},
-			finished: nil,
-		}); err != nil {
-			// Decommissioning failed and won't continue
-			return err
-		}
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			err := listPathRaw(ctx, listPathRawOptions{
+				disks:          disks,
+				bucket:         bName,
+				recursive:      true,
+				forwardTo:      "",
+				minDisks:       len(disks) / 2, // to capture all quorum ratios
+				reportNotFound: false,
+				agreed:         decommissionEntry,
+				partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+					entry, ok := entries.resolve(&resolver)
+					if ok {
+						decommissionEntry(*entry)
+					}
+				},
+				finished: nil,
+			})
+			logger.LogIf(ctx, err)
+			<-parallelWorkers
+		}()
 	}
+	wg.Wait()
 	return nil
 }
 
@@ -754,8 +773,8 @@ func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx in
 }
 
 func (z *erasureServerPools) IsSuspended(idx int) bool {
-	z.poolMetaMutex.Lock()
-	defer z.poolMetaMutex.Unlock()
+	z.poolMetaMutex.RLock()
+	defer z.poolMetaMutex.RUnlock()
 	return z.poolMeta.IsSuspended(idx)
 }
 
@@ -801,12 +820,7 @@ func (z *erasureServerPools) getDecommissionPoolSpaceInfo(idx int) (pi poolSpace
 	if idx+1 > len(z.serverPools) {
 		return pi, errInvalidArgument
 	}
-	info, errs := z.serverPools[idx].StorageInfo(context.Background())
-	for _, err := range errs {
-		if err != nil {
-			return pi, errInvalidArgument
-		}
-	}
+	info, _ := z.serverPools[idx].StorageInfo(context.Background())
 	info.Backend = z.BackendInfo()
 	for _, disk := range info.Disks {
 		if disk.Healing {
@@ -922,6 +936,7 @@ func (z *erasureServerPools) CompleteDecommission(ctx context.Context, idx int)
 	defer z.poolMetaMutex.Unlock()
 
 	if z.poolMeta.DecommissionComplete(idx) {
+		z.decommissionCancelers[idx]() // cancel any active thread.
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -955,6 +970,21 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		}
 	}
 
+	// Create .minio.sys/conifg, .minio.sys/buckets paths if missing,
+	// this code is present to avoid any missing meta buckets on other
+	// pools.
+	for _, metaBucket := range []string{
+		pathJoin(minioMetaBucket, minioConfigPrefix),
+		pathJoin(minioMetaBucket, bucketMetaPrefix),
+	} {
+		var bucketExists BucketExists
+		if err = z.MakeBucketWithLocation(ctx, metaBucket, BucketOptions{}); err != nil {
+			if !errors.As(err, &bucketExists) {
+				return err
+			}
+		}
+	}
+
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to decommission the necessary metadata.
 	buckets = append(buckets, BucketInfo{

cmd/erasure-server-pool.go

@@ -33,6 +33,7 @@ import (
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -128,7 +129,9 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			if !configRetriableErrors(err) {
 				logger.Fatal(err, "Unable to initialize backend")
 			}
-			time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
+			retry := time.Duration(r.Float64() * float64(5*time.Second))
+			logger.LogIf(ctx, fmt.Errorf("Unable to initialize backend: %w, retrying in %s", err, retry))
+			time.Sleep(retry)
 			continue
 		}
 		break
@@ -528,7 +531,7 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context) (StorageInfo, []er
 	return storageInfo, errs
 }
 
-func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32) error {
+func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
 	// Updates must be closed before we return.
 	defer close(updates)
 
@@ -573,7 +576,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 					}
 				}()
 				// Start scanner. Blocks until done.
-				err := erObj.nsScanner(ctx, allBuckets, bf, wantCycle, updates)
+				err := erObj.nsScanner(ctx, allBuckets, bf, wantCycle, updates, healScanMode)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					mu.Lock()
@@ -661,6 +664,9 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
+			if z.IsSuspended(index) {
+				return nil
+			}
 			return z.serverPools[index].MakeBucketWithLocation(ctx, bucket, opts)
 		}, index)
 	}
@@ -672,7 +678,8 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 			if _, ok := err.(BucketExists); !ok {
 				// Delete created buckets, ignoring errors.
 				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
-					Force: false, NoRecreate: true,
+					Force:      false,
+					NoRecreate: true,
 				})
 			}
 			return err
@@ -813,6 +820,11 @@ func (z *erasureServerPools) getLatestObjectInfoWithIdx(ctx context.Context, buc
 			// should be returned upwards.
 			return res.oi, res.zIdx, err
 		}
+		// When its a delete marker and versionID is empty
+		// we should simply return the error right away.
+		if res.oi.DeleteMarker && opts.VersionID == "" {
+			return res.oi, res.zIdx, err
+		}
 	}
 
 	object = decodeDirObject(object)
@@ -1146,6 +1158,14 @@ func maxKeysPlusOne(maxKeys int, addOne bool) int {
 func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
 	var loi ListObjectsInfo
 
+	// Automatically remove the object/version is an expiry lifecycle rule can be applied
+	lc, _ := globalLifecycleSys.Get(bucket)
+	if lc != nil {
+		if !lc.HasActiveRules(prefix, true) {
+			lc = nil
+		}
+	}
+
 	if len(prefix) > 0 && maxKeys == 1 && delimiter == "" && marker == "" {
 		// Optimization for certain applications like
 		// - Cohesity
@@ -1156,6 +1176,13 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		// to avoid the need for ListObjects().
 		objInfo, err := z.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
 		if err == nil {
+			if lc != nil {
+				action := evalActionFromLifecycle(ctx, *lc, objInfo, false)
+				switch action {
+				case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
+					return loi, nil
+				}
+			}
 			loi.Objects = append(loi.Objects, objInfo)
 			return loi, nil
 		}
@@ -1170,6 +1197,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		InclDeleted: false,
 		AskDisks:    globalAPIConfig.getListQuorum(),
 	}
+
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
 		if !isErrBucketNotFound(err) {
@@ -1478,6 +1506,9 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
+			if z.IsSuspended(index) {
+				return nil
+			}
 			return z.serverPools[index].DeleteBucket(ctx, bucket, opts)
 		}, index)
 	}

cmd/erasure.go

@@ -344,7 +344,7 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
 
 // nsScanner will start scanning buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
-func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache) error {
+func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
 	if len(buckets) == 0 {
 		return nil
 	}
@@ -490,7 +490,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 				// Calc usage
 				before := cache.Info.LastUpdate
 				var err error
-				cache, err = disk.NSScanner(ctx, cache, updates)
+				cache, err = disk.NSScanner(ctx, cache, updates, healScanMode)
 				cache.Info.BloomFilter = nil
 				if err != nil {
 					if !cache.Info.LastUpdate.IsZero() && cache.Info.LastUpdate.After(before) {

cmd/fs-v1.go

@@ -235,7 +235,7 @@ func (fs *FSObjects) StorageInfo(ctx context.Context) (StorageInfo, []error) {
 }
 
 // NSScanner returns data usage stats of the current FS deployment
-func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32) error {
+func (fs *FSObjects) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, _ madmin.HealScanMode) error {
 	defer close(updates)
 	// Load bucket totals
 	var totalCache dataUsageCache
@@ -396,7 +396,7 @@ func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUs
 		}
 
 		return sizeSummary{totalSize: fi.Size(), versions: 1}, nil
-	})
+	}, 0)
 
 	return cache, err
 }

cmd/gateway-unsupported.go

@@ -48,7 +48,7 @@ func (a GatewayUnsupported) LocalStorageInfo(ctx context.Context) (StorageInfo,
 }
 
 // NSScanner - scanner is not implemented for gateway
-func (a GatewayUnsupported) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32) error {
+func (a GatewayUnsupported) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, scanMode madmin.HealScanMode) error {
 	logger.CriticalIf(ctx, errors.New("not implemented"))
 	return NotImplemented{}
 }

cmd/global-heal.go

@@ -44,8 +44,7 @@ func newBgHealSequence() *healSequence {
 
 	hs := madmin.HealOpts{
 		// Remove objects that do not have read-quorum
-		Remove:   healDeleteDangling,
-		ScanMode: globalHealConfig.ScanMode(),
+		Remove: healDeleteDangling,
 	}
 
 	return &healSequence{
@@ -165,7 +164,7 @@ func mustGetHealSequence(ctx context.Context) *healSequence {
 // healErasureSet lists and heals all objects in a specific erasure set
 func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, tracker *healingTracker) error {
 	bgSeq := mustGetHealSequence(ctx)
-	scanMode := globalHealConfig.ScanMode()
+	scanMode := madmin.HealNormalScan
 
 	// Make sure to copy since `buckets slice`
 	// is modified in place by tracker.

cmd/globals.go

@@ -102,7 +102,7 @@ const (
 	GlobalStaleUploadsCleanupInterval = time.Hour * 6 // 6 hrs.
 
 	// Refresh interval to update in-memory iam config cache.
-	globalRefreshIAMInterval = 5 * time.Minute
+	globalRefreshIAMInterval = 10 * time.Minute
 
 	// Limit of location constraint XML for unauthenticated PUT bucket operations.
 	maxLocationConstraintSize = 3 * humanize.MiByte
@@ -190,7 +190,7 @@ var (
 	globalBucketTargetSys    *BucketTargetSys
 	// globalAPIConfig controls S3 API requests throttling,
 	// healthcheck readiness deadlines and cors settings.
-	globalAPIConfig = apiConfig{listQuorum: 3}
+	globalAPIConfig = apiConfig{listQuorum: "strict"}
 
 	globalStorageClass storageclass.Config
 	globalLDAPConfig   xldap.Config
@@ -345,6 +345,13 @@ var (
 	globalIsCICD bool
 
 	globalRootDiskThreshold uint64
+
+	// Used for collecting stats for netperf
+	globalNetPerfMinDuration     = time.Second * 10
+	globalNetPerfRX              netPerfRX
+	globalObjectPerfBucket       = "minio-perf-test-tmp-bucket"
+	globalObjectPerfUserMetadata = "X-Amz-Meta-Minio-Object-Perf" // Clients can set this to bypass S3 API service freeze. Used by object pref tests.
+
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -38,7 +38,7 @@ type apiConfig struct {
 	requestsDeadline time.Duration
 	requestsPool     chan struct{}
 	clusterDeadline  time.Duration
-	listQuorum       int
+	listQuorum       string
 	corsAllowOrigins []string
 	// total drives per erasure set across pools.
 	totalDriveCount          int
@@ -127,7 +127,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		}
 	}
 
-	if cap(t.requestsPool) < apiRequestsMaxPerNode {
+	if cap(t.requestsPool) != apiRequestsMaxPerNode {
 		// Only replace if needed.
 		// Existing requests will use the previous limit,
 		// but new requests will use the new limit.
@@ -136,7 +136,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode)
 	}
 	t.requestsDeadline = cfg.RequestsDeadline
-	t.listQuorum = cfg.GetListQuorum()
+	t.listQuorum = cfg.ListQuorum
 	if globalReplicationPool != nil &&
 		cfg.ReplicationWorkers != t.replicationWorkers {
 		globalReplicationPool.ResizeFailedWorkers(cfg.ReplicationFailedWorkers)
@@ -170,7 +170,7 @@ func (t *apiConfig) shouldGzipObjects() bool {
 	return t.gzipObjects
 }
 
-func (t *apiConfig) getListQuorum() int {
+func (t *apiConfig) getListQuorum() string {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
@@ -246,10 +246,12 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		globalHTTPStats.incS3RequestsIncoming()
 
-		if val := globalServiceFreeze.Load(); val != nil {
-			if unlock, ok := val.(chan struct{}); ok && unlock != nil {
-				// Wait until unfrozen.
-				<-unlock
+		if r.Header.Get(globalObjectPerfUserMetadata) == "" {
+			if val := globalServiceFreeze.Load(); val != nil {
+				if unlock, ok := val.(chan struct{}); ok && unlock != nil {
+					// Wait until unfrozen.
+					<-unlock
+				}
 			}
 		}
 

cmd/httprange.go

@@ -174,3 +174,26 @@ func (h *HTTPRangeSpec) String(resourceSize int64) string {
 	}
 	return fmt.Sprintf("%d-%d", off, off+length-1)
 }
+
+// ToHeader returns the Range header value.
+func (h *HTTPRangeSpec) ToHeader() (string, error) {
+	if h == nil {
+		return "", nil
+	}
+	start := strconv.Itoa(int(h.Start))
+	end := strconv.Itoa(int(h.End))
+	switch {
+	case h.Start >= 0 && h.End >= 0:
+		if h.Start > h.End {
+			return "", errInvalidRange
+		}
+	case h.IsSuffixLength:
+		end = strconv.Itoa(int(h.Start * -1))
+		start = ""
+	case h.Start > -1:
+		end = ""
+	default:
+		return "", fmt.Errorf("does not have valid range value")
+	}
+	return fmt.Sprintf("bytes=%s-%s", start, end), nil
+}

cmd/httprange_test.go

@@ -101,3 +101,46 @@ func TestHTTPRequestRangeSpec(t *testing.T) {
 		t.Errorf("Case %d: Expected errInvalidRange but: %v %v %d %d %v", i, rs, err1, o, l, err2)
 	}
 }
+
+func TestHTTPRequestRangeToHeader(t *testing.T) {
+	validRangeSpecs := []struct {
+		spec        string
+		errExpected bool
+	}{
+		{"bytes=0-", false},
+		{"bytes=1-", false},
+
+		{"bytes=0-9", false},
+		{"bytes=1-10", false},
+		{"bytes=1-1", false},
+		{"bytes=2-5", false},
+
+		{"bytes=-5", false},
+		{"bytes=-1", false},
+		{"bytes=-1000", false},
+		{"bytes=", true},
+		{"bytes= ", true},
+		{"byte=", true},
+		{"bytes=A-B", true},
+		{"bytes=1-B", true},
+		{"bytes=B-1", true},
+		{"bytes=-1-1", true},
+	}
+	for i, testCase := range validRangeSpecs {
+		rs, err := parseRequestRangeSpec(testCase.spec)
+		if err != nil {
+			if !testCase.errExpected || err == nil && testCase.errExpected {
+				t.Errorf("unexpected err: %v", err)
+			}
+			continue
+		}
+		h, err := rs.ToHeader()
+		if err != nil && !testCase.errExpected || err == nil && testCase.errExpected {
+			t.Errorf("expected error with invalid range: %v", err)
+		}
+		if h != testCase.spec {
+			t.Errorf("Case %d: translated to incorrect header: %s expected: %s",
+				i, h, testCase.spec)
+		}
+	}
+}

cmd/iam-object-store.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"path"
 	"strings"
 	"sync"
@@ -405,6 +406,136 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
 	return nil
 }
 
+var (
+	usersListKey                   = "/users/"
+	svcAccListKey                  = "/service-accounts/"
+	groupsListKey                  = "/groups/"
+	policiesListKey                = "/policies/"
+	stsListKey                     = "/sts/"
+	policyDBUsersListKey           = "/policydb/users/"
+	policyDBSTSUsersListKey        = "/policydb/sts-users/"
+	policyDBServiceAccountsListKey = "/policydb/service-accounts/"
+	policyDBGroupsListKey          = "/policydb/groups/"
+
+	allListKeys = []string{
+		usersListKey,
+		svcAccListKey,
+		groupsListKey,
+		policiesListKey,
+		stsListKey,
+		policyDBUsersListKey,
+		policyDBSTSUsersListKey,
+		policyDBServiceAccountsListKey,
+		policyDBGroupsListKey,
+	}
+)
+
+func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[string][]string, error) {
+	res := make(map[string][]string)
+
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix) {
+		if item.Err != nil {
+			return nil, item.Err
+		}
+
+		found := false
+		for _, listKey := range allListKeys {
+			if strings.HasPrefix(item.Item, listKey) {
+				found = true
+				name := strings.TrimPrefix(item.Item, listKey)
+				res[listKey] = append(res[listKey], name)
+				break
+			}
+		}
+
+		if !found && !(item.Item == "config/config.json" || item.Item == "/format.json" || strings.Contains(item.Item, "config/history/")) {
+			logger.LogIf(ctx, fmt.Errorf("unknown type of IAM file listed: %v", item.Item))
+		}
+	}
+	return res, nil
+}
+
+// Assumes cache is locked by caller.
+func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache) error {
+	listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
+	if err != nil {
+		return err
+	}
+
+	// Loads things in the same order as `LoadIAMCache()`
+
+	policiesList := listedConfigItems[policiesListKey]
+	for _, item := range policiesList {
+		policyName := path.Dir(item)
+		if err := iamOS.loadPolicyDoc(ctx, policyName, cache.iamPolicyDocsMap); err != nil && err != errNoSuchPolicy {
+			return err
+		}
+	}
+	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
+
+	if iamOS.usersSysType == MinIOUsersSysType {
+
+		regUsersList := listedConfigItems[usersListKey]
+		for _, item := range regUsersList {
+			userName := path.Dir(item)
+			if err := iamOS.loadUser(ctx, userName, regUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
+				return err
+			}
+		}
+
+		groupsList := listedConfigItems[groupsListKey]
+		for _, item := range groupsList {
+			group := path.Dir(item)
+			if err := iamOS.loadGroup(ctx, group, cache.iamGroupsMap); err != nil && err != errNoSuchGroup {
+				return err
+			}
+		}
+	}
+
+	userPolicyMappingsList := listedConfigItems[policyDBUsersListKey]
+	for _, item := range userPolicyMappingsList {
+		userName := strings.TrimSuffix(item, ".json")
+		if err := iamOS.loadMappedPolicy(ctx, userName, regUser, false, cache.iamUserPolicyMap); err != nil && err != errNoSuchPolicy {
+			return err
+		}
+	}
+
+	groupPolicyMappingsList := listedConfigItems[policyDBGroupsListKey]
+	for _, item := range groupPolicyMappingsList {
+		groupName := strings.TrimSuffix(item, ".json")
+		if err := iamOS.loadMappedPolicy(ctx, groupName, regUser, true, cache.iamGroupPolicyMap); err != nil && err != errNoSuchPolicy {
+			return err
+		}
+	}
+
+	svcAccList := listedConfigItems[svcAccListKey]
+	for _, item := range svcAccList {
+		userName := path.Dir(item)
+		if err := iamOS.loadUser(ctx, userName, svcUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
+			return err
+		}
+	}
+
+	stsUsersList := listedConfigItems[stsListKey]
+	for _, item := range stsUsersList {
+		userName := path.Dir(item)
+		if err := iamOS.loadUser(ctx, userName, stsUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
+			return err
+		}
+	}
+
+	stsPolicyMappingsList := listedConfigItems[policyDBSTSUsersListKey]
+	for _, item := range stsPolicyMappingsList {
+		stsName := strings.TrimSuffix(item, ".json")
+		if err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, cache.iamUserPolicyMap); err != nil && err != errNoSuchPolicy {
+			return err
+		}
+	}
+
+	cache.buildUserGroupMemberships()
+	return nil
+}
+
 func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
 	return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
 }

cmd/iam-store.go

@@ -432,48 +432,56 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
 	cache := store.lock()
 	defer store.unlock()
 
-	if err := store.loadPolicyDocs(ctx, newCache.iamPolicyDocsMap); err != nil {
-		return err
-	}
-
-	// Sets default canned policies, if none are set.
-	setDefaultCannedPolicies(newCache.iamPolicyDocsMap)
-
-	if store.getUsersSysType() == MinIOUsersSysType {
-		if err := store.loadUsers(ctx, regUser, newCache.iamUsersMap); err != nil {
+	if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok {
+		err := iamOS.loadAllFromObjStore(ctx, newCache)
+		if err != nil {
 			return err
 		}
-		if err := store.loadGroups(ctx, newCache.iamGroupsMap); err != nil {
+	} else {
+
+		if err := store.loadPolicyDocs(ctx, newCache.iamPolicyDocsMap); err != nil {
 			return err
 		}
-	}
 
-	// load polices mapped to users
-	if err := store.loadMappedPolicies(ctx, regUser, false, newCache.iamUserPolicyMap); err != nil {
-		return err
-	}
+		// Sets default canned policies, if none are set.
+		setDefaultCannedPolicies(newCache.iamPolicyDocsMap)
 
-	// load policies mapped to groups
-	if err := store.loadMappedPolicies(ctx, regUser, true, newCache.iamGroupPolicyMap); err != nil {
-		return err
-	}
+		if store.getUsersSysType() == MinIOUsersSysType {
+			if err := store.loadUsers(ctx, regUser, newCache.iamUsersMap); err != nil {
+				return err
+			}
+			if err := store.loadGroups(ctx, newCache.iamGroupsMap); err != nil {
+				return err
+			}
+		}
 
-	// load service accounts
-	if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil {
-		return err
-	}
+		// load polices mapped to users
+		if err := store.loadMappedPolicies(ctx, regUser, false, newCache.iamUserPolicyMap); err != nil {
+			return err
+		}
 
-	// load STS temp users
-	if err := store.loadUsers(ctx, stsUser, newCache.iamUsersMap); err != nil {
-		return err
-	}
+		// load policies mapped to groups
+		if err := store.loadMappedPolicies(ctx, regUser, true, newCache.iamGroupPolicyMap); err != nil {
+			return err
+		}
 
-	// load STS policy mappings
-	if err := store.loadMappedPolicies(ctx, stsUser, false, newCache.iamUserPolicyMap); err != nil {
-		return err
-	}
+		// load service accounts
+		if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil {
+			return err
+		}
 
-	newCache.buildUserGroupMemberships()
+		// load STS temp users
+		if err := store.loadUsers(ctx, stsUser, newCache.iamUsersMap); err != nil {
+			return err
+		}
+
+		// load STS policy mappings
+		if err := store.loadMappedPolicies(ctx, stsUser, false, newCache.iamUserPolicyMap); err != nil {
+			return err
+		}
+
+		newCache.buildUserGroupMemberships()
+	}
 
 	cache.iamGroupPolicyMap = newCache.iamGroupPolicyMap
 	cache.iamGroupsMap = newCache.iamGroupsMap
@@ -1456,13 +1464,28 @@ func (store *IAMStoreSys) GetAllParentUsers() map[string]string {
 
 	res := map[string]string{}
 	for _, cred := range cache.iamUsersMap {
-		if cred.IsServiceAccount() || cred.IsTemp() {
+		if (cred.IsServiceAccount() || cred.IsTemp()) && cred.SessionToken != "" {
 			parentUser := cred.ParentUser
-			if cred.SessionToken != "" {
-				claims, err := getClaimsFromToken(cred.SessionToken)
+
+			var (
+				err    error
+				claims map[string]interface{}
+			)
+
+			if cred.IsServiceAccount() {
+				claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, cred.SecretKey)
 				if err != nil {
-					continue
+					claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, globalActiveCred.SecretKey)
 				}
+			} else if cred.IsTemp() {
+				claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, globalActiveCred.SecretKey)
+			}
+
+			if err != nil {
+				continue
+			}
+
+			if len(claims) > 0 {
 				if v, ok := claims[subClaim]; ok {
 					subFromToken, ok := v.(string)
 					if ok {
@@ -1470,6 +1493,11 @@ func (store *IAMStoreSys) GetAllParentUsers() map[string]string {
 					}
 				}
 			}
+
+			if parentUser == "" {
+				continue
+			}
+
 			if _, ok := res[parentUser]; !ok {
 				res[parentUser] = cred.ParentUser
 			}
@@ -1561,6 +1589,7 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		return errNoSuchServiceAccount
 	}
 
+	currentSecretKey := cr.SecretKey
 	if opts.secretKey != "" {
 		if !auth.IsSecretKeyValid(opts.secretKey) {
 			return auth.ErrInvalidSecretKeyLength
@@ -1582,20 +1611,21 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		return errors.New("unknown account status value")
 	}
 
-	if opts.sessionPolicy != nil {
-		m, err := getClaimsFromToken(cr.SessionToken)
-		if err != nil {
-			return fmt.Errorf("unable to get svc acc claims: %v", err)
-		}
+	m, err := getClaimsFromTokenWithSecret(cr.SessionToken, currentSecretKey)
+	if err != nil {
+		return fmt.Errorf("unable to get svc acc claims: %v", err)
+	}
 
-		err = opts.sessionPolicy.Validate()
-		if err != nil {
+	if opts.sessionPolicy != nil {
+		if err := opts.sessionPolicy.Validate(); err != nil {
 			return err
 		}
+
 		policyBuf, err := json.Marshal(opts.sessionPolicy)
 		if err != nil {
 			return err
 		}
+
 		if len(policyBuf) > 16*humanize.KiByte {
 			return fmt.Errorf("Session policy should not exceed 16 KiB characters")
 		}
@@ -1603,10 +1633,11 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		// Overwrite session policy claims.
 		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
 		m[iamPolicyClaimNameSA()] = "embedded-policy"
-		cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, globalActiveCred.SecretKey)
-		if err != nil {
-			return err
-		}
+	}
+
+	cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, cr.SecretKey)
+	if err != nil {
+		return err
 	}
 
 	u := newUserIdentity(cr)

cmd/iam.go

@@ -29,6 +29,7 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
@@ -37,6 +38,7 @@ import (
 	"github.com/minio/minio/internal/arn"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	etcd "go.etcd.io/etcd/client/v3"
@@ -76,6 +78,12 @@ type IAMSys struct {
 
 	// configLoaded will be closed and remain so after first load.
 	configLoaded chan struct{}
+
+	// metrics
+	LastRefreshTimeUnixNano         uint64
+	LastRefreshDurationMilliseconds uint64
+	TotalRefreshSuccesses           uint64
+	TotalRefreshFailures            uint64
 }
 
 // IAMUserType represents a user type inside MinIO server
@@ -183,10 +191,17 @@ func (sys *IAMSys) Initialized() bool {
 
 // Load - loads all credentials, policies and policy mappings.
 func (sys *IAMSys) Load(ctx context.Context) error {
+	loadStartTime := time.Now()
 	err := sys.store.LoadIAMCache(ctx)
 	if err != nil {
+		atomic.AddUint64(&sys.TotalRefreshFailures, 1)
 		return err
 	}
+	loadDuration := time.Since(loadStartTime)
+
+	atomic.StoreUint64(&sys.LastRefreshDurationMilliseconds, uint64(loadDuration.Milliseconds()))
+	atomic.StoreUint64(&sys.LastRefreshTimeUnixNano, uint64(loadStartTime.Add(loadDuration).UnixNano()))
+	atomic.AddUint64(&sys.TotalRefreshSuccesses, 1)
 
 	select {
 	case <-sys.configLoaded:
@@ -198,6 +213,8 @@ func (sys *IAMSys) Load(ctx context.Context) error {
 
 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, iamRefreshInterval time.Duration) {
+	iamInitStart := time.Now()
+
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -268,6 +285,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		break
 	}
 
+	iamLoadStart := time.Now()
+
 	// Load IAM data from storage.
 	for {
 		if err := sys.Load(retryCtx); err != nil {
@@ -332,6 +351,9 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	}
 
 	sys.printIAMRoles()
+
+	now := time.Now()
+	logger.Info("Finished loading IAM sub-system (took %.1fs of %.1fs to load data).", now.Sub(iamLoadStart).Seconds(), now.Sub(iamInitStart).Seconds())
 }
 
 // Prints IAM role ARNs.
@@ -363,22 +385,33 @@ func (sys *IAMSys) watch(ctx context.Context) {
 	if ok {
 		ch := watcher.watch(ctx, iamConfigPrefix)
 		for event := range ch {
-			// we simply log errors
-			err := sys.loadWatchedEvent(ctx, event)
-			logger.LogIf(ctx, err)
+			if err := sys.loadWatchedEvent(ctx, event); err != nil {
+				// we simply log errors
+				logger.LogIf(ctx, fmt.Errorf("Failure in loading watch event: %v", err))
+			}
 		}
 		return
 	}
 
+	var maxRefreshDurationSecondsForLog float64 = 10
+
 	// Fall back to loading all items periodically
 	ticker := time.NewTicker(sys.iamRefreshInterval)
 	defer ticker.Stop()
 	for {
 		select {
 		case <-ticker.C:
+			refreshStart := time.Now()
 			if err := sys.Load(ctx); err != nil {
-				logger.LogIf(ctx, err)
+				logger.LogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (took %.2fs): %v", time.Since(refreshStart).Seconds(), err))
+			} else {
+				took := time.Since(refreshStart).Seconds()
+				if took > maxRefreshDurationSecondsForLog {
+					// Log if we took a lot of time to load.
+					logger.Info("IAM refresh took %.2fs", took)
+				}
 			}
+
 		case <-ctx.Done():
 			return
 		}
@@ -799,14 +832,17 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 		}
 	}
 
-	var cred auth.Credentials
-
+	var accessKey, secretKey string
 	var err error
 	if len(opts.accessKey) > 0 {
-		cred, err = auth.CreateNewCredentialsWithMetadata(opts.accessKey, opts.secretKey, m, globalActiveCred.SecretKey)
+		accessKey, secretKey = opts.accessKey, opts.secretKey
 	} else {
-		cred, err = auth.GetNewCredentialsWithMetadata(m, globalActiveCred.SecretKey)
+		accessKey, secretKey, err = auth.GenerateCredentials()
+		if err != nil {
+			return auth.Credentials{}, err
+		}
 	}
+	cred, err := auth.CreateNewCredentialsWithMetadata(accessKey, secretKey, m, secretKey)
 	if err != nil {
 		return auth.Credentials{}, err
 	}
@@ -880,9 +916,12 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (aut
 
 	var embeddedPolicy *iampolicy.Policy
 
-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
 	if err != nil {
-		return auth.Credentials{}, nil, err
+		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		if err != nil {
+			return auth.Credentials{}, nil, err
+		}
 	}
 	pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
 	sp, spok := jwtClaims.Lookup(iampolicy.SessionPolicyName)
@@ -915,9 +954,12 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
 		return nil, errNoSuchServiceAccount
 	}
 
-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
 	if err != nil {
-		return nil, err
+		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		if err != nil {
+			return nil, err
+		}
 	}
 	return jwtClaims.Map(), nil
 }
@@ -1063,12 +1105,28 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 		if _, ok := parentUserToCredsMap[cred.ParentUser]; !ok {
 			// Try to find the ldapUsername for this
 			// parentUser by extracting JWT claims
-			jwtClaims, err := auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
-			if err != nil {
-				// skip this cred - session token seems
-				// invalid
+			var (
+				jwtClaims *jwt.MapClaims
+				err       error
+			)
+
+			if cred.SessionToken == "" {
 				continue
 			}
+
+			if cred.IsServiceAccount() {
+				jwtClaims, err = auth.ExtractClaims(cred.SessionToken, cred.SecretKey)
+				if err != nil {
+					jwtClaims, err = auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
+				}
+			} else {
+				jwtClaims, err = auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
+			}
+			if err != nil {
+				// skip this cred - session token seems invalid
+				continue
+			}
+
 			ldapUsername, ok := jwtClaims.Lookup(ldapUserN)
 			if !ok {
 				// skip this cred - we dont have the

cmd/jwt.go

@@ -132,7 +132,7 @@ func authenticateURL(accessKey, secretKey string) (string, error) {
 // Check if the request is authenticated.
 // Returns nil if the request is authenticated. errNoAuthToken if token missing.
 // Returns errAuthentication for all other errors.
-func webRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, bool, error) {
+func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, bool, error) {
 	token, err := jwtreq.AuthorizationHeaderExtractor.ExtractToken(req)
 	if err != nil {
 		if err == jwtreq.ErrNoTokenInRequest {

cmd/jwt_test.go

@@ -149,7 +149,7 @@ func TestWebRequestAuthenticate(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		_, _, _, gotErr := webRequestAuthenticate(testCase.req)
+		_, _, _, gotErr := metricsRequestAuthenticate(testCase.req)
 		if testCase.expectedErr != gotErr {
 			t.Errorf("Test %d, expected err %s, got %s", i+1, testCase.expectedErr, gotErr)
 		}

cmd/metacache-bucket_test.go

@@ -42,7 +42,7 @@ func Benchmark_bucketMetacache_findCache(b *testing.B) {
 			FilterPrefix: "",
 			Marker:       "",
 			Limit:        0,
-			AskDisks:     0,
+			AskDisks:     "strict",
 			Recursive:    false,
 			Separator:    slashSeparator,
 			Create:       true,
@@ -59,7 +59,7 @@ func Benchmark_bucketMetacache_findCache(b *testing.B) {
 			FilterPrefix: "",
 			Marker:       "",
 			Limit:        0,
-			AskDisks:     0,
+			AskDisks:     "strict",
 			Recursive:    false,
 			Separator:    slashSeparator,
 			Create:       true,

cmd/metacache-entries.go

@@ -191,6 +191,16 @@ func (e *metaCacheEntry) fileInfo(bucket string) (FileInfo, error) {
 		}, nil
 	}
 	if e.cached != nil {
+		if len(e.cached.versions) == 0 {
+			// This special case is needed to handle xlMeta.versions == 0
+			return FileInfo{
+				Volume:   bucket,
+				Name:     e.name,
+				Deleted:  true,
+				IsLatest: true,
+				ModTime:  timeSentinel1970,
+			}, nil
+		}
 		return e.cached.ToFileInfo(bucket, e.name, "")
 	}
 	return getFileInfo(e.metadata, bucket, e.name, "", false)
@@ -271,10 +281,16 @@ func (m metaCacheEntries) shallowClone() metaCacheEntries {
 }
 
 type metadataResolutionParams struct {
-	dirQuorum int    // Number if disks needed for a directory to 'exist'.
-	objQuorum int    // Number of disks needed for an object to 'exist'.
-	bucket    string // Name of the bucket. Used for generating cached fileinfo.
-	strict    bool   // Versions must match exactly, including all metadata.
+	dirQuorum int // Number if disks needed for a directory to 'exist'.
+	objQuorum int // Number of disks needed for an object to 'exist'.
+
+	// An optimization request only an 'n' amount of versions from xl.meta
+	// to avoid resolving all versions to figure out the latest 'version'
+	// for ListObjects, ListObjectsV2
+	requestedVersions int
+
+	bucket string // Name of the bucket. Used for generating cached fileinfo.
+	strict bool   // Versions must match exactly, including all metadata.
 
 	// Reusable slice for resolution
 	candidates [][]xlMetaV2ShallowVersion
@@ -362,7 +378,7 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 		reusable: true,
 		cached:   &xlMetaV2{metaV: selected.cached.metaV},
 	}
-	selected.cached.versions = mergeXLV2Versions(r.objQuorum, r.strict, r.candidates...)
+	selected.cached.versions = mergeXLV2Versions(r.objQuorum, r.strict, r.requestedVersions, r.candidates...)
 	if len(selected.cached.versions) == 0 {
 		return nil, false
 	}
@@ -379,11 +395,11 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 
 // firstFound returns the first found and the number of set entries.
 func (m metaCacheEntries) firstFound() (first *metaCacheEntry, n int) {
-	for _, entry := range m {
+	for i, entry := range m {
 		if entry.name != "" {
 			n++
 			if first == nil {
-				first = &entry
+				first = &m[i]
 			}
 		}
 	}
@@ -488,7 +504,7 @@ func (m *metaCacheEntriesSorted) fileInfoVersions(bucket, prefix, delimiter, aft
 	return versions
 }
 
-// fileInfoVersions converts the metadata to FileInfoVersions where possible.
+// fileInfos converts the metadata to ObjectInfo where possible.
 // Metadata that cannot be decoded is skipped.
 func (m *metaCacheEntriesSorted) fileInfos(bucket, prefix, delimiter string) (objects []ObjectInfo) {
 	objects = make([]ObjectInfo, 0, m.len())

cmd/metacache-server-pool.go

@@ -28,6 +28,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -110,7 +111,7 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 	// If we don't have a list id we must ask the server if it has a cache or create a new.
 	if o.ID != "" && !o.Transient {
 		// Create or ping with handout...
-		rpc := globalNotificationSys.restClientFromHash(o.Bucket)
+		rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
 		ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
 		defer cancel()
 		var c *metacache
@@ -198,7 +199,7 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 		}
 		entries.truncate(0)
 		go func() {
-			rpc := globalNotificationSys.restClientFromHash(o.Bucket)
+			rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
 			if rpc != nil {
 				ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
 				defer cancel()
@@ -289,6 +290,14 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 	}
 	mu.Unlock()
 
+	// Do lifecycle filtering.
+	if o.lcFilter != nil {
+		filterIn := make(chan metaCacheEntry, 10)
+		go filterLifeCycle(ctx, o.Bucket, o.lcFilter, filterIn, results)
+		// Replace results.
+		results = filterIn
+	}
+
 	// Gather results to a single channel.
 	err := mergeEntryChannels(ctx, inputs, results, func(existing, other *metaCacheEntry) (replace bool) {
 		// Pick object over directory
@@ -298,21 +307,20 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 		if !existing.isDir() && other.isDir() {
 			return false
 		}
-
-		eFIV, err := existing.fileInfo(o.Bucket)
+		eMeta, err := existing.xlmeta()
 		if err != nil {
 			return true
 		}
-		oFIV, err := other.fileInfo(o.Bucket)
+		oMeta, err := other.xlmeta()
 		if err != nil {
 			return false
 		}
 		// Replace if modtime is newer
-		if !oFIV.ModTime.Equal(eFIV.ModTime) {
-			return oFIV.ModTime.After(eFIV.ModTime)
+		if !oMeta.latestModtime().Equal(oMeta.latestModtime()) {
+			return oMeta.latestModtime().After(eMeta.latestModtime())
 		}
 		// Use NumVersions as a final tiebreaker.
-		return oFIV.NumVersions > eFIV.NumVersions
+		return len(oMeta.versions) > len(eMeta.versions)
 	})
 
 	cancelList()
@@ -346,6 +354,44 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 	return nil
 }
 
+// filterLifeCycle will filter out objects if the most recent
+// version should be deleted by lifecycle.
+// out will be closed when there are no more results.
+// When 'in' is closed or the context is canceled the
+// function closes 'out' and exits.
+func filterLifeCycle(ctx context.Context, bucket string, lc *lifecycle.Lifecycle, in <-chan metaCacheEntry, out chan<- metaCacheEntry) {
+	defer close(out)
+	for {
+		var obj metaCacheEntry
+		var ok bool
+		select {
+		case <-ctx.Done():
+			return
+		case obj, ok = <-in:
+			if !ok {
+				return
+			}
+		}
+
+		fi, err := obj.fileInfo(bucket)
+		if err != nil {
+			continue
+		}
+		objInfo := fi.ToObjectInfo(bucket, obj.name)
+		action := evalActionFromLifecycle(ctx, *lc, objInfo, false)
+		switch action {
+		case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
+			// Skip this entry.
+			continue
+		}
+		select {
+		case <-ctx.Done():
+			return
+		case out <- obj:
+		}
+	}
+}
+
 func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions) (entries metaCacheEntriesSorted, err error) {
 	// Use ID as the object name...
 	o.pool = z.getAvailablePoolIdx(ctx, minioMetaBucket, o.ID, 10<<20)
@@ -369,7 +415,7 @@ func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions
 	filteredResults := o.gatherResults(ctx, outCh)
 
 	mc := o.newMetacache()
-	meta := metaCacheRPC{meta: &mc, cancel: cancel, rpc: globalNotificationSys.restClientFromHash(o.Bucket), o: *o}
+	meta := metaCacheRPC{meta: &mc, cancel: cancel, rpc: globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix)), o: *o}
 
 	// Save listing...
 	go func() {

cmd/metacache-set.go

@@ -32,6 +32,7 @@ import (
 	"time"
 
 	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
@@ -64,10 +65,8 @@ type listPathOptions struct {
 	// Limit the number of results.
 	Limit int
 
-	// The number of disks to ask. Special values:
-	// 0 uses default number of disks.
-	// -1 use at least 50% of disks or at least the default number.
-	AskDisks int
+	// The number of disks to ask.
+	AskDisks string
 
 	// InclDeleted will keep all entries where latest version is a delete marker.
 	InclDeleted bool
@@ -96,6 +95,11 @@ type listPathOptions struct {
 
 	// pool and set of where the cache is located.
 	pool, set int
+
+	// lcFilter performs filtering based on lifecycle.
+	// This will filter out objects if the most recent version should be deleted by lifecycle.
+	// Is not transferred across request calls.
+	lcFilter *lifecycle.Lifecycle
 }
 
 func init() {
@@ -189,7 +193,11 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
 		}
 		if resCh != nil {
 			resErr = io.EOF
-			resCh <- results
+			select {
+			case <-ctx.Done():
+				// Nobody wants it.
+			case resCh <- results:
+			}
 		}
 	}()
 	return func() (metaCacheEntriesSorted, error) {
@@ -351,7 +359,7 @@ func (r *metacacheReader) filter(o listPathOptions) (entries metaCacheEntriesSor
 
 func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) {
 	retries := 0
-	rpc := globalNotificationSys.restClientFromHash(o.Bucket)
+	rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix))
 
 	for {
 		if contextCanceled(ctx) {
@@ -531,21 +539,43 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 	}
 }
 
+// getListQuorum interprets list quorum values and returns appropriate
+// acceptable quorum expected for list operations
+func getListQuorum(quorum string, driveCount int) int {
+	switch quorum {
+	case "disk":
+		// smallest possible value, generally meant for testing.
+		return 1
+	case "reduced":
+		return 2
+	case "strict":
+		return driveCount
+	}
+	// Defaults to (driveCount+1)/2 drives per set, defaults to "optimal" value
+	if driveCount > 0 {
+		return (driveCount + 1) / 2
+	} // "3" otherwise.
+	return 3
+}
+
 // Will return io.EOF if continuing would not yield more results.
 func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) (err error) {
 	defer close(results)
 	o.debugf(color.Green("listPath:")+" with options: %#v", o)
 
-	askDisks := o.AskDisks
-	listingQuorum := o.AskDisks - 1
-	disks := er.getDisks()
+	// get non-healing disks for listing
+	disks, _ := er.getOnlineDisksWithHealing()
+	askDisks := getListQuorum(o.AskDisks, er.setDriveCount)
 	var fallbackDisks []StorageAPI
 
 	// Special case: ask all disks if the drive count is 4
-	if askDisks <= 0 || er.setDriveCount == 4 {
-		askDisks = len(disks)          // with 'strict' quorum list on all online disks.
-		listingQuorum = len(disks) / 2 // keep this such that we can list all objects with different quorum ratio.
+	if er.setDriveCount == 4 || askDisks > len(disks) {
+		askDisks = len(disks) // use all available drives
 	}
+
+	// However many we ask, versions must exist on ~50%
+	listingQuorum := (askDisks + 1) / 2
+
 	if askDisks > 0 && len(disks) > askDisks {
 		rand.Shuffle(len(disks), func(i, j int) {
 			disks[i], disks[j] = disks[j], disks[i]
@@ -561,6 +591,13 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul
 		bucket:    o.Bucket,
 	}
 
+	// Maximum versions requested for "latest" object
+	// resolution on versioned buckets, this is to be only
+	// used when o.Versioned is false
+	if !o.Versioned {
+		resolver.requestedVersions = 1
+	}
+
 	ctxDone := ctx.Done()
 	return listPathRaw(ctx, listPathRawOptions{
 		disks:         disks,
@@ -789,14 +826,30 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	fallback := func(err error) bool {
+	// Keep track of fallback disks
+	var fdMu sync.Mutex
+	fds := opts.fallbackDisks
+	fallback := func(err error) StorageAPI {
 		switch err.(type) {
 		case StorageErr:
-			// all supported disk errors
-			// attempt a fallback.
-			return true
+			// Attempt to grab a fallback disk
+			fdMu.Lock()
+			defer fdMu.Unlock()
+			if len(fds) == 0 {
+				return nil
+			}
+			fdsCopy := fds
+			for _, fd := range fdsCopy {
+				// Grab a fallback disk
+				fds = fds[1:]
+				if fd != nil && fd.IsOnline() {
+					return fd
+				}
+			}
 		}
-		return false
+		// Either no more disks for fallback or
+		// not a storage error.
+		return nil
 	}
 	askDisks := len(disks)
 	readers := make([]*metacacheReader, askDisks)
@@ -825,25 +878,20 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 			}
 
 			// fallback only when set.
-			if len(opts.fallbackDisks) > 0 && fallback(werr) {
+			for fd := fallback(werr); fd != nil; {
 				// This fallback is only set when
 				// askDisks is less than total
 				// number of disks per set.
-				for _, fd := range opts.fallbackDisks {
-					if fd == nil {
-						continue
-					}
-					werr = fd.WalkDir(ctx, WalkDirOptions{
-						Bucket:         opts.bucket,
-						BaseDir:        opts.path,
-						Recursive:      opts.recursive,
-						ReportNotFound: opts.reportNotFound,
-						FilterPrefix:   opts.filterPrefix,
-						ForwardTo:      opts.forwardTo,
-					}, w)
-					if werr == nil {
-						break
-					}
+				werr = fd.WalkDir(ctx, WalkDirOptions{
+					Bucket:         opts.bucket,
+					BaseDir:        opts.path,
+					Recursive:      opts.recursive,
+					ReportNotFound: opts.reportNotFound,
+					FilterPrefix:   opts.filterPrefix,
+					ForwardTo:      opts.forwardTo,
+				}, w)
+				if werr == nil {
+					break
 				}
 			}
 			w.CloseWithError(werr)

cmd/metacache-walk.go

@@ -138,6 +138,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			// Forward some errors?
 			return nil
 		}
+		diskHealthCheckOK(ctx, err)
 		if len(entries) == 0 {
 			return nil
 		}
@@ -179,6 +180,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				s.walkReadMu.Lock()
 				meta.metadata, err = s.readMetadata(ctx, pathJoin(volumeDir, current, entry))
 				s.walkReadMu.Unlock()
+				diskHealthCheckOK(ctx, err)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					continue
@@ -196,6 +198,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				s.walkReadMu.Lock()
 				meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, current, entry))
 				s.walkReadMu.Unlock()
+				diskHealthCheckOK(ctx, err)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					continue
@@ -257,6 +260,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			s.walkReadMu.Lock()
 			meta.metadata, err = s.readMetadata(ctx, pathJoin(volumeDir, meta.name, xlStorageFormatFile))
 			s.walkReadMu.Unlock()
+			diskHealthCheckOK(ctx, err)
 			switch {
 			case err == nil:
 				// It was an object
@@ -266,6 +270,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				out <- meta
 			case osIsNotExist(err), isSysErrIsDir(err):
 				meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1))
+				diskHealthCheckOK(ctx, err)
 				if err == nil {
 					// It was an object
 					out <- meta
@@ -303,16 +308,12 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	return scanDir(opts.BaseDir)
 }
 
-func (p *xlStorageDiskIDCheck) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error {
-	defer p.updateStorageMetrics(storageMetricWalkDir, opts.Bucket, opts.BaseDir)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWalkDir, opts.Bucket, opts.BaseDir)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.WalkDir(ctx, opts, wr)
 }

cmd/metrics-v2.go

@@ -62,6 +62,7 @@ func init() {
 		getS3TTFBMetric(),
 		getILMNodeMetrics(),
 		getScannerNodeMetrics(),
+		getIAMNodeMetrics(),
 	}
 
 	allMetricsGroups := func() (allMetrics []*MetricsGroup) {
@@ -121,6 +122,7 @@ const (
 	quotaSubsystem            MetricSubsystem = "quota"
 	ilmSubsystem              MetricSubsystem = "ilm"
 	scannerSubsystem          MetricSubsystem = "scanner"
+	iamSubsystem              MetricSubsystem = "iam"
 )
 
 // MetricName are the individual names for the metric.
@@ -1266,6 +1268,62 @@ func getScannerNodeMetrics() *MetricsGroup {
 	return mg
 }
 
+func getIAMNodeMetrics() *MetricsGroup {
+	mg := &MetricsGroup{}
+	mg.RegisterRead(func(_ context.Context) (metrics []Metric) {
+		lastSyncTime := atomic.LoadUint64(&globalIAMSys.LastRefreshTimeUnixNano)
+		var sinceLastSyncMillis uint64
+		if lastSyncTime != 0 {
+			sinceLastSyncMillis = (uint64(time.Now().UnixNano()) - lastSyncTime) / uint64(time.Millisecond)
+		}
+
+		metrics = []Metric{
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "last_sync_duration_millis",
+					Help:      "Last successful IAM data sync duration in milliseconds",
+					Type:      gaugeMetric,
+				},
+				Value: float64(atomic.LoadUint64(&globalIAMSys.LastRefreshDurationMilliseconds)),
+			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "since_last_sync_millis",
+					Help:      "Time (in milliseconds) since last successful IAM data sync. This is set to 0 until the first sync after server start.",
+					Type:      gaugeMetric,
+				},
+				Value: float64(sinceLastSyncMillis),
+			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "sync_successes",
+					Help:      "Number of successful IAM data syncs since server start.",
+					Type:      counterMetric,
+				},
+				Value: float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshSuccesses)),
+			},
+			{
+				Description: MetricDescription{
+					Namespace: nodeMetricNamespace,
+					Subsystem: iamSubsystem,
+					Name:      "sync_failures",
+					Help:      "Number of failed IAM data syncs since server start.",
+					Type:      counterMetric,
+				},
+				Value: float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshFailures)),
+			},
+		}
+		return metrics
+	})
+	return mg
+}
+
 func getMinioVersionMetrics() *MetricsGroup {
 	mg := &MetricsGroup{}
 	mg.RegisterRead(func(_ context.Context) (metrics []Metric) {
@@ -1651,45 +1709,24 @@ func getClusterTierMetrics() *MetricsGroup {
 		cacheInterval: 10 * time.Second,
 	}
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
-		if globalTierConfigMgr.Empty() {
-			return
-		}
 		objLayer := newObjectLayerFn()
 		if objLayer == nil || globalIsGateway {
 			return
 		}
+		if globalTierConfigMgr.Empty() {
+			return
+		}
 
 		dui, err := loadDataUsageFromBackend(GlobalContext, objLayer)
 		if err != nil {
 			return
 		}
-		// data usage has not captured any data yet.
-		if dui.LastUpdate.IsZero() {
+		// data usage has not captured any tier stats yet.
+		if dui.TierStats == nil {
 			return
 		}
 
-		// e.g minio_cluster_ilm_transitioned_bytes{tier="S3TIER-1"}=136314880
-		//     minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1
-		//     minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3
-		for tier, st := range dui.TierStats.Tiers {
-			metrics = append(metrics, Metric{
-				Description:    getClusterTransitionedBytesMD(),
-				Value:          float64(st.TotalSize),
-				VariableLabels: map[string]string{"tier": tier},
-			})
-			metrics = append(metrics, Metric{
-				Description:    getClusterTransitionedObjectsMD(),
-				Value:          float64(st.NumObjects),
-				VariableLabels: map[string]string{"tier": tier},
-			})
-			metrics = append(metrics, Metric{
-				Description:    getClusterTransitionedVersionsMD(),
-				Value:          float64(st.NumVersions),
-				VariableLabels: map[string]string{"tier": tier},
-			})
-		}
-
-		return metrics
+		return dui.tierMetrics()
 	})
 	return mg
 }

cmd/metrics.go

@@ -674,7 +674,7 @@ func metricsHandler() http.Handler {
 // AuthMiddleware checks if the bearer token is valid and authorized.
 func AuthMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		claims, groups, owner, authErr := webRequestAuthenticate(r)
+		claims, groups, owner, authErr := metricsRequestAuthenticate(r)
 		if authErr != nil || !claims.VerifyIssuer("prometheus", true) {
 			w.WriteHeader(http.StatusForbidden)
 			return

cmd/mrf.go

@@ -185,7 +185,7 @@ func (m *mrfState) healRoutine() {
 	defer idler.Stop()
 
 	mrfHealingOpts := madmin.HealOpts{
-		ScanMode: globalHealConfig.ScanMode(),
+		ScanMode: madmin.HealNormalScan,
 		Remove:   healDeleteDangling,
 	}
 

cmd/namespace-lock.go

@@ -170,6 +170,10 @@ func (di *distLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeout
 	}) {
 		timeout.LogFailure()
 		cancel()
+		switch err := newCtx.Err(); err {
+		case context.Canceled:
+			return LockContext{ctx: ctx, cancel: func() {}}, err
+		}
 		return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{}
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
@@ -195,6 +199,10 @@ func (di *distLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeou
 	}) {
 		timeout.LogFailure()
 		cancel()
+		switch err := newCtx.Err(); err {
+		case context.Canceled:
+			return LockContext{ctx: ctx, cancel: func() {}}, err
+		}
 		return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{}
 	}
 	timeout.LogSuccess(UTCNow().Sub(start))
@@ -247,6 +255,10 @@ func (li *localLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeou
 					li.ns.unlock(li.volume, li.paths[si], readLock)
 				}
 			}
+			switch err := ctx.Err(); err {
+			case context.Canceled:
+				return LockContext{}, err
+			}
 			return LockContext{}, OperationTimedOut{}
 		}
 		success[i] = 1
@@ -280,6 +292,10 @@ func (li *localLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeo
 					li.ns.unlock(li.volume, li.paths[si], readLock)
 				}
 			}
+			switch err := ctx.Err(); err {
+			case context.Canceled:
+				return LockContext{}, err
+			}
 			return LockContext{}, OperationTimedOut{}
 		}
 		success[i] = 1

cmd/naughty-disk_test.go

@@ -22,6 +22,8 @@ import (
 	"io"
 	"sync"
 	"time"
+
+	"github.com/minio/madmin-go"
 )
 
 // naughtyDisk wraps a POSIX disk and returns programmed errors
@@ -110,8 +112,8 @@ func (d *naughtyDisk) SetDiskID(id string) {
 	d.disk.SetDiskID(id)
 }
 
-func (d *naughtyDisk) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (info dataUsageCache, err error) {
-	return d.disk.NSScanner(ctx, cache, updates)
+func (d *naughtyDisk) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode) (info dataUsageCache, err error) {
+	return d.disk.NSScanner(ctx, cache, updates, scanMode)
 }
 
 func (d *naughtyDisk) DiskInfo(ctx context.Context) (info DiskInfo, err error) {

cmd/notification.go

@@ -1553,6 +1553,58 @@ func (sys *NotificationSys) ServiceFreeze(ctx context.Context, freeze bool) []No
 	return nerrs
 }
 
+// Netperf - perform mesh style network throughput test
+func (sys *NotificationSys) Netperf(ctx context.Context, duration time.Duration) []madmin.NetperfNodeResult {
+	length := len(sys.allPeerClients)
+	if length == 0 {
+		// For single node erasure setup.
+		return nil
+	}
+	results := make([]madmin.NetperfNodeResult, length)
+
+	scheme := "http"
+	if globalIsTLS {
+		scheme = "https"
+	}
+
+	var wg sync.WaitGroup
+	for index := range sys.peerClients {
+		if sys.peerClients[index] == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(index int) {
+			defer wg.Done()
+			r, err := sys.peerClients[index].Netperf(ctx, duration)
+			u := &url.URL{
+				Scheme: scheme,
+				Host:   sys.peerClients[index].host.String(),
+			}
+			if err != nil {
+				results[index].Error = err.Error()
+			} else {
+				results[index] = r
+			}
+			results[index].Endpoint = u.String()
+		}(index)
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		r := netperf(ctx, duration)
+		u := &url.URL{
+			Scheme: scheme,
+			Host:   globalLocalNodeName,
+		}
+		results[len(results)-1] = r
+		results[len(results)-1].Endpoint = u.String()
+	}()
+	wg.Wait()
+
+	return results
+}
+
 // Speedtest run GET/PUT tests at input concurrency for requested object size,
 // optionally you can extend the tests longer with time.Duration.
 func (sys *NotificationSys) Speedtest(ctx context.Context, size int,

cmd/object-api-errors.go

@@ -701,3 +701,8 @@ func isErrMethodNotAllowed(err error) bool {
 	var methodNotAllowed MethodNotAllowed
 	return errors.As(err, &methodNotAllowed)
 }
+
+func isErrInvalidRange(err error) bool {
+	_, ok := err.(InvalidRange)
+	return ok
+}

cmd/object-api-interface.go

@@ -171,7 +171,7 @@ type ObjectLayer interface {
 
 	// Storage operations.
 	Shutdown(context.Context) error
-	NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32) error
+	NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, scanMode madmin.HealScanMode) error
 	BackendInfo() madmin.BackendInfo
 	StorageInfo(ctx context.Context) (StorageInfo, []error)
 	LocalStorageInfo(ctx context.Context) (StorageInfo, []error)

cmd/object-api-utils.go

@@ -386,13 +386,13 @@ func getHostFromSrv(records []dns.SrvRecord) (host string) {
 }
 
 // IsCompressed returns true if the object is marked as compressed.
-func (o ObjectInfo) IsCompressed() bool {
+func (o *ObjectInfo) IsCompressed() bool {
 	_, ok := o.UserDefined[ReservedMetadataPrefix+"compression"]
 	return ok
 }
 
 // IsCompressedOK returns whether the object is compressed and can be decompressed.
-func (o ObjectInfo) IsCompressedOK() (bool, error) {
+func (o *ObjectInfo) IsCompressedOK() (bool, error) {
 	scheme, ok := o.UserDefined[ReservedMetadataPrefix+"compression"]
 	if !ok {
 		return false, nil
@@ -404,17 +404,8 @@ func (o ObjectInfo) IsCompressedOK() (bool, error) {
 	return true, fmt.Errorf("unknown compression scheme: %s", scheme)
 }
 
-// GetActualETag - returns the actual etag of the stored object
-// decrypts SSE objects.
-func (o ObjectInfo) GetActualETag(h http.Header) string {
-	if _, ok := crypto.IsEncrypted(o.UserDefined); !ok {
-		return o.ETag
-	}
-	return getDecryptedETag(h, o, false)
-}
-
 // GetActualSize - returns the actual size of the stored object
-func (o ObjectInfo) GetActualSize() (int64, error) {
+func (o *ObjectInfo) GetActualSize() (int64, error) {
 	if o.IsCompressed() {
 		sizeStr, ok := o.UserDefined[ReservedMetadataPrefix+"actual-size"]
 		if !ok {

cmd/object-handlers.go

@@ -206,6 +206,9 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 			Start:          offset,
 			End:            offset + length,
 		}
+		if length == -1 {
+			rs.End = -1
+		}
 
 		return getObjectNInfo(ctx, bucket, object, rs, r.Header, readLock, opts)
 	}
@@ -412,20 +415,29 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 	if err != nil {
 		var (
 			reader *GetObjectReader
-			proxy  bool
+			proxy  proxyResult
+			perr   error
 		)
 		proxytgts := getproxyTargets(ctx, bucket, object, opts)
 		if !proxytgts.Empty() {
 			// proxy to replication target if active-active replication is in place.
-			reader, proxy = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
-			if reader != nil && proxy {
+			reader, proxy, perr = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
+			if perr != nil && !isErrObjectNotFound(ErrorRespToObjectError(perr, bucket, object)) &&
+				!isErrVersionNotFound(ErrorRespToObjectError(perr, bucket, object)) {
+				logger.LogIf(ctx, fmt.Errorf("Replication proxy failed for %s/%s(%s) - %w", bucket, object, opts.VersionID, perr))
+			}
+			if reader != nil && proxy.Proxy && perr == nil {
 				gr = reader
 			}
 		}
-		if reader == nil || !proxy {
+		if reader == nil || !proxy.Proxy {
 			if isErrPreconditionFailed(err) {
 				return
 			}
+			if proxy.Err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, proxy.Err), r.URL)
+				return
+			}
 			if globalBucketVersioningSys.Enabled(bucket) && gr != nil {
 				if !gr.ObjInfo.VersionPurgeStatus.Empty() {
 					// Shows the replication status of a permanent delete of a version
@@ -631,22 +643,28 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
 		return
 	}
+	// Get request range.
+	var rs *HTTPRangeSpec
+	rangeHeader := r.Header.Get(xhttp.Range)
 
 	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		var (
-			proxy bool
+			proxy proxyResult
 			oi    ObjectInfo
 		)
 		// proxy HEAD to replication target if active-active replication configured on bucket
 		proxytgts := getproxyTargets(ctx, bucket, object, opts)
 		if !proxytgts.Empty() {
-			oi, proxy = proxyHeadToReplicationTarget(ctx, bucket, object, opts, proxytgts)
-			if proxy {
+			if rangeHeader != "" {
+				rs, _ = parseRequestRangeSpec(rangeHeader)
+			}
+			oi, proxy = proxyHeadToReplicationTarget(ctx, bucket, object, rs, opts, proxytgts)
+			if proxy.Proxy {
 				objInfo = oi
 			}
 		}
-		if !proxy {
+		if !proxy.Proxy {
 			if globalBucketVersioningSys.Enabled(bucket) {
 				switch {
 				case !objInfo.VersionPurgeStatus.Empty():
@@ -703,9 +721,6 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		return
 	}
 
-	// Get request range.
-	var rs *HTTPRangeSpec
-	rangeHeader := r.Header.Get(xhttp.Range)
 	if rangeHeader != "" {
 		// Both 'Range' and 'partNumber' cannot be specified at the same time
 		if opts.PartNumber > 0 {
@@ -2889,12 +2904,20 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	}
 
 	etag := partInfo.ETag
-	switch kind, encrypted := crypto.IsEncrypted(mi.UserDefined); {
-	case encrypted:
+	if kind, encrypted := crypto.IsEncrypted(mi.UserDefined); encrypted {
 		switch kind {
+		case crypto.S3KMS:
+			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
+			w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, mi.KMSKeyID())
+			if kmsCtx, ok := mi.UserDefined[crypto.MetaContext]; ok {
+				w.Header().Set(xhttp.AmzServerSideEncryptionKmsContext, kmsCtx)
+			}
+			if len(etag) >= 32 && strings.Count(etag, "-") != 1 {
+				etag = etag[len(etag)-32:]
+			}
 		case crypto.S3:
 			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
-			etag = tryDecryptETag(objectEncryptionKey[:], etag, false)
+			etag, _ = DecryptETag(objectEncryptionKey, ObjectInfo{ETag: etag})
 		case crypto.SSEC:
 			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm))
 			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
@@ -3002,34 +3025,29 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	var ssec bool
-	if _, ok := crypto.IsEncrypted(listPartsInfo.UserDefined); ok && objectAPI.IsEncryptionSupported() {
-		var key []byte
-		if crypto.SSEC.IsEncrypted(listPartsInfo.UserDefined) {
-			ssec = true
-		}
+	// We have to adjust the size of encrypted parts since encrypted parts
+	// are slightly larger due to encryption overhead.
+	// Further, we have to adjust the ETags of parts when using SSE-S3.
+	// Due to AWS S3, SSE-S3 encrypted parts return the plaintext ETag
+	// being the content MD5 of that particular part. This is not the
+	// case for SSE-C and SSE-KMS objects.
+	if kind, ok := crypto.IsEncrypted(listPartsInfo.UserDefined); ok && objectAPI.IsEncryptionSupported() {
 		var objectEncryptionKey []byte
-		if crypto.S3.IsEncrypted(listPartsInfo.UserDefined) {
-			// Calculating object encryption key
-			objectEncryptionKey, err = decryptObjectInfo(key, bucket, object, listPartsInfo.UserDefined)
+		if kind == crypto.S3 {
+			objectEncryptionKey, err = decryptObjectInfo(nil, bucket, object, listPartsInfo.UserDefined)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
 		}
-		for i := range listPartsInfo.Parts {
-			curp := listPartsInfo.Parts[i]
-			curp.ETag = tryDecryptETag(objectEncryptionKey, curp.ETag, ssec)
-			if !ssec {
-				var partSize uint64
-				partSize, err = sio.DecryptedSize(uint64(curp.Size))
-				if err != nil {
-					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-					return
-				}
-				curp.Size = int64(partSize)
+		for i, p := range listPartsInfo.Parts {
+			listPartsInfo.Parts[i].ETag = tryDecryptETag(objectEncryptionKey, p.ETag, kind != crypto.S3)
+			size, err := sio.DecryptedSize(uint64(p.Size))
+			if err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				return
 			}
-			listPartsInfo.Parts[i] = curp
+			listPartsInfo.Parts[i].Size = int64(size)
 		}
 	}
 
@@ -3161,66 +3179,6 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		return
 	}
 
-	var objectEncryptionKey []byte
-	var isEncrypted, ssec bool
-	if objectAPI.IsEncryptionSupported() {
-		mi, err := objectAPI.GetMultipartInfo(ctx, bucket, object, uploadID, ObjectOptions{})
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
-		}
-		if _, ok := crypto.IsEncrypted(mi.UserDefined); ok {
-			var key []byte
-			isEncrypted = true
-			ssec = crypto.SSEC.IsEncrypted(mi.UserDefined)
-			if crypto.S3.IsEncrypted(mi.UserDefined) {
-				// Calculating object encryption key
-				objectEncryptionKey, err = decryptObjectInfo(key, bucket, object, mi.UserDefined)
-				if err != nil {
-					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-					return
-				}
-			}
-		}
-	}
-
-	partsMap := make(map[string]PartInfo)
-	if isEncrypted {
-		maxParts := 10000
-		listPartsInfo, err := objectAPI.ListObjectParts(ctx, bucket, object, uploadID, 0, maxParts, ObjectOptions{})
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
-		}
-		for _, part := range listPartsInfo.Parts {
-			partsMap[strconv.Itoa(part.PartNumber)] = part
-		}
-	}
-
-	// Complete parts.
-	completeParts := make([]CompletePart, 0, len(complMultipartUpload.Parts))
-	originalCompleteParts := make([]CompletePart, 0, len(complMultipartUpload.Parts))
-	for _, part := range complMultipartUpload.Parts {
-		part.ETag = canonicalizeETag(part.ETag)
-		originalCompleteParts = append(originalCompleteParts, part)
-		if isEncrypted {
-			// ETag is stored in the backend in encrypted form. Validate client sent ETag with
-			// decrypted ETag.
-			if bkPartInfo, ok := partsMap[strconv.Itoa(part.PartNumber)]; ok {
-				bkETag := tryDecryptETag(objectEncryptionKey, bkPartInfo.ETag, ssec)
-				if bkETag != part.ETag {
-					writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPart), r.URL)
-					return
-				}
-				part.ETag = bkPartInfo.ETag
-			}
-		}
-		completeParts = append(completeParts, part)
-	}
-
-	// Calculate s3 compatible md5sum for complete multipart.
-	s3MD5 := getCompleteMultipartMD5(originalCompleteParts)
-
 	completeMultiPartUpload := objectAPI.CompleteMultipartUpload
 	if api.CacheAPI() != nil {
 		completeMultiPartUpload = api.CacheAPI().CompleteMultipartUpload
@@ -3263,14 +3221,68 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		return
 	}
 
-	// preserve ETag if set, or set from parts.
-	if _, ok := opts.UserDefined["etag"]; !ok {
-		opts.UserDefined["etag"] = s3MD5
+	// First, we compute the ETag of the multipart object.
+	// The ETag of a multi-part object is always:
+	//   ETag := MD5(ETag_p1, ETag_p2, ...)+"-N"   (N being the number of parts)
+	//
+	// This is independent of encryption. An encrypted multipart
+	// object also has an ETag that is the MD5 of its part ETags.
+	// The fact the in case of encryption the ETag of a part is
+	// not the MD5 of the part content does not change that.
+	var completeETags []etag.ETag
+	for _, part := range complMultipartUpload.Parts {
+		ETag, err := etag.Parse(part.ETag)
+		if err != nil {
+			continue
+		}
+		completeETags = append(completeETags, ETag)
+	}
+	multipartETag := etag.Multipart(completeETags...)
+	opts.UserDefined["etag"] = multipartETag.String()
+
+	// However, in case of encryption, the persisted part ETags don't match
+	// what we have sent to the client during PutObjectPart. The reason is
+	// that ETags are encrypted. Hence, the client will send a list of complete
+	// part ETags of which non can match the ETag of any part. For example
+	//   ETag (client):          30902184f4e62dd8f98f0aaff810c626
+	//   ETag (server-internal): 20000f00ce5dc16e3f3b124f586ae1d88e9caa1c598415c2759bbb50e84a59f630902184f4e62dd8f98f0aaff810c626
+	//
+	// Therefore, we adjust all ETags sent by the client to match what is stored
+	// on the backend.
+	if objectAPI.IsEncryptionSupported() {
+		mi, err := objectAPI.GetMultipartInfo(ctx, bucket, object, uploadID, ObjectOptions{})
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+
+		if _, ok := crypto.IsEncrypted(mi.UserDefined); ok {
+			const MaxParts = 10000
+			listPartsInfo, err := objectAPI.ListObjectParts(ctx, bucket, object, uploadID, 0, MaxParts, ObjectOptions{})
+			if err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				return
+			}
+			sort.Slice(listPartsInfo.Parts, func(i, j int) bool {
+				return listPartsInfo.Parts[i].PartNumber < listPartsInfo.Parts[j].PartNumber
+			})
+			sort.Slice(complMultipartUpload.Parts, func(i, j int) bool {
+				return complMultipartUpload.Parts[i].PartNumber < complMultipartUpload.Parts[j].PartNumber
+			})
+			for i := range listPartsInfo.Parts {
+				for j := range complMultipartUpload.Parts {
+					if listPartsInfo.Parts[i].PartNumber == complMultipartUpload.Parts[j].PartNumber {
+						complMultipartUpload.Parts[j].ETag = listPartsInfo.Parts[i].ETag
+						continue
+					}
+				}
+			}
+		}
 	}
 
 	w = &whiteSpaceWriter{ResponseWriter: w, Flusher: w.(http.Flusher)}
 	completeDoneCh := sendWhiteSpace(w)
-	objInfo, err := completeMultiPartUpload(ctx, bucket, object, uploadID, completeParts, opts)
+	objInfo, err := completeMultiPartUpload(ctx, bucket, object, uploadID, complMultipartUpload.Parts, opts)
 	// Stop writing white spaces to the client. Note that close(doneCh) style is not used as it
 	// can cause white space to be written after we send XML response in a race condition.
 	headerWritten := <-completeDoneCh

cmd/peer-rest-client.go

@@ -1119,3 +1119,27 @@ func (client *peerRESTClient) GetLastDayTierStats(ctx context.Context) (dailyAll
 	}
 	return dailyAllTierStats(result), nil
 }
+
+// DevNull - Used by netperf to pump data to peer
+func (client *peerRESTClient) DevNull(ctx context.Context, r io.Reader) error {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodDevNull, nil, r, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return err
+}
+
+// Netperf - To initiate netperf on peer
+func (client *peerRESTClient) Netperf(ctx context.Context, duration time.Duration) (madmin.NetperfNodeResult, error) {
+	var result madmin.NetperfNodeResult
+	values := make(url.Values)
+	values.Set(peerRESTDuration, duration.String())
+	respBody, err := client.callWithContext(context.Background(), peerRESTMethodNetperf, values, nil, -1)
+	if err != nil {
+		return result, err
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&result)
+	return result, err
+}

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v20" // Add drivespeedtest
+	peerRESTVersion       = "v21" // Add netperf
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -70,6 +70,8 @@ const (
 	peerRESTMethodReloadSiteReplicationConfig = "/reloadsitereplicationconfig"
 	peerRESTMethodReloadPoolMeta              = "/reloadpoolmeta"
 	peerRESTMethodGetLastDayTierStats         = "/getlastdaytierstats"
+	peerRESTMethodDevNull                     = "/devnull"
+	peerRESTMethodNetperf                     = "/netperf"
 )
 
 const (

cmd/peer-rest-server.go

@@ -27,20 +27,15 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
 	b "github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/event"
-	"github.com/minio/minio/internal/hash"
-	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/randreader"
 	"github.com/tinylib/msgp/msgp"
 )
 
@@ -1143,148 +1138,6 @@ func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// SpeedtestResult return value of the speedtest function
-type SpeedtestResult struct {
-	Endpoint  string
-	Uploads   uint64
-	Downloads uint64
-	Error     string
-}
-
-func newRandomReader(size int) io.Reader {
-	return io.LimitReader(randreader.New(), int64(size))
-}
-
-// Runs the speedtest on local MinIO process.
-func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration, storageClass string) (SpeedtestResult, error) {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return SpeedtestResult{}, errServerNotInitialized
-	}
-
-	var errOnce sync.Once
-	var retError string
-	var wg sync.WaitGroup
-	var totalBytesWritten uint64
-	var totalBytesRead uint64
-
-	objCountPerThread := make([]uint64, concurrent)
-
-	uploadsCtx, uploadsCancel := context.WithCancel(context.Background())
-	defer uploadsCancel()
-
-	go func() {
-		time.Sleep(duration)
-		uploadsCancel()
-	}()
-
-	objNamePrefix := "speedtest/objects/" + uuid.New().String()
-
-	wg.Add(concurrent)
-	for i := 0; i < concurrent; i++ {
-		go func(i int) {
-			defer wg.Done()
-			for {
-				hashReader, err := hash.NewReader(newRandomReader(size),
-					int64(size), "", "", int64(size))
-				if err != nil {
-					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					uploadsCancel()
-					return
-				}
-				reader := NewPutObjReader(hashReader)
-				objInfo, err := objAPI.PutObject(uploadsCtx, minioMetaBucket, fmt.Sprintf("%s.%d.%d",
-					objNamePrefix, i, objCountPerThread[i]), reader, ObjectOptions{
-					UserDefined: map[string]string{
-						xhttp.AmzStorageClass: storageClass,
-					},
-					Speedtest: true,
-				})
-				if err != nil {
-					objCountPerThread[i]--
-					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					uploadsCancel()
-					return
-				}
-				atomic.AddUint64(&totalBytesWritten, uint64(objInfo.Size))
-				objCountPerThread[i]++
-			}
-		}(i)
-	}
-	wg.Wait()
-
-	// We already saw write failures, no need to proceed into read's
-	if retError != "" {
-		return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
-	}
-
-	downloadsCtx, downloadsCancel := context.WithCancel(context.Background())
-	defer downloadsCancel()
-	go func() {
-		time.Sleep(duration)
-		downloadsCancel()
-	}()
-
-	wg.Add(concurrent)
-	for i := 0; i < concurrent; i++ {
-		go func(i int) {
-			defer wg.Done()
-			var j uint64
-			if objCountPerThread[i] == 0 {
-				return
-			}
-			for {
-				if objCountPerThread[i] == j {
-					j = 0
-				}
-				r, err := objAPI.GetObjectNInfo(downloadsCtx, minioMetaBucket, fmt.Sprintf("%s.%d.%d",
-					objNamePrefix, i, j), nil, nil, noLock, ObjectOptions{})
-				if err != nil {
-					if isErrObjectNotFound(err) {
-						continue
-					}
-					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					downloadsCancel()
-					return
-				}
-				n, err := io.Copy(ioutil.Discard, r)
-				r.Close()
-				if err == nil {
-					// Only capture success criteria - do not
-					// have to capture failed reads, truncated
-					// reads etc.
-					atomic.AddUint64(&totalBytesRead, uint64(n))
-				}
-				if err != nil {
-					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					downloadsCancel()
-					return
-				}
-				j++
-			}
-		}(i)
-	}
-	wg.Wait()
-
-	return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
-}
-
 func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("invalid request"))
@@ -1384,6 +1237,50 @@ func (s *peerRESTServer) DriveSpeedTestHandler(w http.ResponseWriter, r *http.Re
 	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
 }
 
+// DevNull - everything goes to io.Discard
+func (s *peerRESTServer) DevNull(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	globalNetPerfRX.Connect()
+	defer globalNetPerfRX.Disconnect()
+
+	connectTime := time.Now()
+	ctx := newContext(r, w, "DevNull")
+	for {
+		n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
+		atomic.AddUint64(&globalNetPerfRX.RX, uint64(n))
+		if err != nil && err != io.EOF {
+			// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
+			// would mean the network is not stable. Logging here will help in debugging network issues.
+			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
+				logger.LogIf(ctx, err)
+			}
+		}
+		if err != nil {
+			break
+		}
+	}
+}
+
+// Netperf - perform netperf
+func (s *peerRESTServer) Netperf(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	durationStr := r.Form.Get(peerRESTDuration)
+	duration, err := time.ParseDuration(durationStr)
+	if err != nil || duration.Seconds() == 0 {
+		duration = time.Second * 10
+	}
+	result := netperf(r.Context(), duration.Round(time.Second))
+	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
+}
+
 // registerPeerRESTHandlers - register peer rest router.
 func registerPeerRESTHandlers(router *mux.Router) {
 	server := &peerRESTServer{}
@@ -1431,6 +1328,8 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedtest).HandlerFunc(httpTraceHdrs(server.SpeedtestHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveSpeedTest).HandlerFunc(httpTraceHdrs(server.DriveSpeedTestHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(httpTraceHdrs(server.Netperf))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDevNull).HandlerFunc(httpTraceHdrs(server.DevNull))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadSiteReplicationConfig).HandlerFunc(httpTraceHdrs(server.ReloadSiteReplicationConfigHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadPoolMeta).HandlerFunc(httpTraceHdrs(server.ReloadPoolMetaHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLastDayTierStats).HandlerFunc(httpTraceHdrs(server.GetLastDayTierStatsHandler))

cmd/perf-tests.go

@@ -0,0 +1,298 @@
+// Copyright (c) 2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/google/uuid"
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/pkg/randreader"
+)
+
+// SpeedtestResult return value of the speedtest function
+type SpeedtestResult struct {
+	Endpoint  string
+	Uploads   uint64
+	Downloads uint64
+	Error     string
+}
+
+func newRandomReader(size int) io.Reader {
+	return io.LimitReader(randreader.New(), int64(size))
+}
+
+// Runs the speedtest on local MinIO process.
+func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration, storageClass string) (SpeedtestResult, error) {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return SpeedtestResult{}, errServerNotInitialized
+	}
+
+	var errOnce sync.Once
+	var retError string
+	var wg sync.WaitGroup
+	var totalBytesWritten uint64
+	var totalBytesRead uint64
+
+	region := globalSite.Region
+	if region == "" {
+		region = "us-east-1"
+	}
+
+	client, err := minio.New(globalLocalNodeName, &minio.Options{
+		Creds:     credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""),
+		Secure:    globalIsTLS,
+		Transport: globalProxyTransport,
+		Region:    region,
+	})
+	if err != nil {
+		return SpeedtestResult{}, err
+	}
+
+	objCountPerThread := make([]uint64, concurrent)
+
+	uploadsCtx, uploadsCancel := context.WithCancel(context.Background())
+	defer uploadsCancel()
+
+	go func() {
+		time.Sleep(duration)
+		uploadsCancel()
+	}()
+
+	objNamePrefix := uuid.New().String() + "/"
+
+	userMetadata := make(map[string]string)
+	userMetadata[globalObjectPerfUserMetadata] = "true"
+
+	wg.Add(concurrent)
+	for i := 0; i < concurrent; i++ {
+		go func(i int) {
+			defer wg.Done()
+			for {
+				reader := newRandomReader(size)
+				info, err := client.PutObject(uploadsCtx, globalObjectPerfBucket, fmt.Sprintf("%s%d.%d", objNamePrefix, i, objCountPerThread[i]), reader, int64(size), minio.PutObjectOptions{UserMetadata: userMetadata, DisableMultipart: true}) // Bypass S3 API freeze
+				if err != nil {
+					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
+						errOnce.Do(func() {
+							retError = err.Error()
+						})
+					}
+					uploadsCancel()
+					return
+				}
+				atomic.AddUint64(&totalBytesWritten, uint64(info.Size))
+				objCountPerThread[i]++
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	// We already saw write failures, no need to proceed into read's
+	if retError != "" {
+		return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
+	}
+
+	downloadsCtx, downloadsCancel := context.WithCancel(context.Background())
+	defer downloadsCancel()
+	go func() {
+		time.Sleep(duration)
+		downloadsCancel()
+	}()
+
+	wg.Add(concurrent)
+	for i := 0; i < concurrent; i++ {
+		go func(i int) {
+			defer wg.Done()
+			var j uint64
+			if objCountPerThread[i] == 0 {
+				return
+			}
+			for {
+				if objCountPerThread[i] == j {
+					j = 0
+				}
+				opts := minio.GetObjectOptions{}
+				opts.Set(globalObjectPerfUserMetadata, "true") // Bypass S3 API freeze
+				r, err := client.GetObject(downloadsCtx, globalObjectPerfBucket, fmt.Sprintf("%s%d.%d", objNamePrefix, i, j), opts)
+				if err != nil {
+					errResp, ok := err.(minio.ErrorResponse)
+					if ok && errResp.StatusCode == http.StatusNotFound {
+						continue
+					}
+					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
+						errOnce.Do(func() {
+							retError = err.Error()
+						})
+					}
+					downloadsCancel()
+					return
+				}
+				n, err := io.Copy(ioutil.Discard, r)
+				r.Close()
+				if err == nil {
+					// Only capture success criteria - do not
+					// have to capture failed reads, truncated
+					// reads etc.
+					atomic.AddUint64(&totalBytesRead, uint64(n))
+				}
+				if err != nil {
+					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
+						errOnce.Do(func() {
+							retError = err.Error()
+						})
+					}
+					downloadsCancel()
+					return
+				}
+				j++
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
+}
+
+// To collect RX stats during "mc support perf net"
+// RXSample holds the RX bytes for the duration between
+// the last peer to connect and the first peer to disconnect.
+// This is to improve the RX throughput accuracy.
+type netPerfRX struct {
+	RX                uint64    // RX bytes
+	lastToConnect     time.Time // time at which last peer to connect to us
+	firstToDisconnect time.Time // time at which the first peer disconnects from us
+	RXSample          uint64    // RX bytes between lastToConnect and firstToDisconnect
+	activeConnections uint64
+	sync.RWMutex
+}
+
+func (n *netPerfRX) Connect() {
+	n.Lock()
+	defer n.Unlock()
+	n.activeConnections++
+	atomic.StoreUint64(&globalNetPerfRX.RX, 0)
+	n.lastToConnect = time.Now()
+}
+
+func (n *netPerfRX) Disconnect() {
+	n.Lock()
+	defer n.Unlock()
+	n.activeConnections--
+	if n.firstToDisconnect.IsZero() {
+		n.RXSample = atomic.LoadUint64(&n.RX)
+		n.firstToDisconnect = time.Now()
+	}
+}
+
+func (n *netPerfRX) ActiveConnections() uint64 {
+	n.RLock()
+	defer n.RUnlock()
+	return n.activeConnections
+}
+
+func (n *netPerfRX) Reset() {
+	n.RLock()
+	defer n.RUnlock()
+	n.RX = 0
+	n.RXSample = 0
+	n.lastToConnect = time.Time{}
+	n.firstToDisconnect = time.Time{}
+}
+
+// Reader to read random data.
+type netperfReader struct {
+	n   uint64
+	eof chan struct{}
+	buf []byte
+}
+
+func (m *netperfReader) Read(b []byte) (int, error) {
+	select {
+	case <-m.eof:
+		return 0, io.EOF
+	default:
+	}
+	n := copy(b, m.buf)
+	atomic.AddUint64(&m.n, uint64(n))
+	return n, nil
+}
+
+func netperf(ctx context.Context, duration time.Duration) madmin.NetperfNodeResult {
+	r := &netperfReader{eof: make(chan struct{})}
+	r.buf = make([]byte, 128*humanize.KiByte)
+	rand.Read(r.buf)
+
+	connectionsPerPeer := 16
+
+	if len(globalNotificationSys.peerClients) > 16 {
+		// For a large cluster it's enough to have 1 connection per peer to saturate the network.
+		connectionsPerPeer = 1
+	}
+
+	errStr := ""
+	var wg sync.WaitGroup
+	for index := range globalNotificationSys.peerClients {
+		if globalNotificationSys.peerClients[index] == nil {
+			continue
+		}
+		go func(index int) {
+			for i := 0; i < connectionsPerPeer; i++ {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					err := globalNotificationSys.peerClients[index].DevNull(ctx, r)
+					if err != nil {
+						errStr = err.Error()
+					}
+				}()
+			}
+		}(index)
+	}
+
+	time.Sleep(duration)
+	close(r.eof)
+	wg.Wait()
+	for {
+		if globalNetPerfRX.ActiveConnections() == 0 {
+			break
+		}
+		time.Sleep(time.Second)
+	}
+	rx := float64(globalNetPerfRX.RXSample)
+	delta := globalNetPerfRX.firstToDisconnect.Sub(globalNetPerfRX.lastToConnect)
+	if delta < 0 {
+		rx = 0
+		errStr = "network disconnection issues detected"
+	}
+
+	globalNetPerfRX.Reset()
+	return madmin.NetperfNodeResult{Endpoint: "", TX: r.n / uint64(duration.Seconds()), RX: uint64(rx / delta.Seconds()), Error: errStr}
+}

cmd/prepare-storage.go

@@ -19,7 +19,6 @@ package cmd
 
 import (
 	"context"
-	"crypto/tls"
 	"errors"
 	"fmt"
 	"net/http"
@@ -121,31 +120,9 @@ func isServerResolvable(endpoint Endpoint, timeout time.Duration) error {
 		Path:   pathJoin(healthCheckPathPrefix, healthCheckLivenessPath),
 	}
 
-	var tlsConfig *tls.Config
-	if globalIsTLS {
-		tlsConfig = &tls.Config{
-			RootCAs: globalRootCAs,
-		}
-	}
-
 	httpClient := &http.Client{
-		Transport:
-		// For more details about various values used here refer
-		// https://golang.org/pkg/net/http/#Transport documentation
-		&http.Transport{
-			Proxy:                 http.ProxyFromEnvironment,
-			DialContext:           xhttp.NewCustomDialContext(3 * time.Second),
-			ResponseHeaderTimeout: 3 * time.Second,
-			TLSHandshakeTimeout:   3 * time.Second,
-			ExpectContinueTimeout: 3 * time.Second,
-			TLSClientConfig:       tlsConfig,
-			// Go net/http automatically unzip if content-type is
-			// gzip disable this feature, as we are always interested
-			// in raw stream.
-			DisableCompression: true,
-		},
+		Transport: globalInternodeTransport,
 	}
-	defer httpClient.CloseIdleConnections()
 
 	ctx, cancel := context.WithTimeout(GlobalContext, timeout)
 

cmd/server-main.go

@@ -28,6 +28,7 @@ import (
 	"math/rand"
 	"os"
 	"os/signal"
+	"runtime"
 	"strings"
 	"sync"
 	"syscall"
@@ -299,6 +300,8 @@ func configRetriableErrors(err error) bool {
 }
 
 func initServer(ctx context.Context, newObject ObjectLayer) error {
+	t1 := time.Now()
+
 	// Once the config is fully loaded, initialize the new object layer.
 	setObjectLayer(newObject)
 
@@ -351,7 +354,7 @@ func initServer(ctx context.Context, newObject ObjectLayer) error {
 				// All successful return.
 				if globalIsDistErasure {
 					// These messages only meant primarily for distributed setup, so only log during distributed setup.
-					logger.Info("All MinIO sub-systems initialized successfully")
+					logger.Info("All MinIO sub-systems initialized successfully in %s", time.Since(t1))
 				}
 				return nil
 			}
@@ -448,6 +451,17 @@ func serverMain(ctx *cli.Context) {
 	// Set system resources to maximum.
 	setMaxResources()
 
+	// Verify kernel release and version.
+	if oldLinux() {
+		logger.Info(color.RedBold("WARNING: Detected Linux kernel version older than 4.0.0 release, there are some known potential performance problems with this kernel version. MinIO recommends a minimum of 4.x.x linux kernel version for best performance"))
+	}
+
+	maxProcs := runtime.GOMAXPROCS(0)
+	cpuProcs := runtime.NumCPU()
+	if maxProcs < cpuProcs {
+		logger.Info(color.RedBold("WARNING: Detected GOMAXPROCS(%d) < NumCPU(%d), please make sure to provide all PROCS to MinIO for optimal performance", maxProcs, cpuProcs))
+	}
+
 	// Configure server.
 	handler, err := configureServerHandler(globalEndpoints)
 	if err != nil {

cmd/server-rlimit.go

@@ -21,10 +21,28 @@ import (
 	"runtime"
 	"runtime/debug"
 
+	"github.com/minio/minio/internal/kernel"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/sys"
 )
 
+func oldLinux() bool {
+	currentKernel, err := kernel.CurrentVersion()
+	if err != nil {
+		// Could not probe the kernel version
+		return false
+	}
+
+	if currentKernel == 0 {
+		// We could not get any valid value return false
+		return false
+	}
+
+	// legacy linux indicator for printing warnings
+	// about older Linux kernels and Go runtime.
+	return currentKernel < kernel.Version(4, 0, 0)
+}
+
 func setMaxResources() (err error) {
 	// Set the Go runtime max threads threshold to 90% of kernel setting.
 	sysMaxThreads, mErr := sys.GetMaxThreads()

cmd/storage-interface.go

@@ -21,6 +21,8 @@ import (
 	"context"
 	"io"
 	"time"
+
+	"github.com/minio/madmin-go"
 )
 
 // StorageAPI interface.
@@ -29,20 +31,42 @@ type StorageAPI interface {
 	String() string
 
 	// Storage operations.
-	IsOnline() bool      // Returns true if disk is online.
-	LastConn() time.Time // Returns the last time this disk (re)-connected
 
+	// Returns true if disk is online and its valid i.e valid format.json.
+	// This has nothing to do with if the drive is hung or not responding.
+	// For that individual storage API calls will fail properly. The purpose
+	// of this function is to know if the "drive" has "format.json" or not
+	// if it has a "format.json" then is it correct "format.json" or not.
+	IsOnline() bool
+
+	// Returns the last time this disk (re)-connected
+	LastConn() time.Time
+
+	// Indicates if disk is local or not.
 	IsLocal() bool
-	Hostname() string   // Returns host name if remote host.
-	Endpoint() Endpoint // Returns endpoint.
 
+	// Returns hostname if disk is remote.
+	Hostname() string
+
+	// Returns the entire endpoint.
+	Endpoint() Endpoint
+
+	// Close the disk, mark it purposefully closed, only implemented for remote disks.
 	Close() error
-	GetDiskID() (string, error)
-	SetDiskID(id string)
-	Healing() *healingTracker // Returns nil if disk is not healing.
 
+	// Returns the unique 'uuid' of this disk.
+	GetDiskID() (string, error)
+
+	// Set a unique 'uuid' for this disk, only used when
+	// disk is replaced and formatted.
+	SetDiskID(id string)
+
+	// Returns healing information for a newly replaced disk,
+	// returns 'nil' once healing is complete or if the disk
+	// has never been replaced.
+	Healing() *healingTracker
 	DiskInfo(ctx context.Context) (info DiskInfo, err error)
-	NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error)
+	NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode) (dataUsageCache, error)
 
 	// Volume operations.
 	MakeVol(ctx context.Context, volume string) (err error)
@@ -120,7 +144,7 @@ func (p *unrecognizedDisk) Healing() *healingTracker {
 	return nil
 }
 
-func (p *unrecognizedDisk) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error) {
+func (p *unrecognizedDisk) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode) (dataUsageCache, error) {
 	return dataUsageCache{}, errDiskNotFound
 }
 

cmd/storage-rest-client.go

@@ -33,6 +33,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/minio/madmin-go"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/rest"
@@ -207,13 +208,15 @@ func (client *storageRESTClient) Healing() *healingTracker {
 	return val.(*healingTracker)
 }
 
-func (client *storageRESTClient) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error) {
+func (client *storageRESTClient) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode) (dataUsageCache, error) {
 	defer close(updates)
 	pr, pw := io.Pipe()
 	go func() {
 		pw.CloseWithError(cache.serializeTo(pw))
 	}()
-	respBody, err := client.call(ctx, storageRESTMethodNSScanner, url.Values{}, pr, -1)
+	vals := make(url.Values)
+	vals.Set(storageRESTScanMode, strconv.Itoa(int(scanMode)))
+	respBody, err := client.call(ctx, storageRESTMethodNSScanner, vals, pr, -1)
 	defer xhttp.DrainBody(respBody)
 	pr.CloseWithError(err)
 	if err != nil {

cmd/storage-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	storageRESTVersion       = "v43" // Added DiskMTime field for FileInfo
+	storageRESTVersion       = "v44" // Added heal scan mode in NSScanner
 	storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
 	storageRESTPrefix        = minioReservedBucketPath + "/storage"
 )
@@ -79,4 +79,5 @@ const (
 	storageRESTDiskID         = "disk-id"
 	storageRESTForceDelete    = "force-delete"
 	storageRESTGlob           = "glob"
+	storageRESTScanMode       = "scan-mode"
 )

cmd/storage-rest-server.go

@@ -40,6 +40,7 @@ import (
 
 	jwtreq "github.com/golang-jwt/jwt/v4/request"
 	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/config"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
@@ -52,7 +53,7 @@ var errDiskStale = errors.New("disk stale")
 
 // To abstract a disk over network.
 type storageRESTServer struct {
-	storage *xlStorage
+	storage *xlStorageDiskIDCheck
 }
 
 func (s *storageRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
@@ -179,10 +180,17 @@ func (s *storageRESTServer) NSScannerHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	scanMode, err := strconv.Atoi(r.Form.Get(storageRESTScanMode))
+	if err != nil {
+		logger.LogIf(r.Context(), err)
+		s.writeErrorResponse(w, err)
+		return
+	}
+
 	setEventStreamHeaders(w)
 
 	var cache dataUsageCache
-	err := cache.deserialize(r.Body)
+	err = cache.deserialize(r.Body)
 	if err != nil {
 		logger.LogIf(r.Context(), err)
 		s.writeErrorResponse(w, err)
@@ -220,7 +228,7 @@ func (s *storageRESTServer) NSScannerHandler(w http.ResponseWriter, r *http.Requ
 			}
 		}
 	}()
-	usageInfo, err := s.storage.NSScanner(ctx, cache, updates)
+	usageInfo, err := s.storage.NSScanner(ctx, cache, updates, madmin.HealScanMode(scanMode))
 	if err != nil {
 		respW.Flush()
 		resp.CloseWithError(err)
@@ -1233,7 +1241,8 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 
 			endpoint := storage.Endpoint()
 
-			server := &storageRESTServer{storage: storage}
+			server := &storageRESTServer{storage: newXLStorageDiskIDCheck(storage)}
+			server.storage.SetDiskID(storage.diskID)
 
 			subrouter := router.PathPrefix(path.Join(storageRESTPrefix, endpoint.Path)).Subrouter()
 

cmd/tier.go

@@ -148,6 +148,9 @@ func (config *TierConfigMgr) Verify(ctx context.Context, tier string) error {
 
 // Empty returns if tier targets are empty
 func (config *TierConfigMgr) Empty() bool {
+	if config == nil {
+		return true
+	}
 	return len(config.ListTiers()) == 0
 }
 

cmd/utils.go

@@ -633,6 +633,7 @@ func NewGatewayHTTPTransportWithClientCerts(clientCert, clientKey string) *http.
 				err.Error()))
 		}
 		if c != nil {
+			c.UpdateReloadDuration(10 * time.Second)
 			c.ReloadOnSignal(syscall.SIGHUP) // allow reloads upon SIGHUP
 			transport.TLSClientConfig.GetClientCertificate = c.GetClientCertificate
 		}

cmd/xl-storage-disk-id-check.go

@@ -19,13 +19,19 @@ package cmd
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"io"
+	"math/rand"
+	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/logger"
 )
 
 //go:generate stringer -type=storageMetric -trimprefix=storageMetric $GOFILE
@@ -65,14 +71,12 @@ const (
 
 // Detects change in underlying disk.
 type xlStorageDiskIDCheck struct {
-	// fields position optimized for memory please
-	// do not re-order them, if you add new fields
-	// please use `fieldalignment ./...` to check
-	// if your changes are not causing any problems.
-	storage      StorageAPI
+	// apiCalls should be placed first so alignment is guaranteed for atomic operations.
+	apiCalls     [storageMetricLast]uint64
 	apiLatencies [storageMetricLast]*lockedLastMinuteLatency
 	diskID       string
-	apiCalls     [storageMetricLast]uint64
+	storage      *xlStorage
+	health       *diskHealthTracker
 }
 
 func (p *xlStorageDiskIDCheck) getMetrics() DiskMetrics {
@@ -109,6 +113,7 @@ func (e *lockedLastMinuteLatency) value() uint64 {
 func newXLStorageDiskIDCheck(storage *xlStorage) *xlStorageDiskIDCheck {
 	xl := xlStorageDiskIDCheck{
 		storage: storage,
+		health:  newDiskHealthTracker(),
 	}
 	for i := range xl.apiLatencies[:] {
 		xl.apiLatencies[i] = &lockedLastMinuteLatency{}
@@ -148,7 +153,7 @@ func (p *xlStorageDiskIDCheck) Healing() *healingTracker {
 	return p.storage.Healing()
 }
 
-func (p *xlStorageDiskIDCheck) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error) {
+func (p *xlStorageDiskIDCheck) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode) (dataUsageCache, error) {
 	if contextCanceled(ctx) {
 		return dataUsageCache{}, ctx.Err()
 	}
@@ -156,7 +161,7 @@ func (p *xlStorageDiskIDCheck) NSScanner(ctx context.Context, cache dataUsageCac
 	if err := p.checkDiskStale(); err != nil {
 		return dataUsageCache{}, err
 	}
-	return p.storage.NSScanner(ctx, cache, updates)
+	return p.storage.NSScanner(ctx, cache, updates, scanMode)
 }
 
 func (p *xlStorageDiskIDCheck) GetDiskLoc() (poolIdx, setIdx, diskIdx int) {
@@ -215,25 +220,31 @@ func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context) (info DiskInfo, err
 			return info, errDiskNotFound
 		}
 	}
+
+	if p.health.isFaulty() {
+		// if disk is already faulty return faulty for 'mc admin info' output and prometheus alerts.
+		return info, errFaultyDisk
+	}
+
 	return info, nil
 }
 
 func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
-	defer p.updateStorageMetrics(storageMetricMakeVolBulk, volumes...)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricMakeVolBulk, volumes...)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
+
 	return p.storage.MakeVolBulk(ctx, volumes...)
 }
 
 func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err error) {
-	defer p.updateStorageMetrics(storageMetricMakeVol, volume)()
-
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricMakeVol, volume)
+	if err != nil {
+		return err
+	}
+	defer done(&err)
 	if contextCanceled(ctx) {
 		return ctx.Err()
 	}
@@ -244,167 +255,122 @@ func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err
 	return p.storage.MakeVol(ctx, volume)
 }
 
-func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) ([]VolInfo, error) {
-	defer p.updateStorageMetrics(storageMetricListVols, "/")()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) (vi []VolInfo, err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricListVols, "/")
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
+
 	return p.storage.ListVols(ctx)
 }
 
 func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
-	defer p.updateStorageMetrics(storageMetricStatVol, volume)()
-
-	if contextCanceled(ctx) {
-		return VolInfo{}, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricStatVol, volume)
+	if err != nil {
 		return vol, err
 	}
+	defer done(&err)
+
 	return p.storage.StatVol(ctx, volume)
 }
 
 func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
-	defer p.updateStorageMetrics(storageMetricDeleteVol, volume)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVol, volume)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
+
 	return p.storage.DeleteVol(ctx, volume, forceDelete)
 }
 
-func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error) {
-	defer p.updateStorageMetrics(storageMetricListDir, volume, dirPath)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) (s []string, err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricListDir, volume, dirPath)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.ListDir(ctx, volume, dirPath, count)
 }
 
 func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
-	defer p.updateStorageMetrics(storageMetricReadFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return 0, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadFile, volume, path)
+	if err != nil {
 		return 0, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadFile(ctx, volume, path, offset, buf, verifier)
 }
 
 func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) {
-	defer p.updateStorageMetrics(storageMetricAppendFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricAppendFile, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.AppendFile(ctx, volume, path, buf)
 }
 
-func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
-	defer p.updateStorageMetrics(storageMetricCreateFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricCreateFile, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.CreateFile(ctx, volume, path, size, reader)
 }
 
 func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
-	defer p.updateStorageMetrics(storageMetricReadFileStream, volume, path)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadFileStream, volume, path)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadFileStream(ctx, volume, path, offset, length)
 }
 
-func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
-	defer p.updateStorageMetrics(storageMetricRenameFile, srcVolume, srcPath, dstVolume, dstPath)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricRenameFile, srcVolume, srcPath, dstVolume, dstPath)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath)
 }
 
-func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) error {
-	defer p.updateStorageMetrics(storageMetricRenameData, srcPath, fi.DataDir, dstVolume, dstPath)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricRenameData, srcPath, fi.DataDir, dstVolume, dstPath)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath)
 }
 
 func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
-	defer p.updateStorageMetrics(storageMetricCheckParts, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricCheckParts, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.CheckParts(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, recursive bool) (err error) {
-	defer p.updateStorageMetrics(storageMetricDelete, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDelete, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.Delete(ctx, volume, path, recursive)
 }
@@ -417,136 +383,102 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string
 	if len(versions) > 0 {
 		path = versions[0].Name
 	}
-
-	defer p.updateStorageMetrics(storageMetricDeleteVersions, volume, path)()
-
 	errs = make([]error, len(versions))
-
-	if contextCanceled(ctx) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVersions, volume, path)
+	if err != nil {
 		for i := range errs {
 			errs[i] = ctx.Err()
 		}
 		return errs
 	}
-
-	if err := p.checkDiskStale(); err != nil {
-		for i := range errs {
-			errs[i] = err
+	defer done(&err)
+	errs = p.storage.DeleteVersions(ctx, volume, versions)
+	for i := range errs {
+		if errs[i] != nil {
+			err = errs[i]
+			break
 		}
-		return errs
 	}
 
-	return p.storage.DeleteVersions(ctx, volume, versions)
+	return errs
 }
 
-func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
-	defer p.updateStorageMetrics(storageMetricVerifyFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricVerifyFile, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.VerifyFile(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) {
-	defer p.updateStorageMetrics(storageMetricWriteAll, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWriteAll, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.WriteAll(ctx, volume, path, b)
 }
 
 func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) (err error) {
-	defer p.updateStorageMetrics(storageMetricDeleteVersion, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVersion, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.DeleteVersion(ctx, volume, path, fi, forceDelMarker)
 }
 
 func (p *xlStorageDiskIDCheck) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
-	defer p.updateStorageMetrics(storageMetricUpdateMetadata, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricUpdateMetadata, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.UpdateMetadata(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
-	defer p.updateStorageMetrics(storageMetricWriteMetadata, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWriteMetadata, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.WriteMetadata(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (fi FileInfo, err error) {
-	defer p.updateStorageMetrics(storageMetricReadVersion, volume, path)()
-
-	if contextCanceled(ctx) {
-		return fi, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadVersion, volume, path)
+	if err != nil {
 		return fi, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadVersion(ctx, volume, path, versionID, readData)
 }
 
 func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
-	defer p.updateStorageMetrics(storageMetricReadAll, volume, path)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadAll, volume, path)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadAll(ctx, volume, path)
 }
 
 func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
-	defer p.updateStorageMetrics(storageMetricStatInfoFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricStatInfoFile, volume, path)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.StatInfoFile(ctx, volume, path, glob)
 }
@@ -565,10 +497,10 @@ func storageTrace(s storageMetric, startTime time.Time, duration time.Duration,
 }
 
 // Update storage metrics
-func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func() {
+func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func(err *error) {
 	startTime := time.Now()
 	trace := globalTrace.NumSubscribers() > 0
-	return func() {
+	return func(err *error) {
 		duration := time.Since(startTime)
 
 		atomic.AddUint64(&p.apiCalls[s], 1)
@@ -579,3 +511,317 @@ func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...st
 		}
 	}
 }
+
+const (
+	diskHealthOK = iota
+	diskHealthFaulty
+)
+
+// diskMaxConcurrent is the maximum number of running concurrent operations
+// for local and (incoming) remote disk ops respectively.
+var diskMaxConcurrent = 512
+
+func init() {
+	if s, ok := os.LookupEnv("_MINIO_DISK_MAX_CONCURRENT"); ok && s != "" {
+		var err error
+		diskMaxConcurrent, err = strconv.Atoi(s)
+		if err != nil {
+			logger.Fatal(err, "invalid _MINIO_DISK_MAX_CONCURRENT value")
+		}
+	}
+}
+
+type diskHealthTracker struct {
+	// atomic time of last success
+	lastSuccess int64
+
+	// atomic time of last time a token was grabbed.
+	lastStarted int64
+
+	// Atomic status of disk.
+	status int32
+
+	// Atomic number of requests blocking for a token.
+	blocked int32
+
+	// Concurrency tokens.
+	tokens chan struct{}
+}
+
+// newDiskHealthTracker creates a new disk health tracker.
+func newDiskHealthTracker() *diskHealthTracker {
+	d := diskHealthTracker{
+		lastSuccess: time.Now().UnixNano(),
+		lastStarted: time.Now().UnixNano(),
+		status:      diskHealthOK,
+		tokens:      make(chan struct{}, diskMaxConcurrent),
+	}
+	for i := 0; i < diskMaxConcurrent; i++ {
+		d.tokens <- struct{}{}
+	}
+	return &d
+}
+
+// logSuccess will update the last successful operation time.
+func (d *diskHealthTracker) logSuccess() {
+	atomic.StoreInt64(&d.lastSuccess, time.Now().UnixNano())
+}
+
+func (d *diskHealthTracker) isFaulty() bool {
+	return atomic.LoadInt32(&d.status) == diskHealthFaulty
+}
+
+type (
+	healthDiskCtxKey   struct{}
+	healthDiskCtxValue struct {
+		lastSuccess *int64
+	}
+)
+
+// logSuccess will update the last successful operation time.
+func (h *healthDiskCtxValue) logSuccess() {
+	atomic.StoreInt64(h.lastSuccess, time.Now().UnixNano())
+}
+
+// noopDoneFunc is a no-op done func.
+// Can be reused.
+var noopDoneFunc = func(_ *error) {}
+
+// TrackDiskHealth for this request.
+// When a non-nil error is returned 'done' MUST be called
+// with the status of the response, if it corresponds to disk health.
+// If the pointer sent to done is non-nil AND the error
+// is either nil or io.EOF the disk is considered good.
+// So if unsure if the disk status is ok, return nil as a parameter to done.
+// Shadowing will work as long as return error is named: https://go.dev/play/p/sauq86SsTN2
+func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMetric, paths ...string) (c context.Context, done func(*error), err error) {
+	done = noopDoneFunc
+	if contextCanceled(ctx) {
+		return ctx, done, ctx.Err()
+	}
+
+	// Return early if disk is faulty already.
+	if atomic.LoadInt32(&p.health.status) == diskHealthFaulty {
+		return ctx, done, errFaultyDisk
+	}
+
+	// Verify if the disk is not stale
+	// - missing format.json (unformatted drive)
+	// - format.json is valid but invalid 'uuid'
+	if err = p.checkDiskStale(); err != nil {
+		return ctx, done, err
+	}
+
+	// Disallow recursive tracking to avoid deadlocks.
+	if ctx.Value(healthDiskCtxKey{}) != nil {
+		done = p.updateStorageMetrics(s, paths...)
+		return ctx, done, nil
+	}
+
+	select {
+	case <-ctx.Done():
+		return ctx, done, ctx.Err()
+	case <-p.health.tokens:
+		// Fast path, got token.
+	default:
+		// We ran out of tokens, check health before blocking.
+		err = p.waitForToken(ctx)
+		if err != nil {
+			return ctx, done, err
+		}
+	}
+	// We only progress here if we got a token.
+
+	atomic.StoreInt64(&p.health.lastStarted, time.Now().UnixNano())
+	ctx = context.WithValue(ctx, healthDiskCtxKey{}, &healthDiskCtxValue{lastSuccess: &p.health.lastSuccess})
+	si := p.updateStorageMetrics(s, paths...)
+	t := time.Now()
+	var once sync.Once
+	return ctx, func(errp *error) {
+		once.Do(func() {
+			if false {
+				var ers string
+				if errp != nil {
+					err := *errp
+					ers = fmt.Sprint(err)
+				}
+				fmt.Println(time.Now().Format(time.RFC3339), "op", s, "took", time.Since(t), "result:", ers, "disk:", p.storage.String(), "path:", strings.Join(paths, "/"))
+			}
+			p.health.tokens <- struct{}{}
+			if errp != nil {
+				err := *errp
+				if err != nil && !errors.Is(err, io.EOF) {
+					return
+				}
+				p.health.logSuccess()
+			}
+			si(errp)
+		})
+	}, nil
+}
+
+// waitForToken will wait for a token, while periodically
+// checking the disk status.
+// If nil is returned a token was picked up.
+func (p *xlStorageDiskIDCheck) waitForToken(ctx context.Context) (err error) {
+	atomic.AddInt32(&p.health.blocked, 1)
+	defer func() {
+		atomic.AddInt32(&p.health.blocked, -1)
+	}()
+	// Avoid stampeding herd...
+	ticker := time.NewTicker(5*time.Second + time.Duration(rand.Int63n(int64(5*time.Second))))
+	defer ticker.Stop()
+	for {
+		err = p.checkHealth(ctx)
+		if err != nil {
+			return err
+		}
+		select {
+		case <-ticker.C:
+		// Ticker expired, check health again.
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-p.health.tokens:
+			return nil
+		}
+	}
+}
+
+// checkHealth should only be called when tokens have run out.
+// This will check if disk should be taken offline.
+func (p *xlStorageDiskIDCheck) checkHealth(ctx context.Context) (err error) {
+	if atomic.LoadInt32(&p.health.status) == diskHealthFaulty {
+		return errFaultyDisk
+	}
+	// Check if there are tokens.
+	if len(p.health.tokens) > 0 {
+		return nil
+	}
+
+	const maxTimeSinceLastSuccess = 30 * time.Second
+	const minTimeSinceLastOpStarted = 15 * time.Second
+
+	// To avoid stampeding herd (100s of simultaneous starting requests)
+	// there must be a delay between the last started request and now
+	// for the last lastSuccess to be useful.
+	t := time.Since(time.Unix(0, atomic.LoadInt64(&p.health.lastStarted)))
+	if t < minTimeSinceLastOpStarted {
+		return nil
+	}
+
+	// If also more than 15 seconds since last success, take disk offline.
+	t = time.Since(time.Unix(0, atomic.LoadInt64(&p.health.lastSuccess)))
+	if t > maxTimeSinceLastSuccess {
+		if atomic.CompareAndSwapInt32(&p.health.status, diskHealthOK, diskHealthFaulty) {
+			logger.LogAlwaysIf(ctx, fmt.Errorf("taking disk %s offline, time since last response %v", p.storage.String(), t.Round(time.Millisecond)))
+			go p.monitorDiskStatus()
+		}
+		return errFaultyDisk
+	}
+	return nil
+}
+
+// monitorDiskStatus should be called once when a drive has been marked offline.
+// Once the disk has been deemed ok, it will return to online status.
+func (p *xlStorageDiskIDCheck) monitorDiskStatus() {
+	t := time.NewTicker(5 * time.Second)
+	defer t.Stop()
+	fn := mustGetUUID()
+	for range t.C {
+		if len(p.health.tokens) == 0 {
+			// Queue is still full, no need to check.
+			continue
+		}
+		err := p.storage.WriteAll(context.Background(), minioMetaTmpBucket, fn, []byte{10000: 42})
+		if err != nil {
+			continue
+		}
+		b, err := p.storage.ReadAll(context.Background(), minioMetaTmpBucket, fn)
+		if err != nil || len(b) != 10001 {
+			continue
+		}
+		err = p.storage.Delete(context.Background(), minioMetaTmpBucket, fn, false)
+		if err == nil {
+			logger.Info("Able to read+write, bringing disk %s online.", p.storage.String())
+			atomic.StoreInt32(&p.health.status, diskHealthOK)
+			return
+		}
+	}
+}
+
+// diskHealthCheckOK will check if the provided error is nil
+// and update disk status if good.
+// For convenience a bool is returned to indicate any error state
+// that is not io.EOF.
+func diskHealthCheckOK(ctx context.Context, err error) bool {
+	// Check if context has a disk health check.
+	tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue)
+	if !ok {
+		// No tracker, return
+		return err == nil || errors.Is(err, io.EOF)
+	}
+	if err == nil || errors.Is(err, io.EOF) {
+		tracker.logSuccess()
+		return true
+	}
+	return false
+}
+
+// diskHealthWrapper provides either a io.Reader or io.Writer
+// that updates status of the provided tracker.
+// Use through diskHealthReader or diskHealthWriter.
+type diskHealthWrapper struct {
+	tracker *healthDiskCtxValue
+	r       io.Reader
+	w       io.Writer
+}
+
+func (d *diskHealthWrapper) Read(p []byte) (int, error) {
+	if d.r == nil {
+		return 0, fmt.Errorf("diskHealthWrapper: Read with no reader")
+	}
+	n, err := d.r.Read(p)
+	if err == nil || err == io.EOF && n > 0 {
+		d.tracker.logSuccess()
+	}
+	return n, err
+}
+
+func (d *diskHealthWrapper) Write(p []byte) (int, error) {
+	if d.w == nil {
+		return 0, fmt.Errorf("diskHealthWrapper: Write with no writer")
+	}
+	n, err := d.w.Write(p)
+	if err == nil && n == len(p) {
+		d.tracker.logSuccess()
+	}
+	return n, err
+}
+
+// diskHealthReader provides a wrapper that will update disk health on
+// ctx, on every successful read.
+// This should only be used directly at the os/syscall level,
+// otherwise buffered operations may return false health checks.
+func diskHealthReader(ctx context.Context, r io.Reader) io.Reader {
+	// Check if context has a disk health check.
+	tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue)
+	if !ok {
+		// No need to wrap
+		return r
+	}
+	return &diskHealthWrapper{r: r, tracker: tracker}
+}
+
+// diskHealthWriter provides a wrapper that will update disk health on
+// ctx, on every successful write.
+// This should only be used directly at the os/syscall level,
+// otherwise buffered operations may return false health checks.
+func diskHealthWriter(ctx context.Context, w io.Writer) io.Writer {
+	// Check if context has a disk health check.
+	tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue)
+	if !ok {
+		// No need to wrap
+		return w
+	}
+	return &diskHealthWrapper{w: w, tracker: tracker}
+}

cmd/xl-storage-format-utils.go

@@ -18,6 +18,8 @@
 package cmd
 
 import (
+	"errors"
+
 	"github.com/zeebo/xxh3"
 )
 
@@ -57,7 +59,19 @@ func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVers
 		}
 		versions, err = xlMeta.ListVersions(volume, path)
 	}
-	if err != nil || len(versions) == 0 {
+	if err == nil && len(versions) == 0 {
+		// This special case is needed to handle len(xlMeta.versions) == 0
+		versions = []FileInfo{
+			{
+				Volume:   volume,
+				Name:     path,
+				Deleted:  true,
+				IsLatest: true,
+				ModTime:  timeSentinel1970,
+			},
+		}
+	}
+	if err != nil {
 		return FileInfoVersions{}, err
 	}
 
@@ -76,11 +90,33 @@ func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, data bool) (F
 	if buf, data := isIndexedMetaV2(xlMetaBuf); buf != nil {
 		inData = data
 		fi, err = buf.ToFileInfo(volume, path, versionID)
+		if len(buf) != 0 && errors.Is(err, errFileNotFound) {
+			// This special case is needed to handle len(xlMeta.versions) == 0
+			return FileInfo{
+				Volume:    volume,
+				Name:      path,
+				VersionID: versionID,
+				Deleted:   true,
+				IsLatest:  true,
+				ModTime:   timeSentinel1970,
+			}, nil
+		}
 	} else {
 		var xlMeta xlMetaV2
 		if err := xlMeta.LoadOrConvert(xlMetaBuf); err != nil {
 			return FileInfo{}, err
 		}
+		if len(xlMeta.versions) == 0 {
+			// This special case is needed to handle len(xlMeta.versions) == 0
+			return FileInfo{
+				Volume:    volume,
+				Name:      path,
+				VersionID: versionID,
+				Deleted:   true,
+				IsLatest:  true,
+				ModTime:   timeSentinel1970,
+			}, nil
+		}
 		inData = xlMeta.data
 		fi, err = xlMeta.ToFileInfo(volume, path, versionID)
 	}

cmd/xl-storage-format-v2.go

@@ -595,6 +595,7 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
 		}
 	}
 	fi.ReplicationState = getInternalReplicationState(fi.Metadata)
+	fi.Deleted = !fi.VersionPurgeStatus().Empty()
 	replStatus := fi.ReplicationState.CompositeReplicationStatus()
 	if replStatus != "" {
 		fi.Metadata[xhttp.AmzBucketReplicationStatus] = string(replStatus)
@@ -658,7 +659,7 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
 	buf := metaDataPoolGet()[:initial]
 	_, err := io.ReadFull(r, buf)
 	if err != nil {
-		return nil, fmt.Errorf("readXLMetaNoData.ReadFull: %w", err)
+		return nil, fmt.Errorf("readXLMetaNoData(io.ReadFull): %w", err)
 	}
 	readMore := func(n int64) error {
 		has := int64(len(buf))
@@ -679,9 +680,9 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
 		if err != nil {
 			if errors.Is(err, io.EOF) {
 				// Returned if we read nothing.
-				return fmt.Errorf("readXLMetaNoData.readMore: %w", io.ErrUnexpectedEOF)
+				err = io.ErrUnexpectedEOF
 			}
-			return fmt.Errorf("readXLMetaNoData.readMore: %w", err)
+			return fmt.Errorf("readXLMetaNoData(readMore): %w", err)
 		}
 		return nil
 	}
@@ -699,7 +700,7 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
 		case 1, 2, 3:
 			sz, tmp, err := msgp.ReadBytesHeader(tmp)
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("readXLMetaNoData(read_meta): uknown metadata version %w", err)
 			}
 			want := int64(sz) + int64(len(buf)-len(tmp))
 
@@ -720,10 +721,14 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
 				return nil, err
 			}
 
+			if int64(len(buf)) < want {
+				return nil, fmt.Errorf("buffer shorter than expected (buflen: %d, want: %d): %w", len(buf), want, errFileCorrupt)
+			}
+
 			tmp = buf[want:]
 			_, after, err := msgp.ReadUint32Bytes(tmp)
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("readXLMetaNoData(read_meta): unknown metadata version %w", err)
 			}
 			want += int64(len(tmp) - len(after))
 
@@ -1150,7 +1155,7 @@ func (x *xlMetaV2) sortByModTime() {
 // DeleteVersion deletes the version specified by version id.
 // returns to the caller which dataDir to delete, also
 // indicates if this is the last version.
-func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
+func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) {
 	// This is a situation where versionId is explicitly
 	// specified as "null", as we do not save "null"
 	// string it is considered empty. But empty also
@@ -1164,7 +1169,7 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 	if fi.VersionID != "" {
 		uv, err = uuid.Parse(fi.VersionID)
 		if err != nil {
-			return "", false, errFileVersionNotFound
+			return "", errFileVersionNotFound
 		}
 	}
 
@@ -1179,7 +1184,7 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 			},
 		}
 		if !ventry.Valid() {
-			return "", false, errors.New("internal error: invalid version entry generated")
+			return "", errors.New("internal error: invalid version entry generated")
 		}
 	}
 	updateVersion := false
@@ -1225,18 +1230,18 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 		case LegacyType:
 			ver, err := x.getIdx(i)
 			if err != nil {
-				return "", false, err
+				return "", err
 			}
 			x.versions = append(x.versions[:i], x.versions[i+1:]...)
 			if fi.Deleted {
 				err = x.addVersion(ventry)
 			}
-			return ver.ObjectV1.DataDir, len(x.versions) == 0, err
+			return ver.ObjectV1.DataDir, err
 		case DeleteType:
 			if updateVersion {
 				ver, err := x.getIdx(i)
 				if err != nil {
-					return "", false, err
+					return "", err
 				}
 				if len(ver.DeleteMarker.MetaSys) == 0 {
 					ver.DeleteMarker.MetaSys = make(map[string][]byte)
@@ -1258,26 +1263,26 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 					ver.DeleteMarker.MetaSys[k] = []byte(v)
 				}
 				err = x.setIdx(i, *ver)
-				return "", len(x.versions) == 0, err
+				return "", err
 			}
 			var err error
 			x.versions = append(x.versions[:i], x.versions[i+1:]...)
 			if fi.MarkDeleted && (fi.VersionPurgeStatus().Empty() || (fi.VersionPurgeStatus() != Complete)) {
 				err = x.addVersion(ventry)
 			}
-			return "", len(x.versions) == 0, err
+			return "", err
 		case ObjectType:
 			if updateVersion && !fi.Deleted {
 				ver, err := x.getIdx(i)
 				if err != nil {
-					return "", false, err
+					return "", err
 				}
 				ver.ObjectV2.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal)
 				for k, v := range fi.ReplicationState.ResetStatusesMap {
 					ver.ObjectV2.MetaSys[k] = []byte(v)
 				}
 				err = x.setIdx(i, *ver)
-				return "", len(x.versions) == 0, err
+				return uuid.UUID(ver.ObjectV2.DataDir).String(), err
 			}
 		}
 	}
@@ -1288,7 +1293,7 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 		}
 		ver, err := x.getIdx(i)
 		if err != nil {
-			return "", false, err
+			return "", err
 		}
 		switch {
 		case fi.ExpireRestored:
@@ -1314,16 +1319,16 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 		if x.SharedDataDirCount(ver.ObjectV2.VersionID, ver.ObjectV2.DataDir) > 0 {
 			// Found that another version references the same dataDir
 			// we shouldn't remove it, and only remove the version instead
-			return "", len(x.versions) == 0, nil
+			return "", nil
 		}
-		return uuid.UUID(ver.ObjectV2.DataDir).String(), len(x.versions) == 0, err
+		return uuid.UUID(ver.ObjectV2.DataDir).String(), err
 	}
 
 	if fi.Deleted {
 		err = x.addVersion(ventry)
-		return "", false, err
+		return "", err
 	}
-	return "", false, errFileVersionNotFound
+	return "", errFileVersionNotFound
 }
 
 // xlMetaDataDirDecoder is a shallow decoder for decoding object datadir only.
@@ -1686,7 +1691,7 @@ func (x xlMetaV2) ListVersions(volume, path string) ([]FileInfo, error) {
 // Quorum must be the minimum number of matching metadata files.
 // Quorum should be > 1 and <= len(versions).
 // If strict is set to false, entries that match type
-func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVersion) (merged []xlMetaV2ShallowVersion) {
+func mergeXLV2Versions(quorum int, strict bool, requestedVersions int, versions ...[]xlMetaV2ShallowVersion) (merged []xlMetaV2ShallowVersion) {
 	if quorum <= 0 {
 		quorum = 1
 	}
@@ -1703,6 +1708,8 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
 	// Shallow copy input
 	versions = append(make([][]xlMetaV2ShallowVersion, 0, len(versions)), versions...)
 
+	var nVersions int // captures all non-free versions
+
 	// Our result
 	merged = make([]xlMetaV2ShallowVersion, 0, len(versions[0]))
 	tops := make([]xlMetaV2ShallowVersion, len(versions))
@@ -1739,6 +1746,12 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
 			latest = tops[0]
 			latestCount = len(tops)
 			merged = append(merged, latest)
+
+			// Calculate latest 'n' non-free versions.
+			if !latest.header.FreeVersion() {
+				nVersions++
+			}
+
 		} else {
 			// Find latest.
 			for i, ver := range tops {
@@ -1802,6 +1815,11 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
 			}
 			if latestCount >= quorum {
 				merged = append(merged, latest)
+
+				// Calculate latest 'n' non-free versions.
+				if !latest.header.FreeVersion() {
+					nVersions++
+				}
 			}
 		}
 
@@ -1835,7 +1853,13 @@ func mergeXLV2Versions(quorum int, strict bool, versions ...[]xlMetaV2ShallowVer
 				break
 			}
 		}
+
+		if requestedVersions > 0 && requestedVersions == nVersions {
+			merged = append(merged, versions[0]...)
+			break
+		}
 	}
+
 	// Sanity check. Enable if duplicates show up.
 	if false {
 		found := make(map[[16]byte]struct{})

cmd/xl-storage-format-v2_test.go

@@ -18,10 +18,16 @@
 package cmd
 
 import (
+	"bufio"
 	"bytes"
+	"compress/gzip"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
+	"math/rand"
+	"os"
+	"reflect"
 	"sort"
 	"testing"
 	"time"
@@ -34,6 +40,29 @@ import (
 	"github.com/minio/minio/internal/ioutil"
 )
 
+func TestReadXLMetaNoData(t *testing.T) {
+	f, err := os.Open("testdata/xl.meta-corrupt.gz")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	gz, err := gzip.NewReader(bufio.NewReader(f))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	buf, err := io.ReadAll(gz)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = readXLMetaNoData(bytes.NewReader(buf), int64(len(buf)))
+	if err == nil {
+		t.Fatal("expected error but returned success")
+	}
+}
+
 func TestXLV2FormatData(t *testing.T) {
 	failOnErr := func(err error) {
 		t.Helper()
@@ -366,7 +395,7 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
 	count := len(testCases)
 	for i := 4; i < len(testCases); i++ {
 		tc := testCases[i]
-		dataDir, _, err := xl.DeleteVersion(fileInfos[i])
+		dataDir, err := xl.DeleteVersion(fileInfos[i])
 		failOnErr(count+1, err)
 		if dataDir != tc.expectedDataDir {
 			t.Fatalf("Expected %s but got %s", tc.expectedDataDir, dataDir)
@@ -375,6 +404,55 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
 	}
 }
 
+func Benchmark_mergeXLV2Versions(b *testing.B) {
+	data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
+	if err != nil {
+		b.Fatal(err)
+	}
+	dec, _ := zstd.NewReader(nil)
+	data, err = dec.DecodeAll(data, nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	var xl xlMetaV2
+	if err = xl.LoadOrConvert(data); err != nil {
+		b.Fatal(err)
+	}
+
+	vers := make([][]xlMetaV2ShallowVersion, 16)
+	for i := range vers {
+		vers[i] = xl.versions
+	}
+
+	b.Run("requested-none", func(b *testing.B) {
+		b.ReportAllocs()
+		b.ResetTimer()
+		b.SetBytes(855) // number of versions...
+		for i := 0; i < b.N; i++ {
+			mergeXLV2Versions(8, false, 0, vers...)
+		}
+	})
+
+	b.Run("requested-v1", func(b *testing.B) {
+		b.ReportAllocs()
+		b.ResetTimer()
+		b.SetBytes(855) // number of versions...
+		for i := 0; i < b.N; i++ {
+			mergeXLV2Versions(8, false, 1, vers...)
+		}
+	})
+
+	b.Run("requested-v2", func(b *testing.B) {
+		b.ReportAllocs()
+		b.ResetTimer()
+		b.SetBytes(855) // number of versions...
+		for i := 0; i < b.N; i++ {
+			mergeXLV2Versions(8, false, 1, vers...)
+		}
+	})
+}
+
 func Benchmark_xlMetaV2Shallow_Load(b *testing.B) {
 	data, err := ioutil.ReadFile("testdata/xl.meta-v1.2.zst")
 	if err != nil {
@@ -398,6 +476,7 @@ func Benchmark_xlMetaV2Shallow_Load(b *testing.B) {
 			}
 		}
 	})
+
 	b.Run("indexed", func(b *testing.B) {
 		var xl xlMetaV2
 		err = xl.Load(data)
@@ -524,7 +603,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
 
 	for i := range vers {
 		t.Run(fmt.Sprintf("non-strict-q%d", i), func(t *testing.T) {
-			merged := mergeXLV2Versions(i, false, vers...)
+			merged := mergeXLV2Versions(i, false, 0, vers...)
 			if len(merged) == 0 {
 				t.Error("Did not get any results")
 				return
@@ -536,7 +615,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
 			}
 		})
 		t.Run(fmt.Sprintf("strict-q%d", i), func(t *testing.T) {
-			merged := mergeXLV2Versions(i, true, vers...)
+			merged := mergeXLV2Versions(i, true, 0, vers...)
 			if len(merged) == 0 {
 				t.Error("Did not get any results")
 				return
@@ -558,7 +637,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
 				}
 				vMod = append(vMod, newVers)
 			}
-			merged := mergeXLV2Versions(i, false, vMod...)
+			merged := mergeXLV2Versions(i, false, 0, vMod...)
 			if len(merged) == 0 {
 				t.Error("Did not get any results")
 				return
@@ -580,7 +659,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
 				}
 				vMod = append(vMod, newVers)
 			}
-			merged := mergeXLV2Versions(i, false, vMod...)
+			merged := mergeXLV2Versions(i, false, 0, vMod...)
 			if len(merged) == 0 && i < 2 {
 				t.Error("Did not get any results")
 				return
@@ -606,7 +685,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
 				}
 				vMod = append(vMod, newVers)
 			}
-			merged := mergeXLV2Versions(i, false, vMod...)
+			merged := mergeXLV2Versions(i, false, 0, vMod...)
 			if len(merged) == 0 {
 				t.Error("Did not get any results")
 				return
@@ -628,7 +707,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
 				}
 				vMod = append(vMod, newVers)
 			}
-			merged := mergeXLV2Versions(i, false, vMod...)
+			merged := mergeXLV2Versions(i, false, 0, vMod...)
 			if len(merged) == 0 && i < 2 {
 				t.Error("Did not get any results")
 				return
@@ -654,7 +733,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
 				}
 				vMod = append(vMod, newVers)
 			}
-			merged := mergeXLV2Versions(i, true, vMod...)
+			merged := mergeXLV2Versions(i, true, 0, vMod...)
 			if len(merged) == 0 && i < 2 {
 				t.Error("Did not get any results")
 				return
@@ -680,7 +759,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
 				}
 				vMod = append(vMod, newVers)
 			}
-			merged := mergeXLV2Versions(i, true, vMod...)
+			merged := mergeXLV2Versions(i, true, 0, vMod...)
 			if len(merged) == 0 && i < 2 {
 				t.Error("Did not get any results")
 				return
@@ -706,7 +785,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
 				}
 				vMod = append(vMod, newVers)
 			}
-			merged := mergeXLV2Versions(i, true, vMod...)
+			merged := mergeXLV2Versions(i, true, 0, vMod...)
 			if len(merged) == 0 && i < 2 {
 				t.Error("Did not get any results")
 				return
@@ -732,7 +811,7 @@ func Test_mergeXLV2Versions(t *testing.T) {
 				}
 				vMod = append(vMod, newVers)
 			}
-			merged := mergeXLV2Versions(i, true, vMod...)
+			merged := mergeXLV2Versions(i, true, 0, vMod...)
 			if len(merged) == 0 && i < 2 {
 				t.Error("Did not get any results")
 				return
@@ -749,3 +828,128 @@ func Test_mergeXLV2Versions(t *testing.T) {
 		})
 	}
 }
+
+func Test_mergeXLV2Versions2(t *testing.T) {
+	vDelMarker := xlMetaV2ShallowVersion{header: xlMetaV2VersionHeader{
+		VersionID: [16]byte{2},
+		ModTime:   1500,
+		Signature: [4]byte{5, 6, 7, 8},
+		Type:      DeleteType,
+		Flags:     0,
+	}}
+	vDelMarker.meta, _ = base64.StdEncoding.DecodeString("gqRUeXBlAqZEZWxPYmqDoklExBCvwGEaY+BAO4B4vyG5ERorpU1UaW1l0xbgJlsWE9IHp01ldGFTeXOA")
+
+	vObj := xlMetaV2ShallowVersion{header: xlMetaV2VersionHeader{
+		VersionID: [16]byte{1},
+		ModTime:   1000,
+		Signature: [4]byte{1, 2, 3, 4},
+		Type:      ObjectType,
+		Flags:     xlFlagUsesDataDir | xlFlagInlineData,
+	}}
+	vObj.meta, _ = base64.StdEncoding.DecodeString("gqRUeXBlAaVWMk9iat4AEaJJRMQQEkaOteYCSrWB3nqppSIKTqRERGlyxBAO8fXSJ5RI+YEtsp8KneVVpkVjQWxnbwGjRWNNDKNFY04Ep0VjQlNpemXSABAAAKdFY0luZGV4BaZFY0Rpc3TcABAFBgcICQoLDA0ODxABAgMEqENTdW1BbGdvAahQYXJ0TnVtc5EBqVBhcnRFVGFnc8CpUGFydFNpemVzkdEBL6pQYXJ0QVNpemVzkdEBL6RTaXpl0QEvpU1UaW1l0xbgJhIa6ABvp01ldGFTeXOBvHgtbWluaW8taW50ZXJuYWwtaW5saW5lLWRhdGHEBHRydWWnTWV0YVVzcoKsY29udGVudC10eXBluGFwcGxpY2F0aW9uL29jdGV0LXN0cmVhbaRldGFn2SBlYTIxMDE2MmVlYjRhZGMzMWZmOTg0Y2I3NDRkNmFmNg==")
+
+	testCases := []struct {
+		name        string
+		input       [][]xlMetaV2ShallowVersion
+		quorum      int
+		reqVersions int
+		want        []xlMetaV2ShallowVersion
+	}{
+		{
+			name: "obj-on-one",
+			input: [][]xlMetaV2ShallowVersion{
+				0: {vDelMarker, vObj}, // disk 0
+				1: {vDelMarker},       // disk 1
+				2: {vDelMarker},       // disk 2
+			},
+			quorum:      2,
+			reqVersions: 0,
+			want:        []xlMetaV2ShallowVersion{vDelMarker},
+		},
+		{
+			name: "obj-on-two",
+			input: [][]xlMetaV2ShallowVersion{
+				0: {vDelMarker, vObj}, // disk 0
+				1: {vDelMarker, vObj}, // disk 1
+				2: {vDelMarker},       // disk 2
+			},
+			quorum:      2,
+			reqVersions: 0,
+			want:        []xlMetaV2ShallowVersion{vDelMarker, vObj},
+		},
+		{
+			name: "obj-on-all",
+			input: [][]xlMetaV2ShallowVersion{
+				0: {vDelMarker, vObj}, // disk 0
+				1: {vDelMarker, vObj}, // disk 1
+				2: {vDelMarker, vObj}, // disk 2
+			},
+			quorum:      2,
+			reqVersions: 0,
+			want:        []xlMetaV2ShallowVersion{vDelMarker, vObj},
+		},
+		{
+			name: "del-on-one",
+			input: [][]xlMetaV2ShallowVersion{
+				0: {vDelMarker, vObj}, // disk 0
+				1: {vObj},             // disk 1
+				2: {vObj},             // disk 2
+			},
+			quorum:      2,
+			reqVersions: 0,
+			want:        []xlMetaV2ShallowVersion{vObj},
+		},
+		{
+			name: "del-on-two",
+			input: [][]xlMetaV2ShallowVersion{
+				0: {vDelMarker, vObj}, // disk 0
+				1: {vDelMarker, vObj}, // disk 1
+				2: {vObj},             // disk 2
+			},
+			quorum:      2,
+			reqVersions: 0,
+			want:        []xlMetaV2ShallowVersion{vDelMarker, vObj},
+		},
+		{
+			name: "del-on-two-16stripe",
+			input: [][]xlMetaV2ShallowVersion{
+				0:  {vObj},             // disk 0
+				1:  {vDelMarker, vObj}, // disk 1
+				2:  {vDelMarker, vObj}, // disk 2
+				3:  {vDelMarker, vObj}, // disk 3
+				4:  {vDelMarker, vObj}, // disk 4
+				5:  {vDelMarker, vObj}, // disk 5
+				6:  {vDelMarker, vObj}, // disk 6
+				7:  {vDelMarker, vObj}, // disk 7
+				8:  {vDelMarker, vObj}, // disk 8
+				9:  {vDelMarker, vObj}, // disk 9
+				10: {vObj},             // disk 10
+				11: {vDelMarker, vObj}, // disk 11
+				12: {vDelMarker, vObj}, // disk 12
+				13: {vDelMarker, vObj}, // disk 13
+				14: {vDelMarker, vObj}, // disk 14
+				15: {vDelMarker, vObj}, // disk 15
+			},
+			quorum:      7,
+			reqVersions: 0,
+			want:        []xlMetaV2ShallowVersion{vDelMarker, vObj},
+		},
+	}
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			// Run multiple times, shuffling the input order.
+			for i := int64(0); i < 50; i++ {
+				t.Run(fmt.Sprint(i), func(t *testing.T) {
+					rng := rand.New(rand.NewSource(i))
+					rng.Shuffle(len(test.input), func(i, j int) {
+						test.input[i], test.input[j] = test.input[j], test.input[i]
+					})
+					got := mergeXLV2Versions(test.quorum, true, 0, test.input...)
+					if !reflect.DeepEqual(test.want, got) {
+						t.Errorf("want %v != got %v", test.want, got)
+					}
+				})
+			}
+		})
+	}
+}

cmd/xl-storage-format_test.go

@@ -433,7 +433,7 @@ func BenchmarkXlMetaV2Shallow(b *testing.B) {
 					// Update a random version.
 					fi.VersionID = ids[rng.Intn(size)]
 					// Delete...
-					_, _, err = xl.DeleteVersion(fi)
+					_, err = xl.DeleteVersion(fi)
 					if err != nil {
 						b.Fatal(err)
 					}

cmd/xl-storage-free-version_test.go

@@ -102,7 +102,7 @@ func TestFreeVersion(t *testing.T) {
 	tierfi.TransitionedObjName = mustGetUUID()
 	tierfi.TransitionTier = "MINIOTIER-1"
 	var err error
-	_, _, err = xl.DeleteVersion(tierfi)
+	_, err = xl.DeleteVersion(tierfi)
 	fatalErr(err)
 	report()
 
@@ -124,7 +124,7 @@ func TestFreeVersion(t *testing.T) {
 	newtierfi.TransitionStatus = ""
 	newtierfi.SetTierFreeVersionID(fvIDs[1])
 	report()
-	_, _, err = xl.DeleteVersion(newtierfi)
+	_, err = xl.DeleteVersion(newtierfi)
 	report()
 	fatalErr(err)
 
@@ -141,7 +141,7 @@ func TestFreeVersion(t *testing.T) {
 	freefi := newtierfi
 	for _, fvID := range fvIDs {
 		freefi.VersionID = fvID
-		_, _, err = xl.DeleteVersion(freefi)
+		_, err = xl.DeleteVersion(freefi)
 		fatalErr(err)
 	}
 	report()

cmd/xl-storage.go

@@ -38,6 +38,7 @@ import (
 
 	"github.com/dustin/go-humanize"
 	jsoniter "github.com/json-iterator/go"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/disk"
@@ -49,11 +50,12 @@ import (
 
 const (
 	nullVersionID = "null"
-	// Really large streams threshold per shard.
-	reallyLargeFileThreshold = 64 * humanize.MiByte // Optimized for HDDs
+	// Largest streams threshold per shard.
+	largestFileThreshold = 64 * humanize.MiByte // Optimized for HDDs
 
 	// Small file threshold below which data accompanies metadata from storage layer.
 	smallFileThreshold = 128 * humanize.KiByte // Optimized for NVMe/SSDs
+
 	// For hardrives it is possible to set this to a lower value to avoid any
 	// spike in latency. But currently we are simply keeping it optimal for SSDs.
 
@@ -397,6 +399,9 @@ func (s *xlStorage) readMetadataWithDMTime(ctx context.Context, itemPath string)
 		}
 	}
 	buf, err := readXLMetaNoData(f, stat.Size())
+	if err != nil {
+		return nil, stat.ModTime().UTC(), fmt.Errorf("%w -> %s", err, itemPath)
+	}
 	return buf, stat.ModTime().UTC(), err
 }
 
@@ -405,7 +410,7 @@ func (s *xlStorage) readMetadata(ctx context.Context, itemPath string) ([]byte,
 	return buf, err
 }
 
-func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error) {
+func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode) (dataUsageCache, error) {
 	// Updates must be closed before we return.
 	defer close(updates)
 	var lc *lifecycle.Lifecycle
@@ -520,7 +525,7 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 			item.applyTierObjSweep(ctx, objAPI, oi)
 		}
 		return sizeS, nil
-	})
+	}, scanMode)
 	if err != nil {
 		return dataUsageInfo, err
 	}
@@ -683,9 +688,11 @@ func (s *xlStorage) SetDiskID(id string) {
 
 func (s *xlStorage) MakeVolBulk(ctx context.Context, volumes ...string) error {
 	for _, volume := range volumes {
-		if err := s.MakeVol(ctx, volume); err != nil && !errors.Is(err, errVolumeExists) {
+		err := s.MakeVol(ctx, volume)
+		if err != nil && !errors.Is(err, errVolumeExists) {
 			return err
 		}
+		diskHealthCheckOK(ctx, err)
 	}
 	return nil
 }
@@ -880,22 +887,15 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 		return err
 	}
 
-	var (
-		dataDir     string
-		lastVersion bool
-		updated     bool
-	)
-
 	for _, fi := range fis {
-		dataDir, lastVersion, err = xlMeta.DeleteVersion(fi)
+		dataDir, err := xlMeta.DeleteVersion(fi)
 		if err != nil {
 			if !fi.Deleted && (err == errFileNotFound || err == errFileVersionNotFound) {
-				// Ignore these since
+				// Ignore these since they do not exist
 				continue
 			}
 			return err
 		}
-		updated = true
 		if dataDir != "" {
 			versionID := fi.VersionID
 			if versionID == "" {
@@ -918,11 +918,8 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 		}
 	}
 
+	lastVersion := len(xlMeta.versions) == 0
 	if !lastVersion {
-		if !updated {
-			return nil
-		}
-
 		buf, err = xlMeta.AppendTo(metaDataPoolGet())
 		defer metaDataPoolPut(buf)
 		if err != nil {
@@ -933,12 +930,7 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 	}
 
 	// Move xl.meta to trash
-	filePath := pathJoin(volumeDir, path, xlStorageFormatFile)
-	if err = checkPathLength(filePath); err != nil {
-		return err
-	}
-
-	err = s.moveToTrash(filePath, false)
+	err = s.moveToTrash(pathJoin(volumeDir, path, xlStorageFormatFile), false)
 	if err == nil || err == errFileNotFound {
 		s.deleteFile(volumeDir, pathJoin(volumeDir, path), false)
 	}
@@ -958,6 +950,7 @@ func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions
 		if err := s.deleteVersions(ctx, volume, fiv.Name, fiv.Versions...); err != nil {
 			errs[i] = err
 		}
+		diskHealthCheckOK(ctx, errs[i])
 	}
 
 	return errs
@@ -1021,7 +1014,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		return err
 	}
 
-	dataDir, lastVersion, err := xlMeta.DeleteVersion(fi)
+	dataDir, err := xlMeta.DeleteVersion(fi)
 	if err != nil {
 		return err
 	}
@@ -1046,7 +1039,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		}
 	}
 
-	if !lastVersion {
+	if len(xlMeta.versions) != 0 {
 		buf, err = xlMeta.AppendTo(metaDataPoolGet())
 		defer metaDataPoolPut(buf)
 		if err != nil {
@@ -1397,7 +1390,7 @@ func (s *xlStorage) readAllData(ctx context.Context, volumeDir string, filePath
 		buf = make([]byte, sz)
 	}
 	// Read file...
-	_, err = io.ReadFull(r, buf)
+	_, err = io.ReadFull(diskHealthReader(ctx, r), buf)
 
 	return buf, stat.ModTime().UTC(), osErrToFileErr(err)
 }
@@ -1708,7 +1701,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 	r := struct {
 		io.Reader
 		io.Closer
-	}{Reader: io.LimitReader(or, length), Closer: closeWrapper(func() error {
+	}{Reader: io.LimitReader(diskHealthReader(ctx, or), length), Closer: closeWrapper(func() error {
 		if !alignment || offset+length%xioutil.DirectioAlignSize != 0 {
 			// invalidate page-cache for unaligned reads.
 			if !globalAPIConfig.isDisableODirect() {
@@ -1798,8 +1791,8 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 	}()
 
 	var bufp *[]byte
-	if fileSize > 0 && fileSize >= reallyLargeFileThreshold {
-		// use a larger 4MiB buffer for really large streams.
+	if fileSize > 0 && fileSize >= largestFileThreshold {
+		// use a larger 4MiB buffer for a really large streams.
 		bufp = xioutil.ODirectPoolXLarge.Get().(*[]byte)
 		defer xioutil.ODirectPoolXLarge.Put(bufp)
 	} else {
@@ -1807,7 +1800,7 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 		defer xioutil.ODirectPoolLarge.Put(bufp)
 	}
 
-	written, err := xioutil.CopyAligned(w, r, *bufp, fileSize)
+	written, err := xioutil.CopyAligned(diskHealthWriter(ctx, w), r, *bufp, fileSize, w)
 	if err != nil {
 		return err
 	}
@@ -2039,7 +2032,8 @@ func skipAccessChecks(volume string) (ok bool) {
 // RenameData - rename source path to destination path atomically, metadata and data directory.
 func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) (err error) {
 	defer func() {
-		if err != nil {
+		if err != nil && !contextCanceled(ctx) {
+			// Only log these errors if context is not yet canceled.
 			logger.LogIf(ctx, fmt.Errorf("srcVolume: %s, srcPath: %s, dstVolume: %s:, dstPath: %s - error %v",
 				srcVolume, srcPath,
 				dstVolume, dstPath,
@@ -2283,6 +2277,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 			}
 			return osErrToFileErr(err)
 		}
+		diskHealthCheckOK(ctx, err)
 
 		// renameAll only for objects that have xl.meta not saved inline.
 		if len(fi.Data) == 0 && fi.Size > 0 {
@@ -2420,7 +2415,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
 	return nil
 }
 
-func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
+func (s *xlStorage) bitrotVerify(ctx context.Context, partPath string, partSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
 	// Open the file for reading.
 	file, err := Open(partPath)
 	if err != nil {
@@ -2435,7 +2430,7 @@ func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlg
 		// for healing code to fix this file.
 		return err
 	}
-	return bitrotVerify(file, fi.Size(), partSize, algo, sum, shardSize)
+	return bitrotVerify(diskHealthReader(ctx, file), fi.Size(), partSize, algo, sum, shardSize)
 }
 
 func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
@@ -2460,7 +2455,7 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File
 	for _, part := range fi.Parts {
 		checksumInfo := erasure.GetChecksumInfo(part.Number)
 		partPath := pathJoin(volumeDir, path, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
-		if err := s.bitrotVerify(partPath,
+		if err := s.bitrotVerify(ctx, partPath,
 			erasure.ShardFileSize(part.Size),
 			checksumInfo.Algorithm,
 			checksumInfo.Hash, erasure.ShardSize()); err != nil {

cmd/xl-storage_test.go

@@ -1884,7 +1884,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
 	if err := storage.WriteAll(context.Background(), volName, fileName, data); err != nil {
 		t.Fatal(err)
 	}
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1894,12 +1894,12 @@ func TestXLStorageVerifyFile(t *testing.T) {
 	}
 
 	// Check if VerifyFile reports the incorrect file length (the correct length is `size+1`)
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 
 	// Check if bitrot fails
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size+1, algo, hashBytes, 0); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size+1, algo, hashBytes, 0); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 
@@ -1928,7 +1928,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	w.(io.Closer).Close()
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, nil, shardSize); err != nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, nil, shardSize); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1943,10 +1943,10 @@ func TestXLStorageVerifyFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	f.Close()
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, nil, shardSize); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, nil, shardSize); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size+1, algo, nil, shardSize); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size+1, algo, nil, shardSize); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 }

docs/hotfixes.md

@@ -88,9 +88,9 @@ Date:   Mon Nov 8 08:41:27 2021 -0800
 λ git cherry-pick 4f3317effea38c203c358af9cb5ce3c0e4173976
 ```
 
-*A self contained **patch** usually applies fine on the hotfix branch during backports as long it is self contained. There are situations however this may lead to conflicts and the patch will not cleanly apply. Conflicts might be trivial which can be resolved easily, when conflicts seem to be non-trivial or touches the part of the code-base the developer is not aware of - to get additional clarity reaching out of #hack on MinIOHQ slack channel is advised.*
+*A self contained **patch** usually applies fine on the hotfix branch during backports as long it is self contained. There are situations however this may lead to conflicts and the patch will not cleanly apply. Conflicts might be trivial which can be resolved easily, when conflicts seem to be non-trivial or touches the part of the code-base the developer is not confident - to get additional clarity reach out to #hack on MinIOHQ slack channel. Hasty changes must be avoided, minor fixes and logs may be added to hotfix branches but this should not be followed as practice.*
 
-Once the **patch** is successfully applied, developer must run tests to alidate the fix that was backported by running following tests, locally.
+Once the **patch** is successfully applied, developer must run tests to validate the fix that was backported by running following tests, locally.
 
 Unit tests
 
@@ -116,16 +116,16 @@ At this point in time the backport is ready to be submitted as a pull request to
 
 To add a hotfix tag to the binary version and embed the relevant `commit-id` following build helpers are available
 
-#### Builds the hotfix binary
+#### Builds the hotfix binary and uploads to https;//dl.min.io
 
 ```
-λ CRED_DIR=/media/builder/minio make hotfix
+λ CRED_DIR=/media/builder/minio make hotfix-push
 ```
 
-#### Builds the hotfix container
+#### Builds the hotfix container and pushes to docker.io/minio/minio
 
 ```
-λ CRED_DIR=/media/builder/minio make docker-hotfix
+λ CRED_DIR=/media/builder/minio make docker-hotfix-push
 ```
 
 Once this has been provided to the customer relevant binary will be uploaded from our *release server* securely, directly to <https://dl.minio.io/server/minio/hotfixes/archive/>

docs/kms/README.md

@@ -120,6 +120,20 @@ Encrypted :
   X-Amz-Server-Side-Encryption: AES256
 ```
 
+## Encrypted Private Key
+
+MinIO supports encrypted KES client private keys. Therefore, you can use
+an password-protected private keys for `MINIO_KMS_KES_KEY_FILE`. 
+
+When using password-protected private keys for accessing KES you need to
+provide the password via:
+```
+export MINIO_KMS_KES_KEY_PASSWORD=<your-password>
+``` 
+
+Note that MinIO only supports encrypted private keys - not encrypted certificates.
+Certificates are no secrets and sent in plaintext as part of the TLS handshake.
+
 ## Explore Further
 
 - [Use `mc` with MinIO Server](https://docs.min.io/docs/minio-client-quickstart-guide)

docs/kms/kes-config.toml

@@ -1,33 +0,0 @@
-# The address:port of the kes server - i.e. on the local machine.
-address = "127.0.0.1:7373"
-
-[tls]
-key  = "./kes-tls.key"
-cert = "./kes-tls.crt" 
-
-[policy.minio]
-paths = [
-          "/v1/key/create/minio-*",
-          "/v1/key/generate/minio-*",
-          "/v1/key/decrypt/minio-*"
-        ]
-identities = [ "dd46485bedc9ad2909d2e8f9017216eec4413bc5c64b236d992f7ec19c843c5f" ]
-
-[cache.expiry]
-all    = "5m" 
-unused = "20s" 
-
-[keystore.vault]
-address   = "https://127.0.0.1:8200"  # The Vault endpoint - i.e. https://127.0.0.1:8200
-name      = "minio"                   # The domain resp. prefix at Vault's K/V backend 
-
-[keystore.vault.approle]
-id     = ""    # Your AppRole Role ID 
-secret = ""    # Your AppRole Secret ID
-retry  = "15s" # Duration until the server tries to re-authenticate after connection loss.
-
-[keystore.vault.tls]
-ca = "./vault-tls.crt" # Since we use self-signed certificates
-
-[keystore.vault.status]
-ping = "10s"

docs/metrics/prometheus/list.md

@@ -55,7 +55,7 @@ These metrics can be from any MinIO server once per collection.
 | `minio_s3_requests_error_total`              | Total number S3 requests with errors                                                                                |
 | `minio_s3_requests_inflight_total`           | Total number of S3 requests currently in flight                                                                     |
 | `minio_s3_requests_total`                    | Total number S3 requests                                                                                            |
-| `minio_s3_time_ttbf_seconds_distribution`    | Distribution of the time to first byte across API calls.                                                            |
+| `minio_s3_time_ttfb_seconds_distribution`    | Distribution of the time to first byte across API calls.                                                            |
 | `minio_s3_traffic_received_bytes`            | Total number of s3 bytes received.                                                                                  |
 | `minio_s3_traffic_sent_bytes`                | Total number of s3 bytes sent                                                                                       |
 | `minio_software_commit_info`                 | Git commit hash for the MinIO release.                                                                              |

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,14 +2,14 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-02-26T02-54-46Z
+  image: quay.io/minio/minio:RELEASE.2022-04-08T19-44-35Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"
     - "9001"
-  environment:
-    MINIO_ROOT_USER: minio
-    MINIO_ROOT_PASSWORD: minio123
+  # environment:
+    # MINIO_ROOT_USER: minioadmin
+    # MINIO_ROOT_PASSWORD: minioadmin
   healthcheck:
     test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
     interval: 30s

docs/orchestration/docker-compose/nginx.conf

@@ -49,6 +49,7 @@ http {
         client_max_body_size 0;
         # To disable buffering
         proxy_buffering off;
+        proxy_request_buffering off;
 
         location / {
             proxy_set_header Host $http_host;
@@ -78,6 +79,7 @@ http {
         client_max_body_size 0;
         # To disable buffering
         proxy_buffering off;
+        proxy_request_buffering off;
 
         location / {
             proxy_set_header Host $http_host;

docs/site-replication/run-multi-site-minio-idp.sh

@@ -173,6 +173,17 @@ if [ $? -ne 0 ]; then
     exit_1;
 fi
 
+err_minio2=$(./mc stat minio2/newbucket/xxx --json | jq -r .error.cause.message)
+if [ $? -ne 0 ]; then
+    echo "expecting object to be missing. exiting.."
+    exit_1;
+fi
+
+if [ "${err_minio2}" != "Object does not exist" ]; then
+    echo "expected to see Object does not exist error, exiting..."
+    exit_1;
+fi
+
 ./mc cp README.md minio2/newbucket/
 
 sleep 5

docs/sts/ldap.md

@@ -34,17 +34,17 @@ KEY:
 identity_ldap  enable LDAP SSO support
 
 ARGS:
-MINIO_IDENTITY_LDAP_SERVER_ADDR*            (address)   AD/LDAP server address e.g. "myldapserver.com:636"
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN          (string)    DN for LDAP read-only service account used to perform DN and group lookups
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD    (string)    Password for LDAP read-only service account used to perform DN and group lookups
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN  (string)    ";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER   (string)    Search filter to lookup user DN
-MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER     (string)    search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"
-MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN    (list)      ";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"
-MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY         (on|off)    trust server TLS without verification, defaults to "off" (verify)
-MINIO_IDENTITY_LDAP_SERVER_INSECURE         (on|off)    allow plain text connection to AD/LDAP server, defaults to "off"
-MINIO_IDENTITY_LDAP_SERVER_STARTTLS         (on|off)    use StartTLS connection to AD/LDAP server, defaults to "off"
-MINIO_IDENTITY_LDAP_COMMENT                 (sentence)  optionally add a comment to this setting
+MINIO_IDENTITY_LDAP_SERVER_ADDR*             (address)   AD/LDAP server address e.g. "myldapserver.com:636"
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN*          (string)    DN for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD     (string)    Password for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN*  (list)      ";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER*   (string)    Search filter to lookup user DN
+MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER      (string)    search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"
+MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN     (list)      ";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"
+MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY          (on|off)    trust server TLS without verification, defaults to "off" (verify)
+MINIO_IDENTITY_LDAP_SERVER_INSECURE          (on|off)    allow plain text connection to AD/LDAP server, defaults to "off"
+MINIO_IDENTITY_LDAP_SERVER_STARTTLS          (on|off)    use StartTLS connection to AD/LDAP server, defaults to "off"
+MINIO_IDENTITY_LDAP_COMMENT                  (sentence)  optionally add a comment to this setting
 ```
 
 ### LDAP server connectivity
@@ -69,8 +69,8 @@ If a self-signed certificate is being used, the certificate can be added to MinI
 A low-privilege read-only LDAP service account is configured in the MinIO server by providing the account's Distinguished Name (DN) and password. This service account is used to perform directory lookups as needed.
 
 ```
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN          (string)    DN for LDAP read-only service account used to perform DN and group lookups
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD    (string)    Password for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN*          (string)    DN for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD     (string)    Password for LDAP read-only service account used to perform DN and group lookups
 ```
 
 If you set an empty lookup bind password, the lookup bind will use the unauthenticated authentication mechanism, as described in [RFC 4513 Section 5.1.2](https://tools.ietf.org/html/rfc4513#section-5.1.2).
@@ -80,8 +80,8 @@ If you set an empty lookup bind password, the lookup bind will use the unauthent
 When a user provides their LDAP credentials, MinIO runs a lookup query to find the user's Distinguished Name (DN). The search filter and base DN used in this lookup query are configured via the following variables:
 
 ```
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN  (string)    Base LDAP DN to search for user DN
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER   (string)    Search filter to lookup user DN
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN*  (list)      ";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER*   (string)    Search filter to lookup user DN
 ```
 
 The search filter must use the LDAP username to find the user DN. This is done via [variable substitution](#variable-substitution-in-configuration-strings).
@@ -110,7 +110,7 @@ export MINIO_IDENTITY_LDAP_SERVER_ADDR=myldapserver.com:636
 export MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN='cn=admin,dc=min,dc=io'
 export MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD=admin
 export MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN='ou=hwengg,dc=min,dc=io'
-export MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER='(uid=%s,cn=accounts,dc=min,dc=io)'
+export MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER='(uid=%s)'
 export MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY=on
 ```
 

go.mod

@@ -27,9 +27,9 @@ require (
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.2
 	github.com/go-ldap/ldap/v3 v3.2.4
-	github.com/go-openapi/loads v0.21.0
+	github.com/go-openapi/loads v0.21.1
 	github.com/go-sql-driver/mysql v1.6.0
-	github.com/golang-jwt/jwt/v4 v4.2.0
+	github.com/golang-jwt/jwt/v4 v4.3.0
 	github.com/gomodule/redigo v1.8.8
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
@@ -45,15 +45,15 @@ require (
 	github.com/lib/pq v1.10.4
 	github.com/miekg/dns v1.1.46
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.15.0
+	github.com/minio/console v0.15.9
 	github.com/minio/csvparser v1.0.0
-	github.com/minio/dperf v0.3.2
+	github.com/minio/dperf v0.3.4
 	github.com/minio/highwayhash v1.0.2
-	github.com/minio/kes v0.18.0
+	github.com/minio/kes v0.19.0
 	github.com/minio/madmin-go v1.3.5
 	github.com/minio/minio-go/v7 v7.0.23
 	github.com/minio/parquet-go v1.1.0
-	github.com/minio/pkg v1.1.18
+	github.com/minio/pkg v1.1.20
 	github.com/minio/selfupdate v0.4.0
 	github.com/minio/sha256-simd v1.0.0
 	github.com/minio/simdjson-go v0.4.2
@@ -61,8 +61,8 @@ require (
 	github.com/minio/zipindex v0.2.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/montanaflynn/stats v0.6.6
-	github.com/nats-io/nats-server/v2 v2.7.2
-	github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d
+	github.com/nats-io/nats-server/v2 v2.7.4
+	github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d
 	github.com/nats-io/stan.go v0.10.2
 	github.com/ncw/directio v1.0.5
 	github.com/nsqio/go-nsq v1.0.8
@@ -75,7 +75,7 @@ require (
 	github.com/rs/cors v1.7.0
 	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
 	github.com/secure-io/sio-go v0.3.1
-	github.com/shirou/gopsutil/v3 v3.21.12
+	github.com/shirou/gopsutil/v3 v3.22.2
 	github.com/streadway/amqp v1.0.0
 	github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e
 	github.com/valyala/bytebufferpool v1.0.0
@@ -85,11 +85,11 @@ require (
 	go.etcd.io/etcd/api/v3 v3.5.2
 	go.etcd.io/etcd/client/v3 v3.5.2
 	go.uber.org/atomic v1.9.0
-	go.uber.org/zap v1.19.1
-	golang.org/x/crypto v0.0.0-20220128200615-198e4374d7ed
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
-	golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27
-	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
+	go.uber.org/zap v1.21.0
+	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
+	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
+	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9
+	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
 	google.golang.org/api v0.67.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -102,14 +102,13 @@ require (
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/apache/thrift v0.15.0 // indirect
-	github.com/armon/go-metrics v0.3.3 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
+	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.2.0 // indirect
-	github.com/charmbracelet/bubbles v0.10.0 // indirect
-	github.com/charmbracelet/bubbletea v0.19.3 // indirect
+	github.com/charmbracelet/bubbles v0.10.3 // indirect
+	github.com/charmbracelet/bubbletea v0.20.0 // indirect
 	github.com/charmbracelet/lipgloss v0.5.0 // indirect
-	github.com/containerd/console v1.0.2 // indirect
+	github.com/containerd/console v1.0.3 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -119,35 +118,31 @@ require (
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
-	github.com/frankban/quicktest v1.14.0 // indirect
+	github.com/gdamore/encoding v1.0.0 // indirect
+	github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect
-	github.com/go-logr/logr v1.2.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/analysis v0.21.2 // indirect
 	github.com/go-openapi/errors v0.20.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.6 // indirect
-	github.com/go-openapi/runtime v0.21.1 // indirect
+	github.com/go-openapi/runtime v0.23.1 // indirect
 	github.com/go-openapi/spec v0.20.4 // indirect
-	github.com/go-openapi/strfmt v0.21.1 // indirect
-	github.com/go-openapi/swag v0.19.15 // indirect
-	github.com/go-openapi/validate v0.20.3 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/goccy/go-json v0.8.1 // indirect
+	github.com/go-openapi/strfmt v0.21.2 // indirect
+	github.com/go-openapi/swag v0.21.1 // indirect
+	github.com/go-openapi/validate v0.21.0 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/goccy/go-json v0.9.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-cmp v0.5.7 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
@@ -155,13 +150,13 @@ require (
 	github.com/jcmturner/gofork v1.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
-	github.com/jessevdk/go-flags v1.4.0 // indirect
+	github.com/jessevdk/go-flags v1.5.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.0 // indirect
 	github.com/lestrrat-go/httpcc v1.0.0 // indirect
 	github.com/lestrrat-go/iter v1.0.1 // indirect
-	github.com/lestrrat-go/jwx v1.2.14 // indirect
+	github.com/lestrrat-go/jwx v1.2.19 // indirect
 	github.com/lestrrat-go/option v1.0.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
@@ -174,73 +169,58 @@ require (
 	github.com/minio/argon2 v1.0.0 // indirect
 	github.com/minio/colorjson v1.0.1 // indirect
 	github.com/minio/filepath v1.0.0 // indirect
-	github.com/minio/mc v0.0.0-20220204044644-e048c85d71a7 // indirect
+	github.com/minio/mc v0.0.0-20220302011226-f13defa54577 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/operator v0.0.0-20220110040724-a5d59a342b7f // indirect
-	github.com/mitchellh/mapstructure v1.4.2 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect
+	github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
-	github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0 // indirect
+	github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739 // indirect
 	github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 // indirect
 	github.com/nats-io/nats-streaming-server v0.24.1 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/navidys/tvxwidgets v0.1.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/onsi/ginkgo v1.16.5 // indirect
-	github.com/onsi/gomega v1.16.0 // indirect
-	github.com/pkg/profile v1.6.0 // indirect
-	github.com/pkg/xattr v0.4.4 // indirect
+	github.com/pkg/xattr v0.4.5 // indirect
 	github.com/posener/complete v1.2.3 // indirect
-	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
-	github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 // indirect
+	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
+	github.com/pquerna/cachecontrol v0.1.0 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/rivo/tview v0.0.0-20220216162559-96063d6082f3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rjeczalik/notify v0.9.2 // indirect
-	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/rs/xid v1.3.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/tidwall/gjson v1.12.1 // indirect
+	github.com/tidwall/gjson v1.14.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.3 // indirect
-	github.com/tklauser/go-sysconf v0.3.9 // indirect
-	github.com/tklauser/numcpus v0.3.0 // indirect
-	github.com/unrolled/secure v1.0.9 // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.4.0 // indirect
+	github.com/unrolled/secure v1.10.0 // indirect
 	github.com/xdg/stringprep v1.0.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect
-	go.mongodb.org/mongo-driver v1.7.5 // indirect
+	go.mongodb.org/mongo-driver v1.8.4 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	go.uber.org/multierr v1.7.0 // indirect
-	golang.org/x/mod v0.5.1 // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	golang.org/x/mod v0.4.2 // indirect
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
-	golang.org/x/tools v0.1.8 // indirect
+	golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00 // indirect
+	google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a // indirect
 	google.golang.org/grpc v1.44.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/h2non/filetype.v1 v1.0.5 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.66.3 // indirect
+	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/api v0.23.3 // indirect
-	k8s.io/apimachinery v0.23.3 // indirect
-	k8s.io/client-go v0.23.3 // indirect
-	k8s.io/klog/v2 v2.40.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
-	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
 	maze.io/x/duration v0.0.0-20160924141736-faac084b6075 // indirect
-	sigs.k8s.io/controller-runtime v0.8.0 // indirect
-	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
 )