Bump github.com/nats-io/nats-server/v2 from 2.7.2 to 2.7.4

Fixes: #14531

.github/workflows/go-healing.yml

@@ -37,10 +37,7 @@ jobs:
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
-          MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
-          MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
-          MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
-          MINIO_KMS_KES_KEY_NAME: "my-minio-key"
+          MINIO_KMS_SECRET_KEY: "my-minio-key:oyArl7zlPECEduNbB1KXgdzDn2Bdpvvw0l8VO51HQnY="
           MINIO_KMS_AUTO_ENCRYPTION: on
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0

cmd/admin-handlers.go

@@ -23,6 +23,7 @@ import (
 	"crypto/subtle"
 	"crypto/tls"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"hash/crc32"
 	"io"
@@ -935,6 +936,50 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 }
 
+// NetperfHandler - perform mesh style network throughput test
+func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "NetperfHandler")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if !globalIsDistErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "netperf")
+	lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
+		return
+	}
+	defer nsLock.Unlock(lkctx.Cancel)
+
+	durationStr := r.Form.Get(peerRESTDuration)
+	duration, err := time.ParseDuration(durationStr)
+	if err != nil {
+		duration = globalNetPerfMinDuration
+	}
+
+	if duration < globalNetPerfMinDuration {
+		// We need sample size of minimum 10 secs.
+		duration = globalNetPerfMinDuration
+	}
+
+	duration = duration.Round(time.Second)
+
+	results := globalNotificationSys.Netperf(ctx, duration)
+	enc := json.NewEncoder(w)
+	if err := enc.Encode(madmin.NetperfResult{NodeResults: results}); err != nil {
+		return
+	}
+}
+
 // SpeedtestHandler - Deprecated. See ObjectSpeedtestHandler
 func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	a.ObjectSpeedtestHandler(w, r)
@@ -2427,7 +2472,9 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		}
 		return nil
 	})
-	logger.LogIf(ctx, err)
+	if !errors.Is(err, errFileNotFound) {
+		logger.LogIf(ctx, err)
+	}
 }
 
 func createHostAnonymizerForFSMode() map[string]string {

cmd/admin-heal-ops.go

@@ -700,8 +700,9 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	}
 	if source.opts != nil {
 		task.opts = *source.opts
+	} else {
+		task.opts.ScanMode = globalHealConfig.ScanMode()
 	}
-	task.opts.ScanMode = globalHealConfig.ScanMode()
 
 	h.mutex.Lock()
 	h.scannedItemsMap[healType]++

cmd/admin-router.go

@@ -224,7 +224,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedtestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedtestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(httpTraceHdrs(adminAPI.DriveSpeedtestHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetSpeedtestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetperfHandler))
 
 		// HTTP Trace
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(gz(http.HandlerFunc(adminAPI.TraceHandler)))

cmd/api-headers.go

@@ -126,6 +126,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 
 	// Set all other user defined metadata.
 	for k, v := range objInfo.UserDefined {
+		// Empty values for object lock and retention can be skipped.
+		if v == "" && equals(k, xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate) {
+			continue
+		}
+
 		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			// Do not need to send any internal metadata
 			// values to client.

cmd/background-newdisks-heal-ops.go

@@ -263,7 +263,7 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 
 	if drivesToHeal := globalBackgroundHealState.healDriveCount(); drivesToHeal > 0 {
-		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content...",
+		logger.Info(fmt.Sprintf("Found drives to heal %d, waiting until %s to heal the content - use 'mc admin heal alias/ --verbose' to check the status",
 			drivesToHeal, defaultMonitorNewDiskInterval))
 
 		// Heal any disk format and metadata early, if possible.
@@ -277,14 +277,6 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 		}
 	}
 
-	if err := bgSeq.healDiskMeta(objAPI); err != nil {
-		if newObjectLayerFn() != nil {
-			// log only in situations, when object layer
-			// has fully initialized.
-			logger.LogIf(bgSeq.ctx, err)
-		}
-	}
-
 	go monitorLocalDisksAndHeal(ctx, z, bgSeq)
 }
 
@@ -333,7 +325,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 				// Ensure that reformatting disks is finished
 				bgSeq.queueHealTask(healSource{bucket: nopHeal}, madmin.HealItemMetadata)
 
-				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
+				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal - 'mc admin heal alias/ --verbose' to check the status.",
 					len(healDisks)))
 
 				erasureSetInPoolDisksToHeal = make([]map[int][]StorageAPI, len(z.serverPools))
@@ -404,13 +396,15 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 					go func(setIndex int, disks []StorageAPI) {
 						defer wg.Done()
 						for _, disk := range disks {
-							logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(i+1))
+							if serverDebugLog {
+								logger.Info("Healing disk '%v' on %s pool", disk, humanize.Ordinal(i+1))
+							}
 
 							// So someone changed the drives underneath, healing tracker missing.
 							tracker, err := loadHealingTracker(ctx, disk)
 							if err != nil {
-								logger.Info("Healing tracker missing on '%s', disk was swapped again on %s pool",
-									disk, humanize.Ordinal(i+1))
+								logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', disk was swapped again on %s pool: %w",
+									disk, humanize.Ordinal(i+1), err))
 								tracker = newHealingTracker(disk)
 							}
 
@@ -438,12 +432,14 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 								continue
 							}
 
-							logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
-								humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
-							logger.Info("Summary:\n")
-							tracker.printTo(os.Stdout)
+							if serverDebugLog {
+								logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
+									humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
+								logger.Info("Summary:\n")
+								tracker.printTo(os.Stdout)
+								logger.Info("\n")
+							}
 							logger.LogIf(ctx, tracker.delete(ctx))
-							logger.Info("\n")
 
 							// Only upon success pop the healed disk.
 							globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())

cmd/bootstrap-peer-server.go

@@ -206,7 +206,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 		for _, clnt := range clnts {
 			if err := clnt.Verify(ctx, srcCfg); err != nil {
 				if !isNetworkError(err) {
-					logger.Info(fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err).Error())
+					logger.LogIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
 				}
 				offlineEndpoints = append(offlineEndpoints, clnt.String())
 				continue

cmd/bucket-encryption-handlers.go

@@ -206,6 +206,15 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
+	// Call site replication hook.
+	//
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:      madmin.SRBucketMetaTypeSSEConfig,
+		Bucket:    bucket,
+		SSEConfig: nil,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 	writeSuccessNoContent(w)
 }

cmd/bucket-handlers.go

@@ -1270,6 +1270,17 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 				return
 			}
+			rcfg, err := getReplicationConfig(ctx, bucket)
+			switch {
+			case err != nil:
+				if _, ok := err.(BucketReplicationConfigNotFound); !ok {
+					writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+					return
+				}
+			case rcfg.HasActiveRules("", true):
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+				return
+			}
 		}
 	}
 

cmd/bucket-replication-utils.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
@@ -27,6 +28,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
 )
 
 //go:generate msgp -file=$GOFILE
@@ -699,3 +701,26 @@ func newBucketResyncStatus(bucket string) BucketReplicationResyncStatus {
 		Version:    resyncMetaVersion,
 	}
 }
+
+var contentRangeRegexp = regexp.MustCompile(`bytes ([0-9]+)-([0-9]+)/([0-9]+|\\*)`)
+
+// parse size from content-range header
+func parseSizeFromContentRange(h http.Header) (sz int64, err error) {
+	cr := h.Get(xhttp.ContentRange)
+	if cr == "" {
+		return sz, fmt.Errorf("Content-Range not set")
+	}
+	parts := contentRangeRegexp.FindStringSubmatch(cr)
+	if len(parts) != 4 {
+		return sz, fmt.Errorf("invalid Content-Range header %s", cr)
+	}
+	if parts[3] == "*" {
+		return -1, nil
+	}
+	var usz uint64
+	usz, err = strconv.ParseUint(parts[3], 10, 64)
+	if err != nil {
+		return sz, err
+	}
+	return int64(usz), nil
+}

cmd/bucket-replication.go

@@ -1531,16 +1531,21 @@ func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
 	go globalReplicationStats.loadInitialReplicationMetrics(ctx)
 }
 
+type proxyResult struct {
+	Proxy bool
+	Err   error
+}
+
 // get Reader from replication target if active-active replication is in place and
 // this node returns a 404
-func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (gr *GetObjectReader, proxy bool) {
-	tgt, oi, proxy := proxyHeadToRepTarget(ctx, bucket, object, opts, proxyTargets)
-	if !proxy {
-		return nil, false
+func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (gr *GetObjectReader, proxy proxyResult, err error) {
+	tgt, oi, proxy := proxyHeadToRepTarget(ctx, bucket, object, rs, opts, proxyTargets)
+	if !proxy.Proxy {
+		return nil, proxy, nil
 	}
-	fn, off, length, err := NewGetObjectReader(rs, oi, opts)
+	fn, _, _, err := NewGetObjectReader(nil, oi, opts)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	gopts := miniogo.GetObjectOptions{
 		VersionID:            opts.VersionID,
@@ -1548,30 +1553,40 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 		Internal: miniogo.AdvancedGetOptions{
 			ReplicationProxyRequest: "true",
 		},
+		PartNumber: opts.PartNumber,
 	}
 	// get correct offsets for encrypted object
-	if off >= 0 && length >= 0 {
-		if err := gopts.SetRange(off, off+length-1); err != nil {
-			return nil, false
+	if rs != nil {
+		h, err := rs.ToHeader()
+		if err != nil {
+			return nil, proxy, err
 		}
+		gopts.Set(xhttp.Range, h)
 	}
 	// Make sure to match ETag when proxying.
 	if err = gopts.SetMatchETag(oi.ETag); err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	c := miniogo.Core{Client: tgt.Client}
-	obj, _, _, err := c.GetObject(ctx, bucket, object, gopts)
+	obj, _, h, err := c.GetObject(ctx, bucket, object, gopts)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	closeReader := func() { obj.Close() }
-
 	reader, err := fn(obj, h, closeReader)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	reader.ObjInfo = oi.Clone()
-	return reader, true
+	if rs != nil {
+		contentSize, err := parseSizeFromContentRange(h)
+		if err != nil {
+			return nil, proxy, err
+		}
+		reader.ObjInfo.Size = contentSize
+	}
+
+	return reader, proxyResult{Proxy: true}, nil
 }
 
 func getproxyTargets(ctx context.Context, bucket, object string, opts ObjectOptions) (tgts *madmin.BucketTargets) {
@@ -1590,11 +1605,11 @@ func getproxyTargets(ctx context.Context, bucket, object string, opts ObjectOpti
 	return tgts
 }
 
-func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgt *TargetClient, oi ObjectInfo, proxy bool) {
+func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgt *TargetClient, oi ObjectInfo, proxy proxyResult) {
 	// this option is set when active-active replication is in place between site A -> B,
 	// and site B does not have the object yet.
 	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
-		return nil, oi, false
+		return nil, oi, proxy
 	}
 	for _, t := range proxyTargets.Targets {
 		tgt = globalBucketTargetSys.GetRemoteTargetClient(ctx, t.Arn)
@@ -1612,9 +1627,22 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 			Internal: miniogo.AdvancedGetOptions{
 				ReplicationProxyRequest: "true",
 			},
+			PartNumber: opts.PartNumber,
 		}
+		if rs != nil {
+			h, err := rs.ToHeader()
+			if err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
+				continue
+			}
+			gopts.Set(xhttp.Range, h)
+		}
+
 		objInfo, err := tgt.StatObject(ctx, t.TargetBucket, object, gopts)
 		if err != nil {
+			if isErrInvalidRange(ErrorRespToObjectError(err, bucket, object)) {
+				return nil, oi, proxyResult{Err: err}
+			}
 			continue
 		}
 
@@ -1645,15 +1673,15 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 		if ok {
 			oi.ContentEncoding = ce
 		}
-		return tgt, oi, true
+		return tgt, oi, proxyResult{Proxy: true}
 	}
-	return nil, oi, false
+	return nil, oi, proxy
 }
 
 // get object info from replication target if active-active replication is in place and
 // this node returns a 404
-func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (oi ObjectInfo, proxy bool) {
-	_, oi, proxy = proxyHeadToRepTarget(ctx, bucket, object, opts, proxyTargets)
+func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (oi ObjectInfo, proxy proxyResult) {
+	_, oi, proxy = proxyHeadToRepTarget(ctx, bucket, object, rs, opts, proxyTargets)
 	return oi, proxy
 }
 

cmd/common-main.go

@@ -327,7 +327,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	logStartupMessage(prepareUpdateMessage("\nRun `mc admin update`", lrTime.Sub(crTime)))
+	logger.Info(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
 }
 
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
@@ -762,7 +762,7 @@ func handleCommonEnvVars() {
 				"         Please use %s and %s",
 				config.EnvAccessKey, config.EnvSecretKey,
 				config.EnvRootUser, config.EnvRootPassword)
-			logStartupMessage(color.RedBold(msg))
+			logger.Info(color.RedBold(msg))
 		}
 		globalActiveCred = cred
 	}
@@ -827,13 +827,6 @@ func handleCommonEnvVars() {
 	}
 }
 
-func logStartupMessage(msg string) {
-	if globalConsoleSys != nil {
-		globalConsoleSys.Send(msg, string(logger.All))
-	}
-	logger.StartupMessage(msg)
-}
-
 func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secureConn bool, err error) {
 	if !(isFile(getPublicCertFile()) && isFile(getPrivateKeyFile())) {
 		return nil, nil, false, nil
@@ -910,6 +903,9 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 	}
 	secureConn = true
 
+	// Certs might be symlinks, reload them every 10 seconds.
+	manager.UpdateReloadDuration(10 * time.Second)
+
 	// syscall.SIGHUP to reload the certs.
 	manager.ReloadOnSignal(syscall.SIGHUP)
 

cmd/config-current.go

@@ -529,7 +529,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 
 	if globalSTSTLSConfig.InsecureSkipVerify {
-		logger.Info("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify)
+		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
 	}
 
 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],

cmd/config-encrypted.go

@@ -74,7 +74,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 		if err != nil {
 			return err
 		}
-		logger.Info("Attempting to re-encrypt IAM users and policies on etcd with %q (%s)", stat.DefaultKey, stat.Name)
+		logger.Info(fmt.Sprintf("Attempting to re-encrypt IAM users and policies on etcd with %q (%s)", stat.DefaultKey, stat.Name))
 	}
 
 	listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
@@ -143,7 +143,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
 		if err != nil {
 			return err
 		}
-		logger.Info("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name)
+		logger.Info(fmt.Sprintf("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name))
 	}
 
 	var marker string

cmd/data-scanner.go

@@ -403,7 +403,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		if filter != nil && ok && existing.Compacted {
 			// If folder isn't in filter and we have data, skip it completely.
 			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
-				if f.healObjectSelect == 0 || !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
+				if f.healObjectSelect == 0 || !thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
 					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
 					f.updateCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
 					if f.dataUsageScannerDebug {
@@ -482,7 +482,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				debug:       f.dataUsageScannerDebug,
 				lifeCycle:   activeLifeCycle,
 				replication: replicationCfg,
-				heal:        thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
+				heal:        thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
 			// if the drive belongs to an erasure set
 			// that is already being healed, skip the
@@ -609,13 +609,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// and the entry itself is compacted.
 			if !into.Compacted && f.oldCache.isCompacted(h) {
 				if !h.mod(f.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
-					if f.healObjectSelect == 0 || !h.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
+					if f.healObjectSelect == 0 || !h.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
 						// Transfer and add as child...
 						f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
 						into.addChild(h)
 						continue
 					}
-					folder.objectHealProbDiv = dataUsageUpdateDirCycles
+					folder.objectHealProbDiv = f.healFolderInclude
 				}
 			}
 			scanFolder(folder)

cmd/data-usage-cache.go

@@ -369,6 +369,17 @@ func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool {
 	return uint32(xxhash.Sum64String(string(h)))%cycles == cycle%cycles
 }
 
+// modAlt returns true if the hash mod cycles == cycle.
+// This is out of sync with mod.
+// If cycles is 0 false is always returned.
+// If cycles is 1 true is always returned (as expected).
+func (h dataUsageHash) modAlt(cycle uint32, cycles uint32) bool {
+	if cycles <= 1 {
+		return cycles == 1
+	}
+	return uint32(xxhash.Sum64String(string(h))>>32)%(cycles) == cycle%cycles
+}
+
 // addChild will add a child based on its hash.
 // If it already exists it will not be added again.
 func (e *dataUsageEntry) addChild(hash dataUsageHash) {

cmd/data-usage-utils.go

@@ -92,32 +92,37 @@ type DataUsageInfo struct {
 }
 
 func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
-	if globalTierConfigMgr.Empty() {
+	if dui.TierStats == nil {
 		return nil
 	}
 
-	ts := make(map[string]madmin.TierStats)
-	// Add configured remote tiers
-	for tier := range globalTierConfigMgr.Tiers {
-		ts[tier] = madmin.TierStats{}
+	cfgs := globalTierConfigMgr.ListTiers()
+	if len(cfgs) == 0 {
+		return nil
 	}
+
+	ts := make(map[string]madmin.TierStats, len(cfgs)+1)
+	infos := make([]madmin.TierInfo, 0, len(ts))
+
 	// Add STANDARD (hot-tier)
 	ts[minioHotTier] = madmin.TierStats{}
+	infos = append(infos, madmin.TierInfo{
+		Name: minioHotTier,
+		Type: "internal",
+	})
+	// Add configured remote tiers
+	for _, cfg := range cfgs {
+		ts[cfg.Name] = madmin.TierStats{}
+		infos = append(infos, madmin.TierInfo{
+			Name: cfg.Name,
+			Type: cfg.Type.String(),
+		})
+	}
 
 	ts = dui.TierStats.adminStats(ts)
-	infos := make([]madmin.TierInfo, 0, len(ts))
-	for tier, st := range ts {
-		var tierType string
-		if tier == minioHotTier {
-			tierType = "internal"
-		} else {
-			tierType = globalTierConfigMgr.Tiers[tier].Type.String()
-		}
-		infos = append(infos, madmin.TierInfo{
-			Name:  tier,
-			Type:  tierType,
-			Stats: st,
-		})
+	for i := range infos {
+		info := infos[i]
+		infos[i].Stats = ts[info.Name]
 	}
 
 	sort.Slice(infos, func(i, j int) bool {
@@ -131,3 +136,31 @@ func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
 	})
 	return infos
 }
+
+func (dui DataUsageInfo) tierMetrics() (metrics []Metric) {
+	if dui.TierStats == nil {
+		return nil
+	}
+	// e.g minio_cluster_ilm_transitioned_bytes{tier="S3TIER-1"}=136314880
+	//     minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1
+	//     minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3
+	for tier, st := range dui.TierStats.Tiers {
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedBytesMD(),
+			Value:          float64(st.TotalSize),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedObjectsMD(),
+			Value:          float64(st.NumObjects),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedVersionsMD(),
+			Value:          float64(st.NumVersions),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+	}
+
+	return metrics
+}

cmd/disk-cache-backend.go

@@ -24,6 +24,7 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -252,16 +253,17 @@ func (c *diskCache) diskUsageLow() bool {
 // Returns if the disk usage reaches  or exceeds configured cache quota when size is added.
 // If current usage without size exceeds high watermark a GC is automatically queued.
 func (c *diskCache) diskSpaceAvailable(size int64) bool {
+	reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
+	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
+
 	gcTriggerPct := c.quotaPct * c.highWatermark / 100
 	di, err := disk.GetInfo(c.dir)
 	if err != nil {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
-		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 		logger.LogIf(ctx, err)
 		return false
 	}
 	if di.Total == 0 {
-		logger.Info("diskCache: Received 0 total disk size")
+		logger.LogIf(ctx, errors.New("diskCache: Received 0 total disk size"))
 		return false
 	}
 	usedPercent := float64(di.Used) * 100 / float64(di.Total)

cmd/disk-cache.go

@@ -31,6 +31,7 @@ import (
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/cache"
+	"github.com/minio/minio/internal/disk"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -594,12 +595,23 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 	if err != nil {
 		return nil, false, err
 	}
+	var warningMsg string
 	for i, dir := range config.Drives {
 		// skip diskCache creation for cache drives missing a format.json
 		if formats[i] == nil {
 			caches = append(caches, nil)
 			continue
 		}
+		if !globalIsCICD && len(warningMsg) == 0 {
+			rootDsk, err := disk.IsRootDisk(dir, "/")
+			if err != nil {
+				warningMsg = fmt.Sprintf("Invalid cache dir %s err : %s", dir, err.Error())
+			}
+			if rootDsk {
+				warningMsg = fmt.Sprintf("cache dir cannot be part of root disk: %s", dir)
+			}
+		}
+
 		if err := checkAtimeSupport(dir); err != nil {
 			return nil, false, fmt.Errorf("Atime support required for disk caching, atime check failed with %w", err)
 		}
@@ -610,11 +622,14 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 		}
 		caches = append(caches, cache)
 	}
+	if warningMsg != "" {
+		logger.Info(color.Yellow(fmt.Sprintf("WARNING: Usage of root disk for disk caching is deprecated: %s", warningMsg)))
+	}
 	return caches, migrating, nil
 }
 
 func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
-	logStartupMessage(color.Blue("Cache migration initiated ...."))
+	logger.Info(color.Blue("Cache migration initiated ...."))
 
 	g := errgroup.WithNErrs(len(c.cache))
 	for index, dc := range c.cache {
@@ -643,7 +658,7 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 
 	// update migration status
 	c.migrating = false
-	logStartupMessage(color.Blue("Cache migration completed successfully."))
+	logger.Info(color.Blue("Cache migration completed successfully."))
 }
 
 // PutObject - caches the uploaded object for single Put operations

cmd/erasure-healing-common.go

@@ -83,7 +83,10 @@ func commonTime(modTimes []time.Time) (modTime time.Time) {
 }
 
 // Beginning of unix time is treated as sentinel value here.
-var timeSentinel = time.Unix(0, 0).UTC()
+var (
+	timeSentinel     = time.Unix(0, 0).UTC()
+	timeSentinel1970 = time.Unix(0, 1).UTC() // 1970 used for special cases when xlmeta.version == 0
+)
 
 // Boot modTimes up to disk count, setting the value to time sentinel.
 func bootModtimes(diskCount int) []time.Time {

cmd/erasure-healing.go

@@ -931,5 +931,12 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	}
 
 	// Heal the object.
-	return er.healObject(healCtx, bucket, object, versionID, opts)
+	hr, err = er.healObject(healCtx, bucket, object, versionID, opts)
+	if errors.Is(err, errFileCorrupt) && opts.ScanMode != madmin.HealDeepScan {
+		// Instead of returning an error when a bitrot error is detected
+		// during a normal heal scan, heal again with bitrot flag enabled.
+		opts.ScanMode = madmin.HealDeepScan
+		hr, err = er.healObject(healCtx, bucket, object, versionID, opts)
+	}
+	return hr, err
 }

cmd/erasure-healing_test.go

@@ -382,6 +382,13 @@ func TestHealingDanglingObject(t *testing.T) {
 		copy(disks, newDisks)
 		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock()
 	}
+	getDisk := func(n int) StorageAPI {
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Lock()
+		disk := disks[n]
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock()
+		return disk
+	}
+
 	// Remove 4 disks.
 	setDisks(nil, nil, nil, nil)
 
@@ -440,8 +447,8 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	setDisks(orgDisks[:4]...)
-
-	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk := getDisk(0)
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -458,7 +465,8 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -488,7 +496,8 @@ func TestHealingDanglingObject(t *testing.T) {
 
 	setDisks(orgDisks[:4]...)
 
-	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -505,7 +514,8 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -807,7 +817,140 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	}
 }
 
-func TestHealObjectCorrupted(t *testing.T) {
+func TestHealObjectCorruptedXLMeta(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resetGlobalHealState()
+	defer resetGlobalHealState()
+
+	nDisks := 16
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	// Everything is fine, should return nil
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+	data := bytes.Repeat([]byte("a"), 5*1024*1024)
+	var opts ObjectOptions
+
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	// Create an object with multiple parts uploaded in decreasing
+	// part number.
+	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	if err != nil {
+		t.Fatalf("Failed to create a multipart upload - %v", err)
+	}
+
+	var uploadedParts []CompletePart
+	for _, partID := range []int{2, 1} {
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		if err1 != nil {
+			t.Fatalf("Failed to upload a part - %v", err1)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Failed to complete multipart upload - %v", err)
+	}
+
+	z := objLayer.(*erasureServerPools)
+	er := z.serverPools[0].sets[0]
+	erasureDisks := er.getDisks()
+	firstDisk := erasureDisks[0]
+
+	// Test 1: Remove the object backend files from the first disk.
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	if err != nil {
+		t.Fatalf("Failed to delete a file - %v", err)
+	}
+
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+	if err != nil {
+		t.Fatalf("Failed to heal object - %v", err)
+	}
+
+	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
+		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+	}
+
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	nfi1, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	if !reflect.DeepEqual(fi, nfi1) {
+		t.Fatalf("FileInfo not equal after healing")
+	}
+
+	// Test 2: Test with a corrupted xl.meta
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), []byte("abcd"))
+	if err != nil {
+		t.Errorf("Failure during creating part.1 - %v", err)
+	}
+
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+	if err != nil {
+		t.Errorf("Expected nil but received %v", err)
+	}
+
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	nfi2, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	if !reflect.DeepEqual(fi, nfi2) {
+		t.Fatalf("FileInfo not equal after healing")
+	}
+
+	// Test 3: checks if HealObject returns an error when xl.meta is not found
+	// in more than read quorum number of disks, to create a corrupted situation.
+	for i := 0; i <= nfi2.Erasure.DataBlocks; i++ {
+		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	}
+
+	// Try healing now, expect to receive errFileNotFound.
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	if err != nil {
+		if _, ok := err.(ObjectNotFound); !ok {
+			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+		}
+	}
+
+	// since majority of xl.meta's are not available, object should be successfully deleted.
+	_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+	if _, ok := err.(ObjectNotFound); !ok {
+		t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+	}
+}
+
+func TestHealObjectCorruptedParts(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -867,7 +1010,26 @@ func TestHealObjectCorrupted(t *testing.T) {
 	er := z.serverPools[0].sets[0]
 	erasureDisks := er.getDisks()
 	firstDisk := erasureDisks[0]
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	secondDisk := erasureDisks[1]
+
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if err != nil {
+		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+	}
+
+	part1Disk1Origin, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	if err != nil {
+		t.Fatalf("Failed to read a file - %v", err)
+	}
+
+	part1Disk2Origin, err := secondDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	if err != nil {
+		t.Fatalf("Failed to read a file - %v", err)
+	}
+
+	// Test 1, remove part.1
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -877,85 +1039,67 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	part1Replaced, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
-		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+	if !reflect.DeepEqual(part1Disk1Origin, part1Replaced) {
+		t.Fatalf("part.1 not healed correctly")
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	// Test 2, Corrupt part.1
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte("foobytes"))
 	if err != nil {
-		t.Errorf("Failure during deleting part.1 - %v", err)
+		t.Fatalf("Failed to write a file - %v", err)
 	}
 
-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
-		t.Errorf("Failure during creating part.1 - %v", err)
+		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	part1Replaced, err = firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Errorf("Expected nil but received %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	if !reflect.DeepEqual(part1Disk1Origin, part1Replaced) {
+		t.Fatalf("part.1 not healed correctly")
+	}
+
+	// Test 3, Corrupt one part and remove data in another disk
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte("foobytes"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to write a file - %v", err)
 	}
 
-	if !reflect.DeepEqual(fi, nfi) {
-		t.Fatalf("FileInfo not equal after healing")
-	}
-
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+	err = secondDisk.Delete(context.Background(), bucket, object, true)
 	if err != nil {
-		t.Errorf("Failure during deleting part.1 - %v", err)
+		t.Fatalf("Failed to delete a file - %v", err)
 	}
 
-	bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
+	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
 	if err != nil {
-		t.Errorf("Failure during creating part.1 - %v", err)
+		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+	partReconstructed, err := firstDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Errorf("Expected nil but received %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+	if !reflect.DeepEqual(part1Disk1Origin, partReconstructed) {
+		t.Fatalf("part.1 not healed correctly")
+	}
+
+	partReconstructed, err = secondDisk.ReadAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
 	if err != nil {
-		t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		t.Fatalf("Failed to read a file - %v", err)
 	}
 
-	if !reflect.DeepEqual(fi, nfi) {
-		t.Fatalf("FileInfo not equal after healing")
-	}
-
-	// Test 4: checks if HealObject returns an error when xl.meta is not found
-	// in more than read quorum number of disks, to create a corrupted situation.
-	for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
-		erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
-	}
-
-	// Try healing now, expect to receive errFileNotFound.
-	_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
-	if err != nil {
-		if _, ok := err.(ObjectNotFound); !ok {
-			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
-		}
-	}
-
-	// since majority of xl.meta's are not available, object should be successfully deleted.
-	_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
-	if _, ok := err.(ObjectNotFound); !ok {
-		t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+	if !reflect.DeepEqual(part1Disk2Origin, partReconstructed) {
+		t.Fatalf("part.1 not healed correctly")
 	}
 }
 
@@ -1017,7 +1161,8 @@ func TestHealObjectErasure(t *testing.T) {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}
 
-	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+	// Delete the whole object folder
+	err = firstDisk.Delete(context.Background(), bucket, object, true)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}

cmd/erasure-object.go

@@ -343,10 +343,8 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 				switch {
 				case errors.Is(err, errFileNotFound):
 					scan = madmin.HealNormalScan
-					logger.Info("Healing required, triggering async heal missing shards for %s", pathJoin(bucket, object, fi.VersionID))
 				case errors.Is(err, errFileCorrupt):
 					scan = madmin.HealDeepScan
-					logger.Info("Healing required, triggering async heal bitrot for %s", pathJoin(bucket, object, fi.VersionID))
 				}
 				switch scan {
 				case madmin.HealNormalScan, madmin.HealDeepScan:

cmd/erasure-object_test.go

@@ -128,6 +128,86 @@ func TestErasureDeleteObjectBasic(t *testing.T) {
 	removeRoots(fsDirs)
 }
 
+func TestDeleteObjectsVersioned(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	obj, fsDirs, err := prepareErasure(ctx, 16)
+	if err != nil {
+		t.Fatal("Unable to initialize 'Erasure' object layer.", err)
+	}
+	// Remove all dirs.
+	for _, dir := range fsDirs {
+		defer os.RemoveAll(dir)
+	}
+
+	type testCaseType struct {
+		bucket string
+		object string
+	}
+
+	bucketName := "bucket"
+	testCases := []testCaseType{
+		{bucketName, "dir/obj1"},
+		{bucketName, "dir/obj1"},
+	}
+
+	err = obj.MakeBucketWithLocation(ctx, bucketName, BucketOptions{
+		VersioningEnabled: true,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	names := make([]ObjectToDelete, len(testCases))
+	for i, testCase := range testCases {
+		objInfo, err := obj.PutObject(ctx, testCase.bucket, testCase.object,
+			mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{
+				Versioned: true,
+			})
+		if err != nil {
+			t.Fatalf("Erasure Object upload failed: <ERROR> %s", err)
+		}
+		names[i] = ObjectToDelete{
+			ObjectV: ObjectV{
+				ObjectName: objInfo.Name,
+				VersionID:  objInfo.VersionID,
+			},
+		}
+
+	}
+	names = append(names, ObjectToDelete{
+		ObjectV: ObjectV{
+			ObjectName: "dir/obj1",
+			VersionID:  mustGetUUID(), // add a non-existent UUID.
+		},
+	})
+
+	_, delErrs := obj.DeleteObjects(ctx, bucketName, names, ObjectOptions{
+		Versioned: true,
+	})
+	for i := range delErrs {
+		if delErrs[i] != nil {
+			t.Errorf("Failed to remove object `%v` with the error: `%v`", names[i], delErrs[i])
+		}
+	}
+
+	for i, test := range testCases {
+		_, statErr := obj.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{
+			VersionID: names[i].ObjectV.VersionID,
+		})
+		switch statErr.(type) {
+		case VersionNotFound:
+		default:
+			t.Fatalf("Object %s is not removed", test.bucket+SlashSeparator+test.object)
+		}
+	}
+
+	if _, err = ioutil.ReadFile(pathJoin(fsDirs[0], bucketName, "dir/obj1", "xl.meta")); err == nil {
+		t.Fatalf("xl.meta still present after removal")
+	}
+}
+
 func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()

cmd/erasure-server-pool-decom.go

@@ -593,7 +593,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				return
 			}
 
-			// We need a reversed order for Decommissioning,
+			// We need a reversed order for decommissioning,
 			// to create the appropriate stack.
 			versionsSorter(fivs.Versions).reverse()
 
@@ -955,6 +955,18 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		}
 	}
 
+	// Create .minio.sys/conifg, .minio.sys/buckets paths if missing,
+	// this code is present to avoid any missing meta buckets on other
+	// pools.
+	for _, metaBucket := range []string{
+		pathJoin(minioMetaBucket, minioConfigPrefix),
+		pathJoin(minioMetaBucket, bucketMetaPrefix),
+	} {
+		if err = z.MakeBucketWithLocation(ctx, metaBucket, BucketOptions{}); err != nil {
+			return err
+		}
+	}
+
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to decommission the necessary metadata.
 	buckets = append(buckets, BucketInfo{

cmd/erasure-server-pool.go

@@ -661,6 +661,9 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
+			if z.IsSuspended(index) {
+				return nil
+			}
 			return z.serverPools[index].MakeBucketWithLocation(ctx, bucket, opts)
 		}, index)
 	}
@@ -671,7 +674,10 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 		if err != nil {
 			if _, ok := err.(BucketExists); !ok {
 				// Delete created buckets, ignoring errors.
-				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: true, NoRecreate: true})
+				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
+					Force:      false,
+					NoRecreate: true,
+				})
 			}
 			return err
 		}
@@ -1476,6 +1482,9 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
+			if z.IsSuspended(index) {
+				return nil
+			}
 			return z.serverPools[index].DeleteBucket(ctx, bucket, opts)
 		}, index)
 	}
@@ -1730,14 +1739,13 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 // HealObjectFn closure function heals the object.
 type HealObjectFn func(bucket, object, versionID string) error
 
-func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects, healEntry func(metaCacheEntry) error, errCh chan<- error) {
+func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects, healEntry func(metaCacheEntry) error) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
 	disks, _ := set.getOnlineDisksWithHealing()
 	if len(disks) == 0 {
-		errCh <- errors.New("listAndHeal: No non-healing disks found")
-		return
+		return errors.New("listAndHeal: No non-healing disks found")
 	}
 
 	// How to resolve partial results.
@@ -1763,8 +1771,8 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 		reportNotFound: false,
 		agreed: func(entry metaCacheEntry) {
 			if err := healEntry(entry); err != nil {
-				errCh <- err
-				return
+				logger.LogIf(ctx, err)
+				cancel()
 			}
 		},
 		partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
@@ -1776,7 +1784,8 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 			}
 
 			if err := healEntry(*entry); err != nil {
-				errCh <- err
+				logger.LogIf(ctx, err)
+				cancel()
 				return
 			}
 		},
@@ -1784,13 +1793,13 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 	}
 
 	if err := listPathRaw(ctx, lopts); err != nil {
-		errCh <- fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
-		return
+		return fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
 	}
+
+	return nil
 }
 
 func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error {
-	errCh := make(chan error)
 	healEntry := func(entry metaCacheEntry) error {
 		if entry.isDir() {
 			return nil
@@ -1829,34 +1838,33 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	go func() {
-		defer close(errCh)
-
+	var poolErrs [][]error
+	for idx, erasureSet := range z.serverPools {
+		if z.IsSuspended(idx) {
+			continue
+		}
+		errs := make([]error, len(erasureSet.sets))
 		var wg sync.WaitGroup
-		for idx, erasureSet := range z.serverPools {
-			if z.IsSuspended(idx) {
-				continue
-			}
-			for _, set := range erasureSet.sets {
-				wg.Add(1)
-				go func(set *erasureObjects) {
-					defer wg.Done()
+		for idx, set := range erasureSet.sets {
+			wg.Add(1)
+			go func(idx int, set *erasureObjects) {
+				defer wg.Done()
 
-					listAndHeal(ctx, bucket, prefix, set, healEntry, errCh)
-				}(set)
-			}
+				errs[idx] = listAndHeal(ctx, bucket, prefix, set, healEntry)
+			}(idx, set)
 		}
 		wg.Wait()
-	}()
-	var err error
-	for e := range errCh {
-		// Save first non-nil error.
-		if e != nil && err != nil {
-			err = e
-			cancel()
+		poolErrs = append(poolErrs, errs)
+	}
+	for _, errs := range poolErrs {
+		for _, err := range errs {
+			if err == nil {
+				continue
+			}
+			return err
 		}
 	}
-	return err
+	return nil
 }
 
 func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {

cmd/erasure-sets.go

@@ -221,7 +221,6 @@ func (s *erasureSets) connectDisks() {
 			if err != nil {
 				if endpoint.IsLocal && errors.Is(err, errUnformattedDisk) {
 					globalBackgroundHealState.pushHealLocalDisks(endpoint)
-					logger.Info(fmt.Sprintf("Found unformatted drive %s, attempting to heal...", endpoint))
 				} else {
 					printEndpointError(endpoint, err, true)
 				}
@@ -229,7 +228,6 @@ func (s *erasureSets) connectDisks() {
 			}
 			if disk.IsLocal() && disk.Healing() != nil {
 				globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint())
-				logger.Info(fmt.Sprintf("Found the drive %s that needs healing, attempting to heal...", disk))
 			}
 			s.erasureDisksMu.RLock()
 			setIndex, diskIndex, err := findDiskIndex(s.format, format)
@@ -279,7 +277,7 @@ func (s *erasureSets) connectDisks() {
 			if !justConnected {
 				continue
 			}
-			globalMRFState.newSetReconnected(setIndex, s.poolIndex)
+			globalMRFState.newSetReconnected(s.poolIndex, setIndex)
 		}
 	}()
 }
@@ -1256,9 +1254,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 		if storageDisks[i] != nil && infos[i].RootDisk {
 			// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
 			// defective drive we should not heal a path on the root disk.
-			logger.Info("Disk `%s` the same as the system root disk.\n"+
-				"Disk will not be used. Please supply a separate disk and restart the server.",
-				storageDisks[i].String())
+			logger.LogIf(GlobalContext, fmt.Errorf("Disk `%s` is part of root disk, will not be used", storageDisks[i]))
 			storageDisks[i] = nil
 		}
 	}

cmd/erasure.go

@@ -29,7 +29,6 @@ import (
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/bpool"
-	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
@@ -353,7 +352,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 	// Collect disks we can use.
 	disks, healing := er.getOnlineDisksWithHealing()
 	if len(disks) == 0 {
-		logger.Info(color.Green("data-scanner:") + " all disks are offline or being healed, skipping scanner")
+		logger.LogIf(ctx, errors.New("data-scanner: all disks are offline or being healed, skipping scanner cycle"))
 		return nil
 	}
 

cmd/gateway-main.go

@@ -377,7 +377,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	}
 
 	// TODO: remove the following line by June 1st.
-	logStartupMessage(
+	logger.Info(
 		color.RedBold(`
 ===================================================================================
 **** WARNING: MinIO Gateway will be removed by June 1st from MinIO repository *****

cmd/gateway-startup-msg.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/logger"
 )
 
 // Prints the formatted startup message.
@@ -45,7 +46,7 @@ func printGatewayStartupMessage(apiEndPoints []string, backendType string) {
 	if globalMinioConsolePortAuto && globalBrowserEnabled {
 		msg := fmt.Sprintf("\nWARNING: Console endpoint is listening on a dynamic port (%s), please use --console-address \":PORT\" to choose a static port.",
 			globalMinioConsolePort)
-		logStartupMessage(color.RedBold(msg))
+		logger.Info(color.RedBold(msg))
 	}
 }
 
@@ -57,19 +58,19 @@ func printGatewayCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 
 	// Colorize the message and print.
-	logStartupMessage(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
+	logger.Info(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
 	if color.IsTerminal() && !globalCLIContext.Anonymous {
-		logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+		logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 	}
 	printEventNotifiers()
 
 	if globalBrowserEnabled {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
-		logStartupMessage(color.Blue("\nConsole: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
+		logger.Info(color.Blue("\nConsole: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
 		if color.IsTerminal() && !globalCLIContext.Anonymous {
-			logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-			logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+			logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+			logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 		}
 	}
 }

cmd/global-heal.go

@@ -330,8 +330,6 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		default:
 			tracker.bucketDone(bucket)
 			logger.LogIf(ctx, tracker.update(ctx))
-			logger.Info("Healing bucket %s content on %s erasure set complete",
-				bucket, humanize.Ordinal(tracker.SetIndex+1))
 		}
 	}
 	tracker.Object = ""

cmd/globals.go

@@ -345,6 +345,11 @@ var (
 	globalIsCICD bool
 
 	globalRootDiskThreshold uint64
+
+	// Used for collecting stats for netperf
+	globalNetPerfMinDuration = time.Second * 10
+	globalNetPerfRX          netPerfRX
+
 	// Add new variable global values here.
 )
 

cmd/httprange.go

@@ -174,3 +174,26 @@ func (h *HTTPRangeSpec) String(resourceSize int64) string {
 	}
 	return fmt.Sprintf("%d-%d", off, off+length-1)
 }
+
+// ToHeader returns the Range header value.
+func (h *HTTPRangeSpec) ToHeader() (string, error) {
+	if h == nil {
+		return "", nil
+	}
+	start := strconv.Itoa(int(h.Start))
+	end := strconv.Itoa(int(h.End))
+	switch {
+	case h.Start >= 0 && h.End >= 0:
+		if h.Start > h.End {
+			return "", errInvalidRange
+		}
+	case h.IsSuffixLength:
+		end = strconv.Itoa(int(h.Start * -1))
+		start = ""
+	case h.Start > -1:
+		end = ""
+	default:
+		return "", fmt.Errorf("does not have valid range value")
+	}
+	return fmt.Sprintf("bytes=%s-%s", start, end), nil
+}

cmd/httprange_test.go

@@ -101,3 +101,46 @@ func TestHTTPRequestRangeSpec(t *testing.T) {
 		t.Errorf("Case %d: Expected errInvalidRange but: %v %v %d %d %v", i, rs, err1, o, l, err2)
 	}
 }
+
+func TestHTTPRequestRangeToHeader(t *testing.T) {
+	validRangeSpecs := []struct {
+		spec        string
+		errExpected bool
+	}{
+		{"bytes=0-", false},
+		{"bytes=1-", false},
+
+		{"bytes=0-9", false},
+		{"bytes=1-10", false},
+		{"bytes=1-1", false},
+		{"bytes=2-5", false},
+
+		{"bytes=-5", false},
+		{"bytes=-1", false},
+		{"bytes=-1000", false},
+		{"bytes=", true},
+		{"bytes= ", true},
+		{"byte=", true},
+		{"bytes=A-B", true},
+		{"bytes=1-B", true},
+		{"bytes=B-1", true},
+		{"bytes=-1-1", true},
+	}
+	for i, testCase := range validRangeSpecs {
+		rs, err := parseRequestRangeSpec(testCase.spec)
+		if err != nil {
+			if !testCase.errExpected || err == nil && testCase.errExpected {
+				t.Errorf("unexpected err: %v", err)
+			}
+			continue
+		}
+		h, err := rs.ToHeader()
+		if err != nil && !testCase.errExpected || err == nil && testCase.errExpected {
+			t.Errorf("expected error with invalid range: %v", err)
+		}
+		if h != testCase.spec {
+			t.Errorf("Case %d: translated to incorrect header: %s expected: %s",
+				i, h, testCase.spec)
+		}
+	}
+}

cmd/iam.go

@@ -349,7 +349,7 @@ func (sys *IAMSys) printIAMRoles() {
 		msgs = append(msgs, color.Bold(arn))
 	}
 
-	logStartupMessage(fmt.Sprintf("%s %s", color.Blue("IAM Roles:"), strings.Join(msgs, " ")))
+	logger.Info(fmt.Sprintf("%s %s", color.Blue("IAM Roles:"), strings.Join(msgs, " ")))
 }
 
 // HasWatcher - returns if the IAM system has a watcher to be notified of

cmd/main.go

@@ -44,7 +44,7 @@ var GlobalFlags = []cli.Flag{
 	},
 	cli.BoolFlag{
 		Name:  "quiet",
-		Usage: "disable startup information",
+		Usage: "disable startup and info messages",
 	},
 	cli.BoolFlag{
 		Name:  "anonymous",
@@ -52,7 +52,7 @@ var GlobalFlags = []cli.Flag{
 	},
 	cli.BoolFlag{
 		Name:  "json",
-		Usage: "output server logs and startup information in json format",
+		Usage: "output logs in JSON format",
 	},
 	// Deprecated flag, so its hidden now, existing deployments will keep working.
 	cli.BoolFlag{

cmd/metacache-entries.go

@@ -191,6 +191,16 @@ func (e *metaCacheEntry) fileInfo(bucket string) (FileInfo, error) {
 		}, nil
 	}
 	if e.cached != nil {
+		if len(e.cached.versions) == 0 {
+			// This special case is needed to handle xlMeta.versions == 0
+			return FileInfo{
+				Volume:   bucket,
+				Name:     e.name,
+				Deleted:  true,
+				IsLatest: true,
+				ModTime:  timeSentinel1970,
+			}, nil
+		}
 		return e.cached.ToFileInfo(bucket, e.name, "")
 	}
 	return getFileInfo(e.metadata, bucket, e.name, "", false)
@@ -379,11 +389,11 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 
 // firstFound returns the first found and the number of set entries.
 func (m metaCacheEntries) firstFound() (first *metaCacheEntry, n int) {
-	for _, entry := range m {
+	for i, entry := range m {
 		if entry.name != "" {
 			n++
 			if first == nil {
-				first = &entry
+				first = &m[i]
 			}
 		}
 	}

cmd/metacache-walk.go

@@ -138,6 +138,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			// Forward some errors?
 			return nil
 		}
+		diskHealthCheckOK(ctx, err)
 		if len(entries) == 0 {
 			return nil
 		}
@@ -179,6 +180,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				s.walkReadMu.Lock()
 				meta.metadata, err = s.readMetadata(ctx, pathJoin(volumeDir, current, entry))
 				s.walkReadMu.Unlock()
+				diskHealthCheckOK(ctx, err)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					continue
@@ -196,6 +198,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				s.walkReadMu.Lock()
 				meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, current, entry))
 				s.walkReadMu.Unlock()
+				diskHealthCheckOK(ctx, err)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					continue
@@ -257,6 +260,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			s.walkReadMu.Lock()
 			meta.metadata, err = s.readMetadata(ctx, pathJoin(volumeDir, meta.name, xlStorageFormatFile))
 			s.walkReadMu.Unlock()
+			diskHealthCheckOK(ctx, err)
 			switch {
 			case err == nil:
 				// It was an object
@@ -266,6 +270,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				out <- meta
 			case osIsNotExist(err), isSysErrIsDir(err):
 				meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1))
+				diskHealthCheckOK(ctx, err)
 				if err == nil {
 					// It was an object
 					out <- meta
@@ -303,16 +308,12 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	return scanDir(opts.BaseDir)
 }
 
-func (p *xlStorageDiskIDCheck) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error {
-	defer p.updateStorageMetrics(storageMetricWalkDir, opts.Bucket, opts.BaseDir)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWalkDir, opts.Bucket, opts.BaseDir)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.WalkDir(ctx, opts, wr)
 }

cmd/metrics-v2.go

@@ -1663,33 +1663,12 @@ func getClusterTierMetrics() *MetricsGroup {
 		if err != nil {
 			return
 		}
-		// data usage has not captured any data yet.
-		if dui.LastUpdate.IsZero() {
+		// data usage has not captured any tier stats yet.
+		if dui.TierStats == nil {
 			return
 		}
 
-		// e.g minio_cluster_ilm_transitioned_bytes{tier="S3TIER-1"}=136314880
-		//     minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1
-		//     minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3
-		for tier, st := range dui.TierStats.Tiers {
-			metrics = append(metrics, Metric{
-				Description:    getClusterTransitionedBytesMD(),
-				Value:          float64(st.TotalSize),
-				VariableLabels: map[string]string{"tier": tier},
-			})
-			metrics = append(metrics, Metric{
-				Description:    getClusterTransitionedObjectsMD(),
-				Value:          float64(st.NumObjects),
-				VariableLabels: map[string]string{"tier": tier},
-			})
-			metrics = append(metrics, Metric{
-				Description:    getClusterTransitionedVersionsMD(),
-				Value:          float64(st.NumVersions),
-				VariableLabels: map[string]string{"tier": tier},
-			})
-		}
-
-		return metrics
+		return dui.tierMetrics()
 	})
 	return mg
 }

cmd/namespace-lock.go

@@ -225,6 +225,7 @@ func (n *nsLockMap) NewNSLock(lockers func() ([]dsync.NetLocker, string), volume
 	if n.isDistErasure {
 		drwmutex := dsync.NewDRWMutex(&dsync.Dsync{
 			GetLockers: lockers,
+			Timeouts:   dsync.DefaultTimeouts,
 		}, pathsJoinPrefix(volume, paths...)...)
 		return &distLockInstance{drwmutex, opsID}
 	}

cmd/notification.go

@@ -1553,6 +1553,58 @@ func (sys *NotificationSys) ServiceFreeze(ctx context.Context, freeze bool) []No
 	return nerrs
 }
 
+// Netperf - perform mesh style network throughput test
+func (sys *NotificationSys) Netperf(ctx context.Context, duration time.Duration) []madmin.NetperfNodeResult {
+	length := len(sys.allPeerClients)
+	if length == 0 {
+		// For single node erasure setup.
+		return nil
+	}
+	results := make([]madmin.NetperfNodeResult, length)
+
+	scheme := "http"
+	if globalIsTLS {
+		scheme = "https"
+	}
+
+	var wg sync.WaitGroup
+	for index := range sys.peerClients {
+		if sys.peerClients[index] == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(index int) {
+			defer wg.Done()
+			r, err := sys.peerClients[index].Netperf(ctx, duration)
+			u := &url.URL{
+				Scheme: scheme,
+				Host:   sys.peerClients[index].host.String(),
+			}
+			if err != nil {
+				results[index].Error = err.Error()
+			} else {
+				results[index] = r
+			}
+			results[index].Endpoint = u.String()
+		}(index)
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		r := netperf(ctx, duration)
+		u := &url.URL{
+			Scheme: scheme,
+			Host:   globalLocalNodeName,
+		}
+		results[len(results)-1] = r
+		results[len(results)-1].Endpoint = u.String()
+	}()
+	wg.Wait()
+
+	return results
+}
+
 // Speedtest run GET/PUT tests at input concurrency for requested object size,
 // optionally you can extend the tests longer with time.Duration.
 func (sys *NotificationSys) Speedtest(ctx context.Context, size int,

cmd/object-api-errors.go

@@ -701,3 +701,8 @@ func isErrMethodNotAllowed(err error) bool {
 	var methodNotAllowed MethodNotAllowed
 	return errors.As(err, &methodNotAllowed)
 }
+
+func isErrInvalidRange(err error) bool {
+	_, ok := err.(InvalidRange)
+	return ok
+}

cmd/object-handlers.go

@@ -412,20 +412,28 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 	if err != nil {
 		var (
 			reader *GetObjectReader
-			proxy  bool
+			proxy  proxyResult
 		)
 		proxytgts := getproxyTargets(ctx, bucket, object, opts)
 		if !proxytgts.Empty() {
 			// proxy to replication target if active-active replication is in place.
-			reader, proxy = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
-			if reader != nil && proxy {
+			reader, proxy, err = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
+			if err != nil && !isErrObjectNotFound(ErrorRespToObjectError(err, bucket, object)) &&
+				!isErrVersionNotFound(ErrorRespToObjectError(err, bucket, object)) {
+				logger.LogIf(ctx, fmt.Errorf("Replication proxy failed for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
+			}
+			if reader != nil && proxy.Proxy && err == nil {
 				gr = reader
 			}
 		}
-		if reader == nil || !proxy {
+		if reader == nil || !proxy.Proxy {
 			if isErrPreconditionFailed(err) {
 				return
 			}
+			if proxy.Err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, proxy.Err), r.URL)
+				return
+			}
 			if globalBucketVersioningSys.Enabled(bucket) && gr != nil {
 				if !gr.ObjInfo.VersionPurgeStatus.Empty() {
 					// Shows the replication status of a permanent delete of a version
@@ -631,22 +639,28 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
 		return
 	}
+	// Get request range.
+	var rs *HTTPRangeSpec
+	rangeHeader := r.Header.Get(xhttp.Range)
 
 	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		var (
-			proxy bool
+			proxy proxyResult
 			oi    ObjectInfo
 		)
 		// proxy HEAD to replication target if active-active replication configured on bucket
 		proxytgts := getproxyTargets(ctx, bucket, object, opts)
 		if !proxytgts.Empty() {
-			oi, proxy = proxyHeadToReplicationTarget(ctx, bucket, object, opts, proxytgts)
-			if proxy {
+			if rangeHeader != "" {
+				rs, _ = parseRequestRangeSpec(rangeHeader)
+			}
+			oi, proxy = proxyHeadToReplicationTarget(ctx, bucket, object, rs, opts, proxytgts)
+			if proxy.Proxy {
 				objInfo = oi
 			}
 		}
-		if !proxy {
+		if !proxy.Proxy {
 			if globalBucketVersioningSys.Enabled(bucket) {
 				switch {
 				case !objInfo.VersionPurgeStatus.Empty():
@@ -703,9 +717,6 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		return
 	}
 
-	// Get request range.
-	var rs *HTTPRangeSpec
-	rangeHeader := r.Header.Get(xhttp.Range)
 	if rangeHeader != "" {
 		// Both 'Range' and 'partNumber' cannot be specified at the same time
 		if opts.PartNumber > 0 {

cmd/peer-rest-client.go

@@ -1119,3 +1119,27 @@ func (client *peerRESTClient) GetLastDayTierStats(ctx context.Context) (dailyAll
 	}
 	return dailyAllTierStats(result), nil
 }
+
+// DevNull - Used by netperf to pump data to peer
+func (client *peerRESTClient) DevNull(ctx context.Context, r io.Reader) error {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodDevNull, nil, r, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return err
+}
+
+// Netperf - To initiate netperf on peer
+func (client *peerRESTClient) Netperf(ctx context.Context, duration time.Duration) (madmin.NetperfNodeResult, error) {
+	var result madmin.NetperfNodeResult
+	values := make(url.Values)
+	values.Set(peerRESTDuration, duration.String())
+	respBody, err := client.callWithContext(context.Background(), peerRESTMethodNetperf, values, nil, -1)
+	if err != nil {
+		return result, err
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&result)
+	return result, err
+}

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v20" // Add drivespeedtest
+	peerRESTVersion       = "v21" // Add netperf
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -70,6 +70,8 @@ const (
 	peerRESTMethodReloadSiteReplicationConfig = "/reloadsitereplicationconfig"
 	peerRESTMethodReloadPoolMeta              = "/reloadpoolmeta"
 	peerRESTMethodGetLastDayTierStats         = "/getlastdaytierstats"
+	peerRESTMethodDevNull                     = "/devnull"
+	peerRESTMethodNetperf                     = "/netperf"
 )
 
 const (

cmd/peer-rest-server.go

@@ -27,20 +27,15 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
 	b "github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/event"
-	"github.com/minio/minio/internal/hash"
-	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/randreader"
 	"github.com/tinylib/msgp/msgp"
 )
 
@@ -1143,148 +1138,6 @@ func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// SpeedtestResult return value of the speedtest function
-type SpeedtestResult struct {
-	Endpoint  string
-	Uploads   uint64
-	Downloads uint64
-	Error     string
-}
-
-func newRandomReader(size int) io.Reader {
-	return io.LimitReader(randreader.New(), int64(size))
-}
-
-// Runs the speedtest on local MinIO process.
-func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration, storageClass string) (SpeedtestResult, error) {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return SpeedtestResult{}, errServerNotInitialized
-	}
-
-	var errOnce sync.Once
-	var retError string
-	var wg sync.WaitGroup
-	var totalBytesWritten uint64
-	var totalBytesRead uint64
-
-	objCountPerThread := make([]uint64, concurrent)
-
-	uploadsCtx, uploadsCancel := context.WithCancel(context.Background())
-	defer uploadsCancel()
-
-	go func() {
-		time.Sleep(duration)
-		uploadsCancel()
-	}()
-
-	objNamePrefix := "speedtest/objects/" + uuid.New().String()
-
-	wg.Add(concurrent)
-	for i := 0; i < concurrent; i++ {
-		go func(i int) {
-			defer wg.Done()
-			for {
-				hashReader, err := hash.NewReader(newRandomReader(size),
-					int64(size), "", "", int64(size))
-				if err != nil {
-					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					uploadsCancel()
-					return
-				}
-				reader := NewPutObjReader(hashReader)
-				objInfo, err := objAPI.PutObject(uploadsCtx, minioMetaBucket, fmt.Sprintf("%s.%d.%d",
-					objNamePrefix, i, objCountPerThread[i]), reader, ObjectOptions{
-					UserDefined: map[string]string{
-						xhttp.AmzStorageClass: storageClass,
-					},
-					Speedtest: true,
-				})
-				if err != nil {
-					objCountPerThread[i]--
-					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					uploadsCancel()
-					return
-				}
-				atomic.AddUint64(&totalBytesWritten, uint64(objInfo.Size))
-				objCountPerThread[i]++
-			}
-		}(i)
-	}
-	wg.Wait()
-
-	// We already saw write failures, no need to proceed into read's
-	if retError != "" {
-		return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
-	}
-
-	downloadsCtx, downloadsCancel := context.WithCancel(context.Background())
-	defer downloadsCancel()
-	go func() {
-		time.Sleep(duration)
-		downloadsCancel()
-	}()
-
-	wg.Add(concurrent)
-	for i := 0; i < concurrent; i++ {
-		go func(i int) {
-			defer wg.Done()
-			var j uint64
-			if objCountPerThread[i] == 0 {
-				return
-			}
-			for {
-				if objCountPerThread[i] == j {
-					j = 0
-				}
-				r, err := objAPI.GetObjectNInfo(downloadsCtx, minioMetaBucket, fmt.Sprintf("%s.%d.%d",
-					objNamePrefix, i, j), nil, nil, noLock, ObjectOptions{})
-				if err != nil {
-					if isErrObjectNotFound(err) {
-						continue
-					}
-					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					downloadsCancel()
-					return
-				}
-				n, err := io.Copy(ioutil.Discard, r)
-				r.Close()
-				if err == nil {
-					// Only capture success criteria - do not
-					// have to capture failed reads, truncated
-					// reads etc.
-					atomic.AddUint64(&totalBytesRead, uint64(n))
-				}
-				if err != nil {
-					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					downloadsCancel()
-					return
-				}
-				j++
-			}
-		}(i)
-	}
-	wg.Wait()
-
-	return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
-}
-
 func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("invalid request"))
@@ -1384,6 +1237,50 @@ func (s *peerRESTServer) DriveSpeedTestHandler(w http.ResponseWriter, r *http.Re
 	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
 }
 
+// DevNull - everything goes to io.Discard
+func (s *peerRESTServer) DevNull(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	globalNetPerfRX.Connect()
+	defer globalNetPerfRX.Disconnect()
+
+	connectTime := time.Now()
+	ctx := newContext(r, w, "DevNull")
+	for {
+		n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
+		atomic.AddUint64(&globalNetPerfRX.RX, uint64(n))
+		if err != nil && err != io.EOF {
+			// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
+			// would mean the network is not stable. Logging here will help in debugging network issues.
+			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
+				logger.LogIf(ctx, err)
+			}
+		}
+		if err != nil {
+			break
+		}
+	}
+}
+
+// Netperf - perform netperf
+func (s *peerRESTServer) Netperf(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	durationStr := r.Form.Get(peerRESTDuration)
+	duration, err := time.ParseDuration(durationStr)
+	if err != nil || duration.Seconds() == 0 {
+		duration = time.Second * 10
+	}
+	result := netperf(r.Context(), duration.Round(time.Second))
+	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
+}
+
 // registerPeerRESTHandlers - register peer rest router.
 func registerPeerRESTHandlers(router *mux.Router) {
 	server := &peerRESTServer{}
@@ -1431,6 +1328,8 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedtest).HandlerFunc(httpTraceHdrs(server.SpeedtestHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveSpeedTest).HandlerFunc(httpTraceHdrs(server.DriveSpeedTestHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(httpTraceHdrs(server.Netperf))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDevNull).HandlerFunc(httpTraceHdrs(server.DevNull))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadSiteReplicationConfig).HandlerFunc(httpTraceHdrs(server.ReloadSiteReplicationConfigHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadPoolMeta).HandlerFunc(httpTraceHdrs(server.ReloadPoolMetaHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLastDayTierStats).HandlerFunc(httpTraceHdrs(server.GetLastDayTierStatsHandler))

cmd/perf-tests.go

@@ -0,0 +1,295 @@
+// Copyright (c) 2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/google/uuid"
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/hash"
+	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/pkg/randreader"
+)
+
+// SpeedtestResult return value of the speedtest function
+type SpeedtestResult struct {
+	Endpoint  string
+	Uploads   uint64
+	Downloads uint64
+	Error     string
+}
+
+func newRandomReader(size int) io.Reader {
+	return io.LimitReader(randreader.New(), int64(size))
+}
+
+// Runs the speedtest on local MinIO process.
+func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration, storageClass string) (SpeedtestResult, error) {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return SpeedtestResult{}, errServerNotInitialized
+	}
+
+	var errOnce sync.Once
+	var retError string
+	var wg sync.WaitGroup
+	var totalBytesWritten uint64
+	var totalBytesRead uint64
+
+	objCountPerThread := make([]uint64, concurrent)
+
+	uploadsCtx, uploadsCancel := context.WithCancel(context.Background())
+	defer uploadsCancel()
+
+	go func() {
+		time.Sleep(duration)
+		uploadsCancel()
+	}()
+
+	objNamePrefix := "speedtest/objects/" + uuid.New().String()
+
+	wg.Add(concurrent)
+	for i := 0; i < concurrent; i++ {
+		go func(i int) {
+			defer wg.Done()
+			for {
+				hashReader, err := hash.NewReader(newRandomReader(size),
+					int64(size), "", "", int64(size))
+				if err != nil {
+					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
+						errOnce.Do(func() {
+							retError = err.Error()
+						})
+					}
+					uploadsCancel()
+					return
+				}
+				reader := NewPutObjReader(hashReader)
+				objInfo, err := objAPI.PutObject(uploadsCtx, minioMetaBucket, fmt.Sprintf("%s.%d.%d",
+					objNamePrefix, i, objCountPerThread[i]), reader, ObjectOptions{
+					UserDefined: map[string]string{
+						xhttp.AmzStorageClass: storageClass,
+					},
+					Speedtest: true,
+				})
+				if err != nil {
+					objCountPerThread[i]--
+					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
+						errOnce.Do(func() {
+							retError = err.Error()
+						})
+					}
+					uploadsCancel()
+					return
+				}
+				atomic.AddUint64(&totalBytesWritten, uint64(objInfo.Size))
+				objCountPerThread[i]++
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	// We already saw write failures, no need to proceed into read's
+	if retError != "" {
+		return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
+	}
+
+	downloadsCtx, downloadsCancel := context.WithCancel(context.Background())
+	defer downloadsCancel()
+	go func() {
+		time.Sleep(duration)
+		downloadsCancel()
+	}()
+
+	wg.Add(concurrent)
+	for i := 0; i < concurrent; i++ {
+		go func(i int) {
+			defer wg.Done()
+			var j uint64
+			if objCountPerThread[i] == 0 {
+				return
+			}
+			for {
+				if objCountPerThread[i] == j {
+					j = 0
+				}
+				r, err := objAPI.GetObjectNInfo(downloadsCtx, minioMetaBucket, fmt.Sprintf("%s.%d.%d",
+					objNamePrefix, i, j), nil, nil, noLock, ObjectOptions{})
+				if err != nil {
+					if isErrObjectNotFound(err) {
+						continue
+					}
+					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
+						errOnce.Do(func() {
+							retError = err.Error()
+						})
+					}
+					downloadsCancel()
+					return
+				}
+				n, err := io.Copy(ioutil.Discard, r)
+				r.Close()
+				if err == nil {
+					// Only capture success criteria - do not
+					// have to capture failed reads, truncated
+					// reads etc.
+					atomic.AddUint64(&totalBytesRead, uint64(n))
+				}
+				if err != nil {
+					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
+						errOnce.Do(func() {
+							retError = err.Error()
+						})
+					}
+					downloadsCancel()
+					return
+				}
+				j++
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
+}
+
+// To collect RX stats during "mc support perf net"
+// RXSample holds the RX bytes for the duration between
+// the last peer to connect and the first peer to disconnect.
+// This is to improve the RX throughput accuracy.
+type netPerfRX struct {
+	RX                uint64    // RX bytes
+	lastToConnect     time.Time // time at which last peer to connect to us
+	firstToDisconnect time.Time // time at which the first peer disconnects from us
+	RXSample          uint64    // RX bytes between lastToConnect and firstToDisconnect
+	activeConnections uint64
+	sync.RWMutex
+}
+
+func (n *netPerfRX) Connect() {
+	n.Lock()
+	defer n.Unlock()
+	n.activeConnections++
+	atomic.StoreUint64(&globalNetPerfRX.RX, 0)
+	n.lastToConnect = time.Now()
+}
+
+func (n *netPerfRX) Disconnect() {
+	n.Lock()
+	defer n.Unlock()
+	n.activeConnections--
+	if n.firstToDisconnect.IsZero() {
+		n.RXSample = atomic.LoadUint64(&n.RX)
+		n.firstToDisconnect = time.Now()
+	}
+}
+
+func (n *netPerfRX) ActiveConnections() uint64 {
+	n.RLock()
+	defer n.RUnlock()
+	return n.activeConnections
+}
+
+func (n *netPerfRX) Reset() {
+	n.RLock()
+	defer n.RUnlock()
+	n.RX = 0
+	n.RXSample = 0
+	n.lastToConnect = time.Time{}
+	n.firstToDisconnect = time.Time{}
+}
+
+// Reader to read random data.
+type netperfReader struct {
+	n   uint64
+	eof chan struct{}
+	buf []byte
+}
+
+func (m *netperfReader) Read(b []byte) (int, error) {
+	select {
+	case <-m.eof:
+		return 0, io.EOF
+	default:
+	}
+	n := copy(b, m.buf)
+	atomic.AddUint64(&m.n, uint64(n))
+	return n, nil
+}
+
+func netperf(ctx context.Context, duration time.Duration) madmin.NetperfNodeResult {
+	r := &netperfReader{eof: make(chan struct{})}
+	r.buf = make([]byte, 128*humanize.KiByte)
+	rand.Read(r.buf)
+
+	connectionsPerPeer := 16
+
+	if len(globalNotificationSys.peerClients) > 16 {
+		// For a large cluster it's enough to have 1 connection per peer to saturate the network.
+		connectionsPerPeer = 1
+	}
+
+	errStr := ""
+	var wg sync.WaitGroup
+	for index := range globalNotificationSys.peerClients {
+		if globalNotificationSys.peerClients[index] == nil {
+			continue
+		}
+		go func(index int) {
+			for i := 0; i < connectionsPerPeer; i++ {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					err := globalNotificationSys.peerClients[index].DevNull(ctx, r)
+					if err != nil {
+						errStr = err.Error()
+					}
+				}()
+			}
+		}(index)
+	}
+
+	time.Sleep(duration)
+	close(r.eof)
+	wg.Wait()
+	for {
+		if globalNetPerfRX.ActiveConnections() == 0 {
+			break
+		}
+		time.Sleep(time.Second)
+	}
+	rx := float64(globalNetPerfRX.RXSample)
+	delta := globalNetPerfRX.firstToDisconnect.Sub(globalNetPerfRX.lastToConnect)
+	if delta < 0 {
+		rx = 0
+		errStr = "network disconnection issues detected"
+	}
+
+	globalNetPerfRX.Reset()
+	return madmin.NetperfNodeResult{Endpoint: "", TX: r.n / uint64(duration.Seconds()), RX: uint64(rx / delta.Seconds()), Error: errStr}
+}

cmd/server-main.go

@@ -448,6 +448,11 @@ func serverMain(ctx *cli.Context) {
 	// Set system resources to maximum.
 	setMaxResources()
 
+	// Verify kernel release and version.
+	if oldLinux() {
+		logger.Info(color.RedBold("WARNING: Detected Linux kernel version older than 4.0.0 release, there are some known potential performance problems with this kernel version. MinIO recommends a minimum of 4.x.x linux kernel version for best performance"))
+	}
+
 	// Configure server.
 	handler, err := configureServerHandler(globalEndpoints)
 	if err != nil {
@@ -507,11 +512,11 @@ func serverMain(ctx *cli.Context) {
 	if globalActiveCred.Equal(auth.DefaultCredentials) {
 		msg := fmt.Sprintf("WARNING: Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables",
 			globalActiveCred)
-		logStartupMessage(color.RedBold(msg))
+		logger.Info(color.RedBold(msg))
 	}
 
 	if !globalCLIContext.StrictS3Compat {
-		logStartupMessage(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
+		logger.Info(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
 	}
 
 	if err = initServer(GlobalContext, newObject); err != nil {
@@ -595,7 +600,7 @@ func serverMain(ctx *cli.Context) {
 
 		// initialize the new disk cache objects.
 		if globalCacheConfig.Enabled {
-			logStartupMessage(color.Yellow("WARNING: Disk caching is deprecated for single/multi drive MinIO setups. Please migrate to using MinIO S3 gateway instead of disk caching"))
+			logger.Info(color.Yellow("WARNING: Disk caching is deprecated for single/multi drive MinIO setups. Please migrate to using MinIO S3 gateway instead of disk caching"))
 			var cacheAPI CacheObjectLayer
 			cacheAPI, err = newServerCacheObjects(GlobalContext, globalCacheConfig)
 			logger.FatalIf(err, "Unable to initialize disk caching")

cmd/server-rlimit.go

@@ -21,10 +21,28 @@ import (
 	"runtime"
 	"runtime/debug"
 
+	"github.com/minio/minio/internal/kernel"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/sys"
 )
 
+func oldLinux() bool {
+	currentKernel, err := kernel.CurrentVersion()
+	if err != nil {
+		// Could not probe the kernel version
+		return false
+	}
+
+	if currentKernel == 0 {
+		// We could not get any valid value return false
+		return false
+	}
+
+	// legacy linux indicator for printing warnings
+	// about older Linux kernels and Go runtime.
+	return currentKernel < kernel.Version(4, 0, 0)
+}
+
 func setMaxResources() (err error) {
 	// Set the Go runtime max threads threshold to 90% of kernel setting.
 	sysMaxThreads, mErr := sys.GetMaxThreads()
@@ -44,7 +62,8 @@ func setMaxResources() (err error) {
 	}
 
 	if maxLimit < 4096 && runtime.GOOS != globalWindowsOSName {
-		logger.Info("WARNING: maximum file descriptor limit %d is too low for production servers. At least 4096 is recommended. Fix with \"ulimit -n 4096\"", maxLimit)
+		logger.Info("WARNING: maximum file descriptor limit %d is too low for production servers. At least 4096 is recommended. Fix with \"ulimit -n 4096\"",
+			maxLimit)
 	}
 
 	if err = sys.SetMaxOpenFileLimit(maxLimit, maxLimit); err != nil {

cmd/server-startup-msg.go

@@ -45,9 +45,9 @@ func mustGetStorageInfo(objAPI ObjectLayer) StorageInfo {
 // Prints the formatted startup message.
 func printStartupMessage(apiEndpoints []string, err error) {
 	if err != nil {
-		logStartupMessage(color.RedBold("Server startup failed with '%v'", err))
-		logStartupMessage(color.RedBold("Not all features may be available on this server"))
-		logStartupMessage(color.RedBold("Please use 'mc admin' commands to further investigate this issue"))
+		if globalConsoleSys != nil {
+			globalConsoleSys.Send(fmt.Sprintf("Server startup failed with '%v', some features may be missing", err), string(logger.All))
+		}
 	}
 
 	strippedAPIEndpoints := stripStandardPorts(apiEndpoints, globalMinioHost)
@@ -76,7 +76,7 @@ func printStartupMessage(apiEndpoints []string, err error) {
 	if globalMinioConsolePortAuto && globalBrowserEnabled {
 		msg := fmt.Sprintf("\nWARNING: Console endpoint is listening on a dynamic port (%s), please use --console-address \":PORT\" to choose a static port.",
 			globalMinioConsolePort)
-		logStartupMessage(color.RedBold(msg))
+		logger.Info(color.RedBold(msg))
 	}
 }
 
@@ -131,29 +131,29 @@ func printServerCommonMsg(apiEndpoints []string) {
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 
 	// Colorize the message and print.
-	logStartupMessage(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
-	if color.IsTerminal() && !globalCLIContext.Anonymous {
-		logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-		logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+	logger.Info(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr)))
+	if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
+		logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+		logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 		if region != "" {
-			logStartupMessage(color.Blue("Region: ") + color.Bold(fmt.Sprintf(getFormatStr(len(region), 2), region)))
+			logger.Info(color.Blue("Region: ") + color.Bold(fmt.Sprintf(getFormatStr(len(region), 2), region)))
 		}
 	}
 	printEventNotifiers()
 
 	if globalBrowserEnabled {
 		consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ")
-		logStartupMessage(color.Blue("\nConsole: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
-		if color.IsTerminal() && !globalCLIContext.Anonymous {
-			logStartupMessage(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
-			logStartupMessage(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
+		logger.Info(color.Blue("\nConsole: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr)))
+		if color.IsTerminal() && (!globalCLIContext.Anonymous && !globalCLIContext.JSON) {
+			logger.Info(color.Blue("RootUser: ") + color.Bold(fmt.Sprintf("%s ", cred.AccessKey)))
+			logger.Info(color.Blue("RootPass: ") + color.Bold(fmt.Sprintf("%s ", cred.SecretKey)))
 		}
 	}
 }
 
 // Prints startup message for Object API acces, prints link to our SDK documentation.
 func printObjectAPIMsg() {
-	logStartupMessage(color.Blue("\nDocumentation: ") + "https://docs.min.io")
+	logger.Info(color.Blue("\nDocumentation: ") + "https://docs.min.io")
 }
 
 // Prints bucket notification configurations.
@@ -172,7 +172,7 @@ func printEventNotifiers() {
 		arnMsg += color.Bold(fmt.Sprintf("%s ", arn))
 	}
 
-	logStartupMessage(arnMsg)
+	logger.Info(arnMsg)
 }
 
 // Prints startup message for command line access. Prints link to our documentation
@@ -185,15 +185,15 @@ func printCLIAccessMsg(endPoint string, alias string) {
 
 	// Configure 'mc', following block prints platform specific information for minio client.
 	if color.IsTerminal() && !globalCLIContext.Anonymous {
-		logStartupMessage(color.Blue("\nCommand-line: ") + mcQuickStartGuide)
+		logger.Info(color.Blue("\nCommand-line: ") + mcQuickStartGuide)
 		if runtime.GOOS == globalWindowsOSName {
 			mcMessage := fmt.Sprintf("$ mc.exe alias set %s %s %s %s", alias,
 				endPoint, cred.AccessKey, cred.SecretKey)
-			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+			logger.Info(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
 		} else {
 			mcMessage := fmt.Sprintf("$ mc alias set %s %s %s %s", alias,
 				endPoint, cred.AccessKey, cred.SecretKey)
-			logStartupMessage(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
+			logger.Info(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage))
 		}
 	}
 }
@@ -220,10 +220,7 @@ func getStorageInfoMsg(storageInfo StorageInfo) string {
 // Prints startup message of storage capacity and erasure information.
 func printStorageInfo(storageInfo StorageInfo) {
 	if msg := getStorageInfoMsg(storageInfo); msg != "" {
-		if globalCLIContext.Quiet {
-			logger.Info(msg)
-		}
-		logStartupMessage(msg)
+		logger.Info(msg)
 	}
 }
 
@@ -231,5 +228,5 @@ func printCacheStorageInfo(storageInfo CacheStorageInfo) {
 	msg := fmt.Sprintf("%s %s Free, %s Total", color.Blue("Cache Capacity:"),
 		humanize.IBytes(storageInfo.Free),
 		humanize.IBytes(storageInfo.Total))
-	logStartupMessage(msg)
+	logger.Info(msg)
 }

cmd/site-replication.go

@@ -189,7 +189,7 @@ func (c *SiteReplicationSys) Init(ctx context.Context, objAPI ObjectLayer) error
 	c.RLock()
 	defer c.RUnlock()
 	if c.enabled {
-		logger.Info("Cluster Replication initialized.")
+		logger.Info("Cluster replication initialized")
 	}
 
 	return err

cmd/speedtest.go

@@ -19,8 +19,6 @@ package cmd
 
 import (
 	"context"
-	"os"
-	"path/filepath"
 	"sort"
 	"time"
 
@@ -28,8 +26,6 @@ import (
 	"github.com/minio/madmin-go"
 )
 
-var dperfUUID = mustGetUUID()
-
 type speedTestOpts struct {
 	throughputSize   int
 	concurrencyStart int
@@ -171,29 +167,23 @@ func driveSpeedTest(ctx context.Context, opts madmin.DriveSpeedTestOpts) madmin.
 		BlockSize: opts.BlockSize,
 		FileSize:  opts.FileSize,
 	}
-	paths := func() []string {
-		toRet := []string{}
-		paths := globalEndpoints.LocalDisksPaths()
-		for _, p := range paths {
-			dperfFilename := filepath.Join(p, ".minio.sys", "tmp", dperfUUID)
-			if _, err := os.Stat(dperfFilename); err == nil {
-				// file exists, so remove it
-				os.Remove(dperfFilename)
-			}
-			toRet = append(toRet, dperfFilename)
+
+	localPaths := globalEndpoints.LocalDisksPaths()
+	paths := func() (tmpPaths []string) {
+		for _, lp := range localPaths {
+			tmpPaths = append(tmpPaths, pathJoin(lp, minioMetaTmpBucket))
 		}
-		return toRet
+		return tmpPaths
 	}()
 
 	perfs, err := perf.Run(ctx, paths...)
 	return madmin.DriveSpeedTestResult{
 		Endpoint: globalLocalNodeName,
 		Version:  Version,
-		DrivePerf: func() []madmin.DrivePerf {
-			results := []madmin.DrivePerf{}
-			for _, r := range perfs {
+		DrivePerf: func() (results []madmin.DrivePerf) {
+			for idx, r := range perfs {
 				result := madmin.DrivePerf{
-					Path:            r.Path,
+					Path:            localPaths[idx],
 					ReadThroughput:  r.ReadThroughput,
 					WriteThroughput: r.WriteThroughput,
 					Error: func() string {

cmd/storage-interface.go

@@ -29,17 +29,40 @@ type StorageAPI interface {
 	String() string
 
 	// Storage operations.
-	IsOnline() bool      // Returns true if disk is online.
-	LastConn() time.Time // Returns the last time this disk (re)-connected
 
+	// Returns true if disk is online and its valid i.e valid format.json.
+	// This has nothing to do with if the drive is hung or not responding.
+	// For that individual storage API calls will fail properly. The purpose
+	// of this function is to know if the "drive" has "format.json" or not
+	// if it has a "format.json" then is it correct "format.json" or not.
+	IsOnline() bool
+
+	// Returns the last time this disk (re)-connected
+	LastConn() time.Time
+
+	// Indicates if disk is local or not.
 	IsLocal() bool
-	Hostname() string   // Returns host name if remote host.
-	Endpoint() Endpoint // Returns endpoint.
 
+	// Returns hostname if disk is remote.
+	Hostname() string
+
+	// Returns the entire endpoint.
+	Endpoint() Endpoint
+
+	// Close the disk, mark it purposefully closed, only implemented for remote disks.
 	Close() error
+
+	// Returns the unique 'uuid' of this disk.
 	GetDiskID() (string, error)
+
+	// Set a unique 'uuid' for this disk, only used when
+	// disk is replaced and formatted.
 	SetDiskID(id string)
-	Healing() *healingTracker // Returns nil if disk is not healing.
+
+	// Returns healing information for a newly replaced disk,
+	// returns 'nil' once healing is complete or if the disk
+	// has never been replaced.
+	Healing() *healingTracker
 
 	DiskInfo(ctx context.Context) (info DiskInfo, err error)
 	NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error)

cmd/storage-rest-server.go

@@ -52,7 +52,7 @@ var errDiskStale = errors.New("disk stale")
 
 // To abstract a disk over network.
 type storageRESTServer struct {
-	storage *xlStorage
+	storage *xlStorageDiskIDCheck
 }
 
 func (s *storageRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
@@ -109,7 +109,7 @@ func storageServerRequestValidate(r *http.Request) error {
 }
 
 // IsValid - To authenticate and verify the time difference.
-func (s *storageRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
+func (s *storageRESTServer) IsAuthValid(w http.ResponseWriter, r *http.Request) bool {
 	if s.storage == nil {
 		s.writeErrorResponse(w, errDiskNotFound)
 		return false
@@ -120,6 +120,15 @@ func (s *storageRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool
 		return false
 	}
 
+	return true
+}
+
+// IsValid - To authenticate and check if the disk-id in the request corresponds to the underlying disk.
+func (s *storageRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
+	if !s.IsAuthValid(w, r) {
+		return false
+	}
+
 	if err := r.ParseForm(); err != nil {
 		s.writeErrorResponse(w, err)
 		return false
@@ -155,7 +164,7 @@ func (s *storageRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request
 
 // DiskInfoHandler - returns disk info.
 func (s *storageRESTServer) DiskInfoHandler(w http.ResponseWriter, r *http.Request) {
-	if !s.IsValid(w, r) {
+	if !s.IsAuthValid(w, r) {
 		return
 	}
 	info, err := s.storage.DiskInfo(r.Context())
@@ -1224,7 +1233,8 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 
 			endpoint := storage.Endpoint()
 
-			server := &storageRESTServer{storage: storage}
+			server := &storageRESTServer{storage: newXLStorageDiskIDCheck(storage)}
+			server.storage.SetDiskID(storage.diskID)
 
 			subrouter := router.PathPrefix(path.Join(storageRESTPrefix, endpoint.Path)).Subrouter()
 

cmd/update-notifier.go

@@ -41,6 +41,10 @@ func prepareUpdateMessage(downloadURL string, older time.Duration) string {
 	t := time.Time{}
 	newerThan := humanize.RelTime(t, t.Add(older), "ago", "")
 
+	if globalCLIContext.JSON {
+		return fmt.Sprintf("You are running an older version of MinIO released %s, update: %s", newerThan, downloadURL)
+	}
+
 	// Return the nicely colored and formatted update message.
 	return colorizeUpdateMessage(downloadURL, newerThan)
 }

cmd/utils.go

@@ -633,6 +633,7 @@ func NewGatewayHTTPTransportWithClientCerts(clientCert, clientKey string) *http.
 				err.Error()))
 		}
 		if c != nil {
+			c.UpdateReloadDuration(10 * time.Second)
 			c.ReloadOnSignal(syscall.SIGHUP) // allow reloads upon SIGHUP
 			transport.TLSClientConfig.GetClientCertificate = c.GetClientCertificate
 		}

cmd/xl-storage-disk-id-check.go

@@ -19,13 +19,19 @@ package cmd
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"io"
+	"math/rand"
+	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/logger"
 )
 
 //go:generate stringer -type=storageMetric -trimprefix=storageMetric $GOFILE
@@ -69,10 +75,11 @@ type xlStorageDiskIDCheck struct {
 	// do not re-order them, if you add new fields
 	// please use `fieldalignment ./...` to check
 	// if your changes are not causing any problems.
-	storage      StorageAPI
+	storage      *xlStorage
 	apiLatencies [storageMetricLast]*lockedLastMinuteLatency
 	diskID       string
 	apiCalls     [storageMetricLast]uint64
+	health       *diskHealthTracker
 }
 
 func (p *xlStorageDiskIDCheck) getMetrics() DiskMetrics {
@@ -109,6 +116,7 @@ func (e *lockedLastMinuteLatency) value() uint64 {
 func newXLStorageDiskIDCheck(storage *xlStorage) *xlStorageDiskIDCheck {
 	xl := xlStorageDiskIDCheck{
 		storage: storage,
+		health:  newDiskHealthTracker(),
 	}
 	for i := range xl.apiLatencies[:] {
 		xl.apiLatencies[i] = &lockedLastMinuteLatency{}
@@ -215,25 +223,31 @@ func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context) (info DiskInfo, err
 			return info, errDiskNotFound
 		}
 	}
+
+	if p.health.isFaulty() {
+		// if disk is already faulty return faulty for 'mc admin info' output and prometheus alerts.
+		return info, errFaultyDisk
+	}
+
 	return info, nil
 }
 
 func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
-	defer p.updateStorageMetrics(storageMetricMakeVolBulk, volumes...)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricMakeVolBulk, volumes...)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
+
 	return p.storage.MakeVolBulk(ctx, volumes...)
 }
 
 func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err error) {
-	defer p.updateStorageMetrics(storageMetricMakeVol, volume)()
-
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricMakeVol, volume)
+	if err != nil {
+		return err
+	}
+	defer done(&err)
 	if contextCanceled(ctx) {
 		return ctx.Err()
 	}
@@ -244,167 +258,122 @@ func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err
 	return p.storage.MakeVol(ctx, volume)
 }
 
-func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) ([]VolInfo, error) {
-	defer p.updateStorageMetrics(storageMetricListVols, "/")()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) (vi []VolInfo, err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricListVols, "/")
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
+
 	return p.storage.ListVols(ctx)
 }
 
 func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
-	defer p.updateStorageMetrics(storageMetricStatVol, volume)()
-
-	if contextCanceled(ctx) {
-		return VolInfo{}, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricStatVol, volume)
+	if err != nil {
 		return vol, err
 	}
+	defer done(&err)
+
 	return p.storage.StatVol(ctx, volume)
 }
 
 func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
-	defer p.updateStorageMetrics(storageMetricDeleteVol, volume)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVol, volume)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
+
 	return p.storage.DeleteVol(ctx, volume, forceDelete)
 }
 
-func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error) {
-	defer p.updateStorageMetrics(storageMetricListDir, volume, dirPath)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) (s []string, err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricListDir, volume, dirPath)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.ListDir(ctx, volume, dirPath, count)
 }
 
 func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
-	defer p.updateStorageMetrics(storageMetricReadFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return 0, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadFile, volume, path)
+	if err != nil {
 		return 0, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadFile(ctx, volume, path, offset, buf, verifier)
 }
 
 func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) {
-	defer p.updateStorageMetrics(storageMetricAppendFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricAppendFile, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.AppendFile(ctx, volume, path, buf)
 }
 
-func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
-	defer p.updateStorageMetrics(storageMetricCreateFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricCreateFile, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.CreateFile(ctx, volume, path, size, reader)
 }
 
 func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
-	defer p.updateStorageMetrics(storageMetricReadFileStream, volume, path)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadFileStream, volume, path)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadFileStream(ctx, volume, path, offset, length)
 }
 
-func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
-	defer p.updateStorageMetrics(storageMetricRenameFile, srcVolume, srcPath, dstVolume, dstPath)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricRenameFile, srcVolume, srcPath, dstVolume, dstPath)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath)
 }
 
-func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) error {
-	defer p.updateStorageMetrics(storageMetricRenameData, srcPath, fi.DataDir, dstVolume, dstPath)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricRenameData, srcPath, fi.DataDir, dstVolume, dstPath)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath)
 }
 
 func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
-	defer p.updateStorageMetrics(storageMetricCheckParts, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricCheckParts, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.CheckParts(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, recursive bool) (err error) {
-	defer p.updateStorageMetrics(storageMetricDelete, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDelete, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.Delete(ctx, volume, path, recursive)
 }
@@ -417,136 +386,102 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string
 	if len(versions) > 0 {
 		path = versions[0].Name
 	}
-
-	defer p.updateStorageMetrics(storageMetricDeleteVersions, volume, path)()
-
 	errs = make([]error, len(versions))
-
-	if contextCanceled(ctx) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVersions, volume, path)
+	if err != nil {
 		for i := range errs {
 			errs[i] = ctx.Err()
 		}
 		return errs
 	}
-
-	if err := p.checkDiskStale(); err != nil {
-		for i := range errs {
-			errs[i] = err
+	defer done(&err)
+	errs = p.storage.DeleteVersions(ctx, volume, versions)
+	for i := range errs {
+		if errs[i] != nil {
+			err = errs[i]
+			break
 		}
-		return errs
 	}
 
-	return p.storage.DeleteVersions(ctx, volume, versions)
+	return errs
 }
 
-func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
-	defer p.updateStorageMetrics(storageMetricVerifyFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricVerifyFile, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.VerifyFile(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) {
-	defer p.updateStorageMetrics(storageMetricWriteAll, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWriteAll, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.WriteAll(ctx, volume, path, b)
 }
 
 func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) (err error) {
-	defer p.updateStorageMetrics(storageMetricDeleteVersion, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVersion, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.DeleteVersion(ctx, volume, path, fi, forceDelMarker)
 }
 
 func (p *xlStorageDiskIDCheck) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
-	defer p.updateStorageMetrics(storageMetricUpdateMetadata, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricUpdateMetadata, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.UpdateMetadata(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
-	defer p.updateStorageMetrics(storageMetricWriteMetadata, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWriteMetadata, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.WriteMetadata(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (fi FileInfo, err error) {
-	defer p.updateStorageMetrics(storageMetricReadVersion, volume, path)()
-
-	if contextCanceled(ctx) {
-		return fi, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadVersion, volume, path)
+	if err != nil {
 		return fi, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadVersion(ctx, volume, path, versionID, readData)
 }
 
 func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
-	defer p.updateStorageMetrics(storageMetricReadAll, volume, path)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadAll, volume, path)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadAll(ctx, volume, path)
 }
 
 func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
-	defer p.updateStorageMetrics(storageMetricStatInfoFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricStatInfoFile, volume, path)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.StatInfoFile(ctx, volume, path, glob)
 }
@@ -565,10 +500,10 @@ func storageTrace(s storageMetric, startTime time.Time, duration time.Duration,
 }
 
 // Update storage metrics
-func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func() {
+func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func(err *error) {
 	startTime := time.Now()
 	trace := globalTrace.NumSubscribers() > 0
-	return func() {
+	return func(err *error) {
 		duration := time.Since(startTime)
 
 		atomic.AddUint64(&p.apiCalls[s], 1)
@@ -579,3 +514,317 @@ func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...st
 		}
 	}
 }
+
+const (
+	diskHealthOK = iota
+	diskHealthFaulty
+)
+
+// diskMaxConcurrent is the maximum number of running concurrent operations
+// for local and (incoming) remote disk ops respectively.
+var diskMaxConcurrent = 50
+
+func init() {
+	if s, ok := os.LookupEnv("_MINIO_DISK_MAX_CONCURRENT"); ok && s != "" {
+		var err error
+		diskMaxConcurrent, err = strconv.Atoi(s)
+		if err != nil {
+			logger.Fatal(err, "invalid _MINIO_DISK_MAX_CONCURRENT value")
+		}
+	}
+}
+
+type diskHealthTracker struct {
+	// atomic time of last success
+	lastSuccess int64
+
+	// atomic time of last time a token was grabbed.
+	lastStarted int64
+
+	// Atomic status of disk.
+	status int32
+
+	// Atomic number of requests blocking for a token.
+	blocked int32
+
+	// Concurrency tokens.
+	tokens chan struct{}
+}
+
+// newDiskHealthTracker creates a new disk health tracker.
+func newDiskHealthTracker() *diskHealthTracker {
+	d := diskHealthTracker{
+		lastSuccess: time.Now().UnixNano(),
+		lastStarted: time.Now().UnixNano(),
+		status:      diskHealthOK,
+		tokens:      make(chan struct{}, diskMaxConcurrent),
+	}
+	for i := 0; i < diskMaxConcurrent; i++ {
+		d.tokens <- struct{}{}
+	}
+	return &d
+}
+
+// logSuccess will update the last successful operation time.
+func (d *diskHealthTracker) logSuccess() {
+	atomic.StoreInt64(&d.lastSuccess, time.Now().UnixNano())
+}
+
+func (d *diskHealthTracker) isFaulty() bool {
+	return atomic.LoadInt32(&d.status) == diskHealthFaulty
+}
+
+type (
+	healthDiskCtxKey   struct{}
+	healthDiskCtxValue struct {
+		lastSuccess *int64
+	}
+)
+
+// logSuccess will update the last successful operation time.
+func (h *healthDiskCtxValue) logSuccess() {
+	atomic.StoreInt64(h.lastSuccess, time.Now().UnixNano())
+}
+
+// noopDoneFunc is a no-op done func.
+// Can be reused.
+var noopDoneFunc = func(_ *error) {}
+
+// TrackDiskHealth for this request.
+// When a non-nil error is returned 'done' MUST be called
+// with the status of the response, if it corresponds to disk health.
+// If the pointer sent to done is non-nil AND the error
+// is either nil or io.EOF the disk is considered good.
+// So if unsure if the disk status is ok, return nil as a parameter to done.
+// Shadowing will work as long as return error is named: https://go.dev/play/p/sauq86SsTN2
+func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMetric, paths ...string) (c context.Context, done func(*error), err error) {
+	done = noopDoneFunc
+	if contextCanceled(ctx) {
+		return ctx, done, ctx.Err()
+	}
+
+	// Return early if disk is faulty already.
+	if atomic.LoadInt32(&p.health.status) == diskHealthFaulty {
+		return ctx, done, errFaultyDisk
+	}
+
+	// Verify if the disk is not stale
+	// - missing format.json (unformatted drive)
+	// - format.json is valid but invalid 'uuid'
+	if err = p.checkDiskStale(); err != nil {
+		return ctx, done, err
+	}
+
+	// Disallow recursive tracking to avoid deadlocks.
+	if ctx.Value(healthDiskCtxKey{}) != nil {
+		done = p.updateStorageMetrics(s, paths...)
+		return ctx, done, nil
+	}
+
+	select {
+	case <-ctx.Done():
+		return ctx, done, ctx.Err()
+	case <-p.health.tokens:
+		// Fast path, got token.
+	default:
+		// We ran out of tokens, check health before blocking.
+		err = p.waitForToken(ctx)
+		if err != nil {
+			return ctx, done, err
+		}
+	}
+	// We only progress here if we got a token.
+
+	atomic.StoreInt64(&p.health.lastStarted, time.Now().UnixNano())
+	ctx = context.WithValue(ctx, healthDiskCtxKey{}, &healthDiskCtxValue{lastSuccess: &p.health.lastSuccess})
+	si := p.updateStorageMetrics(s, paths...)
+	t := time.Now()
+	var once sync.Once
+	return ctx, func(errp *error) {
+		once.Do(func() {
+			if false {
+				var ers string
+				if errp != nil {
+					err := *errp
+					ers = fmt.Sprint(err)
+				}
+				fmt.Println(time.Now().Format(time.RFC3339), "op", s, "took", time.Since(t), "result:", ers, "disk:", p.storage.String(), "path:", strings.Join(paths, "/"))
+			}
+			p.health.tokens <- struct{}{}
+			if errp != nil {
+				err := *errp
+				if err != nil && !errors.Is(err, io.EOF) {
+					return
+				}
+				p.health.logSuccess()
+			}
+			si(errp)
+		})
+	}, nil
+}
+
+// waitForToken will wait for a token, while periodically
+// checking the disk status.
+// If nil is returned a token was picked up.
+func (p *xlStorageDiskIDCheck) waitForToken(ctx context.Context) (err error) {
+	atomic.AddInt32(&p.health.blocked, 1)
+	defer func() {
+		atomic.AddInt32(&p.health.blocked, -1)
+	}()
+	// Avoid stampeding herd...
+	ticker := time.NewTicker(5*time.Second + time.Duration(rand.Int63n(int64(5*time.Second))))
+	defer ticker.Stop()
+	for {
+		err = p.checkHealth(ctx)
+		if err != nil {
+			return err
+		}
+		select {
+		case <-ticker.C:
+		// Ticker expired, check health again.
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-p.health.tokens:
+			return nil
+		}
+	}
+}
+
+// checkHealth should only be called when tokens have run out.
+// This will check if disk should be taken offline.
+func (p *xlStorageDiskIDCheck) checkHealth(ctx context.Context) (err error) {
+	if atomic.LoadInt32(&p.health.status) == diskHealthFaulty {
+		return errFaultyDisk
+	}
+	// Check if there are tokens.
+	if len(p.health.tokens) > 0 {
+		return nil
+	}
+
+	const maxTimeSinceLastSuccess = 30 * time.Second
+	const minTimeSinceLastOpStarted = 15 * time.Second
+
+	// To avoid stampeding herd (100s of simultaneous starting requests)
+	// there must be a delay between the last started request and now
+	// for the last lastSuccess to be useful.
+	t := time.Since(time.Unix(0, atomic.LoadInt64(&p.health.lastStarted)))
+	if t < minTimeSinceLastOpStarted {
+		return nil
+	}
+
+	// If also more than 15 seconds since last success, take disk offline.
+	t = time.Since(time.Unix(0, atomic.LoadInt64(&p.health.lastSuccess)))
+	if t > maxTimeSinceLastSuccess {
+		if atomic.CompareAndSwapInt32(&p.health.status, diskHealthOK, diskHealthFaulty) {
+			logger.LogAlwaysIf(ctx, fmt.Errorf("taking disk %s offline, time since last response %v", p.storage.String(), t.Round(time.Millisecond)))
+			go p.monitorDiskStatus()
+		}
+		return errFaultyDisk
+	}
+	return nil
+}
+
+// monitorDiskStatus should be called once when a drive has been marked offline.
+// Once the disk has been deemed ok, it will return to online status.
+func (p *xlStorageDiskIDCheck) monitorDiskStatus() {
+	t := time.NewTicker(5 * time.Second)
+	defer t.Stop()
+	fn := mustGetUUID()
+	for range t.C {
+		if len(p.health.tokens) == 0 {
+			// Queue is still full, no need to check.
+			continue
+		}
+		err := p.storage.WriteAll(context.Background(), minioMetaTmpBucket, fn, []byte{10000: 42})
+		if err != nil {
+			continue
+		}
+		b, err := p.storage.ReadAll(context.Background(), minioMetaTmpBucket, fn)
+		if err != nil || len(b) != 10001 {
+			continue
+		}
+		err = p.storage.Delete(context.Background(), minioMetaTmpBucket, fn, false)
+		if err == nil {
+			logger.Info("Able to read+write, bringing disk %s online.", p.storage.String())
+			atomic.StoreInt32(&p.health.status, diskHealthOK)
+			return
+		}
+	}
+}
+
+// diskHealthCheckOK will check if the provided error is nil
+// and update disk status if good.
+// For convenience a bool is returned to indicate any error state
+// that is not io.EOF.
+func diskHealthCheckOK(ctx context.Context, err error) bool {
+	// Check if context has a disk health check.
+	tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue)
+	if !ok {
+		// No tracker, return
+		return err == nil || errors.Is(err, io.EOF)
+	}
+	if err == nil || errors.Is(err, io.EOF) {
+		tracker.logSuccess()
+		return true
+	}
+	return false
+}
+
+// diskHealthWrapper provides either a io.Reader or io.Writer
+// that updates status of the provided tracker.
+// Use through diskHealthReader or diskHealthWriter.
+type diskHealthWrapper struct {
+	tracker *healthDiskCtxValue
+	r       io.Reader
+	w       io.Writer
+}
+
+func (d *diskHealthWrapper) Read(p []byte) (int, error) {
+	if d.r == nil {
+		return 0, fmt.Errorf("diskHealthWrapper: Read with no reader")
+	}
+	n, err := d.r.Read(p)
+	if err == nil || err == io.EOF && n > 0 {
+		d.tracker.logSuccess()
+	}
+	return n, err
+}
+
+func (d *diskHealthWrapper) Write(p []byte) (int, error) {
+	if d.w == nil {
+		return 0, fmt.Errorf("diskHealthWrapper: Write with no writer")
+	}
+	n, err := d.w.Write(p)
+	if err == nil && n == len(p) {
+		d.tracker.logSuccess()
+	}
+	return n, err
+}
+
+// diskHealthReader provides a wrapper that will update disk health on
+// ctx, on every successful read.
+// This should only be used directly at the os/syscall level,
+// otherwise buffered operations may return false health checks.
+func diskHealthReader(ctx context.Context, r io.Reader) io.Reader {
+	// Check if context has a disk health check.
+	tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue)
+	if !ok {
+		// No need to wrap
+		return r
+	}
+	return &diskHealthWrapper{r: r, tracker: tracker}
+}
+
+// diskHealthWriter provides a wrapper that will update disk health on
+// ctx, on every successful write.
+// This should only be used directly at the os/syscall level,
+// otherwise buffered operations may return false health checks.
+func diskHealthWriter(ctx context.Context, w io.Writer) io.Writer {
+	// Check if context has a disk health check.
+	tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue)
+	if !ok {
+		// No need to wrap
+		return w
+	}
+	return &diskHealthWrapper{w: w, tracker: tracker}
+}

cmd/xl-storage-format-utils.go

@@ -18,6 +18,8 @@
 package cmd
 
 import (
+	"errors"
+
 	"github.com/zeebo/xxh3"
 )
 
@@ -57,7 +59,19 @@ func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string) (FileInfoVers
 		}
 		versions, err = xlMeta.ListVersions(volume, path)
 	}
-	if err != nil || len(versions) == 0 {
+	if err == nil && len(versions) == 0 {
+		// This special case is needed to handle len(xlMeta.versions) == 0
+		versions = []FileInfo{
+			{
+				Volume:   volume,
+				Name:     path,
+				Deleted:  true,
+				IsLatest: true,
+				ModTime:  timeSentinel1970,
+			},
+		}
+	}
+	if err != nil {
 		return FileInfoVersions{}, err
 	}
 
@@ -76,11 +90,33 @@ func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, data bool) (F
 	if buf, data := isIndexedMetaV2(xlMetaBuf); buf != nil {
 		inData = data
 		fi, err = buf.ToFileInfo(volume, path, versionID)
+		if len(buf) != 0 && errors.Is(err, errFileNotFound) {
+			// This special case is needed to handle len(xlMeta.versions) == 0
+			return FileInfo{
+				Volume:    volume,
+				Name:      path,
+				VersionID: versionID,
+				Deleted:   true,
+				IsLatest:  true,
+				ModTime:   timeSentinel1970,
+			}, nil
+		}
 	} else {
 		var xlMeta xlMetaV2
 		if err := xlMeta.LoadOrConvert(xlMetaBuf); err != nil {
 			return FileInfo{}, err
 		}
+		if len(xlMeta.versions) == 0 {
+			// This special case is needed to handle len(xlMeta.versions) == 0
+			return FileInfo{
+				Volume:    volume,
+				Name:      path,
+				VersionID: versionID,
+				Deleted:   true,
+				IsLatest:  true,
+				ModTime:   timeSentinel1970,
+			}, nil
+		}
 		inData = xlMeta.data
 		fi, err = xlMeta.ToFileInfo(volume, path, versionID)
 	}

cmd/xl-storage-format-v2.go

@@ -880,7 +880,7 @@ func (x *xlMetaV2) loadIndexed(buf xlMetaBuf, data xlMetaInlineData) error {
 	x.metaV = metaV
 	if err = x.data.validate(); err != nil {
 		x.data.repair()
-		logger.Info("xlMetaV2.loadIndexed: data validation failed: %v. %d entries after repair", err, x.data.entries())
+		logger.LogIf(GlobalContext, fmt.Errorf("xlMetaV2.loadIndexed: data validation failed: %v. %d entries after repair", err, x.data.entries()))
 	}
 
 	return decodeVersions(buf, versions, func(i int, hdr, meta []byte) error {
@@ -929,7 +929,7 @@ func (x *xlMetaV2) loadLegacy(buf []byte) error {
 			x.data = buf
 			if err = x.data.validate(); err != nil {
 				x.data.repair()
-				logger.Info("xlMetaV2.Load: data validation failed: %v. %d entries after repair", err, x.data.entries())
+				logger.LogIf(GlobalContext, fmt.Errorf("xlMetaV2.Load: data validation failed: %v. %d entries after repair", err, x.data.entries()))
 			}
 		default:
 			return errors.New("unknown minor metadata version")
@@ -1150,7 +1150,7 @@ func (x *xlMetaV2) sortByModTime() {
 // DeleteVersion deletes the version specified by version id.
 // returns to the caller which dataDir to delete, also
 // indicates if this is the last version.
-func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
+func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) {
 	// This is a situation where versionId is explicitly
 	// specified as "null", as we do not save "null"
 	// string it is considered empty. But empty also
@@ -1164,7 +1164,7 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 	if fi.VersionID != "" {
 		uv, err = uuid.Parse(fi.VersionID)
 		if err != nil {
-			return "", false, errFileVersionNotFound
+			return "", errFileVersionNotFound
 		}
 	}
 
@@ -1179,7 +1179,7 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 			},
 		}
 		if !ventry.Valid() {
-			return "", false, errors.New("internal error: invalid version entry generated")
+			return "", errors.New("internal error: invalid version entry generated")
 		}
 	}
 	updateVersion := false
@@ -1225,18 +1225,18 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 		case LegacyType:
 			ver, err := x.getIdx(i)
 			if err != nil {
-				return "", false, err
+				return "", err
 			}
 			x.versions = append(x.versions[:i], x.versions[i+1:]...)
 			if fi.Deleted {
 				err = x.addVersion(ventry)
 			}
-			return ver.ObjectV1.DataDir, len(x.versions) == 0, err
+			return ver.ObjectV1.DataDir, err
 		case DeleteType:
 			if updateVersion {
 				ver, err := x.getIdx(i)
 				if err != nil {
-					return "", false, err
+					return "", err
 				}
 				if len(ver.DeleteMarker.MetaSys) == 0 {
 					ver.DeleteMarker.MetaSys = make(map[string][]byte)
@@ -1258,26 +1258,26 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 					ver.DeleteMarker.MetaSys[k] = []byte(v)
 				}
 				err = x.setIdx(i, *ver)
-				return "", len(x.versions) == 0, err
+				return "", err
 			}
 			var err error
 			x.versions = append(x.versions[:i], x.versions[i+1:]...)
 			if fi.MarkDeleted && (fi.VersionPurgeStatus().Empty() || (fi.VersionPurgeStatus() != Complete)) {
 				err = x.addVersion(ventry)
 			}
-			return "", len(x.versions) == 0, err
+			return "", err
 		case ObjectType:
 			if updateVersion && !fi.Deleted {
 				ver, err := x.getIdx(i)
 				if err != nil {
-					return "", false, err
+					return "", err
 				}
 				ver.ObjectV2.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal)
 				for k, v := range fi.ReplicationState.ResetStatusesMap {
 					ver.ObjectV2.MetaSys[k] = []byte(v)
 				}
 				err = x.setIdx(i, *ver)
-				return "", len(x.versions) == 0, err
+				return "", err
 			}
 		}
 	}
@@ -1288,7 +1288,7 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 		}
 		ver, err := x.getIdx(i)
 		if err != nil {
-			return "", false, err
+			return "", err
 		}
 		switch {
 		case fi.ExpireRestored:
@@ -1314,16 +1314,16 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, bool, error) {
 		if x.SharedDataDirCount(ver.ObjectV2.VersionID, ver.ObjectV2.DataDir) > 0 {
 			// Found that another version references the same dataDir
 			// we shouldn't remove it, and only remove the version instead
-			return "", len(x.versions) == 0, nil
+			return "", nil
 		}
-		return uuid.UUID(ver.ObjectV2.DataDir).String(), len(x.versions) == 0, err
+		return uuid.UUID(ver.ObjectV2.DataDir).String(), err
 	}
 
 	if fi.Deleted {
 		err = x.addVersion(ventry)
-		return "", false, err
+		return "", err
 	}
-	return "", false, errFileVersionNotFound
+	return "", errFileVersionNotFound
 }
 
 // xlMetaDataDirDecoder is a shallow decoder for decoding object datadir only.

cmd/xl-storage-format-v2_test.go

@@ -366,7 +366,7 @@ func TestDeleteVersionWithSharedDataDir(t *testing.T) {
 	count := len(testCases)
 	for i := 4; i < len(testCases); i++ {
 		tc := testCases[i]
-		dataDir, _, err := xl.DeleteVersion(fileInfos[i])
+		dataDir, err := xl.DeleteVersion(fileInfos[i])
 		failOnErr(count+1, err)
 		if dataDir != tc.expectedDataDir {
 			t.Fatalf("Expected %s but got %s", tc.expectedDataDir, dataDir)

cmd/xl-storage-format_test.go

@@ -433,7 +433,7 @@ func BenchmarkXlMetaV2Shallow(b *testing.B) {
 					// Update a random version.
 					fi.VersionID = ids[rng.Intn(size)]
 					// Delete...
-					_, _, err = xl.DeleteVersion(fi)
+					_, err = xl.DeleteVersion(fi)
 					if err != nil {
 						b.Fatal(err)
 					}

cmd/xl-storage-free-version_test.go

@@ -102,7 +102,7 @@ func TestFreeVersion(t *testing.T) {
 	tierfi.TransitionedObjName = mustGetUUID()
 	tierfi.TransitionTier = "MINIOTIER-1"
 	var err error
-	_, _, err = xl.DeleteVersion(tierfi)
+	_, err = xl.DeleteVersion(tierfi)
 	fatalErr(err)
 	report()
 
@@ -124,7 +124,7 @@ func TestFreeVersion(t *testing.T) {
 	newtierfi.TransitionStatus = ""
 	newtierfi.SetTierFreeVersionID(fvIDs[1])
 	report()
-	_, _, err = xl.DeleteVersion(newtierfi)
+	_, err = xl.DeleteVersion(newtierfi)
 	report()
 	fatalErr(err)
 
@@ -141,7 +141,7 @@ func TestFreeVersion(t *testing.T) {
 	freefi := newtierfi
 	for _, fvID := range fvIDs {
 		freefi.VersionID = fvID
-		_, _, err = xl.DeleteVersion(freefi)
+		_, err = xl.DeleteVersion(freefi)
 		fatalErr(err)
 	}
 	report()

cmd/xl-storage.go

@@ -49,11 +49,12 @@ import (
 
 const (
 	nullVersionID = "null"
-	// Really large streams threshold per shard.
-	reallyLargeFileThreshold = 64 * humanize.MiByte // Optimized for HDDs
+	// Largest streams threshold per shard.
+	largestFileThreshold = 64 * humanize.MiByte // Optimized for HDDs
 
 	// Small file threshold below which data accompanies metadata from storage layer.
 	smallFileThreshold = 128 * humanize.KiByte // Optimized for NVMe/SSDs
+
 	// For hardrives it is possible to set this to a lower value to avoid any
 	// spike in latency. But currently we are simply keeping it optimal for SSDs.
 
@@ -683,9 +684,11 @@ func (s *xlStorage) SetDiskID(id string) {
 
 func (s *xlStorage) MakeVolBulk(ctx context.Context, volumes ...string) error {
 	for _, volume := range volumes {
-		if err := s.MakeVol(ctx, volume); err != nil && !errors.Is(err, errVolumeExists) {
+		err := s.MakeVol(ctx, volume)
+		if err != nil && !errors.Is(err, errVolumeExists) {
 			return err
 		}
+		diskHealthCheckOK(ctx, err)
 	}
 	return nil
 }
@@ -880,22 +883,15 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 		return err
 	}
 
-	var (
-		dataDir     string
-		lastVersion bool
-		updated     bool
-	)
-
 	for _, fi := range fis {
-		dataDir, lastVersion, err = xlMeta.DeleteVersion(fi)
+		dataDir, err := xlMeta.DeleteVersion(fi)
 		if err != nil {
 			if !fi.Deleted && (err == errFileNotFound || err == errFileVersionNotFound) {
-				// Ignore these since
+				// Ignore these since they do not exist
 				continue
 			}
 			return err
 		}
-		updated = true
 		if dataDir != "" {
 			versionID := fi.VersionID
 			if versionID == "" {
@@ -918,11 +914,8 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 		}
 	}
 
+	lastVersion := len(xlMeta.versions) == 0
 	if !lastVersion {
-		if !updated {
-			return nil
-		}
-
 		buf, err = xlMeta.AppendTo(metaDataPoolGet())
 		defer metaDataPoolPut(buf)
 		if err != nil {
@@ -933,12 +926,7 @@ func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis
 	}
 
 	// Move xl.meta to trash
-	filePath := pathJoin(volumeDir, path, xlStorageFormatFile)
-	if err = checkPathLength(filePath); err != nil {
-		return err
-	}
-
-	err = s.moveToTrash(filePath, false)
+	err = s.moveToTrash(pathJoin(volumeDir, path, xlStorageFormatFile), false)
 	if err == nil || err == errFileNotFound {
 		s.deleteFile(volumeDir, pathJoin(volumeDir, path), false)
 	}
@@ -958,6 +946,7 @@ func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions
 		if err := s.deleteVersions(ctx, volume, fiv.Name, fiv.Versions...); err != nil {
 			errs[i] = err
 		}
+		diskHealthCheckOK(ctx, errs[i])
 	}
 
 	return errs
@@ -1021,7 +1010,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		return err
 	}
 
-	dataDir, lastVersion, err := xlMeta.DeleteVersion(fi)
+	dataDir, err := xlMeta.DeleteVersion(fi)
 	if err != nil {
 		return err
 	}
@@ -1046,7 +1035,7 @@ func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi F
 		}
 	}
 
-	if !lastVersion {
+	if len(xlMeta.versions) != 0 {
 		buf, err = xlMeta.AppendTo(metaDataPoolGet())
 		defer metaDataPoolPut(buf)
 		if err != nil {
@@ -1397,7 +1386,7 @@ func (s *xlStorage) readAllData(ctx context.Context, volumeDir string, filePath
 		buf = make([]byte, sz)
 	}
 	// Read file...
-	_, err = io.ReadFull(r, buf)
+	_, err = io.ReadFull(diskHealthReader(ctx, r), buf)
 
 	return buf, stat.ModTime().UTC(), osErrToFileErr(err)
 }
@@ -1708,7 +1697,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 	r := struct {
 		io.Reader
 		io.Closer
-	}{Reader: io.LimitReader(or, length), Closer: closeWrapper(func() error {
+	}{Reader: io.LimitReader(diskHealthReader(ctx, or), length), Closer: closeWrapper(func() error {
 		if !alignment || offset+length%xioutil.DirectioAlignSize != 0 {
 			// invalidate page-cache for unaligned reads.
 			if !globalAPIConfig.isDisableODirect() {
@@ -1798,8 +1787,8 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 	}()
 
 	var bufp *[]byte
-	if fileSize > 0 && fileSize >= reallyLargeFileThreshold {
-		// use a larger 4MiB buffer for really large streams.
+	if fileSize > 0 && fileSize >= largestFileThreshold {
+		// use a larger 4MiB buffer for a really large streams.
 		bufp = xioutil.ODirectPoolXLarge.Get().(*[]byte)
 		defer xioutil.ODirectPoolXLarge.Put(bufp)
 	} else {
@@ -1807,7 +1796,7 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 		defer xioutil.ODirectPoolLarge.Put(bufp)
 	}
 
-	written, err := xioutil.CopyAligned(w, r, *bufp, fileSize)
+	written, err := xioutil.CopyAligned(diskHealthWriter(ctx, w), r, *bufp, fileSize, w)
 	if err != nil {
 		return err
 	}
@@ -2039,7 +2028,8 @@ func skipAccessChecks(volume string) (ok bool) {
 // RenameData - rename source path to destination path atomically, metadata and data directory.
 func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) (err error) {
 	defer func() {
-		if err != nil {
+		if err != nil && !contextCanceled(ctx) {
+			// Only log these errors if context is not yet canceled.
 			logger.LogIf(ctx, fmt.Errorf("srcVolume: %s, srcPath: %s, dstVolume: %s:, dstPath: %s - error %v",
 				srcVolume, srcPath,
 				dstVolume, dstPath,
@@ -2283,6 +2273,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 			}
 			return osErrToFileErr(err)
 		}
+		diskHealthCheckOK(ctx, err)
 
 		// renameAll only for objects that have xl.meta not saved inline.
 		if len(fi.Data) == 0 && fi.Size > 0 {
@@ -2420,7 +2411,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
 	return nil
 }
 
-func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
+func (s *xlStorage) bitrotVerify(ctx context.Context, partPath string, partSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
 	// Open the file for reading.
 	file, err := Open(partPath)
 	if err != nil {
@@ -2435,7 +2426,7 @@ func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlg
 		// for healing code to fix this file.
 		return err
 	}
-	return bitrotVerify(file, fi.Size(), partSize, algo, sum, shardSize)
+	return bitrotVerify(diskHealthReader(ctx, file), fi.Size(), partSize, algo, sum, shardSize)
 }
 
 func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
@@ -2460,7 +2451,7 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File
 	for _, part := range fi.Parts {
 		checksumInfo := erasure.GetChecksumInfo(part.Number)
 		partPath := pathJoin(volumeDir, path, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
-		if err := s.bitrotVerify(partPath,
+		if err := s.bitrotVerify(ctx, partPath,
 			erasure.ShardFileSize(part.Size),
 			checksumInfo.Algorithm,
 			checksumInfo.Hash, erasure.ShardSize()); err != nil {

cmd/xl-storage_test.go

@@ -1884,7 +1884,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
 	if err := storage.WriteAll(context.Background(), volName, fileName, data); err != nil {
 		t.Fatal(err)
 	}
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1894,12 +1894,12 @@ func TestXLStorageVerifyFile(t *testing.T) {
 	}
 
 	// Check if VerifyFile reports the incorrect file length (the correct length is `size+1`)
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 
 	// Check if bitrot fails
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size+1, algo, hashBytes, 0); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size+1, algo, hashBytes, 0); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 
@@ -1928,7 +1928,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	w.(io.Closer).Close()
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, nil, shardSize); err != nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, nil, shardSize); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1943,10 +1943,10 @@ func TestXLStorageVerifyFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	f.Close()
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, nil, shardSize); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, nil, shardSize); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size+1, algo, nil, shardSize); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size+1, algo, nil, shardSize); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 }

docs/debugging/README.md

@@ -30,18 +30,18 @@ mc admin trace --all --verbose myminio
 
 ## Subnet Health
 
-Subnet Health diagnostics help ensure that the underlying infrastructure that runs MinIO is configured correctly, and is functioning properly. This test is one-shot long running one, that is recommended to be run as soon as the cluster is first provisioned, and each time a failure scenario is encountered. Note that the test incurs majority of the available resources on the system. Care must be taken when using this to debug failure scenario, so as to prevent larger outages. Health tests can be triggered using `mc admin subnet health` command.
+Subnet Health diagnostics help ensure that the underlying infrastructure that runs MinIO is configured correctly, and is functioning properly. This test is one-shot long running one, that is recommended to be run as soon as the cluster is first provisioned, and each time a failure scenario is encountered. Note that the test incurs majority of the available resources on the system. Care must be taken when using this to debug failure scenario, so as to prevent larger outages. Health tests can be triggered using `mc support diagnostics` command.
 
 Example:
 
 ```sh
-minio server /data
+minio server /data{1...4}
 ```
 
 The command takes no flags
 
 ```sh
-mc admin subnet health myminio
+mc support diagnostics myminio/
 ```
 
 The output printed will be of the form
@@ -84,14 +84,14 @@ Executing `xl-meta` will look for an `xl.meta` in the current folder and decode
 
 ### Remotely Inspecting backend data
 
-`mc admin inspect` allows collecting files based on *path* from all backend drives. Matching files will be collected in a zip file with their respective host+drive+path. A MinIO host from October 2021 or later is required for full functionality. Syntax is `mc admin inspect ALIAS/path/to/files`. This can for example be used to collect `xl.meta` from objects that are misbehaving. To collect `xl.meta` from a specific object, for example placed at `ALIAS/bucket/path/to/file.txt` append `/xl.meta`, for instance `mc admin inspect ALIAS/bucket/path/to/file.txt/xl.meta`. All files can be collected, so this can also be used to retrieve `part.*` files, etc.
+`mc support inspect` allows collecting files based on *path* from all backend drives. Matching files will be collected in a zip file with their respective host+drive+path. A MinIO host from October 2021 or later is required for full functionality. Syntax is `mc support inspect ALIAS/path/to/files`. This can for example be used to collect `xl.meta` from objects that are misbehaving. To collect `xl.meta` from a specific object, for example placed at `ALIAS/bucket/path/to/file.txt` append `/xl.meta`, for instance `mc support inspect ALIAS/bucket/path/to/file.txt/xl.meta`. All files can be collected, so this can also be used to retrieve `part.*` files, etc.
 
-Wildcards can be used, for example `mc admin inspect ALIAS/bucket/path/**/xl.meta` will collect all `xl.meta` recursively. `mc admin inspect ALIAS/bucket/path/to/file.txt/*/part.*` will collect parts for all versions for the object located at `bucket/path/to/file.txt`.
+Wildcards can be used, for example `mc support inspect ALIAS/bucket/path/**/xl.meta` will collect all `xl.meta` recursively. `mc support inspect ALIAS/bucket/path/to/file.txt/*/part.*` will collect parts for all versions for the object located at `bucket/path/to/file.txt`.
 
 `xl-meta` accepts zip files as input and will output all `xl.meta` files found within the archive. For example:
 
 ```
-$ mc admin inspect play/test123/test*/xl.meta
+$ mc support inspect play/test123/test*/xl.meta
 mc: File data successfully downloaded as inspect.6f96b336.zip
 $ xl-meta inspect.6f96b336.zip
 {
@@ -105,7 +105,7 @@ $ xl-meta inspect.6f96b336.zip
 Optionally `--encrypt` can be specified. This will output an encrypted file and a decryption key:
 
 ```
-$ mc admin inspect --encrypt play/test123/test*/*/part.*
+$ mc support inspect --encrypt play/test123/test*/*/part.*
 mc: Encrypted file data successfully downloaded as inspect.ad2b43d8.enc
 mc: Decryption key: ad2b43d847fdb14e54c5836200177f7158b3f745433525f5d23c0e0208e50c9948540b54
 

docs/gateway/s3.md

@@ -70,7 +70,7 @@ Minimum permissions required if you wish to provide restricted access with your
                 "s3:GetObject",
                 "s3:ListBucket",
                 "s3:DeleteObject",
-                "s3:HeadBucket"
+                "s3:GetBucketAcl",
             ],
             "Resource": [
                 "arn:aws:s3:::mybucket",

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,14 +2,14 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-02-18T01-50-10Z
+  image: quay.io/minio/minio:RELEASE.2022-03-11T11-08-23Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"
     - "9001"
-  environment:
-    MINIO_ROOT_USER: minio
-    MINIO_ROOT_PASSWORD: minio123
+  # environment:
+    # MINIO_ROOT_USER: minioadmin
+    # MINIO_ROOT_PASSWORD: minioadmin
   healthcheck:
     test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
     interval: 30s

docs/orchestration/docker-compose/nginx.conf

@@ -49,6 +49,7 @@ http {
         client_max_body_size 0;
         # To disable buffering
         proxy_buffering off;
+        proxy_request_buffering off;
 
         location / {
             proxy_set_header Host $http_host;
@@ -78,6 +79,7 @@ http {
         client_max_body_size 0;
         # To disable buffering
         proxy_buffering off;
+        proxy_request_buffering off;
 
         location / {
             proxy_set_header Host $http_host;

docs/site-replication/run-multi-site-minio-idp.sh

@@ -173,6 +173,17 @@ if [ $? -ne 0 ]; then
     exit_1;
 fi
 
+err_minio2=$(./mc stat minio2/newbucket/xxx --json | jq -r .error.cause.message)
+if [ $? -ne 0 ]; then
+    echo "expecting object to be missing. exiting.."
+    exit_1;
+fi
+
+if [ "${err_minio2}" != "Object does not exist" ]; then
+    echo "expected to see Object does not exist error, exiting..."
+    exit_1;
+fi
+
 ./mc cp README.md minio2/newbucket/
 
 sleep 5

docs/sts/ldap.md

@@ -34,17 +34,17 @@ KEY:
 identity_ldap  enable LDAP SSO support
 
 ARGS:
-MINIO_IDENTITY_LDAP_SERVER_ADDR*            (address)   AD/LDAP server address e.g. "myldapserver.com:636"
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN          (string)    DN for LDAP read-only service account used to perform DN and group lookups
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD    (string)    Password for LDAP read-only service account used to perform DN and group lookups
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN  (string)    ";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER   (string)    Search filter to lookup user DN
-MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER     (string)    search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"
-MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN    (list)      ";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"
-MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY         (on|off)    trust server TLS without verification, defaults to "off" (verify)
-MINIO_IDENTITY_LDAP_SERVER_INSECURE         (on|off)    allow plain text connection to AD/LDAP server, defaults to "off"
-MINIO_IDENTITY_LDAP_SERVER_STARTTLS         (on|off)    use StartTLS connection to AD/LDAP server, defaults to "off"
-MINIO_IDENTITY_LDAP_COMMENT                 (sentence)  optionally add a comment to this setting
+MINIO_IDENTITY_LDAP_SERVER_ADDR*             (address)   AD/LDAP server address e.g. "myldapserver.com:636"
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN*          (string)    DN for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD     (string)    Password for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN*  (list)      ";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER*   (string)    Search filter to lookup user DN
+MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER      (string)    search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"
+MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN     (list)      ";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"
+MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY          (on|off)    trust server TLS without verification, defaults to "off" (verify)
+MINIO_IDENTITY_LDAP_SERVER_INSECURE          (on|off)    allow plain text connection to AD/LDAP server, defaults to "off"
+MINIO_IDENTITY_LDAP_SERVER_STARTTLS          (on|off)    use StartTLS connection to AD/LDAP server, defaults to "off"
+MINIO_IDENTITY_LDAP_COMMENT                  (sentence)  optionally add a comment to this setting
 ```
 
 ### LDAP server connectivity
@@ -69,8 +69,8 @@ If a self-signed certificate is being used, the certificate can be added to MinI
 A low-privilege read-only LDAP service account is configured in the MinIO server by providing the account's Distinguished Name (DN) and password. This service account is used to perform directory lookups as needed.
 
 ```
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN          (string)    DN for LDAP read-only service account used to perform DN and group lookups
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD    (string)    Password for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN*          (string)    DN for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD     (string)    Password for LDAP read-only service account used to perform DN and group lookups
 ```
 
 If you set an empty lookup bind password, the lookup bind will use the unauthenticated authentication mechanism, as described in [RFC 4513 Section 5.1.2](https://tools.ietf.org/html/rfc4513#section-5.1.2).
@@ -80,8 +80,8 @@ If you set an empty lookup bind password, the lookup bind will use the unauthent
 When a user provides their LDAP credentials, MinIO runs a lookup query to find the user's Distinguished Name (DN). The search filter and base DN used in this lookup query are configured via the following variables:
 
 ```
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN  (string)    Base LDAP DN to search for user DN
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER   (string)    Search filter to lookup user DN
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN*  (list)      ";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER*   (string)    Search filter to lookup user DN
 ```
 
 The search filter must use the LDAP username to find the user DN. This is done via [variable substitution](#variable-substitution-in-configuration-strings).

go.mod

@@ -6,7 +6,7 @@ require (
 	cloud.google.com/go/storage v1.10.0
 	github.com/Azure/azure-pipeline-go v0.2.2
 	github.com/Azure/azure-storage-blob-go v0.10.0
-	github.com/Shopify/sarama v1.27.2
+	github.com/Shopify/sarama v1.30.0
 	github.com/alecthomas/participle v0.2.1
 	github.com/bcicen/jstream v1.0.1
 	github.com/beevik/ntp v0.3.0
@@ -15,7 +15,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2
 	github.com/cheggaaa/pb v1.0.29
 	github.com/colinmarc/hdfs/v2 v2.2.0
-	github.com/coredns/coredns v1.4.0
+	github.com/coredns/coredns v1.9.0
 	github.com/coreos/go-oidc v2.1.0+incompatible
 	github.com/cosnicolaou/pbzip2 v1.0.1
 	github.com/dchest/siphash v1.2.1
@@ -27,9 +27,9 @@ require (
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.2
 	github.com/go-ldap/ldap/v3 v3.2.4
-	github.com/go-openapi/loads v0.21.0
+	github.com/go-openapi/loads v0.21.1
 	github.com/go-sql-driver/mysql v1.6.0
-	github.com/golang-jwt/jwt/v4 v4.1.0
+	github.com/golang-jwt/jwt/v4 v4.3.0
 	github.com/gomodule/redigo v1.8.8
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
@@ -37,76 +37,79 @@ require (
 	github.com/inconshreveable/mousetrap v1.0.0
 	github.com/jcmturner/gokrb5/v8 v8.4.2
 	github.com/json-iterator/go v1.1.12
-	github.com/klauspost/compress v1.13.6
-	github.com/klauspost/cpuid/v2 v2.0.9
+	github.com/klauspost/compress v1.14.4
+	github.com/klauspost/cpuid/v2 v2.0.11
 	github.com/klauspost/pgzip v1.2.5
 	github.com/klauspost/readahead v1.4.0
 	github.com/klauspost/reedsolomon v1.9.15
 	github.com/lib/pq v1.10.4
-	github.com/miekg/dns v1.1.43
+	github.com/miekg/dns v1.1.46
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.14.8
+	github.com/minio/console v0.15.2
 	github.com/minio/csvparser v1.0.0
-	github.com/minio/dperf v0.2.1-0.20220126071331-eefa6468e289
+	github.com/minio/dperf v0.3.4
 	github.com/minio/highwayhash v1.0.2
-	github.com/minio/kes v0.14.0
-	github.com/minio/madmin-go v1.3.4
+	github.com/minio/kes v0.18.0
+	github.com/minio/madmin-go v1.3.5
 	github.com/minio/minio-go/v7 v7.0.23
 	github.com/minio/parquet-go v1.1.0
-	github.com/minio/pkg v1.1.16
+	github.com/minio/pkg v1.1.20
 	github.com/minio/selfupdate v0.4.0
 	github.com/minio/sha256-simd v1.0.0
-	github.com/minio/simdjson-go v0.2.1
+	github.com/minio/simdjson-go v0.4.2
 	github.com/minio/sio v0.3.0
 	github.com/minio/zipindex v0.2.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/montanaflynn/stats v0.6.6
-	github.com/nats-io/nats-server/v2 v2.7.2
-	github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d
+	github.com/nats-io/nats-server/v2 v2.7.4
+	github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d
 	github.com/nats-io/stan.go v0.10.2
 	github.com/ncw/directio v1.0.5
 	github.com/nsqio/go-nsq v1.0.8
 	github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9
-	github.com/pierrec/lz4 v2.6.0+incompatible
+	github.com/pierrec/lz4 v2.6.1+incompatible
 	github.com/pkg/errors v0.9.1
-	github.com/prometheus/client_golang v1.11.0
+	github.com/prometheus/client_golang v1.12.1
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/procfs v0.7.3
 	github.com/rs/cors v1.7.0
 	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
 	github.com/secure-io/sio-go v0.3.1
-	github.com/shirou/gopsutil/v3 v3.21.12
+	github.com/shirou/gopsutil/v3 v3.22.2
 	github.com/streadway/amqp v1.0.0
 	github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e
 	github.com/valyala/bytebufferpool v1.0.0
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	github.com/yargevad/filepathx v1.0.0
 	github.com/zeebo/xxh3 v1.0.0
-	go.etcd.io/etcd/api/v3 v3.5.1
-	go.etcd.io/etcd/client/v3 v3.5.1
+	go.etcd.io/etcd/api/v3 v3.5.2
+	go.etcd.io/etcd/client/v3 v3.5.2
 	go.uber.org/atomic v1.9.0
-	go.uber.org/zap v1.19.1
-	golang.org/x/crypto v0.0.0-20220128200615-198e4374d7ed
-	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f
-	golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27
-	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
-	google.golang.org/api v0.58.0
+	go.uber.org/zap v1.21.0
+	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
+	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
+	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9
+	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
+	google.golang.org/api v0.67.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
-	cloud.google.com/go v0.94.1 // indirect
+	cloud.google.com/go v0.100.2 // indirect
+	cloud.google.com/go/compute v0.1.0 // indirect
+	cloud.google.com/go/iam v0.2.0 // indirect
 	github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c // indirect
 	github.com/PuerkitoBio/purell v1.1.1 // indirect
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/apache/thrift v0.15.0 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
+	github.com/armon/go-metrics v0.3.3 // indirect
+	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.2.0 // indirect
-	github.com/charmbracelet/bubbles v0.10.0 // indirect
-	github.com/charmbracelet/bubbletea v0.19.3 // indirect
-	github.com/charmbracelet/lipgloss v0.4.1-0.20220204041308-bf2912e703f6 // indirect
-	github.com/containerd/console v1.0.2 // indirect
+	github.com/charmbracelet/bubbles v0.10.3 // indirect
+	github.com/charmbracelet/bubbletea v0.20.0 // indirect
+	github.com/charmbracelet/lipgloss v0.5.0 // indirect
+	github.com/containerd/console v1.0.3 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -115,37 +118,34 @@ require (
 	github.com/eapache/go-resiliency v1.2.0 // indirect
 	github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect
 	github.com/eapache/queue v1.1.0 // indirect
-	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/frankban/quicktest v1.14.0 // indirect
+	github.com/gdamore/encoding v1.0.0 // indirect
+	github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect
-	github.com/go-logr/logr v1.2.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/analysis v0.21.2 // indirect
 	github.com/go-openapi/errors v0.20.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.6 // indirect
-	github.com/go-openapi/runtime v0.21.1 // indirect
+	github.com/go-openapi/runtime v0.23.1 // indirect
 	github.com/go-openapi/spec v0.20.4 // indirect
-	github.com/go-openapi/strfmt v0.21.1 // indirect
-	github.com/go-openapi/swag v0.19.15 // indirect
-	github.com/go-openapi/validate v0.20.3 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/goccy/go-json v0.8.1 // indirect
+	github.com/go-openapi/strfmt v0.21.2 // indirect
+	github.com/go-openapi/swag v0.21.1 // indirect
+	github.com/go-openapi/validate v0.21.0 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/goccy/go-json v0.9.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
-	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golang/snappy v0.0.3 // indirect
-	github.com/google/go-cmp v0.5.6 // indirect
-	github.com/google/gofuzz v1.1.0 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
-	github.com/googleapis/gnostic v0.5.1 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
@@ -153,92 +153,78 @@ require (
 	github.com/jcmturner/gofork v1.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
-	github.com/jessevdk/go-flags v1.4.0 // indirect
+	github.com/jessevdk/go-flags v1.5.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.0 // indirect
 	github.com/lestrrat-go/httpcc v1.0.0 // indirect
 	github.com/lestrrat-go/iter v1.0.1 // indirect
-	github.com/lestrrat-go/jwx v1.2.14 // indirect
+	github.com/lestrrat-go/jwx v1.2.19 // indirect
 	github.com/lestrrat-go/option v1.0.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
-	github.com/mailru/easyjson v0.7.6 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
-	github.com/mb0/glob v0.0.0-20160210091149-1eb79d2de6c4 // indirect
 	github.com/minio/argon2 v1.0.0 // indirect
 	github.com/minio/colorjson v1.0.1 // indirect
-	github.com/minio/direct-csi v1.3.5-0.20210601185811-f7776f7961bf // indirect
 	github.com/minio/filepath v1.0.0 // indirect
-	github.com/minio/mc v0.0.0-20220204044644-e048c85d71a7 // indirect
+	github.com/minio/mc v0.0.0-20220302011226-f13defa54577 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/operator v0.0.0-20220110040724-a5d59a342b7f // indirect
-	github.com/mitchellh/mapstructure v1.4.2 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect
+	github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
-	github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0 // indirect
+	github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739 // indirect
 	github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 // indirect
 	github.com/nats-io/nats-streaming-server v0.24.1 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/navidys/tvxwidgets v0.1.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/pkg/profile v1.6.0 // indirect
-	github.com/pkg/xattr v0.4.4 // indirect
+	github.com/pkg/xattr v0.4.5 // indirect
 	github.com/posener/complete v1.2.3 // indirect
-	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
 	github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/rivo/tview v0.0.0-20220216162559-96063d6082f3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rjeczalik/notify v0.9.2 // indirect
 	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/rs/xid v1.3.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/tidwall/gjson v1.12.1 // indirect
+	github.com/tidwall/gjson v1.14.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.3 // indirect
-	github.com/tklauser/go-sysconf v0.3.9 // indirect
-	github.com/tklauser/numcpus v0.3.0 // indirect
-	github.com/unrolled/secure v1.0.9 // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.4.0 // indirect
+	github.com/unrolled/secure v1.10.0 // indirect
 	github.com/xdg/stringprep v1.0.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	go.etcd.io/etcd/client/pkg/v3 v3.5.1 // indirect
-	go.mongodb.org/mongo-driver v1.7.5 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect
+	go.mongodb.org/mongo-driver v1.8.4 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	go.uber.org/multierr v1.7.0 // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	go.uber.org/multierr v1.8.0 // indirect
+	golang.org/x/mod v0.5.1 // indirect
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/tools v0.1.8 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20211223182754-3ac035c7e7cb // indirect
-	google.golang.org/grpc v1.43.0 // indirect
+	google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a // indirect
+	google.golang.org/grpc v1.44.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/h2non/filetype.v1 v1.0.5 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.66.3 // indirect
-	gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
-	gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
-	gopkg.in/jcmturner/gokrb5.v7 v7.5.0 // indirect
-	gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
-	gopkg.in/square/go-jose.v2 v2.3.1 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/api v0.21.1 // indirect
-	k8s.io/apimachinery v0.21.1 // indirect
-	k8s.io/client-go v0.21.1 // indirect
-	k8s.io/klog/v2 v2.30.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 // indirect
-	k8s.io/utils v0.0.0-20201110183641-67b214c5f920 // indirect
+	gopkg.in/ini.v1 v1.66.4 // indirect
+	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	maze.io/x/duration v0.0.0-20160924141736-faac084b6075 // indirect
-	sigs.k8s.io/controller-runtime v0.8.0 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.1.0 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
 )

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 3.5.7
-appVersion: RELEASE.2022-02-18T01-50-10Z
+version: 3.5.9
+appVersion: RELEASE.2022-03-03T21-21-16Z
 keywords:
   - minio
   - storage

helm/minio/templates/deployment.yaml

@@ -44,8 +44,10 @@ spec:
 {{ toYaml .Values.podLabels | indent 8 }}
 {{- end }}
       annotations:
+{{- if not .Values.ignoreChartChecksums }}
         checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+{{- end }}
 {{- if .Values.podAnnotations }}
 {{ toYaml .Values.podAnnotations | trimSuffix "\n" | indent 8 }}
 {{- end }}
@@ -58,6 +60,9 @@ spec:
         runAsUser: {{ .Values.securityContext.runAsUser }}
         runAsGroup: {{ .Values.securityContext.runAsGroup }}
         fsGroup: {{ .Values.securityContext.fsGroup }}
+        {{- if and (ge .Capabilities.KubeVersion.Major "1") (ge .Capabilities.KubeVersion.Minor 20) }}
+        fsGroupChangePolicy: {{ .Values.securityContext.fsGroupChangePolicy }}
+        {{- end }}
 {{- end }}
 {{ if .Values.serviceAccount.create }}
       serviceAccountName: {{ .Values.serviceAccount.name }}

helm/minio/templates/gateway-deployment.yaml

@@ -44,8 +44,10 @@ spec:
 {{ toYaml .Values.podLabels | indent 8 }}
 {{- end }}
       annotations:
+{{- if not .Values.ignoreChartChecksums }}
         checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+{{- end }}
 {{- if .Values.podAnnotations }}
 {{ toYaml .Values.podAnnotations | trimSuffix "\n" | indent 8 }}
 {{- end }}

helm/minio/templates/statefulset.yaml

@@ -38,6 +38,7 @@ apiVersion: {{ template "minio.statefulset.apiVersion" . }}
 kind: StatefulSet
 metadata:
   name: {{ template "minio.fullname" . }}
+  namespace: {{ .Release.Namespace | quote }}
   labels:
     app: {{ template "minio.name" . }}
     chart: {{ template "minio.chart" . }}
@@ -70,8 +71,10 @@ spec:
 {{ toYaml .Values.podLabels | indent 8 }}
 {{- end }}
       annotations:
+{{- if not .Values.ignoreChartChecksums }}
         checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }}
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+{{- end }}
 {{- if .Values.podAnnotations }}
 {{ toYaml .Values.podAnnotations | trimSuffix "\n" | indent 8 }}
 {{- end }}
@@ -84,6 +87,9 @@ spec:
         runAsUser: {{ .Values.securityContext.runAsUser }}
         runAsGroup: {{ .Values.securityContext.runAsGroup }}
         fsGroup: {{ .Values.securityContext.fsGroup }}
+        {{- if and (ge .Capabilities.KubeVersion.Major "1") (ge .Capabilities.KubeVersion.Minor 20) }}
+        fsGroupChangePolicy: {{ .Values.securityContext.fsGroupChangePolicy }}
+        {{- end }}
 {{- end }}
 {{ if .Values.serviceAccount.create }}
       serviceAccountName: {{ .Values.serviceAccount.name }}

helm/minio/values.yaml

@@ -37,6 +37,12 @@ additionalLabels: []
 ## Additional annotations to include with deployment or statefulset
 additionalAnnotations: []
 
+## Typically the deployment/statefulset includes checksums of secrets/config,
+## So that when these change on a subsequent helm install, the deployment/statefulset 
+## is restarted. This can result in unnecessary restarts under GitOps tooling such as 
+## flux, so set to "true" to disable this behaviour.
+ignoreChartChecksums: false
+
 ## Additional arguments to pass to minio binary
 extraArgs: []
 
@@ -231,6 +237,7 @@ securityContext:
   runAsUser: 1000
   runAsGroup: 1000
   fsGroup: 1000
+  fsGroupChangePolicy: "OnRootMismatch"
 
 # Additational pod annotations
 podAnnotations: {}

index.yaml

@@ -1,9 +1,53 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2022-03-03T21-21-16Z
+    created: "2022-03-03T15:28:46.71872892-08:00"
+    description: Multi-Cloud Object Storage
+    digest: 6fda968d3fdfd60470c0055a4e1a3bd8e5aee9ad0af5ba2fb7b7b926fdc9e4a0
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.5.9.tgz
+    version: 3.5.9
+  - apiVersion: v1
+    appVersion: RELEASE.2022-02-26T02-54-46Z
+    created: "2022-03-03T15:28:46.717568339-08:00"
+    description: Multi-Cloud Object Storage
+    digest: 8e015369048a3a82bbd53ad36696786f18561c6b25d14eee9e2c93a7336cef46
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.5.8.tgz
+    version: 3.5.8
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2022-02-20T00:54:28.640234004-08:00"
+    created: "2022-03-03T15:28:46.716258212-08:00"
     description: Multi-Cloud Object Storage
     digest: cb3543fe748e5f0d59b3ccf4ab9af8e10b731405ae445d1f5715e30013632373
     home: https://min.io
@@ -25,7 +69,7 @@ entries:
     version: 3.5.7
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2022-02-20T00:54:28.638878844-08:00"
+    created: "2022-03-03T15:28:46.714897775-08:00"
     description: Multi-Cloud Object Storage
     digest: f2e359fa5eefffc59abb3d14a8fa94b11ddeaa99f6cd8dd5f40f4e04121000d6
     home: https://min.io
@@ -47,7 +91,7 @@ entries:
     version: 3.5.6
   - apiVersion: v1
     appVersion: RELEASE.2022-02-16T00-35-27Z
-    created: "2022-02-20T00:54:28.637144796-08:00"
+    created: "2022-03-03T15:28:46.713557317-08:00"
     description: Multi-Cloud Object Storage
     digest: 529d56cca9d83a3d0e5672e63b6e87b5bcbe10a6b45f7a55ba998cceb32f9c81
     home: https://min.io
@@ -69,7 +113,7 @@ entries:
     version: 3.5.5
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2022-02-20T00:54:28.635349503-08:00"
+    created: "2022-03-03T15:28:46.711659592-08:00"
     description: Multi-Cloud Object Storage
     digest: 3d530598f8ece67bec5b7f990d206584893987c713502f9228e4ee24b5535414
     home: https://min.io
@@ -91,7 +135,7 @@ entries:
     version: 3.5.4
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2022-02-20T00:54:28.634018167-08:00"
+    created: "2022-03-03T15:28:46.710394206-08:00"
     description: Multi-Cloud Object Storage
     digest: 53937031348b29615f07fc4869b2d668391d8ba9084630a497abd7a7dea9dfb0
     home: https://min.io
@@ -113,7 +157,7 @@ entries:
     version: 3.5.3
   - apiVersion: v1
     appVersion: RELEASE.2022-02-07T08-17-33Z
-    created: "2022-02-20T00:54:28.632869049-08:00"
+    created: "2022-03-03T15:28:46.70926804-08:00"
     description: Multi-Cloud Object Storage
     digest: 68d643414ff0d565716c5715034fcbf1af262e041915a5c02eb51ec1a65c1ea0
     home: https://min.io
@@ -135,7 +179,7 @@ entries:
     version: 3.5.2
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2022-02-20T00:54:28.631933558-08:00"
+    created: "2022-03-03T15:28:46.708131603-08:00"
     description: Multi-Cloud Object Storage
     digest: a3e855ed0f31233b989fffd775a29d6fbfa0590089010ff16783fd7f142ef6e7
     home: https://min.io
@@ -157,7 +201,7 @@ entries:
     version: 3.5.1
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2022-02-20T00:54:28.630945491-08:00"
+    created: "2022-03-03T15:28:46.70701956-08:00"
     description: Multi-Cloud Object Storage
     digest: b1b0ae3c54b4260a698753e11d7781bb8ddc67b7e3fbf0af82796e4cd4ef92a3
     home: https://min.io
@@ -179,7 +223,7 @@ entries:
     version: 3.5.0
   - apiVersion: v1
     appVersion: RELEASE.2022-01-28T02-28-16Z
-    created: "2022-02-20T00:54:28.629592126-08:00"
+    created: "2022-03-03T15:28:46.705883001-08:00"
     description: Multi-Cloud Object Storage
     digest: fecf25d2d3fb208c6f894fed642a60780a570b7f6d0adddde846af7236dc80aa
     home: https://min.io
@@ -201,7 +245,7 @@ entries:
     version: 3.4.8
   - apiVersion: v1
     appVersion: RELEASE.2022-01-25T19-56-04Z
-    created: "2022-02-20T00:54:28.628016327-08:00"
+    created: "2022-03-03T15:28:46.704205655-08:00"
     description: Multi-Cloud Object Storage
     digest: c78008caa5ce98f64c887630f59d0cbd481cb3f19a7d4e9d3e81bf4e1e45cadc
     home: https://min.io
@@ -223,7 +267,7 @@ entries:
     version: 3.4.7
   - apiVersion: v1
     appVersion: RELEASE.2022-01-08T03-11-54Z
-    created: "2022-02-20T00:54:28.626913064-08:00"
+    created: "2022-03-03T15:28:46.702952316-08:00"
     description: Multi-Cloud Object Storage
     digest: 8f2e2691bf897f74ff094dd370ec56ba9d417e5e8926710c14c2ba346330238d
     home: https://min.io
@@ -245,7 +289,7 @@ entries:
     version: 3.4.6
   - apiVersion: v1
     appVersion: RELEASE.2022-01-04T07-41-07Z
-    created: "2022-02-20T00:54:28.62588412-08:00"
+    created: "2022-03-03T15:28:46.701754571-08:00"
     description: Multi-Cloud Object Storage
     digest: bacd140f0016fab35f516bde787da6449b3a960c071fad9e4b6563118033ac84
     home: https://min.io
@@ -267,7 +311,7 @@ entries:
     version: 3.4.5
   - apiVersion: v1
     appVersion: RELEASE.2021-12-29T06-49-06Z
-    created: "2022-02-20T00:54:28.624861936-08:00"
+    created: "2022-03-03T15:28:46.700632058-08:00"
     description: Multi-Cloud Object Storage
     digest: 48a453ea5ffeef25933904caefd9470bfb26224dfc2d1096bd0031467ba53007
     home: https://min.io
@@ -289,7 +333,7 @@ entries:
     version: 3.4.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-02-20T00:54:28.623814605-08:00"
+    created: "2022-03-03T15:28:46.699429459-08:00"
     description: Multi-Cloud Object Storage
     digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60
     home: https://min.io
@@ -311,7 +355,7 @@ entries:
     version: 3.4.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-02-20T00:54:28.622541682-08:00"
+    created: "2022-03-03T15:28:46.698226992-08:00"
     description: Multi-Cloud Object Storage
     digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff
     home: https://min.io
@@ -333,7 +377,7 @@ entries:
     version: 3.4.2
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-02-20T00:54:28.621089075-08:00"
+    created: "2022-03-03T15:28:46.696842536-08:00"
     description: Multi-Cloud Object Storage
     digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37
     home: https://min.io
@@ -355,7 +399,7 @@ entries:
     version: 3.4.1
   - apiVersion: v1
     appVersion: RELEASE.2021-12-18T04-42-33Z
-    created: "2022-02-20T00:54:28.620157854-08:00"
+    created: "2022-03-03T15:28:46.695365796-08:00"
     description: Multi-Cloud Object Storage
     digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453
     home: https://min.io
@@ -377,7 +421,7 @@ entries:
     version: 3.4.0
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-02-20T00:54:28.619133895-08:00"
+    created: "2022-03-03T15:28:46.694266323-08:00"
     description: Multi-Cloud Object Storage
     digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894
     home: https://min.io
@@ -399,7 +443,7 @@ entries:
     version: 3.3.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-02-20T00:54:28.618198183-08:00"
+    created: "2022-03-03T15:28:46.693191851-08:00"
     description: Multi-Cloud Object Storage
     digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08
     home: https://min.io
@@ -421,7 +465,7 @@ entries:
     version: 3.3.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-02-20T00:54:28.617135773-08:00"
+    created: "2022-03-03T15:28:46.692145802-08:00"
     description: Multi-Cloud Object Storage
     digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb
     home: https://min.io
@@ -443,7 +487,7 @@ entries:
     version: 3.3.2
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-02-20T00:54:28.615904828-08:00"
+    created: "2022-03-03T15:28:46.690906843-08:00"
     description: Multi-Cloud Object Storage
     digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4
     home: https://min.io
@@ -465,7 +509,7 @@ entries:
     version: 3.3.1
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-02-20T00:54:28.612038975-08:00"
+    created: "2022-03-03T15:28:46.689306119-08:00"
     description: Multi-Cloud Object Storage
     digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4
     home: https://min.io
@@ -487,7 +531,7 @@ entries:
     version: 3.3.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-13T00-23-17Z
-    created: "2022-02-20T00:54:28.611164475-08:00"
+    created: "2022-03-03T15:28:46.688201137-08:00"
     description: Multi-Cloud Object Storage
     digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054
     home: https://min.io
@@ -509,7 +553,7 @@ entries:
     version: 3.2.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-10T16-53-30Z
-    created: "2022-02-20T00:54:28.610186101-08:00"
+    created: "2022-03-03T15:28:46.686861384-08:00"
     description: Multi-Cloud Object Storage
     digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8
     home: https://min.io
@@ -531,7 +575,7 @@ entries:
     version: 3.1.9
   - apiVersion: v1
     appVersion: RELEASE.2021-10-06T23-36-31Z
-    created: "2022-02-20T00:54:28.60906994-08:00"
+    created: "2022-03-03T15:28:46.685072158-08:00"
     description: Multi-Cloud Object Storage
     digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159
     home: https://min.io
@@ -553,7 +597,7 @@ entries:
     version: 3.1.8
   - apiVersion: v1
     appVersion: RELEASE.2021-10-02T16-31-05Z
-    created: "2022-02-20T00:54:28.607903236-08:00"
+    created: "2022-03-03T15:28:46.683398819-08:00"
     description: Multi-Cloud Object Storage
     digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11
     home: https://min.io
@@ -575,7 +619,7 @@ entries:
     version: 3.1.7
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-02-20T00:54:28.606802509-08:00"
+    created: "2022-03-03T15:28:46.681758267-08:00"
     description: Multi-Cloud Object Storage
     digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310
     home: https://min.io
@@ -597,7 +641,7 @@ entries:
     version: 3.1.6
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-02-20T00:54:28.604984477-08:00"
+    created: "2022-03-03T15:28:46.680475021-08:00"
     description: Multi-Cloud Object Storage
     digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a
     home: https://min.io
@@ -619,7 +663,7 @@ entries:
     version: 3.1.5
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-02-20T00:54:28.603900379-08:00"
+    created: "2022-03-03T15:28:46.678759552-08:00"
     description: Multi-Cloud Object Storage
     digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0
     home: https://min.io
@@ -641,7 +685,7 @@ entries:
     version: 3.1.4
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-02-20T00:54:28.602881772-08:00"
+    created: "2022-03-03T15:28:46.677658732-08:00"
     description: Multi-Cloud Object Storage
     digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb
     home: https://min.io
@@ -663,7 +707,7 @@ entries:
     version: 3.1.3
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-02-20T00:54:28.601813678-08:00"
+    created: "2022-03-03T15:28:46.676660879-08:00"
     description: Multi-Cloud Object Storage
     digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd
     home: https://min.io
@@ -685,7 +729,7 @@ entries:
     version: 3.1.2
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-02-20T00:54:28.600623339-08:00"
+    created: "2022-03-03T15:28:46.675832123-08:00"
     description: Multi-Cloud Object Storage
     digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
     home: https://min.io
@@ -707,7 +751,7 @@ entries:
     version: 3.1.1
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-02-20T00:54:28.599490987-08:00"
+    created: "2022-03-03T15:28:46.674997142-08:00"
     description: Multi-Cloud Object Storage
     digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
     home: https://min.io
@@ -729,7 +773,7 @@ entries:
     version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2022-02-20T00:54:28.598263205-08:00"
+    created: "2022-03-03T15:28:46.673959757-08:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -751,7 +795,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-02-20T00:54:28.596502271-08:00"
+    created: "2022-03-03T15:28:46.672774395-08:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -773,7 +817,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-02-20T00:54:28.595483746-08:00"
+    created: "2022-03-03T15:28:46.671624084-08:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -795,7 +839,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-02-20T00:54:28.594540025-08:00"
+    created: "2022-03-03T15:28:46.670794348-08:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -815,7 +859,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-02-20T00:54:28.593484039-08:00"
+    created: "2022-03-03T15:28:46.669829694-08:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -835,7 +879,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-02-20T00:54:28.592493968-08:00"
+    created: "2022-03-03T15:28:46.668829946-08:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -855,7 +899,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-02-20T00:54:28.589744382-08:00"
+    created: "2022-03-03T15:28:46.667869747-08:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -875,7 +919,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-02-20T00:54:28.58862602-08:00"
+    created: "2022-03-03T15:28:46.666658153-08:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -895,7 +939,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-02-20T00:54:28.575940182-08:00"
+    created: "2022-03-03T15:28:46.662395153-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -915,7 +959,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-02-20T00:54:28.574968066-08:00"
+    created: "2022-03-03T15:28:46.661418704-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -935,7 +979,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2022-02-20T00:54:28.573939461-08:00"
+    created: "2022-03-03T15:28:46.660257537-08:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -953,4 +997,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2022-02-20T00:54:28.572717004-08:00"
+generated: "2022-03-03T15:28:46.658942785-08:00"

internal/config/cache/config.go

@@ -115,7 +115,6 @@ func parseCacheDrives(drives string) ([]string, error) {
 			endpoints = append(endpoints, d)
 		}
 	}
-
 	for _, d := range endpoints {
 		if !filepath.IsAbs(d) {
 			return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)

internal/config/identity/ldap/help.go

@@ -31,7 +31,6 @@ var (
 		config.HelpKV{
 			Key:         LookupBindDN,
 			Description: `DN for LDAP read-only service account used to perform DN and group lookups`,
-			Optional:    true,
 			Type:        "string",
 			Sensitive:   true,
 		},
@@ -45,13 +44,11 @@ var (
 		config.HelpKV{
 			Key:         UserDNSearchBaseDN,
 			Description: `";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"`,
-			Optional:    true,
 			Type:        "list",
 		},
 		config.HelpKV{
 			Key:         UserDNSearchFilter,
 			Description: `Search filter to lookup user DN`,
-			Optional:    true,
 			Type:        "string",
 		},
 		config.HelpKV{

internal/config/notify/parse.go

@@ -138,6 +138,12 @@ func fetchSubSysTargets(ctx context.Context, cfg config.Config,
 				}
 				_ = newTarget.Close()
 			}
+			if err = targetList.Add(newTarget); err != nil {
+				logger.LogIf(context.Background(), err)
+				if returnOnTargetError {
+					return targetsOffline, err
+				}
+			}
 		}
 	case config.NotifyESSubSys:
 		esTargets, err := GetNotifyES(cfg[config.NotifyESSubSys], transport)

internal/dsync/drwmutex.go

@@ -43,32 +43,61 @@ func log(format string, data ...interface{}) {
 	}
 }
 
-// dRWMutexAcquireTimeout - tolerance limit to wait for lock acquisition before.
-const drwMutexAcquireTimeout = 1 * time.Second // 1 second.
+const (
+	// dRWMutexAcquireTimeout - default tolerance limit to wait for lock acquisition before.
+	drwMutexAcquireTimeout = 1 * time.Second // 1 second.
 
-// dRWMutexRefreshTimeout - timeout for the refresh call
-const drwMutexRefreshCallTimeout = 5 * time.Second
+	// dRWMutexRefreshTimeout - default timeout for the refresh call
+	drwMutexRefreshCallTimeout = 5 * time.Second
 
-// dRWMutexUnlockTimeout - timeout for the unlock call
-const drwMutexUnlockCallTimeout = 30 * time.Second
+	// dRWMutexUnlockTimeout - default timeout for the unlock call
+	drwMutexUnlockCallTimeout = 30 * time.Second
 
-// dRWMutexForceUnlockTimeout - timeout for the unlock call
-const drwMutexForceUnlockCallTimeout = 30 * time.Second
+	// dRWMutexForceUnlockTimeout - default timeout for the unlock call
+	drwMutexForceUnlockCallTimeout = 30 * time.Second
 
-// dRWMutexRefreshInterval - the interval between two refresh calls
-const drwMutexRefreshInterval = 10 * time.Second
+	// dRWMutexRefreshInterval - default the interval between two refresh calls
+	drwMutexRefreshInterval = 10 * time.Second
 
-const drwMutexInfinite = 1<<63 - 1
+	lockRetryInterval = 1 * time.Second
+
+	drwMutexInfinite = 1<<63 - 1
+)
+
+// Timeouts are timeouts for specific operations.
+type Timeouts struct {
+	// Acquire - tolerance limit to wait for lock acquisition before.
+	Acquire time.Duration
+
+	// RefreshCall - timeout for the refresh call
+	RefreshCall time.Duration
+
+	// UnlockCall - timeout for the unlock call
+	UnlockCall time.Duration
+
+	// ForceUnlockCall - timeout for the force unlock call
+	ForceUnlockCall time.Duration
+}
+
+// DefaultTimeouts contains default timeouts.
+var DefaultTimeouts = Timeouts{
+	Acquire:         drwMutexAcquireTimeout,
+	RefreshCall:     drwMutexUnlockCallTimeout,
+	UnlockCall:      drwMutexRefreshCallTimeout,
+	ForceUnlockCall: drwMutexForceUnlockCallTimeout,
+}
 
 // A DRWMutex is a distributed mutual exclusion lock.
 type DRWMutex struct {
-	Names         []string
-	writeLocks    []string // Array of nodes that granted a write lock
-	readLocks     []string // Array of array of nodes that granted reader locks
-	rng           *rand.Rand
-	m             sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
-	clnt          *Dsync
-	cancelRefresh context.CancelFunc
+	Names             []string
+	writeLocks        []string // Array of nodes that granted a write lock
+	readLocks         []string // Array of array of nodes that granted reader locks
+	rng               *rand.Rand
+	m                 sync.Mutex // Mutex to prevent multiple simultaneous locks from this node
+	clnt              *Dsync
+	cancelRefresh     context.CancelFunc
+	refreshInterval   time.Duration
+	lockRetryInterval time.Duration
 }
 
 // Granted - represents a structure of a granted lock.
@@ -90,11 +119,13 @@ func NewDRWMutex(clnt *Dsync, names ...string) *DRWMutex {
 	restClnts, _ := clnt.GetLockers()
 	sort.Strings(names)
 	return &DRWMutex{
-		writeLocks: make([]string, len(restClnts)),
-		readLocks:  make([]string, len(restClnts)),
-		Names:      names,
-		clnt:       clnt,
-		rng:        rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())}),
+		writeLocks:        make([]string, len(restClnts)),
+		readLocks:         make([]string, len(restClnts)),
+		Names:             names,
+		clnt:              clnt,
+		rng:               rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())}),
+		refreshInterval:   drwMutexRefreshInterval,
+		lockRetryInterval: lockRetryInterval,
 	}
 }
 
@@ -146,10 +177,6 @@ func (dm *DRWMutex) GetRLock(ctx context.Context, cancel context.CancelFunc, id,
 	return dm.lockBlocking(ctx, cancel, id, source, isReadLock, opts)
 }
 
-const (
-	lockRetryInterval = 1 * time.Second
-)
-
 // lockBlocking will try to acquire either a read or a write lock
 //
 // The function will loop using a built-in timing randomized back-off
@@ -209,7 +236,7 @@ func (dm *DRWMutex) lockBlocking(ctx context.Context, lockLossCallback func(), i
 				return locked
 			}
 
-			time.Sleep(time.Duration(dm.rng.Float64() * float64(lockRetryInterval)))
+			time.Sleep(time.Duration(dm.rng.Float64() * float64(dm.lockRetryInterval)))
 		}
 	}
 }
@@ -224,7 +251,7 @@ func (dm *DRWMutex) startContinousLockRefresh(lockLossCallback func(), id, sourc
 	go func() {
 		defer cancel()
 
-		refreshTimer := time.NewTimer(drwMutexRefreshInterval)
+		refreshTimer := time.NewTimer(dm.refreshInterval)
 		defer refreshTimer.Stop()
 
 		for {
@@ -232,7 +259,7 @@ func (dm *DRWMutex) startContinousLockRefresh(lockLossCallback func(), id, sourc
 			case <-ctx.Done():
 				return
 			case <-refreshTimer.C:
-				refreshTimer.Reset(drwMutexRefreshInterval)
+				refreshTimer.Reset(dm.refreshInterval)
 
 				noQuorum, err := refreshLock(ctx, dm.clnt, id, source, quorum)
 				if err == nil && noQuorum {
@@ -250,7 +277,7 @@ func (dm *DRWMutex) startContinousLockRefresh(lockLossCallback func(), id, sourc
 }
 
 func forceUnlock(ctx context.Context, ds *Dsync, id string) {
-	ctx, cancel := context.WithTimeout(ctx, drwMutexForceUnlockCallTimeout)
+	ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.ForceUnlockCall)
 	defer cancel()
 
 	restClnts, _ := ds.GetLockers()
@@ -300,7 +327,7 @@ func refreshLock(ctx context.Context, ds *Dsync, id, source string, quorum int)
 				return
 			}
 
-			ctx, cancel := context.WithTimeout(ctx, drwMutexRefreshCallTimeout)
+			ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.RefreshCall)
 			defer cancel()
 
 			refreshed, err := c.Refresh(ctx, args)
@@ -379,7 +406,7 @@ func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, is
 	}
 
 	// Combined timeout for the lock attempt.
-	ctx, cancel := context.WithTimeout(ctx, drwMutexAcquireTimeout)
+	ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.Acquire)
 	defer cancel()
 	for index, c := range restClnts {
 		wg.Add(1)
@@ -573,7 +600,7 @@ func (dm *DRWMutex) Unlock() {
 
 	isReadLock := false
 	for !releaseAll(dm.clnt, tolerance, owner, &locks, isReadLock, restClnts, dm.Names...) {
-		time.Sleep(time.Duration(dm.rng.Float64() * float64(lockRetryInterval)))
+		time.Sleep(time.Duration(dm.rng.Float64() * float64(dm.lockRetryInterval)))
 	}
 }
 
@@ -614,7 +641,7 @@ func (dm *DRWMutex) RUnlock() {
 
 	isReadLock := true
 	for !releaseAll(dm.clnt, tolerance, owner, &locks, isReadLock, restClnts, dm.Names...) {
-		time.Sleep(time.Duration(dm.rng.Float64() * float64(lockRetryInterval)))
+		time.Sleep(time.Duration(dm.rng.Float64() * float64(dm.lockRetryInterval)))
 	}
 }
 
@@ -635,7 +662,7 @@ func sendRelease(ds *Dsync, c NetLocker, owner string, uid string, isReadLock bo
 		Resources: names,
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), drwMutexUnlockCallTimeout)
+	ctx, cancel := context.WithTimeout(context.Background(), ds.Timeouts.UnlockCall)
 	defer cancel()
 
 	if isReadLock {

internal/dsync/drwmutex_test.go

@@ -47,13 +47,13 @@ func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	// fmt.Println("2nd read lock acquired, waiting...")
 
 	go func() {
-		time.Sleep(2 * time.Second)
+		time.Sleep(2 * testDrwMutexAcquireTimeout)
 		drwm.RUnlock()
 		// fmt.Println("1st read lock released, waiting...")
 	}()
 
 	go func() {
-		time.Sleep(3 * time.Second)
+		time.Sleep(3 * testDrwMutexAcquireTimeout)
 		drwm.RUnlock()
 		// fmt.Println("2nd read lock released, waiting...")
 	}()
@@ -63,7 +63,7 @@ func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	locked = drwm.GetLock(ctx3, cancel3, id, source, Options{Timeout: duration})
 	if locked {
 		// fmt.Println("Write lock acquired, waiting...")
-		time.Sleep(time.Second)
+		time.Sleep(testDrwMutexAcquireTimeout)
 
 		drwm.Unlock()
 	}
@@ -72,7 +72,7 @@ func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 }
 
 func TestSimpleWriteLockAcquired(t *testing.T) {
-	locked := testSimpleWriteLock(t, 5*time.Second)
+	locked := testSimpleWriteLock(t, 10*testDrwMutexAcquireTimeout)
 
 	expected := true
 	if locked != expected {
@@ -81,7 +81,7 @@ func TestSimpleWriteLockAcquired(t *testing.T) {
 }
 
 func TestSimpleWriteLockTimedOut(t *testing.T) {
-	locked := testSimpleWriteLock(t, time.Second)
+	locked := testSimpleWriteLock(t, testDrwMutexAcquireTimeout)
 
 	expected := false
 	if locked != expected {
@@ -99,7 +99,7 @@ func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	}
 
 	go func() {
-		time.Sleep(2 * time.Second)
+		time.Sleep(3 * testDrwMutexAcquireTimeout)
 		drwm.Unlock()
 		// fmt.Println("Initial write lock released, waiting...")
 	}()
@@ -109,7 +109,7 @@ func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 	locked = drwm.GetLock(ctx2, cancel2, id, source, Options{Timeout: duration})
 	if locked {
 		// fmt.Println("2nd write lock acquired, waiting...")
-		time.Sleep(time.Second)
+		time.Sleep(testDrwMutexAcquireTimeout)
 
 		drwm.Unlock()
 	}
@@ -118,7 +118,7 @@ func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) {
 }
 
 func TestDualWriteLockAcquired(t *testing.T) {
-	locked := testDualWriteLock(t, 5*time.Second)
+	locked := testDualWriteLock(t, 10*testDrwMutexAcquireTimeout)
 
 	expected := true
 	if locked != expected {
@@ -127,7 +127,7 @@ func TestDualWriteLockAcquired(t *testing.T) {
 }
 
 func TestDualWriteLockTimedOut(t *testing.T) {
-	locked := testDualWriteLock(t, time.Second)
+	locked := testDualWriteLock(t, testDrwMutexAcquireTimeout)
 
 	expected := false
 	if locked != expected {
@@ -214,25 +214,27 @@ func writer(rwm *DRWMutex, numIterations int, activity *int32, cdone chan bool)
 }
 
 // Borrowed from rwmutex_test.go
-func HammerRWMutex(gomaxprocs, numReaders, numIterations int) {
-	runtime.GOMAXPROCS(gomaxprocs)
-	// Number of active readers + 10000 * number of active writers.
-	var activity int32
-	rwm := NewDRWMutex(ds, "test")
-	cdone := make(chan bool)
-	go writer(rwm, numIterations, &activity, cdone)
-	var i int
-	for i = 0; i < numReaders/2; i++ {
-		go reader(rwm, numIterations, &activity, cdone)
-	}
-	go writer(rwm, numIterations, &activity, cdone)
-	for ; i < numReaders; i++ {
-		go reader(rwm, numIterations, &activity, cdone)
-	}
-	// Wait for the 2 writers and all readers to finish.
-	for i := 0; i < 2+numReaders; i++ {
-		<-cdone
-	}
+func hammerRWMutex(t *testing.T, gomaxprocs, numReaders, numIterations int) {
+	t.Run(fmt.Sprintf("%d-%d-%d", gomaxprocs, numReaders, numIterations), func(t *testing.T) {
+		runtime.GOMAXPROCS(gomaxprocs)
+		// Number of active readers + 10000 * number of active writers.
+		var activity int32
+		rwm := NewDRWMutex(ds, "test")
+		cdone := make(chan bool)
+		go writer(rwm, numIterations, &activity, cdone)
+		var i int
+		for i = 0; i < numReaders/2; i++ {
+			go reader(rwm, numIterations, &activity, cdone)
+		}
+		go writer(rwm, numIterations, &activity, cdone)
+		for ; i < numReaders; i++ {
+			go reader(rwm, numIterations, &activity, cdone)
+		}
+		// Wait for the 2 writers and all readers to finish.
+		for i := 0; i < 2+numReaders; i++ {
+			<-cdone
+		}
+	})
 }
 
 // Borrowed from rwmutex_test.go
@@ -242,16 +244,16 @@ func TestRWMutex(t *testing.T) {
 	if testing.Short() {
 		n = 5
 	}
-	HammerRWMutex(1, 1, n)
-	HammerRWMutex(1, 3, n)
-	HammerRWMutex(1, 10, n)
-	HammerRWMutex(4, 1, n)
-	HammerRWMutex(4, 3, n)
-	HammerRWMutex(4, 10, n)
-	HammerRWMutex(10, 1, n)
-	HammerRWMutex(10, 3, n)
-	HammerRWMutex(10, 10, n)
-	HammerRWMutex(10, 5, n)
+	hammerRWMutex(t, 1, 1, n)
+	hammerRWMutex(t, 1, 3, n)
+	hammerRWMutex(t, 1, 10, n)
+	hammerRWMutex(t, 4, 1, n)
+	hammerRWMutex(t, 4, 3, n)
+	hammerRWMutex(t, 4, 10, n)
+	hammerRWMutex(t, 10, 1, n)
+	hammerRWMutex(t, 10, 3, n)
+	hammerRWMutex(t, 10, 10, n)
+	hammerRWMutex(t, 10, 5, n)
 }
 
 // Borrowed from rwmutex_test.go
@@ -267,12 +269,13 @@ func TestUnlockPanic(t *testing.T) {
 
 // Borrowed from rwmutex_test.go
 func TestUnlockPanic2(t *testing.T) {
+	mu := NewDRWMutex(ds, "test-unlock-panic-2")
 	defer func() {
 		if recover() == nil {
 			t.Fatalf("unlock of unlocked RWMutex did not panic")
 		}
+		mu.RUnlock() // Unlock, so -test.count > 1 works
 	}()
-	mu := NewDRWMutex(ds, "test-unlock-panic-2")
 	mu.RLock(id, source)
 	mu.Unlock()
 }
@@ -290,12 +293,13 @@ func TestRUnlockPanic(t *testing.T) {
 
 // Borrowed from rwmutex_test.go
 func TestRUnlockPanic2(t *testing.T) {
+	mu := NewDRWMutex(ds, "test-runlock-panic-2")
 	defer func() {
 		if recover() == nil {
 			t.Fatalf("read unlock of unlocked RWMutex did not panic")
 		}
+		mu.Unlock() // Unlock, so -test.count > 1 works
 	}()
-	mu := NewDRWMutex(ds, "test-runlock-panic-2")
 	mu.Lock(id, source)
 	mu.RUnlock()
 }

internal/dsync/dsync.go

@@ -22,4 +22,7 @@ package dsync
 type Dsync struct {
 	// List of rest client objects, one per lock server.
 	GetLockers func() ([]NetLocker, string)
+
+	// Timeouts to apply.
+	Timeouts Timeouts
 }

internal/dsync/dsync_test.go

@@ -28,6 +28,14 @@ import (
 	"github.com/google/uuid"
 )
 
+const (
+	testDrwMutexAcquireTimeout         = 250 * time.Millisecond
+	testDrwMutexRefreshCallTimeout     = 250 * time.Millisecond
+	testDrwMutexUnlockCallTimeout      = 250 * time.Millisecond
+	testDrwMutexForceUnlockCallTimeout = 250 * time.Millisecond
+	testDrwMutexRefreshInterval        = 100 * time.Millisecond
+)
+
 // TestMain initializes the testing framework
 func TestMain(m *testing.M) {
 	startLockServers()
@@ -40,6 +48,12 @@ func TestMain(m *testing.M) {
 
 	ds = &Dsync{
 		GetLockers: func() ([]NetLocker, string) { return clnts, uuid.New().String() },
+		Timeouts: Timeouts{
+			Acquire:         testDrwMutexAcquireTimeout,
+			RefreshCall:     testDrwMutexRefreshCallTimeout,
+			UnlockCall:      testDrwMutexUnlockCallTimeout,
+			ForceUnlockCall: testDrwMutexForceUnlockCallTimeout,
+		},
 	}
 
 	code := m.Run()
@@ -53,7 +67,7 @@ func TestSimpleLock(t *testing.T) {
 	dm.Lock(id, source)
 
 	// fmt.Println("Lock acquired, waiting...")
-	time.Sleep(2500 * time.Millisecond)
+	time.Sleep(testDrwMutexRefreshCallTimeout)
 
 	dm.Unlock()
 }
@@ -91,7 +105,7 @@ func TestTwoSimultaneousLocksForSameResource(t *testing.T) {
 
 	// Release lock after 10 seconds
 	go func() {
-		time.Sleep(10 * time.Second)
+		time.Sleep(5 * testDrwMutexAcquireTimeout)
 		// fmt.Println("Unlocking dm1")
 
 		dm1st.Unlock()
@@ -100,7 +114,7 @@ func TestTwoSimultaneousLocksForSameResource(t *testing.T) {
 	dm2nd.Lock(id, source)
 
 	// fmt.Printf("2nd lock obtained after 1st lock is released\n")
-	time.Sleep(2500 * time.Millisecond)
+	time.Sleep(testDrwMutexRefreshCallTimeout * 2)
 
 	dm2nd.Unlock()
 }
@@ -112,14 +126,17 @@ func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 	dm3rd := NewDRWMutex(ds, "aap")
 
 	dm1st.Lock(id, source)
-
+	started := time.Now()
+	var expect time.Duration
 	// Release lock after 10 seconds
 	go func() {
-		time.Sleep(10 * time.Second)
+		// TOTAL
+		time.Sleep(2 * testDrwMutexAcquireTimeout)
 		// fmt.Println("Unlocking dm1")
 
 		dm1st.Unlock()
 	}()
+	expect += 2 * testDrwMutexAcquireTimeout
 
 	var wg sync.WaitGroup
 	wg.Add(2)
@@ -131,7 +148,7 @@ func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 
 		// Release lock after 10 seconds
 		go func() {
-			time.Sleep(2500 * time.Millisecond)
+			time.Sleep(2 * testDrwMutexAcquireTimeout)
 			// fmt.Println("Unlocking dm2")
 
 			dm2nd.Unlock()
@@ -140,10 +157,11 @@ func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 		dm3rd.Lock(id, source)
 
 		// fmt.Printf("3rd lock obtained after 1st & 2nd locks are released\n")
-		time.Sleep(2500 * time.Millisecond)
+		time.Sleep(testDrwMutexRefreshCallTimeout)
 
 		dm3rd.Unlock()
 	}()
+	expect += 2*testDrwMutexAcquireTimeout + testDrwMutexRefreshCallTimeout
 
 	go func() {
 		defer wg.Done()
@@ -152,7 +170,7 @@ func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 
 		// Release lock after 10 seconds
 		go func() {
-			time.Sleep(2500 * time.Millisecond)
+			time.Sleep(2 * testDrwMutexAcquireTimeout)
 			// fmt.Println("Unlocking dm3")
 
 			dm3rd.Unlock()
@@ -161,12 +179,19 @@ func TestThreeSimultaneousLocksForSameResource(t *testing.T) {
 		dm2nd.Lock(id, source)
 
 		// fmt.Printf("2nd lock obtained after 1st & 3rd locks are released\n")
-		time.Sleep(2500 * time.Millisecond)
+		time.Sleep(testDrwMutexRefreshCallTimeout)
 
 		dm2nd.Unlock()
 	}()
+	expect += 2*testDrwMutexAcquireTimeout + testDrwMutexRefreshCallTimeout
 
 	wg.Wait()
+	// We expect at least 3 x 2 x testDrwMutexAcquireTimeout to have passed
+	elapsed := time.Since(started)
+	if elapsed < expect {
+		t.Errorf("expected at least %v time have passed, however %v passed", expect, elapsed)
+	}
+	t.Logf("expected at least %v time have passed, %v passed", expect, elapsed)
 }
 
 // Test two locks for different resources, both succeed
@@ -176,14 +201,8 @@ func TestTwoSimultaneousLocksForDifferentResources(t *testing.T) {
 
 	dm1.Lock(id, source)
 	dm2.Lock(id, source)
-
-	// fmt.Println("Both locks acquired, waiting...")
-	time.Sleep(2500 * time.Millisecond)
-
 	dm1.Unlock()
 	dm2.Unlock()
-
-	time.Sleep(10 * time.Millisecond)
 }
 
 // Test refreshing lock - refresh should always return true
@@ -194,22 +213,19 @@ func TestSuccessfulLockRefresh(t *testing.T) {
 	}
 
 	dm := NewDRWMutex(ds, "aap")
-	contextCanceled := make(chan struct{})
+	dm.refreshInterval = testDrwMutexRefreshInterval
 
-	ctx, cl := context.WithCancel(context.Background())
-	cancel := func() {
-		cl()
-		close(contextCanceled)
-	}
+	ctx, cancel := context.WithCancel(context.Background())
 
 	if !dm.GetLock(ctx, cancel, id, source, Options{Timeout: 5 * time.Minute}) {
 		t.Fatal("GetLock() should be successful")
 	}
 
-	timer := time.NewTimer(drwMutexRefreshInterval * 2)
+	// Make it run twice.
+	timer := time.NewTimer(testDrwMutexRefreshInterval * 2)
 
 	select {
-	case <-contextCanceled:
+	case <-ctx.Done():
 		t.Fatal("Lock context canceled which is not expected")
 	case <-timer.C:
 	}
@@ -231,6 +247,7 @@ func TestFailedRefreshLock(t *testing.T) {
 	}
 
 	dm := NewDRWMutex(ds, "aap")
+	dm.refreshInterval = 500 * time.Millisecond
 	var wg sync.WaitGroup
 	wg.Add(1)
 
@@ -261,14 +278,14 @@ func TestUnlockShouldNotTimeout(t *testing.T) {
 	}
 
 	dm := NewDRWMutex(ds, "aap")
-
+	dm.refreshInterval = testDrwMutexUnlockCallTimeout
 	if !dm.GetLock(context.Background(), nil, id, source, Options{Timeout: 5 * time.Minute}) {
 		t.Fatal("GetLock() should be successful")
 	}
 
 	// Add delay to lock server responses to ensure that lock does not timeout
 	for i := range lockServers {
-		lockServers[i].setResponseDelay(2 * drwMutexUnlockCallTimeout)
+		lockServers[i].setResponseDelay(5 * testDrwMutexUnlockCallTimeout)
 		defer lockServers[i].setResponseDelay(0)
 	}
 
@@ -278,7 +295,7 @@ func TestUnlockShouldNotTimeout(t *testing.T) {
 		unlockReturned <- struct{}{}
 	}()
 
-	timer := time.NewTimer(2 * drwMutexUnlockCallTimeout)
+	timer := time.NewTimer(2 * testDrwMutexUnlockCallTimeout)
 	defer timer.Stop()
 
 	select {

internal/event/target/webhook.go

@@ -119,10 +119,17 @@ func (target *WebhookTarget) IsActive() (bool, error) {
 		}
 		return false, err
 	}
+	tokens := strings.Fields(target.args.AuthToken)
+	switch len(tokens) {
+	case 2:
+		req.Header.Set("Authorization", target.args.AuthToken)
+	case 1:
+		req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
+	}
 
 	resp, err := target.httpClient.Do(req)
 	if err != nil {
-		if xnet.IsNetworkOrHostDown(err, false) || errors.Is(err, context.DeadlineExceeded) {
+		if xnet.IsNetworkOrHostDown(err, true) {
 			return false, errNotConnected
 		}
 		return false, err
@@ -133,7 +140,8 @@ func (target *WebhookTarget) IsActive() (bool, error) {
 	return true, nil
 }
 
-// Save - saves the events to the store if queuestore is configured, which will be replayed when the wenhook connection is active.
+// Save - saves the events to the store if queuestore is configured,
+// which will be replayed when the webhook connection is active.
 func (target *WebhookTarget) Save(eventData event.Event) error {
 	if target.store != nil {
 		return target.store.Put(eventData)

internal/ioutil/ioutil.go

@@ -254,14 +254,14 @@ const DirectioAlignSize = 4096
 // used with DIRECT I/O based file descriptor and it is expected that
 // input writer *os.File not a generic io.Writer. Make sure to have
 // the file opened for writes with syscall.O_DIRECT flag.
-func CopyAligned(w *os.File, r io.Reader, alignedBuf []byte, totalSize int64) (int64, error) {
+func CopyAligned(w io.Writer, r io.Reader, alignedBuf []byte, totalSize int64, file *os.File) (int64, error) {
 	// Writes remaining bytes in the buffer.
-	writeUnaligned := func(w *os.File, buf []byte) (remainingWritten int64, err error) {
+	writeUnaligned := func(w io.Writer, buf []byte) (remainingWritten int64, err error) {
 		// Disable O_DIRECT on fd's on unaligned buffer
 		// perform an amortized Fdatasync(fd) on the fd at
 		// the end, this is performed by the caller before
 		// closing 'w'.
-		if err = disk.DisableDirectIO(w); err != nil {
+		if err = disk.DisableDirectIO(file); err != nil {
 			return remainingWritten, err
 		}
 		// Since w is *os.File io.Copy shall use ReadFrom() call.

internal/kernel/kernel.go

@@ -0,0 +1,130 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build linux
+// +build linux
+
+package kernel
+
+import (
+	"fmt"
+	"io/ioutil"
+	"regexp"
+	"strconv"
+	"strings"
+	"syscall"
+)
+
+var versionRegex = regexp.MustCompile(`^(\d+)\.(\d+).(\d+).*$`)
+
+// VersionFromRelease converts a release string with format
+// 4.4.2[-1] to a kernel version number in LINUX_VERSION_CODE format.
+// That is, for kernel "a.b.c", the version number will be (a<<16 + b<<8 + c)
+func VersionFromRelease(releaseString string) (uint32, error) {
+	versionParts := versionRegex.FindStringSubmatch(releaseString)
+	if len(versionParts) != 4 {
+		return 0, fmt.Errorf("got invalid release version %q (expected format '4.3.2-1')", releaseString)
+	}
+	major, err := strconv.Atoi(versionParts[1])
+	if err != nil {
+		return 0, err
+	}
+
+	minor, err := strconv.Atoi(versionParts[2])
+	if err != nil {
+		return 0, err
+	}
+
+	patch, err := strconv.Atoi(versionParts[3])
+	if err != nil {
+		return 0, err
+	}
+	return Version(major, minor, patch), nil
+}
+
+// Version implements KERNEL_VERSION equivalent macro
+// #define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
+func Version(major, minor, patch int) uint32 {
+	if patch > 255 {
+		patch = 255
+	}
+	out := major<<16 + minor<<8 + patch
+	return uint32(out)
+}
+
+func currentVersionUname() (uint32, error) {
+	var buf syscall.Utsname
+	if err := syscall.Uname(&buf); err != nil {
+		return 0, err
+	}
+	releaseString := strings.Trim(utsnameStr(buf.Release[:]), "\x00")
+	return VersionFromRelease(releaseString)
+}
+
+func currentVersionUbuntu() (uint32, error) {
+	procVersion, err := ioutil.ReadFile("/proc/version_signature")
+	if err != nil {
+		return 0, err
+	}
+	var u1, u2, releaseString string
+	_, err = fmt.Sscanf(string(procVersion), "%s %s %s", &u1, &u2, &releaseString)
+	if err != nil {
+		return 0, err
+	}
+	return VersionFromRelease(releaseString)
+}
+
+var debianVersionRegex = regexp.MustCompile(`.* SMP Debian (\d+\.\d+.\d+-\d+)(?:\+[[:alnum:]]*)?.*`)
+
+func parseDebianVersion(str string) (uint32, error) {
+	match := debianVersionRegex.FindStringSubmatch(str)
+	if len(match) != 2 {
+		return 0, fmt.Errorf("failed to parse kernel version from /proc/version: %s", str)
+	}
+	return VersionFromRelease(match[1])
+}
+
+func currentVersionDebian() (uint32, error) {
+	procVersion, err := ioutil.ReadFile("/proc/version")
+	if err != nil {
+		return 0, fmt.Errorf("error reading /proc/version: %s", err)
+	}
+
+	return parseDebianVersion(string(procVersion))
+}
+
+// CurrentVersion returns the current kernel version in
+// LINUX_VERSION_CODE format (see VersionFromRelease())
+func CurrentVersion() (uint32, error) {
+	// We need extra checks for Debian and Ubuntu as they modify
+	// the kernel version patch number for compatibility with
+	// out-of-tree modules. Linux perf tools do the same for Ubuntu
+	// systems: https://github.com/torvalds/linux/commit/d18acd15c
+	//
+	// See also:
+	// https://kernel-team.pages.debian.net/kernel-handbook/ch-versions.html
+	// https://wiki.ubuntu.com/Kernel/FAQ
+	version, err := currentVersionUbuntu()
+	if err == nil {
+		return version, nil
+	}
+	version, err = currentVersionDebian()
+	if err == nil {
+		return version, nil
+	}
+	return currentVersionUname()
+}

internal/kernel/kernel_other.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build !linux
+// +build !linux
+
+package kernel
+
+// VersionFromRelease only implemented on Linux.
+func VersionFromRelease(_ string) (uint32, error) {
+	return 0, nil
+}
+
+// Version only implemented on Linux.
+func Version(_, _, _ int) uint32 {
+	return 0
+}
+
+// CurrentVersion only implemented on Linux.
+func CurrentVersion() (uint32, error) {
+	return 0, nil
+}

internal/kernel/kernel_test.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build linux
+// +build linux
+
+package kernel
+
+import "testing"
+
+var testData = []struct {
+	success       bool
+	releaseString string
+	kernelVersion uint32
+}{
+	{true, "4.1.2-3", 262402},
+	{true, "4.8.14-200.fc24.x86_64", 264206},
+	{true, "4.1.2-3foo", 262402},
+	{true, "4.1.2foo-1", 262402},
+	{true, "4.1.2-rkt-v1", 262402},
+	{true, "4.1.2rkt-v1", 262402},
+	{true, "4.1.2-3 foo", 262402},
+	{true, "3.10.0-1062.el7.x86_64", 199168},
+	{true, "3.0.0", 196608},
+	{true, "2.6.32", 132640},
+	{true, "5.13.0-30-generic", 331008},
+	{true, "5.10.0-1052-oem", 330240},
+	{false, "foo 4.1.2-3", 0},
+	{true, "4.1.2", 262402},
+	{false, ".4.1.2", 0},
+	{false, "4.1.", 0},
+	{false, "4.1", 0},
+}
+
+func TestVersionFromRelease(t *testing.T) {
+	for _, test := range testData {
+		version, err := VersionFromRelease(test.releaseString)
+		if err != nil && test.success {
+			t.Errorf("expected %q to success: %s", test.releaseString, err)
+		} else if err == nil && !test.success {
+			t.Errorf("expected %q to fail", test.releaseString)
+		}
+		if version != test.kernelVersion {
+			t.Errorf("expected kernel version %d, got %d", test.kernelVersion, version)
+		}
+	}
+}
+
+func TestParseDebianVersion(t *testing.T) {
+	for _, tc := range []struct {
+		success       bool
+		releaseString string
+		kernelVersion uint32
+	}{
+		// 4.9.168
+		{true, "Linux version 4.9.0-9-amd64 (debian-kernel@lists.debian.org) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) ) #1 SMP Debian 4.9.168-1+deb9u3 (2019-06-16)", 264616},
+		// 4.9.88
+		{true, "Linux ip-10-0-75-49 4.9.0-6-amd64 #1 SMP Debian 4.9.88-1+deb9u1 (2018-05-07) x86_64 GNU/Linux", 264536},
+		// 3.0.4
+		{true, "Linux version 3.16.0-9-amd64 (debian-kernel@lists.debian.org) (gcc version 4.9.2 (Debian 4.9.2-10+deb8u2) ) #1 SMP Debian 3.16.68-1 (2019-05-22)", 200772},
+		// Invalid
+		{false, "Linux version 4.9.125-linuxkit (root@659b6d51c354) (gcc version 6.4.0 (Alpine 6.4.0) ) #1 SMP Fri Sep 7 08:20:28 UTC 2018", 0},
+	} {
+		version, err := parseDebianVersion(tc.releaseString)
+		if err != nil && tc.success {
+			t.Errorf("expected %q to success: %s", tc.releaseString, err)
+		} else if err == nil && !tc.success {
+			t.Errorf("expected %q to fail", tc.releaseString)
+		}
+		if version != tc.kernelVersion {
+			t.Errorf("expected kernel version %d, got %d", tc.kernelVersion, version)
+		}
+	}
+}

internal/kernel/kernel_utsname_int8.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build (linux && 386) || (linux && amd64) || (linux && arm64) || (linux && mips64) || (linux && mips)
+// +build linux,386 linux,amd64 linux,arm64 linux,mips64 linux,mips
+
+package kernel
+
+func utsnameStr(in []int8) string {
+	out := make([]byte, 0, len(in))
+	for i := 0; i < len(in); i++ {
+		if in[i] == 0x00 {
+			break
+		}
+		out = append(out, byte(in[i]))
+	}
+	return string(out)
+}

internal/kernel/kernel_utsname_uint8.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build (linux && arm) || (linux && ppc64) || (linux && ppc64le) || (linux && s390x)
+// +build linux,arm linux,ppc64 linux,ppc64le linux,s390x
+
+package kernel
+
+func utsnameStr(in []uint8) string {
+	out := make([]byte, 0, len(in))
+	for i := 0; i < len(in); i++ {
+		if in[i] == 0x00 {
+			break
+		}
+		out = append(out, byte(in[i]))
+	}
+	return string(out)
+}

internal/logger/console.go

@@ -169,7 +169,6 @@ func (i infoMsg) json(msg string, args ...interface{}) {
 }
 
 func (i infoMsg) quiet(msg string, args ...interface{}) {
-	i.pretty(msg, args...)
 }
 
 func (i infoMsg) pretty(msg string, args ...interface{}) {
@@ -222,22 +221,3 @@ func Error(msg string, data ...interface{}) {
 func Info(msg string, data ...interface{}) {
 	consoleLog(info, msg, data...)
 }
-
-var startupMessage startUpMsg
-
-type startUpMsg struct{}
-
-func (s startUpMsg) json(msg string, args ...interface{}) {
-}
-
-func (s startUpMsg) quiet(msg string, args ...interface{}) {
-}
-
-func (s startUpMsg) pretty(msg string, args ...interface{}) {
-	c.Printf(msg, args...)
-}
-
-// StartupMessage :
-func StartupMessage(msg string, data ...interface{}) {
-	consoleLog(startupMessage, msg, data...)
-}

internal/s3select/csv/record.go

@@ -59,6 +59,7 @@ func (r *Record) Get(name string) (*sql.Value, error) {
 			}
 			return sql.FromBytes([]byte(r.csvRecord[idx])), nil
 		}
+		// TODO: Return Missing?
 		return nil, fmt.Errorf("column %v not found", name)
 	}
 

internal/s3select/json/record.go

@@ -89,6 +89,8 @@ func (r *Record) Set(name string, value *sql.Value) (sql.Record, error) {
 		v = s
 	} else if value.IsNull() {
 		v = nil
+	} else if value.IsMissing() {
+		return r, nil
 	} else if b, ok := value.ToBytes(); ok {
 		// This can either be raw json or a CSV value.
 		// Only treat objects and arrays as JSON.

internal/s3select/select_test.go

@@ -479,6 +479,89 @@ func TestJSONQueries(t *testing.T) {
 			query:      `select * from s3object[*] as s where s.request.header['User-Agent'] is not null`,
 			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}`,
 			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:       "is-not-missing",
+			query:      `select * from s3object[*] as s where s.request.header['User-Agent'] is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:  "is-not-missing-2",
+			query: `select * from s3object[*] as s where s.request.header is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:  "is-not-missing-null",
+			query: `select * from s3object[*] as s where s.request.header is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":null}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":null}}`,
+		},
+		{
+			name:       "is-not-missing",
+			query:      `select * from s3object[*] as s where s.request.header['User-Agent'] is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:  "is-not-missing-2",
+			query: `select * from s3object[*] as s where s.request.header is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:  "is-not-missing-null",
+			query: `select * from s3object[*] as s where s.request.header is not missing`,
+			wantResult: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":null}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":null}}`,
+		},
+
+		{
+			name:       "is-missing",
+			query:      `select * from s3object[*] as s where s.request.header['User-Agent'] is missing`,
+			wantResult: `{"request":{"uri":"/2","header":{}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:       "is-missing-2",
+			query:      `select * from s3object[*] as s where s.request.header is missing`,
+			wantResult: ``,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:       "is-missing",
+			query:      `select * from s3object[*] as s where s.request.header['User-Agent'] = missing`,
+			wantResult: `{"request":{"uri":"/2","header":{}}}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":{}}}`,
+		},
+		{
+			name:       "is-missing-null",
+			query:      `select * from s3object[*] as s where s.request.header is missing`,
+			wantResult: ``,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
+{"request":{"uri":"/2","header":null}}`,
+		},
+		{
+			name:  "is-missing-select",
+			query: `select s.request.header['User-Agent'] as h from s3object[*] as s`,
+			wantResult: `{"h":"test"}
+{}`,
+			withJSON: `{"request":{"uri":"/1","header":{"User-Agent":"test"}}}
 {"request":{"uri":"/2","header":{}}}`,
 		},
 	}

internal/s3select/sql/evaluate.go

@@ -449,6 +449,8 @@ func jsonToValue(result interface{}) (*Value, error) {
 		return FromArray(rval), nil
 	case nil:
 		return FromNull(), nil
+	case Missing:
+		return FromMissing(), nil
 	}
 	return nil, fmt.Errorf("Unhandled value type: %T", result)
 }
@@ -493,6 +495,8 @@ func (e *LitValue) evalNode(_ Record) (res *Value, err error) {
 		return FromString(string(*e.String)), nil
 	case e.Boolean != nil:
 		return FromBool(bool(*e.Boolean)), nil
+	case e.Missing:
+		return FromMissing(), nil
 	}
 	return FromNull(), nil
 }

internal/s3select/sql/jsonpath.go

@@ -52,12 +52,12 @@ func jsonpathEval(p []*JSONPathElement, v interface{}) (r interface{}, flat bool
 				}
 			}
 			// Key not found - return nil result
-			return nil, false, nil
+			return Missing{}, false, nil
 		case simdjson.Object:
 			elem := kvs.FindKey(key, nil)
 			if elem == nil {
 				// Key not found - return nil result
-				return nil, false, nil
+				return Missing{}, false, nil
 			}
 			val, err := IterToValue(elem.Iter)
 			if err != nil {