Add extra LDAP configuration validation (#14535)

- The result now contains suggestions on fixing common configuration issues.
- These suggestions will subsequently be exposed in console/mc

.github/workflows/go-healing.yml

@@ -37,10 +37,7 @@ jobs:
         env:
           CGO_ENABLED: 0
           GO111MODULE: on
-          MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
-          MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
-          MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
-          MINIO_KMS_KES_KEY_NAME: "my-minio-key"
+          MINIO_KMS_SECRET_KEY: "my-minio-key:oyArl7zlPECEduNbB1KXgdzDn2Bdpvvw0l8VO51HQnY="
           MINIO_KMS_AUTO_ENCRYPTION: on
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0

cmd/admin-handlers.go

@@ -936,6 +936,50 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 }
 
+// NetperfHandler - perform mesh style network throughput test
+func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "NetperfHandler")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if !globalIsDistErasure {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "netperf")
+	lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
+		return
+	}
+	defer nsLock.Unlock(lkctx.Cancel)
+
+	durationStr := r.Form.Get(peerRESTDuration)
+	duration, err := time.ParseDuration(durationStr)
+	if err != nil {
+		duration = globalNetPerfMinDuration
+	}
+
+	if duration < globalNetPerfMinDuration {
+		// We need sample size of minimum 10 secs.
+		duration = globalNetPerfMinDuration
+	}
+
+	duration = duration.Round(time.Second)
+
+	results := globalNotificationSys.Netperf(ctx, duration)
+	enc := json.NewEncoder(w)
+	if err := enc.Encode(madmin.NetperfResult{NodeResults: results}); err != nil {
+		return
+	}
+}
+
 // SpeedtestHandler - Deprecated. See ObjectSpeedtestHandler
 func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	a.ObjectSpeedtestHandler(w, r)
@@ -1010,8 +1054,15 @@ func (a adminAPIHandlers) ObjectSpeedtestHandler(w http.ResponseWriter, r *http.
 		autotune = false
 	}
 
+	err = objectAPI.MakeBucketWithLocation(ctx, globalObjectPerfBucket, BucketOptions{})
+	if _, ok := err.(BucketExists); !ok {
+		// Only BucketExists error can be ignored.
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	deleteBucket := func() {
-		objectAPI.DeleteBucket(context.Background(), pathJoin(minioMetaBucket, "speedtest"), DeleteBucketOptions{
+		objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
 			Force:      true,
 			NoRecreate: true,
 		})

cmd/admin-router.go

@@ -224,7 +224,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedtestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedtestHandler))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(httpTraceHdrs(adminAPI.DriveSpeedtestHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetSpeedtestHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetperfHandler))
 
 		// HTTP Trace
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(gz(http.HandlerFunc(adminAPI.TraceHandler)))

cmd/auth-handler.go

@@ -197,13 +197,7 @@ func mustGetClaimsFromToken(r *http.Request) map[string]interface{} {
 	return claims
 }
 
-// Fetch claims in the security token returned by the client.
-func getClaimsFromToken(token string) (map[string]interface{}, error) {
-	if token == "" {
-		claims := xjwt.NewMapClaims()
-		return claims.Map(), nil
-	}
-
+func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{}, error) {
 	// JWT token for x-amz-security-token is signed with admin
 	// secret key, temporary credentials become invalid if
 	// server admin credentials change. This is done to ensure
@@ -212,9 +206,15 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 	// hijacking the policies. We need to make sure that this is
 	// based an admin credential such that token cannot be decoded
 	// on the client side and is treated like an opaque value.
-	claims, err := auth.ExtractClaims(token, globalActiveCred.SecretKey)
+	claims, err := auth.ExtractClaims(token, secret)
 	if err != nil {
-		return nil, errAuthentication
+		if subtle.ConstantTimeCompare([]byte(secret), []byte(globalActiveCred.SecretKey)) == 1 {
+			return nil, errAuthentication
+		}
+		claims, err = auth.ExtractClaims(token, globalActiveCred.SecretKey)
+		if err != nil {
+			return nil, errAuthentication
+		}
 	}
 
 	// If OPA is set, return without any further checks.
@@ -241,23 +241,50 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 	return claims.Map(), nil
 }
 
+// Fetch claims in the security token returned by the client.
+func getClaimsFromToken(token string) (map[string]interface{}, error) {
+	return getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey)
+}
+
 // Fetch claims in the security token returned by the client and validate the token.
 func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]interface{}, APIErrorCode) {
 	token := getSessionToken(r)
 	if token != "" && cred.AccessKey == "" {
+		// x-amz-security-token is not allowed for anonymous access.
 		return nil, ErrNoAccessKey
 	}
-	if cred.IsServiceAccount() && token == "" {
-		token = cred.SessionToken
-	}
-	if subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+
+	if token == "" && cred.IsTemp() {
+		// Temporary credentials should always have x-amz-security-token
 		return nil, ErrInvalidToken
 	}
-	claims, err := getClaimsFromToken(token)
-	if err != nil {
-		return nil, toAPIErrorCode(r.Context(), err)
+
+	if token != "" && !cred.IsTemp() {
+		// x-amz-security-token should not present for static credentials.
+		return nil, ErrInvalidToken
 	}
-	return claims, ErrNone
+
+	if cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+		// validate token for temporary credentials only.
+		return nil, ErrInvalidToken
+	}
+
+	secret := globalActiveCred.SecretKey
+	if cred.IsServiceAccount() {
+		token = cred.SessionToken
+		secret = cred.SecretKey
+	}
+
+	if token != "" {
+		claims, err := getClaimsFromTokenWithSecret(token, secret)
+		if err != nil {
+			return nil, toAPIErrorCode(r.Context(), err)
+		}
+		return claims, ErrNone
+	}
+
+	claims := xjwt.NewMapClaims()
+	return claims.Map(), ErrNone
 }
 
 // Check request auth type verifies the incoming http request

cmd/background-newdisks-heal-ops.go

@@ -277,14 +277,6 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 		}
 	}
 
-	if err := bgSeq.healDiskMeta(objAPI); err != nil {
-		if newObjectLayerFn() != nil {
-			// log only in situations, when object layer
-			// has fully initialized.
-			logger.LogIf(bgSeq.ctx, err)
-		}
-	}
-
 	go monitorLocalDisksAndHeal(ctx, z, bgSeq)
 }
 

cmd/bucket-encryption-handlers.go

@@ -206,6 +206,15 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
+	// Call site replication hook.
+	//
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:      madmin.SRBucketMetaTypeSSEConfig,
+		Bucket:    bucket,
+		SSEConfig: nil,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 	writeSuccessNoContent(w)
 }

cmd/bucket-handlers.go

@@ -1270,6 +1270,17 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 				return
 			}
+			rcfg, err := getReplicationConfig(ctx, bucket)
+			switch {
+			case err != nil:
+				if _, ok := err.(BucketReplicationConfigNotFound); !ok {
+					writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+					return
+				}
+			case rcfg.HasActiveRules("", true):
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+				return
+			}
 		}
 	}
 

cmd/bucket-replication-utils.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"fmt"
+	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
@@ -27,6 +28,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/bucket/replication"
+	xhttp "github.com/minio/minio/internal/http"
 )
 
 //go:generate msgp -file=$GOFILE
@@ -699,3 +701,26 @@ func newBucketResyncStatus(bucket string) BucketReplicationResyncStatus {
 		Version:    resyncMetaVersion,
 	}
 }
+
+var contentRangeRegexp = regexp.MustCompile(`bytes ([0-9]+)-([0-9]+)/([0-9]+|\\*)`)
+
+// parse size from content-range header
+func parseSizeFromContentRange(h http.Header) (sz int64, err error) {
+	cr := h.Get(xhttp.ContentRange)
+	if cr == "" {
+		return sz, fmt.Errorf("Content-Range not set")
+	}
+	parts := contentRangeRegexp.FindStringSubmatch(cr)
+	if len(parts) != 4 {
+		return sz, fmt.Errorf("invalid Content-Range header %s", cr)
+	}
+	if parts[3] == "*" {
+		return -1, nil
+	}
+	var usz uint64
+	usz, err = strconv.ParseUint(parts[3], 10, 64)
+	if err != nil {
+		return sz, err
+	}
+	return int64(usz), nil
+}

cmd/bucket-replication.go

@@ -425,7 +425,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		wg.Add(1)
 		go func(index int, tgt *TargetClient) {
 			defer wg.Done()
-			rinfo := replicateDeleteToTarget(ctx, dobj, objectAPI, tgt)
+			rinfo := replicateDeleteToTarget(ctx, dobj, tgt)
 			rinfos.Targets[index] = rinfo
 		}(idx, tgt)
 	}
@@ -482,7 +482,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	}
 }
 
-func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer, tgt *TargetClient) (rinfo replicatedTargetInfo) {
+func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) {
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
 		versionID = dobj.VersionID
@@ -1531,16 +1531,21 @@ func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
 	go globalReplicationStats.loadInitialReplicationMetrics(ctx)
 }
 
+type proxyResult struct {
+	Proxy bool
+	Err   error
+}
+
 // get Reader from replication target if active-active replication is in place and
 // this node returns a 404
-func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (gr *GetObjectReader, proxy bool) {
-	tgt, oi, proxy := proxyHeadToRepTarget(ctx, bucket, object, opts, proxyTargets)
-	if !proxy {
-		return nil, false
+func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (gr *GetObjectReader, proxy proxyResult, err error) {
+	tgt, oi, proxy := proxyHeadToRepTarget(ctx, bucket, object, rs, opts, proxyTargets)
+	if !proxy.Proxy {
+		return nil, proxy, nil
 	}
-	fn, off, length, err := NewGetObjectReader(rs, oi, opts)
+	fn, _, _, err := NewGetObjectReader(nil, oi, opts)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	gopts := miniogo.GetObjectOptions{
 		VersionID:            opts.VersionID,
@@ -1548,30 +1553,40 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 		Internal: miniogo.AdvancedGetOptions{
 			ReplicationProxyRequest: "true",
 		},
+		PartNumber: opts.PartNumber,
 	}
 	// get correct offsets for encrypted object
-	if off >= 0 && length >= 0 {
-		if err := gopts.SetRange(off, off+length-1); err != nil {
-			return nil, false
+	if rs != nil {
+		h, err := rs.ToHeader()
+		if err != nil {
+			return nil, proxy, err
 		}
+		gopts.Set(xhttp.Range, h)
 	}
 	// Make sure to match ETag when proxying.
 	if err = gopts.SetMatchETag(oi.ETag); err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	c := miniogo.Core{Client: tgt.Client}
-	obj, _, _, err := c.GetObject(ctx, bucket, object, gopts)
+	obj, _, h, err := c.GetObject(ctx, bucket, object, gopts)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	closeReader := func() { obj.Close() }
-
 	reader, err := fn(obj, h, closeReader)
 	if err != nil {
-		return nil, false
+		return nil, proxy, err
 	}
 	reader.ObjInfo = oi.Clone()
-	return reader, true
+	if rs != nil {
+		contentSize, err := parseSizeFromContentRange(h)
+		if err != nil {
+			return nil, proxy, err
+		}
+		reader.ObjInfo.Size = contentSize
+	}
+
+	return reader, proxyResult{Proxy: true}, nil
 }
 
 func getproxyTargets(ctx context.Context, bucket, object string, opts ObjectOptions) (tgts *madmin.BucketTargets) {
@@ -1590,11 +1605,11 @@ func getproxyTargets(ctx context.Context, bucket, object string, opts ObjectOpti
 	return tgts
 }
 
-func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgt *TargetClient, oi ObjectInfo, proxy bool) {
+func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgt *TargetClient, oi ObjectInfo, proxy proxyResult) {
 	// this option is set when active-active replication is in place between site A -> B,
 	// and site B does not have the object yet.
 	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
-		return nil, oi, false
+		return nil, oi, proxy
 	}
 	for _, t := range proxyTargets.Targets {
 		tgt = globalBucketTargetSys.GetRemoteTargetClient(ctx, t.Arn)
@@ -1612,9 +1627,22 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 			Internal: miniogo.AdvancedGetOptions{
 				ReplicationProxyRequest: "true",
 			},
+			PartNumber: opts.PartNumber,
 		}
+		if rs != nil {
+			h, err := rs.ToHeader()
+			if err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
+				continue
+			}
+			gopts.Set(xhttp.Range, h)
+		}
+
 		objInfo, err := tgt.StatObject(ctx, t.TargetBucket, object, gopts)
 		if err != nil {
+			if isErrInvalidRange(ErrorRespToObjectError(err, bucket, object)) {
+				return nil, oi, proxyResult{Err: err}
+			}
 			continue
 		}
 
@@ -1645,15 +1673,15 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 		if ok {
 			oi.ContentEncoding = ce
 		}
-		return tgt, oi, true
+		return tgt, oi, proxyResult{Proxy: true}
 	}
-	return nil, oi, false
+	return nil, oi, proxy
 }
 
 // get object info from replication target if active-active replication is in place and
 // this node returns a 404
-func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (oi ObjectInfo, proxy bool) {
-	_, oi, proxy = proxyHeadToRepTarget(ctx, bucket, object, opts, proxyTargets)
+func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (oi ObjectInfo, proxy proxyResult) {
+	_, oi, proxy = proxyHeadToRepTarget(ctx, bucket, object, rs, opts, proxyTargets)
 	return oi, proxy
 }
 

cmd/data-scanner.go

@@ -403,7 +403,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		if filter != nil && ok && existing.Compacted {
 			// If folder isn't in filter and we have data, skip it completely.
 			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
-				if f.healObjectSelect == 0 || !thisHash.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
+				if f.healObjectSelect == 0 || !thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
 					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
 					f.updateCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
 					if f.dataUsageScannerDebug {
@@ -482,7 +482,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				debug:       f.dataUsageScannerDebug,
 				lifeCycle:   activeLifeCycle,
 				replication: replicationCfg,
-				heal:        thisHash.mod(f.oldCache.Info.NextCycle, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
+				heal:        thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healObjectSelect/folder.objectHealProbDiv) && globalIsErasure,
 			}
 			// if the drive belongs to an erasure set
 			// that is already being healed, skip the
@@ -609,13 +609,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// and the entry itself is compacted.
 			if !into.Compacted && f.oldCache.isCompacted(h) {
 				if !h.mod(f.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
-					if f.healObjectSelect == 0 || !h.mod(f.oldCache.Info.NextCycle, f.healFolderInclude/folder.objectHealProbDiv) {
+					if f.healObjectSelect == 0 || !h.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
 						// Transfer and add as child...
 						f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
 						into.addChild(h)
 						continue
 					}
-					folder.objectHealProbDiv = dataUsageUpdateDirCycles
+					folder.objectHealProbDiv = f.healFolderInclude
 				}
 			}
 			scanFolder(folder)

cmd/data-usage-cache.go

@@ -369,6 +369,17 @@ func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool {
 	return uint32(xxhash.Sum64String(string(h)))%cycles == cycle%cycles
 }
 
+// modAlt returns true if the hash mod cycles == cycle.
+// This is out of sync with mod.
+// If cycles is 0 false is always returned.
+// If cycles is 1 true is always returned (as expected).
+func (h dataUsageHash) modAlt(cycle uint32, cycles uint32) bool {
+	if cycles <= 1 {
+		return cycles == 1
+	}
+	return uint32(xxhash.Sum64String(string(h))>>32)%(cycles) == cycle%cycles
+}
+
 // addChild will add a child based on its hash.
 // If it already exists it will not be added again.
 func (e *dataUsageEntry) addChild(hash dataUsageHash) {

cmd/data-usage-utils.go

@@ -92,32 +92,37 @@ type DataUsageInfo struct {
 }
 
 func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
-	if globalTierConfigMgr.Empty() {
+	if dui.TierStats == nil {
 		return nil
 	}
 
-	ts := make(map[string]madmin.TierStats)
-	// Add configured remote tiers
-	for tier := range globalTierConfigMgr.Tiers {
-		ts[tier] = madmin.TierStats{}
+	cfgs := globalTierConfigMgr.ListTiers()
+	if len(cfgs) == 0 {
+		return nil
 	}
+
+	ts := make(map[string]madmin.TierStats, len(cfgs)+1)
+	infos := make([]madmin.TierInfo, 0, len(ts))
+
 	// Add STANDARD (hot-tier)
 	ts[minioHotTier] = madmin.TierStats{}
+	infos = append(infos, madmin.TierInfo{
+		Name: minioHotTier,
+		Type: "internal",
+	})
+	// Add configured remote tiers
+	for _, cfg := range cfgs {
+		ts[cfg.Name] = madmin.TierStats{}
+		infos = append(infos, madmin.TierInfo{
+			Name: cfg.Name,
+			Type: cfg.Type.String(),
+		})
+	}
 
 	ts = dui.TierStats.adminStats(ts)
-	infos := make([]madmin.TierInfo, 0, len(ts))
-	for tier, st := range ts {
-		var tierType string
-		if tier == minioHotTier {
-			tierType = "internal"
-		} else {
-			tierType = globalTierConfigMgr.Tiers[tier].Type.String()
-		}
-		infos = append(infos, madmin.TierInfo{
-			Name:  tier,
-			Type:  tierType,
-			Stats: st,
-		})
+	for i := range infos {
+		info := infos[i]
+		infos[i].Stats = ts[info.Name]
 	}
 
 	sort.Slice(infos, func(i, j int) bool {
@@ -131,3 +136,31 @@ func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
 	})
 	return infos
 }
+
+func (dui DataUsageInfo) tierMetrics() (metrics []Metric) {
+	if dui.TierStats == nil {
+		return nil
+	}
+	// e.g minio_cluster_ilm_transitioned_bytes{tier="S3TIER-1"}=136314880
+	//     minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1
+	//     minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3
+	for tier, st := range dui.TierStats.Tiers {
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedBytesMD(),
+			Value:          float64(st.TotalSize),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedObjectsMD(),
+			Value:          float64(st.NumObjects),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+		metrics = append(metrics, Metric{
+			Description:    getClusterTransitionedVersionsMD(),
+			Value:          float64(st.NumVersions),
+			VariableLabels: map[string]string{"tier": tier},
+		})
+	}
+
+	return metrics
+}

cmd/disk-cache.go

@@ -31,6 +31,7 @@ import (
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/cache"
+	"github.com/minio/minio/internal/disk"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -594,12 +595,23 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 	if err != nil {
 		return nil, false, err
 	}
+	var warningMsg string
 	for i, dir := range config.Drives {
 		// skip diskCache creation for cache drives missing a format.json
 		if formats[i] == nil {
 			caches = append(caches, nil)
 			continue
 		}
+		if !globalIsCICD && len(warningMsg) == 0 {
+			rootDsk, err := disk.IsRootDisk(dir, "/")
+			if err != nil {
+				warningMsg = fmt.Sprintf("Invalid cache dir %s err : %s", dir, err.Error())
+			}
+			if rootDsk {
+				warningMsg = fmt.Sprintf("cache dir cannot be part of root disk: %s", dir)
+			}
+		}
+
 		if err := checkAtimeSupport(dir); err != nil {
 			return nil, false, fmt.Errorf("Atime support required for disk caching, atime check failed with %w", err)
 		}
@@ -610,6 +622,9 @@ func newCache(config cache.Config) ([]*diskCache, bool, error) {
 		}
 		caches = append(caches, cache)
 	}
+	if warningMsg != "" {
+		logger.Info(color.Yellow(fmt.Sprintf("WARNING: Usage of root disk for disk caching is deprecated: %s", warningMsg)))
+	}
 	return caches, migrating, nil
 }
 

cmd/erasure-bucket.go

@@ -39,8 +39,10 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
 	defer NSUpdated(bucket, slashSeparator)
 
 	// Verify if bucket is valid.
-	if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
-		return BucketNameInvalid{Bucket: bucket}
+	if !isMinioMetaBucketName(bucket) {
+		if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
+			return BucketNameInvalid{Bucket: bucket}
+		}
 	}
 
 	storageDisks := er.getDisks()

cmd/erasure-healing_test.go

@@ -382,6 +382,13 @@ func TestHealingDanglingObject(t *testing.T) {
 		copy(disks, newDisks)
 		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock()
 	}
+	getDisk := func(n int) StorageAPI {
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Lock()
+		disk := disks[n]
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock()
+		return disk
+	}
+
 	// Remove 4 disks.
 	setDisks(nil, nil, nil, nil)
 
@@ -440,8 +447,8 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	setDisks(orgDisks[:4]...)
-
-	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk := getDisk(0)
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -458,7 +465,8 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -488,7 +496,8 @@ func TestHealingDanglingObject(t *testing.T) {
 
 	setDisks(orgDisks[:4]...)
 
-	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -505,7 +514,8 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
+	disk = getDisk(0)
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/erasure-object.go

@@ -493,11 +493,6 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 		return objInfo, toObjectErr(err, bucket, object)
 	}
 	objInfo = fi.ToObjectInfo(bucket, object)
-	if opts.VersionID != "" && !fi.VersionPurgeStatus().Empty() {
-		// Make sure to return object info to provide extra information.
-		return objInfo, toObjectErr(errMethodNotAllowed, bucket, object)
-	}
-
 	if fi.Deleted {
 		if opts.VersionID == "" || opts.DeleteMarker {
 			return objInfo, toObjectErr(errFileNotFound, bucket, object)

cmd/erasure-server-pool-decom.go

@@ -593,7 +593,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 				return
 			}
 
-			// We need a reversed order for Decommissioning,
+			// We need a reversed order for decommissioning,
 			// to create the appropriate stack.
 			versionsSorter(fivs.Versions).reverse()
 
@@ -955,6 +955,21 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		}
 	}
 
+	// Create .minio.sys/conifg, .minio.sys/buckets paths if missing,
+	// this code is present to avoid any missing meta buckets on other
+	// pools.
+	for _, metaBucket := range []string{
+		pathJoin(minioMetaBucket, minioConfigPrefix),
+		pathJoin(minioMetaBucket, bucketMetaPrefix),
+	} {
+		var bucketExists BucketExists
+		if err = z.MakeBucketWithLocation(ctx, metaBucket, BucketOptions{}); err != nil {
+			if !errors.As(err, &bucketExists) {
+				return err
+			}
+		}
+	}
+
 	// Buckets data are dispersed in multiple zones/sets, make
 	// sure to decommission the necessary metadata.
 	buckets = append(buckets, BucketInfo{

cmd/erasure-server-pool.go

@@ -661,6 +661,9 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
+			if z.IsSuspended(index) {
+				return nil
+			}
 			return z.serverPools[index].MakeBucketWithLocation(ctx, bucket, opts)
 		}, index)
 	}
@@ -672,7 +675,8 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 			if _, ok := err.(BucketExists); !ok {
 				// Delete created buckets, ignoring errors.
 				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
-					Force: false, NoRecreate: true,
+					Force:      false,
+					NoRecreate: true,
 				})
 			}
 			return err
@@ -1478,6 +1482,9 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
+			if z.IsSuspended(index) {
+				return nil
+			}
 			return z.serverPools[index].DeleteBucket(ctx, bucket, opts)
 		}, index)
 	}

cmd/globals.go

@@ -345,6 +345,12 @@ var (
 	globalIsCICD bool
 
 	globalRootDiskThreshold uint64
+
+	// Used for collecting stats for netperf
+	globalNetPerfMinDuration = time.Second * 10
+	globalNetPerfRX          netPerfRX
+	globalObjectPerfBucket   = "minio-perf-test-tmp-bucket"
+
 	// Add new variable global values here.
 )
 

cmd/httprange.go

@@ -174,3 +174,26 @@ func (h *HTTPRangeSpec) String(resourceSize int64) string {
 	}
 	return fmt.Sprintf("%d-%d", off, off+length-1)
 }
+
+// ToHeader returns the Range header value.
+func (h *HTTPRangeSpec) ToHeader() (string, error) {
+	if h == nil {
+		return "", nil
+	}
+	start := strconv.Itoa(int(h.Start))
+	end := strconv.Itoa(int(h.End))
+	switch {
+	case h.Start >= 0 && h.End >= 0:
+		if h.Start > h.End {
+			return "", errInvalidRange
+		}
+	case h.IsSuffixLength:
+		end = strconv.Itoa(int(h.Start * -1))
+		start = ""
+	case h.Start > -1:
+		end = ""
+	default:
+		return "", fmt.Errorf("does not have valid range value")
+	}
+	return fmt.Sprintf("bytes=%s-%s", start, end), nil
+}

cmd/httprange_test.go

@@ -101,3 +101,46 @@ func TestHTTPRequestRangeSpec(t *testing.T) {
 		t.Errorf("Case %d: Expected errInvalidRange but: %v %v %d %d %v", i, rs, err1, o, l, err2)
 	}
 }
+
+func TestHTTPRequestRangeToHeader(t *testing.T) {
+	validRangeSpecs := []struct {
+		spec        string
+		errExpected bool
+	}{
+		{"bytes=0-", false},
+		{"bytes=1-", false},
+
+		{"bytes=0-9", false},
+		{"bytes=1-10", false},
+		{"bytes=1-1", false},
+		{"bytes=2-5", false},
+
+		{"bytes=-5", false},
+		{"bytes=-1", false},
+		{"bytes=-1000", false},
+		{"bytes=", true},
+		{"bytes= ", true},
+		{"byte=", true},
+		{"bytes=A-B", true},
+		{"bytes=1-B", true},
+		{"bytes=B-1", true},
+		{"bytes=-1-1", true},
+	}
+	for i, testCase := range validRangeSpecs {
+		rs, err := parseRequestRangeSpec(testCase.spec)
+		if err != nil {
+			if !testCase.errExpected || err == nil && testCase.errExpected {
+				t.Errorf("unexpected err: %v", err)
+			}
+			continue
+		}
+		h, err := rs.ToHeader()
+		if err != nil && !testCase.errExpected || err == nil && testCase.errExpected {
+			t.Errorf("expected error with invalid range: %v", err)
+		}
+		if h != testCase.spec {
+			t.Errorf("Case %d: translated to incorrect header: %s expected: %s",
+				i, h, testCase.spec)
+		}
+	}
+}

cmd/iam-store.go

@@ -1456,13 +1456,28 @@ func (store *IAMStoreSys) GetAllParentUsers() map[string]string {
 
 	res := map[string]string{}
 	for _, cred := range cache.iamUsersMap {
-		if cred.IsServiceAccount() || cred.IsTemp() {
+		if (cred.IsServiceAccount() || cred.IsTemp()) && cred.SessionToken != "" {
 			parentUser := cred.ParentUser
-			if cred.SessionToken != "" {
-				claims, err := getClaimsFromToken(cred.SessionToken)
+
+			var (
+				err    error
+				claims map[string]interface{}
+			)
+
+			if cred.IsServiceAccount() {
+				claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, cred.SecretKey)
 				if err != nil {
-					continue
+					claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, globalActiveCred.SecretKey)
 				}
+			} else if cred.IsTemp() {
+				claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, globalActiveCred.SecretKey)
+			}
+
+			if err != nil {
+				continue
+			}
+
+			if len(claims) > 0 {
 				if v, ok := claims[subClaim]; ok {
 					subFromToken, ok := v.(string)
 					if ok {
@@ -1470,6 +1485,11 @@ func (store *IAMStoreSys) GetAllParentUsers() map[string]string {
 					}
 				}
 			}
+
+			if parentUser == "" {
+				continue
+			}
+
 			if _, ok := res[parentUser]; !ok {
 				res[parentUser] = cred.ParentUser
 			}
@@ -1561,6 +1581,7 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		return errNoSuchServiceAccount
 	}
 
+	currentSecretKey := cr.SecretKey
 	if opts.secretKey != "" {
 		if !auth.IsSecretKeyValid(opts.secretKey) {
 			return auth.ErrInvalidSecretKeyLength
@@ -1582,20 +1603,21 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		return errors.New("unknown account status value")
 	}
 
-	if opts.sessionPolicy != nil {
-		m, err := getClaimsFromToken(cr.SessionToken)
-		if err != nil {
-			return fmt.Errorf("unable to get svc acc claims: %v", err)
-		}
+	m, err := getClaimsFromTokenWithSecret(cr.SessionToken, currentSecretKey)
+	if err != nil {
+		return fmt.Errorf("unable to get svc acc claims: %v", err)
+	}
 
-		err = opts.sessionPolicy.Validate()
-		if err != nil {
+	if opts.sessionPolicy != nil {
+		if err := opts.sessionPolicy.Validate(); err != nil {
 			return err
 		}
+
 		policyBuf, err := json.Marshal(opts.sessionPolicy)
 		if err != nil {
 			return err
 		}
+
 		if len(policyBuf) > 16*humanize.KiByte {
 			return fmt.Errorf("Session policy should not exceed 16 KiB characters")
 		}
@@ -1603,10 +1625,11 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		// Overwrite session policy claims.
 		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
 		m[iamPolicyClaimNameSA()] = "embedded-policy"
-		cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, globalActiveCred.SecretKey)
-		if err != nil {
-			return err
-		}
+	}
+
+	cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m, cr.SecretKey)
+	if err != nil {
+		return err
 	}
 
 	u := newUserIdentity(cr)

cmd/iam.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/arn"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
+	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	etcd "go.etcd.io/etcd/client/v3"
@@ -799,14 +800,17 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 		}
 	}
 
-	var cred auth.Credentials
-
+	var accessKey, secretKey string
 	var err error
 	if len(opts.accessKey) > 0 {
-		cred, err = auth.CreateNewCredentialsWithMetadata(opts.accessKey, opts.secretKey, m, globalActiveCred.SecretKey)
+		accessKey, secretKey = opts.accessKey, opts.secretKey
 	} else {
-		cred, err = auth.GetNewCredentialsWithMetadata(m, globalActiveCred.SecretKey)
+		accessKey, secretKey, err = auth.GenerateCredentials()
+		if err != nil {
+			return auth.Credentials{}, err
+		}
 	}
+	cred, err := auth.CreateNewCredentialsWithMetadata(accessKey, secretKey, m, secretKey)
 	if err != nil {
 		return auth.Credentials{}, err
 	}
@@ -880,9 +884,12 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (aut
 
 	var embeddedPolicy *iampolicy.Policy
 
-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
 	if err != nil {
-		return auth.Credentials{}, nil, err
+		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		if err != nil {
+			return auth.Credentials{}, nil, err
+		}
 	}
 	pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
 	sp, spok := jwtClaims.Lookup(iampolicy.SessionPolicyName)
@@ -915,9 +922,12 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
 		return nil, errNoSuchServiceAccount
 	}
 
-	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, sa.SecretKey)
 	if err != nil {
-		return nil, err
+		jwtClaims, err = auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+		if err != nil {
+			return nil, err
+		}
 	}
 	return jwtClaims.Map(), nil
 }
@@ -1063,12 +1073,28 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 		if _, ok := parentUserToCredsMap[cred.ParentUser]; !ok {
 			// Try to find the ldapUsername for this
 			// parentUser by extracting JWT claims
-			jwtClaims, err := auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
-			if err != nil {
-				// skip this cred - session token seems
-				// invalid
+			var (
+				jwtClaims *jwt.MapClaims
+				err       error
+			)
+
+			if cred.SessionToken == "" {
 				continue
 			}
+
+			if cred.IsServiceAccount() {
+				jwtClaims, err = auth.ExtractClaims(cred.SessionToken, cred.SecretKey)
+				if err != nil {
+					jwtClaims, err = auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
+				}
+			} else {
+				jwtClaims, err = auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey)
+			}
+			if err != nil {
+				// skip this cred - session token seems invalid
+				continue
+			}
+
 			ldapUsername, ok := jwtClaims.Lookup(ldapUserN)
 			if !ok {
 				// skip this cred - we dont have the

cmd/jwt.go

@@ -132,7 +132,7 @@ func authenticateURL(accessKey, secretKey string) (string, error) {
 // Check if the request is authenticated.
 // Returns nil if the request is authenticated. errNoAuthToken if token missing.
 // Returns errAuthentication for all other errors.
-func webRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, bool, error) {
+func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, bool, error) {
 	token, err := jwtreq.AuthorizationHeaderExtractor.ExtractToken(req)
 	if err != nil {
 		if err == jwtreq.ErrNoTokenInRequest {

cmd/jwt_test.go

@@ -149,7 +149,7 @@ func TestWebRequestAuthenticate(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		_, _, _, gotErr := webRequestAuthenticate(testCase.req)
+		_, _, _, gotErr := metricsRequestAuthenticate(testCase.req)
 		if testCase.expectedErr != gotErr {
 			t.Errorf("Test %d, expected err %s, got %s", i+1, testCase.expectedErr, gotErr)
 		}

cmd/metacache-entries.go

@@ -389,11 +389,11 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 
 // firstFound returns the first found and the number of set entries.
 func (m metaCacheEntries) firstFound() (first *metaCacheEntry, n int) {
-	for _, entry := range m {
+	for i, entry := range m {
 		if entry.name != "" {
 			n++
 			if first == nil {
-				first = &entry
+				first = &m[i]
 			}
 		}
 	}

cmd/metacache-walk.go

@@ -138,6 +138,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			// Forward some errors?
 			return nil
 		}
+		diskHealthCheckOK(ctx, err)
 		if len(entries) == 0 {
 			return nil
 		}
@@ -179,6 +180,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				s.walkReadMu.Lock()
 				meta.metadata, err = s.readMetadata(ctx, pathJoin(volumeDir, current, entry))
 				s.walkReadMu.Unlock()
+				diskHealthCheckOK(ctx, err)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					continue
@@ -196,6 +198,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				s.walkReadMu.Lock()
 				meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, current, entry))
 				s.walkReadMu.Unlock()
+				diskHealthCheckOK(ctx, err)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					continue
@@ -257,6 +260,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			s.walkReadMu.Lock()
 			meta.metadata, err = s.readMetadata(ctx, pathJoin(volumeDir, meta.name, xlStorageFormatFile))
 			s.walkReadMu.Unlock()
+			diskHealthCheckOK(ctx, err)
 			switch {
 			case err == nil:
 				// It was an object
@@ -266,6 +270,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				out <- meta
 			case osIsNotExist(err), isSysErrIsDir(err):
 				meta.metadata, err = xioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1))
+				diskHealthCheckOK(ctx, err)
 				if err == nil {
 					// It was an object
 					out <- meta
@@ -303,16 +308,12 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	return scanDir(opts.BaseDir)
 }
 
-func (p *xlStorageDiskIDCheck) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error {
-	defer p.updateStorageMetrics(storageMetricWalkDir, opts.Bucket, opts.BaseDir)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWalkDir, opts.Bucket, opts.BaseDir)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.WalkDir(ctx, opts, wr)
 }

cmd/metrics-v2.go

@@ -1663,33 +1663,12 @@ func getClusterTierMetrics() *MetricsGroup {
 		if err != nil {
 			return
 		}
-		// data usage has not captured any data yet.
-		if dui.LastUpdate.IsZero() {
+		// data usage has not captured any tier stats yet.
+		if dui.TierStats == nil {
 			return
 		}
 
-		// e.g minio_cluster_ilm_transitioned_bytes{tier="S3TIER-1"}=136314880
-		//     minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1
-		//     minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3
-		for tier, st := range dui.TierStats.Tiers {
-			metrics = append(metrics, Metric{
-				Description:    getClusterTransitionedBytesMD(),
-				Value:          float64(st.TotalSize),
-				VariableLabels: map[string]string{"tier": tier},
-			})
-			metrics = append(metrics, Metric{
-				Description:    getClusterTransitionedObjectsMD(),
-				Value:          float64(st.NumObjects),
-				VariableLabels: map[string]string{"tier": tier},
-			})
-			metrics = append(metrics, Metric{
-				Description:    getClusterTransitionedVersionsMD(),
-				Value:          float64(st.NumVersions),
-				VariableLabels: map[string]string{"tier": tier},
-			})
-		}
-
-		return metrics
+		return dui.tierMetrics()
 	})
 	return mg
 }

cmd/metrics.go

@@ -674,7 +674,7 @@ func metricsHandler() http.Handler {
 // AuthMiddleware checks if the bearer token is valid and authorized.
 func AuthMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		claims, groups, owner, authErr := webRequestAuthenticate(r)
+		claims, groups, owner, authErr := metricsRequestAuthenticate(r)
 		if authErr != nil || !claims.VerifyIssuer("prometheus", true) {
 			w.WriteHeader(http.StatusForbidden)
 			return

cmd/notification.go

@@ -1553,6 +1553,58 @@ func (sys *NotificationSys) ServiceFreeze(ctx context.Context, freeze bool) []No
 	return nerrs
 }
 
+// Netperf - perform mesh style network throughput test
+func (sys *NotificationSys) Netperf(ctx context.Context, duration time.Duration) []madmin.NetperfNodeResult {
+	length := len(sys.allPeerClients)
+	if length == 0 {
+		// For single node erasure setup.
+		return nil
+	}
+	results := make([]madmin.NetperfNodeResult, length)
+
+	scheme := "http"
+	if globalIsTLS {
+		scheme = "https"
+	}
+
+	var wg sync.WaitGroup
+	for index := range sys.peerClients {
+		if sys.peerClients[index] == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(index int) {
+			defer wg.Done()
+			r, err := sys.peerClients[index].Netperf(ctx, duration)
+			u := &url.URL{
+				Scheme: scheme,
+				Host:   sys.peerClients[index].host.String(),
+			}
+			if err != nil {
+				results[index].Error = err.Error()
+			} else {
+				results[index] = r
+			}
+			results[index].Endpoint = u.String()
+		}(index)
+	}
+
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		r := netperf(ctx, duration)
+		u := &url.URL{
+			Scheme: scheme,
+			Host:   globalLocalNodeName,
+		}
+		results[len(results)-1] = r
+		results[len(results)-1].Endpoint = u.String()
+	}()
+	wg.Wait()
+
+	return results
+}
+
 // Speedtest run GET/PUT tests at input concurrency for requested object size,
 // optionally you can extend the tests longer with time.Duration.
 func (sys *NotificationSys) Speedtest(ctx context.Context, size int,

cmd/object-api-errors.go

@@ -701,3 +701,8 @@ func isErrMethodNotAllowed(err error) bool {
 	var methodNotAllowed MethodNotAllowed
 	return errors.As(err, &methodNotAllowed)
 }
+
+func isErrInvalidRange(err error) bool {
+	_, ok := err.(InvalidRange)
+	return ok
+}

cmd/object-handlers.go

@@ -206,6 +206,9 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 			Start:          offset,
 			End:            offset + length,
 		}
+		if length == -1 {
+			rs.End = -1
+		}
 
 		return getObjectNInfo(ctx, bucket, object, rs, r.Header, readLock, opts)
 	}
@@ -412,20 +415,28 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 	if err != nil {
 		var (
 			reader *GetObjectReader
-			proxy  bool
+			proxy  proxyResult
 		)
 		proxytgts := getproxyTargets(ctx, bucket, object, opts)
 		if !proxytgts.Empty() {
 			// proxy to replication target if active-active replication is in place.
-			reader, proxy = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
-			if reader != nil && proxy {
+			reader, proxy, err = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts)
+			if err != nil && !isErrObjectNotFound(ErrorRespToObjectError(err, bucket, object)) &&
+				!isErrVersionNotFound(ErrorRespToObjectError(err, bucket, object)) {
+				logger.LogIf(ctx, fmt.Errorf("Replication proxy failed for %s/%s(%s) - %w", bucket, object, opts.VersionID, err))
+			}
+			if reader != nil && proxy.Proxy && err == nil {
 				gr = reader
 			}
 		}
-		if reader == nil || !proxy {
+		if reader == nil || !proxy.Proxy {
 			if isErrPreconditionFailed(err) {
 				return
 			}
+			if proxy.Err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, proxy.Err), r.URL)
+				return
+			}
 			if globalBucketVersioningSys.Enabled(bucket) && gr != nil {
 				if !gr.ObjInfo.VersionPurgeStatus.Empty() {
 					// Shows the replication status of a permanent delete of a version
@@ -631,22 +642,28 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error))
 		return
 	}
+	// Get request range.
+	var rs *HTTPRangeSpec
+	rangeHeader := r.Header.Get(xhttp.Range)
 
 	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		var (
-			proxy bool
+			proxy proxyResult
 			oi    ObjectInfo
 		)
 		// proxy HEAD to replication target if active-active replication configured on bucket
 		proxytgts := getproxyTargets(ctx, bucket, object, opts)
 		if !proxytgts.Empty() {
-			oi, proxy = proxyHeadToReplicationTarget(ctx, bucket, object, opts, proxytgts)
-			if proxy {
+			if rangeHeader != "" {
+				rs, _ = parseRequestRangeSpec(rangeHeader)
+			}
+			oi, proxy = proxyHeadToReplicationTarget(ctx, bucket, object, rs, opts, proxytgts)
+			if proxy.Proxy {
 				objInfo = oi
 			}
 		}
-		if !proxy {
+		if !proxy.Proxy {
 			if globalBucketVersioningSys.Enabled(bucket) {
 				switch {
 				case !objInfo.VersionPurgeStatus.Empty():
@@ -703,9 +720,6 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		return
 	}
 
-	// Get request range.
-	var rs *HTTPRangeSpec
-	rangeHeader := r.Header.Get(xhttp.Range)
 	if rangeHeader != "" {
 		// Both 'Range' and 'partNumber' cannot be specified at the same time
 		if opts.PartNumber > 0 {

cmd/peer-rest-client.go

@@ -1119,3 +1119,27 @@ func (client *peerRESTClient) GetLastDayTierStats(ctx context.Context) (dailyAll
 	}
 	return dailyAllTierStats(result), nil
 }
+
+// DevNull - Used by netperf to pump data to peer
+func (client *peerRESTClient) DevNull(ctx context.Context, r io.Reader) error {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodDevNull, nil, r, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return err
+}
+
+// Netperf - To initiate netperf on peer
+func (client *peerRESTClient) Netperf(ctx context.Context, duration time.Duration) (madmin.NetperfNodeResult, error) {
+	var result madmin.NetperfNodeResult
+	values := make(url.Values)
+	values.Set(peerRESTDuration, duration.String())
+	respBody, err := client.callWithContext(context.Background(), peerRESTMethodNetperf, values, nil, -1)
+	if err != nil {
+		return result, err
+	}
+	defer http.DrainBody(respBody)
+	err = gob.NewDecoder(respBody).Decode(&result)
+	return result, err
+}

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v20" // Add drivespeedtest
+	peerRESTVersion       = "v21" // Add netperf
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -70,6 +70,8 @@ const (
 	peerRESTMethodReloadSiteReplicationConfig = "/reloadsitereplicationconfig"
 	peerRESTMethodReloadPoolMeta              = "/reloadpoolmeta"
 	peerRESTMethodGetLastDayTierStats         = "/getlastdaytierstats"
+	peerRESTMethodDevNull                     = "/devnull"
+	peerRESTMethodNetperf                     = "/netperf"
 )
 
 const (

cmd/peer-rest-server.go

@@ -27,20 +27,15 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
-	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
 	b "github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/event"
-	"github.com/minio/minio/internal/hash"
-	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/randreader"
 	"github.com/tinylib/msgp/msgp"
 )
 
@@ -1143,148 +1138,6 @@ func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// SpeedtestResult return value of the speedtest function
-type SpeedtestResult struct {
-	Endpoint  string
-	Uploads   uint64
-	Downloads uint64
-	Error     string
-}
-
-func newRandomReader(size int) io.Reader {
-	return io.LimitReader(randreader.New(), int64(size))
-}
-
-// Runs the speedtest on local MinIO process.
-func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration, storageClass string) (SpeedtestResult, error) {
-	objAPI := newObjectLayerFn()
-	if objAPI == nil {
-		return SpeedtestResult{}, errServerNotInitialized
-	}
-
-	var errOnce sync.Once
-	var retError string
-	var wg sync.WaitGroup
-	var totalBytesWritten uint64
-	var totalBytesRead uint64
-
-	objCountPerThread := make([]uint64, concurrent)
-
-	uploadsCtx, uploadsCancel := context.WithCancel(context.Background())
-	defer uploadsCancel()
-
-	go func() {
-		time.Sleep(duration)
-		uploadsCancel()
-	}()
-
-	objNamePrefix := "speedtest/objects/" + uuid.New().String()
-
-	wg.Add(concurrent)
-	for i := 0; i < concurrent; i++ {
-		go func(i int) {
-			defer wg.Done()
-			for {
-				hashReader, err := hash.NewReader(newRandomReader(size),
-					int64(size), "", "", int64(size))
-				if err != nil {
-					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					uploadsCancel()
-					return
-				}
-				reader := NewPutObjReader(hashReader)
-				objInfo, err := objAPI.PutObject(uploadsCtx, minioMetaBucket, fmt.Sprintf("%s.%d.%d",
-					objNamePrefix, i, objCountPerThread[i]), reader, ObjectOptions{
-					UserDefined: map[string]string{
-						xhttp.AmzStorageClass: storageClass,
-					},
-					Speedtest: true,
-				})
-				if err != nil {
-					objCountPerThread[i]--
-					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					uploadsCancel()
-					return
-				}
-				atomic.AddUint64(&totalBytesWritten, uint64(objInfo.Size))
-				objCountPerThread[i]++
-			}
-		}(i)
-	}
-	wg.Wait()
-
-	// We already saw write failures, no need to proceed into read's
-	if retError != "" {
-		return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
-	}
-
-	downloadsCtx, downloadsCancel := context.WithCancel(context.Background())
-	defer downloadsCancel()
-	go func() {
-		time.Sleep(duration)
-		downloadsCancel()
-	}()
-
-	wg.Add(concurrent)
-	for i := 0; i < concurrent; i++ {
-		go func(i int) {
-			defer wg.Done()
-			var j uint64
-			if objCountPerThread[i] == 0 {
-				return
-			}
-			for {
-				if objCountPerThread[i] == j {
-					j = 0
-				}
-				r, err := objAPI.GetObjectNInfo(downloadsCtx, minioMetaBucket, fmt.Sprintf("%s.%d.%d",
-					objNamePrefix, i, j), nil, nil, noLock, ObjectOptions{})
-				if err != nil {
-					if isErrObjectNotFound(err) {
-						continue
-					}
-					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					downloadsCancel()
-					return
-				}
-				n, err := io.Copy(ioutil.Discard, r)
-				r.Close()
-				if err == nil {
-					// Only capture success criteria - do not
-					// have to capture failed reads, truncated
-					// reads etc.
-					atomic.AddUint64(&totalBytesRead, uint64(n))
-				}
-				if err != nil {
-					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
-						errOnce.Do(func() {
-							retError = err.Error()
-						})
-					}
-					downloadsCancel()
-					return
-				}
-				j++
-			}
-		}(i)
-	}
-	wg.Wait()
-
-	return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
-}
-
 func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
 		s.writeErrorResponse(w, errors.New("invalid request"))
@@ -1384,6 +1237,50 @@ func (s *peerRESTServer) DriveSpeedTestHandler(w http.ResponseWriter, r *http.Re
 	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
 }
 
+// DevNull - everything goes to io.Discard
+func (s *peerRESTServer) DevNull(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	globalNetPerfRX.Connect()
+	defer globalNetPerfRX.Disconnect()
+
+	connectTime := time.Now()
+	ctx := newContext(r, w, "DevNull")
+	for {
+		n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
+		atomic.AddUint64(&globalNetPerfRX.RX, uint64(n))
+		if err != nil && err != io.EOF {
+			// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
+			// would mean the network is not stable. Logging here will help in debugging network issues.
+			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
+				logger.LogIf(ctx, err)
+			}
+		}
+		if err != nil {
+			break
+		}
+	}
+}
+
+// Netperf - perform netperf
+func (s *peerRESTServer) Netperf(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	durationStr := r.Form.Get(peerRESTDuration)
+	duration, err := time.ParseDuration(durationStr)
+	if err != nil || duration.Seconds() == 0 {
+		duration = time.Second * 10
+	}
+	result := netperf(r.Context(), duration.Round(time.Second))
+	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
+}
+
 // registerPeerRESTHandlers - register peer rest router.
 func registerPeerRESTHandlers(router *mux.Router) {
 	server := &peerRESTServer{}
@@ -1431,6 +1328,8 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedtest).HandlerFunc(httpTraceHdrs(server.SpeedtestHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveSpeedTest).HandlerFunc(httpTraceHdrs(server.DriveSpeedTestHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(httpTraceHdrs(server.Netperf))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDevNull).HandlerFunc(httpTraceHdrs(server.DevNull))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadSiteReplicationConfig).HandlerFunc(httpTraceHdrs(server.ReloadSiteReplicationConfigHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadPoolMeta).HandlerFunc(httpTraceHdrs(server.ReloadPoolMetaHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLastDayTierStats).HandlerFunc(httpTraceHdrs(server.GetLastDayTierStatsHandler))

cmd/perf-tests.go

@@ -0,0 +1,287 @@
+// Copyright (c) 2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/dustin/go-humanize"
+	"github.com/google/uuid"
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/pkg/randreader"
+)
+
+// SpeedtestResult return value of the speedtest function
+type SpeedtestResult struct {
+	Endpoint  string
+	Uploads   uint64
+	Downloads uint64
+	Error     string
+}
+
+func newRandomReader(size int) io.Reader {
+	return io.LimitReader(randreader.New(), int64(size))
+}
+
+// Runs the speedtest on local MinIO process.
+func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration, storageClass string) (SpeedtestResult, error) {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return SpeedtestResult{}, errServerNotInitialized
+	}
+
+	var errOnce sync.Once
+	var retError string
+	var wg sync.WaitGroup
+	var totalBytesWritten uint64
+	var totalBytesRead uint64
+
+	client, err := minio.NewCore(globalLocalNodeName, &minio.Options{
+		Creds:     credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""),
+		Secure:    globalIsTLS,
+		Transport: globalProxyTransport,
+	})
+	if err != nil {
+		return SpeedtestResult{}, err
+	}
+
+	objCountPerThread := make([]uint64, concurrent)
+
+	uploadsCtx, uploadsCancel := context.WithCancel(context.Background())
+	defer uploadsCancel()
+
+	go func() {
+		time.Sleep(duration)
+		uploadsCancel()
+	}()
+
+	objNamePrefix := uuid.New().String() + "/"
+
+	wg.Add(concurrent)
+	for i := 0; i < concurrent; i++ {
+		go func(i int) {
+			defer wg.Done()
+			for {
+				reader := newRandomReader(size)
+				_, err := client.PutObject(uploadsCtx, globalObjectPerfBucket, fmt.Sprintf("%s/%d.%d", objNamePrefix, i, objCountPerThread[i]), reader, int64(size), "", "", minio.PutObjectOptions{})
+				if err != nil {
+					if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) {
+						errOnce.Do(func() {
+							retError = err.Error()
+						})
+					}
+					uploadsCancel()
+					return
+				}
+				atomic.AddUint64(&totalBytesWritten, uint64(size))
+				objCountPerThread[i]++
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	// We already saw write failures, no need to proceed into read's
+	if retError != "" {
+		return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
+	}
+
+	downloadsCtx, downloadsCancel := context.WithCancel(context.Background())
+	defer downloadsCancel()
+	go func() {
+		time.Sleep(duration)
+		downloadsCancel()
+	}()
+
+	wg.Add(concurrent)
+	for i := 0; i < concurrent; i++ {
+		go func(i int) {
+			defer wg.Done()
+			var j uint64
+			if objCountPerThread[i] == 0 {
+				return
+			}
+			for {
+				if objCountPerThread[i] == j {
+					j = 0
+				}
+				r, _, _, err := client.GetObject(downloadsCtx, globalObjectPerfBucket, fmt.Sprintf("%s/%d.%d", objNamePrefix, i, j), minio.GetObjectOptions{})
+				if err != nil {
+					errResp, ok := err.(minio.ErrorResponse)
+					if ok && errResp.StatusCode == http.StatusNotFound {
+						continue
+					}
+					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
+						errOnce.Do(func() {
+							retError = err.Error()
+						})
+					}
+					downloadsCancel()
+					return
+				}
+				n, err := io.Copy(ioutil.Discard, r)
+				r.Close()
+				if err == nil {
+					// Only capture success criteria - do not
+					// have to capture failed reads, truncated
+					// reads etc.
+					atomic.AddUint64(&totalBytesRead, uint64(n))
+				}
+				if err != nil {
+					if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) {
+						errOnce.Do(func() {
+							retError = err.Error()
+						})
+					}
+					downloadsCancel()
+					return
+				}
+				j++
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
+}
+
+// To collect RX stats during "mc support perf net"
+// RXSample holds the RX bytes for the duration between
+// the last peer to connect and the first peer to disconnect.
+// This is to improve the RX throughput accuracy.
+type netPerfRX struct {
+	RX                uint64    // RX bytes
+	lastToConnect     time.Time // time at which last peer to connect to us
+	firstToDisconnect time.Time // time at which the first peer disconnects from us
+	RXSample          uint64    // RX bytes between lastToConnect and firstToDisconnect
+	activeConnections uint64
+	sync.RWMutex
+}
+
+func (n *netPerfRX) Connect() {
+	n.Lock()
+	defer n.Unlock()
+	n.activeConnections++
+	atomic.StoreUint64(&globalNetPerfRX.RX, 0)
+	n.lastToConnect = time.Now()
+}
+
+func (n *netPerfRX) Disconnect() {
+	n.Lock()
+	defer n.Unlock()
+	n.activeConnections--
+	if n.firstToDisconnect.IsZero() {
+		n.RXSample = atomic.LoadUint64(&n.RX)
+		n.firstToDisconnect = time.Now()
+	}
+}
+
+func (n *netPerfRX) ActiveConnections() uint64 {
+	n.RLock()
+	defer n.RUnlock()
+	return n.activeConnections
+}
+
+func (n *netPerfRX) Reset() {
+	n.RLock()
+	defer n.RUnlock()
+	n.RX = 0
+	n.RXSample = 0
+	n.lastToConnect = time.Time{}
+	n.firstToDisconnect = time.Time{}
+}
+
+// Reader to read random data.
+type netperfReader struct {
+	n   uint64
+	eof chan struct{}
+	buf []byte
+}
+
+func (m *netperfReader) Read(b []byte) (int, error) {
+	select {
+	case <-m.eof:
+		return 0, io.EOF
+	default:
+	}
+	n := copy(b, m.buf)
+	atomic.AddUint64(&m.n, uint64(n))
+	return n, nil
+}
+
+func netperf(ctx context.Context, duration time.Duration) madmin.NetperfNodeResult {
+	r := &netperfReader{eof: make(chan struct{})}
+	r.buf = make([]byte, 128*humanize.KiByte)
+	rand.Read(r.buf)
+
+	connectionsPerPeer := 16
+
+	if len(globalNotificationSys.peerClients) > 16 {
+		// For a large cluster it's enough to have 1 connection per peer to saturate the network.
+		connectionsPerPeer = 1
+	}
+
+	errStr := ""
+	var wg sync.WaitGroup
+	for index := range globalNotificationSys.peerClients {
+		if globalNotificationSys.peerClients[index] == nil {
+			continue
+		}
+		go func(index int) {
+			for i := 0; i < connectionsPerPeer; i++ {
+				wg.Add(1)
+				go func() {
+					defer wg.Done()
+					err := globalNotificationSys.peerClients[index].DevNull(ctx, r)
+					if err != nil {
+						errStr = err.Error()
+					}
+				}()
+			}
+		}(index)
+	}
+
+	time.Sleep(duration)
+	close(r.eof)
+	wg.Wait()
+	for {
+		if globalNetPerfRX.ActiveConnections() == 0 {
+			break
+		}
+		time.Sleep(time.Second)
+	}
+	rx := float64(globalNetPerfRX.RXSample)
+	delta := globalNetPerfRX.firstToDisconnect.Sub(globalNetPerfRX.lastToConnect)
+	if delta < 0 {
+		rx = 0
+		errStr = "network disconnection issues detected"
+	}
+
+	globalNetPerfRX.Reset()
+	return madmin.NetperfNodeResult{Endpoint: "", TX: r.n / uint64(duration.Seconds()), RX: uint64(rx / delta.Seconds()), Error: errStr}
+}

cmd/server-main.go

@@ -448,6 +448,11 @@ func serverMain(ctx *cli.Context) {
 	// Set system resources to maximum.
 	setMaxResources()
 
+	// Verify kernel release and version.
+	if oldLinux() {
+		logger.Info(color.RedBold("WARNING: Detected Linux kernel version older than 4.0.0 release, there are some known potential performance problems with this kernel version. MinIO recommends a minimum of 4.x.x linux kernel version for best performance"))
+	}
+
 	// Configure server.
 	handler, err := configureServerHandler(globalEndpoints)
 	if err != nil {

cmd/server-rlimit.go

@@ -21,10 +21,28 @@ import (
 	"runtime"
 	"runtime/debug"
 
+	"github.com/minio/minio/internal/kernel"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/sys"
 )
 
+func oldLinux() bool {
+	currentKernel, err := kernel.CurrentVersion()
+	if err != nil {
+		// Could not probe the kernel version
+		return false
+	}
+
+	if currentKernel == 0 {
+		// We could not get any valid value return false
+		return false
+	}
+
+	// legacy linux indicator for printing warnings
+	// about older Linux kernels and Go runtime.
+	return currentKernel < kernel.Version(4, 0, 0)
+}
+
 func setMaxResources() (err error) {
 	// Set the Go runtime max threads threshold to 90% of kernel setting.
 	sysMaxThreads, mErr := sys.GetMaxThreads()

cmd/storage-interface.go

@@ -29,17 +29,40 @@ type StorageAPI interface {
 	String() string
 
 	// Storage operations.
-	IsOnline() bool      // Returns true if disk is online.
-	LastConn() time.Time // Returns the last time this disk (re)-connected
 
+	// Returns true if disk is online and its valid i.e valid format.json.
+	// This has nothing to do with if the drive is hung or not responding.
+	// For that individual storage API calls will fail properly. The purpose
+	// of this function is to know if the "drive" has "format.json" or not
+	// if it has a "format.json" then is it correct "format.json" or not.
+	IsOnline() bool
+
+	// Returns the last time this disk (re)-connected
+	LastConn() time.Time
+
+	// Indicates if disk is local or not.
 	IsLocal() bool
-	Hostname() string   // Returns host name if remote host.
-	Endpoint() Endpoint // Returns endpoint.
 
+	// Returns hostname if disk is remote.
+	Hostname() string
+
+	// Returns the entire endpoint.
+	Endpoint() Endpoint
+
+	// Close the disk, mark it purposefully closed, only implemented for remote disks.
 	Close() error
+
+	// Returns the unique 'uuid' of this disk.
 	GetDiskID() (string, error)
+
+	// Set a unique 'uuid' for this disk, only used when
+	// disk is replaced and formatted.
 	SetDiskID(id string)
-	Healing() *healingTracker // Returns nil if disk is not healing.
+
+	// Returns healing information for a newly replaced disk,
+	// returns 'nil' once healing is complete or if the disk
+	// has never been replaced.
+	Healing() *healingTracker
 
 	DiskInfo(ctx context.Context) (info DiskInfo, err error)
 	NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry) (dataUsageCache, error)

cmd/storage-rest-server.go

@@ -52,7 +52,7 @@ var errDiskStale = errors.New("disk stale")
 
 // To abstract a disk over network.
 type storageRESTServer struct {
-	storage *xlStorage
+	storage *xlStorageDiskIDCheck
 }
 
 func (s *storageRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
@@ -1233,7 +1233,8 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 
 			endpoint := storage.Endpoint()
 
-			server := &storageRESTServer{storage: storage}
+			server := &storageRESTServer{storage: newXLStorageDiskIDCheck(storage)}
+			server.storage.SetDiskID(storage.diskID)
 
 			subrouter := router.PathPrefix(path.Join(storageRESTPrefix, endpoint.Path)).Subrouter()
 

cmd/xl-storage-disk-id-check.go

@@ -19,13 +19,19 @@ package cmd
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"io"
+	"math/rand"
+	"os"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/logger"
 )
 
 //go:generate stringer -type=storageMetric -trimprefix=storageMetric $GOFILE
@@ -69,10 +75,11 @@ type xlStorageDiskIDCheck struct {
 	// do not re-order them, if you add new fields
 	// please use `fieldalignment ./...` to check
 	// if your changes are not causing any problems.
-	storage      StorageAPI
+	storage      *xlStorage
 	apiLatencies [storageMetricLast]*lockedLastMinuteLatency
 	diskID       string
 	apiCalls     [storageMetricLast]uint64
+	health       *diskHealthTracker
 }
 
 func (p *xlStorageDiskIDCheck) getMetrics() DiskMetrics {
@@ -109,6 +116,7 @@ func (e *lockedLastMinuteLatency) value() uint64 {
 func newXLStorageDiskIDCheck(storage *xlStorage) *xlStorageDiskIDCheck {
 	xl := xlStorageDiskIDCheck{
 		storage: storage,
+		health:  newDiskHealthTracker(),
 	}
 	for i := range xl.apiLatencies[:] {
 		xl.apiLatencies[i] = &lockedLastMinuteLatency{}
@@ -215,25 +223,31 @@ func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context) (info DiskInfo, err
 			return info, errDiskNotFound
 		}
 	}
+
+	if p.health.isFaulty() {
+		// if disk is already faulty return faulty for 'mc admin info' output and prometheus alerts.
+		return info, errFaultyDisk
+	}
+
 	return info, nil
 }
 
 func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
-	defer p.updateStorageMetrics(storageMetricMakeVolBulk, volumes...)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricMakeVolBulk, volumes...)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
+
 	return p.storage.MakeVolBulk(ctx, volumes...)
 }
 
 func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err error) {
-	defer p.updateStorageMetrics(storageMetricMakeVol, volume)()
-
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricMakeVol, volume)
+	if err != nil {
+		return err
+	}
+	defer done(&err)
 	if contextCanceled(ctx) {
 		return ctx.Err()
 	}
@@ -244,167 +258,122 @@ func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err
 	return p.storage.MakeVol(ctx, volume)
 }
 
-func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) ([]VolInfo, error) {
-	defer p.updateStorageMetrics(storageMetricListVols, "/")()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) (vi []VolInfo, err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricListVols, "/")
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
+
 	return p.storage.ListVols(ctx)
 }
 
 func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) {
-	defer p.updateStorageMetrics(storageMetricStatVol, volume)()
-
-	if contextCanceled(ctx) {
-		return VolInfo{}, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricStatVol, volume)
+	if err != nil {
 		return vol, err
 	}
+	defer done(&err)
+
 	return p.storage.StatVol(ctx, volume)
 }
 
 func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) {
-	defer p.updateStorageMetrics(storageMetricDeleteVol, volume)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVol, volume)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
+
 	return p.storage.DeleteVol(ctx, volume, forceDelete)
 }
 
-func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) ([]string, error) {
-	defer p.updateStorageMetrics(storageMetricListDir, volume, dirPath)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, volume, dirPath string, count int) (s []string, err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricListDir, volume, dirPath)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.ListDir(ctx, volume, dirPath, count)
 }
 
 func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
-	defer p.updateStorageMetrics(storageMetricReadFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return 0, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadFile, volume, path)
+	if err != nil {
 		return 0, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadFile(ctx, volume, path, offset, buf, verifier)
 }
 
 func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) {
-	defer p.updateStorageMetrics(storageMetricAppendFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricAppendFile, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.AppendFile(ctx, volume, path, buf)
 }
 
-func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
-	defer p.updateStorageMetrics(storageMetricCreateFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricCreateFile, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.CreateFile(ctx, volume, path, size, reader)
 }
 
 func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
-	defer p.updateStorageMetrics(storageMetricReadFileStream, volume, path)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadFileStream, volume, path)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadFileStream(ctx, volume, path, offset, length)
 }
 
-func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
-	defer p.updateStorageMetrics(storageMetricRenameFile, srcVolume, srcPath, dstVolume, dstPath)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricRenameFile, srcVolume, srcPath, dstVolume, dstPath)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath)
 }
 
-func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) error {
-	defer p.updateStorageMetrics(storageMetricRenameData, srcPath, fi.DataDir, dstVolume, dstPath)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricRenameData, srcPath, fi.DataDir, dstVolume, dstPath)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath)
 }
 
 func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (err error) {
-	defer p.updateStorageMetrics(storageMetricCheckParts, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricCheckParts, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.CheckParts(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, recursive bool) (err error) {
-	defer p.updateStorageMetrics(storageMetricDelete, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDelete, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.Delete(ctx, volume, path, recursive)
 }
@@ -417,136 +386,102 @@ func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string
 	if len(versions) > 0 {
 		path = versions[0].Name
 	}
-
-	defer p.updateStorageMetrics(storageMetricDeleteVersions, volume, path)()
-
 	errs = make([]error, len(versions))
-
-	if contextCanceled(ctx) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVersions, volume, path)
+	if err != nil {
 		for i := range errs {
 			errs[i] = ctx.Err()
 		}
 		return errs
 	}
-
-	if err := p.checkDiskStale(); err != nil {
-		for i := range errs {
-			errs[i] = err
+	defer done(&err)
+	errs = p.storage.DeleteVersions(ctx, volume, versions)
+	for i := range errs {
+		if errs[i] != nil {
+			err = errs[i]
+			break
 		}
-		return errs
 	}
 
-	return p.storage.DeleteVersions(ctx, volume, versions)
+	return errs
 }
 
-func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error {
-	defer p.updateStorageMetrics(storageMetricVerifyFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err := p.checkDiskStale(); err != nil {
+func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricVerifyFile, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.VerifyFile(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) {
-	defer p.updateStorageMetrics(storageMetricWriteAll, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWriteAll, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.WriteAll(ctx, volume, path, b)
 }
 
 func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) (err error) {
-	defer p.updateStorageMetrics(storageMetricDeleteVersion, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVersion, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.DeleteVersion(ctx, volume, path, fi, forceDelMarker)
 }
 
 func (p *xlStorageDiskIDCheck) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
-	defer p.updateStorageMetrics(storageMetricUpdateMetadata, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricUpdateMetadata, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.UpdateMetadata(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
-	defer p.updateStorageMetrics(storageMetricWriteMetadata, volume, path)()
-
-	if contextCanceled(ctx) {
-		return ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWriteMetadata, volume, path)
+	if err != nil {
 		return err
 	}
+	defer done(&err)
 
 	return p.storage.WriteMetadata(ctx, volume, path, fi)
 }
 
 func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (fi FileInfo, err error) {
-	defer p.updateStorageMetrics(storageMetricReadVersion, volume, path)()
-
-	if contextCanceled(ctx) {
-		return fi, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadVersion, volume, path)
+	if err != nil {
 		return fi, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadVersion(ctx, volume, path, versionID, readData)
 }
 
 func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) {
-	defer p.updateStorageMetrics(storageMetricReadAll, volume, path)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadAll, volume, path)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.ReadAll(ctx, volume, path)
 }
 
 func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
-	defer p.updateStorageMetrics(storageMetricStatInfoFile, volume, path)()
-
-	if contextCanceled(ctx) {
-		return nil, ctx.Err()
-	}
-
-	if err = p.checkDiskStale(); err != nil {
+	ctx, done, err := p.TrackDiskHealth(ctx, storageMetricStatInfoFile, volume, path)
+	if err != nil {
 		return nil, err
 	}
+	defer done(&err)
 
 	return p.storage.StatInfoFile(ctx, volume, path, glob)
 }
@@ -565,10 +500,10 @@ func storageTrace(s storageMetric, startTime time.Time, duration time.Duration,
 }
 
 // Update storage metrics
-func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func() {
+func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func(err *error) {
 	startTime := time.Now()
 	trace := globalTrace.NumSubscribers() > 0
-	return func() {
+	return func(err *error) {
 		duration := time.Since(startTime)
 
 		atomic.AddUint64(&p.apiCalls[s], 1)
@@ -579,3 +514,317 @@ func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...st
 		}
 	}
 }
+
+const (
+	diskHealthOK = iota
+	diskHealthFaulty
+)
+
+// diskMaxConcurrent is the maximum number of running concurrent operations
+// for local and (incoming) remote disk ops respectively.
+var diskMaxConcurrent = 50
+
+func init() {
+	if s, ok := os.LookupEnv("_MINIO_DISK_MAX_CONCURRENT"); ok && s != "" {
+		var err error
+		diskMaxConcurrent, err = strconv.Atoi(s)
+		if err != nil {
+			logger.Fatal(err, "invalid _MINIO_DISK_MAX_CONCURRENT value")
+		}
+	}
+}
+
+type diskHealthTracker struct {
+	// atomic time of last success
+	lastSuccess int64
+
+	// atomic time of last time a token was grabbed.
+	lastStarted int64
+
+	// Atomic status of disk.
+	status int32
+
+	// Atomic number of requests blocking for a token.
+	blocked int32
+
+	// Concurrency tokens.
+	tokens chan struct{}
+}
+
+// newDiskHealthTracker creates a new disk health tracker.
+func newDiskHealthTracker() *diskHealthTracker {
+	d := diskHealthTracker{
+		lastSuccess: time.Now().UnixNano(),
+		lastStarted: time.Now().UnixNano(),
+		status:      diskHealthOK,
+		tokens:      make(chan struct{}, diskMaxConcurrent),
+	}
+	for i := 0; i < diskMaxConcurrent; i++ {
+		d.tokens <- struct{}{}
+	}
+	return &d
+}
+
+// logSuccess will update the last successful operation time.
+func (d *diskHealthTracker) logSuccess() {
+	atomic.StoreInt64(&d.lastSuccess, time.Now().UnixNano())
+}
+
+func (d *diskHealthTracker) isFaulty() bool {
+	return atomic.LoadInt32(&d.status) == diskHealthFaulty
+}
+
+type (
+	healthDiskCtxKey   struct{}
+	healthDiskCtxValue struct {
+		lastSuccess *int64
+	}
+)
+
+// logSuccess will update the last successful operation time.
+func (h *healthDiskCtxValue) logSuccess() {
+	atomic.StoreInt64(h.lastSuccess, time.Now().UnixNano())
+}
+
+// noopDoneFunc is a no-op done func.
+// Can be reused.
+var noopDoneFunc = func(_ *error) {}
+
+// TrackDiskHealth for this request.
+// When a non-nil error is returned 'done' MUST be called
+// with the status of the response, if it corresponds to disk health.
+// If the pointer sent to done is non-nil AND the error
+// is either nil or io.EOF the disk is considered good.
+// So if unsure if the disk status is ok, return nil as a parameter to done.
+// Shadowing will work as long as return error is named: https://go.dev/play/p/sauq86SsTN2
+func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMetric, paths ...string) (c context.Context, done func(*error), err error) {
+	done = noopDoneFunc
+	if contextCanceled(ctx) {
+		return ctx, done, ctx.Err()
+	}
+
+	// Return early if disk is faulty already.
+	if atomic.LoadInt32(&p.health.status) == diskHealthFaulty {
+		return ctx, done, errFaultyDisk
+	}
+
+	// Verify if the disk is not stale
+	// - missing format.json (unformatted drive)
+	// - format.json is valid but invalid 'uuid'
+	if err = p.checkDiskStale(); err != nil {
+		return ctx, done, err
+	}
+
+	// Disallow recursive tracking to avoid deadlocks.
+	if ctx.Value(healthDiskCtxKey{}) != nil {
+		done = p.updateStorageMetrics(s, paths...)
+		return ctx, done, nil
+	}
+
+	select {
+	case <-ctx.Done():
+		return ctx, done, ctx.Err()
+	case <-p.health.tokens:
+		// Fast path, got token.
+	default:
+		// We ran out of tokens, check health before blocking.
+		err = p.waitForToken(ctx)
+		if err != nil {
+			return ctx, done, err
+		}
+	}
+	// We only progress here if we got a token.
+
+	atomic.StoreInt64(&p.health.lastStarted, time.Now().UnixNano())
+	ctx = context.WithValue(ctx, healthDiskCtxKey{}, &healthDiskCtxValue{lastSuccess: &p.health.lastSuccess})
+	si := p.updateStorageMetrics(s, paths...)
+	t := time.Now()
+	var once sync.Once
+	return ctx, func(errp *error) {
+		once.Do(func() {
+			if false {
+				var ers string
+				if errp != nil {
+					err := *errp
+					ers = fmt.Sprint(err)
+				}
+				fmt.Println(time.Now().Format(time.RFC3339), "op", s, "took", time.Since(t), "result:", ers, "disk:", p.storage.String(), "path:", strings.Join(paths, "/"))
+			}
+			p.health.tokens <- struct{}{}
+			if errp != nil {
+				err := *errp
+				if err != nil && !errors.Is(err, io.EOF) {
+					return
+				}
+				p.health.logSuccess()
+			}
+			si(errp)
+		})
+	}, nil
+}
+
+// waitForToken will wait for a token, while periodically
+// checking the disk status.
+// If nil is returned a token was picked up.
+func (p *xlStorageDiskIDCheck) waitForToken(ctx context.Context) (err error) {
+	atomic.AddInt32(&p.health.blocked, 1)
+	defer func() {
+		atomic.AddInt32(&p.health.blocked, -1)
+	}()
+	// Avoid stampeding herd...
+	ticker := time.NewTicker(5*time.Second + time.Duration(rand.Int63n(int64(5*time.Second))))
+	defer ticker.Stop()
+	for {
+		err = p.checkHealth(ctx)
+		if err != nil {
+			return err
+		}
+		select {
+		case <-ticker.C:
+		// Ticker expired, check health again.
+		case <-ctx.Done():
+			return ctx.Err()
+		case <-p.health.tokens:
+			return nil
+		}
+	}
+}
+
+// checkHealth should only be called when tokens have run out.
+// This will check if disk should be taken offline.
+func (p *xlStorageDiskIDCheck) checkHealth(ctx context.Context) (err error) {
+	if atomic.LoadInt32(&p.health.status) == diskHealthFaulty {
+		return errFaultyDisk
+	}
+	// Check if there are tokens.
+	if len(p.health.tokens) > 0 {
+		return nil
+	}
+
+	const maxTimeSinceLastSuccess = 30 * time.Second
+	const minTimeSinceLastOpStarted = 15 * time.Second
+
+	// To avoid stampeding herd (100s of simultaneous starting requests)
+	// there must be a delay between the last started request and now
+	// for the last lastSuccess to be useful.
+	t := time.Since(time.Unix(0, atomic.LoadInt64(&p.health.lastStarted)))
+	if t < minTimeSinceLastOpStarted {
+		return nil
+	}
+
+	// If also more than 15 seconds since last success, take disk offline.
+	t = time.Since(time.Unix(0, atomic.LoadInt64(&p.health.lastSuccess)))
+	if t > maxTimeSinceLastSuccess {
+		if atomic.CompareAndSwapInt32(&p.health.status, diskHealthOK, diskHealthFaulty) {
+			logger.LogAlwaysIf(ctx, fmt.Errorf("taking disk %s offline, time since last response %v", p.storage.String(), t.Round(time.Millisecond)))
+			go p.monitorDiskStatus()
+		}
+		return errFaultyDisk
+	}
+	return nil
+}
+
+// monitorDiskStatus should be called once when a drive has been marked offline.
+// Once the disk has been deemed ok, it will return to online status.
+func (p *xlStorageDiskIDCheck) monitorDiskStatus() {
+	t := time.NewTicker(5 * time.Second)
+	defer t.Stop()
+	fn := mustGetUUID()
+	for range t.C {
+		if len(p.health.tokens) == 0 {
+			// Queue is still full, no need to check.
+			continue
+		}
+		err := p.storage.WriteAll(context.Background(), minioMetaTmpBucket, fn, []byte{10000: 42})
+		if err != nil {
+			continue
+		}
+		b, err := p.storage.ReadAll(context.Background(), minioMetaTmpBucket, fn)
+		if err != nil || len(b) != 10001 {
+			continue
+		}
+		err = p.storage.Delete(context.Background(), minioMetaTmpBucket, fn, false)
+		if err == nil {
+			logger.Info("Able to read+write, bringing disk %s online.", p.storage.String())
+			atomic.StoreInt32(&p.health.status, diskHealthOK)
+			return
+		}
+	}
+}
+
+// diskHealthCheckOK will check if the provided error is nil
+// and update disk status if good.
+// For convenience a bool is returned to indicate any error state
+// that is not io.EOF.
+func diskHealthCheckOK(ctx context.Context, err error) bool {
+	// Check if context has a disk health check.
+	tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue)
+	if !ok {
+		// No tracker, return
+		return err == nil || errors.Is(err, io.EOF)
+	}
+	if err == nil || errors.Is(err, io.EOF) {
+		tracker.logSuccess()
+		return true
+	}
+	return false
+}
+
+// diskHealthWrapper provides either a io.Reader or io.Writer
+// that updates status of the provided tracker.
+// Use through diskHealthReader or diskHealthWriter.
+type diskHealthWrapper struct {
+	tracker *healthDiskCtxValue
+	r       io.Reader
+	w       io.Writer
+}
+
+func (d *diskHealthWrapper) Read(p []byte) (int, error) {
+	if d.r == nil {
+		return 0, fmt.Errorf("diskHealthWrapper: Read with no reader")
+	}
+	n, err := d.r.Read(p)
+	if err == nil || err == io.EOF && n > 0 {
+		d.tracker.logSuccess()
+	}
+	return n, err
+}
+
+func (d *diskHealthWrapper) Write(p []byte) (int, error) {
+	if d.w == nil {
+		return 0, fmt.Errorf("diskHealthWrapper: Write with no writer")
+	}
+	n, err := d.w.Write(p)
+	if err == nil && n == len(p) {
+		d.tracker.logSuccess()
+	}
+	return n, err
+}
+
+// diskHealthReader provides a wrapper that will update disk health on
+// ctx, on every successful read.
+// This should only be used directly at the os/syscall level,
+// otherwise buffered operations may return false health checks.
+func diskHealthReader(ctx context.Context, r io.Reader) io.Reader {
+	// Check if context has a disk health check.
+	tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue)
+	if !ok {
+		// No need to wrap
+		return r
+	}
+	return &diskHealthWrapper{r: r, tracker: tracker}
+}
+
+// diskHealthWriter provides a wrapper that will update disk health on
+// ctx, on every successful write.
+// This should only be used directly at the os/syscall level,
+// otherwise buffered operations may return false health checks.
+func diskHealthWriter(ctx context.Context, w io.Writer) io.Writer {
+	// Check if context has a disk health check.
+	tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue)
+	if !ok {
+		// No need to wrap
+		return w
+	}
+	return &diskHealthWrapper{w: w, tracker: tracker}
+}

cmd/xl-storage-format-v2.go

@@ -595,6 +595,7 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
 		}
 	}
 	fi.ReplicationState = getInternalReplicationState(fi.Metadata)
+	fi.Deleted = !fi.VersionPurgeStatus().Empty()
 	replStatus := fi.ReplicationState.CompositeReplicationStatus()
 	if replStatus != "" {
 		fi.Metadata[xhttp.AmzBucketReplicationStatus] = string(replStatus)
@@ -658,7 +659,7 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
 	buf := metaDataPoolGet()[:initial]
 	_, err := io.ReadFull(r, buf)
 	if err != nil {
-		return nil, fmt.Errorf("readXLMetaNoData.ReadFull: %w", err)
+		return nil, fmt.Errorf("readXLMetaNoData(io.ReadFull): %w", err)
 	}
 	readMore := func(n int64) error {
 		has := int64(len(buf))
@@ -679,9 +680,9 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
 		if err != nil {
 			if errors.Is(err, io.EOF) {
 				// Returned if we read nothing.
-				return fmt.Errorf("readXLMetaNoData.readMore: %w", io.ErrUnexpectedEOF)
+				err = io.ErrUnexpectedEOF
 			}
-			return fmt.Errorf("readXLMetaNoData.readMore: %w", err)
+			return fmt.Errorf("readXLMetaNoData(readMore): %w", err)
 		}
 		return nil
 	}
@@ -699,7 +700,7 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
 		case 1, 2, 3:
 			sz, tmp, err := msgp.ReadBytesHeader(tmp)
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("readXLMetaNoData(read_meta): uknown metadata version %w", err)
 			}
 			want := int64(sz) + int64(len(buf)-len(tmp))
 
@@ -720,10 +721,14 @@ func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) {
 				return nil, err
 			}
 
+			if int64(len(buf)) < want {
+				return nil, fmt.Errorf("buffer shorter than expected (buflen: %d, want: %d): %w", len(buf), want, errFileCorrupt)
+			}
+
 			tmp = buf[want:]
 			_, after, err := msgp.ReadUint32Bytes(tmp)
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("readXLMetaNoData(read_meta): unknown metadata version %w", err)
 			}
 			want += int64(len(tmp) - len(after))
 
@@ -1277,7 +1282,7 @@ func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) {
 					ver.ObjectV2.MetaSys[k] = []byte(v)
 				}
 				err = x.setIdx(i, *ver)
-				return "", err
+				return uuid.UUID(ver.ObjectV2.DataDir).String(), err
 			}
 		}
 	}

cmd/xl-storage-format-v2_test.go

@@ -18,10 +18,13 @@
 package cmd
 
 import (
+	"bufio"
 	"bytes"
+	"compress/gzip"
 	"encoding/json"
 	"fmt"
 	"io"
+	"os"
 	"sort"
 	"testing"
 	"time"
@@ -34,6 +37,29 @@ import (
 	"github.com/minio/minio/internal/ioutil"
 )
 
+func TestReadXLMetaNoData(t *testing.T) {
+	f, err := os.Open("testdata/xl.meta-corrupt.gz")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	gz, err := gzip.NewReader(bufio.NewReader(f))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	buf, err := io.ReadAll(gz)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = readXLMetaNoData(bytes.NewReader(buf), int64(len(buf)))
+	if err == nil {
+		t.Fatal("expected error but returned success")
+	}
+}
+
 func TestXLV2FormatData(t *testing.T) {
 	failOnErr := func(err error) {
 		t.Helper()

cmd/xl-storage.go

@@ -49,11 +49,12 @@ import (
 
 const (
 	nullVersionID = "null"
-	// Really large streams threshold per shard.
-	reallyLargeFileThreshold = 64 * humanize.MiByte // Optimized for HDDs
+	// Largest streams threshold per shard.
+	largestFileThreshold = 64 * humanize.MiByte // Optimized for HDDs
 
 	// Small file threshold below which data accompanies metadata from storage layer.
 	smallFileThreshold = 128 * humanize.KiByte // Optimized for NVMe/SSDs
+
 	// For hardrives it is possible to set this to a lower value to avoid any
 	// spike in latency. But currently we are simply keeping it optimal for SSDs.
 
@@ -397,6 +398,9 @@ func (s *xlStorage) readMetadataWithDMTime(ctx context.Context, itemPath string)
 		}
 	}
 	buf, err := readXLMetaNoData(f, stat.Size())
+	if err != nil {
+		return nil, stat.ModTime().UTC(), fmt.Errorf("%w -> %s", err, itemPath)
+	}
 	return buf, stat.ModTime().UTC(), err
 }
 
@@ -683,9 +687,11 @@ func (s *xlStorage) SetDiskID(id string) {
 
 func (s *xlStorage) MakeVolBulk(ctx context.Context, volumes ...string) error {
 	for _, volume := range volumes {
-		if err := s.MakeVol(ctx, volume); err != nil && !errors.Is(err, errVolumeExists) {
+		err := s.MakeVol(ctx, volume)
+		if err != nil && !errors.Is(err, errVolumeExists) {
 			return err
 		}
+		diskHealthCheckOK(ctx, err)
 	}
 	return nil
 }
@@ -943,6 +949,7 @@ func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions
 		if err := s.deleteVersions(ctx, volume, fiv.Name, fiv.Versions...); err != nil {
 			errs[i] = err
 		}
+		diskHealthCheckOK(ctx, errs[i])
 	}
 
 	return errs
@@ -1382,7 +1389,7 @@ func (s *xlStorage) readAllData(ctx context.Context, volumeDir string, filePath
 		buf = make([]byte, sz)
 	}
 	// Read file...
-	_, err = io.ReadFull(r, buf)
+	_, err = io.ReadFull(diskHealthReader(ctx, r), buf)
 
 	return buf, stat.ModTime().UTC(), osErrToFileErr(err)
 }
@@ -1693,7 +1700,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 	r := struct {
 		io.Reader
 		io.Closer
-	}{Reader: io.LimitReader(or, length), Closer: closeWrapper(func() error {
+	}{Reader: io.LimitReader(diskHealthReader(ctx, or), length), Closer: closeWrapper(func() error {
 		if !alignment || offset+length%xioutil.DirectioAlignSize != 0 {
 			// invalidate page-cache for unaligned reads.
 			if !globalAPIConfig.isDisableODirect() {
@@ -1783,8 +1790,8 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 	}()
 
 	var bufp *[]byte
-	if fileSize > 0 && fileSize >= reallyLargeFileThreshold {
-		// use a larger 4MiB buffer for really large streams.
+	if fileSize > 0 && fileSize >= largestFileThreshold {
+		// use a larger 4MiB buffer for a really large streams.
 		bufp = xioutil.ODirectPoolXLarge.Get().(*[]byte)
 		defer xioutil.ODirectPoolXLarge.Put(bufp)
 	} else {
@@ -1792,7 +1799,7 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 		defer xioutil.ODirectPoolLarge.Put(bufp)
 	}
 
-	written, err := xioutil.CopyAligned(w, r, *bufp, fileSize)
+	written, err := xioutil.CopyAligned(diskHealthWriter(ctx, w), r, *bufp, fileSize, w)
 	if err != nil {
 		return err
 	}
@@ -2269,6 +2276,7 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 			}
 			return osErrToFileErr(err)
 		}
+		diskHealthCheckOK(ctx, err)
 
 		// renameAll only for objects that have xl.meta not saved inline.
 		if len(fi.Data) == 0 && fi.Size > 0 {
@@ -2406,7 +2414,7 @@ func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolum
 	return nil
 }
 
-func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
+func (s *xlStorage) bitrotVerify(ctx context.Context, partPath string, partSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error {
 	// Open the file for reading.
 	file, err := Open(partPath)
 	if err != nil {
@@ -2421,7 +2429,7 @@ func (s *xlStorage) bitrotVerify(partPath string, partSize int64, algo BitrotAlg
 		// for healing code to fix this file.
 		return err
 	}
-	return bitrotVerify(file, fi.Size(), partSize, algo, sum, shardSize)
+	return bitrotVerify(diskHealthReader(ctx, file), fi.Size(), partSize, algo, sum, shardSize)
 }
 
 func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (err error) {
@@ -2446,7 +2454,7 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File
 	for _, part := range fi.Parts {
 		checksumInfo := erasure.GetChecksumInfo(part.Number)
 		partPath := pathJoin(volumeDir, path, fi.DataDir, fmt.Sprintf("part.%d", part.Number))
-		if err := s.bitrotVerify(partPath,
+		if err := s.bitrotVerify(ctx, partPath,
 			erasure.ShardFileSize(part.Size),
 			checksumInfo.Algorithm,
 			checksumInfo.Hash, erasure.ShardSize()); err != nil {

cmd/xl-storage_test.go

@@ -1884,7 +1884,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
 	if err := storage.WriteAll(context.Background(), volName, fileName, data); err != nil {
 		t.Fatal(err)
 	}
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1894,12 +1894,12 @@ func TestXLStorageVerifyFile(t *testing.T) {
 	}
 
 	// Check if VerifyFile reports the incorrect file length (the correct length is `size+1`)
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 
 	// Check if bitrot fails
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size+1, algo, hashBytes, 0); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size+1, algo, hashBytes, 0); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 
@@ -1928,7 +1928,7 @@ func TestXLStorageVerifyFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	w.(io.Closer).Close()
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, nil, shardSize); err != nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, nil, shardSize); err != nil {
 		t.Fatal(err)
 	}
 
@@ -1943,10 +1943,10 @@ func TestXLStorageVerifyFile(t *testing.T) {
 		t.Fatal(err)
 	}
 	f.Close()
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size, algo, nil, shardSize); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size, algo, nil, shardSize); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
-	if err := storage.storage.(*xlStorage).bitrotVerify(pathJoin(path, volName, fileName), size+1, algo, nil, shardSize); err == nil {
+	if err := storage.storage.bitrotVerify(context.Background(), pathJoin(path, volName, fileName), size+1, algo, nil, shardSize); err == nil {
 		t.Fatal("expected to fail bitrot check")
 	}
 }

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2022-03-03T21-21-16Z
+  image: quay.io/minio/minio:RELEASE.2022-03-14T18-25-24Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/site-replication/run-multi-site-minio-idp.sh

@@ -173,6 +173,17 @@ if [ $? -ne 0 ]; then
     exit_1;
 fi
 
+err_minio2=$(./mc stat minio2/newbucket/xxx --json | jq -r .error.cause.message)
+if [ $? -ne 0 ]; then
+    echo "expecting object to be missing. exiting.."
+    exit_1;
+fi
+
+if [ "${err_minio2}" != "Object does not exist" ]; then
+    echo "expected to see Object does not exist error, exiting..."
+    exit_1;
+fi
+
 ./mc cp README.md minio2/newbucket/
 
 sleep 5

docs/sts/ldap.md

@@ -34,17 +34,17 @@ KEY:
 identity_ldap  enable LDAP SSO support
 
 ARGS:
-MINIO_IDENTITY_LDAP_SERVER_ADDR*            (address)   AD/LDAP server address e.g. "myldapserver.com:636"
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN          (string)    DN for LDAP read-only service account used to perform DN and group lookups
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD    (string)    Password for LDAP read-only service account used to perform DN and group lookups
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN  (string)    ";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER   (string)    Search filter to lookup user DN
-MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER     (string)    search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"
-MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN    (list)      ";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"
-MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY         (on|off)    trust server TLS without verification, defaults to "off" (verify)
-MINIO_IDENTITY_LDAP_SERVER_INSECURE         (on|off)    allow plain text connection to AD/LDAP server, defaults to "off"
-MINIO_IDENTITY_LDAP_SERVER_STARTTLS         (on|off)    use StartTLS connection to AD/LDAP server, defaults to "off"
-MINIO_IDENTITY_LDAP_COMMENT                 (sentence)  optionally add a comment to this setting
+MINIO_IDENTITY_LDAP_SERVER_ADDR*             (address)   AD/LDAP server address e.g. "myldapserver.com:636"
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN*          (string)    DN for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD     (string)    Password for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN*  (list)      ";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER*   (string)    Search filter to lookup user DN
+MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER      (string)    search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"
+MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN     (list)      ";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"
+MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY          (on|off)    trust server TLS without verification, defaults to "off" (verify)
+MINIO_IDENTITY_LDAP_SERVER_INSECURE          (on|off)    allow plain text connection to AD/LDAP server, defaults to "off"
+MINIO_IDENTITY_LDAP_SERVER_STARTTLS          (on|off)    use StartTLS connection to AD/LDAP server, defaults to "off"
+MINIO_IDENTITY_LDAP_COMMENT                  (sentence)  optionally add a comment to this setting
 ```
 
 ### LDAP server connectivity
@@ -69,8 +69,8 @@ If a self-signed certificate is being used, the certificate can be added to MinI
 A low-privilege read-only LDAP service account is configured in the MinIO server by providing the account's Distinguished Name (DN) and password. This service account is used to perform directory lookups as needed.
 
 ```
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN          (string)    DN for LDAP read-only service account used to perform DN and group lookups
-MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD    (string)    Password for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN*          (string)    DN for LDAP read-only service account used to perform DN and group lookups
+MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD     (string)    Password for LDAP read-only service account used to perform DN and group lookups
 ```
 
 If you set an empty lookup bind password, the lookup bind will use the unauthenticated authentication mechanism, as described in [RFC 4513 Section 5.1.2](https://tools.ietf.org/html/rfc4513#section-5.1.2).
@@ -80,8 +80,8 @@ If you set an empty lookup bind password, the lookup bind will use the unauthent
 When a user provides their LDAP credentials, MinIO runs a lookup query to find the user's Distinguished Name (DN). The search filter and base DN used in this lookup query are configured via the following variables:
 
 ```
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN  (string)    Base LDAP DN to search for user DN
-MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER   (string)    Search filter to lookup user DN
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN*  (list)      ";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"
+MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER*   (string)    Search filter to lookup user DN
 ```
 
 The search filter must use the LDAP username to find the user DN. This is done via [variable substitution](#variable-substitution-in-configuration-strings).
@@ -110,7 +110,7 @@ export MINIO_IDENTITY_LDAP_SERVER_ADDR=myldapserver.com:636
 export MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN='cn=admin,dc=min,dc=io'
 export MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD=admin
 export MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN='ou=hwengg,dc=min,dc=io'
-export MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER='(uid=%s,cn=accounts,dc=min,dc=io)'
+export MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER='(uid=%s)'
 export MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY=on
 ```
 

go.mod

@@ -27,9 +27,9 @@ require (
 	github.com/fatih/color v1.13.0
 	github.com/felixge/fgprof v0.9.2
 	github.com/go-ldap/ldap/v3 v3.2.4
-	github.com/go-openapi/loads v0.21.0
+	github.com/go-openapi/loads v0.21.1
 	github.com/go-sql-driver/mysql v1.6.0
-	github.com/golang-jwt/jwt/v4 v4.2.0
+	github.com/golang-jwt/jwt/v4 v4.3.0
 	github.com/gomodule/redigo v1.8.8
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/mux v1.8.0
@@ -45,9 +45,9 @@ require (
 	github.com/lib/pq v1.10.4
 	github.com/miekg/dns v1.1.46
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.15.0
+	github.com/minio/console v0.15.3
 	github.com/minio/csvparser v1.0.0
-	github.com/minio/dperf v0.3.2
+	github.com/minio/dperf v0.3.4
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.18.0
 	github.com/minio/madmin-go v1.3.5
@@ -61,8 +61,8 @@ require (
 	github.com/minio/zipindex v0.2.1
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/montanaflynn/stats v0.6.6
-	github.com/nats-io/nats-server/v2 v2.7.2
-	github.com/nats-io/nats.go v1.13.1-0.20220121202836-972a071d373d
+	github.com/nats-io/nats-server/v2 v2.7.4
+	github.com/nats-io/nats.go v1.13.1-0.20220308171302-2f2f6968e98d
 	github.com/nats-io/stan.go v0.10.2
 	github.com/ncw/directio v1.0.5
 	github.com/nsqio/go-nsq v1.0.8
@@ -75,7 +75,7 @@ require (
 	github.com/rs/cors v1.7.0
 	github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417
 	github.com/secure-io/sio-go v0.3.1
-	github.com/shirou/gopsutil/v3 v3.21.12
+	github.com/shirou/gopsutil/v3 v3.22.2
 	github.com/streadway/amqp v1.0.0
 	github.com/tinylib/msgp v1.1.7-0.20211026165309-e818a1881b0e
 	github.com/valyala/bytebufferpool v1.0.0
@@ -85,11 +85,11 @@ require (
 	go.etcd.io/etcd/api/v3 v3.5.2
 	go.etcd.io/etcd/client/v3 v3.5.2
 	go.uber.org/atomic v1.9.0
-	go.uber.org/zap v1.19.1
-	golang.org/x/crypto v0.0.0-20220128200615-198e4374d7ed
-	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8
-	golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27
-	golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11
+	go.uber.org/zap v1.21.0
+	golang.org/x/crypto v0.0.0-20220214200702-86341886e292
+	golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b
+	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9
+	golang.org/x/time v0.0.0-20220224211638-0e9765cccd65
 	google.golang.org/api v0.67.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -103,13 +103,13 @@ require (
 	github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect
 	github.com/apache/thrift v0.15.0 // indirect
 	github.com/armon/go-metrics v0.3.3 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
+	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bits-and-blooms/bitset v1.2.0 // indirect
-	github.com/charmbracelet/bubbles v0.10.0 // indirect
-	github.com/charmbracelet/bubbletea v0.19.3 // indirect
+	github.com/charmbracelet/bubbles v0.10.3 // indirect
+	github.com/charmbracelet/bubbletea v0.20.0 // indirect
 	github.com/charmbracelet/lipgloss v0.5.0 // indirect
-	github.com/containerd/console v1.0.2 // indirect
+	github.com/containerd/console v1.0.3 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -120,32 +120,30 @@ require (
 	github.com/eapache/queue v1.1.0 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
 	github.com/frankban/quicktest v1.14.0 // indirect
+	github.com/gdamore/encoding v1.0.0 // indirect
+	github.com/gdamore/tcell/v2 v2.4.1-0.20210905002822-f057f0a857a1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.1 // indirect
-	github.com/go-logr/logr v1.2.0 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-openapi/analysis v0.21.2 // indirect
 	github.com/go-openapi/errors v0.20.2 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.19.6 // indirect
-	github.com/go-openapi/runtime v0.21.1 // indirect
+	github.com/go-openapi/runtime v0.23.1 // indirect
 	github.com/go-openapi/spec v0.20.4 // indirect
-	github.com/go-openapi/strfmt v0.21.1 // indirect
-	github.com/go-openapi/swag v0.19.15 // indirect
-	github.com/go-openapi/validate v0.20.3 // indirect
-	github.com/go-stack/stack v1.8.0 // indirect
-	github.com/goccy/go-json v0.8.1 // indirect
+	github.com/go-openapi/strfmt v0.21.2 // indirect
+	github.com/go-openapi/swag v0.21.1 // indirect
+	github.com/go-openapi/validate v0.21.0 // indirect
+	github.com/go-stack/stack v1.8.1 // indirect
+	github.com/goccy/go-json v0.9.4 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/go-cmp v0.5.7 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20211214055906-6f57359322fd // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
-	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
@@ -155,13 +153,13 @@ require (
 	github.com/jcmturner/gofork v1.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
-	github.com/jessevdk/go-flags v1.4.0 // indirect
+	github.com/jessevdk/go-flags v1.5.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
 	github.com/lestrrat-go/blackmagic v1.0.0 // indirect
 	github.com/lestrrat-go/httpcc v1.0.0 // indirect
 	github.com/lestrrat-go/iter v1.0.1 // indirect
-	github.com/lestrrat-go/jwx v1.2.14 // indirect
+	github.com/lestrrat-go/jwx v1.2.19 // indirect
 	github.com/lestrrat-go/option v1.0.0 // indirect
 	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
 	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
@@ -174,73 +172,59 @@ require (
 	github.com/minio/argon2 v1.0.0 // indirect
 	github.com/minio/colorjson v1.0.1 // indirect
 	github.com/minio/filepath v1.0.0 // indirect
-	github.com/minio/mc v0.0.0-20220204044644-e048c85d71a7 // indirect
+	github.com/minio/mc v0.0.0-20220302011226-f13defa54577 // indirect
 	github.com/minio/md5-simd v1.1.2 // indirect
-	github.com/minio/operator v0.0.0-20220110040724-a5d59a342b7f // indirect
-	github.com/mitchellh/mapstructure v1.4.2 // indirect
+	github.com/mitchellh/mapstructure v1.4.3 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/muesli/ansi v0.0.0-20211018074035-2e021307bc4b // indirect
+	github.com/muesli/ansi v0.0.0-20211031195517-c9f0611b6c70 // indirect
 	github.com/muesli/reflow v0.3.0 // indirect
-	github.com/muesli/termenv v0.11.1-0.20220204035834-5ac8409525e0 // indirect
+	github.com/muesli/termenv v0.11.1-0.20220212125758-44cd13922739 // indirect
 	github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 // indirect
 	github.com/nats-io/nats-streaming-server v0.24.1 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/navidys/tvxwidgets v0.1.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/onsi/ginkgo v1.16.5 // indirect
-	github.com/onsi/gomega v1.16.0 // indirect
-	github.com/pkg/profile v1.6.0 // indirect
-	github.com/pkg/xattr v0.4.4 // indirect
+	github.com/pkg/xattr v0.4.5 // indirect
 	github.com/posener/complete v1.2.3 // indirect
-	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/power-devops/perfstat v0.0.0-20220216144756-c35f1ee13d7c // indirect
 	github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021 // indirect
 	github.com/prometheus/common v0.32.1 // indirect
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
+	github.com/rivo/tview v0.0.0-20220216162559-96063d6082f3 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rjeczalik/notify v0.9.2 // indirect
 	github.com/rogpeppe/go-internal v1.8.1 // indirect
 	github.com/rs/xid v1.3.0 // indirect
 	github.com/sirupsen/logrus v1.8.1 // indirect
-	github.com/tidwall/gjson v1.12.1 // indirect
+	github.com/tidwall/gjson v1.14.0 // indirect
 	github.com/tidwall/match v1.1.1 // indirect
 	github.com/tidwall/pretty v1.2.0 // indirect
 	github.com/tidwall/sjson v1.2.3 // indirect
-	github.com/tklauser/go-sysconf v0.3.9 // indirect
-	github.com/tklauser/numcpus v0.3.0 // indirect
-	github.com/unrolled/secure v1.0.9 // indirect
+	github.com/tklauser/go-sysconf v0.3.10 // indirect
+	github.com/tklauser/numcpus v0.4.0 // indirect
+	github.com/unrolled/secure v1.10.0 // indirect
 	github.com/xdg/stringprep v1.0.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.2 // indirect
-	go.mongodb.org/mongo-driver v1.7.5 // indirect
+	go.mongodb.org/mongo-driver v1.8.4 // indirect
 	go.opencensus.io v0.23.0 // indirect
-	go.uber.org/multierr v1.7.0 // indirect
+	go.uber.org/multierr v1.8.0 // indirect
 	golang.org/x/mod v0.5.1 // indirect
-	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	golang.org/x/tools v0.1.8 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00 // indirect
+	google.golang.org/genproto v0.0.0-20220302033224-9aa15565e42a // indirect
 	google.golang.org/grpc v1.44.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/h2non/filetype.v1 v1.0.5 // indirect
-	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.66.3 // indirect
+	gopkg.in/ini.v1 v1.66.4 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
-	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	k8s.io/api v0.23.3 // indirect
-	k8s.io/apimachinery v0.23.3 // indirect
-	k8s.io/client-go v0.23.3 // indirect
-	k8s.io/klog/v2 v2.40.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
-	k8s.io/utils v0.0.0-20211116205334-6203023598ed // indirect
 	maze.io/x/duration v0.0.0-20160924141736-faac084b6075 // indirect
-	sigs.k8s.io/controller-runtime v0.8.0 // indirect
-	sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.2.1 // indirect
-	sigs.k8s.io/yaml v1.2.0 // indirect
 )

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 3.5.9
-appVersion: RELEASE.2022-03-03T21-21-16Z
+version: 3.6.1
+appVersion: RELEASE.2022-03-14T18-25-24Z
 keywords:
   - minio
   - storage

helm/minio/templates/deployment.yaml

@@ -60,6 +60,9 @@ spec:
         runAsUser: {{ .Values.securityContext.runAsUser }}
         runAsGroup: {{ .Values.securityContext.runAsGroup }}
         fsGroup: {{ .Values.securityContext.fsGroup }}
+        {{- if and (ge .Capabilities.KubeVersion.Major "1") (ge .Capabilities.KubeVersion.Minor "20") }}
+        fsGroupChangePolicy: {{ .Values.securityContext.fsGroupChangePolicy }}
+        {{- end }}
 {{- end }}
 {{ if .Values.serviceAccount.create }}
       serviceAccountName: {{ .Values.serviceAccount.name }}

helm/minio/templates/networkpolicy.yaml

@@ -17,6 +17,7 @@ spec:
   ingress:
     - ports:
         - port: {{ .Values.service.port }}
+        - port: {{ .Values.consoleService.port }}
       {{- if not .Values.networkPolicy.allowExternal }}
       from:
         - podSelector:

helm/minio/templates/statefulset.yaml

@@ -38,6 +38,7 @@ apiVersion: {{ template "minio.statefulset.apiVersion" . }}
 kind: StatefulSet
 metadata:
   name: {{ template "minio.fullname" . }}
+  namespace: {{ .Release.Namespace | quote }}
   labels:
     app: {{ template "minio.name" . }}
     chart: {{ template "minio.chart" . }}
@@ -86,6 +87,9 @@ spec:
         runAsUser: {{ .Values.securityContext.runAsUser }}
         runAsGroup: {{ .Values.securityContext.runAsGroup }}
         fsGroup: {{ .Values.securityContext.fsGroup }}
+        {{- if and (ge .Capabilities.KubeVersion.Major "1") (ge .Capabilities.KubeVersion.Minor "20") }}
+        fsGroupChangePolicy: {{ .Values.securityContext.fsGroupChangePolicy }}
+        {{- end }}
 {{- end }}
 {{ if .Values.serviceAccount.create }}
       serviceAccountName: {{ .Values.serviceAccount.name }}

helm/minio/values.yaml

@@ -14,7 +14,7 @@ clusterDomain: cluster.local
 ##
 image:
   repository: quay.io/minio/minio
-  tag: RELEASE.2021-12-29T06-49-06Z
+  tag: RELEASE.2022-03-14T18-25-24Z
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -25,7 +25,7 @@ imagePullSecrets: []
 ##
 mcImage:
   repository: quay.io/minio/mc
-  tag: RELEASE.2021-12-29T06-52-55Z
+  tag: RELEASE.2022-03-13T22-34-00Z
   pullPolicy: IfNotPresent
 
 ## minio mode, i.e. standalone or distributed or gateway.
@@ -237,6 +237,7 @@ securityContext:
   runAsUser: 1000
   runAsGroup: 1000
   fsGroup: 1000
+  fsGroupChangePolicy: "OnRootMismatch"
 
 # Additational pod annotations
 podAnnotations: {}

index.yaml

@@ -1,9 +1,53 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2022-03-14T18-25-24Z
+    created: "2022-03-14T14:40:15.49716364-07:00"
+    description: Multi-Cloud Object Storage
+    digest: d75b88162bfe54740a233bcecf87328bba2ae23d170bec3a35c828bc6fdc224c
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.6.1.tgz
+    version: 3.6.1
+  - apiVersion: v1
+    appVersion: RELEASE.2022-03-11T23-57-45Z
+    created: "2022-03-14T14:40:15.49596131-07:00"
+    description: Multi-Cloud Object Storage
+    digest: 22e53a1184a21a679bc7d8b94e955777f3506340fc29da5ab0cb6d729bdbde8d
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.6.0.tgz
+    version: 3.6.0
   - apiVersion: v1
     appVersion: RELEASE.2022-03-03T21-21-16Z
-    created: "2022-03-03T15:28:46.71872892-08:00"
+    created: "2022-03-14T14:40:15.494728279-07:00"
     description: Multi-Cloud Object Storage
     digest: 6fda968d3fdfd60470c0055a4e1a3bd8e5aee9ad0af5ba2fb7b7b926fdc9e4a0
     home: https://min.io
@@ -25,7 +69,7 @@ entries:
     version: 3.5.9
   - apiVersion: v1
     appVersion: RELEASE.2022-02-26T02-54-46Z
-    created: "2022-03-03T15:28:46.717568339-08:00"
+    created: "2022-03-14T14:40:15.493481483-07:00"
     description: Multi-Cloud Object Storage
     digest: 8e015369048a3a82bbd53ad36696786f18561c6b25d14eee9e2c93a7336cef46
     home: https://min.io
@@ -47,7 +91,7 @@ entries:
     version: 3.5.8
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2022-03-03T15:28:46.716258212-08:00"
+    created: "2022-03-14T14:40:15.492195816-07:00"
     description: Multi-Cloud Object Storage
     digest: cb3543fe748e5f0d59b3ccf4ab9af8e10b731405ae445d1f5715e30013632373
     home: https://min.io
@@ -69,7 +113,7 @@ entries:
     version: 3.5.7
   - apiVersion: v1
     appVersion: RELEASE.2022-02-18T01-50-10Z
-    created: "2022-03-03T15:28:46.714897775-08:00"
+    created: "2022-03-14T14:40:15.490607172-07:00"
     description: Multi-Cloud Object Storage
     digest: f2e359fa5eefffc59abb3d14a8fa94b11ddeaa99f6cd8dd5f40f4e04121000d6
     home: https://min.io
@@ -91,7 +135,7 @@ entries:
     version: 3.5.6
   - apiVersion: v1
     appVersion: RELEASE.2022-02-16T00-35-27Z
-    created: "2022-03-03T15:28:46.713557317-08:00"
+    created: "2022-03-14T14:40:15.489436101-07:00"
     description: Multi-Cloud Object Storage
     digest: 529d56cca9d83a3d0e5672e63b6e87b5bcbe10a6b45f7a55ba998cceb32f9c81
     home: https://min.io
@@ -113,7 +157,7 @@ entries:
     version: 3.5.5
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2022-03-03T15:28:46.711659592-08:00"
+    created: "2022-03-14T14:40:15.488387455-07:00"
     description: Multi-Cloud Object Storage
     digest: 3d530598f8ece67bec5b7f990d206584893987c713502f9228e4ee24b5535414
     home: https://min.io
@@ -135,7 +179,7 @@ entries:
     version: 3.5.4
   - apiVersion: v1
     appVersion: RELEASE.2022-02-12T00-51-25Z
-    created: "2022-03-03T15:28:46.710394206-08:00"
+    created: "2022-03-14T14:40:15.487290524-07:00"
     description: Multi-Cloud Object Storage
     digest: 53937031348b29615f07fc4869b2d668391d8ba9084630a497abd7a7dea9dfb0
     home: https://min.io
@@ -157,7 +201,7 @@ entries:
     version: 3.5.3
   - apiVersion: v1
     appVersion: RELEASE.2022-02-07T08-17-33Z
-    created: "2022-03-03T15:28:46.70926804-08:00"
+    created: "2022-03-14T14:40:15.486308085-07:00"
     description: Multi-Cloud Object Storage
     digest: 68d643414ff0d565716c5715034fcbf1af262e041915a5c02eb51ec1a65c1ea0
     home: https://min.io
@@ -179,7 +223,7 @@ entries:
     version: 3.5.2
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2022-03-03T15:28:46.708131603-08:00"
+    created: "2022-03-14T14:40:15.48528451-07:00"
     description: Multi-Cloud Object Storage
     digest: a3e855ed0f31233b989fffd775a29d6fbfa0590089010ff16783fd7f142ef6e7
     home: https://min.io
@@ -201,7 +245,7 @@ entries:
     version: 3.5.1
   - apiVersion: v1
     appVersion: RELEASE.2022-02-01T18-00-14Z
-    created: "2022-03-03T15:28:46.70701956-08:00"
+    created: "2022-03-14T14:40:15.483859266-07:00"
     description: Multi-Cloud Object Storage
     digest: b1b0ae3c54b4260a698753e11d7781bb8ddc67b7e3fbf0af82796e4cd4ef92a3
     home: https://min.io
@@ -223,7 +267,7 @@ entries:
     version: 3.5.0
   - apiVersion: v1
     appVersion: RELEASE.2022-01-28T02-28-16Z
-    created: "2022-03-03T15:28:46.705883001-08:00"
+    created: "2022-03-14T14:40:15.48287462-07:00"
     description: Multi-Cloud Object Storage
     digest: fecf25d2d3fb208c6f894fed642a60780a570b7f6d0adddde846af7236dc80aa
     home: https://min.io
@@ -245,7 +289,7 @@ entries:
     version: 3.4.8
   - apiVersion: v1
     appVersion: RELEASE.2022-01-25T19-56-04Z
-    created: "2022-03-03T15:28:46.704205655-08:00"
+    created: "2022-03-14T14:40:15.48186785-07:00"
     description: Multi-Cloud Object Storage
     digest: c78008caa5ce98f64c887630f59d0cbd481cb3f19a7d4e9d3e81bf4e1e45cadc
     home: https://min.io
@@ -267,7 +311,7 @@ entries:
     version: 3.4.7
   - apiVersion: v1
     appVersion: RELEASE.2022-01-08T03-11-54Z
-    created: "2022-03-03T15:28:46.702952316-08:00"
+    created: "2022-03-14T14:40:15.48087288-07:00"
     description: Multi-Cloud Object Storage
     digest: 8f2e2691bf897f74ff094dd370ec56ba9d417e5e8926710c14c2ba346330238d
     home: https://min.io
@@ -289,7 +333,7 @@ entries:
     version: 3.4.6
   - apiVersion: v1
     appVersion: RELEASE.2022-01-04T07-41-07Z
-    created: "2022-03-03T15:28:46.701754571-08:00"
+    created: "2022-03-14T14:40:15.479853704-07:00"
     description: Multi-Cloud Object Storage
     digest: bacd140f0016fab35f516bde787da6449b3a960c071fad9e4b6563118033ac84
     home: https://min.io
@@ -311,7 +355,7 @@ entries:
     version: 3.4.5
   - apiVersion: v1
     appVersion: RELEASE.2021-12-29T06-49-06Z
-    created: "2022-03-03T15:28:46.700632058-08:00"
+    created: "2022-03-14T14:40:15.478862047-07:00"
     description: Multi-Cloud Object Storage
     digest: 48a453ea5ffeef25933904caefd9470bfb26224dfc2d1096bd0031467ba53007
     home: https://min.io
@@ -333,7 +377,7 @@ entries:
     version: 3.4.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-03-03T15:28:46.699429459-08:00"
+    created: "2022-03-14T14:40:15.477471151-07:00"
     description: Multi-Cloud Object Storage
     digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60
     home: https://min.io
@@ -355,7 +399,7 @@ entries:
     version: 3.4.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-20T22-07-16Z
-    created: "2022-03-03T15:28:46.698226992-08:00"
+    created: "2022-03-14T14:40:15.47633795-07:00"
     description: Multi-Cloud Object Storage
     digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff
     home: https://min.io
@@ -377,7 +421,7 @@ entries:
     version: 3.4.2
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-03-03T15:28:46.696842536-08:00"
+    created: "2022-03-14T14:40:15.475103638-07:00"
     description: Multi-Cloud Object Storage
     digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37
     home: https://min.io
@@ -399,7 +443,7 @@ entries:
     version: 3.4.1
   - apiVersion: v1
     appVersion: RELEASE.2021-12-18T04-42-33Z
-    created: "2022-03-03T15:28:46.695365796-08:00"
+    created: "2022-03-14T14:40:15.474020887-07:00"
     description: Multi-Cloud Object Storage
     digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453
     home: https://min.io
@@ -421,7 +465,7 @@ entries:
     version: 3.4.0
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-03-03T15:28:46.694266323-08:00"
+    created: "2022-03-14T14:40:15.472984892-07:00"
     description: Multi-Cloud Object Storage
     digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894
     home: https://min.io
@@ -443,7 +487,7 @@ entries:
     version: 3.3.4
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-03-03T15:28:46.693191851-08:00"
+    created: "2022-03-14T14:40:15.471931345-07:00"
     description: Multi-Cloud Object Storage
     digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08
     home: https://min.io
@@ -465,7 +509,7 @@ entries:
     version: 3.3.3
   - apiVersion: v1
     appVersion: RELEASE.2021-12-10T23-03-39Z
-    created: "2022-03-03T15:28:46.692145802-08:00"
+    created: "2022-03-14T14:40:15.470898908-07:00"
     description: Multi-Cloud Object Storage
     digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb
     home: https://min.io
@@ -487,7 +531,7 @@ entries:
     version: 3.3.2
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-03-03T15:28:46.690906843-08:00"
+    created: "2022-03-14T14:40:15.469451021-07:00"
     description: Multi-Cloud Object Storage
     digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4
     home: https://min.io
@@ -509,7 +553,7 @@ entries:
     version: 3.3.1
   - apiVersion: v1
     appVersion: RELEASE.2021-11-24T23-19-33Z
-    created: "2022-03-03T15:28:46.689306119-08:00"
+    created: "2022-03-14T14:40:15.468486385-07:00"
     description: Multi-Cloud Object Storage
     digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4
     home: https://min.io
@@ -531,7 +575,7 @@ entries:
     version: 3.3.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-13T00-23-17Z
-    created: "2022-03-03T15:28:46.688201137-08:00"
+    created: "2022-03-14T14:40:15.467521348-07:00"
     description: Multi-Cloud Object Storage
     digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054
     home: https://min.io
@@ -553,7 +597,7 @@ entries:
     version: 3.2.0
   - apiVersion: v1
     appVersion: RELEASE.2021-10-10T16-53-30Z
-    created: "2022-03-03T15:28:46.686861384-08:00"
+    created: "2022-03-14T14:40:15.466574032-07:00"
     description: Multi-Cloud Object Storage
     digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8
     home: https://min.io
@@ -575,7 +619,7 @@ entries:
     version: 3.1.9
   - apiVersion: v1
     appVersion: RELEASE.2021-10-06T23-36-31Z
-    created: "2022-03-03T15:28:46.685072158-08:00"
+    created: "2022-03-14T14:40:15.465613223-07:00"
     description: Multi-Cloud Object Storage
     digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159
     home: https://min.io
@@ -597,7 +641,7 @@ entries:
     version: 3.1.8
   - apiVersion: v1
     appVersion: RELEASE.2021-10-02T16-31-05Z
-    created: "2022-03-03T15:28:46.683398819-08:00"
+    created: "2022-03-14T14:40:15.464544916-07:00"
     description: Multi-Cloud Object Storage
     digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11
     home: https://min.io
@@ -619,7 +663,7 @@ entries:
     version: 3.1.7
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-03-03T15:28:46.681758267-08:00"
+    created: "2022-03-14T14:40:15.463112267-07:00"
     description: Multi-Cloud Object Storage
     digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310
     home: https://min.io
@@ -641,7 +685,7 @@ entries:
     version: 3.1.6
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-03-03T15:28:46.680475021-08:00"
+    created: "2022-03-14T14:40:15.462171025-07:00"
     description: Multi-Cloud Object Storage
     digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a
     home: https://min.io
@@ -663,7 +707,7 @@ entries:
     version: 3.1.5
   - apiVersion: v1
     appVersion: RELEASE.2021-09-18T18-09-59Z
-    created: "2022-03-03T15:28:46.678759552-08:00"
+    created: "2022-03-14T14:40:15.461286553-07:00"
     description: Multi-Cloud Object Storage
     digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0
     home: https://min.io
@@ -685,7 +729,7 @@ entries:
     version: 3.1.4
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-03-03T15:28:46.677658732-08:00"
+    created: "2022-03-14T14:40:15.460312039-07:00"
     description: Multi-Cloud Object Storage
     digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb
     home: https://min.io
@@ -707,7 +751,7 @@ entries:
     version: 3.1.3
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-03-03T15:28:46.676660879-08:00"
+    created: "2022-03-14T14:40:15.459224829-07:00"
     description: Multi-Cloud Object Storage
     digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd
     home: https://min.io
@@ -729,7 +773,7 @@ entries:
     version: 3.1.2
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-03-03T15:28:46.675832123-08:00"
+    created: "2022-03-14T14:40:15.458109406-07:00"
     description: Multi-Cloud Object Storage
     digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
     home: https://min.io
@@ -751,7 +795,7 @@ entries:
     version: 3.1.1
   - apiVersion: v1
     appVersion: RELEASE.2021-09-09T21-37-07Z
-    created: "2022-03-03T15:28:46.674997142-08:00"
+    created: "2022-03-14T14:40:15.45711975-07:00"
     description: Multi-Cloud Object Storage
     digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
     home: https://min.io
@@ -773,7 +817,7 @@ entries:
     version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2022-03-03T15:28:46.673959757-08:00"
+    created: "2022-03-14T14:40:15.455680171-07:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -795,7 +839,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-03-03T15:28:46.672774395-08:00"
+    created: "2022-03-14T14:40:15.454794257-07:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -817,7 +861,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-03-03T15:28:46.671624084-08:00"
+    created: "2022-03-14T14:40:15.453889745-07:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -839,7 +883,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2022-03-03T15:28:46.670794348-08:00"
+    created: "2022-03-14T14:40:15.45301116-07:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -859,7 +903,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-03-03T15:28:46.669829694-08:00"
+    created: "2022-03-14T14:40:15.452053963-07:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -879,7 +923,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-03-03T15:28:46.668829946-08:00"
+    created: "2022-03-14T14:40:15.449157156-07:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -899,7 +943,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2022-03-03T15:28:46.667869747-08:00"
+    created: "2022-03-14T14:40:15.447756665-07:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -919,7 +963,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-03-03T15:28:46.666658153-08:00"
+    created: "2022-03-14T14:40:15.44681435-07:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -939,7 +983,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-03-03T15:28:46.662395153-08:00"
+    created: "2022-03-14T14:40:15.445857431-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -959,7 +1003,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2022-03-03T15:28:46.661418704-08:00"
+    created: "2022-03-14T14:40:15.444854784-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -979,7 +1023,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2022-03-03T15:28:46.660257537-08:00"
+    created: "2022-03-14T14:40:15.443860092-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -997,4 +1041,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2022-03-03T15:28:46.658942785-08:00"
+generated: "2022-03-14T14:40:15.442779435-07:00"

internal/config/cache/config.go

@@ -115,7 +115,6 @@ func parseCacheDrives(drives string) ([]string, error) {
 			endpoints = append(endpoints, d)
 		}
 	}
-
 	for _, d := range endpoints {
 		if !filepath.IsAbs(d) {
 			return nil, config.ErrInvalidCacheDrivesValue(nil).Msg("cache dir should be absolute path: %s", d)

internal/config/identity/ldap/config-validator.go

@@ -0,0 +1,271 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ldap
+
+import (
+	"fmt"
+	"strings"
+)
+
+// Result - type for high-level names for the validation status of the config.
+type Result string
+
+// Constant values for Result type.
+const (
+	ConfigOk                       Result = "Config OK"
+	ConnectivityError              Result = "LDAP Server Connection Error"
+	LookupBindError                Result = "LDAP Lookup Bind Error"
+	UserSearchParamsMisconfigured  Result = "User Search Parameters Misconfigured"
+	GroupSearchParamsMisconfigured Result = "Group Search Parameters Misconfigured"
+	UserDNLookupError              Result = "User DN Lookup Error"
+	GroupMembershipsLookupError    Result = "Group Memberships Lookup Error"
+)
+
+// Validation returns feedback on the configuration. The `Suggestion` field
+// needs to be "printed" for friendly display (it can contain escaped newlines
+// `\n`).
+type Validation struct {
+	Result     Result
+	Detail     string
+	Suggestion string
+	ErrCause   error
+}
+
+// Error instance for Validation.
+func (v Validation) Error() string {
+	if v.Result == ConfigOk {
+		return ""
+	}
+	return fmt.Sprintf("%s: %s", string(v.Result), v.Detail)
+}
+
+// IsOk - returns if the validation succeeded.
+func (v Validation) IsOk() bool {
+	return v.Result == ConfigOk
+}
+
+// UserLookupResult returns the DN found for the test user and their group
+// memberships.
+type UserLookupResult struct {
+	DN                 string
+	GroupDNMemberships []string
+}
+
+// Validate validates the LDAP configuration. It can be called with any subset
+// of configuration parameters provided by the user - it will return
+// information on what needs to be done to fix the problem if any.
+//
+// This function updates the UserDNSearchBaseDistNames and
+// GroupSearchBaseDistNames fields of the Config - however this an idempotent
+// operation. This is done to support configuration validation in Console/mc and
+// for tests.
+func (l *Config) Validate() Validation {
+	if !l.Enabled {
+		return Validation{Result: ConfigOk, Detail: "Config is not enabled"}
+	}
+
+	if l.ServerAddr == "" {
+		return Validation{
+			Result:     ConnectivityError,
+			Detail:     "Address is empty",
+			Suggestion: "Set a server address.",
+		}
+	}
+
+	conn, err := l.Connect()
+	if err != nil {
+		return Validation{
+			Result:   ConnectivityError,
+			Detail:   fmt.Sprintf("Could not connect to LDAP server: %v", err),
+			ErrCause: err,
+			Suggestion: `Check:
+    (1) server address
+    (2) TLS parameters, and
+    (3) LDAP server's TLS certificate is trusted by MinIO (when using TLS - highly recommended)`,
+		}
+	}
+	defer conn.Close()
+
+	if l.LookupBindDN == "" {
+		return Validation{
+			Result:     LookupBindError,
+			Detail:     "Lookup Bind UserDN not specified",
+			Suggestion: "Specify LDAP service account credentials for performing lookups.",
+		}
+	}
+	if err := l.lookupBind(conn); err != nil {
+		return Validation{
+			Result:     LookupBindError,
+			ErrCause:   err,
+			Detail:     fmt.Sprintf("Error connecting as LDAP Lookup Bind user: %v", err),
+			Suggestion: "Check LDAP Lookup Bind user credentials and if user is allowed to login",
+		}
+	}
+
+	// Validate User Lookup parameters
+	if l.UserDNSearchBaseDistName == "" {
+		return Validation{
+			Result:     UserSearchParamsMisconfigured,
+			Detail:     "UserDN search base is empty",
+			Suggestion: "Set the UserDN search base to the DN of the directory subtree where users are present",
+		}
+	}
+	l.UserDNSearchBaseDistNames = strings.Split(l.UserDNSearchBaseDistName, dnDelimiter)
+
+	if l.UserDNSearchFilter == "" {
+		return Validation{
+			Result: UserSearchParamsMisconfigured,
+			Detail: "UserDN search filter is empty",
+			Suggestion: `Set the UserDN search filter template:
+    Use "%s" - it will be replaced by the login user name and sent to the LDAP server.
+    For example: "(uid=%s)"`,
+		}
+	}
+	if strings.Contains(l.UserDNSearchFilter, "%d") {
+		return Validation{
+			Result: UserSearchParamsMisconfigured,
+			Detail: "User DN search filter contains `%d`",
+			Suggestion: `User DN search filter is a template where "%s" is replaced by the login username.
+    "%d" is not supported here.
+    Please provide a search filter containing "%s"`,
+		}
+	}
+	if !strings.Contains(l.UserDNSearchFilter, "%s") {
+		return Validation{
+			Result: UserSearchParamsMisconfigured,
+			Detail: "User DN search filter does not contain `%s`",
+			Suggestion: `During login, the user's DN is looked up using the search filter template:
+    "%s" gets replaced by the given username - it must be used.
+    Enter an LDAP search filter containing "%s"`,
+		}
+	}
+
+	// If group lookup is not configured, it's ok.
+	if l.GroupSearchBaseDistName != "" || l.GroupSearchFilter != "" {
+
+		// Validate Group Search parameters as they are given.
+		if l.GroupSearchBaseDistName == "" {
+			return Validation{
+				Result: GroupSearchParamsMisconfigured,
+				Detail: "Group Search Base DN is required.",
+				Suggestion: `Since you entered a value for the Group Search Filter - enter a value for the Group Search Base DN too:
+    Enter this value as the DN of the subtree where groups will be found.`,
+			}
+		}
+		l.GroupSearchBaseDistNames = strings.Split(l.GroupSearchBaseDistName, dnDelimiter)
+
+		if l.GroupSearchFilter == "" {
+			return Validation{
+				Result: GroupSearchParamsMisconfigured,
+				Detail: "Group Search Filter is required.",
+				Suggestion: `Since you entered a value for the Group Search Base DN - enter a value for the Group Search Filter too. This is a template where, before the query is sent to the server:
+    "%s" is replaced with the login username;
+    "%d" is replaced with the DN of the login user.
+    For example: "(&(objectclass=groupOfNames)(memberUid=%s))"`,
+			}
+		}
+
+		if !strings.Contains(l.GroupSearchFilter, "%d") && !strings.Contains(l.GroupSearchFilter, "%s") {
+			return Validation{
+				Result: GroupSearchParamsMisconfigured,
+				Detail: `GroupSearchFilter must contain at least one of "%s" or "%d"`,
+				Suggestion: `During group membership lookup the group search filter template is used:
+    "%s" gets replaced by the given username, and
+    "%d" gets replaced by the user's DN.
+    Either one is needed to find only groups that the user is a member of.
+    Enter an LDAP search filter template using at least one of these.`,
+			}
+		}
+	}
+
+	return Validation{
+		Result: ConfigOk,
+	}
+}
+
+// ValidateLookup takes a test username and performs user and group lookup (if
+// configured) and returns the result. It is to validate the LDAP configuration.
+// The lookup is performed without requiring the password for the test user -
+// and so can be used to test any LDAP user intending to use MinIO.
+func (l *Config) ValidateLookup(testUsername string) (*UserLookupResult, Validation) {
+	if testUsername == "" {
+		return nil, Validation{
+			Result: UserDNLookupError,
+			Detail: "Provided username is empty",
+		}
+	}
+
+	if r := l.Validate(); !r.IsOk() {
+		return nil, r
+	}
+
+	conn, err := l.Connect()
+	if err != nil {
+		return nil, Validation{
+			Result:   ConnectivityError,
+			Detail:   fmt.Sprintf("Could not connect to LDAP server: %v", err),
+			ErrCause: err,
+			Suggestion: `Check:
+    (1) server address
+    (2) TLS parameters, and
+    (3) LDAP server's TLS certificate is trusted by MinIO (when using TLS - highly recommended)`,
+		}
+	}
+	defer conn.Close()
+
+	if err := l.lookupBind(conn); err != nil {
+		return nil, Validation{
+			Result:     LookupBindError,
+			ErrCause:   err,
+			Detail:     fmt.Sprintf("Error connecting as LDAP Lookup Bind user: %v", err),
+			Suggestion: "Check LDAP Lookup Bind user credentials and if user is allowed to login",
+		}
+	}
+
+	// Lookup the given username.
+	dn, err := l.lookupUserDN(conn, testUsername)
+	if err != nil {
+		return nil, Validation{
+			Result:   UserDNLookupError,
+			Detail:   fmt.Sprintf("Got an error when looking up user (%s) DN: %v", testUsername, err),
+			ErrCause: err,
+			Suggestion: `Check if this is a temporary error and try again.
+    Perhaps there is an error in the user search filter or user search base DN.`,
+		}
+	}
+
+	// Lookup groups.
+	groups, err := l.searchForUserGroups(conn, testUsername, dn)
+	if err != nil {
+		return nil, Validation{
+			Result:   GroupMembershipsLookupError,
+			Detail:   fmt.Sprintf("Got an error when looking up groups for user(=>%s, dn=>%s): %v", testUsername, dn, err),
+			ErrCause: err,
+			Suggestion: `Check if this is a temporary error and try again.
+    Perhaps there is an error in the group search filter or group search base DN.`,
+		}
+	}
+
+	return &UserLookupResult{
+			DN:                 dn,
+			GroupDNMemberships: groups,
+		}, Validation{
+			Result: ConfigOk,
+			Detail: "User lookup done.",
+		}
+}

internal/config/identity/ldap/config-validator_test.go

@@ -0,0 +1,189 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ldap
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/minio/minio-go/v7/pkg/set"
+)
+
+const (
+	EnvTestLDAPServer = "LDAP_TEST_SERVER"
+)
+
+func TestConfigValidator(t *testing.T) {
+	ldapServer := os.Getenv(EnvTestLDAPServer)
+	if ldapServer == "" {
+		t.Skip()
+	}
+	testCases := []struct {
+		cfg            Config
+		expectedResult Result
+	}{
+		{
+			cfg: func() Config {
+				v := Config{Enabled: true}
+				return v
+			}(),
+			expectedResult: ConnectivityError,
+		},
+		{
+			cfg: func() Config {
+				v := Config{Enabled: true}
+				v.ServerAddr = ldapServer
+				return v
+			}(),
+			expectedResult: ConnectivityError,
+		},
+		{
+			cfg: func() Config {
+				v := Config{Enabled: true}
+				v.ServerAddr = ldapServer
+				v.serverInsecure = true
+				return v
+			}(),
+			expectedResult: LookupBindError,
+		},
+		{
+			cfg: func() Config {
+				v := Config{Enabled: true}
+				v.ServerAddr = ldapServer
+				v.serverInsecure = true
+				v.LookupBindDN = "cn=admin,dc=min,dc=io"
+				v.LookupBindPassword = "admin1"
+				return v
+			}(),
+			expectedResult: LookupBindError,
+		},
+		{ // Case 4
+			cfg: func() Config {
+				v := Config{Enabled: true}
+				v.ServerAddr = ldapServer
+				v.serverInsecure = true
+				v.LookupBindDN = "cn=admin,dc=min,dc=io"
+				v.LookupBindPassword = "admin"
+				return v
+			}(),
+			expectedResult: UserSearchParamsMisconfigured,
+		},
+		{
+			cfg: func() Config {
+				v := Config{Enabled: true}
+				v.ServerAddr = ldapServer
+				v.serverInsecure = true
+				v.LookupBindDN = "cn=admin,dc=min,dc=io"
+				v.LookupBindPassword = "admin"
+				v.UserDNSearchFilter = "(uid=x)"
+				v.UserDNSearchBaseDistName = "dc=min,dc=io"
+				return v
+			}(),
+			expectedResult: UserSearchParamsMisconfigured,
+		},
+		{
+			cfg: func() Config {
+				v := Config{Enabled: true}
+				v.ServerAddr = ldapServer
+				v.serverInsecure = true
+				v.LookupBindDN = "cn=admin,dc=min,dc=io"
+				v.LookupBindPassword = "admin"
+				v.UserDNSearchFilter = "(uid=%s)"
+				v.UserDNSearchBaseDistName = "dc=min,dc=io"
+				return v
+			}(),
+			expectedResult: ConfigOk,
+		},
+		{ // Case 7
+			cfg: func() Config {
+				v := Config{Enabled: true}
+				v.ServerAddr = ldapServer
+				v.serverInsecure = true
+				v.LookupBindDN = "cn=admin,dc=min,dc=io"
+				v.LookupBindPassword = "admin"
+				v.UserDNSearchFilter = "(uid=%s)"
+				v.UserDNSearchBaseDistName = "dc=min,dc=io"
+				v.GroupSearchBaseDistName = "ou=swengg,dc=min,dc=io"
+				v.GroupSearchFilter = "(&(objectclass=groupofnames)(member=x))"
+				return v
+			}(),
+			expectedResult: GroupSearchParamsMisconfigured,
+		},
+		{
+			cfg: func() Config {
+				v := Config{Enabled: true}
+				v.ServerAddr = ldapServer
+				v.serverInsecure = true
+				v.LookupBindDN = "cn=admin,dc=min,dc=io"
+				v.LookupBindPassword = "admin"
+				v.UserDNSearchFilter = "(uid=%s)"
+				v.UserDNSearchBaseDistName = "dc=min,dc=io"
+				v.GroupSearchFilter = "(&(objectclass=groupofnames)(member=x))"
+				return v
+			}(),
+			expectedResult: GroupSearchParamsMisconfigured,
+		},
+		{ // Case 9
+			cfg: func() Config {
+				v := Config{Enabled: true}
+				v.ServerAddr = ldapServer
+				v.serverInsecure = true
+				v.LookupBindDN = "cn=admin,dc=min,dc=io"
+				v.LookupBindPassword = "admin"
+				v.UserDNSearchFilter = "(uid=%s)"
+				v.UserDNSearchBaseDistName = "dc=min,dc=io"
+				v.GroupSearchBaseDistName = "ou=swengg,dc=min,dc=io"
+				v.GroupSearchFilter = "(&(objectclass=groupofnames)(member=%d))"
+				return v
+			}(),
+			expectedResult: ConfigOk,
+		},
+	}
+
+	expectedDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io"
+	expectedGroups := set.CreateStringSet(
+		"cn=projecta,ou=groups,ou=swengg,dc=min,dc=io",
+		"cn=projectb,ou=groups,ou=swengg,dc=min,dc=io",
+	)
+
+	for i, test := range testCases {
+		result := test.cfg.Validate()
+		if result.Result != test.expectedResult {
+			fmt.Printf("Result: %#v\n", result)
+			t.Fatalf("Case %d: Got `%s` expected `%s`", i, result.Result, string(test.expectedResult))
+		}
+		if result.IsOk() {
+			lookupResult, validationResult := test.cfg.ValidateLookup("dillon")
+			if !validationResult.IsOk() {
+				t.Fatalf("Case %d: Got unexpected validation failure: %#v\n", i, validationResult)
+			}
+			if lookupResult.DN != expectedDN {
+				t.Fatalf("Case %d: Got unexpected DN: %v", i, lookupResult.DN)
+			}
+
+			if test.cfg.GroupSearchFilter == "" {
+				continue
+			}
+
+			if !set.CreateStringSet(lookupResult.GroupDNMemberships...).Equals(expectedGroups) {
+				t.Fatalf("Case %d: Got unexpected groups: %v", i, lookupResult.GroupDNMemberships)
+			}
+		}
+	}
+}

internal/config/identity/ldap/config.go

@@ -18,18 +18,9 @@
 package ldap
 
 import (
-	"crypto/tls"
 	"crypto/x509"
-	"errors"
-	"fmt"
-	"net"
-	"strconv"
-	"strings"
 	"time"
 
-	ldap "github.com/go-ldap/ldap/v3"
-	"github.com/minio/minio-go/v7/pkg/set"
-	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/pkg/env"
 )
@@ -52,12 +43,12 @@ type Config struct {
 
 	// User DN search parameters
 	UserDNSearchBaseDistName  string   `json:"userDNSearchBaseDN"`
-	UserDNSearchBaseDistNames []string `json:"-"`
+	UserDNSearchBaseDistNames []string `json:"-"` // Generated field
 	UserDNSearchFilter        string   `json:"userDNSearchFilter"`
 
 	// Group search parameters
 	GroupSearchBaseDistName  string   `json:"groupSearchBaseDN"`
-	GroupSearchBaseDistNames []string `json:"-"`
+	GroupSearchBaseDistNames []string `json:"-"` // Generated field
 	GroupSearchFilter        string   `json:"groupSearchFilter"`
 
 	// Lookup bind LDAP service account
@@ -151,318 +142,6 @@ var (
 	}
 )
 
-func getGroups(conn *ldap.Conn, sreq *ldap.SearchRequest) ([]string, error) {
-	var groups []string
-	sres, err := conn.Search(sreq)
-	if err != nil {
-		// Check if there is no matching result and return empty slice.
-		// Ref: https://ldap.com/ldap-result-code-reference/
-		if ldap.IsErrorWithCode(err, 32) {
-			return nil, nil
-		}
-		return nil, err
-	}
-	for _, entry := range sres.Entries {
-		// We only queried one attribute,
-		// so we only look up the first one.
-		groups = append(groups, entry.DN)
-	}
-	return groups, nil
-}
-
-func (l *Config) lookupBind(conn *ldap.Conn) error {
-	var err error
-	if l.LookupBindPassword == "" {
-		err = conn.UnauthenticatedBind(l.LookupBindDN)
-	} else {
-		err = conn.Bind(l.LookupBindDN, l.LookupBindPassword)
-	}
-	if ldap.IsErrorWithCode(err, 49) {
-		return fmt.Errorf("LDAP Lookup Bind user invalid credentials error: %w", err)
-	}
-	return err
-}
-
-// lookupUserDN searches for the DN of the user given their username. conn is
-// assumed to be using the lookup bind service account. It is required that the
-// search result in at most one result.
-func (l *Config) lookupUserDN(conn *ldap.Conn, username string) (string, error) {
-	filter := strings.ReplaceAll(l.UserDNSearchFilter, "%s", ldap.EscapeFilter(username))
-	var foundDistNames []string
-	for _, userSearchBase := range l.UserDNSearchBaseDistNames {
-		searchRequest := ldap.NewSearchRequest(
-			userSearchBase,
-			ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
-			filter,
-			[]string{}, // only need DN, so no pass no attributes here
-			nil,
-		)
-
-		searchResult, err := conn.Search(searchRequest)
-		if err != nil {
-			return "", err
-		}
-
-		for _, entry := range searchResult.Entries {
-			foundDistNames = append(foundDistNames, entry.DN)
-		}
-	}
-	if len(foundDistNames) == 0 {
-		return "", fmt.Errorf("User DN for %s not found", username)
-	}
-	if len(foundDistNames) != 1 {
-		return "", fmt.Errorf("Multiple DNs for %s found - please fix the search filter", username)
-	}
-	return foundDistNames[0], nil
-}
-
-func (l *Config) searchForUserGroups(conn *ldap.Conn, username, bindDN string) ([]string, error) {
-	// User groups lookup.
-	var groups []string
-	if l.GroupSearchFilter != "" {
-		for _, groupSearchBase := range l.GroupSearchBaseDistNames {
-			filter := strings.ReplaceAll(l.GroupSearchFilter, "%s", ldap.EscapeFilter(username))
-			filter = strings.ReplaceAll(filter, "%d", ldap.EscapeFilter(bindDN))
-			searchRequest := ldap.NewSearchRequest(
-				groupSearchBase,
-				ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
-				filter,
-				nil,
-				nil,
-			)
-
-			var newGroups []string
-			newGroups, err := getGroups(conn, searchRequest)
-			if err != nil {
-				errRet := fmt.Errorf("Error finding groups of %s: %w", bindDN, err)
-				return nil, errRet
-			}
-
-			groups = append(groups, newGroups...)
-		}
-	}
-
-	return groups, nil
-}
-
-// LookupUserDN searches for the full DN and groups of a given username
-func (l *Config) LookupUserDN(username string) (string, []string, error) {
-	conn, err := l.Connect()
-	if err != nil {
-		return "", nil, err
-	}
-	defer conn.Close()
-
-	// Bind to the lookup user account
-	if err = l.lookupBind(conn); err != nil {
-		return "", nil, err
-	}
-
-	// Lookup user DN
-	bindDN, err := l.lookupUserDN(conn, username)
-	if err != nil {
-		errRet := fmt.Errorf("Unable to find user DN: %w", err)
-		return "", nil, errRet
-	}
-
-	groups, err := l.searchForUserGroups(conn, username, bindDN)
-	if err != nil {
-		return "", nil, err
-	}
-
-	return bindDN, groups, nil
-}
-
-// Bind - binds to ldap, searches LDAP and returns the distinguished name of the
-// user and the list of groups.
-func (l *Config) Bind(username, password string) (string, []string, error) {
-	conn, err := l.Connect()
-	if err != nil {
-		return "", nil, err
-	}
-	defer conn.Close()
-
-	var bindDN string
-	// Bind to the lookup user account
-	if err = l.lookupBind(conn); err != nil {
-		return "", nil, err
-	}
-
-	// Lookup user DN
-	bindDN, err = l.lookupUserDN(conn, username)
-	if err != nil {
-		errRet := fmt.Errorf("Unable to find user DN: %w", err)
-		return "", nil, errRet
-	}
-
-	// Authenticate the user credentials.
-	err = conn.Bind(bindDN, password)
-	if err != nil {
-		errRet := fmt.Errorf("LDAP auth failed for DN %s: %w", bindDN, err)
-		return "", nil, errRet
-	}
-
-	// Bind to the lookup user account again to perform group search.
-	if err = l.lookupBind(conn); err != nil {
-		return "", nil, err
-	}
-
-	// User groups lookup.
-	groups, err := l.searchForUserGroups(conn, username, bindDN)
-	if err != nil {
-		return "", nil, err
-	}
-
-	return bindDN, groups, nil
-}
-
-// Connect connect to ldap server.
-func (l *Config) Connect() (ldapConn *ldap.Conn, err error) {
-	if l == nil {
-		return nil, errors.New("LDAP is not configured")
-	}
-
-	_, _, err = net.SplitHostPort(l.ServerAddr)
-	if err != nil {
-		// User default LDAP port if none specified "636"
-		l.ServerAddr = net.JoinHostPort(l.ServerAddr, "636")
-	}
-
-	if l.serverInsecure {
-		return ldap.Dial("tcp", l.ServerAddr)
-	}
-
-	tlsConfig := &tls.Config{
-		InsecureSkipVerify: l.tlsSkipVerify,
-		RootCAs:            l.rootCAs,
-	}
-
-	if l.serverStartTLS {
-		conn, err := ldap.Dial("tcp", l.ServerAddr)
-		if err != nil {
-			return nil, err
-		}
-		err = conn.StartTLS(tlsConfig)
-		return conn, err
-	}
-
-	return ldap.DialTLS("tcp", l.ServerAddr, tlsConfig)
-}
-
-// GetExpiryDuration - return parsed expiry duration.
-func (l Config) GetExpiryDuration(dsecs string) (time.Duration, error) {
-	if dsecs == "" {
-		return l.stsExpiryDuration, nil
-	}
-
-	d, err := strconv.Atoi(dsecs)
-	if err != nil {
-		return 0, auth.ErrInvalidDuration
-	}
-
-	dur := time.Duration(d) * time.Second
-
-	if dur < minLDAPExpiry || dur > maxLDAPExpiry {
-		return 0, auth.ErrInvalidDuration
-	}
-	return dur, nil
-}
-
-func (l Config) testConnection() error {
-	conn, err := l.Connect()
-	if err != nil {
-		return fmt.Errorf("Error creating connection to LDAP server: %w", err)
-	}
-	defer conn.Close()
-
-	if err = l.lookupBind(conn); err != nil {
-		return fmt.Errorf("Error connecting as LDAP Lookup Bind user: %w", err)
-	}
-	return nil
-}
-
-// IsLDAPUserDN determines if the given string could be a user DN from LDAP.
-func (l Config) IsLDAPUserDN(user string) bool {
-	for _, baseDN := range l.UserDNSearchBaseDistNames {
-		if strings.HasSuffix(user, ","+baseDN) {
-			return true
-		}
-	}
-	return false
-}
-
-// GetNonEligibleUserDistNames - find user accounts (DNs) that are no longer
-// present in the LDAP server or do not meet filter criteria anymore
-func (l *Config) GetNonEligibleUserDistNames(userDistNames []string) ([]string, error) {
-	conn, err := l.Connect()
-	if err != nil {
-		return nil, err
-	}
-	defer conn.Close()
-
-	// Bind to the lookup user account
-	if err = l.lookupBind(conn); err != nil {
-		return nil, err
-	}
-
-	// Evaluate the filter again with generic wildcard instead of  specific values
-	filter := strings.ReplaceAll(l.UserDNSearchFilter, "%s", "*")
-
-	nonExistentUsers := []string{}
-	for _, dn := range userDistNames {
-		searchRequest := ldap.NewSearchRequest(
-			dn,
-			ldap.ScopeBaseObject, ldap.NeverDerefAliases, 0, 0, false,
-			filter,
-			[]string{}, // only need DN, so pass no attributes here
-			nil,
-		)
-
-		searchResult, err := conn.Search(searchRequest)
-		if err != nil {
-			// Object does not exist error?
-			if ldap.IsErrorWithCode(err, 32) {
-				nonExistentUsers = append(nonExistentUsers, dn)
-				continue
-			}
-			return nil, err
-		}
-		if len(searchResult.Entries) == 0 {
-			// DN was not found - this means this user account is
-			// expired.
-			nonExistentUsers = append(nonExistentUsers, dn)
-		}
-	}
-	return nonExistentUsers, nil
-}
-
-// LookupGroupMemberships - for each DN finds the set of LDAP groups they are a
-// member of.
-func (l *Config) LookupGroupMemberships(userDistNames []string, userDNToUsernameMap map[string]string) (map[string]set.StringSet, error) {
-	conn, err := l.Connect()
-	if err != nil {
-		return nil, err
-	}
-	defer conn.Close()
-
-	// Bind to the lookup user account
-	if err = l.lookupBind(conn); err != nil {
-		return nil, err
-	}
-
-	res := make(map[string]set.StringSet, len(userDistNames))
-	for _, userDistName := range userDistNames {
-		username := userDNToUsernameMap[userDistName]
-		groups, err := l.searchForUserGroups(conn, username, userDistName)
-		if err != nil {
-			return nil, err
-		}
-		res[userDistName] = set.CreateStringSet(groups...)
-	}
-
-	return res, nil
-}
-
 // Enabled returns if LDAP config is enabled.
 func Enabled(kvs config.KVS) bool {
 	return kvs.Get(ServerAddr) != ""
@@ -480,6 +159,7 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 	if err = config.CheckValidKeys(config.IdentityLDAPSubSys, kvs, DefaultKVS); err != nil {
 		return l, err
 	}
+
 	ldapServer := env.Get(EnvServerAddr, kvs.Get(ServerAddr))
 	if ldapServer == "" {
 		return l, nil
@@ -510,44 +190,21 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 	}
 
 	// Lookup bind user configuration
-	lookupBindDN := env.Get(EnvLookupBindDN, kvs.Get(LookupBindDN))
-	if lookupBindDN == "" {
-		return l, errors.New("Lookup Bind DN is required")
-	}
-	lookupBindPassword := env.Get(EnvLookupBindPassword, kvs.Get(LookupBindPassword))
-	if lookupBindDN != "" {
-		l.LookupBindDN = lookupBindDN
-		l.LookupBindPassword = lookupBindPassword
-	}
-
-	// Test connection to LDAP server.
-	if err := l.testConnection(); err != nil {
-		return l, fmt.Errorf("Connection test for LDAP server failed: %w", err)
-	}
+	l.LookupBindDN = env.Get(EnvLookupBindDN, kvs.Get(LookupBindDN))
+	l.LookupBindPassword = env.Get(EnvLookupBindPassword, kvs.Get(LookupBindPassword))
 
 	// User DN search configuration
-	userDNSearchBaseDN := env.Get(EnvUserDNSearchBaseDN, kvs.Get(UserDNSearchBaseDN))
-	userDNSearchFilter := env.Get(EnvUserDNSearchFilter, kvs.Get(UserDNSearchFilter))
-	if userDNSearchFilter == "" || userDNSearchBaseDN == "" {
-		return l, errors.New("UserDN search base DN and UserDN search filter are both required")
-	}
-	l.UserDNSearchBaseDistName = userDNSearchBaseDN
-	l.UserDNSearchBaseDistNames = strings.Split(userDNSearchBaseDN, dnDelimiter)
-	l.UserDNSearchFilter = userDNSearchFilter
+	l.UserDNSearchFilter = env.Get(EnvUserDNSearchFilter, kvs.Get(UserDNSearchFilter))
+	l.UserDNSearchBaseDistName = env.Get(EnvUserDNSearchBaseDN, kvs.Get(UserDNSearchBaseDN))
 
 	// Group search params configuration
-	grpSearchFilter := env.Get(EnvGroupSearchFilter, kvs.Get(GroupSearchFilter))
-	grpSearchBaseDN := env.Get(EnvGroupSearchBaseDN, kvs.Get(GroupSearchBaseDN))
+	l.GroupSearchFilter = env.Get(EnvGroupSearchFilter, kvs.Get(GroupSearchFilter))
+	l.GroupSearchBaseDistName = env.Get(EnvGroupSearchBaseDN, kvs.Get(GroupSearchBaseDN))
 
-	// Either all group params must be set or none must be set.
-	if (grpSearchFilter != "" && grpSearchBaseDN == "") || (grpSearchFilter == "" && grpSearchBaseDN != "") {
-		return l, errors.New("All group related parameters must be set")
-	}
-
-	if grpSearchFilter != "" {
-		l.GroupSearchFilter = grpSearchFilter
-		l.GroupSearchBaseDistName = grpSearchBaseDN
-		l.GroupSearchBaseDistNames = strings.Split(l.GroupSearchBaseDistName, dnDelimiter)
+	// Validate and test configuration.
+	valResult := l.Validate()
+	if !valResult.IsOk() {
+		return l, valResult
 	}
 
 	return l, nil

internal/config/identity/ldap/help.go

@@ -31,7 +31,6 @@ var (
 		config.HelpKV{
 			Key:         LookupBindDN,
 			Description: `DN for LDAP read-only service account used to perform DN and group lookups`,
-			Optional:    true,
 			Type:        "string",
 			Sensitive:   true,
 		},
@@ -45,13 +44,11 @@ var (
 		config.HelpKV{
 			Key:         UserDNSearchBaseDN,
 			Description: `";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"`,
-			Optional:    true,
 			Type:        "list",
 		},
 		config.HelpKV{
 			Key:         UserDNSearchFilter,
 			Description: `Search filter to lookup user DN`,
-			Optional:    true,
 			Type:        "string",
 		},
 		config.HelpKV{

internal/config/identity/ldap/ldap.go

@@ -0,0 +1,331 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ldap
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	ldap "github.com/go-ldap/ldap/v3"
+	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/auth"
+)
+
+func getGroups(conn *ldap.Conn, sreq *ldap.SearchRequest) ([]string, error) {
+	var groups []string
+	sres, err := conn.Search(sreq)
+	if err != nil {
+		// Check if there is no matching result and return empty slice.
+		// Ref: https://ldap.com/ldap-result-code-reference/
+		if ldap.IsErrorWithCode(err, 32) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	for _, entry := range sres.Entries {
+		// We only queried one attribute,
+		// so we only look up the first one.
+		groups = append(groups, entry.DN)
+	}
+	return groups, nil
+}
+
+func (l *Config) lookupBind(conn *ldap.Conn) error {
+	var err error
+	if l.LookupBindPassword == "" {
+		err = conn.UnauthenticatedBind(l.LookupBindDN)
+	} else {
+		err = conn.Bind(l.LookupBindDN, l.LookupBindPassword)
+	}
+	if ldap.IsErrorWithCode(err, 49) {
+		return fmt.Errorf("LDAP Lookup Bind user invalid credentials error: %w", err)
+	}
+	return err
+}
+
+// lookupUserDN searches for the DN of the user given their username. conn is
+// assumed to be using the lookup bind service account. It is required that the
+// search result in at most one result.
+func (l *Config) lookupUserDN(conn *ldap.Conn, username string) (string, error) {
+	filter := strings.ReplaceAll(l.UserDNSearchFilter, "%s", ldap.EscapeFilter(username))
+	var foundDistNames []string
+	for _, userSearchBase := range l.UserDNSearchBaseDistNames {
+		searchRequest := ldap.NewSearchRequest(
+			userSearchBase,
+			ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+			filter,
+			[]string{}, // only need DN, so no pass no attributes here
+			nil,
+		)
+
+		searchResult, err := conn.Search(searchRequest)
+		if err != nil {
+			return "", err
+		}
+
+		for _, entry := range searchResult.Entries {
+			foundDistNames = append(foundDistNames, entry.DN)
+		}
+	}
+	if len(foundDistNames) == 0 {
+		return "", fmt.Errorf("User DN for %s not found", username)
+	}
+	if len(foundDistNames) != 1 {
+		return "", fmt.Errorf("Multiple DNs for %s found - please fix the search filter", username)
+	}
+	return foundDistNames[0], nil
+}
+
+func (l *Config) searchForUserGroups(conn *ldap.Conn, username, bindDN string) ([]string, error) {
+	// User groups lookup.
+	var groups []string
+	if l.GroupSearchFilter != "" {
+		for _, groupSearchBase := range l.GroupSearchBaseDistNames {
+			filter := strings.ReplaceAll(l.GroupSearchFilter, "%s", ldap.EscapeFilter(username))
+			filter = strings.ReplaceAll(filter, "%d", ldap.EscapeFilter(bindDN))
+			searchRequest := ldap.NewSearchRequest(
+				groupSearchBase,
+				ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
+				filter,
+				nil,
+				nil,
+			)
+
+			var newGroups []string
+			newGroups, err := getGroups(conn, searchRequest)
+			if err != nil {
+				errRet := fmt.Errorf("Error finding groups of %s: %w", bindDN, err)
+				return nil, errRet
+			}
+
+			groups = append(groups, newGroups...)
+		}
+	}
+
+	return groups, nil
+}
+
+// LookupUserDN searches for the full DN and groups of a given username
+func (l *Config) LookupUserDN(username string) (string, []string, error) {
+	conn, err := l.Connect()
+	if err != nil {
+		return "", nil, err
+	}
+	defer conn.Close()
+
+	// Bind to the lookup user account
+	if err = l.lookupBind(conn); err != nil {
+		return "", nil, err
+	}
+
+	// Lookup user DN
+	bindDN, err := l.lookupUserDN(conn, username)
+	if err != nil {
+		errRet := fmt.Errorf("Unable to find user DN: %w", err)
+		return "", nil, errRet
+	}
+
+	groups, err := l.searchForUserGroups(conn, username, bindDN)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return bindDN, groups, nil
+}
+
+// Bind - binds to ldap, searches LDAP and returns the distinguished name of the
+// user and the list of groups.
+func (l *Config) Bind(username, password string) (string, []string, error) {
+	conn, err := l.Connect()
+	if err != nil {
+		return "", nil, err
+	}
+	defer conn.Close()
+
+	var bindDN string
+	// Bind to the lookup user account
+	if err = l.lookupBind(conn); err != nil {
+		return "", nil, err
+	}
+
+	// Lookup user DN
+	bindDN, err = l.lookupUserDN(conn, username)
+	if err != nil {
+		errRet := fmt.Errorf("Unable to find user DN: %w", err)
+		return "", nil, errRet
+	}
+
+	// Authenticate the user credentials.
+	err = conn.Bind(bindDN, password)
+	if err != nil {
+		errRet := fmt.Errorf("LDAP auth failed for DN %s: %w", bindDN, err)
+		return "", nil, errRet
+	}
+
+	// Bind to the lookup user account again to perform group search.
+	if err = l.lookupBind(conn); err != nil {
+		return "", nil, err
+	}
+
+	// User groups lookup.
+	groups, err := l.searchForUserGroups(conn, username, bindDN)
+	if err != nil {
+		return "", nil, err
+	}
+
+	return bindDN, groups, nil
+}
+
+// Connect connect to ldap server.
+func (l *Config) Connect() (ldapConn *ldap.Conn, err error) {
+	if l == nil {
+		return nil, errors.New("LDAP is not configured")
+	}
+
+	_, _, err = net.SplitHostPort(l.ServerAddr)
+	if err != nil {
+		// User default LDAP port if none specified "636"
+		l.ServerAddr = net.JoinHostPort(l.ServerAddr, "636")
+	}
+
+	if l.serverInsecure {
+		return ldap.Dial("tcp", l.ServerAddr)
+	}
+
+	tlsConfig := &tls.Config{
+		InsecureSkipVerify: l.tlsSkipVerify,
+		RootCAs:            l.rootCAs,
+	}
+
+	if l.serverStartTLS {
+		conn, err := ldap.Dial("tcp", l.ServerAddr)
+		if err != nil {
+			return nil, err
+		}
+		err = conn.StartTLS(tlsConfig)
+		return conn, err
+	}
+
+	return ldap.DialTLS("tcp", l.ServerAddr, tlsConfig)
+}
+
+// GetExpiryDuration - return parsed expiry duration.
+func (l Config) GetExpiryDuration(dsecs string) (time.Duration, error) {
+	if dsecs == "" {
+		return l.stsExpiryDuration, nil
+	}
+
+	d, err := strconv.Atoi(dsecs)
+	if err != nil {
+		return 0, auth.ErrInvalidDuration
+	}
+
+	dur := time.Duration(d) * time.Second
+
+	if dur < minLDAPExpiry || dur > maxLDAPExpiry {
+		return 0, auth.ErrInvalidDuration
+	}
+	return dur, nil
+}
+
+// IsLDAPUserDN determines if the given string could be a user DN from LDAP.
+func (l Config) IsLDAPUserDN(user string) bool {
+	for _, baseDN := range l.UserDNSearchBaseDistNames {
+		if strings.HasSuffix(user, ","+baseDN) {
+			return true
+		}
+	}
+	return false
+}
+
+// GetNonEligibleUserDistNames - find user accounts (DNs) that are no longer
+// present in the LDAP server or do not meet filter criteria anymore
+func (l *Config) GetNonEligibleUserDistNames(userDistNames []string) ([]string, error) {
+	conn, err := l.Connect()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	// Bind to the lookup user account
+	if err = l.lookupBind(conn); err != nil {
+		return nil, err
+	}
+
+	// Evaluate the filter again with generic wildcard instead of  specific values
+	filter := strings.ReplaceAll(l.UserDNSearchFilter, "%s", "*")
+
+	nonExistentUsers := []string{}
+	for _, dn := range userDistNames {
+		searchRequest := ldap.NewSearchRequest(
+			dn,
+			ldap.ScopeBaseObject, ldap.NeverDerefAliases, 0, 0, false,
+			filter,
+			[]string{}, // only need DN, so pass no attributes here
+			nil,
+		)
+
+		searchResult, err := conn.Search(searchRequest)
+		if err != nil {
+			// Object does not exist error?
+			if ldap.IsErrorWithCode(err, 32) {
+				nonExistentUsers = append(nonExistentUsers, dn)
+				continue
+			}
+			return nil, err
+		}
+		if len(searchResult.Entries) == 0 {
+			// DN was not found - this means this user account is
+			// expired.
+			nonExistentUsers = append(nonExistentUsers, dn)
+		}
+	}
+	return nonExistentUsers, nil
+}
+
+// LookupGroupMemberships - for each DN finds the set of LDAP groups they are a
+// member of.
+func (l *Config) LookupGroupMemberships(userDistNames []string, userDNToUsernameMap map[string]string) (map[string]set.StringSet, error) {
+	conn, err := l.Connect()
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+
+	// Bind to the lookup user account
+	if err = l.lookupBind(conn); err != nil {
+		return nil, err
+	}
+
+	res := make(map[string]set.StringSet, len(userDistNames))
+	for _, userDistName := range userDistNames {
+		username := userDNToUsernameMap[userDistName]
+		groups, err := l.searchForUserGroups(conn, username, userDistName)
+		if err != nil {
+			return nil, err
+		}
+		res[userDistName] = set.CreateStringSet(groups...)
+	}
+
+	return res, nil
+}

internal/event/target/webhook.go

@@ -119,10 +119,17 @@ func (target *WebhookTarget) IsActive() (bool, error) {
 		}
 		return false, err
 	}
+	tokens := strings.Fields(target.args.AuthToken)
+	switch len(tokens) {
+	case 2:
+		req.Header.Set("Authorization", target.args.AuthToken)
+	case 1:
+		req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
+	}
 
 	resp, err := target.httpClient.Do(req)
 	if err != nil {
-		if xnet.IsNetworkOrHostDown(err, false) || errors.Is(err, context.DeadlineExceeded) {
+		if xnet.IsNetworkOrHostDown(err, true) {
 			return false, errNotConnected
 		}
 		return false, err
@@ -133,7 +140,8 @@ func (target *WebhookTarget) IsActive() (bool, error) {
 	return true, nil
 }
 
-// Save - saves the events to the store if queuestore is configured, which will be replayed when the wenhook connection is active.
+// Save - saves the events to the store if queuestore is configured,
+// which will be replayed when the webhook connection is active.
 func (target *WebhookTarget) Save(eventData event.Event) error {
 	if target.store != nil {
 		return target.store.Put(eventData)

internal/ioutil/ioutil.go

@@ -254,14 +254,14 @@ const DirectioAlignSize = 4096
 // used with DIRECT I/O based file descriptor and it is expected that
 // input writer *os.File not a generic io.Writer. Make sure to have
 // the file opened for writes with syscall.O_DIRECT flag.
-func CopyAligned(w *os.File, r io.Reader, alignedBuf []byte, totalSize int64) (int64, error) {
+func CopyAligned(w io.Writer, r io.Reader, alignedBuf []byte, totalSize int64, file *os.File) (int64, error) {
 	// Writes remaining bytes in the buffer.
-	writeUnaligned := func(w *os.File, buf []byte) (remainingWritten int64, err error) {
+	writeUnaligned := func(w io.Writer, buf []byte) (remainingWritten int64, err error) {
 		// Disable O_DIRECT on fd's on unaligned buffer
 		// perform an amortized Fdatasync(fd) on the fd at
 		// the end, this is performed by the caller before
 		// closing 'w'.
-		if err = disk.DisableDirectIO(w); err != nil {
+		if err = disk.DisableDirectIO(file); err != nil {
 			return remainingWritten, err
 		}
 		// Since w is *os.File io.Copy shall use ReadFrom() call.

internal/kernel/kernel.go

@@ -0,0 +1,130 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build linux
+// +build linux
+
+package kernel
+
+import (
+	"fmt"
+	"io/ioutil"
+	"regexp"
+	"strconv"
+	"strings"
+	"syscall"
+)
+
+var versionRegex = regexp.MustCompile(`^(\d+)\.(\d+).(\d+).*$`)
+
+// VersionFromRelease converts a release string with format
+// 4.4.2[-1] to a kernel version number in LINUX_VERSION_CODE format.
+// That is, for kernel "a.b.c", the version number will be (a<<16 + b<<8 + c)
+func VersionFromRelease(releaseString string) (uint32, error) {
+	versionParts := versionRegex.FindStringSubmatch(releaseString)
+	if len(versionParts) != 4 {
+		return 0, fmt.Errorf("got invalid release version %q (expected format '4.3.2-1')", releaseString)
+	}
+	major, err := strconv.Atoi(versionParts[1])
+	if err != nil {
+		return 0, err
+	}
+
+	minor, err := strconv.Atoi(versionParts[2])
+	if err != nil {
+		return 0, err
+	}
+
+	patch, err := strconv.Atoi(versionParts[3])
+	if err != nil {
+		return 0, err
+	}
+	return Version(major, minor, patch), nil
+}
+
+// Version implements KERNEL_VERSION equivalent macro
+// #define KERNEL_VERSION(a,b,c) (((a) << 16) + ((b) << 8) + ((c) > 255 ? 255 : (c)))
+func Version(major, minor, patch int) uint32 {
+	if patch > 255 {
+		patch = 255
+	}
+	out := major<<16 + minor<<8 + patch
+	return uint32(out)
+}
+
+func currentVersionUname() (uint32, error) {
+	var buf syscall.Utsname
+	if err := syscall.Uname(&buf); err != nil {
+		return 0, err
+	}
+	releaseString := strings.Trim(utsnameStr(buf.Release[:]), "\x00")
+	return VersionFromRelease(releaseString)
+}
+
+func currentVersionUbuntu() (uint32, error) {
+	procVersion, err := ioutil.ReadFile("/proc/version_signature")
+	if err != nil {
+		return 0, err
+	}
+	var u1, u2, releaseString string
+	_, err = fmt.Sscanf(string(procVersion), "%s %s %s", &u1, &u2, &releaseString)
+	if err != nil {
+		return 0, err
+	}
+	return VersionFromRelease(releaseString)
+}
+
+var debianVersionRegex = regexp.MustCompile(`.* SMP Debian (\d+\.\d+.\d+-\d+)(?:\+[[:alnum:]]*)?.*`)
+
+func parseDebianVersion(str string) (uint32, error) {
+	match := debianVersionRegex.FindStringSubmatch(str)
+	if len(match) != 2 {
+		return 0, fmt.Errorf("failed to parse kernel version from /proc/version: %s", str)
+	}
+	return VersionFromRelease(match[1])
+}
+
+func currentVersionDebian() (uint32, error) {
+	procVersion, err := ioutil.ReadFile("/proc/version")
+	if err != nil {
+		return 0, fmt.Errorf("error reading /proc/version: %s", err)
+	}
+
+	return parseDebianVersion(string(procVersion))
+}
+
+// CurrentVersion returns the current kernel version in
+// LINUX_VERSION_CODE format (see VersionFromRelease())
+func CurrentVersion() (uint32, error) {
+	// We need extra checks for Debian and Ubuntu as they modify
+	// the kernel version patch number for compatibility with
+	// out-of-tree modules. Linux perf tools do the same for Ubuntu
+	// systems: https://github.com/torvalds/linux/commit/d18acd15c
+	//
+	// See also:
+	// https://kernel-team.pages.debian.net/kernel-handbook/ch-versions.html
+	// https://wiki.ubuntu.com/Kernel/FAQ
+	version, err := currentVersionUbuntu()
+	if err == nil {
+		return version, nil
+	}
+	version, err = currentVersionDebian()
+	if err == nil {
+		return version, nil
+	}
+	return currentVersionUname()
+}

internal/kernel/kernel_other.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build !linux
+// +build !linux
+
+package kernel
+
+// VersionFromRelease only implemented on Linux.
+func VersionFromRelease(_ string) (uint32, error) {
+	return 0, nil
+}
+
+// Version only implemented on Linux.
+func Version(_, _, _ int) uint32 {
+	return 0
+}
+
+// CurrentVersion only implemented on Linux.
+func CurrentVersion() (uint32, error) {
+	return 0, nil
+}

internal/kernel/kernel_test.go

@@ -0,0 +1,88 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build linux
+// +build linux
+
+package kernel
+
+import "testing"
+
+var testData = []struct {
+	success       bool
+	releaseString string
+	kernelVersion uint32
+}{
+	{true, "4.1.2-3", 262402},
+	{true, "4.8.14-200.fc24.x86_64", 264206},
+	{true, "4.1.2-3foo", 262402},
+	{true, "4.1.2foo-1", 262402},
+	{true, "4.1.2-rkt-v1", 262402},
+	{true, "4.1.2rkt-v1", 262402},
+	{true, "4.1.2-3 foo", 262402},
+	{true, "3.10.0-1062.el7.x86_64", 199168},
+	{true, "3.0.0", 196608},
+	{true, "2.6.32", 132640},
+	{true, "5.13.0-30-generic", 331008},
+	{true, "5.10.0-1052-oem", 330240},
+	{false, "foo 4.1.2-3", 0},
+	{true, "4.1.2", 262402},
+	{false, ".4.1.2", 0},
+	{false, "4.1.", 0},
+	{false, "4.1", 0},
+}
+
+func TestVersionFromRelease(t *testing.T) {
+	for _, test := range testData {
+		version, err := VersionFromRelease(test.releaseString)
+		if err != nil && test.success {
+			t.Errorf("expected %q to success: %s", test.releaseString, err)
+		} else if err == nil && !test.success {
+			t.Errorf("expected %q to fail", test.releaseString)
+		}
+		if version != test.kernelVersion {
+			t.Errorf("expected kernel version %d, got %d", test.kernelVersion, version)
+		}
+	}
+}
+
+func TestParseDebianVersion(t *testing.T) {
+	for _, tc := range []struct {
+		success       bool
+		releaseString string
+		kernelVersion uint32
+	}{
+		// 4.9.168
+		{true, "Linux version 4.9.0-9-amd64 (debian-kernel@lists.debian.org) (gcc version 6.3.0 20170516 (Debian 6.3.0-18+deb9u1) ) #1 SMP Debian 4.9.168-1+deb9u3 (2019-06-16)", 264616},
+		// 4.9.88
+		{true, "Linux ip-10-0-75-49 4.9.0-6-amd64 #1 SMP Debian 4.9.88-1+deb9u1 (2018-05-07) x86_64 GNU/Linux", 264536},
+		// 3.0.4
+		{true, "Linux version 3.16.0-9-amd64 (debian-kernel@lists.debian.org) (gcc version 4.9.2 (Debian 4.9.2-10+deb8u2) ) #1 SMP Debian 3.16.68-1 (2019-05-22)", 200772},
+		// Invalid
+		{false, "Linux version 4.9.125-linuxkit (root@659b6d51c354) (gcc version 6.4.0 (Alpine 6.4.0) ) #1 SMP Fri Sep 7 08:20:28 UTC 2018", 0},
+	} {
+		version, err := parseDebianVersion(tc.releaseString)
+		if err != nil && tc.success {
+			t.Errorf("expected %q to success: %s", tc.releaseString, err)
+		} else if err == nil && !tc.success {
+			t.Errorf("expected %q to fail", tc.releaseString)
+		}
+		if version != tc.kernelVersion {
+			t.Errorf("expected kernel version %d, got %d", tc.kernelVersion, version)
+		}
+	}
+}

internal/kernel/kernel_utsname_int8.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build (linux && 386) || (linux && amd64) || (linux && arm64) || (linux && mips64) || (linux && mips)
+// +build linux,386 linux,amd64 linux,arm64 linux,mips64 linux,mips
+
+package kernel
+
+func utsnameStr(in []int8) string {
+	out := make([]byte, 0, len(in))
+	for i := 0; i < len(in); i++ {
+		if in[i] == 0x00 {
+			break
+		}
+		out = append(out, byte(in[i]))
+	}
+	return string(out)
+}

internal/kernel/kernel_utsname_uint8.go

@@ -0,0 +1,32 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+//go:build (linux && arm) || (linux && ppc64) || (linux && ppc64le) || (linux && s390x)
+// +build linux,arm linux,ppc64 linux,ppc64le linux,s390x
+
+package kernel
+
+func utsnameStr(in []uint8) string {
+	out := make([]byte, 0, len(in))
+	for i := 0; i < len(in); i++ {
+		if in[i] == 0x00 {
+			break
+		}
+		out = append(out, byte(in[i]))
+	}
+	return string(out)
+}

internal/logger/logger.go

@@ -360,7 +360,10 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) {
 	// Iterate over all logger targets to send the log entry
 	for _, t := range SystemTargets() {
 		if err := t.Send(entry, entry.LogKind); err != nil {
-			LogAlwaysIf(context.Background(), fmt.Errorf("event(%v) was not sent to Logger target (%v): %v", entry, t, err), entry.LogKind)
+			if consoleTgt != nil {
+				entry.Trace.Message = fmt.Sprintf("event(%#v) was not sent to Logger target (%#v): %#v", entry, t, err)
+				consoleTgt.Send(entry, entry.LogKind)
+			}
 		}
 	}
 }

internal/logger/targets.go

@@ -46,7 +46,11 @@ var (
 	// Must be immutable at all times.
 	// Can be swapped to another while holding swapMu
 	systemTargets = []Target{}
-	nTargets      int32 // atomic count of len(targets)
+
+	// This is always set represent /dev/console target
+	consoleTgt Target
+
+	nTargets int32 // atomic count of len(targets)
 )
 
 // SystemTargets returns active targets.
@@ -90,6 +94,11 @@ func AddSystemTarget(t Target) error {
 		return err
 	}
 	swapMu.Lock()
+	if consoleTgt == nil {
+		if t.Type() == types.TargetConsole {
+			consoleTgt = t
+		}
+	}
 	updated := append(make([]Target, 0, len(systemTargets)+1), systemTargets...)
 	updated = append(updated, t)
 	systemTargets = updated

internal/s3select/errors.go

@@ -101,6 +101,15 @@ func errInvalidRequestParameter(err error) *s3Error {
 	}
 }
 
+func errInvalidScanRangeParameter(err error) *s3Error {
+	return &s3Error{
+		code:       "InvalidRequestParameter",
+		message:    "The value of a parameter in ScanRange element is invalid. Check the service API documentation and try again.",
+		statusCode: 400,
+		cause:      err,
+	}
+}
+
 func errObjectSerializationConflict(err error) *s3Error {
 	return &s3Error{
 		code:       "ObjectSerializationConflict",

internal/s3select/select.go

@@ -212,6 +212,69 @@ type RequestProgress struct {
 	Enabled bool `xml:"Enabled"`
 }
 
+// ScanRange represents the ScanRange parameter.
+type ScanRange struct {
+	// Start if byte offset form the start off the file.
+	Start *uint64 `xml:"Start"`
+	// End is the last byte that should be returned, if Start is set,
+	// or the offset from EOF to start reading if start is not present.
+	End *uint64 `xml:"End"`
+}
+
+// Validate if the scan range is valid.
+func (s *ScanRange) Validate() error {
+	if s == nil {
+		return nil
+	}
+	if s.Start == nil && s.End == nil {
+		// This parameter is optional, but when specified, it must not be empty.
+		// Ref: https://docs.aws.amazon.com/AmazonS3/latest/API/API_SelectObjectContent.html#AmazonS3-SelectObjectContent-request-ScanRange
+		return errors.New("ScanRange: No Start or End specified")
+	}
+	if s.Start == nil || s.End == nil {
+		return nil
+	}
+	if *s.Start > *s.End {
+		return errors.New("ScanRange: Start cannot be after end")
+	}
+	return nil
+}
+
+// StartLen returns start offset plus length from range.
+func (s *ScanRange) StartLen() (start, length int64, err error) {
+	if s == nil {
+		return 0, -1, nil
+	}
+	err = s.Validate()
+	if err != nil {
+		return 0, 0, err
+	}
+
+	if s.End == nil && s.Start == nil {
+		// Not valid, but should be caught above.
+		return 0, -1, nil
+	}
+	if s.End == nil {
+		start := int64(*s.Start)
+		if start < 0 {
+			return 0, 0, errors.New("ScanRange: Start after EOF")
+		}
+		return start, -1, nil
+	}
+	if s.Start == nil {
+		// Suffix length
+		end := int64(*s.End)
+		if end < 0 {
+			return 0, 0, errors.New("ScanRange: End bigger than file")
+		}
+		// Suffix length
+		return -end, -1, nil
+	}
+	start = int64(*s.Start)
+	end := int64(*s.End)
+	return start, end - start + 1, nil
+}
+
 // S3Select - filters the contents on a simple structured query language (SQL) statement. It
 // represents elements inside <SelectRequest/> in request XML specified in detail at
 // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html.
@@ -222,6 +285,7 @@ type S3Select struct {
 	Input          InputSerialization  `xml:"InputSerialization"`
 	Output         OutputSerialization `xml:"OutputSerialization"`
 	Progress       RequestProgress     `xml:"RequestProgress"`
+	ScanRange      *ScanRange          `xml:"ScanRange"`
 
 	statement      *sql.SelectStatement
 	progressReader *progressReader
@@ -251,7 +315,9 @@ func (s3Select *S3Select) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e
 
 		return errMalformedXML(err)
 	}
-
+	if err := parsedS3Select.ScanRange.Validate(); err != nil {
+		return errInvalidScanRangeParameter(err)
+	}
 	parsedS3Select.ExpressionType = strings.ToLower(parsedS3Select.ExpressionType)
 	if parsedS3Select.ExpressionType != "sql" {
 		return errInvalidExpressionType(fmt.Errorf("invalid expression type '%v'", parsedS3Select.ExpressionType))
@@ -298,9 +364,13 @@ func (s3Select *S3Select) getProgress() (bytesScanned, bytesProcessed int64) {
 // Open - opens S3 object by using callback for SQL selection query.
 // Currently CSV, JSON and Apache Parquet formats are supported.
 func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadCloser, error)) error {
+	offset, end, err := s3Select.ScanRange.StartLen()
+	if err != nil {
+		return err
+	}
 	switch s3Select.Input.format {
 	case csvFormat:
-		rc, err := getReader(0, -1)
+		rc, err := getReader(offset, end)
 		if err != nil {
 			return err
 		}
@@ -335,7 +405,7 @@ func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadClos
 		s3Select.close = rc.Close
 		return nil
 	case jsonFormat:
-		rc, err := getReader(0, -1)
+		rc, err := getReader(offset, end)
 		if err != nil {
 			return err
 		}
@@ -362,6 +432,10 @@ func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadClos
 		if !strings.EqualFold(os.Getenv("MINIO_API_SELECT_PARQUET"), "on") {
 			return errors.New("parquet format parsing not enabled on server")
 		}
+		if offset != 0 || end != -1 {
+			// Offsets do not make sense in parquet files.
+			return errors.New("parquet format does not support offsets")
+		}
 		var err error
 		s3Select.recordReader, err = parquet.NewReader(getReader, &s3Select.Input.ParquetArgs)
 		return err

internal/s3select/select_test.go

@@ -20,6 +20,7 @@ package s3select
 import (
 	"bytes"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -1374,6 +1375,314 @@ func TestJSONInput(t *testing.T) {
 	}
 }
 
+func TestCSVRanges(t *testing.T) {
+	testInput := []byte(`id,time,num,num2,text
+1,2010-01-01T,7867786,4565.908123,"a text, with comma"
+2,2017-01-02T03:04Z,-5, 0.765111,
+`)
+	testTable := []struct {
+		name       string
+		query      string
+		input      []byte
+		requestXML []byte // override request XML
+		wantResult string
+		wantErr    bool
+	}{
+		{
+			name:  "select-all",
+			input: testInput,
+			query: ``,
+			// Since we are doing offset, no headers are used.
+			wantResult: `{"_1":"2","_2":"2017-01-02T03:04Z","_3":"-5","_4":" 0.765111","_5":""}`,
+			requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
+<SelectObjectContentRequest>
+    <Expression>SELECT * from s3object AS s</Expression>
+    <ExpressionType>SQL</ExpressionType>
+    <InputSerialization>
+        <CompressionType>NONE</CompressionType>
+        <CSV>
+        <FileHeaderInfo>NONE</FileHeaderInfo>
+	    <QuoteCharacter>"</QuoteCharacter>
+        </CSV>
+    </InputSerialization>
+    <OutputSerialization>
+        <JSON>
+        </JSON>
+    </OutputSerialization>
+    <RequestProgress>
+        <Enabled>FALSE</Enabled>
+    </RequestProgress>
+	<ScanRange><Start>76</Start><End>109</End></ScanRange>
+</SelectObjectContentRequest>`),
+		},
+		{
+			name:  "select-remain",
+			input: testInput,
+			// Since we are doing offset, no headers are used.
+			wantResult: `{"_1":"2","_2":"2017-01-02T03:04Z","_3":"-5","_4":" 0.765111","_5":""}`,
+			requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
+<SelectObjectContentRequest>
+    <Expression>SELECT * from s3object AS s</Expression>
+    <ExpressionType>SQL</ExpressionType>
+    <InputSerialization>
+        <CompressionType>NONE</CompressionType>
+        <CSV>
+        <FileHeaderInfo>NONE</FileHeaderInfo>
+	    <QuoteCharacter>"</QuoteCharacter>
+        </CSV>
+    </InputSerialization>
+    <OutputSerialization>
+        <JSON>
+        </JSON>
+    </OutputSerialization>
+    <RequestProgress>
+        <Enabled>FALSE</Enabled>
+    </RequestProgress>
+	<ScanRange><Start>76</Start></ScanRange>
+</SelectObjectContentRequest>`),
+		},
+		{
+			name:  "select-end-bytes",
+			input: testInput,
+			query: ``,
+			// Since we are doing offset, no headers are used.
+			wantResult: `{"_1":"2","_2":"2017-01-02T03:04Z","_3":"-5","_4":" 0.765111","_5":""}`,
+			requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
+<SelectObjectContentRequest>
+    <Expression>SELECT * from s3object AS s</Expression>
+    <ExpressionType>SQL</ExpressionType>
+    <InputSerialization>
+        <CompressionType>NONE</CompressionType>
+        <CSV>
+        <FileHeaderInfo>NONE</FileHeaderInfo>
+	    <QuoteCharacter>"</QuoteCharacter>
+        </CSV>
+    </InputSerialization>
+    <OutputSerialization>
+        <JSON>
+        </JSON>
+    </OutputSerialization>
+    <RequestProgress>
+        <Enabled>FALSE</Enabled>
+    </RequestProgress>
+	<ScanRange><End>35</End></ScanRange>
+</SelectObjectContentRequest>`),
+		},
+		{
+			name:  "select-middle",
+			input: testInput,
+			// Since we are doing offset, no headers are used.
+			wantResult: `{"_1":"a text, with comma"}`,
+			requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
+<SelectObjectContentRequest>
+    <Expression>SELECT * from s3object AS s</Expression>
+    <ExpressionType>SQL</ExpressionType>
+    <InputSerialization>
+        <CompressionType>NONE</CompressionType>
+        <CSV>
+        <FileHeaderInfo>NONE</FileHeaderInfo>
+	    <QuoteCharacter>"</QuoteCharacter>
+        </CSV>
+    </InputSerialization>
+    <OutputSerialization>
+        <JSON>
+        </JSON>
+    </OutputSerialization>
+    <RequestProgress>
+        <Enabled>FALSE</Enabled>
+    </RequestProgress>
+	<ScanRange><Start>56</Start><End>76</End></ScanRange>
+</SelectObjectContentRequest>`),
+		},
+		{
+			name:  "error-end-before-start",
+			input: testInput,
+			// Since we are doing offset, no headers are used.
+			wantResult: ``,
+			wantErr:    true,
+			requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
+<SelectObjectContentRequest>
+    <Expression>SELECT * from s3object AS s</Expression>
+    <ExpressionType>SQL</ExpressionType>
+    <InputSerialization>
+        <CompressionType>NONE</CompressionType>
+        <CSV>
+        <FileHeaderInfo>NONE</FileHeaderInfo>
+	    <QuoteCharacter>"</QuoteCharacter>
+        </CSV>
+    </InputSerialization>
+    <OutputSerialization>
+        <JSON>
+        </JSON>
+    </OutputSerialization>
+    <RequestProgress>
+        <Enabled>FALSE</Enabled>
+    </RequestProgress>
+	<ScanRange><Start>56</Start><End>26</End></ScanRange>
+</SelectObjectContentRequest>`),
+		},
+		{
+			name:  "error-empty",
+			input: testInput,
+			// Since we are doing offset, no headers are used.
+			wantResult: ``,
+			wantErr:    true,
+			requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
+<SelectObjectContentRequest>
+    <Expression>SELECT * from s3object AS s</Expression>
+    <ExpressionType>SQL</ExpressionType>
+    <InputSerialization>
+        <CompressionType>NONE</CompressionType>
+        <CSV>
+        <FileHeaderInfo>NONE</FileHeaderInfo>
+	    <QuoteCharacter>"</QuoteCharacter>
+        </CSV>
+    </InputSerialization>
+    <OutputSerialization>
+        <JSON>
+        </JSON>
+    </OutputSerialization>
+    <RequestProgress>
+        <Enabled>FALSE</Enabled>
+    </RequestProgress>
+	<ScanRange></ScanRange>
+</SelectObjectContentRequest>`),
+		},
+		{
+			name:  "error-after-eof",
+			input: testInput,
+			// Since we are doing offset, no headers are used.
+			wantResult: ``,
+			wantErr:    true,
+			requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
+<SelectObjectContentRequest>
+    <Expression>SELECT * from s3object AS s</Expression>
+    <ExpressionType>SQL</ExpressionType>
+    <InputSerialization>
+        <CompressionType>NONE</CompressionType>
+        <CSV>
+        <FileHeaderInfo>NONE</FileHeaderInfo>
+	    <QuoteCharacter>"</QuoteCharacter>
+        </CSV>
+    </InputSerialization>
+    <OutputSerialization>
+        <JSON>
+        </JSON>
+    </OutputSerialization>
+    <RequestProgress>
+        <Enabled>FALSE</Enabled>
+    </RequestProgress>
+	<ScanRange><Start>2600000</Start></ScanRange>
+</SelectObjectContentRequest>`),
+		},
+		{
+			name:  "error-after-eof",
+			input: testInput,
+			// Since we are doing offset, no headers are used.
+			wantResult: ``,
+			wantErr:    true,
+			requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
+<SelectObjectContentRequest>
+    <Expression>SELECT * from s3object AS s</Expression>
+    <ExpressionType>SQL</ExpressionType>
+    <InputSerialization>
+        <CompressionType>NONE</CompressionType>
+        <CSV>
+        <FileHeaderInfo>NONE</FileHeaderInfo>
+	    <QuoteCharacter>"</QuoteCharacter>
+        </CSV>
+    </InputSerialization>
+    <OutputSerialization>
+        <JSON>
+        </JSON>
+    </OutputSerialization>
+    <RequestProgress>
+        <Enabled>FALSE</Enabled>
+    </RequestProgress>
+	<ScanRange><Start>2600000</Start><End>2600001</End></ScanRange>
+</SelectObjectContentRequest>`),
+		},
+	}
+
+	for _, testCase := range testTable {
+		t.Run(testCase.name, func(t *testing.T) {
+			testReq := testCase.requestXML
+			s3Select, err := NewS3Select(bytes.NewReader(testReq))
+			if err != nil {
+				if !testCase.wantErr {
+					t.Fatal(err)
+				}
+				t.Logf("got expected error: %v", err)
+				return
+			}
+
+			if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
+				in := testCase.input
+				if offset != 0 || length != -1 {
+					// Copy from SelectObjectContentHandler
+					isSuffixLength := false
+					if offset < 0 {
+						isSuffixLength = true
+					}
+
+					if length > 0 {
+						length--
+					}
+
+					rs := &httpRangeSpec{
+						IsSuffixLength: isSuffixLength,
+						Start:          offset,
+						End:            offset + length,
+					}
+					if length == -1 {
+						rs.End = -1
+					}
+					t.Log("input, offset:", offset, "length:", length, "size:", len(in))
+					offset, length, err = rs.GetOffsetLength(int64(len(in)))
+					if err != nil {
+						return nil, err
+					}
+					t.Log("rs:", *rs, "offset:", offset, "length:", length)
+					in = in[offset : offset+length]
+				}
+				return ioutil.NopCloser(bytes.NewBuffer(in)), nil
+			}); err != nil {
+				if !testCase.wantErr {
+					t.Fatal(err)
+				}
+				t.Logf("got expected error: %v", err)
+				return
+			} else if testCase.wantErr {
+				t.Error("did not get expected error")
+				return
+			}
+
+			w := &testResponseWriter{}
+			s3Select.Evaluate(w)
+			s3Select.Close()
+			resp := http.Response{
+				StatusCode:    http.StatusOK,
+				Body:          ioutil.NopCloser(bytes.NewReader(w.response)),
+				ContentLength: int64(len(w.response)),
+			}
+			res, err := minio.NewSelectResults(&resp, "testbucket")
+			if err != nil {
+				t.Error(err)
+				return
+			}
+			got, err := ioutil.ReadAll(res)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+			gotS := strings.TrimSpace(string(got))
+			if !reflect.DeepEqual(gotS, testCase.wantResult) {
+				t.Errorf("received response does not match with expected reply. Query: %s\ngot: %s\nwant:%s", testCase.query, gotS, testCase.wantResult)
+			}
+		})
+	}
+}
+
 func TestParquetInput(t *testing.T) {
 	os.Setenv("MINIO_API_SELECT_PARQUET", "on")
 	defer os.Setenv("MINIO_API_SELECT_PARQUET", "off")
@@ -1728,3 +2037,76 @@ func TestParquetInputSchemaCSV(t *testing.T) {
 		})
 	}
 }
+
+// httpRangeSpec represents a range specification as supported by S3 GET
+// object request.
+//
+// Case 1: Not present -> represented by a nil RangeSpec
+// Case 2: bytes=1-10 (absolute start and end offsets) -> RangeSpec{false, 1, 10}
+// Case 3: bytes=10- (absolute start offset with end offset unspecified) -> RangeSpec{false, 10, -1}
+// Case 4: bytes=-30 (suffix length specification) -> RangeSpec{true, -30, -1}
+type httpRangeSpec struct {
+	// Does the range spec refer to a suffix of the object?
+	IsSuffixLength bool
+
+	// Start and end offset specified in range spec
+	Start, End int64
+}
+
+func (h *httpRangeSpec) GetLength(resourceSize int64) (rangeLength int64, err error) {
+	switch {
+	case resourceSize < 0:
+		return 0, errors.New("Resource size cannot be negative")
+
+	case h == nil:
+		rangeLength = resourceSize
+
+	case h.IsSuffixLength:
+		specifiedLen := -h.Start
+		rangeLength = specifiedLen
+		if specifiedLen > resourceSize {
+			rangeLength = resourceSize
+		}
+
+	case h.Start >= resourceSize:
+		return 0, errors.New("errInvalidRange")
+
+	case h.End > -1:
+		end := h.End
+		if resourceSize <= end {
+			end = resourceSize - 1
+		}
+		rangeLength = end - h.Start + 1
+
+	case h.End == -1:
+		rangeLength = resourceSize - h.Start
+
+	default:
+		return 0, errors.New("Unexpected range specification case")
+	}
+
+	return rangeLength, nil
+}
+
+// GetOffsetLength computes the start offset and length of the range
+// given the size of the resource
+func (h *httpRangeSpec) GetOffsetLength(resourceSize int64) (start, length int64, err error) {
+	if h == nil {
+		// No range specified, implies whole object.
+		return 0, resourceSize, nil
+	}
+
+	length, err = h.GetLength(resourceSize)
+	if err != nil {
+		return 0, 0, err
+	}
+
+	start = h.Start
+	if h.IsSuffixLength {
+		start = resourceSize + h.Start
+		if start < 0 {
+			start = 0
+		}
+	}
+	return start, length, nil
+}