fix: negative STS IAM token TTL value (#10866)

This will make the health check clients 'silent'.
Use `IsNetworkOrHostDown` determine if network is ok so it mimics the functionality in the actual client.

cmd/admin-handlers.go

@@ -794,7 +794,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	case hip.clientToken == "":
 		nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), hip.hs, hip.forceStart)
 		go func() {
-			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh)
+			respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh, objectAPI)
 			hr := healResp{respBytes, apiErr, errMsg}
 			respCh <- hr
 		}()
@@ -1035,7 +1035,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	traceCh := make(chan interface{}, 4000)
 
-	peers := newPeerRestClients(globalEndpoints)
+	peers, _ := newPeerRestClients(globalEndpoints)
 
 	globalHTTPTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool {
 		return mustTrace(entry, trcAll, trcErr)
@@ -1103,7 +1103,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 	logCh := make(chan interface{}, 4000)
 
-	peers := newPeerRestClients(globalEndpoints)
+	peers, _ := newPeerRestClients(globalEndpoints)
 
 	globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
 
@@ -1284,8 +1284,8 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
 	defer cancel()
 
-	nsLock := objectAPI.NewNSLock(ctx, minioMetaBucket, "obd-in-progress")
-	if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "obd-in-progress")
+	if err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
 		errResp(err)
 		return
 	}
@@ -1294,12 +1294,6 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	go func() {
 		defer close(obdInfoCh)
 
-		if log := query.Get("log"); log == "true" {
-			obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, getLocalLogOBD(deadlinedCtx, r))
-			obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, globalNotificationSys.LogOBDInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
-		}
-
 		if cpu := query.Get("syscpu"); cpu == "true" {
 			cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r)
 

cmd/admin-heal-ops.go

@@ -228,7 +228,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 // `keepHealSeqStateDuration`. This function also launches a
 // background routine to clean up heal results after the
 // aforementioned duration.
-func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
+func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLayer) (
 	respBytes []byte, apiErr APIError, errMsg string) {
 
 	if h.forceStarted {
@@ -265,7 +265,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence) (
 	ahs.healSeqMap[hpath] = h
 
 	// Launch top-level background heal go-routine
-	go h.healSequenceStart()
+	go h.healSequenceStart(objAPI)
 
 	clientToken := h.clientToken
 	if globalIsDistErasure {
@@ -609,7 +609,7 @@ func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error {
 // routine for completion, and (2) listens for external stop
 // signals. When either event happens, it sets the finish status for
 // the heal-sequence.
-func (h *healSequence) healSequenceStart() {
+func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
 	// Set status as running
 	h.mutex.Lock()
 	h.currentStatus.Summary = healRunningStatus
@@ -617,7 +617,7 @@ func (h *healSequence) healSequenceStart() {
 	h.mutex.Unlock()
 
 	if h.sourceCh == nil {
-		go h.traverseAndHeal()
+		go h.traverseAndHeal(objAPI)
 	} else {
 		go h.healFromSourceCh()
 	}
@@ -766,28 +766,23 @@ func (h *healSequence) healFromSourceCh() {
 	h.healItemsFromSourceCh()
 }
 
-func (h *healSequence) healDiskMeta() error {
-	// Start with format healing
-	if err := h.healDiskFormat(); err != nil {
-		return err
-	}
-
+func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
 	// Start healing the config prefix.
-	if err := h.healMinioSysMeta(minioConfigPrefix)(); err != nil {
+	if err := h.healMinioSysMeta(objAPI, minioConfigPrefix)(); err != nil {
 		return err
 	}
 
 	// Start healing the bucket config prefix.
-	return h.healMinioSysMeta(bucketConfigPrefix)()
+	return h.healMinioSysMeta(objAPI, bucketConfigPrefix)()
 }
 
-func (h *healSequence) healItems(bucketsOnly bool) error {
-	if err := h.healDiskMeta(); err != nil {
+func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
+	if err := h.healDiskMeta(objAPI); err != nil {
 		return err
 	}
 
 	// Heal buckets and objects
-	return h.healBuckets(bucketsOnly)
+	return h.healBuckets(objAPI, bucketsOnly)
 }
 
 // traverseAndHeal - traverses on-disk data and performs healing
@@ -797,26 +792,20 @@ func (h *healSequence) healItems(bucketsOnly bool) error {
 // quit signal is received, this routine cannot quit immediately and
 // has to wait until a safe point is reached, such as between scanning
 // two objects.
-func (h *healSequence) traverseAndHeal() {
+func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) {
 	bucketsOnly := false // Heals buckets and objects also.
-	h.traverseAndHealDoneCh <- h.healItems(bucketsOnly)
+	h.traverseAndHealDoneCh <- h.healItems(objAPI, bucketsOnly)
 	close(h.traverseAndHealDoneCh)
 }
 
 // healMinioSysMeta - heals all files under a given meta prefix, returns a function
 // which in-turn heals the respective meta directory path and any files in int.
-func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
+func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) func() error {
 	return func() error {
-		// Get current object layer instance.
-		objectAPI := newObjectLayerFn()
-		if objectAPI == nil {
-			return errServerNotInitialized
-		}
-
 		// NOTE: Healing on meta is run regardless
 		// of any bucket being selected, this is to ensure that
 		// meta are always upto date and correct.
-		return objectAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
+		return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string) error {
 			if h.isQuitting() {
 				return errHealStopSignalled
 			}
@@ -843,39 +832,27 @@ func (h *healSequence) healDiskFormat() error {
 		return errHealStopSignalled
 	}
 
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
 	return h.queueHealTask(healSource{bucket: SlashSeparator}, madmin.HealItemMetadata)
 }
 
 // healBuckets - check for all buckets heal or just particular bucket.
-func (h *healSequence) healBuckets(bucketsOnly bool) error {
+func (h *healSequence) healBuckets(objAPI ObjectLayer, bucketsOnly bool) error {
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}
 
 	// 1. If a bucket was specified, heal only the bucket.
 	if h.bucket != "" {
-		return h.healBucket(h.bucket, bucketsOnly)
+		return h.healBucket(objAPI, h.bucket, bucketsOnly)
 	}
 
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
-	buckets, err := objectAPI.ListBucketsHeal(h.ctx)
+	buckets, err := objAPI.ListBucketsHeal(h.ctx)
 	if err != nil {
 		return errFnHealFromAPIErr(h.ctx, err)
 	}
 
 	for _, bucket := range buckets {
-		if err = h.healBucket(bucket.Name, bucketsOnly); err != nil {
+		if err = h.healBucket(objAPI, bucket.Name, bucketsOnly); err != nil {
 			return err
 		}
 	}
@@ -884,13 +861,7 @@ func (h *healSequence) healBuckets(bucketsOnly bool) error {
 }
 
 // healBucket - traverses and heals given bucket
-func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
+func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly bool) error {
 	if err := h.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket); err != nil {
 		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 			return err
@@ -905,7 +876,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		if h.object != "" {
 			// Check if an object named as the objPrefix exists,
 			// and if so heal it.
-			oi, err := objectAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
+			oi, err := objAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
 			if err == nil {
 				if err = h.healObject(bucket, h.object, oi.VersionID); err != nil {
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
@@ -919,7 +890,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 		return nil
 	}
 
-	if err := objectAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
+	if err := objAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil {
 		// Object might have been deleted, by the time heal
 		// was attempted we ignore this object an move on.
 		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
@@ -931,12 +902,6 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 
 // healObject - heal the given object and record result
 func (h *healSequence) healObject(bucket, object, versionID string) error {
-	// Get current object layer instance.
-	objectAPI := newObjectLayerFn()
-	if objectAPI == nil {
-		return errServerNotInitialized
-	}
-
 	if h.isQuitting() {
 		return errHealStopSignalled
 	}

cmd/admin-router.go

@@ -221,5 +221,5 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 
 	// If none of the routes match add default error handler routes
 	adminRouter.NotFoundHandler = httpTraceAll(errorResponseHandler)
-	adminRouter.MethodNotAllowedHandler = httpTraceAll(errorResponseHandler)
+	adminRouter.MethodNotAllowedHandler = httpTraceAll(methodNotAllowedHandler("Admin"))
 }

cmd/api-errors.go

@@ -233,6 +233,7 @@ const (
 	ErrInvalidResourceName
 	ErrServerNotInitialized
 	ErrOperationTimedOut
+	ErrClientDisconnected
 	ErrOperationMaxedOut
 	ErrInvalidRequest
 	// MinIO storage class error codes
@@ -1216,6 +1217,11 @@ var errorCodes = errorCodeMap{
 		Description:    "A timeout occurred while trying to lock a resource, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrClientDisconnected: {
+		Code:           "ClientDisconnected",
+		Description:    "Client disconnected before response was ready",
+		HTTPStatusCode: 499, // No official code, use nginx value.
+	},
 	ErrOperationMaxedOut: {
 		Code:           "SlowDown",
 		Description:    "A timeout exceeded while waiting to proceed with the request, please reduce your request rate",
@@ -1742,6 +1748,16 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
+	// Only return ErrClientDisconnected if the provided context is actually canceled.
+	// This way downstream context.Canceled will still report ErrOperationTimedOut
+	select {
+	case <-ctx.Done():
+		if ctx.Err() == context.Canceled {
+			return ErrClientDisconnected
+		}
+	default:
+	}
+
 	switch err {
 	case errInvalidArgument:
 		apiErr = ErrAdminInvalidArgument

cmd/api-headers.go

@@ -115,10 +115,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	// Set tag count if object has tags
-	tags, _ := url.ParseQuery(objInfo.UserTags)
-	tagCount := len(tags)
-	if tagCount > 0 {
-		w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tagCount)}
+	if len(objInfo.UserTags) > 0 {
+		tags, _ := url.ParseQuery(objInfo.UserTags)
+		if len(tags) > 0 {
+			w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(len(tags))}
+		}
 	}
 
 	// Set all other user defined metadata.

cmd/api-response.go

@@ -35,11 +35,11 @@ import (
 
 const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
-	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
-	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
-	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
-	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
+	iso8601TimeFormat = "2006-01-02T15:04:05.000Z"                     // Reply date format with nanosecond precision.
+	maxObjectList     = metacacheBlockSize - (metacacheBlockSize / 10) // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxDeleteList     = 10000                                          // Limit number of objects deleted in a delete call.
+	maxUploadsList    = 10000                                          // Limit number of uploads in a listUploadsResponse.
+	maxPartsList      = 10000                                          // Limit number of parts in a listPartsResponse.
 )
 
 // LocationResponse - format for location response.

cmd/api-router.go

@@ -44,6 +44,12 @@ func newCachedObjectLayerFn() CacheObjectLayer {
 	return globalCacheObjectAPI
 }
 
+func setObjectLayer(o ObjectLayer) {
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = o
+	globalObjLayerMutex.Unlock()
+}
+
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
@@ -320,7 +326,7 @@ func registerAPIRouter(router *mux.Router) {
 
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))
-	apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler))
+	apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(methodNotAllowedHandler("S3")))
 
 }
 

cmd/auth-handler.go

@@ -333,6 +333,9 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// Populate payload again to handle it in HTTP handler.
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
+	if cred.AccessKey != "" {
+		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
+	}
 
 	if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
 		// Anonymous checks are not meant for ListBuckets action
@@ -616,7 +619,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 // isPutActionAllowed - check if PUT operation is allowed on the resource, this
 // call verifies bucket policies and IAM policies, supports multi user
 // checks etc.
-func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
+func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
 	switch atype {
@@ -637,6 +640,10 @@ func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.R
 		return s3Err
 	}
 
+	if cred.AccessKey != "" {
+		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
+	}
+
 	// Do not check for PutObjectRetentionAction permission,
 	// if mode and retain until date are not set.
 	// Can happen when bucket has default lock config set

cmd/background-newdisks-heal-ops.go

@@ -64,7 +64,8 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 			drivesToHeal, defaultMonitorNewDiskInterval))
 
 		// Heal any disk format and metadata early, if possible.
-		if err := bgSeq.healDiskMeta(); err != nil {
+		// Start with format healing
+		if err := bgSeq.healDiskFormat(); err != nil {
 			if newObjectLayerFn() != nil {
 				// log only in situations, when object layer
 				// has fully initialized.
@@ -73,6 +74,14 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 		}
 	}
 
+	if err := bgSeq.healDiskMeta(objAPI); err != nil {
+		if newObjectLayerFn() != nil {
+			// log only in situations, when object layer
+			// has fully initialized.
+			logger.LogIf(bgSeq.ctx, err)
+		}
+	}
+
 	go monitorLocalDisksAndHeal(ctx, z, bgSeq)
 }
 
@@ -101,7 +110,7 @@ func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 	globalBackgroundHealRoutine = newHealRoutine()
 	go globalBackgroundHealRoutine.run(ctx, objAPI)
 
-	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence())
+	globalBackgroundHealState.LaunchNewHealSequence(newBgHealSequence(), objAPI)
 }
 
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
@@ -175,8 +184,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq *
 
 						logger.Info("Healing disk '%s' on %s zone complete", disk, humanize.Ordinal(i+1))
 
-						if err := disk.DeleteFile(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
-							healingTrackerFilename); err != nil && !errors.Is(err, errFileNotFound) {
+						if err := disk.Delete(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
+							healingTrackerFilename, false); err != nil && !errors.Is(err, errFileNotFound) {
 							logger.LogIf(ctx, err)
 							continue
 						}

cmd/bootstrap-peer-server.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"crypto/tls"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -224,16 +223,7 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
 		Path:   bootstrapRESTPath,
 	}
 
-	var tlsConfig *tls.Config
-	if globalIsSSL {
-		tlsConfig = &tls.Config{
-			ServerName: endpoint.Hostname(),
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout)
-	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
+	restClient := rest.NewClient(serverURL, globalInternodeTransport, newAuthToken)
 	restClient.HealthCheckFn = nil
 
 	return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient}

cmd/bucket-handlers.go

@@ -383,6 +383,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
+	// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
+	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
+
 	// Before proceeding validate if bucket exists.
 	_, err := objectAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
@@ -457,7 +461,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	deletedObjects := make([]DeletedObject, len(deleteObjects.Objects))
 	for i := range errs {
-		dindex := objectsToDelete[deleteList[i]]
+		dindex := objectsToDelete[ObjectToDelete{
+			ObjectName: dObjects[i].ObjectName,
+			VersionID:  dObjects[i].VersionID,
+		}]
 		apiErr := toAPIError(ctx, errs[i])
 		if apiErr.Code == "" || apiErr.Code == "NoSuchKey" || apiErr.Code == "InvalidArgument" {
 			deletedObjects[dindex] = dObjects[i]

cmd/bucket-handlers_test.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"encoding/xml"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -837,7 +836,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		// Verify whether the bucket obtained object is same as the one created.
 		if testCase.expectedContent != nil && !bytes.Equal(testCase.expectedContent, actualContent) {
-			fmt.Println(string(testCase.expectedContent), string(actualContent))
+			t.Log(string(testCase.expectedContent), string(actualContent))
 			t.Errorf("Test %d : MinIO %s: Object content differs from expected value.", i+1, instanceType)
 		}
 	}

cmd/bucket-listobjects-handlers.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -114,15 +113,6 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Forward the request using Source IP or bucket
-	forwardStr := handlers.GetSourceIPFromHeaders(r)
-	if forwardStr == "" {
-		forwardStr = bucket
-	}
-	if proxyRequestByStringHash(ctx, w, r, forwardStr) {
-		return
-	}
-
 	listObjectVersions := objectAPI.ListObjectVersions
 
 	// Inititate a list object versions operation based on the input params.
@@ -145,7 +135,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 // ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 10000)
-// of the objects in a bucket. You can use the request parame<ters as selection
+// of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
 // NOTE: It is recommended that this API to be used for application development.
@@ -185,13 +175,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Analyze continuation token and route the request accordingly
-	var success bool
-	token, success = proxyRequestByToken(ctx, w, r, token)
-	if success {
-		return
-	}
-
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -207,9 +190,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
-		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
-	}
 
 	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
@@ -262,13 +242,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Analyze continuation token and route the request accordingly
-	var success bool
-	token, success = proxyRequestByToken(ctx, w, r, token)
-	if success {
-		return
-	}
-
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -282,13 +255,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
 
-	// The next continuation token has id@node_index format to optimize paginated listing
-	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
-		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
-	}
-
-	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
+	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
 		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
 
@@ -296,18 +263,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
 
-func getLocalNodeIndex() int {
-	if len(globalProxyEndpoints) == 0 {
-		return -1
-	}
-	for i, ep := range globalProxyEndpoints {
-		if ep.IsLocal {
-			return i
-		}
-	}
-	return -1
-}
-
 func parseRequestToken(token string) (subToken string, nodeIndex int) {
 	if token == "" {
 		return token, -1

cmd/bucket-replication.go

@@ -88,7 +88,7 @@ func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object strin
 
 // mustReplicate returns true if object meets replication criteria.
 func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
-	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
+	if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
 		return false
 	}
 	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
@@ -235,7 +235,11 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 	replicationStatus := replication.Complete
 
 	// Setup bandwidth throttling
-	totalNodesCount := len(GetRemotePeers(globalEndpoints)) + 1
+	peers, _ := globalEndpoints.peers()
+	totalNodesCount := len(peers)
+	if totalNodesCount == 0 {
+		totalNodesCount = 1 // For standalone erasure coding
+	}
 	b := target.BandwidthLimit / int64(totalNodesCount)
 	var headerSize int
 	for k, v := range putOpts.Header() {

cmd/config-encrypted.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"time"
 	"unicode/utf8"
@@ -194,7 +193,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 				// Config is already encrypted with right keys
 				continue
 			}
-			return errors.New("config data not in plain-text form or encrypted")
+			return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
 		}
 
 		cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
@@ -274,7 +273,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 					// Config is already encrypted with right keys
 					continue
 				}
-				return errors.New("config data not in plain-text form or encrypted")
+				return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
 			}
 
 			cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)

cmd/config/api/api.go

@@ -34,12 +34,17 @@ const (
 	apiClusterDeadline         = "cluster_deadline"
 	apiCorsAllowOrigin         = "cors_allow_origin"
 	apiRemoteTransportDeadline = "remote_transport_deadline"
+	apiListQuorum              = "list_quorum"
+	apiExtendListCacheLife     = "extend_list_cache_life"
 
 	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
 	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
 	EnvAPIClusterDeadline         = "MINIO_API_CLUSTER_DEADLINE"
 	EnvAPICorsAllowOrigin         = "MINIO_API_CORS_ALLOW_ORIGIN"
 	EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
+	EnvAPIListQuorum              = "MINIO_API_LIST_QUORUM"
+	EnvAPIExtendListCacheLife     = "MINIO_API_EXTEND_LIST_CACHE_LIFE"
+	EnvAPISecureCiphers           = "MINIO_API_SECURE_CIPHERS"
 )
 
 // Deprecated key and ENVs
@@ -71,6 +76,14 @@ var (
 			Key:   apiRemoteTransportDeadline,
 			Value: "2h",
 		},
+		config.KV{
+			Key:   apiListQuorum,
+			Value: "optimal",
+		},
+		config.KV{
+			Key:   apiExtendListCacheLife,
+			Value: "0s",
+		},
 	}
 )
 
@@ -81,6 +94,8 @@ type Config struct {
 	ClusterDeadline         time.Duration `json:"cluster_deadline"`
 	CorsAllowOrigin         []string      `json:"cors_allow_origin"`
 	RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
+	ListQuorum              string        `json:"list_strict_quorum"`
+	ExtendListLife          time.Duration `json:"extend_list_cache_life"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -94,6 +109,24 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 	return json.Unmarshal(data, &aux)
 }
 
+// GetListQuorum interprets list quorum values and returns appropriate
+// acceptable quorum expected for list operations
+func (sCfg Config) GetListQuorum() int {
+	switch sCfg.ListQuorum {
+	case "optimal":
+		return 3
+	case "reduced":
+		return 2
+	case "disk":
+		// smallest possible value, generally meant for testing.
+		return 1
+	case "strict":
+		return -1
+	}
+	// Defaults to 3 drives per set.
+	return 3
+}
+
 // LookupConfig - lookup api config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	// remove this since we have removed this already.
@@ -130,11 +163,25 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
+	listQuorum := env.Get(EnvAPIListQuorum, kvs.Get(apiListQuorum))
+	switch listQuorum {
+	case "strict", "optimal", "reduced", "disk":
+	default:
+		return cfg, errors.New("invalid value for list strict quorum")
+	}
+
+	listLife, err := time.ParseDuration(env.Get(EnvAPIExtendListCacheLife, kvs.Get(apiExtendListCacheLife)))
+	if err != nil {
+		return cfg, err
+	}
+
 	return Config{
 		RequestsMax:             requestsMax,
 		RequestsDeadline:        requestsDeadline,
 		ClusterDeadline:         clusterDeadline,
 		CorsAllowOrigin:         corsAllowOrigin,
 		RemoteTransportDeadline: remoteTransportDeadline,
+		ListQuorum:              listQuorum,
+		ExtendListLife:          listLife,
 	}, nil
 }

cmd/config/cache/config.go

@@ -28,16 +28,17 @@ import (
 
 // Config represents cache config settings
 type Config struct {
-	Enabled       bool     `json:"-"`
-	Drives        []string `json:"drives"`
-	Expiry        int      `json:"expiry"`
-	MaxUse        int      `json:"maxuse"`
-	Quota         int      `json:"quota"`
-	Exclude       []string `json:"exclude"`
-	After         int      `json:"after"`
-	WatermarkLow  int      `json:"watermark_low"`
-	WatermarkHigh int      `json:"watermark_high"`
-	Range         bool     `json:"range"`
+	Enabled         bool     `json:"-"`
+	Drives          []string `json:"drives"`
+	Expiry          int      `json:"expiry"`
+	MaxUse          int      `json:"maxuse"`
+	Quota           int      `json:"quota"`
+	Exclude         []string `json:"exclude"`
+	After           int      `json:"after"`
+	WatermarkLow    int      `json:"watermark_low"`
+	WatermarkHigh   int      `json:"watermark_high"`
+	Range           bool     `json:"range"`
+	CommitWriteback bool     `json:"-"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
@@ -152,3 +153,13 @@ func parseCacheExcludes(excludes string) ([]string, error) {
 
 	return excludesSlice, nil
 }
+
+func parseCacheCommitMode(commitStr string) (bool, error) {
+	switch strings.ToLower(commitStr) {
+	case "writeback":
+		return true, nil
+	case "writethrough":
+		return false, nil
+	}
+	return false, config.ErrInvalidCacheCommitValue(nil).Msg("cache commit value must be `writeback` or `writethrough`")
+}

cmd/config/cache/help.go

@@ -74,5 +74,11 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         Commit,
+			Description: `set to control cache commit behavior, defaults to "writethrough"`,
+			Optional:    true,
+			Type:        "string",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -35,6 +35,7 @@ const (
 	WatermarkLow  = "watermark_low"
 	WatermarkHigh = "watermark_high"
 	Range         = "range"
+	Commit        = "commit"
 
 	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
 	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
@@ -45,6 +46,7 @@ const (
 	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
 	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
 	EnvCacheRange         = "MINIO_CACHE_RANGE"
+	EnvCacheCommit        = "MINIO_CACHE_COMMIT"
 
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
@@ -53,6 +55,7 @@ const (
 	DefaultAfter         = "0"
 	DefaultWaterMarkLow  = "70"
 	DefaultWaterMarkHigh = "80"
+	DefaultCacheCommit   = "writethrough"
 )
 
 // DefaultKVS - default KV settings for caching.
@@ -90,6 +93,10 @@ var (
 			Key:   Range,
 			Value: config.EnableOn,
 		},
+		config.KV{
+			Key:   Commit,
+			Value: DefaultCacheCommit,
+		},
 	}
 )
 
@@ -210,6 +217,12 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		}
 		cfg.Range = rng
 	}
+	if commit := env.Get(EnvCacheCommit, kvs.Get(Commit)); commit != "" {
+		cfg.CommitWriteback, err = parseCacheCommitMode(commit)
+		if err != nil {
+			return cfg, err
+		}
+	}
 
 	return cfg, nil
 }

cmd/config/errors.go

@@ -102,6 +102,12 @@ var (
 		"MINIO_CACHE_RANGE: Valid expected value is `on` or `off`",
 	)
 
+	ErrInvalidCacheCommitValue = newErrFn(
+		"Invalid cache commit value",
+		"Please check the passed value",
+		"MINIO_CACHE_COMMIT: Valid expected value is `writeback` or `writethrough`",
+	)
+
 	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
 		"Invalid rotating credentials",
 		"Please set correct rotating credentials in the environment for decryption",

cmd/config/storageclass/help.go

@@ -33,6 +33,12 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         ClassDMA,
+			Description: `enable O_DIRECT for both read and write, defaults to "write" e.g. "read+write"`,
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

cmd/config/storageclass/storage-class.go

@@ -18,6 +18,7 @@ package storageclass
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -32,17 +33,26 @@ const (
 	RRS = "REDUCED_REDUNDANCY"
 	// Standard storage class
 	STANDARD = "STANDARD"
+	// DMA storage class
+	DMA = "DMA"
+
+	// Valid values are "write" and "read+write"
+	DMAWrite     = "write"
+	DMAReadWrite = "read+write"
 )
 
 // Standard constats for config info storage class
 const (
 	ClassStandard = "standard"
 	ClassRRS      = "rrs"
+	ClassDMA      = "dma"
 
 	// Reduced redundancy storage class environment variable
 	RRSEnv = "MINIO_STORAGE_CLASS_RRS"
 	// Standard storage class environment variable
 	StandardEnv = "MINIO_STORAGE_CLASS_STANDARD"
+	// DMA storage class environment variable
+	DMAEnv = "MINIO_STORAGE_CLASS_DMA"
 
 	// Supported storage class scheme is EC
 	schemePrefix = "EC"
@@ -52,6 +62,9 @@ const (
 
 	// Default RRS parity is always minimum parity.
 	defaultRRSParity = minParityDisks
+
+	// Default DMA value
+	defaultDMA = DMAWrite
 )
 
 // DefaultKVS - default storage class config
@@ -65,18 +78,24 @@ var (
 			Key:   ClassRRS,
 			Value: "EC:2",
 		},
+		config.KV{
+			Key:   ClassDMA,
+			Value: defaultDMA,
+		},
 	}
 )
 
 // StorageClass - holds storage class information
 type StorageClass struct {
 	Parity int
+	DMA    string
 }
 
 // Config storage class configuration
 type Config struct {
 	Standard StorageClass `json:"standard"`
 	RRS      StorageClass `json:"rrs"`
+	DMA      StorageClass `json:"dma"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -93,7 +112,7 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 // IsValid - returns true if input string is a valid
 // storage class kind supported.
 func IsValid(sc string) bool {
-	return sc == RRS || sc == STANDARD
+	return sc == RRS || sc == STANDARD || sc == DMA
 }
 
 // UnmarshalText unmarshals storage class from its textual form into
@@ -103,6 +122,14 @@ func (sc *StorageClass) UnmarshalText(b []byte) error {
 	if scStr == "" {
 		return nil
 	}
+	if scStr == DMAWrite {
+		sc.DMA = DMAWrite
+		return nil
+	}
+	if scStr == DMAReadWrite {
+		sc.DMA = DMAReadWrite
+		return nil
+	}
 	s, err := parseStorageClass(scStr)
 	if err != nil {
 		return err
@@ -116,14 +143,14 @@ func (sc *StorageClass) MarshalText() ([]byte, error) {
 	if sc.Parity != 0 {
 		return []byte(fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)), nil
 	}
-	return []byte(""), nil
+	return []byte(sc.DMA), nil
 }
 
 func (sc *StorageClass) String() string {
 	if sc.Parity != 0 {
 		return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)
 	}
-	return ""
+	return sc.DMA
 }
 
 // Parses given storageClassEnv and returns a storageClass structure.
@@ -212,6 +239,11 @@ func (sCfg Config) GetParityForSC(sc string) (parity int) {
 	}
 }
 
+// GetDMA - returns DMA configuration.
+func (sCfg Config) GetDMA() string {
+	return sCfg.DMA.DMA
+}
+
 // Enabled returns if etcd is enabled.
 func Enabled(kvs config.KVS) bool {
 	ssc := kvs.Get(ClassStandard)
@@ -231,6 +263,7 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 
 	ssc := env.Get(StandardEnv, kvs.Get(ClassStandard))
 	rrsc := env.Get(RRSEnv, kvs.Get(ClassRRS))
+	dma := env.Get(DMAEnv, kvs.Get(ClassDMA))
 	// Check for environment variables and parse into storageClass struct
 	if ssc != "" {
 		cfg.Standard, err = parseStorageClass(ssc)
@@ -252,6 +285,14 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 		cfg.RRS.Parity = defaultRRSParity
 	}
 
+	if dma == "" {
+		dma = defaultDMA
+	}
+	if dma != DMAReadWrite && dma != DMAWrite {
+		return Config{}, errors.New(`valid dma values are "read-write" and "write"`)
+	}
+	cfg.DMA.DMA = dma
+
 	// Validation is done after parsing both the storage classes. This is needed because we need one
 	// storage class value to deduce the correct value of the other storage class.
 	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, setDriveCount); err != nil {

cmd/crypto/kes.go

@@ -177,11 +177,10 @@ func (kes *kesService) CreateKey(keyID string) error { return kes.client.CreateK
 // named key referenced by keyID. It also binds the generated key
 // cryptographically to the provided context.
 func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
-	var context bytes.Buffer
-	ctx.WriteTo(&context)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	var plainKey []byte
-	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
+	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context)
 	if err != nil {
 		return key, nil, err
 	}
@@ -200,11 +199,10 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
-	var context bytes.Buffer
-	ctx.WriteTo(&context)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	var plainKey []byte
-	plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context.Bytes())
+	plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context)
 	if err != nil {
 		return key, err
 	}
@@ -415,7 +413,7 @@ func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.
 		}
 
 		// If the error is not temp. / retryable => fail the request immediately.
-		if !xnet.IsNetworkOrHostDown(err) &&
+		if !xnet.IsNetworkOrHostDown(err, false) &&
 			!errors.Is(err, io.EOF) &&
 			!errors.Is(err, io.ErrUnexpectedEOF) &&
 			!errors.Is(err, context.DeadlineExceeded) {

cmd/crypto/key.go

@@ -103,7 +103,6 @@ func (key ObjectKey) Seal(extKey, iv [32]byte, domain, bucket, object string) Se
 func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucket, object string) error {
 	var (
 		unsealConfig sio.Config
-		decryptedKey bytes.Buffer
 	)
 	switch sealedKey.Algorithm {
 	default:
@@ -122,10 +121,9 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 		unsealConfig = sio.Config{MinVersion: sio.Version10, Key: sha.Sum(nil)}
 	}
 
-	if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil {
+	if out, err := sio.DecryptBuffer(key[:0], sealedKey.Key[:], unsealConfig); len(out) != 32 || err != nil {
 		return ErrSecretKeyMismatch
 	}
-	copy(key[:], decryptedKey.Bytes())
 	return nil
 }
 
@@ -165,11 +163,7 @@ func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) {
 	if !IsETagSealed(etag) {
 		return etag, nil
 	}
-	var buffer bytes.Buffer
 	mac := hmac.New(sha256.New, key[:])
 	mac.Write([]byte("SSE-etag"))
-	if _, err := sio.Decrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
-		return nil, err
-	}
-	return buffer.Bytes(), nil
+	return sio.DecryptBuffer(make([]byte, 0, len(etag)), etag, sio.Config{Key: mac.Sum(nil)})
 }

cmd/crypto/kms.go

@@ -39,6 +39,8 @@ type Context map[string]string
 //
 // WriteTo sorts the context keys and writes the sorted
 // key-value pairs as canonical JSON object to w.
+//
+// Note that neither keys nor values are escaped for JSON.
 func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	sortedKeys := make(sort.StringSlice, 0, len(c))
 	for k := range c {
@@ -67,6 +69,53 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	return n + int64(nn), err
 }
 
+// AppendTo appends the context in a canonical from to dst.
+//
+// AppendTo sorts the context keys and writes the sorted
+// key-value pairs as canonical JSON object to w.
+//
+// Note that neither keys nor values are escaped for JSON.
+func (c Context) AppendTo(dst []byte) (output []byte) {
+	if len(c) == 0 {
+		return append(dst, '{', '}')
+	}
+
+	// out should not escape.
+	out := bytes.NewBuffer(dst)
+
+	// No need to copy+sort
+	if len(c) == 1 {
+		for k, v := range c {
+			out.WriteString(`{"`)
+			out.WriteString(k)
+			out.WriteString(`":"`)
+			out.WriteString(v)
+			out.WriteString(`"}`)
+		}
+		return out.Bytes()
+	}
+
+	sortedKeys := make([]string, 0, len(c))
+	for k := range c {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Strings(sortedKeys)
+
+	out.WriteByte('{')
+	for i, k := range sortedKeys {
+		out.WriteByte('"')
+		out.WriteString(k)
+		out.WriteString(`":"`)
+		out.WriteString(c[k])
+		out.WriteByte('"')
+		if i < len(sortedKeys)-1 {
+			out.WriteByte(',')
+		}
+	}
+	out.WriteByte('}')
+	return out.Bytes()
+}
+
 // KMS represents an active and authenticted connection
 // to a Key-Management-Service. It supports generating
 // data key generation and unsealing of KMS-generated
@@ -155,13 +204,12 @@ func (kms *masterKeyKMS) Info() (info KMSInfo) {
 
 func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
 	var (
-		buffer     bytes.Buffer
 		derivedKey = kms.deriveKey(keyID, ctx)
 	)
-	if n, err := sio.Decrypt(&buffer, bytes.NewReader(sealedKey), sio.Config{Key: derivedKey[:]}); err != nil || n != 32 {
+	out, err := sio.DecryptBuffer(key[:0], sealedKey, sio.Config{Key: derivedKey[:]})
+	if err != nil || len(out) != 32 {
 		return key, err // TODO(aead): upgrade sio to use sio.Error
 	}
-	copy(key[:], buffer.Bytes())
 	return key, nil
 }
 
@@ -171,7 +219,7 @@ func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte)
 	}
 	mac := hmac.New(sha256.New, kms.masterKey[:])
 	mac.Write([]byte(keyID))
-	context.WriteTo(mac)
+	mac.Write(context.AppendTo(make([]byte, 0, 128)))
 	mac.Sum(key[:0])
 	return key
 }

cmd/crypto/kms_test.go

@@ -16,6 +16,7 @@ package crypto
 
 import (
 	"bytes"
+	"fmt"
 	"path"
 	"strings"
 	"testing"
@@ -83,3 +84,32 @@ func TestContextWriteTo(t *testing.T) {
 		}
 	}
 }
+
+func TestContextAppendTo(t *testing.T) {
+	for i, test := range contextWriteToTests {
+		dst := make([]byte, 0, 1024)
+		dst = test.Context.AppendTo(dst)
+		if s := string(dst); s != test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON)
+		}
+		// Append one more
+		dst = test.Context.AppendTo(dst)
+		if s := string(dst); s != test.ExpectedJSON+test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON+test.ExpectedJSON)
+		}
+	}
+}
+
+func BenchmarkContext_AppendTo(b *testing.B) {
+	tests := []Context{{}, {"bucket": "warp-benchmark-bucket"}, {"0": "1", "-": "2", ".": "#"}, {"34trg": "dfioutr89", "ikjfdghkjf": "jkedfhgfjkhg", "sdfhsdjkh": "if88889", "asddsirfh804": "kjfdshgdfuhgfg78-45604586#$%"}}
+	for _, test := range tests {
+		b.Run(fmt.Sprintf("%d-elems", len(test)), func(b *testing.B) {
+			dst := make([]byte, 0, 1024)
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				dst = test.AppendTo(dst[:0])
+			}
+		})
+	}
+}

cmd/crypto/metadata.go

@@ -204,15 +204,17 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 
 	// Check whether all extracted values are well-formed
-	iv, err := base64.StdEncoding.DecodeString(b64IV)
-	if err != nil || len(iv) != 32 {
+	var iv [32]byte
+	n, err := base64.StdEncoding.Decode(iv[:], []byte(b64IV))
+	if err != nil || n != 32 {
 		return keyID, kmsKey, sealedKey, errInvalidInternalIV
 	}
 	if algorithm != SealAlgorithm {
 		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
 	}
-	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
-	if err != nil || len(encryptedKey) != 64 {
+	var encryptedKey [64]byte
+	n, err = base64.StdEncoding.Decode(encryptedKey[:], []byte(b64SealedKey))
+	if err != nil || n != 64 {
 		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
 	}
 	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
@@ -223,8 +225,8 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 
 	sealedKey.Algorithm = algorithm
-	copy(sealedKey.IV[:], iv)
-	copy(sealedKey.Key[:], encryptedKey)
+	sealedKey.IV = iv
+	sealedKey.Key = encryptedKey
 	return keyID, kmsKey, sealedKey, nil
 }
 

cmd/crypto/vault.go

@@ -15,7 +15,6 @@
 package crypto
 
 import (
-	"bytes"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -224,11 +223,10 @@ func (v *vaultService) CreateKey(keyID string) error {
 // named key referenced by keyID. It also binds the generated key
 // cryptographically to the provided context.
 func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
-		"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context": base64.StdEncoding.EncodeToString(context),
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
 	if err != nil {
@@ -260,12 +258,11 @@ func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
 		"ciphertext": string(sealedKey),
-		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context":    base64.StdEncoding.EncodeToString(context),
 	}
 
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
@@ -294,12 +291,11 @@ func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (v *vaultService) UpdateKey(keyID string, sealedKey []byte, ctx Context) (rotatedKey []byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
 		"ciphertext": string(sealedKey),
-		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context":    base64.StdEncoding.EncodeToString(context),
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/rewrap/%s", keyID), payload)
 	if err != nil {

cmd/data-crawler.go

@@ -69,10 +69,10 @@ func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 // There should only ever be one crawler running per cluster.
 func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 	// Make sure only 1 crawler is running on the cluster.
-	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "runDataCrawler.lock")
+	locker := objAPI.NewNSLock(minioMetaBucket, "runDataCrawler.lock")
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	for {
-		err := locker.GetLock(dataCrawlerLeaderLockTimeout)
+		err := locker.GetLock(ctx, dataCrawlerLeaderLockTimeout)
 		if err != nil {
 			time.Sleep(time.Duration(r.Float64() * float64(dataCrawlStartDelay)))
 			continue

cmd/data-update-tracker.go

@@ -40,13 +40,13 @@ import (
 
 const (
 	// Estimate bloom filter size. With this many items
-	dataUpdateTrackerEstItems = 1000000
+	dataUpdateTrackerEstItems = 10000000
 	// ... we want this false positive rate:
 	dataUpdateTrackerFP        = 0.99
-	dataUpdateTrackerQueueSize = 10000
+	dataUpdateTrackerQueueSize = 0
 
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 3
+	dataUpdateTrackerVersion      = 4
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
 )
 
@@ -168,6 +168,43 @@ func (d *dataUpdateTracker) current() uint64 {
 	return d.Current.idx
 }
 
+// latestWithDir returns the highest index that contains the directory.
+// This means that any cycle higher than this does NOT contain the entry.
+func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
+	bucket, _ := path2BucketObjectWithBasePath("", dir)
+	if bucket == "" {
+		if d.debug && len(dir) > 0 {
+			logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", dir)
+		}
+		return d.current()
+	}
+	if isReservedOrInvalidBucket(bucket, false) {
+		if d.debug {
+			logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, dir)
+		}
+		return d.current()
+	}
+
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if d.Current.bf.containsDir(dir) || d.Current.idx == 0 {
+		return d.Current.idx
+	}
+	if d.debug {
+		logger.Info("current bloom does NOT contains dir %s", dir)
+	}
+
+	idx := d.Current.idx - 1
+	for {
+		f := d.History.find(idx)
+		if f == nil || f.bf.containsDir(dir) || idx == 0 {
+			break
+		}
+		idx--
+	}
+	return idx
+}
+
 // start will load the current data from the drives start collecting information and
 // start a saver goroutine.
 // All of these will exit when the context is canceled.
@@ -366,7 +403,7 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 		return err
 	}
 	switch tmp[0] {
-	case 1, 2:
+	case 1, 2, 3:
 		logger.Info(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
 		return nil
 	case dataUpdateTrackerVersion:
@@ -440,41 +477,66 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 // start a collector that picks up entries from objectUpdatedCh
 // and adds them  to the current bloom filter.
 func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case in := <-d.input:
-			bucket, _ := path2BucketObjectWithBasePath("", in)
-			if bucket == "" {
-				if d.debug && len(in) > 0 {
-					logger.Info(color.Green("data-usage:")+" no bucket (%s)", in)
-				}
-				continue
+	for in := range d.input {
+		bucket, _ := path2BucketObjectWithBasePath("", in)
+		if bucket == "" {
+			if d.debug && len(in) > 0 {
+				logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in)
 			}
-
-			if isReservedOrInvalidBucket(bucket, false) {
-				if false && d.debug {
-					logger.Info(color.Green("data-usage:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
-				}
-				continue
-			}
-			split := splitPathDeterministic(in)
-
-			// Add all paths until level 3.
-			d.mu.Lock()
-			for i := range split {
-				if d.debug && false {
-					logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
-				}
-				d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
-			}
-			d.dirty = d.dirty || len(split) > 0
-			d.mu.Unlock()
+			continue
 		}
+
+		if isReservedOrInvalidBucket(bucket, false) {
+			if d.debug {
+				logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
+			}
+			continue
+		}
+		split := splitPathDeterministic(in)
+
+		// Add all paths until done.
+		d.mu.Lock()
+		for i := range split {
+			if d.debug {
+				logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+			}
+			d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+		}
+		d.dirty = d.dirty || len(split) > 0
+		d.mu.Unlock()
 	}
 }
 
+// markDirty adds the supplied path to the current bloom filter.
+func (d *dataUpdateTracker) markDirty(in string) {
+	bucket, _ := path2BucketObjectWithBasePath("", in)
+	if bucket == "" {
+		if d.debug && len(in) > 0 {
+			logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in)
+		}
+		return
+	}
+
+	if isReservedOrInvalidBucket(bucket, false) {
+		if d.debug && false {
+			logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
+		}
+		return
+	}
+	split := splitPathDeterministic(in)
+
+	// Add all paths until done.
+	d.mu.Lock()
+	for i := range split {
+		if d.debug {
+			logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+		}
+		d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+	}
+	d.dirty = d.dirty || len(split) > 0
+	d.mu.Unlock()
+}
+
 // find entry with specified index.
 // Returns nil if not found.
 func (d dataUpdateTrackerHistory) find(idx uint64) *dataUpdateFilter {
@@ -534,8 +596,13 @@ func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint6
 // cycleFilter will cycle the bloom filter to start recording to index y if not already.
 // The response will contain a bloom filter starting at index x up to, but not including index y.
 // If y is 0, the response will not update y, but return the currently recorded information
-// from the up until and including current y.
-func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
+// from the oldest (unless 0, then it will be all) until and including current y.
+func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
+	if req.OldestClean != "" {
+		return &bloomFilterResponse{OldestIdx: d.latestWithDir(req.OldestClean)}, nil
+	}
+	current := req.Current
+	oldest := req.Oldest
 	d.mu.Lock()
 	defer d.mu.Unlock()
 	if current == 0 {
@@ -543,7 +610,10 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uin
 			return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
 		}
 		d.History.sort()
-		return d.filterFrom(ctx, d.History[len(d.History)-1].idx, d.Current.idx), nil
+		if oldest == 0 {
+			oldest = d.History[len(d.History)-1].idx
+		}
+		return d.filterFrom(ctx, oldest, d.Current.idx), nil
 	}
 
 	// Move current to history if new one requested
@@ -571,7 +641,7 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uin
 // Trailing slashes are removed.
 // Returns 0 length if no parts are found after trimming.
 func splitPathDeterministic(in string) []string {
-	split := strings.Split(in, SlashSeparator)
+	split := strings.Split(decodeDirObject(in), SlashSeparator)
 
 	// Trim empty start/end
 	for len(split) > 0 {
@@ -587,10 +657,6 @@ func splitPathDeterministic(in string) []string {
 		split = split[:len(split)-1]
 	}
 
-	// Return up to 3 parts.
-	if len(split) > 3 {
-		split = split[:3]
-	}
 	return split
 }
 
@@ -599,6 +665,9 @@ func splitPathDeterministic(in string) []string {
 type bloomFilterRequest struct {
 	Oldest  uint64
 	Current uint64
+	// If set the oldest clean version will be returned in OldestIdx
+	// and the rest of the request will be ignored.
+	OldestClean string
 }
 
 type bloomFilterResponse struct {
@@ -615,10 +684,9 @@ type bloomFilterResponse struct {
 }
 
 // ObjectPathUpdated indicates a path has been updated.
-// The function will never block.
+// The function will block until the entry has been picked up.
 func ObjectPathUpdated(s string) {
-	select {
-	case objectUpdatedCh <- s:
-	default:
+	if intDataUpdateTracker != nil {
+		intDataUpdateTracker.markDirty(s)
 	}
 }

cmd/data-update-tracker_test.go

@@ -169,7 +169,12 @@ func TestDataUpdateTracker(t *testing.T) {
 		})
 	}
 	// Cycle to history
-	_, err = dut.cycleFilter(ctx, 1, 2)
+	req := bloomFilterRequest{
+		Oldest:  1,
+		Current: 2,
+	}
+
+	_, err = dut.cycleFilter(ctx, req)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -200,7 +205,11 @@ func TestDataUpdateTracker(t *testing.T) {
 	if dut.current() != 2 {
 		t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
 	}
-	bfr2, err := dut.cycleFilter(ctx, 1, 3)
+	req = bloomFilterRequest{
+		Oldest:  1,
+		Current: 3,
+	}
+	bfr2, err := dut.cycleFilter(ctx, req)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/disk-cache-backend.go

@@ -19,7 +19,9 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/md5"
 	"crypto/rand"
+	"encoding/base64"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -70,7 +72,9 @@ type cacheMeta struct {
 	// Ranges maps cached range to associated filename.
 	Ranges map[string]string `json:"ranges,omitempty"`
 	// Hits is a counter on the number of times this object has been accessed so far.
-	Hits int `json:"hits,omitempty"`
+	Hits   int    `json:"hits,omitempty"`
+	Bucket string `json:"bucket,omitempty"`
+	Object string `json:"object,omitempty"`
 }
 
 // RangeInfo has the range, file and range length information for a cached range.
@@ -130,19 +134,21 @@ type diskCache struct {
 	online       uint32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 	purgeRunning int32
 
-	triggerGC     chan struct{}
-	dir           string         // caching directory
-	stats         CacheDiskStats // disk cache stats for prometheus
-	quotaPct      int            // max usage in %
-	pool          sync.Pool
-	after         int // minimum accesses before an object is cached.
-	lowWatermark  int
-	highWatermark int
-	enableRange   bool
+	triggerGC        chan struct{}
+	dir              string         // caching directory
+	stats            CacheDiskStats // disk cache stats for prometheus
+	quotaPct         int            // max usage in %
+	pool             sync.Pool
+	after            int // minimum accesses before an object is cached.
+	lowWatermark     int
+	highWatermark    int
+	enableRange      bool
+	commitWriteback  bool
+	retryWritebackCh chan ObjectInfo
 	// nsMutex namespace lock
 	nsMutex *nsLockMap
 	// Object functions pointing to the corresponding functions of backend implementation.
-	NewNSLockFn func(ctx context.Context, cachePath string) RWLocker
+	NewNSLockFn func(cachePath string) RWLocker
 }
 
 // Inits the disk cache dir if it is not initialized already.
@@ -156,15 +162,17 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
 	cache := diskCache{
-		dir:           dir,
-		triggerGC:     make(chan struct{}, 1),
-		stats:         CacheDiskStats{Dir: dir},
-		quotaPct:      quotaPct,
-		after:         config.After,
-		lowWatermark:  config.WatermarkLow,
-		highWatermark: config.WatermarkHigh,
-		enableRange:   config.Range,
-		online:        1,
+		dir:              dir,
+		triggerGC:        make(chan struct{}, 1),
+		stats:            CacheDiskStats{Dir: dir},
+		quotaPct:         quotaPct,
+		after:            config.After,
+		lowWatermark:     config.WatermarkLow,
+		highWatermark:    config.WatermarkHigh,
+		enableRange:      config.Range,
+		commitWriteback:  config.CommitWriteback,
+		retryWritebackCh: make(chan ObjectInfo, 10000),
+		online:           1,
 		pool: sync.Pool{
 			New: func() interface{} {
 				b := disk.AlignedBlock(int(cacheBlkSize))
@@ -174,9 +182,12 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 		nsMutex: newNSLock(false),
 	}
 	go cache.purgeWait(ctx)
+	if cache.commitWriteback {
+		go cache.scanCacheWritebackFailures(ctx)
+	}
 	cache.diskSpaceAvailable(0) // update if cache usage is already high.
-	cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
-		return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
+	cache.NewNSLockFn = func(cachePath string) RWLocker {
+		return cache.nsMutex.NewNSLock(nil, cachePath, "")
 	}
 	return &cache, nil
 }
@@ -323,6 +334,12 @@ func (c *diskCache) purge(ctx context.Context) {
 		// stat all cached file ranges and cacheDataFile.
 		cachedFiles := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, name))
 		objInfo := meta.ToObjectInfo("", "")
+		// prevent gc from clearing un-synced commits. This metadata is present when
+		// cache writeback commit setting is enabled.
+		status, ok := objInfo.UserDefined[writeBackStatusHeader]
+		if ok && status != CommitComplete.String() {
+			return nil
+		}
 		cc := cacheControlOpts(objInfo)
 		for fname, fi := range cachedFiles {
 			if cc != nil {
@@ -419,8 +436,8 @@ func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectI
 // if partial object is cached.
 func (c *diskCache) statCachedMeta(ctx context.Context, cacheObjPath string) (meta *cacheMeta, partial bool, numHits int, err error) {
 
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err = cLock.GetRLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cacheObjPath)
+	if err = cLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return
 	}
 
@@ -501,8 +518,8 @@ func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (meta *c
 // incHitsOnly is true if metadata update is incrementing only the hit counter
 func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
 	cachedPath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cachedPath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cachedPath)
+	if err := cLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		return err
 	}
 	defer cLock.Unlock()
@@ -524,7 +541,11 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	}
 	defer f.Close()
 
-	m := &cacheMeta{Version: cacheMetaVersion}
+	m := &cacheMeta{
+		Version: cacheMetaVersion,
+		Bucket:  bucket,
+		Object:  object,
+	}
 	if err := jsonLoad(f, m); err != nil && err != io.EOF {
 		return err
 	}
@@ -538,12 +559,15 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 			}
 			m.Ranges[rs.String(actualSize)] = rsFileName
 		}
-	} else {
+	}
+	if rs == nil && !incHitsOnly {
 		// this is necessary cleanup of range files if entire object is cached.
-		for _, f := range m.Ranges {
-			removeAll(pathJoin(cachedPath, f))
+		if _, err := os.Stat(pathJoin(cachedPath, cacheDataFile)); err == nil {
+			for _, f := range m.Ranges {
+				removeAll(pathJoin(cachedPath, f))
+			}
+			m.Ranges = nil
 		}
-		m.Ranges = nil
 	}
 	m.Stat.Size = actualSize
 	m.Stat.ModTime = UTCNow()
@@ -558,7 +582,6 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 			m.Meta["etag"] = etag
 		}
 	}
-
 	m.Hits++
 
 	m.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
@@ -570,22 +593,22 @@ func getCacheSHADir(dir, bucket, object string) string {
 }
 
 // Cache data to disk with bitrot checksum added for each block of 1MB
-func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Reader, size uint64) (int64, error) {
+func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Reader, size uint64) (int64, string, error) {
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
-		return 0, err
+		return 0, "", err
 	}
 	filePath := pathJoin(cachePath, fileName)
 
 	if filePath == "" || reader == nil {
-		return 0, errInvalidArgument
+		return 0, "", errInvalidArgument
 	}
 
 	if err := checkPathLength(filePath); err != nil {
-		return 0, err
+		return 0, "", err
 	}
 	f, err := os.Create(filePath)
 	if err != nil {
-		return 0, osErrToFileErr(err)
+		return 0, "", osErrToFileErr(err)
 	}
 	defer f.Close()
 
@@ -595,12 +618,12 @@ func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Rea
 
 	bufp := c.pool.Get().(*[]byte)
 	defer c.pool.Put(bufp)
-
+	md5Hash := md5.New()
 	var n, n2 int
 	for {
 		n, err = io.ReadFull(reader, *bufp)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
-			return 0, err
+			return 0, "", err
 		}
 		eof := err == io.EOF || err == io.ErrUnexpectedEOF
 		if n == 0 && size != 0 {
@@ -609,21 +632,28 @@ func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Rea
 		}
 		h.Reset()
 		if _, err = h.Write((*bufp)[:n]); err != nil {
-			return 0, err
+			return 0, "", err
 		}
 		hashBytes := h.Sum(nil)
+		// compute md5Hash of original data stream if writeback commit to cache
+		if c.commitWriteback {
+			if _, err = md5Hash.Write((*bufp)[:n]); err != nil {
+				return 0, "", err
+			}
+		}
 		if _, err = f.Write(hashBytes); err != nil {
-			return 0, err
+			return 0, "", err
 		}
 		if n2, err = f.Write((*bufp)[:n]); err != nil {
-			return 0, err
+			return 0, "", err
 		}
 		bytesWritten += int64(n2)
 		if eof {
 			break
 		}
 	}
-	return bytesWritten, nil
+
+	return bytesWritten, base64.StdEncoding.EncodeToString(md5Hash.Sum(nil)), nil
 }
 
 func newCacheEncryptReader(content io.Reader, bucket, object string, metadata map[string]string) (r io.Reader, err error) {
@@ -660,41 +690,41 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 }
 
 // Caches the object to disk
-func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
+func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) (oi ObjectInfo, err error) {
 	if !c.diskSpaceAvailable(size) {
 		io.Copy(ioutil.Discard, data)
-		return errDiskFull
+		return oi, errDiskFull
 	}
 	cachePath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cachePath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
-		return err
+	cLock := c.NewNSLockFn(cachePath)
+	if err := cLock.GetLock(ctx, globalOperationTimeout); err != nil {
+		return oi, err
 	}
 	defer cLock.Unlock()
 
 	meta, _, numHits, err := c.statCache(ctx, cachePath)
 	// Case where object not yet cached
 	if os.IsNotExist(err) && c.after >= 1 {
-		return c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
+		return oi, c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
 	}
 	// Case where object already has a cache metadata entry but not yet cached
 	if err == nil && numHits < c.after {
 		cETag := extractETag(meta.Meta)
 		bETag := extractETag(opts.UserDefined)
 		if cETag == bETag {
-			return c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
+			return oi, c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
 		}
 		incHitsOnly = true
 	}
 
 	if rs != nil {
-		return c.putRange(ctx, bucket, object, data, size, rs, opts)
+		return oi, c.putRange(ctx, bucket, object, data, size, rs, opts)
 	}
 	if !c.diskSpaceAvailable(size) {
-		return errDiskFull
+		return oi, errDiskFull
 	}
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
-		return err
+		return oi, err
 	}
 	var metadata = cloneMSS(opts.UserDefined)
 	var reader = data
@@ -702,25 +732,39 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 	if globalCacheKMS != nil {
 		reader, err = newCacheEncryptReader(data, bucket, object, metadata)
 		if err != nil {
-			return err
+			return oi, err
 		}
 		actualSize, _ = sio.EncryptedSize(uint64(size))
 	}
-	n, err := c.bitrotWriteToCache(cachePath, cacheDataFile, reader, actualSize)
+	n, md5sum, err := c.bitrotWriteToCache(cachePath, cacheDataFile, reader, actualSize)
 	if IsErr(err, baseErrs...) {
 		// take the cache drive offline
 		c.setOffline()
 	}
 	if err != nil {
 		removeAll(cachePath)
-		return err
+		return oi, err
 	}
 
 	if actualSize != uint64(n) {
 		removeAll(cachePath)
-		return IncompleteBody{Bucket: bucket, Object: object}
+		return oi, IncompleteBody{Bucket: bucket, Object: object}
 	}
-	return c.saveMetadata(ctx, bucket, object, metadata, n, nil, "", incHitsOnly)
+	if c.commitWriteback {
+		metadata["content-md5"] = md5sum
+		if md5bytes, err := base64.StdEncoding.DecodeString(md5sum); err == nil {
+			metadata["etag"] = hex.EncodeToString(md5bytes)
+		}
+		metadata[writeBackStatusHeader] = CommitPending.String()
+	}
+	return ObjectInfo{
+			Bucket:      bucket,
+			Name:        object,
+			ETag:        metadata["etag"],
+			Size:        n,
+			UserDefined: metadata,
+		},
+		c.saveMetadata(ctx, bucket, object, metadata, n, nil, "", incHitsOnly)
 }
 
 // Caches the range to disk
@@ -751,7 +795,7 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
 
 	}
 	cacheFile := MustGetUUID()
-	n, err := c.bitrotWriteToCache(cachePath, cacheFile, reader, actualSize)
+	n, _, err := c.bitrotWriteToCache(cachePath, cacheFile, reader, actualSize)
 	if IsErr(err, baseErrs...) {
 		// take the cache drive offline
 		c.setOffline()
@@ -866,8 +910,8 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 // Get returns ObjectInfo and reader for object from disk cache
 func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, numHits int, err error) {
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err := cLock.GetRLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cacheObjPath)
+	if err := cLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return nil, numHits, err
 	}
 
@@ -930,8 +974,8 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 
 // Deletes the cached object
 func (c *diskCache) delete(ctx context.Context, cacheObjPath string) (err error) {
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cacheObjPath)
+	if err := cLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		return err
 	}
 	defer cLock.Unlock()
@@ -951,3 +995,36 @@ func (c *diskCache) Exists(ctx context.Context, bucket, object string) bool {
 	}
 	return true
 }
+
+// queues writeback upload failures on server startup
+func (c *diskCache) scanCacheWritebackFailures(ctx context.Context) {
+	defer close(c.retryWritebackCh)
+	filterFn := func(name string, typ os.FileMode) error {
+		if name == minioMetaBucket {
+			// Proceed to next file.
+			return nil
+		}
+		cacheDir := pathJoin(c.dir, name)
+		meta, _, _, err := c.statCachedMeta(ctx, cacheDir)
+		if err != nil {
+			return nil
+		}
+
+		objInfo := meta.ToObjectInfo("", "")
+		status, ok := objInfo.UserDefined[writeBackStatusHeader]
+		if !ok || status == CommitComplete.String() {
+			return nil
+		}
+		select {
+		case c.retryWritebackCh <- objInfo:
+		default:
+		}
+
+		return nil
+	}
+
+	if err := readDirFilterFn(c.dir, filterFn); err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+}

cmd/disk-cache.go

@@ -22,24 +22,47 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/minio/minio/cmd/config/cache"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/sync/errgroup"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
 const (
-	cacheBlkSize    = int64(1 * 1024 * 1024)
-	cacheGCInterval = time.Minute * 30
+	cacheBlkSize          = 1 << 20
+	cacheGCInterval       = time.Minute * 30
+	writeBackStatusHeader = ReservedMetadataPrefixLower + "write-back-status"
+	writeBackRetryHeader  = ReservedMetadataPrefixLower + "write-back-retry"
 )
 
+type cacheCommitStatus string
+
+const (
+	// CommitPending - cache writeback with backend is pending.
+	CommitPending cacheCommitStatus = "pending"
+
+	// CommitComplete - cache writeback completed ok.
+	CommitComplete cacheCommitStatus = "complete"
+
+	// CommitFailed - cache writeback needs a retry.
+	CommitFailed cacheCommitStatus = "failed"
+)
+
+// String returns string representation of status
+func (s cacheCommitStatus) String() string {
+	return string(s)
+}
+
 // CacheStorageInfo - represents total, free capacity of
 // underlying cache storage.
 type CacheStorageInfo struct {
@@ -69,11 +92,14 @@ type cacheObjects struct {
 	exclude []string
 	// number of accesses after which to cache an object
 	after int
+	// commit objects in async manner
+	commitWriteback bool
 	// if true migration is in progress from v1 to v2
 	migrating bool
 	// mutex to protect migration bool
 	migMutex sync.Mutex
-
+	// retry queue for writeback cache mode to reattempt upload to backend
+	wbRetryCh chan ObjectInfo
 	// Cache stats
 	cacheStats *CacheStats
 
@@ -305,12 +331,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		return bkReader, bkErr
 	}
 
-	if rs != nil {
+	if rs != nil && !dcache.enableRange {
 		go func() {
 			// if range caching is disabled, download entire object.
-			if !dcache.enableRange {
-				rs = nil
-			}
+			rs = nil
 			// fill cache in the background for range GET requests
 			bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, rs, h, lockType, opts)
 			if bErr != nil {
@@ -335,9 +359,9 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	teeReader := io.TeeReader(bkReader, pipeWriter)
 	userDefined := getMetadata(bkReader.ObjInfo)
 	go func() {
-		putErr := dcache.Put(ctx, bucket, object,
+		_, putErr := dcache.Put(ctx, bucket, object,
 			io.LimitReader(pipeReader, bkReader.ObjInfo.Size),
-			bkReader.ObjInfo.Size, nil, ObjectOptions{
+			bkReader.ObjInfo.Size, rs, ObjectOptions{
 				UserDefined: userDefined,
 			}, false)
 		// close the write end of the pipe, so the error gets
@@ -631,7 +655,14 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 		dcache.Delete(ctx, bucket, object)
 		return putObjectFn(ctx, bucket, object, r, opts)
 	}
-
+	if c.commitWriteback {
+		oi, err := dcache.Put(ctx, bucket, object, r, r.Size(), nil, opts, false)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		go c.uploadObject(GlobalContext, oi)
+		return oi, nil
+	}
 	objInfo, err = putObjectFn(ctx, bucket, object, r, opts)
 
 	if err == nil {
@@ -649,9 +680,68 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 			}
 		}()
 	}
-
 	return objInfo, err
+}
 
+// upload cached object to backend in async commit mode.
+func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
+	dcache, err := c.getCacheToLoc(ctx, oi.Bucket, oi.Name)
+	if err != nil {
+		// disk cache could not be located.
+		logger.LogIf(ctx, fmt.Errorf("Could not upload %s/%s to backend: %w", oi.Bucket, oi.Name, err))
+		return
+	}
+	cReader, _, bErr := dcache.Get(ctx, oi.Bucket, oi.Name, nil, http.Header{}, ObjectOptions{})
+	if bErr != nil {
+		return
+	}
+	defer cReader.Close()
+
+	if cReader.ObjInfo.ETag != oi.ETag {
+		return
+	}
+	st := cacheCommitStatus(oi.UserDefined[writeBackStatusHeader])
+	if st == CommitComplete || st.String() == "" {
+		return
+	}
+	hashReader, err := hash.NewReader(cReader, oi.Size, "", "", oi.Size, globalCLIContext.StrictS3Compat)
+	if err != nil {
+		return
+	}
+	var opts ObjectOptions
+	opts.UserDefined = make(map[string]string)
+	opts.UserDefined[xhttp.ContentMD5] = oi.UserDefined["content-md5"]
+	objInfo, err := c.InnerPutObjectFn(ctx, oi.Bucket, oi.Name, NewPutObjReader(hashReader, nil, nil), opts)
+	wbCommitStatus := CommitComplete
+	if err != nil {
+		wbCommitStatus = CommitFailed
+	}
+
+	meta := cloneMSS(cReader.ObjInfo.UserDefined)
+	retryCnt := 0
+	if wbCommitStatus == CommitFailed {
+		retryCnt, _ = strconv.Atoi(meta[writeBackRetryHeader])
+		retryCnt++
+		meta[writeBackRetryHeader] = strconv.Itoa(retryCnt)
+	} else {
+		delete(meta, writeBackRetryHeader)
+	}
+	meta[writeBackStatusHeader] = wbCommitStatus.String()
+	meta["etag"] = oi.ETag
+	dcache.SaveMetadata(ctx, oi.Bucket, oi.Name, meta, objInfo.Size, nil, "", false)
+	if retryCnt > 0 {
+		// slow down retries
+		time.Sleep(time.Second * time.Duration(retryCnt%10+1))
+		c.queueWritebackRetry(oi)
+	}
+}
+
+func (c *cacheObjects) queueWritebackRetry(oi ObjectInfo) {
+	select {
+	case c.wbRetryCh <- oi:
+		c.uploadObject(GlobalContext, oi)
+	default:
+	}
 }
 
 // Returns cacheObjects for use by Server.
@@ -662,12 +752,13 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		return nil, err
 	}
 	c := &cacheObjects{
-		cache:      cache,
-		exclude:    config.Exclude,
-		after:      config.After,
-		migrating:  migrateSw,
-		migMutex:   sync.Mutex{},
-		cacheStats: newCacheStats(),
+		cache:           cache,
+		exclude:         config.Exclude,
+		after:           config.After,
+		migrating:       migrateSw,
+		migMutex:        sync.Mutex{},
+		commitWriteback: config.CommitWriteback,
+		cacheStats:      newCacheStats(),
 		InnerGetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
 		},
@@ -701,6 +792,15 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		go c.migrateCacheFromV1toV2(ctx)
 	}
 	go c.gc(ctx)
+	if c.commitWriteback {
+		c.wbRetryCh = make(chan ObjectInfo, 10000)
+		go func() {
+			<-GlobalContext.Done()
+			close(c.wbRetryCh)
+		}()
+		go c.queuePendingWriteback(ctx)
+	}
+
 	return c, nil
 }
 
@@ -726,3 +826,25 @@ func (c *cacheObjects) gc(ctx context.Context) {
 		}
 	}
 }
+
+// queues any pending or failed async commits when server restarts
+func (c *cacheObjects) queuePendingWriteback(ctx context.Context) {
+	for _, dcache := range c.cache {
+		if dcache != nil {
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				case oi, ok := <-dcache.retryWritebackCh:
+					if !ok {
+						goto next
+					}
+					c.queueWritebackRetry(oi)
+				default:
+					time.Sleep(time.Second * 1)
+				}
+			}
+		next:
+		}
+	}
+}

cmd/encryption-v1_test.go

@@ -19,10 +19,14 @@ package cmd
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/json"
+	"fmt"
 	"net/http"
+	"os"
 	"testing"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/klauspost/compress/zstd"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/sio"
@@ -622,3 +626,89 @@ func TestGetDefaultOpts(t *testing.T) {
 		}
 	}
 }
+func Test_decryptObjectInfo(t *testing.T) {
+	var testSet []struct {
+		Bucket  string
+		Name    string
+		UserDef map[string]string
+	}
+	file, err := os.Open("testdata/decryptObjectInfo.json.zst")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	dec, err := zstd.NewReader(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dec.Close()
+	js := json.NewDecoder(dec)
+	err = js.Decode(&testSet)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574")
+	defer os.Setenv("MINIO_KMS_MASTER_KEY", "")
+	GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var dst [32]byte
+	for i := range testSet {
+		t.Run(fmt.Sprint("case-", i), func(t *testing.T) {
+			test := &testSet[i]
+			_, err := decryptObjectInfo(dst[:], test.Bucket, test.Name, test.UserDef)
+			if err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+func Benchmark_decryptObjectInfo(b *testing.B) {
+	var testSet []struct {
+		Bucket  string
+		Name    string
+		UserDef map[string]string
+	}
+	file, err := os.Open("testdata/decryptObjectInfo.json.zst")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer file.Close()
+	dec, err := zstd.NewReader(file)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer dec.Close()
+	js := json.NewDecoder(dec)
+	err = js.Decode(&testSet)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574")
+	defer os.Setenv("MINIO_KMS_MASTER_KEY", "")
+	GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{})
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	b.SetBytes(int64(len(testSet)))
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			var dst [32]byte
+			for i := range testSet {
+				test := &testSet[i]
+				_, err := decryptObjectInfo(dst[:], test.Bucket, test.Name, test.UserDef)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		}
+	})
+}

cmd/endpoint.go

@@ -17,7 +17,9 @@
 package cmd
 
 import (
+	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -26,17 +28,20 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/mountinfo"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // EndpointType - enum for endpoint type.
@@ -269,6 +274,52 @@ func (l EndpointServerSets) Hostnames() []string {
 	return foundSet.ToSlice()
 }
 
+// hostsSorted will return all hosts found.
+// The LOCAL host will be nil, but the indexes of all hosts should
+// remain consistent across the cluster.
+func (l EndpointServerSets) hostsSorted() []*xnet.Host {
+	peers, localPeer := l.peers()
+	sort.Strings(peers)
+	hosts := make([]*xnet.Host, len(peers))
+	for i, hostStr := range peers {
+		if hostStr == localPeer {
+			continue
+		}
+		host, err := xnet.ParseHost(hostStr)
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
+			continue
+		}
+		hosts[i] = host
+	}
+
+	return hosts
+}
+
+// peers will return all peers, including local.
+// The local peer is returned as a separate string.
+func (l EndpointServerSets) peers() (peers []string, local string) {
+	allSet := set.NewStringSet()
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.Type() != URLEndpointType {
+				continue
+			}
+
+			peer := endpoint.Host
+			if endpoint.IsLocal {
+				if _, port := mustSplitHostPort(peer); port == globalMinioPort {
+					local = peer
+				}
+			}
+
+			allSet.Add(peer)
+		}
+	}
+
+	return allSet.ToSlice(), local
+}
+
 // Endpoints - list of same type of endpoint.
 type Endpoints []Endpoint
 
@@ -712,28 +763,6 @@ func GetLocalPeer(endpointServerSets EndpointServerSets) (localPeer string) {
 	return peerSet.ToSlice()[0]
 }
 
-// GetRemotePeers - get hosts information other than this minio service.
-func GetRemotePeers(endpointServerSets EndpointServerSets) []string {
-	peerSet := set.NewStringSet()
-	for _, ep := range endpointServerSets {
-		for _, endpoint := range ep.Endpoints {
-			if endpoint.Type() != URLEndpointType {
-				continue
-			}
-
-			peer := endpoint.Host
-			if endpoint.IsLocal {
-				if _, port := mustSplitHostPort(peer); port == globalMinioPort {
-					continue
-				}
-			}
-
-			peerSet.Add(peer)
-		}
-	}
-	return peerSet.ToSlice()
-}
-
 // GetProxyEndpointLocalIndex returns index of the local proxy endpoint
 func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
 	for i, pep := range proxyEps {
@@ -744,8 +773,74 @@ func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
 	return -1
 }
 
+func httpDo(clnt *http.Client, req *http.Request, f func(*http.Response, error) error) error {
+	ctx, cancel := context.WithTimeout(GlobalContext, 200*time.Millisecond)
+	defer cancel()
+
+	// Run the HTTP request in a goroutine and pass the response to f.
+	c := make(chan error, 1)
+	req = req.WithContext(ctx)
+	go func() { c <- f(clnt.Do(req)) }()
+	select {
+	case <-ctx.Done():
+		<-c // Wait for f to return.
+		return ctx.Err()
+	case err := <-c:
+		return err
+	}
+}
+
+func getOnlineProxyEndpointIdx() int {
+	type reqIndex struct {
+		Request *http.Request
+		Idx     int
+	}
+
+	proxyRequests := make(map[*http.Client]reqIndex, len(globalProxyEndpoints))
+	for i, proxyEp := range globalProxyEndpoints {
+		proxyEp := proxyEp
+		serverURL := &url.URL{
+			Scheme: proxyEp.Scheme,
+			Host:   proxyEp.Host,
+			Path:   pathJoin(healthCheckPathPrefix, healthCheckLivenessPath),
+		}
+
+		req, err := http.NewRequest(http.MethodGet, serverURL.String(), nil)
+		if err != nil {
+			continue
+		}
+
+		proxyRequests[&http.Client{
+			Transport: proxyEp.Transport,
+		}] = reqIndex{
+			Request: req,
+			Idx:     i,
+		}
+	}
+
+	for c, r := range proxyRequests {
+		if err := httpDo(c, r.Request, func(resp *http.Response, err error) error {
+			if err != nil {
+				return err
+			}
+			xhttp.DrainBody(resp.Body)
+			if resp.StatusCode != http.StatusOK {
+				return errors.New(resp.Status)
+			}
+			if v := resp.Header.Get(xhttp.MinIOServerStatus); v == unavailable {
+				return errors.New(v)
+			}
+			return nil
+		}); err != nil {
+			continue
+		}
+		return r.Idx
+	}
+	return -1
+}
+
 // GetProxyEndpoints - get all endpoints that can be used to proxy list request.
-func GetProxyEndpoints(endpointServerSets EndpointServerSets) ([]ProxyEndpoint, error) {
+func GetProxyEndpoints(endpointServerSets EndpointServerSets) []ProxyEndpoint {
 	var proxyEps []ProxyEndpoint
 
 	proxyEpSet := set.NewStringSet()
@@ -779,7 +874,7 @@ func GetProxyEndpoints(endpointServerSets EndpointServerSets) ([]ProxyEndpoint,
 			})
 		}
 	}
-	return proxyEps, nil
+	return proxyEps
 }
 
 func updateDomainIPs(endPoints set.StringSet) {

cmd/endpoint_test.go

@@ -380,24 +380,28 @@ func TestGetRemotePeers(t *testing.T) {
 	testCases := []struct {
 		endpointArgs   []string
 		expectedResult []string
+		expectedLocal  string
 	}{
-		{[]string{"/d1", "/d2", "d3", "d4"}, []string{}},
-		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000"}},
-		{[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000"}},
-		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000"}},
-		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9001", "localhost:9002", "localhost:9003"}},
+		{[]string{"/d1", "/d2", "d3", "d4"}, []string{}, ""},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:9000"}, "localhost:9000"},
+		{[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000", "localhost:9000"}, "localhost:9000"},
+		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000", "localhost:9000"}, "localhost:9000"},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9000", "localhost:9001", "localhost:9002", "localhost:9003"}, "localhost:9000"},
 	}
 
 	for _, testCase := range testCases {
 		zendpoints := mustGetZoneEndpoints(testCase.endpointArgs...)
 		if !zendpoints[0].Endpoints[0].IsLocal {
 			if err := zendpoints[0].Endpoints.UpdateIsLocal(false); err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
+				t.Errorf("error: expected = <nil>, got = %v", err)
 			}
 		}
-		remotePeers := GetRemotePeers(zendpoints)
+		remotePeers, local := zendpoints.peers()
 		if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
-			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
+			t.Errorf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
+		}
+		if local != testCase.expectedLocal {
+			t.Errorf("expected: %v, got: %v", testCase.expectedLocal, local)
 		}
 	}
 }

cmd/erasure-bucket.go

@@ -158,6 +158,7 @@ func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs
 // DeleteBucket - deletes a bucket.
 func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
 	// Collect if all disks report volume not found.
+	defer ObjectPathUpdated(bucket + slashSeparator)
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))

cmd/erasure-common.go

@@ -51,6 +51,7 @@ func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 			}
 			di, err := disks[i-1].DiskInfo(context.Background())
 			if err != nil || di.Healing {
+
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.
 				//

cmd/erasure-encode_test.go

@@ -195,7 +195,7 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
 			if disk == OfflineDisk {
 				continue
 			}
-			disk.DeleteFile(context.Background(), "testbucket", "object")
+			disk.Delete(context.Background(), "testbucket", "object", false)
 			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)

cmd/erasure-healing-common.go

@@ -26,22 +26,25 @@ import (
 // commonTime returns a maximally occurring time from a list of time.
 func commonTime(modTimes []time.Time) (modTime time.Time, count int) {
 	var maxima int // Counter for remembering max occurrence of elements.
-	timeOccurenceMap := make(map[time.Time]int)
+	timeOccurenceMap := make(map[int64]int)
 	// Ignore the uuid sentinel and count the rest.
 	for _, time := range modTimes {
 		if time.Equal(timeSentinel) {
 			continue
 		}
-		timeOccurenceMap[time]++
+		timeOccurenceMap[time.UnixNano()]++
 	}
+
 	// Find the common cardinality from previously collected
 	// occurrences of elements.
-	for time, count := range timeOccurenceMap {
-		if count > maxima || (count == maxima && time.After(modTime)) {
+	for nano, count := range timeOccurenceMap {
+		t := time.Unix(0, nano)
+		if count > maxima || (count == maxima && t.After(modTime)) {
 			maxima = count
-			modTime = time
+			modTime = t
 		}
 	}
+
 	// Return the collected common uuid.
 	return modTime, maxima
 }
@@ -160,6 +163,32 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	dataErrs := make([]error, len(onlineDisks))
 
+	inconsistent := 0
+	for i, meta := range partsMetadata {
+		if !meta.IsValid() {
+			// Since for majority of the cases erasure.Index matches with erasure.Distribution we can
+			// consider the offline disks as consistent.
+			continue
+		}
+		if len(meta.Erasure.Distribution) != len(onlineDisks) {
+			// Erasure distribution seems to have lesser
+			// number of items than number of online disks.
+			inconsistent++
+			continue
+		}
+		if meta.Erasure.Distribution[i] != meta.Erasure.Index {
+			// Mismatch indexes with distribution order
+			inconsistent++
+		}
+	}
+
+	erasureDistributionReliable := true
+	if inconsistent > len(partsMetadata)/2 {
+		// If there are too many inconsistent files, then we can't trust erasure.Distribution (most likely
+		// because of bugs found in CopyObject/PutObjectTags) https://github.com/minio/minio/pull/10772
+		erasureDistributionReliable = false
+	}
+
 	for i, onlineDisk := range onlineDisks {
 		if errs[i] != nil {
 			dataErrs[i] = errs[i]
@@ -169,6 +198,28 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			dataErrs[i] = errDiskNotFound
 			continue
 		}
+		if erasureDistributionReliable {
+			meta := partsMetadata[i]
+			if !meta.IsValid() {
+				continue
+			}
+
+			if len(meta.Erasure.Distribution) != len(onlineDisks) {
+				// Erasure distribution is not the same as onlineDisks
+				// attempt a fix if possible, assuming other entries
+				// might have the right erasure distribution.
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
+				continue
+			}
+
+			// Since erasure.Distribution is trustable we can fix the mismatching erasure.Index
+			if meta.Erasure.Distribution[i] != meta.Erasure.Index {
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
+				continue
+			}
+		}
 
 		switch scanMode {
 		case madmin.HealDeepScan:

cmd/erasure-healing-common_test.go

@@ -211,7 +211,7 @@ func TestListOnlineDisks(t *testing.T) {
 				// and check if that disk
 				// appears in outDatedDisks.
 				tamperedIndex = index
-				dErr := erasureDisks[index].DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+				dErr := erasureDisks[index].Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
 				if dErr != nil {
 					t.Fatalf("Test %d: Failed to delete %s - %v", i+1,
 						filepath.Join(object, "part.1"), dErr)

cmd/erasure-healing.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"path"
 	"sync"
 	"time"
 
@@ -221,17 +222,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	partsMetadata []FileInfo, errs []error, latestFileInfo FileInfo,
 	dryRun bool, remove bool, scanMode madmin.HealScanMode) (result madmin.HealResultItem, err error) {
 
-	for i, metadata := range shufflePartsMetadata(partsMetadata, latestFileInfo.Erasure.Distribution) {
-		if !metadata.IsValid() {
-			continue
-		}
-		if i != metadata.Erasure.Index-1 {
-			// FIXME: fix in the next release for objects which erasure.Index does not match
-			// with expected distribution.
-			return result, fmt.Errorf("Unable to heal object %s, disk ordering issue detected", pathJoin(bucket, object))
-		}
-	}
-
 	dataBlocks := latestFileInfo.Erasure.DataBlocks
 
 	storageDisks := er.getDisks()
@@ -315,6 +305,10 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		})
 	}
 
+	if isAllNotFound(errs) {
+		return defaultHealResult(latestFileInfo, storageDisks, storageEndpoints, errs, bucket, object), nil
+	}
+
 	// If less than read quorum number of disks have all the parts
 	// of the data, we can't reconstruct the erasure-coded data.
 	if numAvailableDisks < dataBlocks {
@@ -353,6 +347,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	if pErr != nil {
 		return result, toObjectErr(pErr, bucket, object)
 	}
+	defer ObjectPathUpdated(pathJoin(bucket, object))
 
 	cleanFileInfo := func(fi FileInfo) FileInfo {
 		// Returns a copy of the 'fi' with checksums and parts nil'ed.
@@ -529,10 +524,11 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 				wg.Add(1)
 				go func(index int, disk StorageAPI) {
 					defer wg.Done()
-					_ = disk.DeleteFile(ctx, bucket, object)
+					_ = disk.Delete(ctx, bucket, object, false)
 				}(index, disk)
 			}
 			wg.Wait()
+			ObjectPathUpdated(path.Join(bucket, object))
 		}
 	}
 
@@ -555,7 +551,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 			hr.After.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateCorrupt}
 		}
 	}
-	if dryRun || danglingObject {
+	if dryRun || danglingObject || isAllNotFound(errs) {
 		return hr, nil
 	}
 	for i, err := range errs {
@@ -660,9 +656,23 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
 	return g.Wait()
 }
 
+// isAllNotFound will return if any element of the error slice is not
+// errFileNotFound, errFileVersionNotFound or errVolumeNotFound.
+// A 0 length slice will always return false.
+func isAllNotFound(errs []error) bool {
+	for _, err := range errs {
+		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) || errors.Is(err, errFileVersionNotFound) {
+			continue
+		}
+		return false
+	}
+	return len(errs) > 0
+}
+
 // ObjectDir is considered dangling/corrupted if any only
 // if total disks - a combination of corrupted and missing
 // files is lesser than N/2+1 number of disks.
+// If no files were found false will be returned.
 func isObjectDirDangling(errs []error) (ok bool) {
 	var found int
 	var notFound int
@@ -679,7 +689,8 @@ func isObjectDirDangling(errs []error) (ok bool) {
 			otherFound++
 		}
 	}
-	return found+foundNotEmpty+otherFound < notFound
+	found = found + foundNotEmpty + otherFound
+	return found < notFound && found > 0
 }
 
 // Object is considered dangling/corrupted if any only
@@ -759,6 +770,10 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	// Read metadata files from all the disks
 	partsMetadata, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID)
 
+	if isAllNotFound(errs) {
+		// Nothing to do
+		return defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs, bucket, object), nil
+	}
 	// Check if the object is dangling, if yes and user requested
 	// remove we simply delete it from namespace.
 	if m, ok := isObjectDangling(partsMetadata, errs, []error{}); ok {

cmd/erasure-healing_test.go

@@ -201,7 +201,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	er := z.serverSets[0].sets[0]
 	erasureDisks := er.getDisks()
 	firstDisk := erasureDisks[0]
-	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -221,12 +221,12 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
 	}
 
-	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
 	if err != nil {
 		t.Errorf("Failure during deleting part.1 - %v", err)
 	}
 
-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader([]byte{}))
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
 	if err != nil {
 		t.Errorf("Failure during creating part.1 - %v", err)
 	}
@@ -246,13 +246,13 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("FileInfo not equal after healing")
 	}
 
-	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
 	if err != nil {
 		t.Errorf("Failure during deleting part.1 - %v", err)
 	}
 
 	bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader(bdata))
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
 	if err != nil {
 		t.Errorf("Failure during creating part.1 - %v", err)
 	}
@@ -275,7 +275,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	// Test 4: checks if HealObject returns an error when xl.meta is not found
 	// in more than read quorum number of disks, to create a corrupted situation.
 	for i := 0; i <= len(er.getDisks())/2; i++ {
-		er.getDisks()[i].DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
+		er.getDisks()[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
 	}
 
 	// Try healing now, expect to receive errFileNotFound.
@@ -351,7 +351,7 @@ func TestHealObjectErasure(t *testing.T) {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}
 
-	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}

cmd/erasure-metadata-utils.go

@@ -153,28 +153,54 @@ func readVersionFromDisks(ctx context.Context, disks []StorageAPI, bucket, objec
 	return metadataArray, g.Wait()
 }
 
-// Return disks ordered by the meta.Erasure.Index information.
-func shuffleDisksByIndex(disks []StorageAPI, metaArr []FileInfo) (shuffledDisks []StorageAPI) {
+func shuffleDisksAndPartsMetadataByIndex(disks []StorageAPI, metaArr []FileInfo, distribution []int) (shuffledDisks []StorageAPI, shuffledPartsMetadata []FileInfo) {
 	shuffledDisks = make([]StorageAPI, len(disks))
+	shuffledPartsMetadata = make([]FileInfo, len(disks))
+	var inconsistent int
 	for i, meta := range metaArr {
 		if disks[i] == nil {
+			// Assuming offline drives as inconsistent,
+			// to be safe and fallback to original
+			// distribution order.
+			inconsistent++
+			continue
+		}
+		// check if erasure distribution order matches the index
+		// position if this is not correct we discard the disk
+		// and move to collect others
+		if distribution[i] != meta.Erasure.Index {
+			inconsistent++ // keep track of inconsistent entries
 			continue
 		}
 		shuffledDisks[meta.Erasure.Index-1] = disks[i]
+		shuffledPartsMetadata[meta.Erasure.Index-1] = metaArr[i]
 	}
-	return shuffledDisks
+
+	// Inconsistent meta info is with in the limit of
+	// expected quorum, proceed with EcIndex based
+	// disk order.
+	if inconsistent < len(disks)/2 {
+		return shuffledDisks, shuffledPartsMetadata
+	}
+
+	// fall back to original distribution based order.
+	return shuffleDisksAndPartsMetadata(disks, metaArr, distribution)
 }
 
-// Return FileInfo slice ordered by the meta.Erasure.Index information.
-func shufflePartsMetadataByIndex(disks []StorageAPI, metaArr []FileInfo) []FileInfo {
-	newMetaArr := make([]FileInfo, len(disks))
-	for i, meta := range metaArr {
-		if disks[i] == nil {
-			continue
-		}
-		newMetaArr[meta.Erasure.Index-1] = metaArr[i]
+// Return shuffled partsMetadata depending on distribution.
+func shuffleDisksAndPartsMetadata(disks []StorageAPI, partsMetadata []FileInfo, distribution []int) (shuffledDisks []StorageAPI, shuffledPartsMetadata []FileInfo) {
+	if distribution == nil {
+		return disks, partsMetadata
 	}
-	return newMetaArr
+	shuffledDisks = make([]StorageAPI, len(disks))
+	shuffledPartsMetadata = make([]FileInfo, len(partsMetadata))
+	// Shuffle slice xl metadata for expected distribution.
+	for index := range partsMetadata {
+		blockIndex := distribution[index]
+		shuffledPartsMetadata[blockIndex-1] = partsMetadata[index]
+		shuffledDisks[blockIndex-1] = disks[index]
+	}
+	return shuffledDisks, shuffledPartsMetadata
 }
 
 // Return shuffled partsMetadata depending on distribution.

cmd/erasure-metadata.go

@@ -90,9 +90,12 @@ func (fi FileInfo) IsValid() bool {
 	}
 	dataBlocks := fi.Erasure.DataBlocks
 	parityBlocks := fi.Erasure.ParityBlocks
+	correctIndexes := (fi.Erasure.Index > 0 &&
+		fi.Erasure.Index <= dataBlocks+parityBlocks &&
+		len(fi.Erasure.Distribution) == (dataBlocks+parityBlocks))
 	return ((dataBlocks >= parityBlocks) &&
 		(dataBlocks != 0) && (parityBlocks != 0) &&
-		(fi.Erasure.Index > 0 && fi.Erasure.Distribution != nil))
+		correctIndexes)
 }
 
 // ToObjectInfo - Converts metadata to object info.
@@ -214,13 +217,15 @@ func (fi FileInfo) ObjectToPartOffset(ctx context.Context, offset int64) (partIn
 
 func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.Time, quorum int) (xmv FileInfo, e error) {
 	metaHashes := make([]string, len(metaArr))
+	h := sha256.New()
 	for i, meta := range metaArr {
 		if meta.IsValid() && meta.ModTime.Equal(modTime) {
-			h := sha256.New()
 			for _, part := range meta.Parts {
 				h.Write([]byte(fmt.Sprintf("part.%d", part.Number)))
 			}
+			h.Write([]byte(fmt.Sprintf("%v", meta.Erasure.Distribution)))
 			metaHashes[i] = hex.EncodeToString(h.Sum(nil))
+			h.Reset()
 		}
 	}
 

cmd/erasure-metadata_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"strconv"
 	"testing"
+	"time"
 
 	humanize "github.com/dustin/go-humanize"
 )
@@ -154,3 +155,48 @@ func TestObjectToPartOffset(t *testing.T) {
 		}
 	}
 }
+
+func TestFindFileInfoInQuorum(t *testing.T) {
+	getNFInfo := func(n int, quorum int, t int64) []FileInfo {
+		fi := newFileInfo("test", 8, 8)
+		fi.AddObjectPart(1, "etag", 100, 100)
+		fi.ModTime = time.Unix(t, 0)
+		fis := make([]FileInfo, n)
+		for i := range fis {
+			fis[i] = fi
+			fis[i].Erasure.Index = i + 1
+			quorum--
+			if quorum == 0 {
+				break
+			}
+		}
+		return fis
+	}
+
+	tests := []struct {
+		fis         []FileInfo
+		modTime     time.Time
+		expectedErr error
+	}{
+		{
+			fis:         getNFInfo(16, 16, 1603863445),
+			modTime:     time.Unix(1603863445, 0),
+			expectedErr: nil,
+		},
+		{
+			fis:         getNFInfo(16, 7, 1603863445),
+			modTime:     time.Unix(1603863445, 0),
+			expectedErr: errErasureReadQuorum,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run("", func(t *testing.T) {
+			_, err := findFileInfoInQuorum(context.Background(), test.fis, test.modTime, 8)
+			if err != test.expectedErr {
+				t.Errorf("Expected %s, got %s", test.expectedErr, err)
+			}
+		})
+	}
+}

cmd/erasure-multipart.go

@@ -81,7 +81,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
 			// requests. xl.meta is the authoritative source of truth on which parts constitute
 			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
-			_ = storageDisks[index].DeleteFile(context.TODO(), minioMetaMultipartBucket, curpartPath)
+			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, false)
 			return nil
 		}, index)
 	}
@@ -302,6 +302,8 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		partsMetadata[i] = fi
 	}
 
+	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(onlineDisks, partsMetadata, fi.Erasure.Distribution)
+
 	var err error
 	// Write updated `xl.meta` to all disks.
 	onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaTmpBucket, tempUploadIDPath, partsMetadata, writeQuorum)
@@ -353,8 +355,8 @@ func (er erasureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObjec
 //
 // Implements S3 compatible Upload Part API.
 func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	if err = uploadIDLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return PartInfo{}, err
 	}
 	readLocked := true
@@ -467,7 +469,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	// PutObjectParts would serialize here updating `xl.meta`
 	uploadIDLock.RUnlock()
 	readLocked = false
-	if err = uploadIDLock.GetLock(globalOperationTimeout); err != nil {
+	if err = uploadIDLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		return PartInfo{}, err
 	}
 	defer uploadIDLock.Unlock()
@@ -548,8 +550,8 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 		UploadID: uploadID,
 	}
 
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	if err := uploadIDLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return MultipartInfo{}, err
 	}
 	defer uploadIDLock.RUnlock()
@@ -596,8 +598,8 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 // ListPartsInfo structure is marshaled directly into XML and
 // replied back to the client.
 func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, e error) {
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	if err := uploadIDLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return ListPartsInfo{}, err
 	}
 	defer uploadIDLock.RUnlock()
@@ -689,8 +691,8 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
 	// Hold read-locks to verify uploaded parts, also disallows
 	// parallel part uploads as well.
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	if err = uploadIDLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer uploadIDLock.RUnlock()
@@ -743,10 +745,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	// Order online disks in accordance with distribution order.
-	onlineDisks = shuffleDisks(onlineDisks, fi.Erasure.Distribution)
-
 	// Order parts metadata in accordance with distribution order.
-	partsMetadata = shufflePartsMetadata(partsMetadata, fi.Erasure.Distribution)
+	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadataByIndex(onlineDisks, partsMetadata, fi.Erasure.Distribution)
 
 	// Save current erasure metadata for validation.
 	var currentFI = fi
@@ -844,8 +844,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	// Hold namespace to complete the transaction
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	if err = lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer lk.Unlock()
@@ -886,8 +886,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 // would be removed from the system, rollback is not possible on this
 // operation.
 func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
-	lk := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		return err
 	}
 	defer lk.Unlock()

cmd/erasure-object.go

@@ -46,9 +46,10 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	if !srcInfo.metadataOnly {
 		return oi, NotImplemented{}
 	}
+
 	defer ObjectPathUpdated(path.Join(dstBucket, dstObject))
-	lk := er.NewNSLock(ctx, dstBucket, dstObject)
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(dstBucket, dstObject)
+	if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer lk.Unlock()
@@ -72,6 +73,8 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
 
+	onlineDisks, metaArr = shuffleDisksAndPartsMetadataByIndex(onlineDisks, metaArr, fi.Erasure.Distribution)
+
 	if fi.Deleted {
 		if srcOpts.VersionID == "" {
 			return oi, toObjectErr(errFileNotFound, srcBucket, srcObject)
@@ -132,15 +135,15 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 
 	// Acquire lock
 	if lockType != noLock {
-		lock := er.NewNSLock(ctx, bucket, object)
+		lock := er.NewNSLock(bucket, object)
 		switch lockType {
 		case writeLock:
-			if err = lock.GetLock(globalOperationTimeout); err != nil {
+			if err = lock.GetLock(ctx, globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.Unlock
 		case readLock:
-			if err = lock.GetRLock(globalOperationTimeout); err != nil {
+			if err = lock.GetRLock(ctx, globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.RUnlock
@@ -174,7 +177,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 
 	pr, pw := io.Pipe()
 	go func() {
-		err := er.getObjectWithFileInfo(ctx, bucket, object, off, length, pw, "", opts, fi, metaArr, onlineDisks)
+		err := er.getObjectWithFileInfo(ctx, bucket, object, off, length, pw, fi, metaArr, onlineDisks)
 		pw.CloseWithError(err)
 	}()
 
@@ -193,8 +196,8 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 // length indicates the total length of the object.
 func (er erasureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
 	// Lock the object before reading.
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return err
 	}
 	defer lk.RUnlock()
@@ -211,16 +214,13 @@ func (er erasureObjects) GetObject(ctx context.Context, bucket, object string, s
 		return errUnexpected
 	}
 
-	return er.getObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
+	return er.getObject(ctx, bucket, object, startOffset, length, writer, opts)
 }
 
-func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions, fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI) error {
-	tmpDisks := onlineDisks
+func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI) error {
 	// Reorder online disks based on erasure distribution order.
-	onlineDisks = shuffleDisksByIndex(tmpDisks, metaArr)
-
 	// Reorder parts metadata based on erasure distribution order.
-	metaArr = shufflePartsMetadataByIndex(tmpDisks, metaArr)
+	onlineDisks, metaArr = shuffleDisksAndPartsMetadataByIndex(onlineDisks, metaArr, fi.Erasure.Distribution)
 
 	// For negative length read everything.
 	if length < 0 {
@@ -324,7 +324,7 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 }
 
 // getObject wrapper for erasure GetObject
-func (er erasureObjects) getObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
+func (er erasureObjects) getObject(ctx context.Context, bucket, object string, startOffset, length int64, writer io.Writer, opts ObjectOptions) error {
 	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
 	if err != nil {
 		return toObjectErr(err, bucket, object)
@@ -337,14 +337,14 @@ func (er erasureObjects) getObject(ctx context.Context, bucket, object string, s
 		return toObjectErr(errMethodNotAllowed, bucket, object)
 	}
 
-	return er.getObjectWithFileInfo(ctx, bucket, object, startOffset, length, writer, etag, opts, fi, metaArr, onlineDisks)
+	return er.getObjectWithFileInfo(ctx, bucket, object, startOffset, length, writer, fi, metaArr, onlineDisks)
 }
 
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
 	// Lock the object before reading.
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return ObjectInfo{}, err
 	}
 	defer lk.RUnlock()
@@ -364,6 +364,24 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	}
 
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
+		if reducedErr == errErasureReadQuorum && bucket != minioMetaBucket {
+			if _, ok := isObjectDangling(metaArr, errs, nil); ok {
+				reducedErr = errFileNotFound
+				if opts.VersionID != "" {
+					reducedErr = errFileVersionNotFound
+				}
+				// Remove the dangling object only when:
+				//  - This is a non versioned bucket
+				//  - This is a versioned bucket and the version ID is passed, the reason
+				//    is that we cannot fetch the ID of the latest version when we don't trust xl.meta
+				if !opts.Versioned || opts.VersionID != "" {
+					er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
+						Name:      object,
+						VersionID: opts.VersionID,
+					})
+				}
+			}
+		}
 		return fi, nil, nil, toObjectErr(reducedErr, bucket, object)
 	}
 
@@ -375,7 +393,6 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	if err != nil {
 		return fi, nil, nil, err
 	}
-
 	return fi, metaArr, onlineDisks, nil
 }
 
@@ -426,6 +443,9 @@ func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry str
 // Similar to rename but renames data from srcEntry to dstEntry at dataDir
 func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dataDir, dstBucket, dstEntry string, writeQuorum int, ignoredErr []error) ([]StorageAPI, error) {
 	dataDir = retainSlash(dataDir)
+	defer ObjectPathUpdated(path.Join(srcBucket, srcEntry))
+	defer ObjectPathUpdated(path.Join(dstBucket, dstEntry))
+
 	g := errgroup.WithNErrs(len(disks))
 
 	// Rename file on all underlying storage disks.
@@ -473,11 +493,12 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, da
 // rename - common function that renamePart and renameObject use to rename
 // the respective underlying storage layer representations.
 func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, writeQuorum int, ignoredErr []error) ([]StorageAPI, error) {
-
 	if isDir {
 		dstEntry = retainSlash(dstEntry)
 		srcEntry = retainSlash(srcEntry)
 	}
+	defer ObjectPathUpdated(path.Join(srcBucket, srcEntry))
+	defer ObjectPathUpdated(path.Join(dstBucket, dstEntry))
 
 	g := errgroup.WithNErrs(len(disks))
 
@@ -584,7 +605,8 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	// Order disks according to erasure distribution
-	onlineDisks := shuffleDisks(storageDisks, fi.Erasure.Distribution)
+	var onlineDisks []StorageAPI
+	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi.Erasure.Distribution)
 
 	erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
 	if err != nil {
@@ -631,8 +653,8 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
 	}
 
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		return ObjectInfo{}, err
 	}
 	defer lk.Unlock()
@@ -704,10 +726,9 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 }
 
 func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, writeQuorum int, fi FileInfo) error {
+	defer ObjectPathUpdated(pathJoin(bucket, object))
 	disks := er.getDisks()
-
 	g := errgroup.WithNErrs(len(disks))
-
 	for index := range disks {
 		index := index
 		g.Go(func() error {
@@ -852,6 +873,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		}
 		errs[objIndex] = reduceWriteQuorumErrs(ctx, diskErrs, objectOpIgnoredErrs, writeQuorums[objIndex])
 		if errs[objIndex] == nil {
+			ObjectPathUpdated(pathJoin(bucket, objects[objIndex].ObjectName))
 			if versions[objIndex].Deleted {
 				dobjects[objIndex] = DeletedObject{
 					DeleteMarker:          versions[objIndex].Deleted,
@@ -891,6 +913,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 // any error as it is not necessary for the handler to reply back a
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+	defer ObjectPathUpdated(path.Join(bucket, object))
 	goi, gerr := er.GetObjectInfo(ctx, bucket, object, opts)
 	if gerr != nil && goi.Name == "" {
 		switch gerr.(type) {
@@ -901,8 +924,8 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	}
 
 	// Acquire a write lock before deleting the object.
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalDeleteOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	if err = lk.GetLock(ctx, globalDeleteOperationTimeout); err != nil {
 		return ObjectInfo{}, err
 	}
 	defer lk.Unlock()
@@ -973,7 +996,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	}
 
 	// List all online disks.
-	_, modTime := listOnlineDisks(disks, metaArr, errs)
+	onlineDisks, modTime := listOnlineDisks(disks, metaArr, errs)
 
 	// Pick latest valid metadata.
 	fi, err := pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
@@ -981,6 +1004,8 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 		return toObjectErr(err, bucket, object)
 	}
 
+	onlineDisks, metaArr = shuffleDisksAndPartsMetadataByIndex(onlineDisks, metaArr, fi.Erasure.Distribution)
+
 	if fi.Deleted {
 		if opts.VersionID == "" {
 			return toObjectErr(errFileNotFound, bucket, object)
@@ -1005,6 +1030,66 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 
 	tempObj := mustGetUUID()
 
+	// Write unique `xl.meta` for each disk.
+	if onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	// Atomically rename metadata from tmp location to destination for each disk.
+	if _, err = renameFileInfo(ctx, onlineDisks, minioMetaTmpBucket, tempObj, bucket, object, writeQuorum); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	return nil
+}
+
+// updateObjectMeta will update the metadata of a file.
+func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object string, meta map[string]string, opts ObjectOptions) error {
+	if len(meta) == 0 {
+		return nil
+	}
+	disks := er.getDisks()
+
+	// Read metadata associated with the object from all disks.
+	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID)
+
+	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
+	if err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	// List all online disks.
+	_, modTime := listOnlineDisks(disks, metaArr, errs)
+
+	// Pick latest valid metadata.
+	fi, err := pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
+	if err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	// Update metadata
+	for k, v := range meta {
+		fi.Metadata[k] = v
+	}
+
+	if fi.Deleted {
+		if opts.VersionID == "" {
+			return toObjectErr(errFileNotFound, bucket, object)
+		}
+		return toObjectErr(errMethodNotAllowed, bucket, object)
+	}
+
+	for i := range metaArr {
+		if errs[i] != nil {
+			// Avoid disks where loading metadata fail
+			continue
+		}
+
+		metaArr[i].Metadata = fi.Metadata
+	}
+
+	tempObj := mustGetUUID()
+
 	// Write unique `xl.meta` for each disk.
 	if disks, err = writeUniqueFileInfo(ctx, disks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil {
 		return toObjectErr(err, bucket, object)

cmd/erasure-server-sets.go

@@ -31,7 +31,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/cmd/config/storageclass"
-	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/dsync"
 	"github.com/minio/minio/pkg/madmin"
@@ -89,8 +88,8 @@ func newErasureServerSets(ctx context.Context, endpointServerSets EndpointServer
 	return z, nil
 }
 
-func (z *erasureServerSets) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	return z.serverSets[0].NewNSLock(ctx, bucket, objects...)
+func (z *erasureServerSets) NewNSLock(bucket string, objects ...string) RWLocker {
+	return z.serverSets[0].NewNSLock(bucket, objects...)
 }
 
 func (z *erasureServerSets) GetAllLockers() []dsync.NetLocker {
@@ -570,8 +569,8 @@ func (z *erasureServerSets) DeleteObjects(ctx context.Context, bucket string, ob
 	}
 
 	// Acquire a bulk write lock across 'objects'
-	multiDeleteLock := z.NewNSLock(ctx, bucket, objSets.ToSlice()...)
-	if err := multiDeleteLock.GetLock(globalOperationTimeout); err != nil {
+	multiDeleteLock := z.NewNSLock(bucket, objSets.ToSlice()...)
+	if err := multiDeleteLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		for i := range derrs {
 			derrs[i] = err
 		}
@@ -659,366 +658,6 @@ func (z *erasureServerSets) ListObjectsV2(ctx context.Context, bucket, prefix, c
 	return listObjectsV2Info, err
 }
 
-func (z *erasureServerSets) listObjectsNonSlash(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, err error) {
-
-	serverSetsEntryChs := make([][]FileInfoCh, 0, len(z.serverSets))
-	serverSetsListTolerancePerSet := make([]int, 0, len(z.serverSets))
-
-	endWalkCh := make(chan struct{})
-	defer close(endWalkCh)
-
-	for _, zone := range z.serverSets {
-		serverSetsEntryChs = append(serverSetsEntryChs,
-			zone.startMergeWalksN(ctx, bucket, prefix, "", true, endWalkCh, zone.listTolerancePerSet, false))
-		if zone.listTolerancePerSet == -1 {
-			serverSetsListTolerancePerSet = append(serverSetsListTolerancePerSet, zone.setDriveCount/2)
-		} else {
-			serverSetsListTolerancePerSet = append(serverSetsListTolerancePerSet, zone.listTolerancePerSet-2)
-		}
-	}
-
-	var objInfos []ObjectInfo
-	var eof bool
-	var prevPrefix string
-
-	serverSetsEntriesInfos := make([][]FileInfo, 0, len(serverSetsEntryChs))
-	serverSetsEntriesValid := make([][]bool, 0, len(serverSetsEntryChs))
-	for _, entryChs := range serverSetsEntryChs {
-		serverSetsEntriesInfos = append(serverSetsEntriesInfos, make([]FileInfo, len(entryChs)))
-		serverSetsEntriesValid = append(serverSetsEntriesValid, make([]bool, len(entryChs)))
-	}
-
-	for {
-		if len(objInfos) == maxKeys {
-			break
-		}
-
-		result, quorumCount, zoneIndex, ok := lexicallySortedEntryZone(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
-		if !ok {
-			eof = true
-			break
-		}
-
-		if quorumCount < serverSetsListTolerancePerSet[zoneIndex] {
-			// Skip entries which are not found on upto expected tolerance
-			continue
-		}
-
-		var objInfo ObjectInfo
-
-		index := strings.Index(strings.TrimPrefix(result.Name, prefix), delimiter)
-		if index == -1 {
-			objInfo = ObjectInfo{
-				IsDir:           false,
-				Bucket:          bucket,
-				Name:            result.Name,
-				ModTime:         result.ModTime,
-				Size:            result.Size,
-				ContentType:     result.Metadata["content-type"],
-				ContentEncoding: result.Metadata["content-encoding"],
-			}
-
-			// Extract etag from metadata.
-			objInfo.ETag = extractETag(result.Metadata)
-
-			// All the parts per object.
-			objInfo.Parts = result.Parts
-
-			// etag/md5Sum has already been extracted. We need to
-			// remove to avoid it from appearing as part of
-			// response headers. e.g, X-Minio-* or X-Amz-*.
-			objInfo.UserDefined = cleanMetadata(result.Metadata)
-
-			// Update storage class
-			if sc, ok := result.Metadata[xhttp.AmzStorageClass]; ok {
-				objInfo.StorageClass = sc
-			} else {
-				objInfo.StorageClass = globalMinioDefaultStorageClass
-			}
-		} else {
-			index = len(prefix) + index + len(delimiter)
-			currPrefix := result.Name[:index]
-			if currPrefix == prevPrefix {
-				continue
-			}
-			prevPrefix = currPrefix
-
-			objInfo = ObjectInfo{
-				Bucket: bucket,
-				Name:   currPrefix,
-				IsDir:  true,
-			}
-		}
-
-		if objInfo.Name <= marker {
-			continue
-		}
-
-		objInfos = append(objInfos, objInfo)
-	}
-
-	result := ListObjectsInfo{}
-	for _, objInfo := range objInfos {
-		if objInfo.IsDir {
-			result.Prefixes = append(result.Prefixes, objInfo.Name)
-			continue
-		}
-		result.Objects = append(result.Objects, objInfo)
-	}
-
-	if !eof {
-		result.IsTruncated = true
-		if len(objInfos) > 0 {
-			result.NextMarker = objInfos[len(objInfos)-1].Name
-		}
-	}
-
-	return result, nil
-}
-
-func (z *erasureServerSets) listObjectsSplunk(ctx context.Context, bucket, prefix, marker string, maxKeys int) (loi ListObjectsInfo, err error) {
-	if strings.Contains(prefix, guidSplunk) {
-		logger.LogIf(ctx, NotImplemented{})
-		return loi, NotImplemented{}
-	}
-
-	recursive := true
-
-	serverSetsEntryChs := make([][]FileInfoCh, 0, len(z.serverSets))
-	serverSetsEndWalkCh := make([]chan struct{}, 0, len(z.serverSets))
-	serverSetsListTolerancePerSet := make([]int, 0, len(z.serverSets))
-
-	for _, zone := range z.serverSets {
-		entryChs, endWalkCh := zone.poolSplunk.Release(listParams{bucket, recursive, marker, prefix})
-		if entryChs == nil {
-			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet, true)
-		}
-		serverSetsEntryChs = append(serverSetsEntryChs, entryChs)
-		serverSetsEndWalkCh = append(serverSetsEndWalkCh, endWalkCh)
-		if zone.listTolerancePerSet == -1 {
-			serverSetsListTolerancePerSet = append(serverSetsListTolerancePerSet, zone.setDriveCount/2)
-		} else {
-			serverSetsListTolerancePerSet = append(serverSetsListTolerancePerSet, zone.listTolerancePerSet-2)
-		}
-	}
-
-	entries := mergeServerSetsEntriesCh(serverSetsEntryChs, maxKeys, serverSetsListTolerancePerSet)
-	if len(entries.Files) == 0 {
-		return loi, nil
-	}
-
-	loi.IsTruncated = entries.IsTruncated
-	if loi.IsTruncated {
-		loi.NextMarker = entries.Files[len(entries.Files)-1].Name
-	}
-
-	for _, entry := range entries.Files {
-		objInfo := entry.ToObjectInfo(bucket, entry.Name)
-		splits := strings.Split(objInfo.Name, guidSplunk)
-		if len(splits) == 0 {
-			loi.Objects = append(loi.Objects, objInfo)
-			continue
-		}
-
-		loi.Prefixes = append(loi.Prefixes, splits[0]+guidSplunk)
-	}
-
-	if loi.IsTruncated {
-		for i, zone := range z.serverSets {
-			zone.poolSplunk.Set(listParams{bucket, recursive, loi.NextMarker, prefix}, serverSetsEntryChs[i],
-				serverSetsEndWalkCh[i])
-		}
-	}
-	return loi, nil
-}
-
-func (z *erasureServerSets) listObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
-	loi := ListObjectsInfo{}
-
-	if err := checkListObjsArgs(ctx, bucket, prefix, marker, z); err != nil {
-		return loi, err
-	}
-
-	// Marker is set validate pre-condition.
-	if marker != "" {
-		// Marker not common with prefix is not implemented. Send an empty response
-		if !HasPrefix(marker, prefix) {
-			return loi, nil
-		}
-	}
-
-	// With max keys of zero we have reached eof, return right here.
-	if maxKeys == 0 {
-		return loi, nil
-	}
-
-	// For delimiter and prefix as '/' we do not list anything at all
-	// since according to s3 spec we stop at the 'delimiter'
-	// along // with the prefix. On a flat namespace with 'prefix'
-	// as '/' we don't have any entries, since all the keys are
-	// of form 'keyName/...'
-	if delimiter == SlashSeparator && prefix == SlashSeparator {
-		return loi, nil
-	}
-
-	// Over flowing count - reset to maxObjectList.
-	if maxKeys < 0 || maxKeys > maxObjectList {
-		maxKeys = maxObjectList
-	}
-
-	if delimiter != SlashSeparator && delimiter != "" {
-		if delimiter == guidSplunk {
-			return z.listObjectsSplunk(ctx, bucket, prefix, marker, maxKeys)
-		}
-		return z.listObjectsNonSlash(ctx, bucket, prefix, marker, delimiter, maxKeys)
-	}
-
-	// Default is recursive, if delimiter is set then list non recursive.
-	recursive := true
-	if delimiter == SlashSeparator {
-		recursive = false
-	}
-
-	serverSetsEntryChs := make([][]FileInfoCh, 0, len(z.serverSets))
-	serverSetsEndWalkCh := make([]chan struct{}, 0, len(z.serverSets))
-	serverSetsListTolerancePerSet := make([]int, 0, len(z.serverSets))
-
-	for _, zone := range z.serverSets {
-		entryChs, endWalkCh := zone.pool.Release(listParams{bucket, recursive, marker, prefix})
-		if entryChs == nil {
-			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet, false)
-		}
-		serverSetsEntryChs = append(serverSetsEntryChs, entryChs)
-		serverSetsEndWalkCh = append(serverSetsEndWalkCh, endWalkCh)
-		if zone.listTolerancePerSet == -1 {
-			serverSetsListTolerancePerSet = append(serverSetsListTolerancePerSet, zone.setDriveCount/2)
-		} else {
-			serverSetsListTolerancePerSet = append(serverSetsListTolerancePerSet, zone.listTolerancePerSet-2)
-		}
-	}
-
-	entries := mergeServerSetsEntriesCh(serverSetsEntryChs, maxKeys, serverSetsListTolerancePerSet)
-	if len(entries.Files) == 0 {
-		return loi, nil
-	}
-
-	loi.IsTruncated = entries.IsTruncated
-	if loi.IsTruncated {
-		loi.NextMarker = entries.Files[len(entries.Files)-1].Name
-	}
-
-	for _, entry := range entries.Files {
-		objInfo := entry.ToObjectInfo(entry.Volume, entry.Name)
-		if HasSuffix(objInfo.Name, SlashSeparator) && !recursive {
-			loi.Prefixes = append(loi.Prefixes, objInfo.Name)
-			continue
-		}
-		loi.Objects = append(loi.Objects, objInfo)
-	}
-	if loi.IsTruncated {
-		for i, zone := range z.serverSets {
-			zone.pool.Set(listParams{bucket, recursive, loi.NextMarker, prefix}, serverSetsEntryChs[i],
-				serverSetsEndWalkCh[i])
-		}
-	}
-	return loi, nil
-}
-
-// Calculate least entry across serverSets and across multiple FileInfo
-// channels, returns the least common entry and the total number of times
-// we found this entry. Additionally also returns a boolean
-// to indicate if the caller needs to call this function
-// again to list the next entry. It is callers responsibility
-// if the caller wishes to list N entries to call lexicallySortedEntry
-// N times until this boolean is 'false'.
-func lexicallySortedEntryZone(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zoneEntriesValid [][]bool) (FileInfo, int, int, bool) {
-	for i, entryChs := range zoneEntryChs {
-		for j := range entryChs {
-			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
-		}
-	}
-
-	var isTruncated = false
-	for _, entriesValid := range zoneEntriesValid {
-		for _, valid := range entriesValid {
-			if !valid {
-				continue
-			}
-			isTruncated = true
-			break
-		}
-		if isTruncated {
-			break
-		}
-	}
-
-	var lentry FileInfo
-	var found bool
-	var zoneIndex = -1
-	// TODO: following loop can be merged with above
-	// loop, explore this possibility.
-	for i, entriesValid := range zoneEntriesValid {
-		for j, valid := range entriesValid {
-			if !valid {
-				continue
-			}
-			if !found {
-				lentry = zoneEntries[i][j]
-				found = true
-				zoneIndex = i
-				continue
-			}
-			str1 := zoneEntries[i][j].Name
-			str2 := lentry.Name
-			if HasSuffix(str1, globalDirSuffix) {
-				str1 = strings.TrimSuffix(str1, globalDirSuffix) + slashSeparator
-			}
-			if HasSuffix(str2, globalDirSuffix) {
-				str2 = strings.TrimSuffix(str2, globalDirSuffix) + slashSeparator
-			}
-
-			if str1 < str2 {
-				lentry = zoneEntries[i][j]
-				zoneIndex = i
-			}
-		}
-	}
-
-	// We haven't been able to find any least entry,
-	// this would mean that we don't have valid entry.
-	if !found {
-		return lentry, 0, zoneIndex, isTruncated
-	}
-
-	lexicallySortedEntryCount := 0
-	for i, entriesValid := range zoneEntriesValid {
-		for j, valid := range entriesValid {
-			if !valid {
-				continue
-			}
-
-			// Entries are duplicated across disks,
-			// we should simply skip such entries.
-			if lentry.Name == zoneEntries[i][j].Name && lentry.ModTime.Equal(zoneEntries[i][j].ModTime) {
-				lexicallySortedEntryCount++
-				continue
-			}
-
-			// Push all entries which are lexically higher
-			// and will be returned later in Pop()
-			zoneEntryChs[i][j].Push(zoneEntries[i][j])
-		}
-	}
-
-	if HasSuffix(lentry.Name, globalDirSuffix) {
-		lentry.Name = strings.TrimSuffix(lentry.Name, globalDirSuffix) + slashSeparator
-	}
-
-	return lentry, lexicallySortedEntryCount, zoneIndex, isTruncated
-}
-
 // Calculate least entry across serverSets and across multiple FileInfoVersions
 // channels, returns the least common entry and the total number of times
 // we found this entry. Additionally also returns a boolean
@@ -1110,236 +749,59 @@ func lexicallySortedEntryZoneVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneE
 	return lentry, lexicallySortedEntryCount, zoneIndex, isTruncated
 }
 
-// mergeServerSetsEntriesVersionsCh - merges FileInfoVersions channel to entries upto maxKeys.
-func mergeServerSetsEntriesVersionsCh(serverSetsEntryChs [][]FileInfoVersionsCh, maxKeys int, serverSetsListTolerancePerSet []int) (entries FilesInfoVersions) {
-	var i = 0
-	serverSetsEntriesInfos := make([][]FileInfoVersions, 0, len(serverSetsEntryChs))
-	serverSetsEntriesValid := make([][]bool, 0, len(serverSetsEntryChs))
-	for _, entryChs := range serverSetsEntryChs {
-		serverSetsEntriesInfos = append(serverSetsEntriesInfos, make([]FileInfoVersions, len(entryChs)))
-		serverSetsEntriesValid = append(serverSetsEntriesValid, make([]bool, len(entryChs)))
-	}
-
-	for {
-		fi, quorumCount, zoneIndex, ok := lexicallySortedEntryZoneVersions(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
-		if !ok {
-			// We have reached EOF across all entryChs, break the loop.
-			break
-		}
-
-		if quorumCount < serverSetsListTolerancePerSet[zoneIndex] {
-			// Skip entries which are not found upto the expected tolerance
-			continue
-		}
-
-		entries.FilesVersions = append(entries.FilesVersions, fi)
-		i++
-		if i == maxKeys {
-			entries.IsTruncated = isTruncatedServerSetsVersions(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
-			break
-		}
-	}
-	return entries
-}
-
-// mergeServerSetsEntriesCh - merges FileInfo channel to entries upto maxKeys.
-func mergeServerSetsEntriesCh(serverSetsEntryChs [][]FileInfoCh, maxKeys int, serverSetsListTolerancePerSet []int) (entries FilesInfo) {
-	var i = 0
-	serverSetsEntriesInfos := make([][]FileInfo, 0, len(serverSetsEntryChs))
-	serverSetsEntriesValid := make([][]bool, 0, len(serverSetsEntryChs))
-	for _, entryChs := range serverSetsEntryChs {
-		serverSetsEntriesInfos = append(serverSetsEntriesInfos, make([]FileInfo, len(entryChs)))
-		serverSetsEntriesValid = append(serverSetsEntriesValid, make([]bool, len(entryChs)))
-	}
-	var prevEntry string
-	for {
-		fi, quorumCount, zoneIndex, ok := lexicallySortedEntryZone(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
-		if !ok {
-			// We have reached EOF across all entryChs, break the loop.
-			break
-		}
-
-		if quorumCount < serverSetsListTolerancePerSet[zoneIndex] {
-			// Skip entries which are not found upto configured tolerance.
-			continue
-		}
-
-		if HasSuffix(fi.Name, slashSeparator) && fi.Name == prevEntry {
-			continue
-		}
-
-		entries.Files = append(entries.Files, fi)
-		i++
-		if i == maxKeys {
-			entries.IsTruncated = isTruncatedServerSets(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
-			break
-		}
-		prevEntry = fi.Name
-	}
-	return entries
-}
-
-func isTruncatedServerSets(zoneEntryChs [][]FileInfoCh, zoneEntries [][]FileInfo, zoneEntriesValid [][]bool) bool {
-	for i, entryChs := range zoneEntryChs {
-		for j := range entryChs {
-			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
-		}
-	}
-
-	var isTruncated = false
-	for _, entriesValid := range zoneEntriesValid {
-		for _, valid := range entriesValid {
-			if valid {
-				isTruncated = true
-				break
-			}
-		}
-		if isTruncated {
-			break
-		}
-	}
-	for i, entryChs := range zoneEntryChs {
-		for j := range entryChs {
-			if zoneEntriesValid[i][j] {
-				zoneEntryChs[i][j].Push(zoneEntries[i][j])
-			}
-		}
-
-	}
-	return isTruncated
-}
-
-func isTruncatedServerSetsVersions(zoneEntryChs [][]FileInfoVersionsCh, zoneEntries [][]FileInfoVersions, zoneEntriesValid [][]bool) bool {
-	for i, entryChs := range zoneEntryChs {
-		for j := range entryChs {
-			zoneEntries[i][j], zoneEntriesValid[i][j] = entryChs[j].Pop()
-		}
-	}
-
-	var isTruncated = false
-	for _, entriesValid := range zoneEntriesValid {
-		for _, valid := range entriesValid {
-			if !valid {
-				continue
-			}
-			isTruncated = true
-			break
-		}
-		if isTruncated {
-			break
-		}
-	}
-	for i, entryChs := range zoneEntryChs {
-		for j := range entryChs {
-			if zoneEntriesValid[i][j] {
-				zoneEntryChs[i][j].Push(zoneEntries[i][j])
-			}
-		}
-	}
-	return isTruncated
-}
-
-func (z *erasureServerSets) listObjectVersions(ctx context.Context, bucket, prefix, marker, versionMarker, delimiter string, maxKeys int) (ListObjectVersionsInfo, error) {
+func (z *erasureServerSets) ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionMarker, delimiter string, maxKeys int) (ListObjectVersionsInfo, error) {
 	loi := ListObjectVersionsInfo{}
-
-	if err := checkListObjsArgs(ctx, bucket, prefix, marker, z); err != nil {
-		return loi, err
-	}
-
-	// Marker is set validate pre-condition.
-	if marker != "" {
-		// Marker not common with prefix is not implemented. Send an empty response
-		if !HasPrefix(marker, prefix) {
-			return loi, nil
-		}
-	}
-
 	if marker == "" && versionMarker != "" {
 		return loi, NotImplemented{}
 	}
-
-	// With max keys of zero we have reached eof, return right here.
-	if maxKeys == 0 {
-		return loi, nil
+	merged, err := z.listPath(ctx, listPathOptions{
+		Bucket:      bucket,
+		Prefix:      prefix,
+		Separator:   delimiter,
+		Limit:       maxKeys,
+		Marker:      marker,
+		InclDeleted: true,
+		AskDisks:    globalAPIConfig.getListQuorum(),
+	})
+	if err != nil && err != io.EOF {
+		return loi, err
 	}
-
-	// For delimiter and prefix as '/' we do not list anything at all
-	// since according to s3 spec we stop at the 'delimiter'
-	// along // with the prefix. On a flat namespace with 'prefix'
-	// as '/' we don't have any entries, since all the keys are
-	// of form 'keyName/...'
-	if delimiter == SlashSeparator && prefix == SlashSeparator {
-		return loi, nil
-	}
-
-	// Over flowing count - reset to maxObjectList.
-	if maxKeys < 0 || maxKeys > maxObjectList {
-		maxKeys = maxObjectList
-	}
-
-	if delimiter != SlashSeparator && delimiter != "" {
-		return loi, NotImplemented{}
-	}
-
-	// Default is recursive, if delimiter is set then list non recursive.
-	recursive := true
-	if delimiter == SlashSeparator {
-		recursive = false
-	}
-
-	serverSetsEntryChs := make([][]FileInfoVersionsCh, 0, len(z.serverSets))
-	serverSetsEndWalkCh := make([]chan struct{}, 0, len(z.serverSets))
-	serverSetsListTolerancePerSet := make([]int, 0, len(z.serverSets))
-	for _, zone := range z.serverSets {
-		entryChs, endWalkCh := zone.poolVersions.Release(listParams{bucket, recursive, marker, prefix})
-		if entryChs == nil {
-			endWalkCh = make(chan struct{})
-			entryChs = zone.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, zone.listTolerancePerSet)
-		}
-		serverSetsEntryChs = append(serverSetsEntryChs, entryChs)
-		serverSetsEndWalkCh = append(serverSetsEndWalkCh, endWalkCh)
-		if zone.listTolerancePerSet == -1 {
-			serverSetsListTolerancePerSet = append(serverSetsListTolerancePerSet, zone.setDriveCount/2)
-		} else {
-			serverSetsListTolerancePerSet = append(serverSetsListTolerancePerSet, zone.listTolerancePerSet-2)
-		}
-	}
-
-	entries := mergeServerSetsEntriesVersionsCh(serverSetsEntryChs, maxKeys, serverSetsListTolerancePerSet)
-	if len(entries.FilesVersions) == 0 {
-		return loi, nil
-	}
-
-	loi.IsTruncated = entries.IsTruncated
-	if loi.IsTruncated {
-		loi.NextMarker = entries.FilesVersions[len(entries.FilesVersions)-1].Name
-	}
-
-	for _, entry := range entries.FilesVersions {
-		for _, version := range entry.Versions {
-			objInfo := version.ToObjectInfo(bucket, entry.Name)
-			if HasSuffix(objInfo.Name, SlashSeparator) && !recursive {
-				loi.Prefixes = append(loi.Prefixes, objInfo.Name)
-				continue
-			}
-			loi.Objects = append(loi.Objects, objInfo)
-		}
+	loi.Objects, loi.Prefixes = merged.fileInfoVersions(bucket, prefix, delimiter, versionMarker)
+	loi.IsTruncated = err == nil && len(loi.Objects) > 0
+	if maxKeys > 0 && len(loi.Objects) > maxKeys {
+		loi.Objects = loi.Objects[:maxKeys]
+		loi.IsTruncated = true
 	}
 	if loi.IsTruncated {
-		for i, zone := range z.serverSets {
-			zone.poolVersions.Set(listParams{bucket, recursive, loi.NextMarker, prefix}, serverSetsEntryChs[i],
-				serverSetsEndWalkCh[i])
-		}
+		last := loi.Objects[len(loi.Objects)-1]
+		loi.NextMarker = encodeMarker(last.Name, merged.listID)
+		loi.NextVersionIDMarker = last.VersionID
 	}
 	return loi, nil
 }
 
-func (z *erasureServerSets) ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionMarker, delimiter string, maxKeys int) (ListObjectVersionsInfo, error) {
-	return z.listObjectVersions(ctx, bucket, prefix, marker, versionMarker, delimiter, maxKeys)
-}
-
 func (z *erasureServerSets) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) {
-	return z.listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys)
+	var loi ListObjectsInfo
+	merged, err := z.listPath(ctx, listPathOptions{
+		Bucket:      bucket,
+		Prefix:      prefix,
+		Separator:   delimiter,
+		Limit:       maxKeys,
+		Marker:      marker,
+		InclDeleted: false,
+		AskDisks:    globalAPIConfig.getListQuorum(),
+	})
+	if err != nil && err != io.EOF {
+		logger.LogIf(ctx, err)
+		return loi, err
+	}
+	// Default is recursive, if delimiter is set then list non recursive.
+	loi.Objects, loi.Prefixes = merged.fileInfos(bucket, prefix, delimiter)
+	loi.IsTruncated = err == nil && len(loi.Objects) > 0
+	if loi.IsTruncated {
+		loi.NextMarker = encodeMarker(loi.Objects[len(loi.Objects)-1].Name, merged.listID)
+	}
+	return loi, nil
 }
 
 func (z *erasureServerSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
@@ -1634,6 +1096,29 @@ func (z *erasureServerSets) DeleteBucket(ctx context.Context, bucket string, for
 	return nil
 }
 
+// deleteAll will delete a bucket+prefix unconditionally across all disks.
+// Note that set distribution is ignored so it should only be used in cases where
+// data is not distributed across sets.
+// Errors are logged but individual disk failures are not returned.
+func (z *erasureServerSets) deleteAll(ctx context.Context, bucket, prefix string) {
+	var wg sync.WaitGroup
+	for _, servers := range z.serverSets {
+		for _, set := range servers.sets {
+			for _, disk := range set.getDisks() {
+				if disk == nil {
+					continue
+				}
+				wg.Add(1)
+				go func(disk StorageAPI) {
+					defer wg.Done()
+					disk.Delete(ctx, bucket, prefix, true)
+				}(disk)
+			}
+		}
+	}
+	wg.Wait()
+}
+
 // This function is used to undo a successful DeleteBucket operation.
 func undoDeleteBucketServerSets(ctx context.Context, bucket string, serverSets []*erasureSets, errs []error) {
 	g := errgroup.WithNErrs(len(serverSets))
@@ -1680,21 +1165,10 @@ func (z *erasureServerSets) ListBuckets(ctx context.Context) (buckets []BucketIn
 	return buckets, nil
 }
 
-func (z *erasureServerSets) ReloadFormat(ctx context.Context, dryRun bool) error {
-	// No locks needed since reload happens in HealFormat under
-	// write lock across all nodes.
-	for _, zone := range z.serverSets {
-		if err := zone.ReloadFormat(ctx, dryRun); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
 func (z *erasureServerSets) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
 	// Acquire lock on format.json
-	formatLock := z.NewNSLock(ctx, minioMetaBucket, formatConfigFile)
-	if err := formatLock.GetLock(globalOperationTimeout); err != nil {
+	formatLock := z.NewNSLock(minioMetaBucket, formatConfigFile)
+	if err := formatLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		return madmin.HealResultItem{}, err
 	}
 	defer formatLock.Unlock()
@@ -1722,15 +1196,6 @@ func (z *erasureServerSets) HealFormat(ctx context.Context, dryRun bool) (madmin
 		r.After.Drives = append(r.After.Drives, result.After.Drives...)
 	}
 
-	// Healing succeeded notify the peers to reload format and re-initialize disks.
-	// We will not notify peers if healing is not required.
-	for _, nerr := range globalNotificationSys.ReloadFormat(dryRun) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-
 	// No heal returned by all serverSets, return errNoHealRequired
 	if countNoHeal == len(z.serverSets) {
 		return r, errNoHealRequired
@@ -1774,74 +1239,51 @@ func (z *erasureServerSets) Walk(ctx context.Context, bucket, prefix string, res
 		return err
 	}
 
-	serverSetsListTolerancePerSet := make([]int, 0, len(z.serverSets))
-	for _, zone := range z.serverSets {
-		if zone.listTolerancePerSet == -1 {
-			serverSetsListTolerancePerSet = append(serverSetsListTolerancePerSet, zone.setDriveCount/2)
-		} else {
-			serverSetsListTolerancePerSet = append(serverSetsListTolerancePerSet, zone.listTolerancePerSet-2)
-		}
-	}
-
 	if opts.WalkVersions {
-		var serverSetsEntryChs [][]FileInfoVersionsCh
-		for _, zone := range z.serverSets {
-			serverSetsEntryChs = append(serverSetsEntryChs, zone.startMergeWalksVersions(ctx, bucket, prefix, "", true, ctx.Done()))
-		}
-
-		var serverSetsEntriesInfos [][]FileInfoVersions
-		var serverSetsEntriesValid [][]bool
-		for _, entryChs := range serverSetsEntryChs {
-			serverSetsEntriesInfos = append(serverSetsEntriesInfos, make([]FileInfoVersions, len(entryChs)))
-			serverSetsEntriesValid = append(serverSetsEntriesValid, make([]bool, len(entryChs)))
-		}
-
 		go func() {
 			defer close(results)
 
+			var marker, versionIDMarker string
 			for {
-				entry, quorumCount, zoneIdx, ok := lexicallySortedEntryZoneVersions(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
-				if !ok {
-					// We have reached EOF across all entryChs, break the loop.
-					return
+				loi, err := z.ListObjectVersions(ctx, bucket, prefix, marker, versionIDMarker, "", 1000)
+				if err != nil {
+					break
 				}
 
-				if quorumCount >= serverSetsListTolerancePerSet[zoneIdx] {
-					for _, version := range entry.Versions {
-						results <- version.ToObjectInfo(bucket, version.Name)
-					}
+				for _, obj := range loi.Objects {
+					results <- obj
 				}
+
+				if !loi.IsTruncated {
+					break
+				}
+
+				marker = loi.NextMarker
+				versionIDMarker = loi.NextVersionIDMarker
 			}
 		}()
-
 		return nil
 	}
 
-	serverSetsEntryChs := make([][]FileInfoCh, 0, len(z.serverSets))
-	for _, zone := range z.serverSets {
-		serverSetsEntryChs = append(serverSetsEntryChs, zone.startMergeWalks(ctx, bucket, prefix, "", true, ctx.Done()))
-	}
-
-	serverSetsEntriesInfos := make([][]FileInfo, 0, len(serverSetsEntryChs))
-	serverSetsEntriesValid := make([][]bool, 0, len(serverSetsEntryChs))
-	for _, entryChs := range serverSetsEntryChs {
-		serverSetsEntriesInfos = append(serverSetsEntriesInfos, make([]FileInfo, len(entryChs)))
-		serverSetsEntriesValid = append(serverSetsEntriesValid, make([]bool, len(entryChs)))
-	}
-
 	go func() {
 		defer close(results)
 
+		var marker string
 		for {
-			entry, quorumCount, zoneIdx, ok := lexicallySortedEntryZone(serverSetsEntryChs, serverSetsEntriesInfos, serverSetsEntriesValid)
-			if !ok {
-				// We have reached EOF across all entryChs, break the loop.
-				return
+			loi, err := z.ListObjects(ctx, bucket, prefix, marker, "", 1000)
+			if err != nil {
+				break
 			}
 
-			if quorumCount >= serverSetsListTolerancePerSet[zoneIdx] {
-				results <- entry.ToObjectInfo(bucket, entry.Name)
+			for _, obj := range loi.Objects {
+				results <- obj
 			}
+
+			if !loi.IsTruncated {
+				break
+			}
+
+			marker = loi.NextMarker
 		}
 	}()
 
@@ -1907,17 +1349,17 @@ func (z *erasureServerSets) HealObjects(ctx context.Context, bucket, prefix stri
 func (z *erasureServerSets) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	object = encodeDirObject(object)
 
-	lk := z.NewNSLock(ctx, bucket, object)
+	lk := z.NewNSLock(bucket, object)
 	if bucket == minioMetaBucket {
 		// For .minio.sys bucket heals we should hold write locks.
-		if err := lk.GetLock(globalOperationTimeout); err != nil {
+		if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
 			return madmin.HealResultItem{}, err
 		}
 		defer lk.Unlock()
 	} else {
 		// Lock the object before healing. Use read lock since healing
 		// will only regenerate parts & xl.meta of outdated disks.
-		if err := lk.GetRLock(globalOperationTimeout); err != nil {
+		if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
 			return madmin.HealResultItem{}, err
 		}
 		defer lk.RUnlock()

cmd/erasure-sets.go

@@ -31,11 +31,9 @@ import (
 	"github.com/dchest/siphash"
 	"github.com/google/uuid"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
 	"github.com/minio/minio/pkg/dsync"
-	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -81,10 +79,8 @@ type erasureSets struct {
 
 	// Total number of sets and the number of disks per set.
 	setCount, setDriveCount int
-	listTolerancePerSet     int
 
-	monitorContextCancel context.CancelFunc
-	disksConnectEvent    chan diskConnectInfo
+	disksConnectEvent chan diskConnectInfo
 
 	// Distribution algorithm of choice.
 	distributionAlgo string
@@ -92,11 +88,6 @@ type erasureSets struct {
 
 	disksStorageInfoCache timedValue
 
-	// Merge tree walk
-	pool         *MergeWalkPool
-	poolSplunk   *MergeWalkPool
-	poolVersions *MergeWalkVersionsPool
-
 	mrfMU         sync.Mutex
 	mrfOperations map[healSource]int
 }
@@ -346,31 +337,21 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 
 	endpointStrings := make([]string, len(endpoints))
 
-	listTolerancePerSet := 3
-	// By default this is off
-	if env.Get("MINIO_API_LIST_STRICT_QUORUM", config.EnableOn) == config.EnableOn {
-		listTolerancePerSet = -1
-	}
-
 	// Initialize the erasure sets instance.
 	s := &erasureSets{
-		sets:                make([]*erasureObjects, setCount),
-		erasureDisks:        make([][]StorageAPI, setCount),
-		erasureLockers:      make([][]dsync.NetLocker, setCount),
-		erasureLockOwner:    GetLocalPeer(globalEndpoints),
-		endpoints:           endpoints,
-		endpointStrings:     endpointStrings,
-		setCount:            setCount,
-		setDriveCount:       setDriveCount,
-		listTolerancePerSet: listTolerancePerSet,
-		format:              format,
-		disksConnectEvent:   make(chan diskConnectInfo),
-		distributionAlgo:    format.Erasure.DistributionAlgo,
-		deploymentID:        uuid.MustParse(format.ID),
-		pool:                NewMergeWalkPool(globalMergeLookupTimeout),
-		poolSplunk:          NewMergeWalkPool(globalMergeLookupTimeout),
-		poolVersions:        NewMergeWalkVersionsPool(globalMergeLookupTimeout),
-		mrfOperations:       make(map[healSource]int),
+		sets:              make([]*erasureObjects, setCount),
+		erasureDisks:      make([][]StorageAPI, setCount),
+		erasureLockers:    make([][]dsync.NetLocker, setCount),
+		erasureLockOwner:  GetLocalPeer(globalEndpoints),
+		endpoints:         endpoints,
+		endpointStrings:   endpointStrings,
+		setCount:          setCount,
+		setDriveCount:     setDriveCount,
+		format:            format,
+		disksConnectEvent: make(chan diskConnectInfo),
+		distributionAlgo:  format.Erasure.DistributionAlgo,
+		deploymentID:      uuid.MustParse(format.ID),
+		mrfOperations:     make(map[healSource]int),
 	}
 
 	mutex := newNSLock(globalIsDistErasure)
@@ -419,11 +400,8 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 			GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
 	}
 
-	mctx, mctxCancel := context.WithCancel(ctx)
-	s.monitorContextCancel = mctxCancel
-
 	// Start the disk monitoring and connect routine.
-	go s.monitorAndConnectEndpoints(mctx, defaultMonitorConnectEndpointInterval)
+	go s.monitorAndConnectEndpoints(ctx, defaultMonitorConnectEndpointInterval)
 	go s.maintainMRFList()
 	go s.healMRFRoutine()
 
@@ -431,11 +409,11 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (s *erasureSets) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
+func (s *erasureSets) NewNSLock(bucket string, objects ...string) RWLocker {
 	if len(objects) == 1 {
-		return s.getHashedSet(objects[0]).NewNSLock(ctx, bucket, objects...)
+		return s.getHashedSet(objects[0]).NewNSLock(bucket, objects...)
 	}
-	return s.getHashedSet("").NewNSLock(ctx, bucket, objects...)
+	return s.getHashedSet("").NewNSLock(bucket, objects...)
 }
 
 // SetDriveCount returns the current drives per set.
@@ -799,6 +777,7 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
 
 	// Check if this request is only metadata update.
 	if cpSrcDstSame && srcInfo.metadataOnly {
+
 		// Version ID is set for the destination and source == destination version ID.
 		// perform an in-place update.
 		if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
@@ -939,10 +918,6 @@ func lexicallySortedEntryVersions(entryChs []FileInfoVersionsCh, entries []FileI
 	return lentry, lexicallySortedEntryCount, isTruncated
 }
 
-func (s *erasureSets) startMergeWalks(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh <-chan struct{}) []FileInfoCh {
-	return s.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, -1, false)
-}
-
 func (s *erasureSets) startMergeWalksVersions(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh <-chan struct{}) []FileInfoVersionsCh {
 	return s.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, -1)
 }
@@ -977,42 +952,6 @@ func (s *erasureSets) startMergeWalksVersionsN(ctx context.Context, bucket, pref
 	return entryChs
 }
 
-// Starts a walk channel across n number of disks and returns a slice of
-// FileInfoCh which can be read from.
-func (s *erasureSets) startMergeWalksN(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh <-chan struct{}, ndisks int, splunk bool) []FileInfoCh {
-	var entryChs []FileInfoCh
-	var wg sync.WaitGroup
-	var mutex sync.Mutex
-	for _, set := range s.sets {
-		// Reset for the next erasure set.
-		for _, disk := range set.getLoadBalancedNDisks(ndisks) {
-			wg.Add(1)
-			go func(disk StorageAPI) {
-				defer wg.Done()
-
-				var entryCh chan FileInfo
-				var err error
-				if splunk {
-					entryCh, err = disk.WalkSplunk(GlobalContext, bucket, prefix, marker, endWalkCh)
-				} else {
-					entryCh, err = disk.Walk(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
-				}
-				if err != nil {
-					// Disk walk returned error, ignore it.
-					return
-				}
-				mutex.Lock()
-				entryChs = append(entryChs, FileInfoCh{
-					Ch: entryCh,
-				})
-				mutex.Unlock()
-			}(disk)
-		}
-	}
-	wg.Wait()
-	return entryChs
-}
-
 func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
 	// In list multipart uploads we are going to treat input prefix as the object,
 	// this means that we are not supporting directory navigation.
@@ -1140,81 +1079,6 @@ func formatsToDrivesInfo(endpoints Endpoints, formats []*formatErasureV3, sErrs
 	return beforeDrives
 }
 
-// Reloads the format from the disk, usually called by a remote peer notifier while
-// healing in a distributed setup.
-func (s *erasureSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
-	storageDisks, errs := initStorageDisksWithErrorsWithoutHealthCheck(s.endpoints)
-	for i, err := range errs {
-		if err != nil && err != errDiskNotFound {
-			return fmt.Errorf("Disk %s: %w", s.endpoints[i], err)
-		}
-	}
-	defer func(storageDisks []StorageAPI) {
-		if err != nil {
-			closeStorageDisks(storageDisks)
-		}
-	}(storageDisks)
-
-	formats, _ := loadFormatErasureAll(storageDisks, false)
-	if err = checkFormatErasureValues(formats, s.setDriveCount); err != nil {
-		return err
-	}
-
-	refFormat, err := getFormatErasureInQuorum(formats)
-	if err != nil {
-		return err
-	}
-
-	s.monitorContextCancel() // turn-off disk monitoring and replace format.
-
-	s.erasureDisksMu.Lock()
-
-	// Replace with new reference format.
-	s.format = refFormat
-
-	// Close all existing disks and reconnect all the disks.
-	for _, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-
-		diskID, err := disk.GetDiskID()
-		if err != nil {
-			continue
-		}
-
-		m, n, err := findDiskIndexByDiskID(refFormat, diskID)
-		if err != nil {
-			continue
-		}
-
-		if s.erasureDisks[m][n] != nil {
-			s.erasureDisks[m][n].Close()
-		}
-
-		s.endpointStrings[m*s.setDriveCount+n] = disk.String()
-		if !disk.IsLocal() {
-			// Enable healthcheck disk for remote endpoint.
-			disk, err = newStorageAPI(disk.Endpoint())
-			if err != nil {
-				continue
-			}
-			disk.SetDiskID(diskID)
-		}
-
-		s.erasureDisks[m][n] = disk
-	}
-
-	s.erasureDisksMu.Unlock()
-
-	mctx, mctxCancel := context.WithCancel(GlobalContext)
-	s.monitorContextCancel = mctxCancel
-
-	go s.monitorAndConnectEndpoints(mctx, defaultMonitorConnectEndpointInterval)
-
-	return nil
-}
-
 // If it is a single node Erasure and all disks are root disks, it is most likely a test setup, else it is a production setup.
 // On a test setup we allow creation of format.json on root disks to help with dev/testing.
 func isTestSetup(infos []DiskInfo, errs []error) bool {
@@ -1334,35 +1198,19 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			}
 		}
 
-		// Save formats `format.json` across all disks.
-		if err = saveFormatErasureAllWithErrs(ctx, storageDisks, sErrs, tmpNewFormats); err != nil {
+		// Save new formats `format.json` on unformatted disks.
+		if err = saveUnformattedFormat(ctx, storageDisks, tmpNewFormats); err != nil {
 			return madmin.HealResultItem{}, err
 		}
 
-		refFormat, err = getFormatErasureInQuorum(tmpNewFormats)
-		if err != nil {
-			return madmin.HealResultItem{}, err
-		}
-
-		s.monitorContextCancel() // turn-off disk monitoring and replace format.
-
 		s.erasureDisksMu.Lock()
 
-		// Replace with new reference format.
-		s.format = refFormat
-
-		// Disconnect/relinquish all existing disks, lockers and reconnect the disks, lockers.
-		for _, disk := range storageDisks {
-			if disk == nil {
+		for index, format := range tmpNewFormats {
+			if format == nil {
 				continue
 			}
 
-			diskID, err := disk.GetDiskID()
-			if err != nil {
-				continue
-			}
-
-			m, n, err := findDiskIndexByDiskID(refFormat, diskID)
+			m, n, err := findDiskIndexByDiskID(refFormat, format.Erasure.This)
 			if err != nil {
 				continue
 			}
@@ -1371,24 +1219,14 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 				s.erasureDisks[m][n].Close()
 			}
 
-			s.endpointStrings[m*s.setDriveCount+n] = disk.String()
-			if !disk.IsLocal() {
-				// Enable healthcheck disk for remote endpoint.
-				disk, err = newStorageAPI(disk.Endpoint())
-				if err != nil {
-					continue
-				}
-				disk.SetDiskID(diskID)
-			}
-			s.erasureDisks[m][n] = disk
-
+			s.erasureDisks[m][n] = storageDisks[index]
+			s.endpointStrings[m*s.setDriveCount+n] = storageDisks[index].String()
 		}
 
-		s.erasureDisksMu.Unlock()
+		// Replace reference format with what was loaded from disks.
+		s.format = refFormat
 
-		mctx, mctxCancel := context.WithCancel(GlobalContext)
-		s.monitorContextCancel = mctxCancel
-		go s.monitorAndConnectEndpoints(mctx, defaultMonitorConnectEndpointInterval)
+		s.erasureDisksMu.Unlock()
 	}
 
 	return res, nil

cmd/erasure-sets_test.go

@@ -197,27 +197,13 @@ func TestNewErasureSets(t *testing.T) {
 // TestHashedLayer - tests the hashed layer which will be returned
 // consistently for a given object name.
 func TestHashedLayer(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	var objs []*erasureObjects
-	for i := 0; i < 16; i++ {
-		obj, fsDirs, err := prepareErasure16(ctx)
-		if err != nil {
-			t.Fatal("Unable to initialize 'Erasure' object layer.", err)
-		}
-		defer obj.Shutdown(ctx)
-
-		// Remove all dirs.
-		for _, dir := range fsDirs {
-			defer os.RemoveAll(dir)
-		}
-
-		z := obj.(*erasureServerSets)
-		objs = append(objs, z.serverSets[0].sets[0])
+	// Test distribution with 16 sets.
+	var objs [16]*erasureObjects
+	for i := range objs {
+		objs[i] = &erasureObjects{}
 	}
 
-	sets := &erasureSets{sets: objs, distributionAlgo: "CRCMOD"}
+	sets := &erasureSets{sets: objs[:], distributionAlgo: "CRCMOD"}
 
 	testCases := []struct {
 		objectName  string

cmd/erasure.go

@@ -68,8 +68,8 @@ type erasureObjects struct {
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (er erasureObjects) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	return er.nsMutex.NewNSLock(ctx, er.getLockers, bucket, objects...)
+func (er erasureObjects) NewNSLock(bucket string, objects ...string) RWLocker {
+	return er.nsMutex.NewNSLock(er.getLockers, bucket, objects...)
 }
 
 // SetDriveCount returns the current drives per set.

cmd/etcd.go

@@ -38,9 +38,23 @@ func etcdErrToErr(err error, etcdEndpoints []string) error {
 	}
 }
 
-func saveKeyEtcd(ctx context.Context, client *etcd.Client, key string, data []byte) error {
+func saveKeyEtcdWithTTL(ctx context.Context, client *etcd.Client, key string, data []byte, ttl int64) error {
 	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
+	lease, err := client.Grant(timeoutCtx, ttl)
+	if err != nil {
+		return etcdErrToErr(err, client.Endpoints())
+	}
+	_, err = client.Put(timeoutCtx, key, string(data), etcd.WithLease(lease.ID))
+	return etcdErrToErr(err, client.Endpoints())
+}
+
+func saveKeyEtcd(ctx context.Context, client *etcd.Client, key string, data []byte, opts ...options) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	if len(opts) > 0 {
+		return saveKeyEtcdWithTTL(ctx, client, key, data, opts[0].ttl)
+	}
 	_, err := client.Put(timeoutCtx, key, string(data))
 	return etcdErrToErr(err, client.Endpoints())
 }

cmd/format-disk-cache.go

@@ -365,7 +365,7 @@ func migrateCacheData(ctx context.Context, c *diskCache, bucket, object, oldfile
 		}
 		actualSize, _ = sio.EncryptedSize(uint64(st.Size()))
 	}
-	_, err = c.bitrotWriteToCache(destDir, cacheDataFile, reader, actualSize)
+	_, _, err = c.bitrotWriteToCache(destDir, cacheDataFile, reader, actualSize)
 	return err
 }
 

cmd/format-erasure.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"encoding/hex"
 	"encoding/json"
@@ -360,10 +359,10 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 	tmpFormat := mustGetUUID()
 
 	// Purge any existing temporary file, okay to ignore errors here.
-	defer disk.DeleteFile(context.TODO(), minioMetaBucket, tmpFormat)
+	defer disk.Delete(context.TODO(), minioMetaBucket, tmpFormat, false)
 
 	// write to unique file.
-	if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil {
+	if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, formatBytes); err != nil {
 		return err
 	}
 
@@ -383,7 +382,7 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 		}
 		return disk.WriteAll(context.TODO(), minioMetaBucket,
 			pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
-			bytes.NewReader(htrackerBytes))
+			htrackerBytes)
 	}
 	return nil
 }
@@ -704,28 +703,18 @@ func initErasureMetaVolumesInLocalDisks(storageDisks []StorageAPI, formats []*fo
 	return nil
 }
 
-// saveFormatErasureAllWithErrs - populates `format.json` on disks in its order.
+// saveUnformattedFormat - populates `format.json` on unformatted disks.
 // also adds `.healing.bin` on the disks which are being actively healed.
-func saveFormatErasureAllWithErrs(ctx context.Context, storageDisks []StorageAPI, fErrs []error, formats []*formatErasureV3) error {
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	// Write `format.json` to all disks.
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if formats[index] == nil {
-				return errDiskNotFound
-			}
-			if errors.Is(fErrs[index], errUnformattedDisk) {
-				return saveFormatErasure(storageDisks[index], formats[index], true)
-			}
-			return nil
-		}, index)
+func saveUnformattedFormat(ctx context.Context, storageDisks []StorageAPI, formats []*formatErasureV3) error {
+	for index, format := range formats {
+		if format == nil {
+			continue
+		}
+		if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
+			return err
+		}
 	}
-
-	writeQuorum := getWriteQuorum(len(storageDisks))
-	// Wait for the routines to finish.
-	return reduceWriteQuorumErrs(ctx, g.Wait(), nil, writeQuorum)
+	return nil
 }
 
 // saveFormatErasureAll - populates `format.json` on disks in its order.

cmd/format-fs.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math/rand"
 	"os"
 	"path"
 	"time"
@@ -27,7 +28,6 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
-	"github.com/minio/minio/pkg/retry"
 )
 
 // FS format version strings.
@@ -340,31 +340,28 @@ func formatFSFixDeploymentID(ctx context.Context, fsFormatPath string) error {
 		return time.Now().Round(time.Second).Sub(formatStartTime).String()
 	}
 
-	retryCtx, cancel := context.WithCancel(ctx)
-	// Indicate to our routine to exit cleanly upon return.
-	defer cancel()
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	var wlk *lock.LockedFile
-	retryCh := retry.NewTimerWithJitter(retryCtx, time.Second, 30*time.Second, retry.MaxJitter)
 	var stop bool
 	for !stop {
 		select {
-		case <-retryCh:
+		case <-ctx.Done():
+			return fmt.Errorf("Initializing FS format stopped gracefully")
+		default:
 			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
 			if err == lock.ErrAlreadyLocked {
 				// Lock already present, sleep and attempt again
 				logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())
+				time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
 				continue
 			}
 			if err != nil {
 				return err
 			}
-			stop = true
-		case <-ctx.Done():
-			return fmt.Errorf("Initializing FS format stopped gracefully")
 		}
+		stop = true
 	}
-
 	defer wlk.Close()
 
 	if err = jsonLoad(wlk, format); err != nil {

cmd/fs-v1-helpers_test.go

@@ -297,7 +297,7 @@ func TestFSDeletes(t *testing.T) {
 			srcPath:     "success-file",
 			expectedErr: nil,
 		},
-		// The file was deleted in the last case, so DeleteFile should fail.
+		// The file was deleted in the last case, so Delete should fail.
 		{
 			basePath:    path,
 			srcVol:      "success-vol",
@@ -423,7 +423,7 @@ func TestFSRemoves(t *testing.T) {
 			expectedErr: nil,
 		},
 		// Test case - 2.
-		// The file was deleted in the last case, so DeleteFile should fail.
+		// The file was deleted in the last case, so Delete should fail.
 		{
 			srcFSPath:   path,
 			srcVol:      "success-vol",

cmd/fs-v1-multipart.go

@@ -714,8 +714,8 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Hold write lock on the object.
-	destLock := fs.NewNSLock(ctx, bucket, object)
-	if err = destLock.GetLock(globalOperationTimeout); err != nil {
+	destLock := fs.NewNSLock(bucket, object)
+	if err = destLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer destLock.Unlock()

cmd/fs-v1-multipart_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sync"
 	"testing"
 	"time"
@@ -195,6 +196,11 @@ func TestCompleteMultipartUpload(t *testing.T) {
 
 // TestCompleteMultipartUpload - test CompleteMultipartUpload
 func TestAbortMultipartUpload(t *testing.T) {
+	if runtime.GOOS == globalWindowsOSName {
+		// Concurrent AbortMultipartUpload() fails on windows
+		t.Skip()
+	}
+
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)
@@ -219,7 +225,6 @@ func TestAbortMultipartUpload(t *testing.T) {
 	if _, err := obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), opts); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
-	time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."
 	if err := obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID, opts); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}

cmd/fs-v1.go

@@ -186,9 +186,9 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (fs *FSObjects) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
+func (fs *FSObjects) NewNSLock(bucket string, objects ...string) RWLocker {
 	// lockers are explicitly 'nil' for FS mode since there are only local lockers
-	return fs.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
+	return fs.nsMutex.NewNSLock(nil, bucket, objects...)
 }
 
 // SetDriveCount no-op
@@ -402,6 +402,7 @@ func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket string,
 		return BucketNameInvalid{Bucket: bucket}
 	}
 
+	defer ObjectPathUpdated(bucket + slashSeparator)
 	atomic.AddInt64(&fs.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt64(&fs.activeIOCount, -1)
@@ -601,8 +602,8 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 	defer ObjectPathUpdated(path.Join(dstBucket, dstObject))
 
 	if !cpSrcDstSame {
-		objectDWLock := fs.NewNSLock(ctx, dstBucket, dstObject)
-		if err := objectDWLock.GetLock(globalOperationTimeout); err != nil {
+		objectDWLock := fs.NewNSLock(dstBucket, dstObject)
+		if err := objectDWLock.GetLock(ctx, globalOperationTimeout); err != nil {
 			return oi, err
 		}
 		defer objectDWLock.Unlock()
@@ -692,15 +693,15 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 
 	if lockType != noLock {
 		// Lock the object before reading.
-		lock := fs.NewNSLock(ctx, bucket, object)
+		lock := fs.NewNSLock(bucket, object)
 		switch lockType {
 		case writeLock:
-			if err = lock.GetLock(globalOperationTimeout); err != nil {
+			if err = lock.GetLock(ctx, globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.Unlock
 		case readLock:
-			if err = lock.GetRLock(globalOperationTimeout); err != nil {
+			if err = lock.GetRLock(ctx, globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.RUnlock
@@ -785,8 +786,8 @@ func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offse
 	}
 
 	// Lock the object before reading.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return err
 	}
@@ -1013,8 +1014,8 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 // getObjectInfoWithLock - reads object metadata and replies back ObjectInfo.
 func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
 	// Lock the object before reading.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer lk.RUnlock()
@@ -1051,8 +1052,8 @@ func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
 
 	oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
 	if err == errCorruptedFormat || err == io.EOF {
-		lk := fs.NewNSLock(ctx, bucket, object)
-		if err = lk.GetLock(globalOperationTimeout); err != nil {
+		lk := fs.NewNSLock(bucket, object)
+		if err = lk.GetLock(ctx, globalOperationTimeout); err != nil {
 			return oi, toObjectErr(err, bucket, object)
 		}
 
@@ -1102,8 +1103,8 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 	}
 
 	// Lock the object.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return objInfo, err
 	}
@@ -1284,8 +1285,8 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string, op
 	}
 
 	// Acquire a write lock before deleting the object.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	if err = lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		return objInfo, err
 	}
 	defer lk.Unlock()
@@ -1535,12 +1536,6 @@ func (fs *FSObjects) DeleteObjectTags(ctx context.Context, bucket, object string
 	return fs.PutObjectTags(ctx, bucket, object, "", opts)
 }
 
-// ReloadFormat - no-op for fs, Valid only for Erasure.
-func (fs *FSObjects) ReloadFormat(ctx context.Context, dryRun bool) error {
-	logger.LogIf(ctx, NotImplemented{})
-	return NotImplemented{}
-}
-
 // HealFormat - no-op for fs, Valid only for Erasure.
 func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
 	logger.LogIf(ctx, NotImplemented{})

cmd/gateway-common.go

@@ -270,7 +270,7 @@ func IsBackendOnline(ctx context.Context, clnt *http.Client, urlStr string) bool
 	resp, err := clnt.Do(req)
 	if err != nil {
 		clnt.CloseIdleConnections()
-		return !xnet.IsNetworkOrHostDown(err)
+		return !xnet.IsNetworkOrHostDown(err, false)
 	}
 	xhttp.DrainBody(resp.Body)
 	return true
@@ -291,7 +291,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		object = params[1]
 	}
 
-	if xnet.IsNetworkOrHostDown(err) {
+	if xnet.IsNetworkOrHostDown(err, false) {
 		return BackendDown{}
 	}
 

cmd/gateway-main.go

@@ -51,8 +51,8 @@ type GatewayLocker struct {
 }
 
 // NewNSLock - implements gateway level locker
-func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	return l.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
+func (l *GatewayLocker) NewNSLock(bucket string, objects ...string) RWLocker {
+	return l.nsMutex.NewNSLock(nil, bucket, objects...)
 }
 
 // Walk - implements common gateway level Walker, to walk on all objects recursively at a prefix

cmd/gateway-unsupported.go

@@ -41,8 +41,8 @@ func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, bf *bloomF
 }
 
 // NewNSLock is a dummy stub for gateway.
-func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	logger.CriticalIf(ctx, errors.New("not implemented"))
+func (a GatewayUnsupported) NewNSLock(bucket string, objects ...string) RWLocker {
+	logger.CriticalIf(context.Background(), errors.New("not implemented"))
 	return nil
 }
 
@@ -160,11 +160,6 @@ func (a GatewayUnsupported) DeleteBucketSSEConfig(ctx context.Context, bucket st
 	return NotImplemented{}
 }
 
-// ReloadFormat - Not implemented stub.
-func (a GatewayUnsupported) ReloadFormat(ctx context.Context, dryRun bool) error {
-	return NotImplemented{}
-}
-
 // HealFormat - Not implemented stub
 func (a GatewayUnsupported) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
 	return madmin.HealResultItem{}, NotImplemented{}

cmd/gateway/s3/gateway-s3-sse.go

@@ -65,8 +65,8 @@ type s3EncObjects struct {
 
 // ListObjects lists all blobs in S3 bucket filtered by prefix
 func (l *s3EncObjects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, e error) {
-	var continuationToken, startAfter string
-	res, err := l.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, maxKeys, false, startAfter)
+	var startAfter string
+	res, err := l.ListObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, false, startAfter)
 	if err != nil {
 		return loi, err
 	}
@@ -91,10 +91,12 @@ func (l *s3EncObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 		if e != nil {
 			return loi, minio.ErrorRespToObjectError(e, bucket)
 		}
+
+		continuationToken = loi.NextContinuationToken
+		isTruncated = loi.IsTruncated
+
 		for _, obj := range loi.Objects {
 			startAfter = obj.Name
-			continuationToken = loi.NextContinuationToken
-			isTruncated = loi.IsTruncated
 
 			if !isGWObject(obj.Name) {
 				continue
@@ -708,16 +710,15 @@ func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, clea
 
 // cleanupStaleUploads removes old custom encryption multipart uploads on backend
 func (l *s3EncObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
-	for {
-		buckets, err := l.s3Objects.ListBuckets(ctx)
-		if err != nil {
-			break
-		}
-		for _, b := range buckets {
-			expParts := l.getStalePartsForBucket(ctx, b.Name, expiry)
-			for k := range expParts {
-				l.s3Objects.DeleteObject(ctx, b.Name, k, minio.ObjectOptions{})
-			}
+	buckets, err := l.s3Objects.ListBuckets(ctx)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+	for _, b := range buckets {
+		expParts := l.getStalePartsForBucket(ctx, b.Name, expiry)
+		for k := range expParts {
+			l.s3Objects.DeleteObject(ctx, b.Name, k, minio.ObjectOptions{})
 		}
 	}
 }
@@ -729,6 +730,7 @@ func (l *s3EncObjects) getStalePartsForBucket(ctx context.Context, bucket string
 	for {
 		loi, err := l.s3Objects.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, 1000, false, startAfter)
 		if err != nil {
+			logger.LogIf(ctx, err)
 			break
 		}
 		for _, obj := range loi.Objects {

cmd/generic-handlers.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"context"
 	"net/http"
 	"strings"
 	"time"
@@ -157,15 +158,48 @@ const (
 	loginPathPrefix         = SlashSeparator + "login"
 )
 
-// Adds redirect rules for incoming requests.
 type redirectHandler struct {
 	handler http.Handler
 }
 
-func setBrowserRedirectHandler(h http.Handler) http.Handler {
+func setRedirectHandler(h http.Handler) http.Handler {
 	return redirectHandler{handler: h}
 }
 
+// Adds redirect rules for incoming requests.
+type browserRedirectHandler struct {
+	handler http.Handler
+}
+
+func setBrowserRedirectHandler(h http.Handler) http.Handler {
+	return browserRedirectHandler{handler: h}
+}
+
+func shouldProxy() bool {
+	if newObjectLayerFn() == nil {
+		return true
+	}
+	return !globalIAMSys.Initialized()
+}
+
+func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if guessIsRPCReq(r) || guessIsBrowserReq(r) ||
+		guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || isAdminReq(r) {
+		h.handler.ServeHTTP(w, r)
+		return
+	}
+	if shouldProxy() {
+		// if this server is still initializing, proxy the request
+		// to any other online servers to avoid 503 for any incoming
+		// API calls.
+		if idx := getOnlineProxyEndpointIdx(); idx >= 0 {
+			proxyRequest(context.TODO(), w, r, globalProxyEndpoints[idx])
+			return
+		}
+	}
+	h.handler.ServeHTTP(w, r)
+}
+
 // Fetch redirect location if urlPath satisfies certain
 // criteria. Some special names are considered to be
 // redirectable, this is purely internal function and
@@ -236,7 +270,7 @@ func guessIsRPCReq(req *http.Request) bool {
 		strings.HasPrefix(req.URL.Path, minioReservedBucketPath+SlashSeparator)
 }
 
-func (h redirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+func (h browserRedirectHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// Re-direction is handled specifically for browser requests.
 	if guessIsBrowserReq(r) {
 		// Fetch the redirect location if any.

cmd/globals.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"crypto/x509"
+	"net/http"
 	"os"
 	"time"
 
@@ -159,7 +160,7 @@ var (
 	globalBucketTargetSys    *BucketTargetSys
 	// globalAPIConfig controls S3 API requests throttling,
 	// healthcheck readiness deadlines and cors settings.
-	globalAPIConfig apiConfig
+	globalAPIConfig = apiConfig{listQuorum: 3}
 
 	globalStorageClass storageclass.Config
 	globalLDAPConfig   xldap.Config
@@ -275,6 +276,8 @@ var (
 
 	globalProxyEndpoints []ProxyEndpoint
 
+	globalInternodeTransport http.RoundTripper
+
 	globalDNSCache *xhttp.DNSCache
 	// Add new variable global values here.
 )

cmd/handler-api.go

@@ -32,6 +32,8 @@ type apiConfig struct {
 	requestsDeadline time.Duration
 	requestsPool     chan struct{}
 	clusterDeadline  time.Duration
+	listQuorum       int
+	extendListLife   time.Duration
 	corsAllowOrigins []string
 }
 
@@ -63,6 +65,22 @@ func (t *apiConfig) init(cfg api.Config, setDriveCount int) {
 
 	t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode)
 	t.requestsDeadline = cfg.RequestsDeadline
+	t.listQuorum = cfg.GetListQuorum()
+	t.extendListLife = cfg.ExtendListLife
+}
+
+func (t *apiConfig) getListQuorum() int {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.listQuorum
+}
+
+func (t *apiConfig) getExtendListLife() time.Duration {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.extendListLife
 }
 
 func (t *apiConfig) getCorsAllowOrigins() []string {
@@ -92,6 +110,9 @@ func (t *apiConfig) getRequestsPool() (chan struct{}, <-chan time.Time) {
 	if t.requestsPool == nil {
 		return nil, nil
 	}
+	if t.requestsDeadline <= 0 {
+		return t.requestsPool, nil
+	}
 
 	return t.requestsPool, time.NewTimer(t.requestsDeadline).C
 }

cmd/handler-utils.go

@@ -424,6 +424,17 @@ func extractAPIVersion(r *http.Request) string {
 	return regexVersion.FindString(r.URL.Path)
 }
 
+func methodNotAllowedHandler(api string) func(w http.ResponseWriter, r *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		code := "XMinio" + api + "VersionMismatch"
+		writeErrorResponseString(r.Context(), w, APIError{
+			Code:           code,
+			Description:    "Not allowed (" + r.Method + " " + r.URL.String() + " on " + api + " API)",
+			HTTPStatusCode: http.StatusMethodNotAllowed,
+		}, r.URL)
+	}
+}
+
 // If none of the http routes match respond with appropriate errors
 func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 	if r.Method == http.MethodOptions {

cmd/healthcheck-handler.go

@@ -20,31 +20,36 @@ import (
 	"context"
 	"net/http"
 	"strconv"
+
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
+const unavailable = "offline"
+
 // ClusterCheckHandler returns if the server is ready for requests.
 func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClusterCheckHandler")
 
-	objLayer := newObjectLayerFn()
-	// Service not initialized yet
-	if objLayer == nil {
+	if shouldProxy() {
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
 
+	objLayer := newObjectLayerFn()
+
 	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline())
 	defer cancel()
 
 	opts := HealthOptions{Maintenance: r.URL.Query().Get("maintenance") == "true"}
 	result := objLayer.Health(ctx, opts)
 	if result.WriteQuorum > 0 {
-		w.Header().Set("X-Minio-Write-Quorum", strconv.Itoa(result.WriteQuorum))
+		w.Header().Set(xhttp.MinIOWriteQuorum, strconv.Itoa(result.WriteQuorum))
 	}
 	if !result.Healthy {
 		// return how many drives are being healed if any
 		if result.HealingDrives > 0 {
-			w.Header().Set("X-Minio-Healing-Drives", strconv.Itoa(result.HealingDrives))
+			w.Header().Set(xhttp.MinIOHealingDrives, strconv.Itoa(result.HealingDrives))
 		}
 		// As a maintenance call we are purposefully asked to be taken
 		// down, this is for orchestrators to know if we can safely
@@ -61,12 +66,19 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 
 // ReadinessCheckHandler Checks if the process is up. Always returns success.
 func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	// TODO: only implement this function to notify that this pod is
-	// busy, at a local scope in future, for now '200 OK'.
+	if shouldProxy() {
+		// Service not initialized yet
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
+	}
+
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
 // LivenessCheckHandler - Checks if the process is up. Always returns success.
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
+	if shouldProxy() {
+		// Service not initialized yet
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
+	}
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }

cmd/http/headers.go

@@ -126,6 +126,12 @@ const (
 
 	// Header indicates if the etag should be preserved by client
 	MinIOSourceETag = "x-minio-source-etag"
+
+	// Writes expected write quorum
+	MinIOWriteQuorum = "x-minio-write-quorum"
+
+	// Reports number of drives currently healing
+	MinIOHealingDrives = "x-minio-healing-drives"
 )
 
 // Common http query params S3 API

cmd/http/server.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/cmd/config"
+	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/pkg/certs"
 	"github.com/minio/minio/pkg/env"
 )
@@ -180,13 +181,9 @@ var secureCipherSuites = []uint16{
 // Go only provides constant-time implementations of Curve25519 and NIST P-256 curve.
 var secureCurves = []tls.CurveID{tls.X25519, tls.CurveP256}
 
-const (
-	enableSecureCiphersEnv = "MINIO_API_SECURE_CIPHERS"
-)
-
 // NewServer - creates new HTTP server using given arguments.
 func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificateFunc) *Server {
-	secureCiphers := env.Get(enableSecureCiphersEnv, config.EnableOn) == config.EnableOn
+	secureCiphers := env.Get(api.EnvAPISecureCiphers, config.EnableOn) == config.EnableOn
 
 	var tlsConfig *tls.Config
 	if getCert != nil {

cmd/iam-etcd-store.go

@@ -99,7 +99,7 @@ func (ies *IAMEtcdStore) runlock() {
 	ies.RUnlock()
 }
 
-func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item interface{}, path string) error {
+func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item interface{}, path string, opts ...options) error {
 	data, err := json.Marshal(item)
 	if err != nil {
 		return err
@@ -110,7 +110,7 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item interface{}, pa
 			return err
 		}
 	}
-	return saveKeyEtcd(ctx, ies.client, path, data)
+	return saveKeyEtcd(ctx, ies.client, path, data, opts...)
 }
 
 func (ies *IAMEtcdStore) loadIAMConfig(ctx context.Context, item interface{}, path string) error {
@@ -566,12 +566,12 @@ func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p
 	return ies.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
 }
 
-func (ies *IAMEtcdStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy) error {
-	return ies.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup))
+func (ies *IAMEtcdStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
+	return ies.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...)
 }
 
-func (ies *IAMEtcdStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity) error {
-	return ies.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType))
+func (ies *IAMEtcdStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
+	return ies.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...)
 }
 
 func (ies *IAMEtcdStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error {

cmd/iam-object-store.go

@@ -204,7 +204,7 @@ func (iamOS *IAMObjectStore) migrateBackendFormat(ctx context.Context) error {
 	return iamOS.migrateToV1(ctx)
 }
 
-func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item interface{}, path string) error {
+func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item interface{}, path string, opts ...options) error {
 	data, err := json.Marshal(item)
 	if err != nil {
 		return err
@@ -421,7 +421,6 @@ func (iamOS *IAMObjectStore) loadAll(ctx context.Context, sys *IAMSys) error {
 	}
 	// Sets default canned policies, if none are set.
 	setDefaultCannedPolicies(sys.iamPolicyDocsMap)
-
 	iamOS.unlock()
 
 	if isMinIOUsersSys {
@@ -513,12 +512,12 @@ func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName strin
 	return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
 }
 
-func (iamOS *IAMObjectStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy) error {
-	return iamOS.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup))
+func (iamOS *IAMObjectStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
+	return iamOS.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...)
 }
 
-func (iamOS *IAMObjectStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity) error {
-	return iamOS.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType))
+func (iamOS *IAMObjectStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
+	return iamOS.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...)
 }
 
 func (iamOS *IAMObjectStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error {

cmd/iam.go

@@ -23,7 +23,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math/rand"
 	"strings"
+	"sync"
 	"time"
 
 	humanize "github.com/dustin/go-humanize"
@@ -33,7 +35,6 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
 	"github.com/minio/minio/pkg/madmin"
-	"github.com/minio/minio/pkg/retry"
 )
 
 // UsersSysType - defines the type of users and groups system that is
@@ -201,6 +202,8 @@ func newMappedPolicy(policy string) MappedPolicy {
 
 // IAMSys - config system.
 type IAMSys struct {
+	sync.Mutex
+
 	usersSysType UsersSysType
 
 	// map of policy names to policy definitions
@@ -230,6 +233,11 @@ const (
 	srvAccUser
 )
 
+// key options
+type options struct {
+	ttl int64 //expiry in seconds
+}
+
 // IAMStorageAPI defines an interface for the IAM persistence layer
 type IAMStorageAPI interface {
 	lock()
@@ -254,13 +262,13 @@ type IAMStorageAPI interface {
 
 	loadAll(context.Context, *IAMSys) error
 
-	saveIAMConfig(ctx context.Context, item interface{}, path string) error
+	saveIAMConfig(ctx context.Context, item interface{}, path string, opts ...options) error
 	loadIAMConfig(ctx context.Context, item interface{}, path string) error
 	deleteIAMConfig(ctx context.Context, path string) error
 
 	savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error
-	saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy) error
-	saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity) error
+	saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error
+	saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error
 	saveGroupInfo(ctx context.Context, group string, gi GroupInfo) error
 
 	deletePolicyDoc(ctx context.Context, policyName string) error
@@ -277,7 +285,7 @@ type IAMStorageAPI interface {
 // simplifies the implementation for group removal. This is called
 // only via IAM notifications.
 func (sys *IAMSys) LoadGroup(objAPI ObjectLayer, group string) error {
-	if objAPI == nil || sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -318,7 +326,7 @@ func (sys *IAMSys) LoadGroup(objAPI ObjectLayer, group string) error {
 
 // LoadPolicy - reloads a specific canned policy from backend disks or etcd.
 func (sys *IAMSys) LoadPolicy(objAPI ObjectLayer, policyName string) error {
-	if objAPI == nil || sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -336,7 +344,7 @@ func (sys *IAMSys) LoadPolicy(objAPI ObjectLayer, policyName string) error {
 // LoadPolicyMapping - loads the mapped policy for a user or group
 // from storage into server memory.
 func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isGroup bool) error {
-	if objAPI == nil || sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -362,7 +370,7 @@ func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isG
 
 // LoadUser - reloads a specific user from backend disks or etcd.
 func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, userType IAMUserType) error {
-	if objAPI == nil || sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -386,7 +394,7 @@ func (sys *IAMSys) LoadUser(objAPI ObjectLayer, accessKey string, userType IAMUs
 
 // LoadServiceAccount - reloads a specific service account from backend disks or etcd.
 func (sys *IAMSys) LoadServiceAccount(accessKey string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -410,6 +418,9 @@ func (sys *IAMSys) doIAMConfigMigration(ctx context.Context) error {
 
 // InitStore initializes IAM stores
 func (sys *IAMSys) InitStore(objAPI ObjectLayer) {
+	sys.Lock()
+	defer sys.Unlock()
+
 	if globalEtcdClient == nil {
 		sys.store = newIAMObjectStore(objAPI)
 	} else {
@@ -421,6 +432,16 @@ func (sys *IAMSys) InitStore(objAPI ObjectLayer) {
 	}
 }
 
+// Initialized check if IAM is initialized
+func (sys *IAMSys) Initialized() bool {
+	if sys == nil {
+		return false
+	}
+	sys.Lock()
+	defer sys.Unlock()
+	return sys.store != nil
+}
+
 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	retryCtx, cancel := context.WithCancel(ctx)
@@ -429,7 +450,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	defer cancel()
 
 	// Hold the lock for migration only.
-	txnLk := objAPI.NewNSLock(retryCtx, minioMetaBucket, minioConfigPrefix+"/iam.lock")
+	txnLk := objAPI.NewNSLock(minioMetaBucket, minioConfigPrefix+"/iam.lock")
 
 	// Initializing IAM sub-system needs a retry mechanism for
 	// the following reasons:
@@ -443,11 +464,15 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 	// allocate dynamic timeout once before the loop
 	iamLockTimeout := newDynamicTimeout(5*time.Second, 3*time.Second)
 
-	for range retry.NewTimerWithJitter(retryCtx, time.Second, 5*time.Second, retry.MaxJitter) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	var err error
+	for {
 		// let one of the server acquire the lock, if not let them timeout.
 		// which shall be retried again by this loop.
-		if err := txnLk.GetLock(iamLockTimeout); err != nil {
+		if err = txnLk.GetLock(retryCtx, iamLockTimeout); err != nil {
 			logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. trying to acquire lock")
+			time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
 			continue
 		}
 
@@ -455,7 +480,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 			// ****  WARNING ****
 			// Migrating to encrypted backend on etcd should happen before initialization of
 			// IAM sub-system, make sure that we do not move the above codeblock elsewhere.
-			if err := migrateIAMConfigsEtcdToEncrypted(ctx, globalEtcdClient); err != nil {
+			if err = migrateIAMConfigsEtcdToEncrypted(ctx, globalEtcdClient); err != nil {
 				txnLk.Unlock()
 				logger.LogIf(ctx, fmt.Errorf("Unable to decrypt an encrypted ETCD backend for IAM users and policies: %w", err))
 				logger.LogIf(ctx, errors.New("IAM sub-system is partially initialized, some users may not be available"))
@@ -469,11 +494,10 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 		}
 
 		// Migrate IAM configuration, if necessary.
-		if err := sys.doIAMConfigMigration(ctx); err != nil {
+		if err = sys.doIAMConfigMigration(ctx); err != nil {
 			txnLk.Unlock()
 			if errors.Is(err, errDiskNotFound) ||
 				errors.Is(err, errConfigNotFound) ||
-				errors.Is(err, context.Canceled) ||
 				errors.Is(err, context.DeadlineExceeded) ||
 				errors.As(err, &rquorum) ||
 				errors.As(err, &wquorum) ||
@@ -491,11 +515,10 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 		break
 	}
 
-	err := sys.store.loadAll(ctx, sys)
+	err = sys.store.loadAll(ctx, sys)
 
 	// Invalidate the old cred always, even upon error to avoid any leakage.
 	globalOldCred = auth.Credentials{}
-
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize IAM sub-system, some users may not be available %w", err))
 	}
@@ -505,7 +528,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 
 // DeletePolicy - deletes a canned policy from backend or etcd.
 func (sys *IAMSys) DeletePolicy(policyName string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -557,7 +580,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 
 // InfoPolicy - expands the canned policy into its JSON structure.
 func (sys *IAMSys) InfoPolicy(policyName string) (iampolicy.Policy, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return iampolicy.Policy{}, errServerNotInitialized
 	}
 
@@ -574,17 +597,23 @@ func (sys *IAMSys) InfoPolicy(policyName string) (iampolicy.Policy, error) {
 
 // ListPolicies - lists all canned policies.
 func (sys *IAMSys) ListPolicies() (map[string]iampolicy.Policy, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
 
 	sys.store.rlock()
-	defer sys.store.runlock()
+	fallback := sys.storeFallback
+	sys.store.runlock()
 
-	if sys.storeFallback {
-		return nil, errIAMNotInitialized
+	if fallback {
+		if err := sys.store.loadAll(context.Background(), sys); err != nil {
+			return nil, err
+		}
 	}
 
+	sys.store.rlock()
+	defer sys.store.runlock()
+
 	policyDocsMap := make(map[string]iampolicy.Policy, len(sys.iamPolicyDocsMap))
 	for k, v := range sys.iamPolicyDocsMap {
 		policyDocsMap[k] = v
@@ -595,7 +624,7 @@ func (sys *IAMSys) ListPolicies() (map[string]iampolicy.Policy, error) {
 
 // SetPolicy - sets a new name policy.
 func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -616,7 +645,7 @@ func (sys *IAMSys) SetPolicy(policyName string, p iampolicy.Policy) error {
 
 // DeleteUser - delete user (only for long-term users not STS users).
 func (sys *IAMSys) DeleteUser(accessKey string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -665,13 +694,14 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	return err
 }
 
-// returns comma separated policy string, from an input policy
-// after validating if there are any current policies which exist
-// on MinIO corresponding to the input.
-func (sys *IAMSys) currentPolicies(policyName string) string {
-	if sys.store == nil {
+// CurrentPolicies - returns comma separated policy string, from
+// an input policy after validating if there are any current
+// policies which exist on MinIO corresponding to the input.
+func (sys *IAMSys) CurrentPolicies(policyName string) string {
+	if !sys.Initialized() {
 		return ""
 	}
+
 	sys.store.rlock()
 	defer sys.store.runlock()
 
@@ -688,13 +718,15 @@ func (sys *IAMSys) currentPolicies(policyName string) string {
 
 // SetTempUser - set temporary user credentials, these credentials have an expiry.
 func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyName string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
 	sys.store.lock()
 	defer sys.store.unlock()
 
+	ttl := int64(cred.Expiration.Sub(UTCNow()).Seconds())
+
 	// If OPA is not set we honor any policy claims for this
 	// temporary user which match with pre-configured canned
 	// policies for this server.
@@ -719,7 +751,7 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 			return nil
 		}
 
-		if err := sys.store.saveMappedPolicy(context.Background(), accessKey, stsUser, false, mp); err != nil {
+		if err := sys.store.saveMappedPolicy(context.Background(), accessKey, stsUser, false, mp, options{ttl: ttl}); err != nil {
 			return err
 		}
 
@@ -727,7 +759,7 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 	}
 
 	u := newUserIdentity(cred)
-	if err := sys.store.saveUserIdentity(context.Background(), accessKey, stsUser, u); err != nil {
+	if err := sys.store.saveUserIdentity(context.Background(), accessKey, stsUser, u, options{ttl: ttl}); err != nil {
 		return err
 	}
 
@@ -737,23 +769,29 @@ func (sys *IAMSys) SetTempUser(accessKey string, cred auth.Credentials, policyNa
 
 // ListUsers - list all users.
 func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
 
-	var users = make(map[string]madmin.UserInfo)
-
-	sys.store.rlock()
-	defer sys.store.runlock()
-
 	if sys.usersSysType != MinIOUsersSysType {
 		return nil, errIAMActionNotAllowed
 	}
 
-	if sys.storeFallback {
-		return nil, errIAMNotInitialized
+	sys.store.rlock()
+	fallback := sys.storeFallback
+	sys.store.runlock()
+
+	if fallback {
+		if err := sys.store.loadAll(context.Background(), sys); err != nil {
+			return nil, err
+		}
 	}
 
+	sys.store.rlock()
+	defer sys.store.runlock()
+
+	var users = make(map[string]madmin.UserInfo)
+
 	for k, v := range sys.iamUsersMap {
 		if !v.IsTemp() && !v.IsServiceAccount() {
 			users[k] = madmin.UserInfo{
@@ -773,7 +811,7 @@ func (sys *IAMSys) ListUsers() (map[string]madmin.UserInfo, error) {
 
 // IsTempUser - returns if given key is a temporary user.
 func (sys *IAMSys) IsTempUser(name string) (bool, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return false, errServerNotInitialized
 	}
 
@@ -790,7 +828,7 @@ func (sys *IAMSys) IsTempUser(name string) (bool, error) {
 
 // IsServiceAccount - returns if given key is a service account
 func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return false, "", errServerNotInitialized
 	}
 
@@ -811,7 +849,7 @@ func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) {
 
 // GetUserInfo - get info on a user.
 func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return u, errServerNotInitialized
 	}
 
@@ -856,8 +894,7 @@ func (sys *IAMSys) GetUserInfo(name string) (u madmin.UserInfo, err error) {
 
 // SetUserStatus - sets current user status, supports disabled or enabled.
 func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus) error {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -902,8 +939,7 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 
 // NewServiceAccount - create a new service account
 func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, sessionPolicy *iampolicy.Policy) (auth.Credentials, error) {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return auth.Credentials{}, errServerNotInitialized
 	}
 
@@ -969,20 +1005,24 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, ses
 
 // ListServiceAccounts - lists all services accounts associated to a specific user
 func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([]string, error) {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
 
+	sys.store.rlock()
+	fallback := sys.storeFallback
+	sys.store.runlock()
+
+	if fallback {
+		if err := sys.store.loadAll(context.Background(), sys); err != nil {
+			return nil, err
+		}
+	}
+
 	sys.store.rlock()
 	defer sys.store.runlock()
 
-	if sys.storeFallback {
-		return nil, errIAMNotInitialized
-	}
-
 	var serviceAccounts []string
-
 	for k, v := range sys.iamUsersMap {
 		if v.IsServiceAccount() && v.ParentUser == accessKey {
 			serviceAccounts = append(serviceAccounts, k)
@@ -994,8 +1034,7 @@ func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([
 
 // GetServiceAccountParent - gets information about a service account
 func (sys *IAMSys) GetServiceAccountParent(ctx context.Context, accessKey string) (string, error) {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return "", errServerNotInitialized
 	}
 
@@ -1011,8 +1050,7 @@ func (sys *IAMSys) GetServiceAccountParent(ctx context.Context, accessKey string
 
 // DeleteServiceAccount - delete a service account
 func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) error {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1040,8 +1078,7 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) e
 
 // SetUser - set user credentials and policy.
 func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1078,18 +1115,17 @@ func (sys *IAMSys) SetUser(accessKey string, uinfo madmin.UserInfo) error {
 
 // SetUserSecretKey - sets user secret key
 func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
-
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
-	sys.store.lock()
-	defer sys.store.unlock()
-
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
 	}
 
+	sys.store.lock()
+	defer sys.store.unlock()
+
 	cred, ok := sys.iamUsersMap[accessKey]
 	if !ok {
 		return errNoSuchUser
@@ -1107,7 +1143,7 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 
 // GetUser - get user credentials
 func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return cred, false
 	}
 
@@ -1173,7 +1209,7 @@ func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 // AddUsersToGroup - adds users to a group, creating the group if
 // needed. No error if user(s) already are in the group.
 func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1181,13 +1217,13 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
 		return errInvalidArgument
 	}
 
-	sys.store.lock()
-	defer sys.store.unlock()
-
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
 	}
 
+	sys.store.lock()
+	defer sys.store.unlock()
+
 	// Validate that all members exist.
 	for _, member := range members {
 		cr, ok := sys.iamUsersMap[member]
@@ -1233,7 +1269,7 @@ func (sys *IAMSys) AddUsersToGroup(group string, members []string) error {
 // RemoveUsersFromGroup - remove users from group. If no users are
 // given, and the group is empty, deletes the group as well.
 func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1241,13 +1277,13 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 		return errInvalidArgument
 	}
 
-	sys.store.lock()
-	defer sys.store.unlock()
-
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
 	}
 
+	sys.store.lock()
+	defer sys.store.unlock()
+
 	// Validate that all members exist.
 	for _, member := range members {
 		cr, ok := sys.iamUsersMap[member]
@@ -1313,17 +1349,17 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 
 // SetGroupStatus - enable/disabled a group
 func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
-	sys.store.lock()
-	defer sys.store.unlock()
-
 	if sys.usersSysType != MinIOUsersSysType {
 		return errIAMActionNotAllowed
 	}
 
+	sys.store.lock()
+	defer sys.store.unlock()
+
 	if group == "" {
 		return errInvalidArgument
 	}
@@ -1348,7 +1384,7 @@ func (sys *IAMSys) SetGroupStatus(group string, enabled bool) error {
 
 // GetGroupDescription - builds up group description
 func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return gd, errServerNotInitialized
 	}
 
@@ -1388,30 +1424,38 @@ func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err e
 
 // ListGroups - lists groups.
 func (sys *IAMSys) ListGroups() (r []string, err error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return r, errServerNotInitialized
 	}
 
-	sys.store.rlock()
-	defer sys.store.runlock()
-
-	if sys.storeFallback {
-		return nil, errIAMNotInitialized
-	}
-
 	if sys.usersSysType != MinIOUsersSysType {
 		return nil, errIAMActionNotAllowed
 	}
 
+	sys.store.rlock()
+	fallback := sys.storeFallback
+	sys.store.runlock()
+
+	if fallback {
+		if err := sys.store.loadAll(context.Background(), sys); err != nil {
+			return nil, err
+		}
+	}
+
+	sys.store.rlock()
+	defer sys.store.runlock()
+
+	r = make([]string, 0, len(sys.iamGroupsMap))
 	for k := range sys.iamGroupsMap {
 		r = append(r, k)
 	}
+
 	return r, nil
 }
 
 // PolicyDBSet - sets a policy for a user or group in the PolicyDB.
 func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
@@ -1477,7 +1521,7 @@ func (sys *IAMSys) policyDBSet(name, policyName string, userType IAMUserType, is
 // be a member of multiple groups, this function returns an array of
 // applicable policies (each group is mapped to at most one policy).
 func (sys *IAMSys) PolicyDBGet(name string, isGroup bool) ([]string, error) {
-	if sys == nil || sys.store == nil {
+	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
 
@@ -1909,9 +1953,6 @@ func (sys *IAMSys) removeGroupFromMembershipsMap(group string) {
 
 // EnableLDAPSys - enable ldap system users type.
 func (sys *IAMSys) EnableLDAPSys() {
-	sys.store.lock()
-	defer sys.store.unlock()
-
 	sys.usersSysType = LDAPUsersSysType
 }
 

cmd/listen-notification-handlers.go

@@ -124,7 +124,7 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	listenCh := make(chan interface{}, 4000)
 
-	peers := newPeerRestClients(globalEndpoints)
+	peers, _ := newPeerRestClients(globalEndpoints)
 
 	globalHTTPListen.Subscribe(listenCh, ctx.Done(), func(evI interface{}) bool {
 		ev, ok := evI.(event.Event)

cmd/lock-rest-client.go

@@ -19,8 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"crypto/tls"
-	"errors"
 	"io"
 	"net/url"
 	"strconv"
@@ -29,6 +27,7 @@ import (
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/dsync"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // lockRESTClient is authenticable lock REST client
@@ -152,25 +151,17 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
 		Path:   pathJoin(lockRESTPrefix, endpoint.Path, lockRESTVersion),
 	}
 
-	var tlsConfig *tls.Config
-	if globalIsSSL {
-		tlsConfig = &tls.Config{
-			ServerName: endpoint.Hostname(),
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout)
-	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
+	restClient := rest.NewClient(serverURL, globalInternodeTransport, newAuthToken)
+	restClient.ExpectTimeouts = true
+	// Use a separate client to avoid recursive calls.
+	healthClient := rest.NewClient(serverURL, globalInternodeTransport, newAuthToken)
+	healthClient.ExpectTimeouts = true
 	restClient.HealthCheckFn = func() bool {
 		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
-		// Instantiate a new rest client for healthcheck
-		// to avoid recursive healthCheckFn()
-		respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, lockRESTMethodHealth, nil, nil, -1)
+		defer cancel()
+		respBody, err := healthClient.Call(ctx, lockRESTMethodHealth, nil, nil, -1)
 		xhttp.DrainBody(respBody)
-		cancel()
-		var ne *rest.NetworkError
-		return !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &ne)
+		return !xnet.IsNetworkOrHostDown(err, false)
 	}
 
 	return &lockRESTClient{endpoint: endpoint, restClient: restClient}

cmd/lock-rest-client_test.go

@@ -29,6 +29,7 @@ func TestLockRESTlient(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error %v", err)
 	}
+
 	lkClient := newlockRESTClient(endpoint)
 	if !lkClient.IsOnline() {
 		t.Fatalf("unexpected error. connection failed")

cmd/logger/logger.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"go/build"
 	"hash"
+	"net/http"
 	"path/filepath"
 	"reflect"
 	"runtime"
@@ -299,9 +300,15 @@ func LogIf(ctx context.Context, err error, errKind ...interface{}) {
 		return
 	}
 
-	if !errors.Is(err, context.Canceled) {
-		logIf(ctx, err, errKind...)
+	if errors.Is(err, context.Canceled) {
+		return
 	}
+
+	if err.Error() == http.ErrServerClosed.Error() || err.Error() == "disk not found" {
+		return
+	}
+
+	logIf(ctx, err, errKind...)
 }
 
 // logIf prints a detailed error message during
@@ -337,7 +344,7 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) {
 	trace := getTrace(3)
 
 	// Get the cause for the Error
-	message := err.Error()
+	message := fmt.Sprintf("%v (%T)", err, err)
 	if req.DeploymentID == "" {
 		req.DeploymentID = globalDeploymentID
 	}

cmd/logger/logonce.go

@@ -18,8 +18,9 @@ package logger
 
 import (
 	"context"
+	"errors"
+	"net/http"
 	"sync"
-
 	"time"
 )
 
@@ -77,5 +78,17 @@ var logOnce = newLogOnceType()
 // id is a unique identifier for related log messages, refer to cmd/notification.go
 // on how it is used.
 func LogOnceIf(ctx context.Context, err error, id interface{}, errKind ...interface{}) {
+	if err == nil {
+		return
+	}
+
+	if errors.Is(err, context.Canceled) {
+		return
+	}
+
+	if err.Error() == http.ErrServerClosed.Error() || err.Error() == "disk not found" {
+		return
+	}
+
 	logOnce.logOnceIf(ctx, err, id, errKind...)
 }

cmd/logger/reqinfo.go

@@ -43,6 +43,7 @@ type ReqInfo struct {
 	API          string   // API name - GetObject PutObject NewMultipartUpload etc.
 	BucketName   string   // Bucket name
 	ObjectName   string   // Object name
+	AccessKey    string   // Access Key
 	tags         []KeyVal // Any additional info not accommodated by above fields
 	sync.RWMutex
 }

cmd/merge-walk-pool.go

@@ -1,353 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2019, 2020 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"reflect"
-	"sync"
-	"time"
-)
-
-const (
-	globalMergeLookupTimeout = time.Minute * 1 // 1 minutes.
-)
-
-// mergeWalkVersions - represents the go routine that does the merge walk versions.
-type mergeWalkVersions struct {
-	added      time.Time
-	entryChs   []FileInfoVersionsCh
-	endWalkCh  chan struct{}   // To signal when mergeWalk go-routine should end.
-	endTimerCh chan<- struct{} // To signal when timer go-routine should end.
-}
-
-// mergeWalk - represents the go routine that does the merge walk.
-type mergeWalk struct {
-	added      time.Time
-	entryChs   []FileInfoCh
-	endWalkCh  chan struct{}   // To signal when mergeWalk go-routine should end.
-	endTimerCh chan<- struct{} // To signal when timer go-routine should end.
-}
-
-// MergeWalkVersionsPool - pool of mergeWalk go routines.
-// A mergeWalk is added to the pool by Set() and removed either by
-// doing a Release() or if the concerned timer goes off.
-// mergeWalkPool's purpose is to maintain active mergeWalk go-routines in a map so that
-// it can be looked up across related list calls.
-type MergeWalkVersionsPool struct {
-	sync.Mutex
-	pool    map[listParams][]mergeWalkVersions
-	timeOut time.Duration
-}
-
-// NewMergeWalkVersionsPool - initialize new tree walk pool for versions.
-func NewMergeWalkVersionsPool(timeout time.Duration) *MergeWalkVersionsPool {
-	tPool := &MergeWalkVersionsPool{
-		pool:    make(map[listParams][]mergeWalkVersions),
-		timeOut: timeout,
-	}
-	return tPool
-}
-
-// Release - similar to mergeWalkPool.Release but for versions.
-func (t *MergeWalkVersionsPool) Release(params listParams) ([]FileInfoVersionsCh, chan struct{}) {
-	t.Lock()
-	defer t.Unlock()
-	walks, ok := t.pool[params] // Pick the valid walks.
-	if !ok || len(walks) == 0 {
-		// Release return nil if params not found.
-		return nil, nil
-	}
-
-	// Pop out the first valid walk entry.
-	walk := walks[0]
-	walks = walks[1:]
-	if len(walks) > 0 {
-		t.pool[params] = walks
-	} else {
-		delete(t.pool, params)
-	}
-	walk.endTimerCh <- struct{}{}
-	return walk.entryChs, walk.endWalkCh
-}
-
-// Set - similar to mergeWalkPool.Set but for file versions
-func (t *MergeWalkVersionsPool) Set(params listParams, resultChs []FileInfoVersionsCh, endWalkCh chan struct{}) {
-	t.Lock()
-	defer t.Unlock()
-
-	// If we are above the limit delete at least one entry from the pool.
-	if len(t.pool) > treeWalkEntryLimit {
-		age := time.Now()
-		var oldest listParams
-		for k, v := range t.pool {
-			if len(v) == 0 {
-				delete(t.pool, k)
-				continue
-			}
-			// The first element is the oldest, so we only check that.
-			e := v[0]
-			if e.added.Before(age) {
-				oldest = k
-				age = e.added
-			}
-		}
-		// Invalidate and delete oldest.
-		if walks, ok := t.pool[oldest]; ok && len(walks) > 0 {
-			endCh := walks[0].endTimerCh
-			endWalkCh := walks[0].endWalkCh
-			if len(walks) > 1 {
-				// Move walks forward
-				copy(walks, walks[1:])
-				walks = walks[:len(walks)-1]
-				t.pool[oldest] = walks
-			} else {
-				// Only entry, just delete.
-				delete(t.pool, oldest)
-			}
-			select {
-			case endCh <- struct{}{}:
-				close(endWalkCh)
-			default:
-			}
-		} else {
-			// Shouldn't happen, but just in case.
-			delete(t.pool, oldest)
-		}
-	}
-
-	// Should be a buffered channel so that Release() never blocks.
-	endTimerCh := make(chan struct{}, 1)
-
-	walkInfo := mergeWalkVersions{
-		added:      UTCNow(),
-		entryChs:   resultChs,
-		endWalkCh:  endWalkCh,
-		endTimerCh: endTimerCh,
-	}
-
-	// Append new walk info.
-	walks := t.pool[params]
-	if len(walks) < treeWalkSameEntryLimit {
-		t.pool[params] = append(walks, walkInfo)
-	} else {
-		// We are at limit, invalidate oldest, move list down and add new as last.
-		select {
-		case walks[0].endTimerCh <- struct{}{}:
-			close(walks[0].endWalkCh)
-		default:
-		}
-		copy(walks, walks[1:])
-		walks[len(walks)-1] = walkInfo
-	}
-
-	// Timer go-routine which times out after t.timeOut seconds.
-	go func(endTimerCh <-chan struct{}, walkInfo mergeWalkVersions) {
-		select {
-		// Wait until timeOut
-		case <-time.After(t.timeOut):
-			// Timeout has expired. Remove the mergeWalk from mergeWalkPool and
-			// end the mergeWalk go-routine.
-			t.Lock()
-			defer t.Unlock()
-			walks, ok := t.pool[params]
-			if ok {
-				// Trick of filtering without allocating
-				// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
-				nwalks := walks[:0]
-				// Look for walkInfo, remove it from the walks list.
-				for _, walk := range walks {
-					if !reflect.DeepEqual(walk, walkInfo) {
-						nwalks = append(nwalks, walk)
-					}
-				}
-				if len(nwalks) == 0 {
-					// No more mergeWalk go-routines associated with listParams
-					// hence remove map entry.
-					delete(t.pool, params)
-				} else {
-					// There are more mergeWalk go-routines associated with listParams
-					// hence save the list in the map.
-					t.pool[params] = nwalks
-				}
-			}
-			// Signal the mergeWalk go-routine to die.
-			close(endWalkCh)
-		case <-endTimerCh:
-			return
-		}
-	}(endTimerCh, walkInfo)
-}
-
-// MergeWalkPool - pool of mergeWalk go routines.
-// A mergeWalk is added to the pool by Set() and removed either by
-// doing a Release() or if the concerned timer goes off.
-// mergeWalkPool's purpose is to maintain active mergeWalk go-routines in a map so that
-// it can be looked up across related list calls.
-type MergeWalkPool struct {
-	sync.Mutex
-	pool    map[listParams][]mergeWalk
-	timeOut time.Duration
-}
-
-// NewMergeWalkPool - initialize new tree walk pool.
-func NewMergeWalkPool(timeout time.Duration) *MergeWalkPool {
-	tPool := &MergeWalkPool{
-		pool:    make(map[listParams][]mergeWalk),
-		timeOut: timeout,
-	}
-	return tPool
-}
-
-// Release - selects a mergeWalk from the pool based on the input
-// listParams, removes it from the pool, and returns the MergeWalkResult
-// channel.
-// Returns nil if listParams does not have an associated mergeWalk.
-func (t *MergeWalkPool) Release(params listParams) ([]FileInfoCh, chan struct{}) {
-	t.Lock()
-	defer t.Unlock()
-	walks, ok := t.pool[params] // Pick the valid walks.
-	if !ok || len(walks) == 0 {
-		// Release return nil if params not found.
-		return nil, nil
-	}
-
-	// Pop out the first valid walk entry.
-	walk := walks[0]
-	walks[0] = mergeWalk{} // clear references.
-	walks = walks[1:]
-	if len(walks) > 0 {
-		t.pool[params] = walks
-	} else {
-		delete(t.pool, params)
-	}
-	walk.endTimerCh <- struct{}{}
-	return walk.entryChs, walk.endWalkCh
-}
-
-// Set - adds a mergeWalk to the mergeWalkPool.
-// Also starts a timer go-routine that ends when:
-// 1) time.After() expires after t.timeOut seconds.
-//    The expiration is needed so that the mergeWalk go-routine resources are freed after a timeout
-//    if the S3 client does only partial listing of objects.
-// 2) Release() signals the timer go-routine to end on endTimerCh.
-//    During listing the timer should not timeout and end the mergeWalk go-routine, hence the
-//    timer go-routine should be ended.
-func (t *MergeWalkPool) Set(params listParams, resultChs []FileInfoCh, endWalkCh chan struct{}) {
-	t.Lock()
-	defer t.Unlock()
-
-	// If we are above the limit delete at least one entry from the pool.
-	if len(t.pool) > treeWalkEntryLimit {
-		age := time.Now()
-		var oldest listParams
-		for k, v := range t.pool {
-			if len(v) == 0 {
-				delete(t.pool, k)
-				continue
-			}
-			// The first element is the oldest, so we only check that.
-			e := v[0]
-			if e.added.Before(age) {
-				oldest = k
-				age = e.added
-			}
-		}
-		// Invalidate and delete oldest.
-		if walks, ok := t.pool[oldest]; ok && len(walks) > 0 {
-			endCh := walks[0].endTimerCh
-			endWalkCh := walks[0].endWalkCh
-			if len(walks) > 1 {
-				// Move walks forward
-				copy(walks, walks[1:])
-				walks = walks[:len(walks)-1]
-				t.pool[oldest] = walks
-			} else {
-				// Only entry, just delete.
-				delete(t.pool, oldest)
-			}
-			select {
-			case endCh <- struct{}{}:
-				close(endWalkCh)
-			default:
-			}
-		} else {
-			// Shouldn't happen, but just in case.
-			delete(t.pool, oldest)
-		}
-	}
-
-	// Should be a buffered channel so that Release() never blocks.
-	endTimerCh := make(chan struct{}, 1)
-	walkInfo := mergeWalk{
-		added:      UTCNow(),
-		entryChs:   resultChs,
-		endWalkCh:  endWalkCh,
-		endTimerCh: endTimerCh,
-	}
-
-	// Append new walk info.
-	walks := t.pool[params]
-	if len(walks) < treeWalkSameEntryLimit {
-		t.pool[params] = append(walks, walkInfo)
-	} else {
-		// We are at limit, invalidate oldest, move list down and add new as last.
-		select {
-		case walks[0].endTimerCh <- struct{}{}:
-			close(walks[0].endWalkCh)
-		default:
-		}
-		copy(walks, walks[1:])
-		walks[len(walks)-1] = walkInfo
-	}
-
-	// Timer go-routine which times out after t.timeOut seconds.
-	go func(endTimerCh <-chan struct{}, walkInfo mergeWalk) {
-		select {
-		// Wait until timeOut
-		case <-time.After(t.timeOut):
-			// Timeout has expired. Remove the mergeWalk from mergeWalkPool and
-			// end the mergeWalk go-routine.
-			t.Lock()
-			defer t.Unlock()
-			walks, ok := t.pool[params]
-			if ok {
-				// Trick of filtering without allocating
-				// https://github.com/golang/go/wiki/SliceTricks#filtering-without-allocating
-				nwalks := walks[:0]
-				// Look for walkInfo, remove it from the walks list.
-				for _, walk := range walks {
-					if !reflect.DeepEqual(walk, walkInfo) {
-						nwalks = append(nwalks, walk)
-					}
-				}
-				if len(nwalks) == 0 {
-					// No more mergeWalk go-routines associated with listParams
-					// hence remove map entry.
-					delete(t.pool, params)
-				} else {
-					// There are more mergeWalk go-routines associated with listParams
-					// hence save the list in the map.
-					t.pool[params] = nwalks
-				}
-			}
-			// Signal the mergeWalk go-routine to die.
-			close(endWalkCh)
-		case <-endTimerCh:
-			return
-		}
-	}(endTimerCh, walkInfo)
-}

cmd/merge-walk-pool_test.go

@@ -1,185 +0,0 @@
-/*
- * MinIO Cloud Storage, (C) 2019,2020 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"testing"
-	"time"
-)
-
-// Test if tree walker go-routine is removed from the pool after timeout
-// and that is available in the pool before the timeout.
-func TestMergeWalkPoolVersionsBasic(t *testing.T) {
-	// Create a treeWalkPool
-	tw := NewMergeWalkVersionsPool(1 * time.Second)
-
-	// Create sample params
-	params := listParams{
-		bucket: "test-bucket",
-	}
-
-	endWalkCh := make(chan struct{})
-	// Add a treeWalk to the pool
-	tw.Set(params, []FileInfoVersionsCh{}, endWalkCh)
-
-	// Wait for treeWalkPool timeout to happen
-	<-time.After(2 * time.Second)
-	if c1, _ := tw.Release(params); c1 != nil {
-		t.Error("treeWalk go-routine must have been freed")
-	}
-
-	// Add the treeWalk back to the pool
-	endWalkCh = make(chan struct{})
-	tw.Set(params, []FileInfoVersionsCh{}, endWalkCh)
-
-	// Release the treeWalk before timeout
-	select {
-	case <-time.After(1 * time.Second):
-		break
-	default:
-		if c1, _ := tw.Release(params); c1 == nil {
-			t.Error("treeWalk go-routine got freed before timeout")
-		}
-	}
-}
-
-// Test if tree walker go-routine is removed from the pool after timeout
-// and that is available in the pool before the timeout.
-func TestMergeWalkPoolBasic(t *testing.T) {
-	// Create a treeWalkPool
-	tw := NewMergeWalkPool(1 * time.Second)
-
-	// Create sample params
-	params := listParams{
-		bucket: "test-bucket",
-	}
-
-	endWalkCh := make(chan struct{})
-	// Add a treeWalk to the pool
-	tw.Set(params, []FileInfoCh{}, endWalkCh)
-
-	// Wait for treeWalkPool timeout to happen
-	<-time.After(2 * time.Second)
-	if c1, _ := tw.Release(params); c1 != nil {
-		t.Error("treeWalk go-routine must have been freed")
-	}
-
-	// Add the treeWalk back to the pool
-	endWalkCh = make(chan struct{})
-	tw.Set(params, []FileInfoCh{}, endWalkCh)
-
-	// Release the treeWalk before timeout
-	select {
-	case <-time.After(1 * time.Second):
-		break
-	default:
-		if c1, _ := tw.Release(params); c1 == nil {
-			t.Error("treeWalk go-routine got freed before timeout")
-		}
-	}
-}
-
-// Test if multiple merge walkers for the same listParams are managed as expected by the pool.
-func TestManyMergeWalksSameParam(t *testing.T) {
-	// Create a treeWalkPool.
-	tw := NewMergeWalkPool(5 * time.Second)
-
-	// Create sample params.
-	params := listParams{
-		bucket: "test-bucket",
-	}
-
-	select {
-	// This timeout is an upper-bound. This is started
-	// before the first treeWalk go-routine's timeout period starts.
-	case <-time.After(5 * time.Second):
-		break
-	default:
-		// Create many treeWalk go-routines for the same params.
-		for i := 0; i < treeWalkSameEntryLimit; i++ {
-			endWalkCh := make(chan struct{})
-			walkChs := make([]FileInfoCh, 0)
-			tw.Set(params, walkChs, endWalkCh)
-		}
-
-		tw.Lock()
-		if walks, ok := tw.pool[params]; ok {
-			if len(walks) != treeWalkSameEntryLimit {
-				t.Error("There aren't as many walks as were Set")
-			}
-		}
-		tw.Unlock()
-		for i := 0; i < treeWalkSameEntryLimit; i++ {
-			tw.Lock()
-			if walks, ok := tw.pool[params]; ok {
-				// Before ith Release we should have n-i treeWalk go-routines.
-				if treeWalkSameEntryLimit-i != len(walks) {
-					t.Error("There aren't as many walks as were Set")
-				}
-			}
-			tw.Unlock()
-			tw.Release(params)
-		}
-	}
-
-}
-
-// Test if multiple merge walkers for the same listParams are managed as expected by the pool
-// but that treeWalkSameEntryLimit is respected.
-func TestManyMergeWalksSameParamPrune(t *testing.T) {
-	// Create a treeWalkPool.
-	tw := NewMergeWalkPool(5 * time.Second)
-
-	// Create sample params.
-	params := listParams{
-		bucket: "test-bucket",
-	}
-
-	select {
-	// This timeout is an upper-bound. This is started
-	// before the first treeWalk go-routine's timeout period starts.
-	case <-time.After(5 * time.Second):
-		break
-	default:
-		// Create many treeWalk go-routines for the same params.
-		for i := 0; i < treeWalkSameEntryLimit*4; i++ {
-			endWalkCh := make(chan struct{})
-			walkChs := make([]FileInfoCh, 0)
-			tw.Set(params, walkChs, endWalkCh)
-		}
-
-		tw.Lock()
-		if walks, ok := tw.pool[params]; ok {
-			if len(walks) > treeWalkSameEntryLimit {
-				t.Error("There aren't as many walks as were Set")
-			}
-		}
-		tw.Unlock()
-		for i := 0; i < treeWalkSameEntryLimit; i++ {
-			tw.Lock()
-			if walks, ok := tw.pool[params]; ok {
-				// Before ith Release we should have n-i treeWalk go-routines.
-				if treeWalkSameEntryLimit-i != len(walks) {
-					t.Error("There aren't as many walks as were Set")
-				}
-			}
-			tw.Unlock()
-			tw.Release(params)
-		}
-	}
-
-}

cmd/metacache-bucket.go

@@ -0,0 +1,448 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"runtime/debug"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/klauspost/compress/s2"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/hash"
+	"github.com/tinylib/msgp/msgp"
+)
+
+//go:generate msgp -file $GOFILE -unexported
+
+// a bucketMetacache keeps track of all caches generated
+// for a bucket.
+type bucketMetacache struct {
+	// Name of bucket
+	bucket string
+
+	// caches indexed by id.
+	caches map[string]metacache
+
+	// Internal state
+	mu        sync.RWMutex `msg:"-"`
+	updated   bool         `msg:"-"`
+	transient bool         `msg:"-"` // bucket used for non-persisted caches.
+}
+
+// newBucketMetacache creates a new bucketMetacache.
+// Optionally remove all existing caches.
+func newBucketMetacache(bucket string, cleanup bool) *bucketMetacache {
+	if cleanup {
+		// Recursively delete all caches.
+		objAPI := newObjectLayerFn()
+		ez, ok := objAPI.(*erasureServerSets)
+		if ok {
+			ctx := context.Background()
+			ez.deleteAll(ctx, minioMetaBucket, metacachePrefixForID(bucket, slashSeparator))
+		}
+	}
+	return &bucketMetacache{
+		bucket: bucket,
+		caches: make(map[string]metacache, 10),
+	}
+}
+
+// loadBucketMetaCache will load the cache from the object layer.
+// If the cache cannot be found a new one is created.
+func loadBucketMetaCache(ctx context.Context, bucket string) (*bucketMetacache, error) {
+	objAPI := newObjectLayerFn()
+	for objAPI == nil {
+		select {
+		case <-ctx.Done():
+			return nil, ctx.Err()
+		case <-time.After(250 * time.Millisecond):
+		}
+		objAPI = newObjectLayerFn()
+		if objAPI == nil {
+			logger.LogIf(ctx, fmt.Errorf("loadBucketMetaCache: object layer not ready. bucket: %q", bucket))
+		}
+	}
+	var meta bucketMetacache
+	var decErr error
+	var wg sync.WaitGroup
+	wg.Add(1)
+
+	r, w := io.Pipe()
+	go func() {
+		defer wg.Done()
+		dec := s2DecPool.Get().(*s2.Reader)
+		dec.Reset(r)
+		decErr = meta.DecodeMsg(msgp.NewReader(dec))
+		dec.Reset(nil)
+		s2DecPool.Put(dec)
+		r.CloseWithError(decErr)
+	}()
+	// Use global context for this.
+	err := objAPI.GetObject(GlobalContext, minioMetaBucket, pathJoin("buckets", bucket, ".metacache", "index.s2"), 0, -1, w, "", ObjectOptions{})
+	logger.LogIf(ctx, w.CloseWithError(err))
+	if err != nil {
+		if isErrObjectNotFound(err) {
+			err = nil
+		} else {
+			logger.LogIf(ctx, err)
+		}
+		if errors.Is(err, InsufficientReadQuorum{}) {
+			// Cache is likely lost. Clean up and return new.
+			return newBucketMetacache(bucket, true), nil
+		}
+		return newBucketMetacache(bucket, false), err
+	}
+	wg.Wait()
+	if decErr != nil {
+		if errors.Is(err, context.Canceled) {
+			return newBucketMetacache(bucket, false), err
+		}
+		// Log the error, but assume the data is lost and return a fresh bucket.
+		// Otherwise a broken cache will never recover.
+		logger.LogIf(ctx, decErr)
+		return newBucketMetacache(bucket, true), nil
+	}
+	// Sanity check...
+	if meta.bucket != bucket {
+		logger.Info("loadBucketMetaCache: loaded cache name mismatch, want %s, got %s. Discarding.", bucket, meta.bucket)
+		return newBucketMetacache(bucket, true), nil
+	}
+	return &meta, nil
+}
+
+// save the bucket cache to the object storage.
+func (b *bucketMetacache) save(ctx context.Context) error {
+	if b.transient {
+		return nil
+	}
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	// Keep lock while we marshal.
+	// We need a write lock since we update 'updated'
+	b.mu.Lock()
+	if !b.updated {
+		b.mu.Unlock()
+		return nil
+	}
+	// Save as s2 compressed msgpack
+	tmp := bytes.NewBuffer(make([]byte, 0, b.Msgsize()))
+	enc := s2.NewWriter(tmp)
+	err := msgp.Encode(enc, b)
+	if err != nil {
+		b.mu.Unlock()
+		return err
+	}
+	err = enc.Close()
+	if err != nil {
+		b.mu.Unlock()
+		return err
+	}
+	b.updated = false
+	b.mu.Unlock()
+
+	hr, err := hash.NewReader(tmp, int64(tmp.Len()), "", "", int64(tmp.Len()), false)
+	if err != nil {
+		return err
+	}
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, pathJoin("buckets", b.bucket, ".metacache", "index.s2"), NewPutObjReader(hr, nil, nil), ObjectOptions{})
+	logger.LogIf(ctx, err)
+	return err
+}
+
+// findCache will attempt to find a matching cache for the provided options.
+// If a cache with the same ID exists already it will be returned.
+// If none can be found a new is created with the provided ID.
+func (b *bucketMetacache) findCache(o listPathOptions) metacache {
+	if b == nil {
+		logger.Info("bucketMetacache.findCache: nil cache for bucket %s", o.Bucket)
+		return metacache{}
+	}
+
+	if o.Bucket != b.bucket && !b.transient {
+		logger.Info("bucketMetacache.findCache: bucket %s does not match this bucket %s", o.Bucket, b.bucket)
+		debug.PrintStack()
+		return metacache{}
+	}
+
+	debugPrint := func(msg string, data ...interface{}) {}
+	if false {
+		debugPrint = logger.Info
+	}
+
+	// Grab a write lock, since we create one if we cannot find one.
+	if o.Create {
+		b.mu.Lock()
+		defer b.mu.Unlock()
+	} else {
+		b.mu.RLock()
+		defer b.mu.RUnlock()
+	}
+
+	// Check if exists already.
+	if c, ok := b.caches[o.ID]; ok {
+		debugPrint("returning existing %v", o.ID)
+		return c
+	}
+
+	var best metacache
+	extend := globalAPIConfig.getExtendListLife()
+	for _, cached := range b.caches {
+		// Never return transient caches if there is no id.
+		if b.transient {
+			break
+		}
+		if cached.status == scanStateError || cached.status == scanStateNone || cached.dataVersion != metacacheStreamVersion {
+			debugPrint("cache %s state or stream version mismatch", cached.id)
+			continue
+		}
+		if cached.startedCycle < o.OldestCycle {
+			debugPrint("cache %s cycle too old", cached.id)
+			continue
+		}
+		// Root of what we are looking for must at least have
+		if !strings.HasPrefix(o.BaseDir, cached.root) {
+			debugPrint("cache %s prefix mismatch, cached:%v, want:%v", cached.id, cached.root, o.BaseDir)
+			continue
+		}
+		// If the existing listing wasn't recursive root must match.
+		if !cached.recursive && o.BaseDir != cached.root {
+			debugPrint("cache %s  non rec prefix mismatch, cached:%v, want:%v", cached.id, cached.root, o.BaseDir)
+			continue
+		}
+		if o.Recursive && !cached.recursive {
+			debugPrint("cache %s not recursive", cached.id)
+			// If this is recursive the cached listing must be as well.
+			continue
+		}
+		if o.Separator != slashSeparator && !cached.recursive {
+			debugPrint("cache %s not slashsep and not recursive", cached.id)
+			// Non slash separator requires recursive.
+			continue
+		}
+		if !cached.finished() && time.Since(cached.lastUpdate) > metacacheMaxRunningAge {
+			debugPrint("cache %s not running, time: %v", cached.id, time.Since(cached.lastUpdate))
+			// Abandoned
+			continue
+		}
+
+		if cached.finished() && cached.endedCycle <= o.OldestCycle {
+			if extend <= 0 {
+				// If scan has ended the oldest requested must be less.
+				debugPrint("cache %s ended and cycle (%v) <= oldest allowed (%v)", cached.id, cached.endedCycle, o.OldestCycle)
+				continue
+			}
+			if time.Since(cached.lastUpdate) > metacacheMaxRunningAge+extend {
+				// Cache ended within bloom cycle, but we can extend the life.
+				debugPrint("cache %s ended (%v) and beyond extended life (%v)", cached.id, cached.lastUpdate, extend+metacacheMaxRunningAge)
+				continue
+			}
+		}
+		if cached.started.Before(best.started) {
+			debugPrint("cache %s disregarded - we have a better", cached.id)
+			// If we already have a newer, keep that.
+			continue
+		}
+		best = cached
+	}
+	if !best.started.IsZero() {
+		if o.Create {
+			best.lastHandout = UTCNow()
+			b.caches[best.id] = best
+			b.updated = true
+		}
+		debugPrint("returning cached %s, status: %v, ended: %v", best.id, best.status, best.ended)
+		return best
+	}
+	if !o.Create {
+		return metacache{
+			id:     o.ID,
+			bucket: o.Bucket,
+			status: scanStateNone,
+		}
+	}
+
+	// Create new and add.
+	best = o.newMetacache()
+	b.caches[o.ID] = best
+	b.updated = true
+	debugPrint("returning new cache %s, bucket: %v", best.id, best.bucket)
+	return best
+}
+
+// cleanup removes redundant and outdated entries.
+func (b *bucketMetacache) cleanup() {
+	// Entries to remove.
+	remove := make(map[string]struct{})
+	currentCycle := intDataUpdateTracker.current()
+
+	debugPrint := func(msg string, data ...interface{}) {}
+	if false {
+		debugPrint = logger.Info
+	}
+
+	b.mu.RLock()
+	for id, cache := range b.caches {
+		if b.transient && time.Since(cache.lastUpdate) > 15*time.Minute && time.Since(cache.lastHandout) > 15*time.Minute {
+			// Keep transient caches only for 1 hour.
+			remove[id] = struct{}{}
+			continue
+		}
+		if !cache.worthKeeping(currentCycle) {
+			debugPrint("cache %s not worth keeping", id)
+			remove[id] = struct{}{}
+			continue
+		}
+		if cache.id != id {
+			logger.Info("cache ID mismatch %s != %s", id, cache.id)
+			remove[id] = struct{}{}
+			continue
+		}
+		if cache.bucket != b.bucket && !b.transient {
+			logger.Info("cache bucket mismatch %s != %s", b.bucket, cache.bucket)
+			remove[id] = struct{}{}
+			continue
+		}
+	}
+
+	// Check all non-deleted against eachother.
+	// O(n*n), but should still be rather quick.
+	for id, cache := range b.caches {
+		if b.transient {
+			break
+		}
+		if _, ok := remove[id]; ok {
+			continue
+		}
+		for _, cache2 := range b.caches {
+			if cache.canBeReplacedBy(&cache2) {
+				debugPrint("cache %s can be replaced by %s", id, cache2.id)
+				remove[id] = struct{}{}
+				break
+			} else {
+				debugPrint("cache %s can be NOT replaced by %s", id, cache2.id)
+			}
+		}
+	}
+
+	b.mu.RUnlock()
+	for id := range remove {
+		b.deleteCache(id)
+	}
+}
+
+// updateCache will update a cache by id.
+// If the cache cannot be found nil is returned.
+// The bucket cache will be locked until the done .
+func (b *bucketMetacache) updateCache(id string) (cache *metacache, done func()) {
+	b.mu.Lock()
+	c, ok := b.caches[id]
+	if !ok {
+		b.mu.Unlock()
+		return nil, func() {}
+	}
+	return &c, func() {
+		c.lastUpdate = UTCNow()
+		b.caches[id] = c
+		b.mu.Unlock()
+	}
+}
+
+// updateCacheEntry will update a cache.
+// Returns the updated status.
+func (b *bucketMetacache) updateCacheEntry(update metacache) (metacache, error) {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	existing, ok := b.caches[update.id]
+	if !ok {
+		logger.Info("updateCacheEntry: bucket %s list id %v not found", b.bucket, update.id)
+		return update, errFileNotFound
+	}
+	existing.update(update)
+	b.caches[update.id] = existing
+	b.updated = true
+	return existing, nil
+}
+
+// getCache will return a clone of a specific metacache.
+// Will return nil if the cache doesn't exist.
+func (b *bucketMetacache) getCache(id string) *metacache {
+	b.mu.RLock()
+	c, ok := b.caches[id]
+	b.mu.RUnlock()
+	if !ok {
+		return nil
+	}
+	return &c
+}
+
+// deleteAll will delete all on disk data for ALL caches.
+// Deletes are performed concurrently.
+func (b *bucketMetacache) deleteAll() {
+	b.mu.Lock()
+	defer b.mu.Unlock()
+
+	ctx := context.Background()
+	ez, ok := newObjectLayerFn().(*erasureServerSets)
+	if !ok {
+		logger.LogIf(ctx, errors.New("bucketMetacache: expected objAPI to be *erasureZones"))
+		return
+	}
+
+	b.updated = true
+	if !b.transient {
+		// Delete all.
+		ez.deleteAll(ctx, minioMetaBucket, metacachePrefixForID(b.bucket, slashSeparator))
+		b.caches = make(map[string]metacache, 10)
+		return
+	}
+
+	// Transient are in different buckets.
+	var wg sync.WaitGroup
+	for id := range b.caches {
+		wg.Add(1)
+		go func(cache metacache) {
+			defer wg.Done()
+			ez.deleteAll(ctx, minioMetaBucket, metacachePrefixForID(cache.bucket, cache.id))
+		}(b.caches[id])
+	}
+	wg.Wait()
+	b.caches = make(map[string]metacache, 10)
+}
+
+// deleteCache will delete a specific cache and all files related to it across the cluster.
+func (b *bucketMetacache) deleteCache(id string) {
+	b.mu.Lock()
+	c, ok := b.caches[id]
+	if ok {
+		delete(b.caches, id)
+		b.updated = true
+	}
+	b.mu.Unlock()
+	if ok {
+		c.delete(context.Background())
+	}
+}

cmd/metacache-bucket_gen.go

@@ -0,0 +1,209 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *bucketMetacache) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "bucket":
+			z.bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "bucket")
+				return
+			}
+		case "caches":
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "caches")
+				return
+			}
+			if z.caches == nil {
+				z.caches = make(map[string]metacache, zb0002)
+			} else if len(z.caches) > 0 {
+				for key := range z.caches {
+					delete(z.caches, key)
+				}
+			}
+			for zb0002 > 0 {
+				zb0002--
+				var za0001 string
+				var za0002 metacache
+				za0001, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "caches")
+					return
+				}
+				err = za0002.DecodeMsg(dc)
+				if err != nil {
+					err = msgp.WrapError(err, "caches", za0001)
+					return
+				}
+				z.caches[za0001] = za0002
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *bucketMetacache) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "bucket"
+	err = en.Append(0x82, 0xa6, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "bucket")
+		return
+	}
+	// write "caches"
+	err = en.Append(0xa6, 0x63, 0x61, 0x63, 0x68, 0x65, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteMapHeader(uint32(len(z.caches)))
+	if err != nil {
+		err = msgp.WrapError(err, "caches")
+		return
+	}
+	for za0001, za0002 := range z.caches {
+		err = en.WriteString(za0001)
+		if err != nil {
+			err = msgp.WrapError(err, "caches")
+			return
+		}
+		err = za0002.EncodeMsg(en)
+		if err != nil {
+			err = msgp.WrapError(err, "caches", za0001)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *bucketMetacache) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "bucket"
+	o = append(o, 0x82, 0xa6, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	o = msgp.AppendString(o, z.bucket)
+	// string "caches"
+	o = append(o, 0xa6, 0x63, 0x61, 0x63, 0x68, 0x65, 0x73)
+	o = msgp.AppendMapHeader(o, uint32(len(z.caches)))
+	for za0001, za0002 := range z.caches {
+		o = msgp.AppendString(o, za0001)
+		o, err = za0002.MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "caches", za0001)
+			return
+		}
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *bucketMetacache) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "bucket":
+			z.bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "bucket")
+				return
+			}
+		case "caches":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "caches")
+				return
+			}
+			if z.caches == nil {
+				z.caches = make(map[string]metacache, zb0002)
+			} else if len(z.caches) > 0 {
+				for key := range z.caches {
+					delete(z.caches, key)
+				}
+			}
+			for zb0002 > 0 {
+				var za0001 string
+				var za0002 metacache
+				zb0002--
+				za0001, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "caches")
+					return
+				}
+				bts, err = za0002.UnmarshalMsg(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "caches", za0001)
+					return
+				}
+				z.caches[za0001] = za0002
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *bucketMetacache) Msgsize() (s int) {
+	s = 1 + 7 + msgp.StringPrefixSize + len(z.bucket) + 7 + msgp.MapHeaderSize
+	if z.caches != nil {
+		for za0001, za0002 := range z.caches {
+			_ = za0002
+			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
+		}
+	}
+	return
+}

cmd/metacache-bucket_gen_test.go

@@ -0,0 +1,123 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalbucketMetacache(t *testing.T) {
+	v := bucketMetacache{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgbucketMetacache(b *testing.B) {
+	v := bucketMetacache{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgbucketMetacache(b *testing.B) {
+	v := bucketMetacache{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalbucketMetacache(b *testing.B) {
+	v := bucketMetacache{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodebucketMetacache(t *testing.T) {
+	v := bucketMetacache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodebucketMetacache Msgsize() is inaccurate")
+	}
+
+	vn := bucketMetacache{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodebucketMetacache(b *testing.B) {
+	v := bucketMetacache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodebucketMetacache(b *testing.B) {
+	v := bucketMetacache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/metacache-entries.go

@@ -0,0 +1,588 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"sort"
+	"strings"
+)
+
+// metaCacheEntry is an object or a directory within an unknown bucket.
+type metaCacheEntry struct {
+	// name is the full name of the object including prefixes
+	name string
+	// Metadata. If none is present it is not an object but only a prefix.
+	// Entries without metadata will only be present in non-recursive scans.
+	metadata []byte
+
+	// cached contains the metadata if decoded.
+	cached *FileInfo
+}
+
+// isDir returns if the entry is representing a prefix directory.
+func (e metaCacheEntry) isDir() bool {
+	return len(e.metadata) == 0
+}
+
+// isObject returns if the entry is representing an object.
+func (e metaCacheEntry) isObject() bool {
+	return len(e.metadata) > 0
+}
+
+// hasPrefix returns whether an entry has a specific prefix
+func (e metaCacheEntry) hasPrefix(s string) bool {
+	return strings.HasPrefix(e.name, s)
+}
+
+// likelyMatches returns if the entries match by comparing name and metadata length.
+func (e *metaCacheEntry) likelyMatches(other *metaCacheEntry) bool {
+	// This should reject 99%
+	if len(e.metadata) != len(other.metadata) || e.name != other.name {
+		return false
+	}
+	return true
+}
+
+// matches returns if the entries match by comparing their latest version fileinfo.
+func (e *metaCacheEntry) matches(other *metaCacheEntry, bucket string) bool {
+	if e == nil && other == nil {
+		return true
+	}
+	if e == nil || other == nil {
+		return false
+	}
+
+	// This should reject 99%
+	if len(e.metadata) != len(other.metadata) || e.name != other.name {
+		return false
+	}
+	eFi, eErr := e.fileInfo(bucket)
+	oFi, oErr := e.fileInfo(bucket)
+	if eErr != nil || oErr != nil {
+		return eErr == oErr
+	}
+	return eFi.ModTime.Equal(oFi.ModTime) && eFi.Size == oFi.Size && eFi.VersionID == oFi.VersionID
+}
+
+// isInDir returns whether the entry is in the dir when considering the separator.
+func (e metaCacheEntry) isInDir(dir, separator string) bool {
+	if len(dir) == 0 {
+		// Root
+		idx := strings.Index(e.name, separator)
+		return idx == -1 || idx == len(e.name)-len(separator)
+	}
+	ext := strings.TrimPrefix(e.name, dir)
+	if len(ext) != len(e.name) {
+		idx := strings.Index(ext, separator)
+		// If separator is not found or is last entry, ok.
+		return idx == -1 || idx == len(ext)-len(separator)
+	}
+	return false
+}
+
+// isLatestDeletemarker returns whether the latest version is a delete marker.
+// If metadata is NOT versioned false will always be returned.
+// If v2 and UNABLE to load metadata true will be returned.
+func (e *metaCacheEntry) isLatestDeletemarker() bool {
+	if e.cached != nil {
+		return e.cached.Deleted
+	}
+	if !isXL2V1Format(e.metadata) {
+		return false
+	}
+	var xlMeta xlMetaV2
+	if err := xlMeta.Load(e.metadata); err != nil || len(xlMeta.Versions) == 0 {
+		return true
+	}
+	return xlMeta.Versions[len(xlMeta.Versions)-1].Type == DeleteType
+}
+
+// fileInfo returns the decoded metadata.
+// If entry is a directory it is returned as that.
+// If versioned the latest version will be returned.
+func (e *metaCacheEntry) fileInfo(bucket string) (*FileInfo, error) {
+	if e.isDir() {
+		return &FileInfo{
+			Volume: bucket,
+			Name:   e.name,
+			Mode:   uint32(os.ModeDir),
+		}, nil
+	}
+	if e.cached == nil {
+		fi, err := getFileInfo(e.metadata, bucket, e.name, "")
+		if err != nil {
+			return nil, err
+		}
+		e.cached = &fi
+	}
+	return e.cached, nil
+}
+
+// fileInfoVersions returns the metadata as FileInfoVersions.
+// If entry is a directory it is returned as that.
+func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, error) {
+	if e.isDir() {
+		return FileInfoVersions{
+			Volume: bucket,
+			Name:   e.name,
+			Versions: []FileInfo{
+				{
+					Volume: bucket,
+					Name:   e.name,
+					Mode:   uint32(os.ModeDir),
+				},
+			},
+		}, nil
+	}
+	return getFileInfoVersions(e.metadata, bucket, e.name)
+}
+
+// metaCacheEntries is a slice of metacache entries.
+type metaCacheEntries []metaCacheEntry
+
+// less function for sorting.
+func (m metaCacheEntries) less(i, j int) bool {
+	return m[i].name < m[j].name
+}
+
+// sort entries by name.
+// m is sorted and a sorted metadata object is returned.
+// Changes to m will also be reflected in the returned object.
+func (m metaCacheEntries) sort() metaCacheEntriesSorted {
+	if m.isSorted() {
+		return metaCacheEntriesSorted{o: m}
+	}
+	sort.Slice(m, m.less)
+	return metaCacheEntriesSorted{o: m}
+}
+
+// isSorted returns whether the objects are sorted.
+// This is usually orders of magnitude faster than actually sorting.
+func (m metaCacheEntries) isSorted() bool {
+	return sort.SliceIsSorted(m, m.less)
+}
+
+// shallowClone will create a shallow clone of the array objects,
+// but object metadata will not be cloned.
+func (m metaCacheEntries) shallowClone() metaCacheEntries {
+	dst := make(metaCacheEntries, len(m))
+	copy(dst, m)
+	return dst
+}
+
+type metadataResolutionParams struct {
+	dirQuorum int    // Number if disks needed for a directory to 'exist'.
+	objQuorum int    // Number of disks needed for an object to 'exist'.
+	bucket    string // Name of the bucket. Used for generating cached fileinfo.
+}
+
+func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCacheEntry, ok bool) {
+	if len(m) == 0 {
+		return nil, false
+	}
+
+	dirExists := 0
+	objExists := 0
+	var selFIV *FileInfo
+	for i := range m {
+		entry := &m[i]
+		if entry.name == "" {
+			continue
+		}
+		if entry.isDir() {
+			dirExists++
+			selected = entry
+			continue
+		}
+
+		// Get new entry metadata
+		objExists++
+		fiv, err := entry.fileInfo(r.bucket)
+		if err != nil {
+			continue
+		}
+		if selFIV == nil {
+			selected = entry
+			selFIV = fiv
+			continue
+		}
+
+		if selected.matches(entry, r.bucket) {
+			continue
+		}
+
+		// Select latest modtime.
+		if fiv.ModTime.After(selFIV.ModTime) {
+			selected = entry
+			selFIV = fiv
+			continue
+		}
+	}
+	// If directory, we need quorum.
+	if dirExists > 0 && dirExists < r.dirQuorum {
+		return nil, false
+	}
+	if objExists < r.objQuorum {
+		return nil, false
+	}
+	// Take the latest selected.
+	return selected, selected != nil
+}
+
+// names will return all names in order.
+// Since this allocates it should not be used in critical functions.
+func (m metaCacheEntries) names() []string {
+	res := make([]string, 0, len(m))
+	for _, obj := range m {
+		res = append(res, obj.name)
+	}
+	return res
+}
+
+// metaCacheEntriesSorted contains metacache entries that are sorted.
+type metaCacheEntriesSorted struct {
+	o metaCacheEntries
+	// list id is not serialized
+	listID string
+}
+
+// writeTo will write all objects to the provided output.
+func (m metaCacheEntriesSorted) writeTo(writer io.Writer) error {
+	w := newMetacacheWriter(writer, 1<<20)
+	if err := w.write(m.o...); err != nil {
+		w.Close()
+		return err
+	}
+	return w.Close()
+}
+
+// shallowClone will create a shallow clone of the array objects,
+// but object metadata will not be cloned.
+func (m metaCacheEntriesSorted) shallowClone() metaCacheEntriesSorted {
+	// We have value receiver so we already have a copy.
+	m.o = m.o.shallowClone()
+	return m
+}
+
+// iterate the entries in order.
+// If the iterator function returns iterating stops.
+func (m *metaCacheEntriesSorted) iterate(fn func(entry metaCacheEntry) (cont bool)) {
+	if m == nil {
+		return
+	}
+	for _, o := range m.o {
+		if !fn(o) {
+			return
+		}
+	}
+}
+
+// fileInfoVersions converts the metadata to FileInfoVersions where possible.
+// Metadata that cannot be decoded is skipped.
+func (m *metaCacheEntriesSorted) fileInfoVersions(bucket, prefix, delimiter, afterV string) (versions []ObjectInfo, commonPrefixes []string) {
+	versions = make([]ObjectInfo, 0, m.len())
+	prevPrefix := ""
+	for _, entry := range m.o {
+		if entry.isObject() {
+			if delimiter != "" {
+				idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter)
+				if idx >= 0 {
+					idx = len(prefix) + idx + len(delimiter)
+					currPrefix := entry.name[:idx]
+					if currPrefix == prevPrefix {
+						continue
+					}
+					prevPrefix = currPrefix
+					commonPrefixes = append(commonPrefixes, currPrefix)
+					continue
+				}
+			}
+
+			fiv, err := entry.fileInfoVersions(bucket)
+			if afterV != "" {
+				// Forward first entry to specified version
+				fiv.forwardPastVersion(afterV)
+				afterV = ""
+			}
+			if err == nil {
+				for _, version := range fiv.Versions {
+					versions = append(versions, version.ToObjectInfo(bucket, entry.name))
+				}
+			}
+			continue
+		}
+
+		if entry.isDir() {
+			if delimiter == "" {
+				continue
+			}
+			idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter)
+			if idx < 0 {
+				continue
+			}
+			idx = len(prefix) + idx + len(delimiter)
+			currPrefix := entry.name[:idx]
+			if currPrefix == prevPrefix {
+				continue
+			}
+			prevPrefix = currPrefix
+			commonPrefixes = append(commonPrefixes, currPrefix)
+			continue
+		}
+	}
+
+	return versions, commonPrefixes
+}
+
+// fileInfoVersions converts the metadata to FileInfoVersions where possible.
+// Metadata that cannot be decoded is skipped.
+func (m *metaCacheEntriesSorted) fileInfos(bucket, prefix, delimiter string) (objects []ObjectInfo, commonPrefixes []string) {
+	objects = make([]ObjectInfo, 0, m.len())
+	prevPrefix := ""
+	for _, entry := range m.o {
+		if entry.isObject() {
+			if delimiter != "" {
+				idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter)
+				if idx >= 0 {
+					idx = len(prefix) + idx + len(delimiter)
+					currPrefix := entry.name[:idx]
+					if currPrefix == prevPrefix {
+						continue
+					}
+					prevPrefix = currPrefix
+					commonPrefixes = append(commonPrefixes, currPrefix)
+					continue
+				}
+			}
+
+			fi, err := entry.fileInfo(bucket)
+			if err == nil {
+				objects = append(objects, fi.ToObjectInfo(bucket, entry.name))
+			}
+			continue
+		}
+		if entry.isDir() {
+			if delimiter == "" {
+				continue
+			}
+			idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter)
+			if idx < 0 {
+				continue
+			}
+			idx = len(prefix) + idx + len(delimiter)
+			currPrefix := entry.name[:idx]
+			if currPrefix == prevPrefix {
+				continue
+			}
+			prevPrefix = currPrefix
+			commonPrefixes = append(commonPrefixes, currPrefix)
+			continue
+		}
+	}
+
+	return objects, commonPrefixes
+}
+
+// forwardTo will truncate m so only entries that are s or after is in the list.
+func (m *metaCacheEntriesSorted) forwardTo(s string) {
+	if s == "" {
+		return
+	}
+	idx := sort.Search(len(m.o), func(i int) bool {
+		return m.o[i].name >= s
+	})
+	m.o = m.o[idx:]
+}
+
+// merge will merge other into m.
+// If the same entries exists in both and metadata matches only one is added,
+// otherwise the entry from m will be placed first.
+// Operation time is expected to be O(n+m).
+func (m *metaCacheEntriesSorted) merge(other metaCacheEntriesSorted, limit int) {
+	merged := make(metaCacheEntries, 0, m.len()+other.len())
+	a := m.entries()
+	b := other.entries()
+	for len(a) > 0 && len(b) > 0 {
+		if a[0].name == b[0].name && bytes.Equal(a[0].metadata, b[0].metadata) {
+			// Same, discard one.
+			merged = append(merged, a[0])
+			a = a[1:]
+			b = b[1:]
+		} else if a[0].name < b[0].name {
+			merged = append(merged, a[0])
+			a = a[1:]
+		} else {
+			merged = append(merged, b[0])
+			b = b[1:]
+		}
+		if limit > 0 && len(merged) >= limit {
+			break
+		}
+	}
+	// Append anything left.
+	if limit < 0 || len(merged) < limit {
+		merged = append(merged, a...)
+		merged = append(merged, b...)
+	}
+	m.o = merged
+}
+
+// filter allows selective filtering with the provided function.
+func (m *metaCacheEntriesSorted) filter(fn func(entry *metaCacheEntry) bool) {
+	dst := m.o[:0]
+	for _, o := range m.o {
+		if fn(&o) {
+			dst = append(dst, o)
+		}
+	}
+	m.o = dst
+}
+
+// filterPrefix will filter m to only contain entries with the specified prefix.
+func (m *metaCacheEntriesSorted) filterPrefix(s string) {
+	if s == "" {
+		return
+	}
+	m.forwardTo(s)
+	for i, o := range m.o {
+		if !o.hasPrefix(s) {
+			m.o = m.o[:i]
+			break
+		}
+	}
+}
+
+// filterObjectsOnly will remove prefix directories.
+// Order is preserved, but the underlying slice is modified.
+func (m *metaCacheEntriesSorted) filterObjectsOnly() {
+	dst := m.o[:0]
+	for _, o := range m.o {
+		if !o.isDir() {
+			dst = append(dst, o)
+		}
+	}
+	m.o = dst
+}
+
+// filterPrefixesOnly will remove objects.
+// Order is preserved, but the underlying slice is modified.
+func (m *metaCacheEntriesSorted) filterPrefixesOnly() {
+	dst := m.o[:0]
+	for _, o := range m.o {
+		if o.isDir() {
+			dst = append(dst, o)
+		}
+	}
+	m.o = dst
+}
+
+// filterRecursiveEntries will keep entries only with the prefix that doesn't contain separator.
+// This can be used to remove recursive listings.
+// To return root elements only set prefix to an empty string.
+// Order is preserved, but the underlying slice is modified.
+func (m *metaCacheEntriesSorted) filterRecursiveEntries(prefix, separator string) {
+	dst := m.o[:0]
+	if prefix != "" {
+		m.forwardTo(prefix)
+		for _, o := range m.o {
+			ext := strings.TrimPrefix(o.name, prefix)
+			if len(ext) != len(o.name) {
+				if !strings.Contains(ext, separator) {
+					dst = append(dst, o)
+				}
+			}
+		}
+	} else {
+		// No prefix, simpler
+		for _, o := range m.o {
+			if !strings.Contains(o.name, separator) {
+				dst = append(dst, o)
+			}
+		}
+	}
+	m.o = dst
+}
+
+// truncate the number of entries to maximum n.
+func (m *metaCacheEntriesSorted) truncate(n int) {
+	if m == nil {
+		return
+	}
+	if len(m.o) > n {
+		m.o = m.o[:n]
+	}
+}
+
+// len returns the number of objects and prefix dirs in m.
+func (m *metaCacheEntriesSorted) len() int {
+	if m == nil {
+		return 0
+	}
+	return len(m.o)
+}
+
+// entries returns the underlying objects as is currently represented.
+func (m *metaCacheEntriesSorted) entries() metaCacheEntries {
+	if m == nil {
+		return nil
+	}
+	return m.o
+}
+
+// deduplicate entries in the list.
+// If compareMeta is set it will be used to resolve conflicts.
+// The function should return whether the existing entry should be replaced with other.
+// If no compareMeta is provided duplicates may be left.
+// This is indicated by the returned boolean.
+func (m *metaCacheEntriesSorted) deduplicate(compareMeta func(existing, other *metaCacheEntry) (replace bool)) (dupesLeft bool) {
+	dst := m.o[:0]
+	for j := range m.o {
+		found := false
+		obj := &m.o[j]
+		for i := len(dst) - 1; i >= 0; i++ {
+			existing := &dst[i]
+			if existing.name != obj.name {
+				break
+			}
+
+			// Use given resolution function first if any.
+			if compareMeta != nil {
+				if compareMeta(existing, obj) {
+					dst[i] = *obj
+				}
+				found = true
+				break
+			}
+			if obj.likelyMatches(existing) {
+				found = true
+				break
+			}
+
+			// Matches, move on.
+			dupesLeft = true
+			continue
+		}
+		if !found {
+			dst = append(dst, *obj)
+		}
+	}
+	m.o = dst
+	return dupesLeft
+}

cmd/metacache-entries_test.go

@@ -0,0 +1,265 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"reflect"
+	"sort"
+	"testing"
+)
+
+func Test_metaCacheEntries_sort(t *testing.T) {
+	entries := loadMetacacheSampleEntries(t)
+
+	o := entries.entries()
+	if !o.isSorted() {
+		t.Fatal("Expected sorted objects")
+	}
+
+	// Swap first and last
+	o[0], o[len(o)-1] = o[len(o)-1], o[0]
+	if o.isSorted() {
+		t.Fatal("Expected unsorted objects")
+	}
+
+	sorted := o.sort()
+	if !o.isSorted() {
+		t.Fatal("Expected sorted o objects")
+	}
+	if !sorted.entries().isSorted() {
+		t.Fatal("Expected sorted wrapped objects")
+	}
+	want := loadMetacacheSampleNames
+	for i, got := range o {
+		if got.name != want[i] {
+			t.Errorf("entry %d, want %q, got %q", i, want[i], got.name)
+		}
+	}
+}
+
+func Test_metaCacheEntries_forwardTo(t *testing.T) {
+	org := loadMetacacheSampleEntries(t)
+	entries := org
+	want := []string{"src/compress/zlib/reader_test.go", "src/compress/zlib/writer.go", "src/compress/zlib/writer_test.go"}
+	entries.forwardTo("src/compress/zlib/reader_test.go")
+	got := entries.entries().names()
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got unexpected result: %#v", got)
+	}
+
+	// Try with prefix
+	entries = org
+	entries.forwardTo("src/compress/zlib/reader_t")
+	got = entries.entries().names()
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got unexpected result: %#v", got)
+	}
+}
+
+func Test_metaCacheEntries_merge(t *testing.T) {
+	org := loadMetacacheSampleEntries(t)
+	a, b := org.shallowClone(), org.shallowClone()
+	be := b.entries()
+	for i := range be {
+		//  Modify b so it isn't deduplicated.
+		be[i].metadata = []byte("something-else")
+	}
+	// Merge b into a
+	a.merge(b, -1)
+	want := append(loadMetacacheSampleNames, loadMetacacheSampleNames...)
+	sort.Strings(want)
+	got := a.entries().names()
+	if len(got) != len(want) {
+		t.Errorf("unexpected count, want %v, got %v", len(want), len(got))
+	}
+
+	for i, name := range got {
+		if want[i] != name {
+			t.Errorf("unexpected name, want %q, got %q", want[i], name)
+		}
+	}
+}
+
+func Test_metaCacheEntries_dedupe(t *testing.T) {
+	org := loadMetacacheSampleEntries(t)
+	a, b := org.shallowClone(), org.shallowClone()
+
+	// Merge b into a
+	a.merge(b, -1)
+	if a.deduplicate(nil) {
+		t.Fatal("deduplicate returned duplicate entries left")
+	}
+	want := loadMetacacheSampleNames
+	got := a.entries().names()
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("got unexpected result: %#v", got)
+	}
+}
+
+func Test_metaCacheEntries_dedupe2(t *testing.T) {
+	org := loadMetacacheSampleEntries(t)
+	a, b := org.shallowClone(), org.shallowClone()
+
+	// Replace metadata in b
+	testMarker := []byte("sampleset")
+	for i := range b.o {
+		b.o[i].metadata = testMarker
+	}
+
+	// Merge b into a
+	a.merge(b, -1)
+	if a.deduplicate(func(existing, other *metaCacheEntry) (replace bool) {
+		a := bytes.Equal(existing.metadata, testMarker)
+		b := bytes.Equal(other.metadata, testMarker)
+		if a == b {
+			t.Fatal("got same number of testmarkers, only one should be given", a, b)
+		}
+		return b
+	}) {
+		t.Fatal("deduplicate returned duplicate entries left, we should always resolve")
+	}
+	want := loadMetacacheSampleNames
+	got := a.entries().names()
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("got unexpected result: %#v", got)
+	}
+}
+
+func Test_metaCacheEntries_filterObjects(t *testing.T) {
+	data := loadMetacacheSampleEntries(t)
+	data.filterObjectsOnly()
+	got := data.entries().names()
+	want := []string{"src/compress/bzip2/bit_reader.go", "src/compress/bzip2/bzip2.go", "src/compress/bzip2/bzip2_test.go", "src/compress/bzip2/huffman.go", "src/compress/bzip2/move_to_front.go", "src/compress/bzip2/testdata/Isaac.Newton-Opticks.txt.bz2", "src/compress/bzip2/testdata/e.txt.bz2", "src/compress/bzip2/testdata/fail-issue5747.bz2", "src/compress/bzip2/testdata/pass-random1.bin", "src/compress/bzip2/testdata/pass-random1.bz2", "src/compress/bzip2/testdata/pass-random2.bin", "src/compress/bzip2/testdata/pass-random2.bz2", "src/compress/bzip2/testdata/pass-sawtooth.bz2", "src/compress/bzip2/testdata/random.data.bz2", "src/compress/flate/deflate.go", "src/compress/flate/deflate_test.go", "src/compress/flate/deflatefast.go", "src/compress/flate/dict_decoder.go", "src/compress/flate/dict_decoder_test.go", "src/compress/flate/example_test.go", "src/compress/flate/flate_test.go", "src/compress/flate/huffman_bit_writer.go", "src/compress/flate/huffman_bit_writer_test.go", "src/compress/flate/huffman_code.go", "src/compress/flate/inflate.go", "src/compress/flate/inflate_test.go", "src/compress/flate/reader_test.go", "src/compress/flate/testdata/huffman-null-max.dyn.expect", "src/compress/flate/testdata/huffman-null-max.dyn.expect-noinput", "src/compress/flate/testdata/huffman-null-max.golden", "src/compress/flate/testdata/huffman-null-max.in", "src/compress/flate/testdata/huffman-null-max.wb.expect", "src/compress/flate/testdata/huffman-null-max.wb.expect-noinput", "src/compress/flate/testdata/huffman-pi.dyn.expect", "src/compress/flate/testdata/huffman-pi.dyn.expect-noinput", "src/compress/flate/testdata/huffman-pi.golden", "src/compress/flate/testdata/huffman-pi.in", "src/compress/flate/testdata/huffman-pi.wb.expect", "src/compress/flate/testdata/huffman-pi.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-1k.dyn.expect", "src/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput", "src/compress/flate/testdata/huffman-rand-1k.golden", "src/compress/flate/testdata/huffman-rand-1k.in", "src/compress/flate/testdata/huffman-rand-1k.wb.expect", "src/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-limit.dyn.expect", "src/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput", "src/compress/flate/testdata/huffman-rand-limit.golden", "src/compress/flate/testdata/huffman-rand-limit.in", "src/compress/flate/testdata/huffman-rand-limit.wb.expect", "src/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-max.golden", "src/compress/flate/testdata/huffman-rand-max.in", "src/compress/flate/testdata/huffman-shifts.dyn.expect", "src/compress/flate/testdata/huffman-shifts.dyn.expect-noinput", "src/compress/flate/testdata/huffman-shifts.golden", "src/compress/flate/testdata/huffman-shifts.in", "src/compress/flate/testdata/huffman-shifts.wb.expect", "src/compress/flate/testdata/huffman-shifts.wb.expect-noinput", "src/compress/flate/testdata/huffman-text-shift.dyn.expect", "src/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput", "src/compress/flate/testdata/huffman-text-shift.golden", "src/compress/flate/testdata/huffman-text-shift.in", "src/compress/flate/testdata/huffman-text-shift.wb.expect", "src/compress/flate/testdata/huffman-text-shift.wb.expect-noinput", "src/compress/flate/testdata/huffman-text.dyn.expect", "src/compress/flate/testdata/huffman-text.dyn.expect-noinput", "src/compress/flate/testdata/huffman-text.golden", "src/compress/flate/testdata/huffman-text.in", "src/compress/flate/testdata/huffman-text.wb.expect", "src/compress/flate/testdata/huffman-text.wb.expect-noinput", "src/compress/flate/testdata/huffman-zero.dyn.expect", "src/compress/flate/testdata/huffman-zero.dyn.expect-noinput", "src/compress/flate/testdata/huffman-zero.golden", "src/compress/flate/testdata/huffman-zero.in", "src/compress/flate/testdata/huffman-zero.wb.expect", "src/compress/flate/testdata/huffman-zero.wb.expect-noinput", "src/compress/flate/testdata/null-long-match.dyn.expect-noinput", "src/compress/flate/testdata/null-long-match.wb.expect-noinput", "src/compress/flate/token.go", "src/compress/flate/writer_test.go", "src/compress/gzip/example_test.go", "src/compress/gzip/gunzip.go", "src/compress/gzip/gunzip_test.go", "src/compress/gzip/gzip.go", "src/compress/gzip/gzip_test.go", "src/compress/gzip/issue14937_test.go", "src/compress/gzip/testdata/issue6550.gz.base64", "src/compress/lzw/reader.go", "src/compress/lzw/reader_test.go", "src/compress/lzw/writer.go", "src/compress/lzw/writer_test.go", "src/compress/testdata/e.txt", "src/compress/testdata/gettysburg.txt", "src/compress/testdata/pi.txt", "src/compress/zlib/example_test.go", "src/compress/zlib/reader.go", "src/compress/zlib/reader_test.go", "src/compress/zlib/writer.go", "src/compress/zlib/writer_test.go"}
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("got unexpected result: %#v", got)
+	}
+}
+
+func Test_metaCacheEntries_filterPrefixes(t *testing.T) {
+	data := loadMetacacheSampleEntries(t)
+	data.filterPrefixesOnly()
+	got := data.entries().names()
+	want := []string{"src/compress/bzip2/", "src/compress/bzip2/testdata/", "src/compress/flate/", "src/compress/flate/testdata/", "src/compress/gzip/", "src/compress/gzip/testdata/", "src/compress/lzw/", "src/compress/testdata/", "src/compress/zlib/"}
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("got unexpected result: %#v", got)
+	}
+}
+
+func Test_metaCacheEntries_filterRecursive(t *testing.T) {
+	data := loadMetacacheSampleEntries(t)
+	data.filterRecursiveEntries("src/compress/bzip2/", slashSeparator)
+	got := data.entries().names()
+	want := []string{"src/compress/bzip2/", "src/compress/bzip2/bit_reader.go", "src/compress/bzip2/bzip2.go", "src/compress/bzip2/bzip2_test.go", "src/compress/bzip2/huffman.go", "src/compress/bzip2/move_to_front.go"}
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("got unexpected result: %#v", got)
+	}
+}
+
+func Test_metaCacheEntries_filterRecursiveRoot(t *testing.T) {
+	data := loadMetacacheSampleEntries(t)
+	data.filterRecursiveEntries("", slashSeparator)
+	got := data.entries().names()
+	want := []string{}
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("got unexpected result: %#v", got)
+	}
+}
+
+func Test_metaCacheEntries_filterRecursiveRootSep(t *testing.T) {
+	data := loadMetacacheSampleEntries(t)
+	// This will remove anything with "bzip2/" in the path since it is separator
+	data.filterRecursiveEntries("", "bzip2/")
+	got := data.entries().names()
+	want := []string{"src/compress/flate/", "src/compress/flate/deflate.go", "src/compress/flate/deflate_test.go", "src/compress/flate/deflatefast.go", "src/compress/flate/dict_decoder.go", "src/compress/flate/dict_decoder_test.go", "src/compress/flate/example_test.go", "src/compress/flate/flate_test.go", "src/compress/flate/huffman_bit_writer.go", "src/compress/flate/huffman_bit_writer_test.go", "src/compress/flate/huffman_code.go", "src/compress/flate/inflate.go", "src/compress/flate/inflate_test.go", "src/compress/flate/reader_test.go", "src/compress/flate/testdata/", "src/compress/flate/testdata/huffman-null-max.dyn.expect", "src/compress/flate/testdata/huffman-null-max.dyn.expect-noinput", "src/compress/flate/testdata/huffman-null-max.golden", "src/compress/flate/testdata/huffman-null-max.in", "src/compress/flate/testdata/huffman-null-max.wb.expect", "src/compress/flate/testdata/huffman-null-max.wb.expect-noinput", "src/compress/flate/testdata/huffman-pi.dyn.expect", "src/compress/flate/testdata/huffman-pi.dyn.expect-noinput", "src/compress/flate/testdata/huffman-pi.golden", "src/compress/flate/testdata/huffman-pi.in", "src/compress/flate/testdata/huffman-pi.wb.expect", "src/compress/flate/testdata/huffman-pi.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-1k.dyn.expect", "src/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput", "src/compress/flate/testdata/huffman-rand-1k.golden", "src/compress/flate/testdata/huffman-rand-1k.in", "src/compress/flate/testdata/huffman-rand-1k.wb.expect", "src/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-limit.dyn.expect", "src/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput", "src/compress/flate/testdata/huffman-rand-limit.golden", "src/compress/flate/testdata/huffman-rand-limit.in", "src/compress/flate/testdata/huffman-rand-limit.wb.expect", "src/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-max.golden", "src/compress/flate/testdata/huffman-rand-max.in", "src/compress/flate/testdata/huffman-shifts.dyn.expect", "src/compress/flate/testdata/huffman-shifts.dyn.expect-noinput", "src/compress/flate/testdata/huffman-shifts.golden", "src/compress/flate/testdata/huffman-shifts.in", "src/compress/flate/testdata/huffman-shifts.wb.expect", "src/compress/flate/testdata/huffman-shifts.wb.expect-noinput", "src/compress/flate/testdata/huffman-text-shift.dyn.expect", "src/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput", "src/compress/flate/testdata/huffman-text-shift.golden", "src/compress/flate/testdata/huffman-text-shift.in", "src/compress/flate/testdata/huffman-text-shift.wb.expect", "src/compress/flate/testdata/huffman-text-shift.wb.expect-noinput", "src/compress/flate/testdata/huffman-text.dyn.expect", "src/compress/flate/testdata/huffman-text.dyn.expect-noinput", "src/compress/flate/testdata/huffman-text.golden", "src/compress/flate/testdata/huffman-text.in", "src/compress/flate/testdata/huffman-text.wb.expect", "src/compress/flate/testdata/huffman-text.wb.expect-noinput", "src/compress/flate/testdata/huffman-zero.dyn.expect", "src/compress/flate/testdata/huffman-zero.dyn.expect-noinput", "src/compress/flate/testdata/huffman-zero.golden", "src/compress/flate/testdata/huffman-zero.in", "src/compress/flate/testdata/huffman-zero.wb.expect", "src/compress/flate/testdata/huffman-zero.wb.expect-noinput", "src/compress/flate/testdata/null-long-match.dyn.expect-noinput", "src/compress/flate/testdata/null-long-match.wb.expect-noinput", "src/compress/flate/token.go", "src/compress/flate/writer_test.go", "src/compress/gzip/", "src/compress/gzip/example_test.go", "src/compress/gzip/gunzip.go", "src/compress/gzip/gunzip_test.go", "src/compress/gzip/gzip.go", "src/compress/gzip/gzip_test.go", "src/compress/gzip/issue14937_test.go", "src/compress/gzip/testdata/", "src/compress/gzip/testdata/issue6550.gz.base64", "src/compress/lzw/", "src/compress/lzw/reader.go", "src/compress/lzw/reader_test.go", "src/compress/lzw/writer.go", "src/compress/lzw/writer_test.go", "src/compress/testdata/", "src/compress/testdata/e.txt", "src/compress/testdata/gettysburg.txt", "src/compress/testdata/pi.txt", "src/compress/zlib/", "src/compress/zlib/example_test.go", "src/compress/zlib/reader.go", "src/compress/zlib/reader_test.go", "src/compress/zlib/writer.go", "src/compress/zlib/writer_test.go"}
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("got unexpected result: %#v", got)
+	}
+}
+
+func Test_metaCacheEntries_filterPrefix(t *testing.T) {
+	data := loadMetacacheSampleEntries(t)
+	data.filterPrefix("src/compress/bzip2/")
+	got := data.entries().names()
+	want := []string{"src/compress/bzip2/", "src/compress/bzip2/bit_reader.go", "src/compress/bzip2/bzip2.go", "src/compress/bzip2/bzip2_test.go", "src/compress/bzip2/huffman.go", "src/compress/bzip2/move_to_front.go", "src/compress/bzip2/testdata/", "src/compress/bzip2/testdata/Isaac.Newton-Opticks.txt.bz2", "src/compress/bzip2/testdata/e.txt.bz2", "src/compress/bzip2/testdata/fail-issue5747.bz2", "src/compress/bzip2/testdata/pass-random1.bin", "src/compress/bzip2/testdata/pass-random1.bz2", "src/compress/bzip2/testdata/pass-random2.bin", "src/compress/bzip2/testdata/pass-random2.bz2", "src/compress/bzip2/testdata/pass-sawtooth.bz2", "src/compress/bzip2/testdata/random.data.bz2"}
+	if !reflect.DeepEqual(want, got) {
+		t.Errorf("got unexpected result: %#v", got)
+	}
+}
+
+func Test_metaCacheEntry_isInDir(t *testing.T) {
+	tests := []struct {
+		testName string
+		entry    string
+		dir      string
+		sep      string
+		want     bool
+	}{
+		{
+			testName: "basic-file",
+			entry:    "src/file",
+			dir:      "src/",
+			sep:      slashSeparator,
+			want:     true,
+		},
+		{
+			testName: "basic-dir",
+			entry:    "src/dir/",
+			dir:      "src/",
+			sep:      slashSeparator,
+			want:     true,
+		},
+		{
+			testName: "deeper-file",
+			entry:    "src/dir/somewhere.ext",
+			dir:      "src/",
+			sep:      slashSeparator,
+			want:     false,
+		},
+		{
+			testName: "deeper-dir",
+			entry:    "src/dir/somewhere/",
+			dir:      "src/",
+			sep:      slashSeparator,
+			want:     false,
+		},
+		{
+			testName: "root-dir",
+			entry:    "doc/",
+			dir:      "",
+			sep:      slashSeparator,
+			want:     true,
+		},
+		{
+			testName: "root-file",
+			entry:    "word.doc",
+			dir:      "",
+			sep:      slashSeparator,
+			want:     true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.testName, func(t *testing.T) {
+			e := metaCacheEntry{
+				name: tt.entry,
+			}
+			if got := e.isInDir(tt.dir, tt.sep); got != tt.want {
+				t.Errorf("isInDir() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cmd/metacache-manager.go

@@ -0,0 +1,268 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"runtime/debug"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// localMetacacheMgr is the *local* manager for this peer.
+// It should never be used directly since buckets are
+// distributed deterministically.
+// Therefore no cluster locks are required.
+var localMetacacheMgr = &metacacheManager{
+	buckets: make(map[string]*bucketMetacache),
+	trash:   make(map[string]metacache),
+}
+
+type metacacheManager struct {
+	mu      sync.RWMutex
+	init    sync.Once
+	buckets map[string]*bucketMetacache
+	trash   map[string]metacache // Recently deleted lists.
+}
+
+const metacacheManagerTransientBucket = "**transient**"
+
+// initManager will start async saving the cache.
+func (m *metacacheManager) initManager() {
+	// Add a transient bucket.
+	tb := newBucketMetacache(metacacheManagerTransientBucket, false)
+	tb.transient = true
+	m.buckets[metacacheManagerTransientBucket] = tb
+
+	// Start saver when object layer is ready.
+	go func() {
+		objAPI := newObjectLayerFn()
+		for objAPI == nil {
+			time.Sleep(time.Second)
+			objAPI = newObjectLayerFn()
+		}
+		if !globalIsErasure {
+			logger.Info("metacacheManager was initialized in non-erasure mode, skipping save")
+			return
+		}
+
+		t := time.NewTicker(time.Minute)
+		var exit bool
+		bg := context.Background()
+		for !exit {
+			select {
+			case <-t.C:
+			case <-GlobalContext.Done():
+				exit = true
+			}
+			m.mu.RLock()
+			for _, v := range m.buckets {
+				if !exit {
+					v.cleanup()
+				}
+				logger.LogIf(bg, v.save(bg))
+			}
+			m.mu.RUnlock()
+			m.mu.Lock()
+			for k, v := range m.trash {
+				if time.Since(v.lastUpdate) > metacacheMaxRunningAge {
+					v.delete(context.Background())
+					delete(m.trash, k)
+				}
+			}
+			m.mu.Unlock()
+		}
+		m.getTransient().deleteAll()
+	}()
+}
+
+// findCache will get a metacache.
+func (m *metacacheManager) findCache(ctx context.Context, o listPathOptions) metacache {
+	if o.Transient || isReservedOrInvalidBucket(o.Bucket, false) {
+		return m.getTransient().findCache(o)
+	}
+	m.mu.RLock()
+	b, ok := m.buckets[o.Bucket]
+	if ok {
+		m.mu.RUnlock()
+		return b.findCache(o)
+	}
+	if meta, ok := m.trash[o.ID]; ok {
+		m.mu.RUnlock()
+		return meta
+	}
+	m.mu.RUnlock()
+	return m.getBucket(ctx, o.Bucket).findCache(o)
+}
+
+// updateCacheEntry will update non-transient state.
+func (m *metacacheManager) updateCacheEntry(update metacache) (metacache, error) {
+	m.mu.RLock()
+	if meta, ok := m.trash[update.id]; ok {
+		m.mu.RUnlock()
+		return meta, nil
+	}
+
+	b, ok := m.buckets[update.bucket]
+	if ok {
+		m.mu.RUnlock()
+		return b.updateCacheEntry(update)
+	}
+	m.mu.RUnlock()
+	// We should have either a trashed bucket or this
+	return metacache{}, errVolumeNotFound
+}
+
+// getBucket will get a bucket metacache or load it from disk if needed.
+func (m *metacacheManager) getBucket(ctx context.Context, bucket string) *bucketMetacache {
+	m.init.Do(m.initManager)
+
+	// Return a transient bucket for invalid or system buckets.
+	if isReservedOrInvalidBucket(bucket, false) {
+		return m.getTransient()
+	}
+	m.mu.RLock()
+	b, ok := m.buckets[bucket]
+	m.mu.RUnlock()
+	if ok {
+		if b.bucket != bucket {
+			logger.Info("getBucket: cached bucket %s does not match this bucket %s", b.bucket, bucket)
+			debug.PrintStack()
+		}
+		return b
+	}
+
+	m.mu.Lock()
+	// See if someone else fetched it while we waited for the lock.
+	b, ok = m.buckets[bucket]
+	if ok {
+		m.mu.Unlock()
+		if b.bucket != bucket {
+			logger.Info("getBucket: newly cached bucket %s does not match this bucket %s", b.bucket, bucket)
+			debug.PrintStack()
+		}
+		return b
+	}
+
+	// Load bucket. If we fail return the transient bucket.
+	b, err := loadBucketMetaCache(ctx, bucket)
+	if err != nil {
+		m.mu.Unlock()
+		logger.LogIf(ctx, err)
+		return m.getTransient()
+	}
+	if b.bucket != bucket {
+		logger.LogIf(ctx, fmt.Errorf("getBucket: loaded bucket %s does not match this bucket %s", b.bucket, bucket))
+	}
+	m.buckets[bucket] = b
+	m.mu.Unlock()
+	return b
+}
+
+// deleteBucketCache will delete the bucket cache if it exists.
+func (m *metacacheManager) deleteBucketCache(bucket string) {
+	m.init.Do(m.initManager)
+	m.mu.Lock()
+	b, ok := m.buckets[bucket]
+	if !ok {
+		m.mu.Unlock()
+		return
+	}
+	delete(m.buckets, bucket)
+	m.mu.Unlock()
+
+	// Since deletes may take some time we try to do it without
+	// holding lock to m all the time.
+	b.mu.Lock()
+	defer b.mu.Unlock()
+	for k, v := range b.caches {
+		if time.Since(v.lastUpdate) > metacacheMaxRunningAge {
+			v.delete(context.Background())
+			continue
+		}
+		v.error = "Bucket deleted"
+		v.status = scanStateError
+		m.mu.Lock()
+		m.trash[k] = v
+		m.mu.Unlock()
+	}
+}
+
+// deleteAll will delete all caches.
+func (m *metacacheManager) deleteAll() {
+	m.init.Do(m.initManager)
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	for bucket, b := range m.buckets {
+		b.deleteAll()
+		if !b.transient {
+			delete(m.buckets, bucket)
+		}
+	}
+}
+
+// getTransient will return a transient bucket.
+func (m *metacacheManager) getTransient() *bucketMetacache {
+	m.init.Do(m.initManager)
+	m.mu.RLock()
+	bmc := m.buckets[metacacheManagerTransientBucket]
+	m.mu.RUnlock()
+	return bmc
+}
+
+// checkMetacacheState should be used if data is not updating.
+// Should only be called if a failure occurred.
+func (o listPathOptions) checkMetacacheState(ctx context.Context, rpc *peerRESTClient) error {
+	// We operate on a copy...
+	o.Create = false
+	var cache metacache
+	if rpc == nil || o.Transient {
+		cache = localMetacacheMgr.findCache(ctx, o)
+	} else {
+		c, err := rpc.GetMetacacheListing(ctx, o)
+		if err != nil {
+			return err
+		}
+		cache = *c
+	}
+
+	if cache.status == scanStateNone || cache.fileNotFound {
+		return errFileNotFound
+	}
+	if cache.status == scanStateSuccess || cache.status == scanStateStarted {
+		if time.Since(cache.lastUpdate) > metacacheMaxRunningAge {
+			// We got a stale entry, mark error on handling server.
+			err := fmt.Errorf("timeout: list %s not updated", cache.id)
+			cache.error = err.Error()
+			cache.status = scanStateError
+			if rpc == nil || o.Transient {
+				localMetacacheMgr.updateCacheEntry(cache)
+			} else {
+				rpc.UpdateMetacacheListing(ctx, cache)
+			}
+			return err
+		}
+		return nil
+	}
+	if cache.error != "" {
+		return fmt.Errorf("async cache listing failed with: %s", cache.error)
+	}
+	return nil
+}

cmd/metacache-marker.go

@@ -0,0 +1,70 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// markerTagVersion is the marker version.
+// Should not need to be updated unless a fundamental change is made to the marker format.
+const markerTagVersion = "v1"
+
+// parseMarker will parse a marker possibly encoded with encodeMarker
+func parseMarker(s string) (marker, uuid string) {
+	if !strings.Contains(s, "[minio_cache:"+markerTagVersion) {
+		return s, ""
+	}
+	start := strings.LastIndex(s, "[")
+	marker = s[:start]
+	end := strings.LastIndex(s, "]")
+	tag := strings.Trim(s[start:end], "[]")
+	tags := strings.Split(tag, ",")
+	for _, tag := range tags {
+		kv := strings.Split(tag, ":")
+		if len(kv) < 2 {
+			continue
+		}
+		switch kv[0] {
+		case "minio_cache":
+			if kv[1] != markerTagVersion {
+				break
+			}
+		case "id":
+			uuid = kv[1]
+		default:
+			// Ignore unknown
+		}
+	}
+	return
+}
+
+// encodeMarker will encode a uuid and return it as a marker.
+// uuid cannot contain '[', ':' or ','.
+func encodeMarker(marker, uuid string) string {
+	if uuid == "" {
+		return marker
+	}
+	if strings.ContainsAny(uuid, "[:,") {
+		logger.LogIf(context.Background(), fmt.Errorf("encodeMarker: uuid %s contained invalid characters", uuid))
+	}
+	return fmt.Sprintf("%s[minio_cache:%s,id:%s]", marker, markerTagVersion, uuid)
+}

cmd/metacache-server-sets.go

@@ -0,0 +1,202 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"io"
+	"path"
+	"sync"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+// listPath will return the requested entries.
+// If no more entries are in the listing io.EOF is returned,
+// otherwise nil or an unexpected error is returned.
+// The listPathOptions given will be checked and modified internally.
+// Required important fields are Bucket, Prefix, Separator.
+// Other important fields are Limit, Marker.
+// List ID always derived from the Marker.
+func (z *erasureServerSets) listPath(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) {
+	if err := checkListObjsArgs(ctx, o.Bucket, o.Prefix, o.Marker, z); err != nil {
+		return entries, err
+	}
+
+	// Marker is set validate pre-condition.
+	if o.Marker != "" && o.Prefix != "" {
+		// Marker not common with prefix is not implemented. Send an empty response
+		if !HasPrefix(o.Marker, o.Prefix) {
+			return entries, io.EOF
+		}
+	}
+
+	// With max keys of zero we have reached eof, return right here.
+	if o.Limit == 0 {
+		return entries, io.EOF
+	}
+
+	// For delimiter and prefix as '/' we do not list anything at all
+	// since according to s3 spec we stop at the 'delimiter'
+	// along // with the prefix. On a flat namespace with 'prefix'
+	// as '/' we don't have any entries, since all the keys are
+	// of form 'keyName/...'
+	if o.Separator == SlashSeparator && o.Prefix == SlashSeparator {
+		return entries, io.EOF
+	}
+
+	// Over flowing count - reset to maxObjectList.
+	if o.Limit < 0 || o.Limit > maxObjectList {
+		o.Limit = maxObjectList
+	}
+
+	// If delimiter is slashSeparator we must return directories of
+	// the non-recursive scan unless explicitly requested.
+	o.IncludeDirectories = o.Separator == slashSeparator
+	if (o.Separator == slashSeparator || o.Separator == "") && !o.Recursive {
+		o.Recursive = o.Separator != slashSeparator
+		o.Separator = slashSeparator
+	} else {
+		// Default is recursive, if delimiter is set then list non recursive.
+		o.Recursive = true
+	}
+
+	// Decode and get the optional list id from the marker.
+	o.Marker, o.ID = parseMarker(o.Marker)
+	o.Create = o.ID == ""
+	if o.ID == "" {
+		o.ID = mustGetUUID()
+	}
+	o.BaseDir = baseDirFromPrefix(o.Prefix)
+
+	var cache metacache
+	// If we don't have a list id we must ask the server if it has a cache or create a new.
+	if o.Create {
+		o.CurrentCycle = intDataUpdateTracker.current()
+		o.OldestCycle = globalNotificationSys.findEarliestCleanBloomFilter(ctx, path.Join(o.Bucket, o.BaseDir))
+		var cache metacache
+		rpc := globalNotificationSys.restClientFromHash(o.Bucket)
+		if isReservedOrInvalidBucket(o.Bucket, false) {
+			rpc = nil
+			o.Transient = true
+		}
+		if rpc == nil || o.Transient {
+			// Local
+			cache = localMetacacheMgr.findCache(ctx, o)
+		} else {
+			c, err := rpc.GetMetacacheListing(ctx, o)
+			if err != nil {
+				if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+					// Context is canceled, return at once.
+					// request canceled, no entries to return
+					return entries, io.EOF
+				}
+				logger.LogIf(ctx, err)
+				o.Transient = true
+				cache = localMetacacheMgr.findCache(ctx, o)
+			} else {
+				cache = *c
+			}
+		}
+		if cache.fileNotFound {
+			// No cache found, no entries found.
+			return entries, io.EOF
+		}
+		// Only create if we created a new.
+		o.Create = o.ID == cache.id
+		o.ID = cache.id
+	}
+
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+	var errs []error
+	allAtEOF := true
+	asked := 0
+	mu.Lock()
+	// Ask all sets and merge entries.
+	for _, zone := range z.serverSets {
+		for _, set := range zone.sets {
+			wg.Add(1)
+			asked++
+			go func(i int, set *erasureObjects) {
+				defer wg.Done()
+				e, err := set.listPath(ctx, o)
+				mu.Lock()
+				defer mu.Unlock()
+				if err == nil {
+					allAtEOF = false
+				}
+				errs[i] = err
+				entries.merge(e, -1)
+
+				// Resolve non-trivial conflicts
+				entries.deduplicate(func(existing, other *metaCacheEntry) (replace bool) {
+					if existing.isDir() {
+						return false
+					}
+					eFIV, err := existing.fileInfo(o.Bucket)
+					if err != nil {
+						return true
+					}
+					oFIV, err := existing.fileInfo(o.Bucket)
+					if err != nil {
+						return false
+					}
+					return oFIV.ModTime.After(eFIV.ModTime)
+				})
+				if entries.len() > o.Limit {
+					allAtEOF = false
+					entries.truncate(o.Limit)
+				}
+			}(len(errs), set)
+			errs = append(errs, nil)
+		}
+	}
+	mu.Unlock()
+	wg.Wait()
+
+	if isAllNotFound(errs) {
+		// All sets returned not found.
+		// Update master cache with that information.
+		cache.status = scanStateSuccess
+		cache.fileNotFound = true
+		_, err := o.updateMetacacheListing(cache, globalNotificationSys.restClientFromHash(o.Bucket))
+		logger.LogIf(ctx, err)
+		// cache returned not found, entries truncated.
+		return entries, io.EOF
+	}
+
+	for _, err := range errs {
+		if err == nil {
+			allAtEOF = false
+			continue
+		}
+		if err.Error() == io.EOF.Error() {
+			continue
+		}
+		logger.LogIf(ctx, err)
+		return entries, err
+	}
+	truncated := entries.len() > o.Limit || !allAtEOF
+	entries.truncate(o.Limit)
+	entries.listID = o.ID
+	if !truncated {
+		return entries, io.EOF
+	}
+	return entries, nil
+}

cmd/metacache-set.go

@@ -0,0 +1,814 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/gob"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/minio/minio/cmd/config/storageclass"
+	xhttp "github.com/minio/minio/cmd/http"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/console"
+	"github.com/minio/minio/pkg/hash"
+)
+
+type listPathOptions struct {
+	// ID of the listing.
+	// This will be used to persist the list.
+	ID string
+
+	// Bucket of the listing.
+	Bucket string
+
+	// Directory inside the bucket.
+	BaseDir string
+
+	// Scan/return only content with prefix.
+	Prefix string
+
+	// Marker to resume listing.
+	// The response will be the first entry AFTER this object name.
+	Marker string
+
+	// Limit the number of results.
+	Limit int
+
+	// The number of disks to ask. Special values:
+	// 0 uses default number of disks.
+	// -1 use at least 50% of disks or at least the default number.
+	AskDisks int
+
+	// InclDeleted will keep all entries where latest version is a delete marker.
+	InclDeleted bool
+
+	// Scan recursively.
+	// If false only main directory will be scanned.
+	// Should always be true if Separator is n SlashSeparator.
+	Recursive bool
+
+	// Separator to use.
+	Separator string
+
+	// Create indicates that the lister should not attempt to load an existing cache.
+	Create bool
+
+	// CurrentCycle indicates the current bloom cycle.
+	// Will be used if a new scan is started.
+	CurrentCycle uint64
+
+	// OldestCycle indicates the oldest cycle acceptable.
+	OldestCycle uint64
+
+	// Include pure directories.
+	IncludeDirectories bool
+
+	// Transient is set if the cache is transient due to an error or being a reserved bucket.
+	// This means the cache metadata will not be persisted on disk.
+	// A transient result will never be returned from the cache so knowing the list id is required.
+	Transient bool
+}
+
+func init() {
+	gob.Register(listPathOptions{})
+}
+
+// newMetacache constructs a new metacache from the options.
+func (o listPathOptions) newMetacache() metacache {
+	return metacache{
+		id:           o.ID,
+		bucket:       o.Bucket,
+		root:         o.BaseDir,
+		recursive:    o.Recursive,
+		status:       scanStateStarted,
+		error:        "",
+		started:      UTCNow(),
+		lastHandout:  UTCNow(),
+		lastUpdate:   UTCNow(),
+		ended:        time.Time{},
+		startedCycle: o.CurrentCycle,
+		endedCycle:   0,
+		dataVersion:  metacacheStreamVersion,
+	}
+}
+
+// gatherResults will collect all results on the input channel and filter results according to the options.
+// Caller should close the channel when done.
+// The returned function will return the results once there is enough or input is closed.
+func (o *listPathOptions) gatherResults(in <-chan metaCacheEntry) func() (metaCacheEntriesSorted, error) {
+	const debugPrint = false
+	var resultsDone = make(chan metaCacheEntriesSorted)
+	// Copy so we can mutate
+	resCh := resultsDone
+	resErr := io.EOF
+
+	go func() {
+		var results metaCacheEntriesSorted
+		for entry := range in {
+			if resCh == nil {
+				// past limit
+				continue
+			}
+			if !o.IncludeDirectories && entry.isDir() {
+				continue
+			}
+			if debugPrint {
+				console.Infoln("gather got:", entry.name)
+			}
+			if o.Marker != "" && entry.name <= o.Marker {
+				if debugPrint {
+					console.Infoln("pre marker")
+				}
+				continue
+			}
+			if !strings.HasPrefix(entry.name, o.Prefix) {
+				if debugPrint {
+					console.Infoln("not in prefix")
+				}
+				continue
+			}
+			if !o.Recursive && !entry.isInDir(o.Prefix, o.Separator) {
+				if debugPrint {
+					console.Infoln("not in dir", o.Prefix, o.Separator)
+				}
+				continue
+			}
+			if !o.InclDeleted && entry.isObject() {
+				if entry.isLatestDeletemarker() {
+					if debugPrint {
+						console.Infoln("latest delete")
+					}
+					continue
+				}
+			}
+			if o.Limit > 0 && results.len() >= o.Limit {
+				// We have enough and we have more.
+				// Do not return io.EOF
+				if resCh != nil {
+					resErr = nil
+					resCh <- results
+					resCh = nil
+				}
+				continue
+			}
+			if debugPrint {
+				console.Infoln("adding...")
+			}
+			results.o = append(results.o, entry)
+		}
+		if resCh != nil {
+			resErr = io.EOF
+			resCh <- results
+		}
+	}()
+	return func() (metaCacheEntriesSorted, error) {
+		return <-resultsDone, resErr
+	}
+}
+
+// findFirstPart will find the part with 0 being the first that corresponds to the marker in the options.
+// io.ErrUnexpectedEOF is returned if the place containing the marker hasn't been scanned yet.
+// io.EOF indicates the marker is beyond the end of the stream and does not exist.
+func (o *listPathOptions) findFirstPart(fi FileInfo) (int, error) {
+	search := o.Marker
+	if search == "" {
+		search = o.Prefix
+	}
+	if search == "" {
+		return 0, nil
+	}
+	const debugPrint = false
+	if debugPrint {
+		console.Infoln("searching for ", search)
+	}
+	var tmp metacacheBlock
+	i := 0
+	for {
+		partKey := fmt.Sprintf("%s-metacache-part-%d", ReservedMetadataPrefixLower, i)
+		v, ok := fi.Metadata[partKey]
+		if !ok {
+			if debugPrint {
+				console.Infoln("no match in metadata, waiting")
+			}
+			return -1, io.ErrUnexpectedEOF
+		}
+		err := json.Unmarshal([]byte(v), &tmp)
+		if !ok {
+			logger.LogIf(context.Background(), err)
+			return -1, err
+		}
+		if tmp.First == "" && tmp.Last == "" && tmp.EOS {
+			return 0, errFileNotFound
+		}
+		if tmp.First >= search {
+			if debugPrint {
+				console.Infoln("First >= search", v)
+			}
+			return i, nil
+		}
+		if tmp.Last >= search {
+			if debugPrint {
+
+				console.Infoln("Last >= search", v)
+			}
+			return i, nil
+		}
+		if tmp.EOS {
+			if debugPrint {
+				console.Infoln("no match, at EOS", v)
+			}
+			return -3, io.EOF
+		}
+		if debugPrint {
+			console.Infoln("First ", tmp.First, "<", search, " search", i)
+		}
+		i++
+	}
+}
+
+// updateMetacacheListing will update the metacache listing.
+func (o *listPathOptions) updateMetacacheListing(m metacache, rpc *peerRESTClient) (metacache, error) {
+	if o.Transient {
+		return localMetacacheMgr.getTransient().updateCacheEntry(m)
+	}
+	if rpc == nil {
+		return localMetacacheMgr.updateCacheEntry(m)
+	}
+	return rpc.UpdateMetacacheListing(context.Background(), m)
+}
+
+func getMetacacheBlockInfo(fi FileInfo, block int) (*metacacheBlock, error) {
+	var tmp metacacheBlock
+	partKey := fmt.Sprintf("%s-metacache-part-%d", ReservedMetadataPrefixLower, block)
+	v, ok := fi.Metadata[partKey]
+	if !ok {
+		return nil, io.ErrUnexpectedEOF
+	}
+	return &tmp, json.Unmarshal([]byte(v), &tmp)
+}
+
+func metacachePrefixForID(bucket, id string) string {
+	return pathJoin("buckets", bucket, ".metacache", id)
+}
+
+// objectPath returns the object path of the cache.
+func (o *listPathOptions) objectPath(block int) string {
+	return pathJoin(metacachePrefixForID(o.Bucket, o.ID), "block-"+strconv.Itoa(block)+".s2")
+}
+
+// filter will apply the options and return the number of objects requested by the limit.
+// Will return io.EOF if there are no more entries with the same filter.
+// The last entry can be used as a marker to resume the listing.
+func (r *metacacheReader) filter(o listPathOptions) (entries metaCacheEntriesSorted, err error) {
+	const debugPrint = false
+	// Forward to prefix, if any
+	err = r.forwardTo(o.Prefix)
+	if err != nil {
+		return entries, err
+	}
+	if o.Marker != "" {
+		err = r.forwardTo(o.Marker)
+		if err != nil {
+			return entries, err
+		}
+		next, err := r.peek()
+		if err != nil {
+			return entries, err
+		}
+		if next.name == o.Marker {
+			err := r.skip(1)
+			if err != nil {
+				return entries, err
+			}
+		}
+	}
+	if debugPrint {
+		console.Infoln("forwarded to ", o.Prefix, "marker:", o.Marker, "sep:", o.Separator)
+	}
+	// Filter
+	if !o.Recursive {
+		entries.o = make(metaCacheEntries, 0, o.Limit)
+		pastPrefix := false
+		err := r.readFn(func(entry metaCacheEntry) bool {
+			if o.Prefix != "" && !strings.HasPrefix(entry.name, o.Prefix) {
+				// We are past the prefix, don't continue.
+				pastPrefix = true
+				return false
+			}
+			if !o.IncludeDirectories && entry.isDir() {
+				return true
+			}
+			if !entry.isInDir(o.Prefix, o.Separator) {
+				return true
+			}
+			if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() {
+				return entries.len() < o.Limit
+			}
+			entries.o = append(entries.o, entry)
+			return entries.len() < o.Limit
+		})
+		if (err != nil && err.Error() == io.EOF.Error()) || pastPrefix || r.nextEOF() {
+			return entries, io.EOF
+		}
+		return entries, err
+	}
+
+	// We should not need to filter more.
+	return r.readN(o.Limit, o.InclDeleted, o.IncludeDirectories, o.Prefix)
+}
+
+func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) {
+	retries := 0
+	const debugPrint = false
+	rpc := globalNotificationSys.restClientFromHash(o.Bucket)
+	for {
+		select {
+		case <-ctx.Done():
+			return entries, ctx.Err()
+		default:
+		}
+
+		// If many failures, check the cache state.
+		if retries > 10 {
+			err := o.checkMetacacheState(ctx, rpc)
+			if debugPrint {
+				logger.Info("waiting for first part (%s), err: %v", o.objectPath(0), err)
+			}
+			if err != nil {
+				return entries, err
+			}
+			retries = 1
+		}
+
+		const retryDelay = 500 * time.Millisecond
+		// Load first part metadata...
+		// All operations are performed without locks, so we must be careful and allow for failures.
+		// Read metadata associated with the object from a disk.
+		if retries > 0 {
+			disks := er.getOnlineDisks()
+			if len(disks) == 0 {
+				time.Sleep(retryDelay)
+				retries++
+				continue
+			}
+
+			_, err := disks[0].ReadVersion(ctx, minioMetaBucket, o.objectPath(0), "", false)
+			if err != nil {
+				time.Sleep(retryDelay)
+				retries++
+				continue
+			}
+		}
+
+		// Read metadata associated with the object from all disks.
+		fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(0), ObjectOptions{})
+		if err != nil {
+			if err == errFileNotFound || errors.Is(err, errErasureReadQuorum) || errors.Is(err, InsufficientReadQuorum{}) {
+				retries++
+				time.Sleep(retryDelay)
+				continue
+			}
+			if debugPrint {
+				console.Infoln("first getObjectFileInfo", o.objectPath(0), "returned err:", err)
+				console.Infof("err type: %T\n", err)
+			}
+			return entries, err
+		}
+		if fi.Deleted {
+			return entries, errFileNotFound
+		}
+
+		partN, err := o.findFirstPart(fi)
+		switch err {
+		case nil:
+		case io.ErrUnexpectedEOF, errErasureReadQuorum, InsufficientReadQuorum{}:
+			if retries == 10 {
+				err := o.checkMetacacheState(ctx, rpc)
+				if debugPrint {
+					logger.Info("waiting for metadata, err: %v", err)
+				}
+				if err != nil {
+					return entries, err
+				}
+				retries = -1
+			}
+			retries++
+			time.Sleep(retryDelay)
+			continue
+		case io.EOF:
+			return entries, io.EOF
+		}
+
+		// We got a stream to start at.
+		loadedPart := 0
+		var buf bytes.Buffer
+		for {
+			select {
+			case <-ctx.Done():
+				return entries, ctx.Err()
+			default:
+			}
+
+			if partN != loadedPart {
+				if retries > 10 {
+					err := o.checkMetacacheState(ctx, rpc)
+					if debugPrint {
+						logger.Info("waiting for part data (%v), err: %v", o.objectPath(partN), err)
+					}
+					if err != nil {
+						return entries, err
+					}
+					retries = 1
+				}
+
+				if retries > 0 {
+					// Load from one disk only
+					disks := er.getOnlineDisks()
+					if len(disks) == 0 {
+						time.Sleep(retryDelay)
+						retries++
+						continue
+					}
+
+					_, err := disks[0].ReadVersion(ctx, minioMetaBucket, o.objectPath(partN), "", false)
+					if err != nil {
+						time.Sleep(retryDelay)
+						retries++
+						continue
+					}
+				}
+				// Load first part metadata...
+				fi, metaArr, onlineDisks, err = er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(partN), ObjectOptions{})
+				switch err {
+				case errFileNotFound, errErasureReadQuorum, InsufficientReadQuorum{}:
+					time.Sleep(retryDelay)
+					retries++
+					continue
+				default:
+					time.Sleep(retryDelay)
+					retries++
+					continue
+				case nil:
+					loadedPart = partN
+					bi, err := getMetacacheBlockInfo(fi, partN)
+					logger.LogIf(ctx, err)
+					if err == nil {
+						if bi.pastPrefix(o.Prefix) {
+							return entries, io.EOF
+						}
+					}
+				}
+				if fi.Deleted {
+					return entries, io.ErrUnexpectedEOF
+				}
+			}
+			buf.Reset()
+			err := er.getObjectWithFileInfo(ctx, minioMetaBucket, o.objectPath(partN), 0, fi.Size, &buf, fi, metaArr, onlineDisks)
+			switch err {
+			case errFileNotFound, errErasureReadQuorum, InsufficientReadQuorum{}:
+				time.Sleep(retryDelay)
+				retries++
+				continue
+			default:
+				logger.LogIf(ctx, err)
+				return entries, err
+			case nil:
+			}
+			tmp, err := newMetacacheReader(&buf)
+			if err != nil {
+				return entries, err
+			}
+			e, err := tmp.filter(o)
+			entries.o = append(entries.o, e.o...)
+			if o.Limit > 0 && entries.len() > o.Limit {
+				entries.truncate(o.Limit)
+				return entries, nil
+			}
+			switch err {
+			case io.EOF:
+				// We finished at the end of the block.
+				// And should not expect any more results.
+				bi, err := getMetacacheBlockInfo(fi, partN)
+				logger.LogIf(ctx, err)
+				if err != nil || bi.EOS {
+					// We are done and there are no more parts.
+					return entries, io.EOF
+				}
+				if bi.endedPrefix(o.Prefix) {
+					// Nothing more for prefix.
+					return entries, io.EOF
+				}
+				partN++
+				retries = 0
+			case nil:
+				// We stopped within the listing, we are done for now...
+				return entries, nil
+			default:
+				return entries, err
+			}
+		}
+	}
+}
+
+// Will return io.EOF if continuing would not yield more results.
+func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) {
+	const debugPrint = false
+	if debugPrint {
+		console.Printf("listPath with options: %#v", o)
+	}
+	// See if we have the listing stored.
+	if !o.Create {
+		entries, err := er.streamMetadataParts(ctx, o)
+		switch err {
+		case nil, io.EOF, context.Canceled, context.DeadlineExceeded:
+			return entries, err
+		}
+		logger.LogIf(ctx, err)
+		return entries, err
+	}
+
+	meta := o.newMetacache()
+	rpc := globalNotificationSys.restClientFromHash(o.Bucket)
+	var metaMu sync.Mutex
+
+	if debugPrint {
+		console.Println("listPath: scanning bucket:", o.Bucket, "basedir:", o.BaseDir, "prefix:", o.Prefix, "marker:", o.Marker)
+	}
+
+	// Disconnect from call above, but cancel on exit.
+	ctx, cancel := context.WithCancel(GlobalContext)
+	// We need to ask disks.
+	disks := er.getOnlineDisks()
+
+	defer func() {
+		if debugPrint {
+			console.Println("listPath returning:", entries.len(), "err:", err)
+		}
+		if err != nil && err != io.EOF {
+			metaMu.Lock()
+			if meta.status != scanStateError {
+				meta.error = err.Error()
+				meta.status = scanStateError
+			}
+			meta, _ = o.updateMetacacheListing(meta, rpc)
+			metaMu.Unlock()
+			cancel()
+		}
+	}()
+
+	askDisks := o.AskDisks
+	if askDisks == -1 {
+		askDisks = getReadQuorum(er.SetDriveCount())
+	}
+
+	if len(disks) < askDisks {
+		err = InsufficientReadQuorum{}
+		if debugPrint {
+			console.Errorf("listPath: Insufficient disks, %d of %d needed are available", len(disks), askDisks)
+		}
+		logger.LogIf(ctx, fmt.Errorf("listPath: Insufficient disks, %d of %d needed are available", len(disks), askDisks))
+		cancel()
+		return
+	}
+
+	// Select askDisks random disks.
+	if len(disks) > askDisks {
+		disks = disks[:askDisks]
+	}
+
+	var readers = make([]*metacacheReader, askDisks)
+	for i := range disks {
+		r, w := io.Pipe()
+		d := disks[i]
+		readers[i], err = newMetacacheReader(r)
+		if err != nil {
+			cancel()
+			return entries, err
+		}
+		// Send request to each disk.
+		go func() {
+			err := d.WalkDir(ctx, WalkDirOptions{Bucket: o.Bucket, BaseDir: o.BaseDir, Recursive: o.Recursive || o.Separator != SlashSeparator}, w)
+			w.CloseWithError(err)
+			if err != io.EOF {
+				logger.LogIf(ctx, err)
+			}
+		}()
+	}
+
+	// Create output for our results.
+	cacheCh := make(chan metaCacheEntry, metacacheBlockSize)
+
+	// Create filter for results.
+	filterCh := make(chan metaCacheEntry, 100)
+	filteredResults := o.gatherResults(filterCh)
+	closeChannels := func() {
+		close(cacheCh)
+		close(filterCh)
+	}
+
+	go func() {
+		defer cancel()
+		// Save continuous updates
+		go func() {
+			var err error
+			ticker := time.NewTicker(10 * time.Second)
+			defer ticker.Stop()
+			var exit bool
+			for !exit {
+				select {
+				case <-ticker.C:
+				case <-ctx.Done():
+					exit = true
+				}
+				metaMu.Lock()
+				meta.endedCycle = intDataUpdateTracker.current()
+				meta, err = o.updateMetacacheListing(meta, rpc)
+				if meta.status == scanStateError {
+					cancel()
+					exit = true
+				}
+				metaMu.Unlock()
+				logger.LogIf(ctx, err)
+			}
+		}()
+
+		const retryDelay = 200 * time.Millisecond
+		const maxTries = 10
+
+		// Write results to disk.
+		bw := newMetacacheBlockWriter(cacheCh, func(b *metacacheBlock) error {
+			if debugPrint {
+				console.Println("listPath: saving block", b.n, "to", o.objectPath(b.n))
+			}
+			r, err := hash.NewReader(bytes.NewBuffer(b.data), int64(len(b.data)), "", "", int64(len(b.data)), false)
+			logger.LogIf(ctx, err)
+			custom := b.headerKV()
+			custom[xhttp.AmzStorageClass] = storageclass.RRS
+			_, err = er.putObject(ctx, minioMetaBucket, o.objectPath(b.n), NewPutObjReader(r, nil, nil), ObjectOptions{UserDefined: custom})
+			if err != nil {
+				metaMu.Lock()
+				if meta.error != "" {
+					meta.status = scanStateError
+					meta.error = err.Error()
+				}
+				metaMu.Unlock()
+				cancel()
+				return err
+			}
+			if b.n == 0 {
+				return nil
+			}
+			// Update block 0 metadata.
+			var retries int
+			for {
+				err := er.updateObjectMeta(ctx, minioMetaBucket, o.objectPath(0), b.headerKV(), ObjectOptions{})
+				if err == nil {
+					break
+				}
+				switch err.(type) {
+				case ObjectNotFound:
+					return err
+				case InsufficientReadQuorum:
+				default:
+					logger.LogIf(ctx, err)
+				}
+				if retries >= maxTries {
+					return err
+				}
+				retries++
+				time.Sleep(retryDelay)
+			}
+			return nil
+		})
+
+		// How to resolve results.
+		resolver := metadataResolutionParams{
+			dirQuorum: askDisks - 1,
+			objQuorum: askDisks - 1,
+			bucket:    o.Bucket,
+		}
+
+		topEntries := make(metaCacheEntries, len(readers))
+		for {
+			// Get the top entry from each
+			var current metaCacheEntry
+			var atEOF, agree int
+			for i, r := range readers {
+				topEntries[i].name = ""
+				entry, err := r.peek()
+				switch err {
+				case io.EOF:
+					atEOF++
+					continue
+				case nil:
+				default:
+					closeChannels()
+					metaMu.Lock()
+					meta.status = scanStateError
+					meta.error = err.Error()
+					metaMu.Unlock()
+					return
+				}
+				// If no current, add it.
+				if current.name == "" {
+					topEntries[i] = entry
+					current = entry
+					agree++
+					continue
+				}
+				// If exact match, we agree.
+				if current.matches(&entry, o.Bucket) {
+					topEntries[i] = entry
+					agree++
+					continue
+				}
+				// If only the name matches we didn't agree, but add it for resolution.
+				if entry.name == current.name {
+					topEntries[i] = entry
+					continue
+				}
+				// We got different entries
+				if entry.name > current.name {
+					continue
+				}
+				// We got a new, better current.
+				// Clear existing entries.
+				for i := range topEntries[:i] {
+					topEntries[i] = metaCacheEntry{}
+				}
+				agree = 1
+				current = entry
+				topEntries[i] = entry
+			}
+			// Break if all at EOF.
+			if atEOF == len(readers) {
+				break
+			}
+			if agree == len(readers) {
+				// Everybody agreed
+				for _, r := range readers {
+					r.skip(1)
+				}
+				cacheCh <- topEntries[0]
+				filterCh <- topEntries[0]
+				continue
+			}
+
+			// Results Disagree :-(
+			entry, ok := topEntries.resolve(&resolver)
+			if ok {
+				cacheCh <- *entry
+				filterCh <- *entry
+			}
+			// Skip the inputs we used.
+			for i, r := range readers {
+				if topEntries[i].name != "" {
+					r.skip(1)
+				}
+			}
+		}
+
+		// Save success
+		metaMu.Lock()
+		if meta.error == "" {
+			meta.status = scanStateSuccess
+			meta.endedCycle = intDataUpdateTracker.current()
+		}
+		meta, _ = o.updateMetacacheListing(meta, rpc)
+		metaMu.Unlock()
+
+		closeChannels()
+		if err := bw.Close(); err != nil {
+			metaMu.Lock()
+			meta.error = err.Error()
+			meta.status = scanStateError
+			meta, err = o.updateMetacacheListing(meta, rpc)
+			metaMu.Unlock()
+		}
+	}()
+
+	return filteredResults()
+}

cmd/metacache-stream.go

@@ -0,0 +1,844 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"github.com/klauspost/compress/s2"
+	"github.com/minio/minio/cmd/logger"
+	"github.com/tinylib/msgp/msgp"
+)
+
+// metadata stream format:
+//
+// The stream is s2 compressed.
+// https://github.com/klauspost/compress/tree/master/s2#s2-compression
+// This ensures integrity and reduces the size typically by at least 50%.
+//
+// All stream elements are msgpack encoded.
+//
+// 1 Integer, metacacheStreamVersion of the writer.
+// This can be used for managing breaking changes.
+//
+// For each element:
+// 1. Bool. If false at end of stream.
+// 2. String. Name of object. Directories contains a trailing slash.
+// 3. Binary. Blob of metadata. Length 0 on directories.
+// ... Next element.
+//
+// Streams can be assumed to be sorted in ascending order.
+// If the stream ends before a false boolean it can be assumed it was truncated.
+
+const metacacheStreamVersion = 1
+
+// metacacheWriter provides a serializer of metacache objects.
+type metacacheWriter struct {
+	mw        *msgp.Writer
+	creator   func() error
+	closer    func() error
+	blockSize int
+
+	streamErr error
+	streamWg  sync.WaitGroup
+}
+
+// newMetacacheWriter will create a serializer that will write objects in given order to the output.
+// Provide a block size that affects latency. If 0 a default of 128KiB will be used.
+// Block size can be up to 4MiB.
+func newMetacacheWriter(out io.Writer, blockSize int) *metacacheWriter {
+	if blockSize < 8<<10 {
+		blockSize = 128 << 10
+	}
+	w := metacacheWriter{
+		mw:        nil,
+		blockSize: blockSize,
+	}
+	w.creator = func() error {
+		s2w := s2.NewWriter(out, s2.WriterBlockSize(blockSize))
+		w.mw = msgp.NewWriter(s2w)
+		w.creator = nil
+		if err := w.mw.WriteByte(metacacheStreamVersion); err != nil {
+			return err
+		}
+
+		w.closer = func() (err error) {
+			defer func() {
+				cerr := s2w.Close()
+				if err == nil && cerr != nil {
+					err = cerr
+				}
+			}()
+			if w.streamErr != nil {
+				return w.streamErr
+			}
+			if err = w.mw.WriteBool(false); err != nil {
+				return err
+			}
+			return w.mw.Flush()
+		}
+		return nil
+	}
+	return &w
+}
+
+// write one or more objects to the stream in order.
+// It is favorable to send as many objects as possible in a single write,
+// but no more than math.MaxUint32
+func (w *metacacheWriter) write(objs ...metaCacheEntry) error {
+	if w == nil {
+		return errors.New("metacacheWriter: nil writer")
+	}
+	if len(objs) == 0 {
+		return nil
+	}
+	if w.creator != nil {
+		err := w.creator()
+		w.creator = nil
+		if err != nil {
+			return fmt.Errorf("metacacheWriter: unable to create writer: %w", err)
+		}
+		if w.mw == nil {
+			return errors.New("metacacheWriter: writer not initialized")
+		}
+	}
+	for _, o := range objs {
+		if len(o.name) == 0 {
+			return errors.New("metacacheWriter: no name provided")
+		}
+		// Indicate EOS
+		err := w.mw.WriteBool(true)
+		if err != nil {
+			return err
+		}
+		err = w.mw.WriteString(o.name)
+		if err != nil {
+			return err
+		}
+		err = w.mw.WriteBytes(o.metadata)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// stream entries to the output.
+// The returned channel should be closed when done.
+// Any error is reported when closing the metacacheWriter.
+func (w *metacacheWriter) stream() (chan<- metaCacheEntry, error) {
+	if w.creator != nil {
+		err := w.creator()
+		w.creator = nil
+		if err != nil {
+			return nil, fmt.Errorf("metacacheWriter: unable to create writer: %w", err)
+		}
+		if w.mw == nil {
+			return nil, errors.New("metacacheWriter: writer not initialized")
+		}
+	}
+	var objs = make(chan metaCacheEntry, 100)
+	w.streamErr = nil
+	w.streamWg.Add(1)
+	go func() {
+		defer w.streamWg.Done()
+		for o := range objs {
+			if len(o.name) == 0 || w.streamErr != nil {
+				continue
+			}
+			// Indicate EOS
+			err := w.mw.WriteBool(true)
+			if err != nil {
+				w.streamErr = err
+				continue
+			}
+			err = w.mw.WriteString(o.name)
+			if err != nil {
+				w.streamErr = err
+				continue
+			}
+			err = w.mw.WriteBytes(o.metadata)
+			if err != nil {
+				w.streamErr = err
+				continue
+			}
+		}
+	}()
+
+	return objs, nil
+}
+
+// Close and release resources.
+func (w *metacacheWriter) Close() error {
+	if w == nil || w.closer == nil {
+		return nil
+	}
+	w.streamWg.Wait()
+	err := w.closer()
+	w.closer = nil
+	return err
+}
+
+// Reset and start writing to new writer.
+// Close must have been called before this.
+func (w *metacacheWriter) Reset(out io.Writer) {
+	w.streamErr = nil
+	w.creator = func() error {
+		s2w := s2.NewWriter(out, s2.WriterBlockSize(w.blockSize))
+		w.mw = msgp.NewWriter(s2w)
+		w.creator = nil
+		if err := w.mw.WriteByte(metacacheStreamVersion); err != nil {
+			return err
+		}
+
+		w.closer = func() error {
+			if w.streamErr != nil {
+				return w.streamErr
+			}
+			if err := w.mw.WriteBool(false); err != nil {
+				return err
+			}
+			if err := w.mw.Flush(); err != nil {
+				return err
+			}
+			return s2w.Close()
+		}
+		return nil
+	}
+}
+
+var s2DecPool = sync.Pool{New: func() interface{} {
+	return s2.NewReader(nil)
+}}
+
+// metacacheReader allows reading a cache stream.
+type metacacheReader struct {
+	mr      *msgp.Reader
+	current metaCacheEntry
+	err     error // stateful error
+	closer  func()
+	creator func() error
+}
+
+// newMetacacheReader creates a new cache reader.
+// Nothing will be read from the stream yet.
+func newMetacacheReader(r io.Reader) (*metacacheReader, error) {
+	dec := s2DecPool.Get().(*s2.Reader)
+	dec.Reset(r)
+	mr := msgp.NewReader(dec)
+	m := metacacheReader{
+		mr: mr,
+		closer: func() {
+			dec.Reset(nil)
+			s2DecPool.Put(dec)
+		},
+		creator: func() error {
+			v, err := mr.ReadByte()
+			if err != nil {
+				return err
+			}
+			switch v {
+			case metacacheStreamVersion:
+			default:
+				return fmt.Errorf("metacacheReader: Unknown version: %d", v)
+			}
+			return nil
+		},
+	}
+	return &m, nil
+}
+
+func (r *metacacheReader) checkInit() {
+	if r.creator == nil || r.err != nil {
+		return
+	}
+	r.err = r.creator()
+	r.creator = nil
+}
+
+// peek will return the name of the next object.
+// Will return io.EOF if there are no more objects.
+// Should be used sparingly.
+func (r *metacacheReader) peek() (metaCacheEntry, error) {
+	r.checkInit()
+	if r.err != nil {
+		return metaCacheEntry{}, r.err
+	}
+	if r.current.name != "" {
+		return r.current, nil
+	}
+	if more, err := r.mr.ReadBool(); !more {
+		switch err {
+		case nil:
+			r.err = io.EOF
+			return metaCacheEntry{}, io.EOF
+		case io.EOF:
+			r.err = io.ErrUnexpectedEOF
+			return metaCacheEntry{}, io.ErrUnexpectedEOF
+		}
+		r.err = err
+		return metaCacheEntry{}, err
+	}
+
+	var err error
+	if r.current.name, err = r.mr.ReadString(); err != nil {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		r.err = err
+		return metaCacheEntry{}, err
+	}
+	r.current.metadata, err = r.mr.ReadBytes(r.current.metadata[:0])
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	}
+	r.err = err
+	return r.current, err
+}
+
+// next will read one entry from the stream.
+// Generally not recommended for fast operation.
+func (r *metacacheReader) next() (metaCacheEntry, error) {
+	r.checkInit()
+	if r.err != nil {
+		return metaCacheEntry{}, r.err
+	}
+	var m metaCacheEntry
+	var err error
+	if r.current.name != "" {
+		m.name = r.current.name
+		m.metadata = r.current.metadata
+		r.current.name = ""
+		r.current.metadata = nil
+		return m, nil
+	}
+	if more, err := r.mr.ReadBool(); !more {
+		switch err {
+		case nil:
+			r.err = io.EOF
+			return m, io.EOF
+		case io.EOF:
+			r.err = io.ErrUnexpectedEOF
+			return m, io.ErrUnexpectedEOF
+		}
+		r.err = err
+		return m, err
+	}
+	if m.name, err = r.mr.ReadString(); err != nil {
+		if err == io.EOF {
+			err = io.ErrUnexpectedEOF
+		}
+		r.err = err
+		return m, err
+	}
+	m.metadata, err = r.mr.ReadBytes(nil)
+	if err == io.EOF {
+		err = io.ErrUnexpectedEOF
+	}
+	r.err = err
+	return m, err
+}
+
+// next will read one entry from the stream.
+// Generally not recommended for fast operation.
+func (r *metacacheReader) nextEOF() bool {
+	r.checkInit()
+	if r.err != nil {
+		return r.err == io.EOF
+	}
+	if r.current.name != "" {
+		return false
+	}
+	_, err := r.peek()
+	if err != nil {
+		r.err = err
+		return r.err == io.EOF
+	}
+	return false
+}
+
+// forwardTo will forward to the first entry that is >= s.
+// Will return io.EOF if end of stream is reached without finding any.
+func (r *metacacheReader) forwardTo(s string) error {
+	r.checkInit()
+	if r.err != nil {
+		return r.err
+	}
+
+	if s == "" {
+		return nil
+	}
+	if r.current.name != "" {
+		if r.current.name >= s {
+			return nil
+		}
+		r.current.name = ""
+		r.current.metadata = nil
+	}
+	// temporary name buffer.
+	var tmp = make([]byte, 0, 256)
+	for {
+		if more, err := r.mr.ReadBool(); !more {
+			switch err {
+			case nil:
+				r.err = io.EOF
+				return io.EOF
+			case io.EOF:
+				r.err = io.ErrUnexpectedEOF
+				return io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return err
+		}
+		// Read name without allocating more than 1 buffer.
+		sz, err := r.mr.ReadStringHeader()
+		if err != nil {
+			r.err = err
+			return err
+		}
+		if cap(tmp) < int(sz) {
+			tmp = make([]byte, 0, sz+256)
+		}
+		tmp = tmp[:sz]
+		_, err = r.mr.R.ReadFull(tmp)
+		if err != nil {
+			r.err = err
+			return err
+		}
+		if string(tmp) >= s {
+			r.current.name = string(tmp)
+			r.current.metadata, err = r.mr.ReadBytes(nil)
+			return err
+		}
+		// Skip metadata
+		err = r.mr.Skip()
+		if err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return err
+		}
+	}
+}
+
+// readN will return all the requested number of entries in order
+// or all if n < 0.
+// Will return io.EOF if end of stream is reached.
+// If requesting 0 objects nil error will always be returned regardless of at end of stream.
+// Use peek to determine if at end of stream.
+func (r *metacacheReader) readN(n int, inclDeleted, inclDirs bool, prefix string) (metaCacheEntriesSorted, error) {
+	r.checkInit()
+	if n == 0 {
+		return metaCacheEntriesSorted{}, nil
+	}
+	if r.err != nil {
+		return metaCacheEntriesSorted{}, r.err
+	}
+
+	var res metaCacheEntries
+	if n > 0 {
+		res = make(metaCacheEntries, 0, n)
+	}
+	if prefix != "" {
+		if err := r.forwardTo(prefix); err != nil {
+			return metaCacheEntriesSorted{}, err
+		}
+	}
+	next, err := r.peek()
+	if err != nil {
+		return metaCacheEntriesSorted{}, err
+	}
+	if !next.hasPrefix(prefix) {
+		return metaCacheEntriesSorted{}, io.EOF
+	}
+
+	if r.current.name != "" {
+		if (inclDeleted || !r.current.isLatestDeletemarker()) && r.current.hasPrefix(prefix) && (inclDirs || r.current.isObject()) {
+			res = append(res, r.current)
+		}
+		r.current.name = ""
+		r.current.metadata = nil
+	}
+
+	for n < 0 || len(res) < n {
+		if more, err := r.mr.ReadBool(); !more {
+			switch err {
+			case nil:
+				r.err = io.EOF
+				return metaCacheEntriesSorted{o: res}, io.EOF
+			case io.EOF:
+				r.err = io.ErrUnexpectedEOF
+				return metaCacheEntriesSorted{o: res}, io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return metaCacheEntriesSorted{o: res}, err
+		}
+		var err error
+		var meta metaCacheEntry
+		if meta.name, err = r.mr.ReadString(); err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return metaCacheEntriesSorted{o: res}, err
+		}
+		if !meta.hasPrefix(prefix) {
+			r.mr.R.Skip(1)
+			return metaCacheEntriesSorted{o: res}, io.EOF
+		}
+		if meta.metadata, err = r.mr.ReadBytes(nil); err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return metaCacheEntriesSorted{o: res}, err
+		}
+		if !inclDirs && meta.isDir() {
+			continue
+		}
+		if meta.isDir() && !inclDeleted && meta.isLatestDeletemarker() {
+			continue
+		}
+		res = append(res, meta)
+	}
+	return metaCacheEntriesSorted{o: res}, nil
+}
+
+// readAll will return all remaining objects on the dst channel and close it when done.
+// The context allows the operation to be canceled.
+func (r *metacacheReader) readAll(ctx context.Context, dst chan<- metaCacheEntry) error {
+	r.checkInit()
+	if r.err != nil {
+		return r.err
+	}
+	defer close(dst)
+	if r.current.name != "" {
+		select {
+		case <-ctx.Done():
+			r.err = ctx.Err()
+			return ctx.Err()
+		case dst <- r.current:
+		}
+		r.current.name = ""
+		r.current.metadata = nil
+	}
+	for {
+		if more, err := r.mr.ReadBool(); !more {
+			switch err {
+			case io.EOF:
+				err = io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return err
+		}
+
+		var err error
+		var meta metaCacheEntry
+		if meta.name, err = r.mr.ReadString(); err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return err
+		}
+		if meta.metadata, err = r.mr.ReadBytes(nil); err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return err
+		}
+		select {
+		case <-ctx.Done():
+			r.err = ctx.Err()
+			return ctx.Err()
+		case dst <- meta:
+		}
+	}
+}
+
+// readFn will return all remaining objects
+// and provide a callback for each entry read in order
+// as long as true is returned on the callback.
+func (r *metacacheReader) readFn(fn func(entry metaCacheEntry) bool) error {
+	r.checkInit()
+	if r.err != nil {
+		return r.err
+	}
+	if r.current.name != "" {
+		fn(r.current)
+		r.current.name = ""
+		r.current.metadata = nil
+	}
+	for {
+		if more, err := r.mr.ReadBool(); !more {
+			switch err {
+			case io.EOF:
+				r.err = io.ErrUnexpectedEOF
+				return io.ErrUnexpectedEOF
+			case nil:
+				r.err = io.EOF
+				return io.EOF
+			}
+			return err
+		}
+
+		var err error
+		var meta metaCacheEntry
+		if meta.name, err = r.mr.ReadString(); err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return err
+		}
+		if meta.metadata, err = r.mr.ReadBytes(nil); err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return err
+		}
+		// Send it!
+		if !fn(meta) {
+			return nil
+		}
+	}
+}
+
+// readNames will return all the requested number of names in order
+// or all if n < 0.
+// Will return io.EOF if end of stream is reached.
+func (r *metacacheReader) readNames(n int) ([]string, error) {
+	r.checkInit()
+	if r.err != nil {
+		return nil, r.err
+	}
+	if n == 0 {
+		return nil, nil
+	}
+	var res []string
+	if n > 0 {
+		res = make([]string, 0, n)
+	}
+	if r.current.name != "" {
+		res = append(res, r.current.name)
+		r.current.name = ""
+		r.current.metadata = nil
+	}
+	for n < 0 || len(res) < n {
+		if more, err := r.mr.ReadBool(); !more {
+			switch err {
+			case nil:
+				r.err = io.EOF
+				return res, io.EOF
+			case io.EOF:
+				r.err = io.ErrUnexpectedEOF
+				return res, io.ErrUnexpectedEOF
+			}
+			return res, err
+		}
+
+		var err error
+		var name string
+		if name, err = r.mr.ReadString(); err != nil {
+			r.err = err
+			return res, err
+		}
+		if err = r.mr.Skip(); err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return res, err
+		}
+		res = append(res, name)
+	}
+	return res, nil
+}
+
+// skip n entries on the input stream.
+// If there are less entries left io.EOF is returned.
+func (r *metacacheReader) skip(n int) error {
+	r.checkInit()
+	if r.err != nil {
+		return r.err
+	}
+	if n <= 0 {
+		return nil
+	}
+	if r.current.name != "" {
+		n--
+		r.current.name = ""
+		r.current.metadata = nil
+	}
+	for n > 0 {
+		if more, err := r.mr.ReadBool(); !more {
+			switch err {
+			case nil:
+				r.err = io.EOF
+				return io.EOF
+			case io.EOF:
+				r.err = io.ErrUnexpectedEOF
+				return io.ErrUnexpectedEOF
+			}
+			return err
+		}
+
+		if err := r.mr.Skip(); err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return err
+		}
+		if err := r.mr.Skip(); err != nil {
+			if err == io.EOF {
+				err = io.ErrUnexpectedEOF
+			}
+			r.err = err
+			return err
+		}
+		n--
+	}
+	return nil
+}
+
+// Close and release resources.
+func (r *metacacheReader) Close() error {
+	if r == nil || r.closer == nil {
+		return nil
+	}
+	r.closer()
+	r.closer = nil
+	r.creator = nil
+	return nil
+}
+
+// metacacheBlockWriter collects blocks and provides a callaback to store them.
+type metacacheBlockWriter struct {
+	wg           sync.WaitGroup
+	streamErr    error
+	blockEntries int
+}
+
+// newMetacacheBlockWriter provides a streaming block writer.
+// Each block is the size of the capacity of the input channel.
+// The caller should close to indicate the stream has ended.
+func newMetacacheBlockWriter(in <-chan metaCacheEntry, nextBlock func(b *metacacheBlock) error) *metacacheBlockWriter {
+	w := metacacheBlockWriter{blockEntries: cap(in)}
+	w.wg.Add(1)
+	go func() {
+		defer w.wg.Done()
+		var current metacacheBlock
+		var n int
+		var buf bytes.Buffer
+		block := newMetacacheWriter(&buf, 1<<20)
+		finishBlock := func() {
+			err := block.Close()
+			if err != nil {
+				w.streamErr = err
+				return
+			}
+			current.data = buf.Bytes()
+			w.streamErr = nextBlock(&current)
+			// Prepare for next
+			current.n++
+			buf.Reset()
+			block.Reset(&buf)
+			current.First = ""
+		}
+		for o := range in {
+			if len(o.name) == 0 || w.streamErr != nil {
+				continue
+			}
+			if current.First == "" {
+				current.First = o.name
+			}
+
+			if n >= w.blockEntries-1 {
+				finishBlock()
+				n = 0
+			}
+			n++
+
+			w.streamErr = block.write(o)
+			if w.streamErr != nil {
+				continue
+			}
+			current.Last = o.name
+		}
+		if n > 0 || current.n == 0 {
+			current.EOS = true
+			finishBlock()
+		}
+	}()
+	return &w
+}
+
+// Close the stream.
+// The incoming channel must be closed before calling this.
+// Returns the first error the occurred during the writing if any.
+func (w *metacacheBlockWriter) Close() error {
+	w.wg.Wait()
+	return w.streamErr
+}
+
+type metacacheBlock struct {
+	data  []byte
+	n     int
+	First string `json:"f"`
+	Last  string `json:"l"`
+	EOS   bool   `json:"eos,omitempty"`
+}
+
+func (b metacacheBlock) headerKV() map[string]string {
+	v, err := json.Marshal(b)
+	if err != nil {
+		logger.LogIf(context.Background(), err) // Unlikely
+		return nil
+	}
+	return map[string]string{fmt.Sprintf("%s-metacache-part-%d", ReservedMetadataPrefixLower, b.n): string(v)}
+}
+
+// pastPrefix returns true if the given prefix is before start of the block.
+func (b metacacheBlock) pastPrefix(prefix string) bool {
+	if prefix == "" || strings.HasPrefix(b.First, prefix) {
+		return false
+	}
+	// We have checked if prefix matches, so we can do direct compare.
+	return b.First > prefix
+}
+
+// endedPrefix returns true if the given prefix ends within the block.
+func (b metacacheBlock) endedPrefix(prefix string) bool {
+	if prefix == "" || strings.HasPrefix(b.Last, prefix) {
+		return false
+	}
+
+	// We have checked if prefix matches, so we can do direct compare.
+	return b.Last > prefix
+}

cmd/metacache-walk.go

@@ -0,0 +1,237 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strconv"
+	"strings"
+	"sync/atomic"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/logger"
+)
+
+// WalkDirOptions provides options for WalkDir operations.
+type WalkDirOptions struct {
+	// Bucket to crawl
+	Bucket string
+
+	// Directory inside the bucket.
+	BaseDir string
+
+	// Do a full recursive scan.
+	Recursive bool
+}
+
+// WalkDir will traverse a directory and return all entries found.
+// On success a sorted meta cache stream will be returned.
+func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error {
+	atomic.AddInt32(&s.activeIOCount, 1)
+	defer func() {
+		atomic.AddInt32(&s.activeIOCount, -1)
+	}()
+
+	// Verify if volume is valid and it exists.
+	volumeDir, err := s.getVolDir(opts.Bucket)
+	if err != nil {
+		return err
+	}
+
+	// Stat a volume entry.
+	_, err = os.Stat(volumeDir)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return errFaultyDisk
+		}
+		return err
+	}
+
+	// Fast exit track to check if we are listing an object with
+	// a trailing slash, this will avoid to list the object content.
+	if HasSuffix(opts.BaseDir, SlashSeparator) {
+		if st, err := os.Stat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile)); err == nil && st.Mode().IsRegular() {
+			return errFileNotFound
+		}
+	}
+	// Use a small block size to start sending quickly
+	w := newMetacacheWriter(wr, 16<<10)
+	defer w.Close()
+	out, err := w.stream()
+	if err != nil {
+		return err
+	}
+	defer close(out)
+
+	var scanDir func(path string) error
+	scanDir = func(current string) error {
+		entries, err := s.ListDir(ctx, opts.Bucket, current, -1)
+		if err != nil {
+			// Folder could have gone away in-between
+			if err != errVolumeNotFound && err != errFileNotFound {
+				logger.LogIf(ctx, err)
+			}
+			// Forward some errors?
+			return nil
+		}
+
+		for i, entry := range entries {
+			if strings.HasSuffix(entry, slashSeparator) {
+				// Trim slash, maybe compiler is clever?
+				entries[i] = entries[i][:len(entry)-1]
+				continue
+			}
+			// Do do not retain the file.
+			entries[i] = ""
+
+			// If root was an object return it as such.
+			if HasSuffix(entry, xlStorageFormatFile) {
+				var meta metaCacheEntry
+				meta.metadata, err = ioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFile))
+				if err != nil {
+					logger.LogIf(ctx, err)
+					continue
+				}
+				meta.name = strings.TrimSuffix(meta.name, xlStorageFormatFile)
+				meta.name = strings.TrimSuffix(meta.name, SlashSeparator)
+				out <- meta
+				return nil
+			}
+			// Check legacy.
+			if HasSuffix(entry, xlStorageFormatFileV1) {
+				var meta metaCacheEntry
+				meta.metadata, err = ioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1))
+				if err != nil {
+					logger.LogIf(ctx, err)
+					continue
+				}
+				meta.name = strings.TrimSuffix(meta.name, xlStorageFormatFileV1)
+				meta.name = strings.TrimSuffix(meta.name, SlashSeparator)
+				out <- meta
+				return nil
+			}
+			// Skip all other files.
+		}
+
+		// Process in sort order.
+		sort.Strings(entries)
+		dirStack := make([]string, 0, 5)
+		for _, entry := range entries {
+			if entry == "" {
+				continue
+			}
+			meta := metaCacheEntry{name: PathJoin(current, entry)}
+
+			// If directory entry on stack before this, pop it now.
+			for len(dirStack) > 0 && dirStack[len(dirStack)-1] < meta.name {
+				pop := dirStack[len(dirStack)-1]
+				out <- metaCacheEntry{name: pop}
+				if opts.Recursive {
+					// Scan folder we found. Should be in correct sort order where we are.
+					err := scanDir(pop)
+					logger.LogIf(ctx, err)
+				}
+				dirStack = dirStack[:len(dirStack)-1]
+			}
+
+			// All objects will be returned as directories, there has been no object check yet.
+			// Check it by attempting to read metadata.
+			meta.metadata, err = ioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFile))
+			switch {
+			case err == nil:
+				// It was an object
+				out <- meta
+			case os.IsNotExist(err):
+				meta.metadata, err = ioutil.ReadFile(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1))
+				if err == nil {
+					// Maybe rename? Would make it inconsistent across disks though.
+					// os.Rename(pathJoin(volumeDir, meta.name, xlStorageFormatFileV1), pathJoin(volumeDir, meta.name, xlStorageFormatFile))
+					// It was an object
+					out <- meta
+					continue
+				}
+
+				// NOT an object, append to stack (with slash)
+				dirStack = append(dirStack, meta.name+slashSeparator)
+			default:
+				logger.LogIf(ctx, err)
+			}
+		}
+		// If directory entry left on stack, pop it now.
+		for len(dirStack) > 0 {
+			pop := dirStack[len(dirStack)-1]
+			out <- metaCacheEntry{name: pop}
+			if opts.Recursive {
+				// Scan folder we found. Should be in correct sort order where we are.
+				err := scanDir(pop)
+				logger.LogIf(ctx, err)
+			}
+			dirStack = dirStack[:len(dirStack)-1]
+		}
+		return nil
+	}
+
+	// Stream output.
+	return scanDir(opts.BaseDir)
+}
+
+func (p *xlStorageDiskIDCheck) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error {
+	if err := p.checkDiskStale(); err != nil {
+		return err
+	}
+	return p.storage.WalkDir(ctx, opts, wr)
+}
+
+// WalkDir will traverse a directory and return all entries found.
+// On success a meta cache stream will be returned, that should be closed when done.
+func (client *storageRESTClient) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, opts.Bucket)
+	values.Set(storageRESTDirPath, opts.BaseDir)
+	values.Set(storageRESTRecursive, strconv.FormatBool(opts.Recursive))
+	respBody, err := client.call(ctx, storageRESTMethodWalkDir, values, nil, -1)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+	return waitForHTTPStream(respBody, wr)
+}
+
+// WalkDirHandler - remote caller to list files and folders in a requested directory path.
+func (s *storageRESTServer) WalkDirHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	dirPath := vars[storageRESTDirPath]
+	recursive, err := strconv.ParseBool(vars[storageRESTRecursive])
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	writer := streamHTTPResponse(w)
+	writer.CloseWithError(s.storage.WalkDir(r.Context(), WalkDirOptions{Bucket: volume, BaseDir: dirPath, Recursive: recursive}, writer))
+}

cmd/metacache.go

@@ -0,0 +1,185 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/minio/minio/cmd/logger"
+)
+
+type scanStatus uint8
+
+const (
+	scanStateNone scanStatus = iota
+	scanStateStarted
+	scanStateSuccess
+	scanStateError
+
+	// Time in which the initiator of a scan must have reported back.
+	metacacheMaxRunningAge = time.Minute
+
+	// metacacheBlockSize is the number of file/directory entries to have in each block.
+	metacacheBlockSize = 5000
+)
+
+//go:generate msgp -file $GOFILE -unexported
+
+// metacache contains a tracked cache entry.
+type metacache struct {
+	id           string     `msg:"id"`
+	bucket       string     `msg:"b"`
+	root         string     `msg:"root"`
+	recursive    bool       `msg:"rec"`
+	status       scanStatus `msg:"stat"`
+	fileNotFound bool       `msg:"fnf"`
+	error        string     `msg:"err"`
+	started      time.Time  `msg:"st"`
+	ended        time.Time  `msg:"end"`
+	lastUpdate   time.Time  `msg:"u"`
+	lastHandout  time.Time  `msg:"lh"`
+	startedCycle uint64     `msg:"stc"`
+	endedCycle   uint64     `msg:"endc"`
+	dataVersion  uint8      `msg:"v"`
+}
+
+func (m *metacache) finished() bool {
+	return !m.ended.IsZero()
+}
+
+// worthKeeping indicates if the cache by itself is worth keeping.
+func (m *metacache) worthKeeping(currentCycle uint64) bool {
+	if m == nil {
+		return false
+	}
+	cache := m
+	switch {
+	case !cache.finished() && time.Since(cache.lastUpdate) > metacacheMaxRunningAge:
+		// Not finished and update for metacacheMaxRunningAge, discard it.
+		return false
+	case cache.finished() && cache.startedCycle > currentCycle:
+		// Cycle is somehow bigger.
+		return false
+	case cache.finished() && time.Since(cache.lastHandout) > 48*time.Hour:
+		// Keep only for 2 days. Fallback if crawler is clogged.
+		return false
+	case cache.finished() && currentCycle >= dataUsageUpdateDirCycles && cache.startedCycle < currentCycle-dataUsageUpdateDirCycles:
+		// Cycle is too old to be valuable.
+		return false
+	case cache.status == scanStateError || cache.status == scanStateNone:
+		// Remove failed listings after 10 minutes.
+		return time.Since(cache.lastUpdate) < 10*time.Minute
+	}
+	return true
+}
+
+// canBeReplacedBy.
+// Both must pass the worthKeeping check.
+func (m *metacache) canBeReplacedBy(other *metacache) bool {
+	// If the other is older it can never replace.
+	if other.started.Before(m.started) || m.id == other.id {
+		return false
+	}
+	if other.status == scanStateNone || other.status == scanStateError {
+		return false
+	}
+	if m.status == scanStateStarted && time.Since(m.lastUpdate) < metacacheMaxRunningAge {
+		return false
+	}
+	// Keep it around a bit longer.
+	if time.Since(m.lastHandout) < time.Hour || time.Since(m.lastUpdate) < metacacheMaxRunningAge {
+		return false
+	}
+
+	// Go through recursive combinations.
+	switch {
+	case !m.recursive && !other.recursive:
+		// If both not recursive root must match.
+		return m.root == other.root
+	case m.recursive && !other.recursive:
+		// A recursive can never be replaced by a non-recursive
+		return false
+	case !m.recursive && other.recursive:
+		// If other is recursive it must contain this root
+		return strings.HasPrefix(m.root, other.root)
+	case m.recursive && other.recursive:
+		// Similar if both are recursive
+		return strings.HasPrefix(m.root, other.root)
+	}
+	panic("should be unreachable")
+}
+
+// baseDirFromPrefix will return the base directory given an object path.
+// For example an object with name prefix/folder/object.ext will return `prefix/folder/`.
+func baseDirFromPrefix(prefix string) string {
+	b := path.Dir(prefix)
+	if b == "." || b == "./" || b == "/" {
+		b = ""
+	}
+	if !strings.Contains(prefix, slashSeparator) {
+		b = ""
+	}
+	if len(b) > 0 && !strings.HasSuffix(b, slashSeparator) {
+		b += slashSeparator
+	}
+	return b
+}
+
+// update cache with new status.
+// The updates are conditional so multiple callers can update with different states.
+func (m *metacache) update(update metacache) {
+	m.lastUpdate = UTCNow()
+
+	if m.status == scanStateStarted && update.status == scanStateSuccess {
+		m.ended = UTCNow()
+		m.endedCycle = update.endedCycle
+	}
+
+	if m.status == scanStateStarted && update.status != scanStateStarted {
+		m.status = update.status
+	}
+
+	if m.error == "" && update.error != "" {
+		m.error = update.error
+		m.status = scanStateError
+		m.ended = UTCNow()
+	}
+	m.fileNotFound = m.fileNotFound || update.fileNotFound
+}
+
+// delete all cache data on disks.
+func (m *metacache) delete(ctx context.Context) {
+	if m.bucket == "" || m.id == "" {
+		logger.LogIf(ctx, fmt.Errorf("metacache.delete: bucket (%s) or id (%s) empty", m.bucket, m.id))
+	}
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		logger.LogIf(ctx, errors.New("metacache.delete: no object layer"))
+		return
+	}
+	ez, ok := objAPI.(*erasureServerSets)
+	if !ok {
+		logger.LogIf(ctx, errors.New("metacache.delete: expected objAPI to be *erasureServerSets"))
+		return
+	}
+	ez.deleteAll(ctx, minioMetaBucket, metacachePrefixForID(m.bucket, m.id))
+}

cmd/metacache_gen.go

@@ -0,0 +1,495 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *metacache) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "id":
+			z.id, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "id")
+				return
+			}
+		case "b":
+			z.bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "bucket")
+				return
+			}
+		case "root":
+			z.root, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "root")
+				return
+			}
+		case "rec":
+			z.recursive, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "recursive")
+				return
+			}
+		case "stat":
+			{
+				var zb0002 uint8
+				zb0002, err = dc.ReadUint8()
+				if err != nil {
+					err = msgp.WrapError(err, "status")
+					return
+				}
+				z.status = scanStatus(zb0002)
+			}
+		case "fnf":
+			z.fileNotFound, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "fileNotFound")
+				return
+			}
+		case "err":
+			z.error, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "error")
+				return
+			}
+		case "st":
+			z.started, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "started")
+				return
+			}
+		case "end":
+			z.ended, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "ended")
+				return
+			}
+		case "u":
+			z.lastUpdate, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "lastUpdate")
+				return
+			}
+		case "lh":
+			z.lastHandout, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "lastHandout")
+				return
+			}
+		case "stc":
+			z.startedCycle, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "startedCycle")
+				return
+			}
+		case "endc":
+			z.endedCycle, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "endedCycle")
+				return
+			}
+		case "v":
+			z.dataVersion, err = dc.ReadUint8()
+			if err != nil {
+				err = msgp.WrapError(err, "dataVersion")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *metacache) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 14
+	// write "id"
+	err = en.Append(0x8e, 0xa2, 0x69, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.id)
+	if err != nil {
+		err = msgp.WrapError(err, "id")
+		return
+	}
+	// write "b"
+	err = en.Append(0xa1, 0x62)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "bucket")
+		return
+	}
+	// write "root"
+	err = en.Append(0xa4, 0x72, 0x6f, 0x6f, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.root)
+	if err != nil {
+		err = msgp.WrapError(err, "root")
+		return
+	}
+	// write "rec"
+	err = en.Append(0xa3, 0x72, 0x65, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.recursive)
+	if err != nil {
+		err = msgp.WrapError(err, "recursive")
+		return
+	}
+	// write "stat"
+	err = en.Append(0xa4, 0x73, 0x74, 0x61, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint8(uint8(z.status))
+	if err != nil {
+		err = msgp.WrapError(err, "status")
+		return
+	}
+	// write "fnf"
+	err = en.Append(0xa3, 0x66, 0x6e, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.fileNotFound)
+	if err != nil {
+		err = msgp.WrapError(err, "fileNotFound")
+		return
+	}
+	// write "err"
+	err = en.Append(0xa3, 0x65, 0x72, 0x72)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.error)
+	if err != nil {
+		err = msgp.WrapError(err, "error")
+		return
+	}
+	// write "st"
+	err = en.Append(0xa2, 0x73, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.started)
+	if err != nil {
+		err = msgp.WrapError(err, "started")
+		return
+	}
+	// write "end"
+	err = en.Append(0xa3, 0x65, 0x6e, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.ended)
+	if err != nil {
+		err = msgp.WrapError(err, "ended")
+		return
+	}
+	// write "u"
+	err = en.Append(0xa1, 0x75)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.lastUpdate)
+	if err != nil {
+		err = msgp.WrapError(err, "lastUpdate")
+		return
+	}
+	// write "lh"
+	err = en.Append(0xa2, 0x6c, 0x68)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.lastHandout)
+	if err != nil {
+		err = msgp.WrapError(err, "lastHandout")
+		return
+	}
+	// write "stc"
+	err = en.Append(0xa3, 0x73, 0x74, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.startedCycle)
+	if err != nil {
+		err = msgp.WrapError(err, "startedCycle")
+		return
+	}
+	// write "endc"
+	err = en.Append(0xa4, 0x65, 0x6e, 0x64, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.endedCycle)
+	if err != nil {
+		err = msgp.WrapError(err, "endedCycle")
+		return
+	}
+	// write "v"
+	err = en.Append(0xa1, 0x76)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint8(z.dataVersion)
+	if err != nil {
+		err = msgp.WrapError(err, "dataVersion")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *metacache) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 14
+	// string "id"
+	o = append(o, 0x8e, 0xa2, 0x69, 0x64)
+	o = msgp.AppendString(o, z.id)
+	// string "b"
+	o = append(o, 0xa1, 0x62)
+	o = msgp.AppendString(o, z.bucket)
+	// string "root"
+	o = append(o, 0xa4, 0x72, 0x6f, 0x6f, 0x74)
+	o = msgp.AppendString(o, z.root)
+	// string "rec"
+	o = append(o, 0xa3, 0x72, 0x65, 0x63)
+	o = msgp.AppendBool(o, z.recursive)
+	// string "stat"
+	o = append(o, 0xa4, 0x73, 0x74, 0x61, 0x74)
+	o = msgp.AppendUint8(o, uint8(z.status))
+	// string "fnf"
+	o = append(o, 0xa3, 0x66, 0x6e, 0x66)
+	o = msgp.AppendBool(o, z.fileNotFound)
+	// string "err"
+	o = append(o, 0xa3, 0x65, 0x72, 0x72)
+	o = msgp.AppendString(o, z.error)
+	// string "st"
+	o = append(o, 0xa2, 0x73, 0x74)
+	o = msgp.AppendTime(o, z.started)
+	// string "end"
+	o = append(o, 0xa3, 0x65, 0x6e, 0x64)
+	o = msgp.AppendTime(o, z.ended)
+	// string "u"
+	o = append(o, 0xa1, 0x75)
+	o = msgp.AppendTime(o, z.lastUpdate)
+	// string "lh"
+	o = append(o, 0xa2, 0x6c, 0x68)
+	o = msgp.AppendTime(o, z.lastHandout)
+	// string "stc"
+	o = append(o, 0xa3, 0x73, 0x74, 0x63)
+	o = msgp.AppendUint64(o, z.startedCycle)
+	// string "endc"
+	o = append(o, 0xa4, 0x65, 0x6e, 0x64, 0x63)
+	o = msgp.AppendUint64(o, z.endedCycle)
+	// string "v"
+	o = append(o, 0xa1, 0x76)
+	o = msgp.AppendUint8(o, z.dataVersion)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *metacache) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "id":
+			z.id, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "id")
+				return
+			}
+		case "b":
+			z.bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "bucket")
+				return
+			}
+		case "root":
+			z.root, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "root")
+				return
+			}
+		case "rec":
+			z.recursive, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "recursive")
+				return
+			}
+		case "stat":
+			{
+				var zb0002 uint8
+				zb0002, bts, err = msgp.ReadUint8Bytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "status")
+					return
+				}
+				z.status = scanStatus(zb0002)
+			}
+		case "fnf":
+			z.fileNotFound, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "fileNotFound")
+				return
+			}
+		case "err":
+			z.error, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "error")
+				return
+			}
+		case "st":
+			z.started, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "started")
+				return
+			}
+		case "end":
+			z.ended, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ended")
+				return
+			}
+		case "u":
+			z.lastUpdate, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "lastUpdate")
+				return
+			}
+		case "lh":
+			z.lastHandout, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "lastHandout")
+				return
+			}
+		case "stc":
+			z.startedCycle, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "startedCycle")
+				return
+			}
+		case "endc":
+			z.endedCycle, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "endedCycle")
+				return
+			}
+		case "v":
+			z.dataVersion, bts, err = msgp.ReadUint8Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "dataVersion")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *metacache) Msgsize() (s int) {
+	s = 1 + 3 + msgp.StringPrefixSize + len(z.id) + 2 + msgp.StringPrefixSize + len(z.bucket) + 5 + msgp.StringPrefixSize + len(z.root) + 4 + msgp.BoolSize + 5 + msgp.Uint8Size + 4 + msgp.BoolSize + 4 + msgp.StringPrefixSize + len(z.error) + 3 + msgp.TimeSize + 4 + msgp.TimeSize + 2 + msgp.TimeSize + 3 + msgp.TimeSize + 4 + msgp.Uint64Size + 5 + msgp.Uint64Size + 2 + msgp.Uint8Size
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *scanStatus) DecodeMsg(dc *msgp.Reader) (err error) {
+	{
+		var zb0001 uint8
+		zb0001, err = dc.ReadUint8()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = scanStatus(zb0001)
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z scanStatus) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteUint8(uint8(z))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z scanStatus) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendUint8(o, uint8(z))
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *scanStatus) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	{
+		var zb0001 uint8
+		zb0001, bts, err = msgp.ReadUint8Bytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		(*z) = scanStatus(zb0001)
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z scanStatus) Msgsize() (s int) {
+	s = msgp.Uint8Size
+	return
+}

cmd/metacache_gen_test.go

@@ -0,0 +1,123 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalmetacache(t *testing.T) {
+	v := metacache{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgmetacache(b *testing.B) {
+	v := metacache{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgmetacache(b *testing.B) {
+	v := metacache{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalmetacache(b *testing.B) {
+	v := metacache{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodemetacache(t *testing.T) {
+	v := metacache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodemetacache Msgsize() is inaccurate")
+	}
+
+	vn := metacache{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodemetacache(b *testing.B) {
+	v := metacache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodemetacache(b *testing.B) {
+	v := metacache{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/metacache_test.go

@@ -0,0 +1,301 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cmd
+
+import (
+	"testing"
+	"time"
+)
+
+var metaCacheTestsetTimestamp = time.Now()
+
+var metaCacheTestset = []metacache{
+	0: {
+		id:           "case-1-normal",
+		bucket:       "bucket",
+		root:         "folder/prefix",
+		recursive:    false,
+		status:       scanStateSuccess,
+		fileNotFound: false,
+		error:        "",
+		started:      metaCacheTestsetTimestamp,
+		ended:        metaCacheTestsetTimestamp.Add(time.Minute),
+		lastUpdate:   metaCacheTestsetTimestamp.Add(time.Minute),
+		lastHandout:  metaCacheTestsetTimestamp,
+		startedCycle: 10,
+		endedCycle:   10,
+		dataVersion:  metacacheStreamVersion,
+	},
+	1: {
+		id:           "case-2-recursive",
+		bucket:       "bucket",
+		root:         "folder/prefix",
+		recursive:    true,
+		status:       scanStateSuccess,
+		fileNotFound: false,
+		error:        "",
+		started:      metaCacheTestsetTimestamp,
+		ended:        metaCacheTestsetTimestamp.Add(time.Minute),
+		lastUpdate:   metaCacheTestsetTimestamp.Add(time.Minute),
+		lastHandout:  metaCacheTestsetTimestamp,
+		startedCycle: 10,
+		endedCycle:   10,
+		dataVersion:  metacacheStreamVersion,
+	},
+	2: {
+		id:           "case-3-older",
+		bucket:       "bucket",
+		root:         "folder/prefix",
+		recursive:    false,
+		status:       scanStateSuccess,
+		fileNotFound: true,
+		error:        "",
+		started:      metaCacheTestsetTimestamp.Add(-time.Minute),
+		ended:        metaCacheTestsetTimestamp,
+		lastUpdate:   metaCacheTestsetTimestamp,
+		lastHandout:  metaCacheTestsetTimestamp,
+		startedCycle: 10,
+		endedCycle:   10,
+		dataVersion:  metacacheStreamVersion,
+	},
+	3: {
+		id:           "case-4-error",
+		bucket:       "bucket",
+		root:         "folder/prefix",
+		recursive:    false,
+		status:       scanStateError,
+		fileNotFound: false,
+		error:        "an error lol",
+		started:      metaCacheTestsetTimestamp.Add(-20 * time.Minute),
+		ended:        metaCacheTestsetTimestamp.Add(-20 * time.Minute),
+		lastUpdate:   metaCacheTestsetTimestamp.Add(-20 * time.Minute),
+		lastHandout:  metaCacheTestsetTimestamp.Add(-20 * time.Minute),
+		startedCycle: 10,
+		endedCycle:   10,
+		dataVersion:  metacacheStreamVersion,
+	},
+	4: {
+		id:           "case-5-noupdate",
+		bucket:       "bucket",
+		root:         "folder/prefix",
+		recursive:    false,
+		status:       scanStateStarted,
+		fileNotFound: false,
+		error:        "",
+		started:      metaCacheTestsetTimestamp.Add(-time.Minute),
+		ended:        time.Time{},
+		lastUpdate:   metaCacheTestsetTimestamp.Add(-time.Minute),
+		lastHandout:  metaCacheTestsetTimestamp,
+		startedCycle: 10,
+		endedCycle:   10,
+		dataVersion:  metacacheStreamVersion,
+	},
+	5: {
+		id:           "case-6-404notfound",
+		bucket:       "bucket",
+		root:         "folder/notfound",
+		recursive:    true,
+		status:       scanStateSuccess,
+		fileNotFound: true,
+		error:        "",
+		started:      metaCacheTestsetTimestamp,
+		ended:        metaCacheTestsetTimestamp.Add(time.Minute),
+		lastUpdate:   metaCacheTestsetTimestamp.Add(time.Minute),
+		lastHandout:  metaCacheTestsetTimestamp,
+		startedCycle: 10,
+		endedCycle:   10,
+		dataVersion:  metacacheStreamVersion,
+	},
+	6: {
+		id:           "case-7-oldcycle",
+		bucket:       "bucket",
+		root:         "folder/prefix",
+		recursive:    true,
+		status:       scanStateSuccess,
+		fileNotFound: false,
+		error:        "",
+		started:      metaCacheTestsetTimestamp.Add(-10 * time.Minute),
+		ended:        metaCacheTestsetTimestamp.Add(-8 * time.Minute),
+		lastUpdate:   metaCacheTestsetTimestamp.Add(-8 * time.Minute),
+		lastHandout:  metaCacheTestsetTimestamp,
+		startedCycle: 6,
+		endedCycle:   8,
+		dataVersion:  metacacheStreamVersion,
+	},
+	7: {
+		id:           "case-8-running",
+		bucket:       "bucket",
+		root:         "folder/running",
+		recursive:    false,
+		status:       scanStateStarted,
+		fileNotFound: false,
+		error:        "",
+		started:      metaCacheTestsetTimestamp.Add(-1 * time.Minute),
+		ended:        time.Time{},
+		lastUpdate:   metaCacheTestsetTimestamp.Add(-1 * time.Minute),
+		lastHandout:  metaCacheTestsetTimestamp,
+		startedCycle: 10,
+		endedCycle:   0,
+		dataVersion:  metacacheStreamVersion,
+	},
+	8: {
+		id:           "case-8-finished-a-week-ago",
+		bucket:       "bucket",
+		root:         "folder/finished",
+		recursive:    false,
+		status:       scanStateSuccess,
+		fileNotFound: false,
+		error:        "",
+		started:      metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour),
+		ended:        metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour),
+		lastUpdate:   metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour),
+		lastHandout:  metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour),
+		startedCycle: 10,
+		endedCycle:   10,
+		dataVersion:  metacacheStreamVersion,
+	},
+}
+
+func Test_baseDirFromPrefix(t *testing.T) {
+	tests := []struct {
+		name   string
+		prefix string
+		want   string
+	}{
+		{
+			name:   "root",
+			prefix: "object.ext",
+			want:   "",
+		},
+		{
+			name:   "rootdotslash",
+			prefix: "./object.ext",
+			want:   "",
+		},
+		{
+			name:   "rootslash",
+			prefix: "/",
+			want:   "",
+		},
+		{
+			name:   "folder",
+			prefix: "prefix/",
+			want:   "prefix/",
+		},
+		{
+			name:   "folderobj",
+			prefix: "prefix/obj.ext",
+			want:   "prefix/",
+		},
+		{
+			name:   "folderfolderobj",
+			prefix: "prefix/prefix2/obj.ext",
+			want:   "prefix/prefix2/",
+		},
+		{
+			name:   "folderfolder",
+			prefix: "prefix/prefix2/",
+			want:   "prefix/prefix2/",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := baseDirFromPrefix(tt.prefix); got != tt.want {
+				t.Errorf("baseDirFromPrefix() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
+
+func Test_metacache_canBeReplacedBy(t *testing.T) {
+	testAgainst := metacache{
+		id:           "case-1-modified",
+		bucket:       "bucket",
+		root:         "folder/prefix",
+		recursive:    true,
+		status:       scanStateSuccess,
+		fileNotFound: false,
+		error:        "",
+		started:      metaCacheTestsetTimestamp.Add(time.Minute),
+		ended:        metaCacheTestsetTimestamp.Add(2 * time.Minute),
+		lastUpdate:   metaCacheTestsetTimestamp.Add(2 * time.Minute),
+		lastHandout:  metaCacheTestsetTimestamp.Add(time.Minute),
+		startedCycle: 10,
+		endedCycle:   10,
+		dataVersion:  metacacheStreamVersion,
+	}
+	wantResults := []bool{0: true, 1: true, 2: true, 3: true, 4: true, 5: false, 6: true, 7: false, 8: false}
+
+	for i, tt := range metaCacheTestset {
+		t.Run(tt.id, func(t *testing.T) {
+			var want bool
+			if i >= len(wantResults) {
+				t.Logf("no expected result for test #%d", i)
+			} else {
+				want = wantResults[i]
+			}
+			// Add an hour, otherwise it will never be replaced.
+			// We operated on a copy.
+			tt.lastHandout = tt.lastHandout.Add(-2 * time.Hour)
+			tt.lastUpdate = tt.lastHandout.Add(-2 * time.Hour)
+			got := tt.canBeReplacedBy(&testAgainst)
+			if got != want {
+				t.Errorf("#%d: want %v, got %v", i, want, got)
+			}
+		})
+	}
+}
+
+func Test_metacache_finished(t *testing.T) {
+	wantResults := []bool{0: true, 1: true, 2: true, 3: true, 4: false, 5: true, 6: true, 7: false, 8: true}
+
+	for i, tt := range metaCacheTestset {
+		t.Run(tt.id, func(t *testing.T) {
+			var want bool
+			if i >= len(wantResults) {
+				t.Logf("no expected result for test #%d", i)
+			} else {
+				want = wantResults[i]
+			}
+
+			got := tt.finished()
+			if got != want {
+				t.Errorf("#%d: want %v, got %v", i, want, got)
+			}
+		})
+	}
+}
+
+func Test_metacache_worthKeeping(t *testing.T) {
+	wantResults := []bool{0: true, 1: true, 2: true, 3: false, 4: false, 5: true, 6: false, 7: false, 8: false}
+
+	for i, tt := range metaCacheTestset {
+		t.Run(tt.id, func(t *testing.T) {
+			var want bool
+			if i >= len(wantResults) {
+				t.Logf("no expected result for test #%d", i)
+			} else {
+				want = wantResults[i]
+			}
+
+			got := tt.worthKeeping(7 + dataUsageUpdateDirCycles)
+			if got != want {
+				t.Errorf("#%d: want %v, got %v", i, want, got)
+			}
+		})
+	}
+}