fix: infinite loop in cleanupStaleUploads of encrypted MPUs (#10845)

fixes #10588

.github/workflows/codeql.yml

@@ -1,51 +0,0 @@
-name: "Code scanning - action"
-
-on:
-  push:
-  pull_request:
-  schedule:
-    - cron: '0 19 * * 0'
-
-jobs:
-  CodeQL-Build:
-
-    # CodeQL runs on ubuntu-latest and windows-latest
-    runs-on: ubuntu-latest
-
-    steps:
-    - name: Checkout repository
-      uses: actions/checkout@v2
-      with:
-        # We must fetch at least the immediate parents so that if this is
-        # a pull request then we can checkout the head.
-        fetch-depth: 2
-
-    # If this run was triggered by a pull request event, then checkout
-    # the head of the pull request instead of the merge commit.
-    - run: git checkout HEAD^2
-      if: ${{ github.event_name == 'pull_request' }}
-      
-    # Initializes the CodeQL tools for scanning.
-    - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
-      with:
-         languages: go, javascript
-
-    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-    # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
-
-    # ℹ️ Command-line programs to run using the OS shell.
-    # 📚 https://git.io/JvXDl
-
-    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-    #    and modify them (or add more) to build your code if your project
-    #    uses a compiled language
-
-    #- run: |
-    #   make bootstrap
-    #   make release
-
-    - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.14-alpine as builder
+FROM golang:1.15-alpine as builder
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
@@ -13,11 +13,11 @@ RUN  \
 
 FROM alpine:3.12
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 EXPOSE 9000
 

Dockerfile.arm.release

@@ -5,11 +5,11 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY --from=qemu /usr/bin/qemu-arm-static /usr/bin
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 RUN \
      echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \

Dockerfile.arm64.release

@@ -5,11 +5,11 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY --from=qemu /usr/bin/qemu-aarch64-static /usr/bin
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 RUN \
      echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \

Dockerfile.ppc64le.release

@@ -5,11 +5,11 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY --from=qemu /usr/bin/qemu-ppc64le-static /usr/bin
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 RUN \
      echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \

Dockerfile.release

@@ -2,11 +2,11 @@ FROM alpine:3.12
 
 LABEL maintainer="MinIO Inc <dev@min.io>"
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 RUN \
      echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \

Dockerfile.s390x.release

@@ -5,11 +5,11 @@ LABEL maintainer="MinIO Inc <dev@min.io>"
 
 COPY --from=qemu /usr/bin/qemu-s390x-static /usr/bin
 
-ENV MINIO_UPDATE off
 ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_SECRET_KEY_FILE=secret_key \
     MINIO_KMS_MASTER_KEY_FILE=kms_master_key \
-    MINIO_SSE_MASTER_KEY_FILE=sse_master_key
+    MINIO_SSE_MASTER_KEY_FILE=sse_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
 
 RUN \
      echo "http://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && \

README.md

@@ -88,7 +88,7 @@ service minio start
 ```
 
 ## Install from Source
-Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.14](https://golang.org/dl/#stable)
+Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.15](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio

cmd/admin-handlers-config-kv.go

@@ -42,7 +42,7 @@ import (
 
 func validateAdminReqConfigKV(ctx context.Context, w http.ResponseWriter, r *http.Request) (auth.Credentials, ObjectLayer) {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return auth.Credentials{}, nil

cmd/admin-handlers-users.go

@@ -35,8 +35,8 @@ func validateAdminUsersReq(ctx context.Context, w http.ResponseWriter, r *http.R
 	var adminAPIErr APIErrorCode
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return nil, cred
 	}
@@ -386,8 +386,8 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	defer logger.AuditLog(w, r, "AddServiceAccount", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -465,8 +465,8 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 	defer logger.AuditLog(w, r, "ListServiceAccounts", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -520,8 +520,8 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	defer logger.AuditLog(w, r, "DeleteServiceAccount", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
@@ -579,8 +579,8 @@ func (a adminAPIHandlers) AccountUsageInfoHandler(w http.ResponseWriter, r *http
 	defer logger.AuditLog(w, r, "AccountUsageInfo", mustGetClaimsFromToken(r))
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
-	if objectAPI == nil || globalNotificationSys == nil || globalIAMSys == nil {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}

cmd/admin-handlers.go

@@ -35,7 +35,6 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
-
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -86,8 +85,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	if globalInplaceUpdateDisabled {
-		// if MINIO_UPDATE=off - inplace update is disabled, mostly
-		// in containers.
+		// if MINIO_UPDATE=off - inplace update is disabled, mostly in containers.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 		return
 	}
@@ -349,7 +347,7 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
-func lriToLockEntry(l lockRequesterInfo, resource, server string, rquorum, wquorum int) *madmin.LockEntry {
+func lriToLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
 	entry := &madmin.LockEntry{
 		Timestamp:  l.Timestamp,
 		Resource:   resource,
@@ -357,18 +355,17 @@ func lriToLockEntry(l lockRequesterInfo, resource, server string, rquorum, wquor
 		Source:     l.Source,
 		Owner:      l.Owner,
 		ID:         l.UID,
+		Quorum:     l.Quorum,
 	}
 	if l.Writer {
 		entry.Type = "WRITE"
-		entry.Quorum = wquorum
 	} else {
 		entry.Type = "READ"
-		entry.Quorum = rquorum
 	}
 	return entry
 }
 
-func topLockEntries(peerLocks []*PeerLocks, count int, rquorum, wquorum int, stale bool) madmin.LockEntries {
+func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries {
 	entryMap := make(map[string]*madmin.LockEntry)
 	for _, peerLock := range peerLocks {
 		if peerLock == nil {
@@ -380,7 +377,7 @@ func topLockEntries(peerLocks []*PeerLocks, count int, rquorum, wquorum int, sta
 					if val, ok := entryMap[lockReqInfo.UID]; ok {
 						val.ServerList = append(val.ServerList, peerLock.Addr)
 					} else {
-						entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr, rquorum, wquorum)
+						entryMap[lockReqInfo.UID] = lriToLockEntry(lockReqInfo, k, peerLock.Addr)
 					}
 				}
 			}
@@ -388,9 +385,6 @@ func topLockEntries(peerLocks []*PeerLocks, count int, rquorum, wquorum int, sta
 	}
 	var lockEntries madmin.LockEntries
 	for _, v := range entryMap {
-		if len(lockEntries) == count {
-			break
-		}
 		if stale {
 			lockEntries = append(lockEntries, *v)
 			continue
@@ -433,12 +427,13 @@ func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request
 
 	peerLocks := globalNotificationSys.GetLocks(ctx, r)
 
-	rquorum := getReadQuorum(objectAPI.SetDriveCount())
-	wquorum := getWriteQuorum(objectAPI.SetDriveCount())
+	topLocks := topLockEntries(peerLocks, stale)
 
-	topLocks := topLockEntries(peerLocks, count, rquorum, wquorum, stale)
+	// Marshal API response upto requested count.
+	if len(topLocks) > count && count > 0 {
+		topLocks = topLocks[:count]
+	}
 
-	// Marshal API response
 	jsonBytes, err := json.Marshal(topLocks)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -898,7 +893,7 @@ func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Reques
 	var cred auth.Credentials
 	var adminAPIErr APIErrorCode
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return nil, cred
@@ -1040,7 +1035,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	// Use buffered channel to take care of burst sends or slow w.Write()
 	traceCh := make(chan interface{}, 4000)
 
-	peers := newPeerRestClients(globalEndpoints)
+	peers, _ := newPeerRestClients(globalEndpoints)
 
 	globalHTTPTrace.Subscribe(traceCh, ctx.Done(), func(entry interface{}) bool {
 		return mustTrace(entry, trcAll, trcErr)
@@ -1108,7 +1103,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 	logCh := make(chan interface{}, 4000)
 
-	peers := newPeerRestClients(globalEndpoints)
+	peers, _ := newPeerRestClients(globalEndpoints)
 
 	globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
 
@@ -1289,8 +1284,8 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	deadlinedCtx, cancel := context.WithTimeout(ctx, deadline)
 	defer cancel()
 
-	nsLock := objectAPI.NewNSLock(ctx, minioMetaBucket, "obd-in-progress")
-	if err := nsLock.GetLock(newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "obd-in-progress")
+	if err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline)); err != nil { // returns a locked lock
 		errResp(err)
 		return
 	}
@@ -1299,12 +1294,6 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 	go func() {
 		defer close(obdInfoCh)
 
-		if log := query.Get("log"); log == "true" {
-			obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, getLocalLogOBD(deadlinedCtx, r))
-			obdInfo.Logging.ServersLog = append(obdInfo.Logging.ServersLog, globalNotificationSys.LogOBDInfo(deadlinedCtx)...)
-			partialWrite(obdInfo)
-		}
-
 		if cpu := query.Get("syscpu"); cpu == "true" {
 			cpuInfo := getLocalCPUOBDInfo(deadlinedCtx, r)
 
@@ -1424,6 +1413,31 @@ func (a adminAPIHandlers) OBDInfoHandler(w http.ResponseWriter, r *http.Request)
 
 }
 
+// BandwidthMonitorHandler - GET /minio/admin/v3/bandwidth
+// ----------
+// Get bandwidth consumption information
+func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "BandwidthMonitor")
+
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuthType(ctx, r, iampolicy.BandwidthMonitorAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+
+	setEventStreamHeaders(w)
+	bucketsRequestedString := r.URL.Query().Get("buckets")
+	bucketsRequested := strings.Split(bucketsRequestedString, ",")
+	consolidatedReport := globalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...)
+	enc := json.NewEncoder(w)
+	err := enc.Encode(consolidatedReport)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+	}
+	w.(http.Flusher).Flush()
+}
+
 // ServerInfoHandler - GET /minio/admin/v3/info
 // ----------
 // Get server information
@@ -1489,11 +1503,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	mode := "safemode"
-	if newObjectLayerFn() != nil {
-		mode = "online"
-	}
-
+	mode := "online"
 	server := getLocalServerProperty(globalEndpoints, r)
 	servers := globalNotificationSys.ServerInfo()
 	servers = append(servers, server)

cmd/admin-handlers_test.go

@@ -105,8 +105,8 @@ func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error)
 	}
 
 	globalPolicySys = NewPolicySys()
-	objLayer := &erasureZones{zones: make([]*erasureSets, 1)}
-	objLayer.zones[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
+	objLayer := &erasureServerSets{serverSets: make([]*erasureSets, 1)}
+	objLayer.serverSets[0], err = newErasureSets(ctx, endpoints, storageDisks, format)
 	if err != nil {
 		return nil, nil, err
 	}

cmd/admin-heal-ops.go

@@ -808,7 +808,7 @@ func (h *healSequence) traverseAndHeal() {
 func (h *healSequence) healMinioSysMeta(metaPrefix string) func() error {
 	return func() error {
 		// Get current object layer instance.
-		objectAPI := newObjectLayerWithoutSafeModeFn()
+		objectAPI := newObjectLayerFn()
 		if objectAPI == nil {
 			return errServerNotInitialized
 		}
@@ -844,7 +844,7 @@ func (h *healSequence) healDiskFormat() error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -864,7 +864,7 @@ func (h *healSequence) healBuckets(bucketsOnly bool) error {
 	}
 
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -886,7 +886,7 @@ func (h *healSequence) healBuckets(bucketsOnly bool) error {
 // healBucket - traverses and heals given bucket
 func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}
@@ -932,7 +932,7 @@ func (h *healSequence) healBucket(bucket string, bucketsOnly bool) error {
 // healObject - heal the given object and record result
 func (h *healSequence) healObject(bucket, object, versionID string) error {
 	// Get current object layer instance.
-	objectAPI := newObjectLayerWithoutSafeModeFn()
+	objectAPI := newObjectLayerFn()
 	if objectAPI == nil {
 		return errServerNotInitialized
 	}

cmd/admin-router.go

@@ -180,19 +180,19 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 				// PutBucketQuotaConfig
 				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
 					httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
-			}
-			// Bucket replication operations
-			// GetBucketTargetHandler
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
-				httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
-			// SetRemoteTargetHandler
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
-			// SetRemoteTargetHandler
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
-		}
 
+				// Bucket replication operations
+				// GetBucketTargetHandler
+				adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
+					httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+				// SetRemoteTargetHandler
+				adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
+					httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
+				// SetRemoteTargetHandler
+				adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
+					httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
+			}
+		}
 		// -- Top APIs --
 		// Top locks
 		if globalIsDistErasure {
@@ -214,10 +214,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 			// -- OBD API --
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
 				HandlerFunc(httpTraceHdrs(adminAPI.OBDInfoHandler))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
+				HandlerFunc(httpTraceHdrs(adminAPI.BandwidthMonitorHandler))
 		}
 	}
 
 	// If none of the routes match add default error handler routes
 	adminRouter.NotFoundHandler = httpTraceAll(errorResponseHandler)
-	adminRouter.MethodNotAllowedHandler = httpTraceAll(errorResponseHandler)
+	adminRouter.MethodNotAllowedHandler = httpTraceAll(methodNotAllowedHandler("Admin"))
 }

cmd/admin-server-info.go

@@ -24,13 +24,13 @@ import (
 
 // getLocalServerProperty - returns madmin.ServerProperties for only the
 // local endpoints from given list of endpoints
-func getLocalServerProperty(endpointZones EndpointZones, r *http.Request) madmin.ServerProperties {
+func getLocalServerProperty(endpointServerSets EndpointServerSets, r *http.Request) madmin.ServerProperties {
 	addr := r.Host
 	if globalIsDistErasure {
-		addr = GetLocalPeer(endpointZones)
+		addr = GetLocalPeer(endpointServerSets)
 	}
 	network := make(map[string]string)
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			nodeName := endpoint.Host
 			if nodeName == "" {

cmd/api-errors.go

@@ -106,16 +106,17 @@ const (
 	ErrNoSuchCORSConfiguration
 	ErrNoSuchWebsiteConfiguration
 	ErrReplicationConfigurationNotFoundError
-	ErrReplicationDestinationNotFoundError
+	ErrRemoteDestinationNotFoundError
 	ErrReplicationDestinationMissingLock
-	ErrReplicationTargetNotFoundError
+	ErrRemoteTargetNotFoundError
 	ErrReplicationRemoteConnectionError
 	ErrBucketRemoteIdenticalToSource
 	ErrBucketRemoteAlreadyExists
+	ErrBucketRemoteLabelInUse
 	ErrBucketRemoteArnTypeInvalid
 	ErrBucketRemoteArnInvalid
 	ErrBucketRemoteRemoveDisallowed
-	ErrReplicationTargetNotVersionedError
+	ErrRemoteTargetNotVersionedError
 	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
@@ -232,6 +233,7 @@ const (
 	ErrInvalidResourceName
 	ErrServerNotInitialized
 	ErrOperationTimedOut
+	ErrClientDisconnected
 	ErrOperationMaxedOut
 	ErrInvalidRequest
 	// MinIO storage class error codes
@@ -809,9 +811,9 @@ var errorCodes = errorCodeMap{
 		Description:    "The replication configuration was not found",
 		HTTPStatusCode: http.StatusNotFound,
 	},
-	ErrReplicationDestinationNotFoundError: {
-		Code:           "ReplicationDestinationNotFoundError",
-		Description:    "The replication destination bucket does not exist",
+	ErrRemoteDestinationNotFoundError: {
+		Code:           "RemoteDestinationNotFoundError",
+		Description:    "The remote destination bucket does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrReplicationDestinationMissingLock: {
@@ -819,29 +821,34 @@ var errorCodes = errorCodeMap{
 		Description:    "The replication destination bucket does not have object locking enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrReplicationTargetNotFoundError: {
-		Code:           "XminioAdminReplicationTargetNotFoundError",
-		Description:    "The replication target does not exist",
+	ErrRemoteTargetNotFoundError: {
+		Code:           "XMinioAdminRemoteTargetNotFoundError",
+		Description:    "The remote target does not exist",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrReplicationRemoteConnectionError: {
-		Code:           "XminioAdminReplicationRemoteConnectionError",
+		Code:           "XMinioAdminReplicationRemoteConnectionError",
 		Description:    "Remote service endpoint or target bucket not available",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrBucketRemoteIdenticalToSource: {
-		Code:           "XminioAdminRemoteIdenticalToSource",
+		Code:           "XMinioAdminRemoteIdenticalToSource",
 		Description:    "The remote target cannot be identical to source",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBucketRemoteAlreadyExists: {
-		Code:           "XminioAdminBucketRemoteAlreadyExists",
+		Code:           "XMinioAdminBucketRemoteAlreadyExists",
 		Description:    "The remote target already exists",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrBucketRemoteLabelInUse: {
+		Code:           "XMinioAdminBucketRemoteLabelInUse",
+		Description:    "The remote target with this label already exists",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrBucketRemoteRemoveDisallowed: {
 		Code:           "XMinioAdminRemoteRemoveDisallowed",
-		Description:    "Replication configuration exists with this ARN.",
+		Description:    "This ARN is in use by an existing configuration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBucketRemoteArnTypeInvalid: {
@@ -854,9 +861,9 @@ var errorCodes = errorCodeMap{
 		Description:    "The bucket remote ARN does not have correct format",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrReplicationTargetNotVersionedError: {
-		Code:           "ReplicationTargetNotVersionedError",
-		Description:    "The replication target does not have versioning enabled",
+	ErrRemoteTargetNotVersionedError: {
+		Code:           "RemoteTargetNotVersionedError",
+		Description:    "The remote target does not have versioning enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrReplicationSourceNotVersionedError: {
@@ -1210,6 +1217,11 @@ var errorCodes = errorCodeMap{
 		Description:    "A timeout occurred while trying to lock a resource, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrClientDisconnected: {
+		Code:           "ClientDisconnected",
+		Description:    "Client disconnected before response was ready",
+		HTTPStatusCode: 499, // No official code, use nginx value.
+	},
 	ErrOperationMaxedOut: {
 		Code:           "SlowDown",
 		Description:    "A timeout exceeded while waiting to proceed with the request, please reduce your request rate",
@@ -1736,6 +1748,16 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
+	// Only return ErrClientDisconnected if the provided context is actually canceled.
+	// This way downstream context.Canceled will still report ErrOperationTimedOut
+	select {
+	case <-ctx.Done():
+		if ctx.Err() == context.Canceled {
+			return ErrClientDisconnected
+		}
+	default:
+	}
+
 	switch err {
 	case errInvalidArgument:
 		apiErr = ErrAdminInvalidArgument
@@ -1906,24 +1928,26 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminNoSuchQuotaConfiguration
 	case BucketReplicationConfigNotFound:
 		apiErr = ErrReplicationConfigurationNotFoundError
-	case BucketReplicationDestinationNotFound:
-		apiErr = ErrReplicationDestinationNotFoundError
+	case BucketRemoteDestinationNotFound:
+		apiErr = ErrRemoteDestinationNotFoundError
 	case BucketReplicationDestinationMissingLock:
 		apiErr = ErrReplicationDestinationMissingLock
 	case BucketRemoteTargetNotFound:
-		apiErr = ErrReplicationTargetNotFoundError
+		apiErr = ErrRemoteTargetNotFoundError
 	case BucketRemoteConnectionErr:
 		apiErr = ErrReplicationRemoteConnectionError
 	case BucketRemoteAlreadyExists:
 		apiErr = ErrBucketRemoteAlreadyExists
+	case BucketRemoteLabelInUse:
+		apiErr = ErrBucketRemoteLabelInUse
 	case BucketRemoteArnTypeInvalid:
 		apiErr = ErrBucketRemoteArnTypeInvalid
 	case BucketRemoteArnInvalid:
 		apiErr = ErrBucketRemoteArnInvalid
 	case BucketRemoteRemoveDisallowed:
 		apiErr = ErrBucketRemoteRemoveDisallowed
-	case BucketReplicationTargetNotVersioned:
-		apiErr = ErrReplicationTargetNotVersionedError
+	case BucketRemoteTargetNotVersioned:
+		apiErr = ErrRemoteTargetNotVersionedError
 	case BucketReplicationSourceNotVersioned:
 		apiErr = ErrReplicationSourceNotVersionedError
 	case BucketQuotaExceeded:

cmd/api-headers.go

@@ -115,10 +115,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 
 	// Set tag count if object has tags
-	tags, _ := url.ParseQuery(objInfo.UserTags)
-	tagCount := len(tags)
-	if tagCount > 0 {
-		w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tagCount)}
+	if len(objInfo.UserTags) > 0 {
+		tags, _ := url.ParseQuery(objInfo.UserTags)
+		if len(tags) > 0 {
+			w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(len(tags))}
+		}
 	}
 
 	// Set all other user defined metadata.

cmd/api-response.go

@@ -35,11 +35,11 @@ import (
 
 const (
 	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
-	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
-	maxDeleteList     = 10000                      // Limit number of objects deleted in a delete call.
-	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
-	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
+	iso8601TimeFormat = "2006-01-02T15:04:05.000Z"                     // Reply date format with nanosecond precision.
+	maxObjectList     = metacacheBlockSize - (metacacheBlockSize / 10) // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxDeleteList     = 10000                                          // Limit number of objects deleted in a delete call.
+	maxUploadsList    = 10000                                          // Limit number of uploads in a listUploadsResponse.
+	maxPartsList      = 10000                                          // Limit number of parts in a listPartsResponse.
 )
 
 // LocationResponse - format for location response.
@@ -703,10 +703,6 @@ func generateMultiDeleteResponse(quiet bool, deletedObjects []DeletedObject, err
 }
 
 func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) {
-	if newObjectLayerFn() == nil {
-		// Server still in safe mode.
-		w.Header().Set(xhttp.MinIOServerStatus, "safemode")
-	}
 	setCommonHeaders(w)
 	if mType != mimeNone {
 		w.Header().Set(xhttp.ContentType, string(mType))
@@ -773,10 +769,6 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 		// The request is from browser and also if browser
 		// is enabled we need to redirect.
 		if browser && globalBrowserEnabled {
-			if newObjectLayerFn() == nil {
-				// server still in safe mode.
-				w.Header().Set(xhttp.MinIOServerStatus, "safemode")
-			}
 			w.Header().Set(xhttp.Location, minioReservedBucketPath+reqURL.Path)
 			w.WriteHeader(http.StatusTemporaryRedirect)
 			return

cmd/api-router.go

@@ -32,31 +32,24 @@ func newHTTPServerFn() *xhttp.Server {
 	return globalHTTPServer
 }
 
-func newObjectLayerWithoutSafeModeFn() ObjectLayer {
-	globalObjLayerMutex.Lock()
-	defer globalObjLayerMutex.Unlock()
-	return globalObjectAPI
-}
-
 func newObjectLayerFn() ObjectLayer {
 	globalObjLayerMutex.Lock()
 	defer globalObjLayerMutex.Unlock()
-	if globalSafeMode {
-		return nil
-	}
 	return globalObjectAPI
 }
 
 func newCachedObjectLayerFn() CacheObjectLayer {
 	globalObjLayerMutex.Lock()
 	defer globalObjLayerMutex.Unlock()
-
-	if globalSafeMode {
-		return nil
-	}
 	return globalCacheObjectAPI
 }
 
+func setObjectLayer(o ObjectLayer) {
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = o
+	globalObjLayerMutex.Unlock()
+}
+
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
@@ -333,7 +326,7 @@ func registerAPIRouter(router *mux.Router) {
 
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))
-	apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(errorResponseHandler))
+	apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(methodNotAllowedHandler("S3")))
 
 }
 

cmd/auth-handler.go

@@ -333,8 +333,12 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// Populate payload again to handle it in HTTP handler.
 		r.Body = ioutil.NopCloser(bytes.NewReader(payload))
 	}
+	if cred.AccessKey != "" {
+		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
+	}
 
-	if cred.AccessKey == "" {
+	if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
+		// Anonymous checks are not meant for ListBuckets action
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Action:          action,
@@ -378,12 +382,13 @@ func checkRequestAuthTypeToAccessKey(ctx context.Context, r *http.Request, actio
 		// Request is allowed return the appropriate access key.
 		return cred.AccessKey, owner, ErrNone
 	}
+
 	if action == policy.ListBucketVersionsAction {
 		// In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission
 		// verify as a fallback.
 		if globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          iampolicy.Action(policy.ListBucketAction),
+			Action:          iampolicy.ListBucketAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			ObjectName:      objectName,
@@ -554,7 +559,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		if retMode == objectlock.RetGovernance && byPassSet {
 			byPassSet = globalPolicySys.IsAllowed(policy.Args{
 				AccountName:     cred.AccessKey,
-				Action:          policy.Action(policy.BypassGovernanceRetentionAction),
+				Action:          policy.BypassGovernanceRetentionAction,
 				BucketName:      bucketName,
 				ConditionValues: conditions,
 				IsOwner:         false,
@@ -563,7 +568,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		}
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          policy.Action(policy.PutObjectRetentionAction),
+			Action:          policy.PutObjectRetentionAction,
 			BucketName:      bucketName,
 			ConditionValues: conditions,
 			IsOwner:         false,
@@ -586,7 +591,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 	if retMode == objectlock.RetGovernance && byPassSet {
 		byPassSet = globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
-			Action:          policy.BypassGovernanceRetentionAction,
+			Action:          iampolicy.BypassGovernanceRetentionAction,
 			BucketName:      bucketName,
 			ObjectName:      objectName,
 			ConditionValues: conditions,
@@ -596,7 +601,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 	}
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
-		Action:          policy.PutObjectRetentionAction,
+		Action:          iampolicy.PutObjectRetentionAction,
 		BucketName:      bucketName,
 		ConditionValues: conditions,
 		ObjectName:      objectName,
@@ -614,7 +619,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 // isPutActionAllowed - check if PUT operation is allowed on the resource, this
 // call verifies bucket policies and IAM policies, supports multi user
 // checks etc.
-func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
+func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action iampolicy.Action) (s3Err APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
 	switch atype {
@@ -635,6 +640,10 @@ func isPutActionAllowed(atype authType, bucketName, objectName string, r *http.R
 		return s3Err
 	}
 
+	if cred.AccessKey != "" {
+		logger.GetReqInfo(ctx).AccessKey = cred.AccessKey
+	}
+
 	// Do not check for PutObjectRetentionAction permission,
 	// if mode and retain until date are not set.
 	// Can happen when bucket has default lock config set

cmd/background-newdisks-heal-ops.go

@@ -39,7 +39,7 @@ type healingTracker struct {
 }
 
 func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
-	z, ok := objAPI.(*erasureZones)
+	z, ok := objAPI.(*erasureServerSets)
 	if !ok {
 		return
 	}
@@ -107,7 +107,7 @@ func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
 // monitorLocalDisksAndHeal - ensures that detected new disks are healed
 //  1. Only the concerned erasure set will be listed and healed
 //  2. Only the node hosting the disk is responsible to perform the heal
-func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healSequence) {
+func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerSets, bgSeq *healSequence) {
 	// Perform automatic disk healing when a disk is replaced locally.
 	for {
 		select {
@@ -129,8 +129,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 				logger.Info(fmt.Sprintf("Found drives to heal %d, proceeding to heal content...",
 					len(healDisks)))
 
-				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.zones))
-				for i := range z.zones {
+				erasureSetInZoneDisksToHeal = make([]map[int][]StorageAPI, len(z.serverSets))
+				for i := range z.serverSets {
 					erasureSetInZoneDisksToHeal[i] = map[int][]StorageAPI{}
 				}
 			}
@@ -149,7 +149,10 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 				}
 
 				// Calculate the set index where the current endpoint belongs
-				setIndex, _, err := findDiskIndex(z.zones[zoneIdx].format, format)
+				z.serverSets[zoneIdx].erasureDisksMu.RLock()
+				// Protect reading reference format.
+				setIndex, _, err := findDiskIndex(z.serverSets[zoneIdx].format, format)
+				z.serverSets[zoneIdx].erasureDisksMu.RUnlock()
 				if err != nil {
 					printEndpointError(endpoint, err, false)
 					continue
@@ -164,7 +167,7 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 					for _, disk := range disks {
 						logger.Info("Healing disk '%s' on %s zone", disk, humanize.Ordinal(i+1))
 
-						lbDisks := z.zones[i].sets[setIndex].getLoadBalancedNDisks(z.zones[i].listTolerancePerSet)
+						lbDisks := z.serverSets[i].sets[setIndex].getOnlineDisks()
 						if err := healErasureSet(ctx, setIndex, buckets, lbDisks); err != nil {
 							logger.LogIf(ctx, err)
 							continue
@@ -172,8 +175,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureZones, bgSeq *healS
 
 						logger.Info("Healing disk '%s' on %s zone complete", disk, humanize.Ordinal(i+1))
 
-						if err := disk.DeleteFile(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
-							healingTrackerFilename); err != nil {
+						if err := disk.Delete(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix),
+							healingTrackerFilename, false); err != nil && !errors.Is(err, errFileNotFound) {
 							logger.LogIf(ctx, err)
 							continue
 						}

cmd/bootstrap-peer-server.go

@@ -18,9 +18,7 @@ package cmd
 
 import (
 	"context"
-	"crypto/tls"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -54,7 +52,7 @@ type bootstrapRESTServer struct{}
 type ServerSystemConfig struct {
 	MinioPlatform  string
 	MinioRuntime   string
-	MinioEndpoints EndpointZones
+	MinioEndpoints EndpointServerSets
 }
 
 // Diff - returns error on first difference found in two configs.
@@ -161,9 +159,9 @@ func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg ServerSyst
 	return srcCfg.Diff(recvCfg)
 }
 
-func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones) error {
+func verifyServerSystemConfig(ctx context.Context, endpointServerSets EndpointServerSets) error {
 	srcCfg := getServerSystemCfg()
-	clnts := newBootstrapRESTClients(endpointZones)
+	clnts := newBootstrapRESTClients(endpointServerSets)
 	var onlineServers int
 	var offlineEndpoints []string
 	var retries int
@@ -198,10 +196,10 @@ func verifyServerSystemConfig(ctx context.Context, endpointZones EndpointZones)
 	return nil
 }
 
-func newBootstrapRESTClients(endpointZones EndpointZones) []*bootstrapRESTClient {
+func newBootstrapRESTClients(endpointServerSets EndpointServerSets) []*bootstrapRESTClient {
 	seenHosts := set.NewStringSet()
 	var clnts []*bootstrapRESTClient
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			if seenHosts.Contains(endpoint.Host) {
 				continue
@@ -225,26 +223,8 @@ func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
 		Path:   bootstrapRESTPath,
 	}
 
-	var tlsConfig *tls.Config
-	if globalIsSSL {
-		tlsConfig = &tls.Config{
-			ServerName: endpoint.Hostname(),
-			RootCAs:    globalRootCAs,
-		}
-	}
-
-	trFn := newInternodeHTTPTransport(tlsConfig, rest.DefaultTimeout)
-	restClient := rest.NewClient(serverURL, trFn, newAuthToken)
-	restClient.HealthCheckFn = func() bool {
-		ctx, cancel := context.WithTimeout(GlobalContext, restClient.HealthCheckTimeout)
-		// Instantiate a new rest client for healthcheck
-		// to avoid recursive healthCheckFn()
-		respBody, err := rest.NewClient(serverURL, trFn, newAuthToken).Call(ctx, bootstrapRESTMethodHealth, nil, nil, -1)
-		xhttp.DrainBody(respBody)
-		cancel()
-		var ne *rest.NetworkError
-		return !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &ne)
-	}
+	restClient := rest.NewClient(serverURL, globalInternodeTransport, newAuthToken)
+	restClient.HealthCheckFn = nil
 
 	return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient}
 }

cmd/bucket-encryption.go

@@ -34,7 +34,7 @@ func NewBucketSSEConfigSys() *BucketSSEConfigSys {
 // Get - gets bucket encryption config for the given bucket.
 func (sys *BucketSSEConfigSys) Get(bucket string) (*bucketsse.BucketSSEConfig, error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}

cmd/bucket-handlers.go

@@ -383,6 +383,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		return
 	}
 
+	// Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo()
+	// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
+	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
+
 	// Before proceeding validate if bucket exists.
 	_, err := objectAPI.GetBucketInfo(ctx, bucket)
 	if err != nil {
@@ -457,7 +461,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	deletedObjects := make([]DeletedObject, len(deleteObjects.Objects))
 	for i := range errs {
-		dindex := objectsToDelete[deleteList[i]]
+		dindex := objectsToDelete[ObjectToDelete{
+			ObjectName: dObjects[i].ObjectName,
+			VersionID:  dObjects[i].VersionID,
+		}]
 		apiErr := toAPIError(ctx, errs[i])
 		if apiErr.Code == "" || apiErr.Code == "NoSuchKey" || apiErr.Code == "InvalidArgument" {
 			deletedObjects[dindex] = dObjects[i]
@@ -487,19 +494,21 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 
 	// Notify deleted event for objects.
 	for _, dobj := range deletedObjects {
+		eventName := event.ObjectRemovedDelete
+
 		objInfo := ObjectInfo{
 			Name:      dobj.ObjectName,
 			VersionID: dobj.VersionID,
 		}
+
 		if dobj.DeleteMarker {
-			objInfo = ObjectInfo{
-				Name:         dobj.ObjectName,
-				DeleteMarker: dobj.DeleteMarker,
-				VersionID:    dobj.DeleteMarkerVersionID,
-			}
+			objInfo.DeleteMarker = dobj.DeleteMarker
+			objInfo.VersionID = dobj.DeleteMarkerVersionID
+			eventName = event.ObjectRemovedDeleteMarkerCreated
 		}
+
 		sendEvent(eventArgs{
-			EventName:    event.ObjectRemovedDelete,
+			EventName:    eventName,
 			BucketName:   bucket,
 			Object:       objInfo,
 			ReqParams:    extractReqParams(r),

cmd/bucket-handlers_test.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"encoding/xml"
-	"fmt"
 	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
@@ -837,7 +836,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		// Verify whether the bucket obtained object is same as the one created.
 		if testCase.expectedContent != nil && !bytes.Equal(testCase.expectedContent, actualContent) {
-			fmt.Println(string(testCase.expectedContent), string(actualContent))
+			t.Log(string(testCase.expectedContent), string(actualContent))
 			t.Errorf("Test %d : MinIO %s: Object content differs from expected value.", i+1, instanceType)
 		}
 	}

cmd/bucket-lifecycle.go

@@ -31,7 +31,7 @@ type LifecycleSys struct{}
 // Get - gets lifecycle config associated to a given bucket name.
 func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}

cmd/bucket-listobjects-handlers.go

@@ -18,7 +18,6 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"net/http"
 	"strconv"
 	"strings"
@@ -114,15 +113,6 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Forward the request using Source IP or bucket
-	forwardStr := handlers.GetSourceIPFromHeaders(r)
-	if forwardStr == "" {
-		forwardStr = bucket
-	}
-	if proxyRequestByStringHash(ctx, w, r, forwardStr) {
-		return
-	}
-
 	listObjectVersions := objectAPI.ListObjectVersions
 
 	// Inititate a list object versions operation based on the input params.
@@ -145,7 +135,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 // ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 10000)
-// of the objects in a bucket. You can use the request parame<ters as selection
+// of the objects in a bucket. You can use the request parameters as selection
 // criteria to return a subset of the objects in a bucket.
 //
 // NOTE: It is recommended that this API to be used for application development.
@@ -185,13 +175,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		return
 	}
 
-	// Analyze continuation token and route the request accordingly
-	var success bool
-	token, success = proxyRequestByToken(ctx, w, r, token)
-	if success {
-		return
-	}
-
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -207,9 +190,6 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
-		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
-	}
 
 	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
@@ -262,13 +242,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Analyze continuation token and route the request accordingly
-	var success bool
-	token, success = proxyRequestByToken(ctx, w, r, token)
-	if success {
-		return
-	}
-
 	listObjectsV2 := objectAPI.ListObjectsV2
 
 	// Inititate a list objects operation based on the input params.
@@ -282,13 +255,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 
 	concurrentDecryptETag(ctx, listObjectsV2Info.Objects)
 
-	// The next continuation token has id@node_index format to optimize paginated listing
-	nextContinuationToken := listObjectsV2Info.NextContinuationToken
-	if nextContinuationToken != "" && listObjectsV2Info.IsTruncated {
-		nextContinuationToken = fmt.Sprintf("%s@%d", listObjectsV2Info.NextContinuationToken, getLocalNodeIndex())
-	}
-
-	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
+	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
 		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
 
@@ -296,18 +263,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	writeSuccessResponseXML(w, encodeResponse(response))
 }
 
-func getLocalNodeIndex() int {
-	if len(globalProxyEndpoints) == 0 {
-		return -1
-	}
-	for i, ep := range globalProxyEndpoints {
-		if ep.IsLocal {
-			return i
-		}
-	}
-	return -1
-}
-
 func parseRequestToken(token string) (subToken string, nodeIndex int) {
 	if token == "" {
 		return token, -1

cmd/bucket-metadata-sys.go

@@ -49,6 +49,7 @@ func (sys *BucketMetadataSys) Remove(bucket string) {
 	}
 	sys.Lock()
 	delete(sys.metadataMap, bucket)
+	globalBucketMonitor.DeleteBucket(bucket)
 	sys.Unlock()
 }
 
@@ -72,7 +73,7 @@ func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
 // Update update bucket metadata for the specified config file.
 // The configData data should not be modified after being sent here.
 func (sys *BucketMetadataSys) Update(bucket string, configFile string, configData []byte) error {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
@@ -275,7 +276,7 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
 	if globalIsGateway && globalGatewayName == NASBackendGateway {
 		// Only needed in case of NAS gateway.
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -313,7 +314,7 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -373,10 +374,24 @@ func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.Buc
 	return meta.bucketTargetConfig, nil
 }
 
+// GetBucketTarget returns the target for the bucket and arn.
+func (sys *BucketMetadataSys) GetBucketTarget(bucket string, arn string) (madmin.BucketTarget, error) {
+	targets, err := sys.GetBucketTargetsConfig(bucket)
+	if err != nil {
+		return madmin.BucketTarget{}, err
+	}
+	for _, t := range targets.Targets {
+		if t.Arn == arn {
+			return t, nil
+		}
+	}
+	return madmin.BucketTarget{}, errConfigNotFound
+}
+
 // GetConfig returns a specific configuration from the bucket metadata.
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetConfig(bucket string) (BucketMetadata, error) {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return newBucketMetadata(bucket), errServerNotInitialized
 	}
@@ -418,12 +433,12 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 	}
 
 	// Load bucket metadata sys in background
-	go logger.LogIf(ctx, sys.load(ctx, buckets, objAPI))
+	go sys.load(ctx, buckets, objAPI)
 	return nil
 }
 
 // concurrently load bucket metadata to speed up loading bucket metadata.
-func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
 	g := errgroup.WithNErrs(len(buckets))
 	for index := range buckets {
 		index := index
@@ -440,22 +455,20 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 	}
 	for _, err := range g.Wait() {
 		if err != nil {
-			return err
+			logger.LogIf(ctx, err)
 		}
 	}
-	return nil
 }
 
 // Loads bucket metadata for all buckets into BucketMetadataSys.
-func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) error {
+func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
 	count := 100 // load 100 bucket metadata at a time.
 	for {
 		if len(buckets) < count {
-			return sys.concurrentLoad(ctx, buckets, objAPI)
-		}
-		if err := sys.concurrentLoad(ctx, buckets[:count], objAPI); err != nil {
-			return err
+			sys.concurrentLoad(ctx, buckets, objAPI)
+			return
 		}
+		sys.concurrentLoad(ctx, buckets[:count], objAPI)
 		buckets = buckets[count:]
 	}
 }

cmd/bucket-metadata.go

@@ -107,6 +107,10 @@ func newBucketMetadata(name string) BucketMetadata {
 // Load - loads the metadata of bucket by name from ObjectLayer api.
 // If an error is returned the returned metadata will be default initialized.
 func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
+	if name == "" {
+		logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
+		return errors.New("bucket name cannot be empty")
+	}
 	configFile := path.Join(bucketConfigPrefix, name, bucketMetadataFile)
 	data, err := readConfig(ctx, api, configFile)
 	if err != nil {
@@ -128,20 +132,22 @@ func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string)
 	}
 	// OK, parse data.
 	_, err = b.UnmarshalMsg(data[4:])
+	b.Name = name // in-case parsing failed for some reason, make sure bucket name is not empty.
 	return err
 }
 
 // loadBucketMetadata loads and migrates to bucket metadata.
 func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) {
 	b := newBucketMetadata(bucket)
-	err := b.Load(ctx, objectAPI, bucket)
+	err := b.Load(ctx, objectAPI, b.Name)
 	if err == nil {
 		return b, b.convertLegacyConfigs(ctx, objectAPI)
 	}
 
-	if err != errConfigNotFound {
+	if !errors.Is(err, errConfigNotFound) {
 		return b, err
 	}
+
 	// Old bucket without bucket metadata. Hence we migrate existing settings.
 	return b, b.convertLegacyConfigs(ctx, objectAPI)
 }

cmd/bucket-object-lock.go

@@ -33,7 +33,7 @@ type BucketObjectLockSys struct{}
 // Get - Get retention configuration.
 func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return r, errServerNotInitialized
 		}

cmd/bucket-quota.go

@@ -63,7 +63,7 @@ func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota,
 }
 
 func (sys *BucketQuotaSys) check(ctx context.Context, bucket string, size int64) error {
-	objAPI := newObjectLayerWithoutSafeModeFn()
+	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
 	}

cmd/bucket-replication.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/bucket/bandwidth"
 	"github.com/minio/minio/pkg/bucket/replication"
 	"github.com/minio/minio/pkg/event"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
@@ -38,7 +39,7 @@ import (
 // gets replication config associated to a given bucket name.
 func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
 	if globalIsGateway {
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			return nil, errServerNotInitialized
 		}
@@ -52,12 +53,12 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
 // validateReplicationDestination returns error if replication destination bucket missing or not configured
 // It also returns true if replication destination is same as this server.
 func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config) (bool, error) {
-	clnt := globalBucketTargetSys.GetReplicationTargetClient(ctx, rCfg.RoleArn)
+	clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rCfg.RoleArn)
 	if clnt == nil {
 		return false, BucketRemoteTargetNotFound{Bucket: bucket}
 	}
 	if found, _ := clnt.BucketExists(ctx, rCfg.GetDestination().Bucket); !found {
-		return false, BucketReplicationDestinationNotFound{Bucket: rCfg.GetDestination().Bucket}
+		return false, BucketRemoteDestinationNotFound{Bucket: rCfg.GetDestination().Bucket}
 	}
 	if ret, err := globalBucketObjectLockSys.Get(bucket); err == nil {
 		if ret.LockEnabled {
@@ -87,7 +88,7 @@ func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object strin
 
 // mustReplicate returns true if object meets replication criteria.
 func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
-	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
+	if s3Err := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, "", r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
 		return false
 	}
 	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
@@ -119,7 +120,7 @@ func mustReplicater(ctx context.Context, r *http.Request, bucket, object string,
 	return cfg.Replicate(opts)
 }
 
-func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
+func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
 	meta := make(map[string]string)
 	for k, v := range objInfo.UserDefined {
 		if k == xhttp.AmzBucketReplicationStatus {
@@ -139,15 +140,17 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 		sc = objInfo.StorageClass
 	}
 	putOpts = miniogo.PutObjectOptions{
-		UserMetadata:         meta,
-		UserTags:             tag.ToMap(),
-		ContentType:          objInfo.ContentType,
-		ContentEncoding:      objInfo.ContentEncoding,
-		StorageClass:         sc,
-		ReplicationVersionID: objInfo.VersionID,
-		ReplicationStatus:    miniogo.ReplicationStatusReplica,
-		ReplicationMTime:     objInfo.ModTime,
-		ReplicationETag:      objInfo.ETag,
+		UserMetadata:    meta,
+		UserTags:        tag.ToMap(),
+		ContentType:     objInfo.ContentType,
+		ContentEncoding: objInfo.ContentEncoding,
+		StorageClass:    sc,
+		Internal: miniogo.AdvancedPutOptions{
+			SourceVersionID:   objInfo.VersionID,
+			ReplicationStatus: miniogo.ReplicationStatusReplica,
+			SourceMTime:       objInfo.ModTime,
+			SourceETag:        objInfo.ETag,
+		},
 	}
 	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
 		rmode := miniogo.RetentionMode(mode)
@@ -166,6 +169,7 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 	if crypto.S3.IsEncrypted(objInfo.UserDefined) {
 		putOpts.ServerSideEncryption = encrypt.NewSSE()
 	}
+
 	return
 }
 
@@ -180,19 +184,17 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 		logger.LogIf(ctx, err)
 		return
 	}
-
-	tgt := globalBucketTargetSys.GetReplicationTargetClient(ctx, cfg.RoleArn)
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
 	if tgt == nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, cfg.RoleArn))
 		return
 	}
-
 	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
 		VersionID: objInfo.VersionID,
 	})
 	if err != nil {
 		return
 	}
-
 	objInfo = gr.ObjInfo
 	size, err := objInfo.GetActualSize()
 	if err != nil {
@@ -223,11 +225,27 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 			return
 		}
 	}
-	putOpts := putReplicationOpts(dest, objInfo)
 
+	target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
+		return
+	}
+	putOpts := putReplicationOpts(ctx, dest, objInfo)
 	replicationStatus := replication.Complete
-	_, err = tgt.PutObject(ctx, dest.Bucket, object, gr, size, "", "", putOpts)
-	gr.Close()
+
+	// Setup bandwidth throttling
+	peers, _ := globalEndpoints.peers()
+	totalNodesCount := len(peers)
+	b := target.BandwidthLimit / int64(totalNodesCount)
+	var headerSize int
+	for k, v := range putOpts.Header() {
+		headerSize += len(k) + len(v)
+	}
+	r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
+
+	_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, "", "", putOpts)
+	r.Close()
 	if err != nil {
 		replicationStatus = replication.Failed
 	}
@@ -310,7 +328,7 @@ func newReplicationState() *replicationState {
 		globalReplicationConcurrent = 1
 	}
 	rs := &replicationState{
-		replicaCh: make(chan ObjectInfo, globalReplicationConcurrent*2),
+		replicaCh: make(chan ObjectInfo, 10000),
 	}
 	go func() {
 		<-GlobalContext.Done()

cmd/bucket-targets.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"net/http"
 	"sync"
+	"time"
 
 	minio "github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
@@ -98,10 +99,9 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 			return BucketRemoteConnectionErr{Bucket: tgt.TargetBucket}
 		}
 		if vcfg.Status != string(versioning.Enabled) {
-			return BucketReplicationTargetNotVersioned{Bucket: tgt.TargetBucket}
+			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
 		}
 	}
-
 	sys.Lock()
 	defer sys.Unlock()
 
@@ -113,6 +113,9 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 			if t.Arn == tgt.Arn {
 				return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
 			}
+			if t.Label == tgt.Label {
+				return BucketRemoteLabelInUse{Bucket: t.TargetBucket}
+			}
 			newtgts[idx] = *tgt
 			found = true
 			continue
@@ -173,8 +176,8 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	return nil
 }
 
-// GetReplicationTargetClient returns minio-go client for replication target instance
-func (sys *BucketTargetSys) GetReplicationTargetClient(ctx context.Context, arn string) *miniogo.Core {
+// GetRemoteTargetClient returns minio-go client for replication target instance
+func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *miniogo.Core {
 	sys.RLock()
 	defer sys.RUnlock()
 	return sys.arnRemotesMap[arn]
@@ -205,14 +208,14 @@ func (sys *BucketTargetSys) Init(ctx context.Context, buckets []BucketInfo, objA
 	return nil
 }
 
-// UpdateTarget updates target to reflect metadata updates
-func (sys *BucketTargetSys) UpdateTarget(bucket string, cfg *madmin.BucketTargets) {
+// UpdateAllTargets updates target to reflect metadata updates
+func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketTargets) {
 	if sys == nil {
 		return
 	}
 	sys.Lock()
 	defer sys.Unlock()
-	if cfg == nil || cfg.Empty() {
+	if tgts == nil || tgts.Empty() {
 		// remove target and arn association
 		if tgts, ok := sys.targetsMap[bucket]; ok {
 			for _, t := range tgts {
@@ -223,10 +226,10 @@ func (sys *BucketTargetSys) UpdateTarget(bucket string, cfg *madmin.BucketTarget
 		return
 	}
 
-	if len(cfg.Targets) > 0 {
-		sys.targetsMap[bucket] = cfg.Targets
+	if len(tgts.Targets) > 0 {
+		sys.targetsMap[bucket] = tgts.Targets
 	}
-	for _, tgt := range cfg.Targets {
+	for _, tgt := range tgts.Targets {
 		tgtClient, err := sys.getRemoteTargetClient(&tgt)
 		if err != nil {
 			continue
@@ -236,7 +239,7 @@ func (sys *BucketTargetSys) UpdateTarget(bucket string, cfg *madmin.BucketTarget
 			sys.clientsCache[tgtClient.EndpointURL().String()] = tgtClient
 		}
 	}
-	sys.targetsMap[bucket] = cfg.Targets
+	sys.targetsMap[bucket] = tgts.Targets
 }
 
 // create minio-go clients for buckets having remote targets
@@ -279,8 +282,9 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*m
 	creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "")
 
 	getRemoteTargetInstanceTransportOnce.Do(func() {
-		getRemoteTargetInstanceTransport = NewGatewayHTTPTransport()
+		getRemoteTargetInstanceTransport = newGatewayHTTPTransport(1 * time.Hour)
 	})
+
 	core, err := miniogo.NewCore(tcfg.Endpoint, &miniogo.Options{
 		Creds:     creds,
 		Secure:    tcfg.Secure,

cmd/common-main.go

@@ -21,6 +21,7 @@ import (
 	"encoding/gob"
 	"errors"
 	"fmt"
+	"math/rand"
 	"net"
 	"net/url"
 	"os"
@@ -33,6 +34,7 @@ import (
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/certs"
@@ -43,6 +45,9 @@ func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
+	rand.Seed(time.Now().UTC().UnixNano())
+	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second)
+
 	gob.Register(StorageErr(""))
 }
 
@@ -100,11 +105,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	if globalInplaceUpdateDisabled {
-		logStartupMessage(updateMsg)
-	} else {
-		logStartupMessage(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
-	}
+	logStartupMessage(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
 }
 
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {

cmd/config-current.go

@@ -25,9 +25,9 @@ import (
 	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
-	"github.com/minio/minio/cmd/config/crawler"
 	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/etcd"
+	"github.com/minio/minio/cmd/config/heal"
 	xldap "github.com/minio/minio/cmd/config/identity/ldap"
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/config/notify"
@@ -56,7 +56,7 @@ func initHelp() {
 		config.KmsKesSubSys:         crypto.DefaultKesKVS,
 		config.LoggerWebhookSubSys:  logger.DefaultKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditKVS,
-		config.CrawlerSubSys:        crawler.DefaultKVS,
+		config.HealSubSys:           heal.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -109,8 +109,8 @@ func initHelp() {
 			Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.",
 		},
 		config.HelpKV{
-			Key:         config.CrawlerSubSys,
-			Description: "manage continuous disk crawling for bucket disk usage, lifecycle, quota and data integrity checks",
+			Key:         config.HealSubSys,
+			Description: "manage object healing frequency and bitrot verification checks",
 		},
 		config.HelpKV{
 			Key:             config.LoggerWebhookSubSys,
@@ -191,7 +191,7 @@ func initHelp() {
 		config.EtcdSubSys:           etcd.Help,
 		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
-		config.CrawlerSubSys:        crawler.Help,
+		config.HealSubSys:           heal.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.PolicyOPASubSys:      opa.Help,
@@ -253,7 +253,7 @@ func validateConfig(s config.Config, setDriveCount int) error {
 		return err
 	}
 
-	if _, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default]); err != nil {
+	if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
 		return err
 	}
 
@@ -438,9 +438,9 @@ func lookupConfigs(s config.Config, setDriveCount int) {
 			}
 		}
 	}
-	globalCrawlerConfig, err = crawler.LookupConfig(s[config.CrawlerSubSys][config.Default])
+	globalHealConfig, err = heal.LookupConfig(s[config.HealSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to read crawler config: %w", err))
+		logger.LogIf(ctx, fmt.Errorf("Unable to read heal config: %w", err))
 	}
 
 	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())

cmd/config-encrypted.go

@@ -19,7 +19,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"time"
 	"unicode/utf8"
@@ -194,7 +193,7 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 				// Config is already encrypted with right keys
 				continue
 			}
-			return errors.New("config data not in plain-text form or encrypted")
+			return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
 		}
 
 		cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)
@@ -274,7 +273,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, activeCredOld auth.Crede
 					// Config is already encrypted with right keys
 					continue
 				}
-				return errors.New("config data not in plain-text form or encrypted")
+				return fmt.Errorf("Decrypting config failed %w, possibly credentials are incorrect", err)
 			}
 
 			cencdata, err = madmin.EncryptData(globalActiveCred.String(), data)

cmd/config/api/api.go

@@ -34,12 +34,17 @@ const (
 	apiClusterDeadline         = "cluster_deadline"
 	apiCorsAllowOrigin         = "cors_allow_origin"
 	apiRemoteTransportDeadline = "remote_transport_deadline"
+	apiListQuorum              = "list_quorum"
+	apiExtendListCacheLife     = "extend_list_cache_life"
 
 	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
 	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
 	EnvAPIClusterDeadline         = "MINIO_API_CLUSTER_DEADLINE"
 	EnvAPICorsAllowOrigin         = "MINIO_API_CORS_ALLOW_ORIGIN"
 	EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE"
+	EnvAPIListQuorum              = "MINIO_API_LIST_QUORUM"
+	EnvAPIExtendListCacheLife     = "MINIO_API_EXTEND_LIST_CACHE_LIFE"
+	EnvAPISecureCiphers           = "MINIO_API_SECURE_CIPHERS"
 )
 
 // Deprecated key and ENVs
@@ -71,6 +76,14 @@ var (
 			Key:   apiRemoteTransportDeadline,
 			Value: "2h",
 		},
+		config.KV{
+			Key:   apiListQuorum,
+			Value: "optimal",
+		},
+		config.KV{
+			Key:   apiExtendListCacheLife,
+			Value: "0s",
+		},
 	}
 )
 
@@ -81,6 +94,8 @@ type Config struct {
 	ClusterDeadline         time.Duration `json:"cluster_deadline"`
 	CorsAllowOrigin         []string      `json:"cors_allow_origin"`
 	RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
+	ListQuorum              string        `json:"list_strict_quorum"`
+	ExtendListLife          time.Duration `json:"extend_list_cache_life"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -94,6 +109,24 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 	return json.Unmarshal(data, &aux)
 }
 
+// GetListQuorum interprets list quorum values and returns appropriate
+// acceptable quorum expected for list operations
+func (sCfg Config) GetListQuorum() int {
+	switch sCfg.ListQuorum {
+	case "optimal":
+		return 3
+	case "reduced":
+		return 2
+	case "disk":
+		// smallest possible value, generally meant for testing.
+		return 1
+	case "strict":
+		return -1
+	}
+	// Defaults to 3 drives per set.
+	return 3
+}
+
 // LookupConfig - lookup api config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 	// remove this since we have removed this already.
@@ -130,11 +163,25 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
+	listQuorum := env.Get(EnvAPIListQuorum, kvs.Get(apiListQuorum))
+	switch listQuorum {
+	case "strict", "optimal", "reduced", "disk":
+	default:
+		return cfg, errors.New("invalid value for list strict quorum")
+	}
+
+	listLife, err := time.ParseDuration(env.Get(EnvAPIExtendListCacheLife, kvs.Get(apiExtendListCacheLife)))
+	if err != nil {
+		return cfg, err
+	}
+
 	return Config{
 		RequestsMax:             requestsMax,
 		RequestsDeadline:        requestsDeadline,
 		ClusterDeadline:         clusterDeadline,
 		CorsAllowOrigin:         corsAllowOrigin,
 		RemoteTransportDeadline: remoteTransportDeadline,
+		ListQuorum:              listQuorum,
+		ExtendListLife:          listLife,
 	}, nil
 }

cmd/config/cache/config.go

@@ -28,16 +28,17 @@ import (
 
 // Config represents cache config settings
 type Config struct {
-	Enabled       bool     `json:"-"`
-	Drives        []string `json:"drives"`
-	Expiry        int      `json:"expiry"`
-	MaxUse        int      `json:"maxuse"`
-	Quota         int      `json:"quota"`
-	Exclude       []string `json:"exclude"`
-	After         int      `json:"after"`
-	WatermarkLow  int      `json:"watermark_low"`
-	WatermarkHigh int      `json:"watermark_high"`
-	Range         bool     `json:"range"`
+	Enabled         bool     `json:"-"`
+	Drives          []string `json:"drives"`
+	Expiry          int      `json:"expiry"`
+	MaxUse          int      `json:"maxuse"`
+	Quota           int      `json:"quota"`
+	Exclude         []string `json:"exclude"`
+	After           int      `json:"after"`
+	WatermarkLow    int      `json:"watermark_low"`
+	WatermarkHigh   int      `json:"watermark_high"`
+	Range           bool     `json:"range"`
+	CommitWriteback bool     `json:"-"`
 }
 
 // UnmarshalJSON - implements JSON unmarshal interface for unmarshalling
@@ -152,3 +153,13 @@ func parseCacheExcludes(excludes string) ([]string, error) {
 
 	return excludesSlice, nil
 }
+
+func parseCacheCommitMode(commitStr string) (bool, error) {
+	switch strings.ToLower(commitStr) {
+	case "writeback":
+		return true, nil
+	case "writethrough":
+		return false, nil
+	}
+	return false, config.ErrInvalidCacheCommitValue(nil).Msg("cache commit value must be `writeback` or `writethrough`")
+}

cmd/config/cache/help.go

@@ -74,5 +74,11 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         Commit,
+			Description: `set to control cache commit behavior, defaults to "writethrough"`,
+			Optional:    true,
+			Type:        "string",
+		},
 	}
 )

cmd/config/cache/lookup.go

@@ -35,6 +35,7 @@ const (
 	WatermarkLow  = "watermark_low"
 	WatermarkHigh = "watermark_high"
 	Range         = "range"
+	Commit        = "commit"
 
 	EnvCacheDrives        = "MINIO_CACHE_DRIVES"
 	EnvCacheExclude       = "MINIO_CACHE_EXCLUDE"
@@ -45,6 +46,7 @@ const (
 	EnvCacheWatermarkLow  = "MINIO_CACHE_WATERMARK_LOW"
 	EnvCacheWatermarkHigh = "MINIO_CACHE_WATERMARK_HIGH"
 	EnvCacheRange         = "MINIO_CACHE_RANGE"
+	EnvCacheCommit        = "MINIO_CACHE_COMMIT"
 
 	EnvCacheEncryptionMasterKey = "MINIO_CACHE_ENCRYPTION_MASTER_KEY"
 
@@ -53,6 +55,7 @@ const (
 	DefaultAfter         = "0"
 	DefaultWaterMarkLow  = "70"
 	DefaultWaterMarkHigh = "80"
+	DefaultCacheCommit   = "writethrough"
 )
 
 // DefaultKVS - default KV settings for caching.
@@ -90,6 +93,10 @@ var (
 			Key:   Range,
 			Value: config.EnableOn,
 		},
+		config.KV{
+			Key:   Commit,
+			Value: DefaultCacheCommit,
+		},
 	}
 )
 
@@ -210,6 +217,12 @@ func LookupConfig(kvs config.KVS) (Config, error) {
 		}
 		cfg.Range = rng
 	}
+	if commit := env.Get(EnvCacheCommit, kvs.Get(Commit)); commit != "" {
+		cfg.CommitWriteback, err = parseCacheCommitMode(commit)
+		if err != nil {
+			return cfg, err
+		}
+	}
 
 	return cfg, nil
 }

cmd/config/config.go

@@ -76,7 +76,7 @@ const (
 	KmsKesSubSys         = "kms_kes"
 	LoggerWebhookSubSys  = "logger_webhook"
 	AuditWebhookSubSys   = "audit_webhook"
-	CrawlerSubSys        = "crawler"
+	HealSubSys           = "heal"
 
 	// Add new constants here if you add new fields to config.
 )
@@ -113,7 +113,7 @@ var SubSystems = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
-	CrawlerSubSys,
+	HealSubSys,
 	NotifyAMQPSubSys,
 	NotifyESSubSys,
 	NotifyKafkaSubSys,
@@ -140,7 +140,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
-	CrawlerSubSys,
+	HealSubSys,
 }...)
 
 // Constant separators

cmd/config/errors.go

@@ -102,6 +102,12 @@ var (
 		"MINIO_CACHE_RANGE: Valid expected value is `on` or `off`",
 	)
 
+	ErrInvalidCacheCommitValue = newErrFn(
+		"Invalid cache commit value",
+		"Please check the passed value",
+		"MINIO_CACHE_COMMIT: Valid expected value is `writeback` or `writethrough`",
+	)
+
 	ErrInvalidRotatingCredentialsBackendEncrypted = newErrFn(
 		"Invalid rotating credentials",
 		"Please set correct rotating credentials in the environment for decryption",

cmd/config/heal/heal.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package crawler
+package heal
 
 import (
 	"errors"
@@ -24,20 +24,20 @@ import (
 
 // Compression environment variables
 const (
-	BitrotScan = "bitrotscan"
+	Bitrot = "bitrotscan"
 )
 
-// Config represents the crawler settings.
+// Config represents the heal settings.
 type Config struct {
 	// Bitrot will perform bitrot scan on local disk when checking objects.
 	Bitrot bool `json:"bitrotscan"`
 }
 
 var (
-	// DefaultKVS - default KV config for crawler settings
+	// DefaultKVS - default KV config for heal settings
 	DefaultKVS = config.KVS{
 		config.KV{
-			Key:   BitrotScan,
+			Key:   Bitrot,
 			Value: config.EnableOff,
 		},
 	}
@@ -45,7 +45,7 @@ var (
 	// Help provides help for config values
 	Help = config.HelpKVS{
 		config.HelpKV{
-			Key:         BitrotScan,
+			Key:         Bitrot,
 			Description: `perform bitrot scan on disks when checking objects during crawl`,
 			Optional:    true,
 			Type:        "on|off",
@@ -55,12 +55,12 @@ var (
 
 // LookupConfig - lookup config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS) (cfg Config, err error) {
-	if err = config.CheckValidKeys(config.CrawlerSubSys, kvs, DefaultKVS); err != nil {
+	if err = config.CheckValidKeys(config.HealSubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
-	bitrot := kvs.Get(BitrotScan)
+	bitrot := kvs.Get(Bitrot)
 	if bitrot != config.EnableOn && bitrot != config.EnableOff {
-		return cfg, errors.New(BitrotScan + ": must be 'on' or 'off'")
+		return cfg, errors.New(Bitrot + ": must be 'on' or 'off'")
 	}
 	cfg.Bitrot = bitrot == config.EnableOn
 	return cfg, nil

cmd/config/storageclass/help.go

@@ -33,6 +33,12 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         ClassDMA,
+			Description: `enable O_DIRECT for both read and write, defaults to "write" e.g. "read+write"`,
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

cmd/config/storageclass/storage-class.go

@@ -18,6 +18,7 @@ package storageclass
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"strconv"
 	"strings"
@@ -32,17 +33,26 @@ const (
 	RRS = "REDUCED_REDUNDANCY"
 	// Standard storage class
 	STANDARD = "STANDARD"
+	// DMA storage class
+	DMA = "DMA"
+
+	// Valid values are "write" and "read+write"
+	DMAWrite     = "write"
+	DMAReadWrite = "read+write"
 )
 
 // Standard constats for config info storage class
 const (
 	ClassStandard = "standard"
 	ClassRRS      = "rrs"
+	ClassDMA      = "dma"
 
 	// Reduced redundancy storage class environment variable
 	RRSEnv = "MINIO_STORAGE_CLASS_RRS"
 	// Standard storage class environment variable
 	StandardEnv = "MINIO_STORAGE_CLASS_STANDARD"
+	// DMA storage class environment variable
+	DMAEnv = "MINIO_STORAGE_CLASS_DMA"
 
 	// Supported storage class scheme is EC
 	schemePrefix = "EC"
@@ -52,6 +62,9 @@ const (
 
 	// Default RRS parity is always minimum parity.
 	defaultRRSParity = minParityDisks
+
+	// Default DMA value
+	defaultDMA = DMAWrite
 )
 
 // DefaultKVS - default storage class config
@@ -65,18 +78,24 @@ var (
 			Key:   ClassRRS,
 			Value: "EC:2",
 		},
+		config.KV{
+			Key:   ClassDMA,
+			Value: defaultDMA,
+		},
 	}
 )
 
 // StorageClass - holds storage class information
 type StorageClass struct {
 	Parity int
+	DMA    string
 }
 
 // Config storage class configuration
 type Config struct {
 	Standard StorageClass `json:"standard"`
 	RRS      StorageClass `json:"rrs"`
+	DMA      StorageClass `json:"dma"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -93,7 +112,7 @@ func (sCfg *Config) UnmarshalJSON(data []byte) error {
 // IsValid - returns true if input string is a valid
 // storage class kind supported.
 func IsValid(sc string) bool {
-	return sc == RRS || sc == STANDARD
+	return sc == RRS || sc == STANDARD || sc == DMA
 }
 
 // UnmarshalText unmarshals storage class from its textual form into
@@ -103,6 +122,14 @@ func (sc *StorageClass) UnmarshalText(b []byte) error {
 	if scStr == "" {
 		return nil
 	}
+	if scStr == DMAWrite {
+		sc.DMA = DMAWrite
+		return nil
+	}
+	if scStr == DMAReadWrite {
+		sc.DMA = DMAReadWrite
+		return nil
+	}
 	s, err := parseStorageClass(scStr)
 	if err != nil {
 		return err
@@ -116,14 +143,14 @@ func (sc *StorageClass) MarshalText() ([]byte, error) {
 	if sc.Parity != 0 {
 		return []byte(fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)), nil
 	}
-	return []byte(""), nil
+	return []byte(sc.DMA), nil
 }
 
 func (sc *StorageClass) String() string {
 	if sc.Parity != 0 {
 		return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity)
 	}
-	return ""
+	return sc.DMA
 }
 
 // Parses given storageClassEnv and returns a storageClass structure.
@@ -212,6 +239,11 @@ func (sCfg Config) GetParityForSC(sc string) (parity int) {
 	}
 }
 
+// GetDMA - returns DMA configuration.
+func (sCfg Config) GetDMA() string {
+	return sCfg.DMA.DMA
+}
+
 // Enabled returns if etcd is enabled.
 func Enabled(kvs config.KVS) bool {
 	ssc := kvs.Get(ClassStandard)
@@ -231,6 +263,7 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 
 	ssc := env.Get(StandardEnv, kvs.Get(ClassStandard))
 	rrsc := env.Get(RRSEnv, kvs.Get(ClassRRS))
+	dma := env.Get(DMAEnv, kvs.Get(ClassDMA))
 	// Check for environment variables and parse into storageClass struct
 	if ssc != "" {
 		cfg.Standard, err = parseStorageClass(ssc)
@@ -252,6 +285,14 @@ func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) {
 		cfg.RRS.Parity = defaultRRSParity
 	}
 
+	if dma == "" {
+		dma = defaultDMA
+	}
+	if dma != DMAReadWrite && dma != DMAWrite {
+		return Config{}, errors.New(`valid dma values are "read-write" and "write"`)
+	}
+	cfg.DMA.DMA = dma
+
 	// Validation is done after parsing both the storage classes. This is needed because we need one
 	// storage class value to deduce the correct value of the other storage class.
 	if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, setDriveCount); err != nil {

cmd/consolelogger.go

@@ -40,8 +40,8 @@ type HTTPConsoleLoggerSys struct {
 	logBuf   *ring.Ring
 }
 
-func mustGetNodeName(endpointZones EndpointZones) (nodeName string) {
-	host, err := xnet.ParseHost(GetLocalPeer(endpointZones))
+func mustGetNodeName(endpointServerSets EndpointServerSets) (nodeName string) {
+	host, err := xnet.ParseHost(GetLocalPeer(endpointServerSets))
 	if err != nil {
 		logger.FatalIf(err, "Unable to start console logging subsystem")
 	}
@@ -63,8 +63,8 @@ func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys {
 }
 
 // SetNodeName - sets the node name if any after distributed setup has initialized
-func (sys *HTTPConsoleLoggerSys) SetNodeName(endpointZones EndpointZones) {
-	sys.nodeName = mustGetNodeName(endpointZones)
+func (sys *HTTPConsoleLoggerSys) SetNodeName(endpointServerSets EndpointServerSets) {
+	sys.nodeName = mustGetNodeName(endpointServerSets)
 }
 
 // HasLogListeners returns true if console log listeners are registered

cmd/crypto/kes.go

@@ -177,11 +177,10 @@ func (kes *kesService) CreateKey(keyID string) error { return kes.client.CreateK
 // named key referenced by keyID. It also binds the generated key
 // cryptographically to the provided context.
 func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
-	var context bytes.Buffer
-	ctx.WriteTo(&context)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	var plainKey []byte
-	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context.Bytes())
+	plainKey, sealedKey, err = kes.client.GenerateDataKey(keyID, context)
 	if err != nil {
 		return key, nil, err
 	}
@@ -200,11 +199,10 @@ func (kes *kesService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (kes *kesService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
-	var context bytes.Buffer
-	ctx.WriteTo(&context)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	var plainKey []byte
-	plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context.Bytes())
+	plainKey, err = kes.client.DecryptDataKey(keyID, sealedKey, context)
 	if err != nil {
 		return key, err
 	}
@@ -415,7 +413,7 @@ func (c *kesClient) postRetry(path string, body io.ReadSeeker, limit int64) (io.
 		}
 
 		// If the error is not temp. / retryable => fail the request immediately.
-		if !xnet.IsNetworkOrHostDown(err) &&
+		if !xnet.IsNetworkOrHostDown(err, false) &&
 			!errors.Is(err, io.EOF) &&
 			!errors.Is(err, io.ErrUnexpectedEOF) &&
 			!errors.Is(err, context.DeadlineExceeded) {

cmd/crypto/key.go

@@ -103,7 +103,6 @@ func (key ObjectKey) Seal(extKey, iv [32]byte, domain, bucket, object string) Se
 func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucket, object string) error {
 	var (
 		unsealConfig sio.Config
-		decryptedKey bytes.Buffer
 	)
 	switch sealedKey.Algorithm {
 	default:
@@ -122,10 +121,9 @@ func (key *ObjectKey) Unseal(extKey [32]byte, sealedKey SealedKey, domain, bucke
 		unsealConfig = sio.Config{MinVersion: sio.Version10, Key: sha.Sum(nil)}
 	}
 
-	if n, err := sio.Decrypt(&decryptedKey, bytes.NewReader(sealedKey.Key[:]), unsealConfig); n != 32 || err != nil {
+	if out, err := sio.DecryptBuffer(key[:0], sealedKey.Key[:], unsealConfig); len(out) != 32 || err != nil {
 		return ErrSecretKeyMismatch
 	}
-	copy(key[:], decryptedKey.Bytes())
 	return nil
 }
 
@@ -165,11 +163,7 @@ func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) {
 	if !IsETagSealed(etag) {
 		return etag, nil
 	}
-	var buffer bytes.Buffer
 	mac := hmac.New(sha256.New, key[:])
 	mac.Write([]byte("SSE-etag"))
-	if _, err := sio.Decrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil {
-		return nil, err
-	}
-	return buffer.Bytes(), nil
+	return sio.DecryptBuffer(make([]byte, 0, len(etag)), etag, sio.Config{Key: mac.Sum(nil)})
 }

cmd/crypto/kms.go

@@ -39,6 +39,8 @@ type Context map[string]string
 //
 // WriteTo sorts the context keys and writes the sorted
 // key-value pairs as canonical JSON object to w.
+//
+// Note that neither keys nor values are escaped for JSON.
 func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	sortedKeys := make(sort.StringSlice, 0, len(c))
 	for k := range c {
@@ -67,6 +69,53 @@ func (c Context) WriteTo(w io.Writer) (n int64, err error) {
 	return n + int64(nn), err
 }
 
+// AppendTo appends the context in a canonical from to dst.
+//
+// AppendTo sorts the context keys and writes the sorted
+// key-value pairs as canonical JSON object to w.
+//
+// Note that neither keys nor values are escaped for JSON.
+func (c Context) AppendTo(dst []byte) (output []byte) {
+	if len(c) == 0 {
+		return append(dst, '{', '}')
+	}
+
+	// out should not escape.
+	out := bytes.NewBuffer(dst)
+
+	// No need to copy+sort
+	if len(c) == 1 {
+		for k, v := range c {
+			out.WriteString(`{"`)
+			out.WriteString(k)
+			out.WriteString(`":"`)
+			out.WriteString(v)
+			out.WriteString(`"}`)
+		}
+		return out.Bytes()
+	}
+
+	sortedKeys := make([]string, 0, len(c))
+	for k := range c {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Strings(sortedKeys)
+
+	out.WriteByte('{')
+	for i, k := range sortedKeys {
+		out.WriteByte('"')
+		out.WriteString(k)
+		out.WriteString(`":"`)
+		out.WriteString(c[k])
+		out.WriteByte('"')
+		if i < len(sortedKeys)-1 {
+			out.WriteByte(',')
+		}
+	}
+	out.WriteByte('}')
+	return out.Bytes()
+}
+
 // KMS represents an active and authenticted connection
 // to a Key-Management-Service. It supports generating
 // data key generation and unsealing of KMS-generated
@@ -155,13 +204,12 @@ func (kms *masterKeyKMS) Info() (info KMSInfo) {
 
 func (kms *masterKeyKMS) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
 	var (
-		buffer     bytes.Buffer
 		derivedKey = kms.deriveKey(keyID, ctx)
 	)
-	if n, err := sio.Decrypt(&buffer, bytes.NewReader(sealedKey), sio.Config{Key: derivedKey[:]}); err != nil || n != 32 {
+	out, err := sio.DecryptBuffer(key[:0], sealedKey, sio.Config{Key: derivedKey[:]})
+	if err != nil || len(out) != 32 {
 		return key, err // TODO(aead): upgrade sio to use sio.Error
 	}
-	copy(key[:], buffer.Bytes())
 	return key, nil
 }
 
@@ -171,7 +219,7 @@ func (kms *masterKeyKMS) deriveKey(keyID string, context Context) (key [32]byte)
 	}
 	mac := hmac.New(sha256.New, kms.masterKey[:])
 	mac.Write([]byte(keyID))
-	context.WriteTo(mac)
+	mac.Write(context.AppendTo(make([]byte, 0, 128)))
 	mac.Sum(key[:0])
 	return key
 }

cmd/crypto/kms_test.go

@@ -16,6 +16,7 @@ package crypto
 
 import (
 	"bytes"
+	"fmt"
 	"path"
 	"strings"
 	"testing"
@@ -83,3 +84,32 @@ func TestContextWriteTo(t *testing.T) {
 		}
 	}
 }
+
+func TestContextAppendTo(t *testing.T) {
+	for i, test := range contextWriteToTests {
+		dst := make([]byte, 0, 1024)
+		dst = test.Context.AppendTo(dst)
+		if s := string(dst); s != test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON)
+		}
+		// Append one more
+		dst = test.Context.AppendTo(dst)
+		if s := string(dst); s != test.ExpectedJSON+test.ExpectedJSON {
+			t.Errorf("Test %d: JSON representation differ - got: '%s' want: '%s'", i, s, test.ExpectedJSON+test.ExpectedJSON)
+		}
+	}
+}
+
+func BenchmarkContext_AppendTo(b *testing.B) {
+	tests := []Context{{}, {"bucket": "warp-benchmark-bucket"}, {"0": "1", "-": "2", ".": "#"}, {"34trg": "dfioutr89", "ikjfdghkjf": "jkedfhgfjkhg", "sdfhsdjkh": "if88889", "asddsirfh804": "kjfdshgdfuhgfg78-45604586#$%"}}
+	for _, test := range tests {
+		b.Run(fmt.Sprintf("%d-elems", len(test)), func(b *testing.B) {
+			dst := make([]byte, 0, 1024)
+			b.ReportAllocs()
+			b.ResetTimer()
+			for i := 0; i < b.N; i++ {
+				dst = test.AppendTo(dst[:0])
+			}
+		})
+	}
+}

cmd/crypto/metadata.go

@@ -204,15 +204,17 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 
 	// Check whether all extracted values are well-formed
-	iv, err := base64.StdEncoding.DecodeString(b64IV)
-	if err != nil || len(iv) != 32 {
+	var iv [32]byte
+	n, err := base64.StdEncoding.Decode(iv[:], []byte(b64IV))
+	if err != nil || n != 32 {
 		return keyID, kmsKey, sealedKey, errInvalidInternalIV
 	}
 	if algorithm != SealAlgorithm {
 		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
 	}
-	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
-	if err != nil || len(encryptedKey) != 64 {
+	var encryptedKey [64]byte
+	n, err = base64.StdEncoding.Decode(encryptedKey[:], []byte(b64SealedKey))
+	if err != nil || n != 64 {
 		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
 	}
 	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
@@ -223,8 +225,8 @@ func (s3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte
 	}
 
 	sealedKey.Algorithm = algorithm
-	copy(sealedKey.IV[:], iv)
-	copy(sealedKey.Key[:], encryptedKey)
+	sealedKey.IV = iv
+	sealedKey.Key = encryptedKey
 	return keyID, kmsKey, sealedKey, nil
 }
 

cmd/crypto/vault.go

@@ -15,7 +15,6 @@
 package crypto
 
 import (
-	"bytes"
 	"encoding/base64"
 	"errors"
 	"fmt"
@@ -224,11 +223,10 @@ func (v *vaultService) CreateKey(keyID string) error {
 // named key referenced by keyID. It also binds the generated key
 // cryptographically to the provided context.
 func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sealedKey []byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
-		"context": base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context": base64.StdEncoding.EncodeToString(context),
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/datakey/plaintext/%s", keyID), payload)
 	if err != nil {
@@ -260,12 +258,11 @@ func (v *vaultService) GenerateKey(keyID string, ctx Context) (key [32]byte, sea
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (key [32]byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
 		"ciphertext": string(sealedKey),
-		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context":    base64.StdEncoding.EncodeToString(context),
 	}
 
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/decrypt/%s", keyID), payload)
@@ -294,12 +291,11 @@ func (v *vaultService) UnsealKey(keyID string, sealedKey []byte, ctx Context) (k
 // The context must be same context as the one provided while
 // generating the plaintext key / sealedKey.
 func (v *vaultService) UpdateKey(keyID string, sealedKey []byte, ctx Context) (rotatedKey []byte, err error) {
-	var contextStream bytes.Buffer
-	ctx.WriteTo(&contextStream)
+	context := ctx.AppendTo(make([]byte, 0, 128))
 
 	payload := map[string]interface{}{
 		"ciphertext": string(sealedKey),
-		"context":    base64.StdEncoding.EncodeToString(contextStream.Bytes()),
+		"context":    base64.StdEncoding.EncodeToString(context),
 	}
 	s, err := v.client.Logical().Write(fmt.Sprintf("/transit/rewrap/%s", keyID), payload)
 	if err != nil {

cmd/data-crawler.go

@@ -29,7 +29,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/config"
-	"github.com/minio/minio/cmd/config/crawler"
+	"github.com/minio/minio/cmd/config/heal"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
 	"github.com/minio/minio/pkg/bucket/replication"
@@ -53,8 +53,8 @@ const (
 )
 
 var (
-	globalCrawlerConfig          crawler.Config
-	dataCrawlerLeaderLockTimeout = newDynamicTimeout(1*time.Minute, 30*time.Second)
+	globalHealConfig             heal.Config
+	dataCrawlerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 )
 
 // initDataCrawler will start the crawler unless disabled.
@@ -69,10 +69,10 @@ func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 // There should only ever be one crawler running per cluster.
 func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 	// Make sure only 1 crawler is running on the cluster.
-	locker := objAPI.NewNSLock(ctx, minioMetaBucket, "runDataCrawler.lock")
+	locker := objAPI.NewNSLock(minioMetaBucket, "runDataCrawler.lock")
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	for {
-		err := locker.GetLock(dataCrawlerLeaderLockTimeout)
+		err := locker.GetLock(ctx, dataCrawlerLeaderLockTimeout)
 		if err != nil {
 			time.Sleep(time.Duration(r.Float64() * float64(dataCrawlStartDelay)))
 			continue
@@ -135,12 +135,11 @@ type cachedFolder struct {
 }
 
 type folderScanner struct {
-	root               string
-	getSize            getSizeFn
-	oldCache           dataUsageCache
-	newCache           dataUsageCache
-	withFilter         *bloomFilter
-	waitForLowActiveIO func()
+	root       string
+	getSize    getSizeFn
+	oldCache   dataUsageCache
+	newCache   dataUsageCache
+	withFilter *bloomFilter
 
 	dataUsageCrawlMult  float64
 	dataUsageCrawlDebug bool
@@ -155,7 +154,7 @@ type folderScanner struct {
 // The returned cache will always be valid, but may not be updated from the existing.
 // Before each operation waitForLowActiveIO is called which can be used to temporarily halt the crawler.
 // If the supplied context is canceled the function will return at the first chance.
-func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache, waitForLowActiveIO func(), getSize getSizeFn) (dataUsageCache, error) {
+func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache, getSize getSizeFn) (dataUsageCache, error) {
 	t := UTCNow()
 
 	logPrefix := color.Green("data-usage: ")
@@ -183,7 +182,6 @@ func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 		getSize:             getSize,
 		oldCache:            cache,
 		newCache:            dataUsageCache{Info: cache.Info},
-		waitForLowActiveIO:  waitForLowActiveIO,
 		newFolders:          nil,
 		existingFolders:     nil,
 		dataUsageCrawlMult:  delayMult,
@@ -376,7 +374,6 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				}
 			}
 		}
-		f.waitForLowActiveIO()
 		sleepDuration(dataCrawlSleepPerFolder, f.dataUsageCrawlMult)
 
 		cache := dataUsageEntry{}
@@ -424,7 +421,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				}
 				return nil
 			}
-			f.waitForLowActiveIO()
+
 			// Dynamic time delay.
 			t := UTCNow()
 
@@ -462,7 +459,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			continue
 		}
 
-		objAPI := newObjectLayerWithoutSafeModeFn()
+		objAPI := newObjectLayerFn()
 		if objAPI == nil {
 			continue
 		}
@@ -484,14 +481,22 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 		// If that doesn't bring it back we remove the folder and assume it was deleted.
 		// This means that the next run will not look for it.
 		for k := range existing {
-			f.waitForLowActiveIO()
 			bucket, prefix := path2BucketObject(k)
 			if f.dataUsageCrawlDebug {
 				logger.Info(color.Green("folder-scanner:")+" checking disappeared folder: %v/%v", bucket, prefix)
 			}
 
+			// Dynamic time delay.
+			t := UTCNow()
+
 			err = objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{Recursive: true, Remove: healDeleteDangling},
 				func(bucket, object, versionID string) error {
+					// Wait for each heal as per crawler frequency.
+					sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+
+					defer func() {
+						t = UTCNow()
+					}()
 					return bgSeq.queueHealTask(healSource{
 						bucket:    bucket,
 						object:    object,
@@ -499,6 +504,8 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 					}, madmin.HealItemObject)
 				})
 
+			sleepDuration(time.Since(t), f.dataUsageCrawlMult)
+
 			if f.dataUsageCrawlDebug && err != nil {
 				logger.Info(color.Green("healObjects:")+" checking returned value %v", err)
 			}
@@ -535,7 +542,6 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder)
 		default:
 		}
 
-		f.waitForLowActiveIO()
 		if typ&os.ModeDir != 0 {
 			dirStack = append(dirStack, entName)
 			err := readDirFn(path.Join(dirStack...), addDir)
@@ -749,9 +755,14 @@ func (i *crawlItem) applyActions(ctx context.Context, o ObjectLayer, meta action
 		return size
 	}
 
+	eventName := event.ObjectRemovedDelete
+	if obj.DeleteMarker {
+		eventName = event.ObjectRemovedDeleteMarkerCreated
+	}
+
 	// Notify object deleted event.
 	sendEvent(eventArgs{
-		EventName:  event.ObjectRemovedDelete,
+		EventName:  eventName,
 		BucketName: i.bucket,
 		Object:     obj,
 		Host:       "Internal: [ILM-EXPIRY]",

cmd/data-update-tracker.go

@@ -40,13 +40,13 @@ import (
 
 const (
 	// Estimate bloom filter size. With this many items
-	dataUpdateTrackerEstItems = 1000000
+	dataUpdateTrackerEstItems = 10000000
 	// ... we want this false positive rate:
 	dataUpdateTrackerFP        = 0.99
-	dataUpdateTrackerQueueSize = 10000
+	dataUpdateTrackerQueueSize = 0
 
 	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 3
+	dataUpdateTrackerVersion      = 4
 	dataUpdateTrackerSaveInterval = 5 * time.Minute
 )
 
@@ -168,6 +168,43 @@ func (d *dataUpdateTracker) current() uint64 {
 	return d.Current.idx
 }
 
+// latestWithDir returns the highest index that contains the directory.
+// This means that any cycle higher than this does NOT contain the entry.
+func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
+	bucket, _ := path2BucketObjectWithBasePath("", dir)
+	if bucket == "" {
+		if d.debug && len(dir) > 0 {
+			logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", dir)
+		}
+		return d.current()
+	}
+	if isReservedOrInvalidBucket(bucket, false) {
+		if d.debug {
+			logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, dir)
+		}
+		return d.current()
+	}
+
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	if d.Current.bf.containsDir(dir) || d.Current.idx == 0 {
+		return d.Current.idx
+	}
+	if d.debug {
+		logger.Info("current bloom does NOT contains dir %s", dir)
+	}
+
+	idx := d.Current.idx - 1
+	for {
+		f := d.History.find(idx)
+		if f == nil || f.bf.containsDir(dir) || idx == 0 {
+			break
+		}
+		idx--
+	}
+	return idx
+}
+
 // start will load the current data from the drives start collecting information and
 // start a saver goroutine.
 // All of these will exit when the context is canceled.
@@ -366,7 +403,7 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 		return err
 	}
 	switch tmp[0] {
-	case 1, 2:
+	case 1, 2, 3:
 		logger.Info(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
 		return nil
 	case dataUpdateTrackerVersion:
@@ -440,41 +477,66 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 // start a collector that picks up entries from objectUpdatedCh
 // and adds them  to the current bloom filter.
 func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case in := <-d.input:
-			bucket, _ := path2BucketObjectWithBasePath("", in)
-			if bucket == "" {
-				if d.debug && len(in) > 0 {
-					logger.Info(color.Green("data-usage:")+" no bucket (%s)", in)
-				}
-				continue
+	for in := range d.input {
+		bucket, _ := path2BucketObjectWithBasePath("", in)
+		if bucket == "" {
+			if d.debug && len(in) > 0 {
+				logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in)
 			}
-
-			if isReservedOrInvalidBucket(bucket, false) {
-				if false && d.debug {
-					logger.Info(color.Green("data-usage:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
-				}
-				continue
-			}
-			split := splitPathDeterministic(in)
-
-			// Add all paths until level 3.
-			d.mu.Lock()
-			for i := range split {
-				if d.debug && false {
-					logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
-				}
-				d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
-			}
-			d.dirty = d.dirty || len(split) > 0
-			d.mu.Unlock()
+			continue
 		}
+
+		if isReservedOrInvalidBucket(bucket, false) {
+			if d.debug {
+				logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
+			}
+			continue
+		}
+		split := splitPathDeterministic(in)
+
+		// Add all paths until done.
+		d.mu.Lock()
+		for i := range split {
+			if d.debug {
+				logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+			}
+			d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+		}
+		d.dirty = d.dirty || len(split) > 0
+		d.mu.Unlock()
 	}
 }
 
+// markDirty adds the supplied path to the current bloom filter.
+func (d *dataUpdateTracker) markDirty(in string) {
+	bucket, _ := path2BucketObjectWithBasePath("", in)
+	if bucket == "" {
+		if d.debug && len(in) > 0 {
+			logger.Info(color.Green("dataUpdateTracker:")+" no bucket (%s)", in)
+		}
+		return
+	}
+
+	if isReservedOrInvalidBucket(bucket, false) {
+		if d.debug && false {
+			logger.Info(color.Green("dataUpdateTracker:")+" isReservedOrInvalidBucket: %v, entry: %v", bucket, in)
+		}
+		return
+	}
+	split := splitPathDeterministic(in)
+
+	// Add all paths until done.
+	d.mu.Lock()
+	for i := range split {
+		if d.debug {
+			logger.Info(color.Green("dataUpdateTracker:") + " Marking path dirty: " + color.Blue(path.Join(split[:i+1]...)))
+		}
+		d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+	}
+	d.dirty = d.dirty || len(split) > 0
+	d.mu.Unlock()
+}
+
 // find entry with specified index.
 // Returns nil if not found.
 func (d dataUpdateTrackerHistory) find(idx uint64) *dataUpdateFilter {
@@ -534,8 +596,13 @@ func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint6
 // cycleFilter will cycle the bloom filter to start recording to index y if not already.
 // The response will contain a bloom filter starting at index x up to, but not including index y.
 // If y is 0, the response will not update y, but return the currently recorded information
-// from the up until and including current y.
-func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
+// from the oldest (unless 0, then it will be all) until and including current y.
+func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
+	if req.OldestClean != "" {
+		return &bloomFilterResponse{OldestIdx: d.latestWithDir(req.OldestClean)}, nil
+	}
+	current := req.Current
+	oldest := req.Oldest
 	d.mu.Lock()
 	defer d.mu.Unlock()
 	if current == 0 {
@@ -543,7 +610,10 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uin
 			return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
 		}
 		d.History.sort()
-		return d.filterFrom(ctx, d.History[len(d.History)-1].idx, d.Current.idx), nil
+		if oldest == 0 {
+			oldest = d.History[len(d.History)-1].idx
+		}
+		return d.filterFrom(ctx, oldest, d.Current.idx), nil
 	}
 
 	// Move current to history if new one requested
@@ -571,7 +641,7 @@ func (d *dataUpdateTracker) cycleFilter(ctx context.Context, oldest, current uin
 // Trailing slashes are removed.
 // Returns 0 length if no parts are found after trimming.
 func splitPathDeterministic(in string) []string {
-	split := strings.Split(in, SlashSeparator)
+	split := strings.Split(decodeDirObject(in), SlashSeparator)
 
 	// Trim empty start/end
 	for len(split) > 0 {
@@ -587,10 +657,6 @@ func splitPathDeterministic(in string) []string {
 		split = split[:len(split)-1]
 	}
 
-	// Return up to 3 parts.
-	if len(split) > 3 {
-		split = split[:3]
-	}
 	return split
 }
 
@@ -599,6 +665,9 @@ func splitPathDeterministic(in string) []string {
 type bloomFilterRequest struct {
 	Oldest  uint64
 	Current uint64
+	// If set the oldest clean version will be returned in OldestIdx
+	// and the rest of the request will be ignored.
+	OldestClean string
 }
 
 type bloomFilterResponse struct {
@@ -615,10 +684,9 @@ type bloomFilterResponse struct {
 }
 
 // ObjectPathUpdated indicates a path has been updated.
-// The function will never block.
+// The function will block until the entry has been picked up.
 func ObjectPathUpdated(s string) {
-	select {
-	case objectUpdatedCh <- s:
-	default:
+	if intDataUpdateTracker != nil {
+		intDataUpdateTracker.markDirty(s)
 	}
 }

cmd/data-update-tracker_test.go

@@ -169,7 +169,12 @@ func TestDataUpdateTracker(t *testing.T) {
 		})
 	}
 	// Cycle to history
-	_, err = dut.cycleFilter(ctx, 1, 2)
+	req := bloomFilterRequest{
+		Oldest:  1,
+		Current: 2,
+	}
+
+	_, err = dut.cycleFilter(ctx, req)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -200,7 +205,11 @@ func TestDataUpdateTracker(t *testing.T) {
 	if dut.current() != 2 {
 		t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
 	}
-	bfr2, err := dut.cycleFilter(ctx, 1, 3)
+	req = bloomFilterRequest{
+		Oldest:  1,
+		Current: 3,
+	}
+	bfr2, err := dut.cycleFilter(ctx, req)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/data-usage_test.go

@@ -62,7 +62,7 @@ func TestDataUsageUpdate(t *testing.T) {
 		return 0, nil
 	}
 
-	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
+	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -183,7 +183,7 @@ func TestDataUsageUpdate(t *testing.T) {
 		},
 	}
 	createUsageTestFiles(t, base, bucket, files)
-	got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+	got, err = crawlDataFolder(context.Background(), base, got, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -268,7 +268,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+		got, err = crawlDataFolder(context.Background(), base, got, getSize)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -355,7 +355,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		}
 		return 0, nil
 	}
-	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, func() {}, getSize)
+	got, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -465,7 +465,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		},
 	}
 	createUsageTestFiles(t, base, "", files)
-	got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+	got, err = crawlDataFolder(context.Background(), base, got, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -548,7 +548,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = crawlDataFolder(context.Background(), base, got, func() {}, getSize)
+		got, err = crawlDataFolder(context.Background(), base, got, getSize)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -652,7 +652,7 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		}
 		return 0, nil
 	}
-	want, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, func() {}, getSize)
+	want, err := crawlDataFolder(context.Background(), base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/disk-cache-backend.go

@@ -19,7 +19,9 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/md5"
 	"crypto/rand"
+	"encoding/base64"
 	"encoding/hex"
 	"errors"
 	"fmt"
@@ -70,7 +72,9 @@ type cacheMeta struct {
 	// Ranges maps cached range to associated filename.
 	Ranges map[string]string `json:"ranges,omitempty"`
 	// Hits is a counter on the number of times this object has been accessed so far.
-	Hits int `json:"hits,omitempty"`
+	Hits   int    `json:"hits,omitempty"`
+	Bucket string `json:"bucket,omitempty"`
+	Object string `json:"object,omitempty"`
 }
 
 // RangeInfo has the range, file and range length information for a cached range.
@@ -130,19 +134,21 @@ type diskCache struct {
 	online       uint32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
 	purgeRunning int32
 
-	triggerGC     chan struct{}
-	dir           string         // caching directory
-	stats         CacheDiskStats // disk cache stats for prometheus
-	quotaPct      int            // max usage in %
-	pool          sync.Pool
-	after         int // minimum accesses before an object is cached.
-	lowWatermark  int
-	highWatermark int
-	enableRange   bool
+	triggerGC        chan struct{}
+	dir              string         // caching directory
+	stats            CacheDiskStats // disk cache stats for prometheus
+	quotaPct         int            // max usage in %
+	pool             sync.Pool
+	after            int // minimum accesses before an object is cached.
+	lowWatermark     int
+	highWatermark    int
+	enableRange      bool
+	commitWriteback  bool
+	retryWritebackCh chan ObjectInfo
 	// nsMutex namespace lock
 	nsMutex *nsLockMap
 	// Object functions pointing to the corresponding functions of backend implementation.
-	NewNSLockFn func(ctx context.Context, cachePath string) RWLocker
+	NewNSLockFn func(cachePath string) RWLocker
 }
 
 // Inits the disk cache dir if it is not initialized already.
@@ -156,15 +162,17 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 		return nil, fmt.Errorf("Unable to initialize '%s' dir, %w", dir, err)
 	}
 	cache := diskCache{
-		dir:           dir,
-		triggerGC:     make(chan struct{}, 1),
-		stats:         CacheDiskStats{Dir: dir},
-		quotaPct:      quotaPct,
-		after:         config.After,
-		lowWatermark:  config.WatermarkLow,
-		highWatermark: config.WatermarkHigh,
-		enableRange:   config.Range,
-		online:        1,
+		dir:              dir,
+		triggerGC:        make(chan struct{}, 1),
+		stats:            CacheDiskStats{Dir: dir},
+		quotaPct:         quotaPct,
+		after:            config.After,
+		lowWatermark:     config.WatermarkLow,
+		highWatermark:    config.WatermarkHigh,
+		enableRange:      config.Range,
+		commitWriteback:  config.CommitWriteback,
+		retryWritebackCh: make(chan ObjectInfo, 10000),
+		online:           1,
 		pool: sync.Pool{
 			New: func() interface{} {
 				b := disk.AlignedBlock(int(cacheBlkSize))
@@ -174,9 +182,12 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 		nsMutex: newNSLock(false),
 	}
 	go cache.purgeWait(ctx)
+	if cache.commitWriteback {
+		go cache.scanCacheWritebackFailures(ctx)
+	}
 	cache.diskSpaceAvailable(0) // update if cache usage is already high.
-	cache.NewNSLockFn = func(ctx context.Context, cachePath string) RWLocker {
-		return cache.nsMutex.NewNSLock(ctx, nil, cachePath, "")
+	cache.NewNSLockFn = func(cachePath string) RWLocker {
+		return cache.nsMutex.NewNSLock(nil, cachePath, "")
 	}
 	return &cache, nil
 }
@@ -323,6 +334,12 @@ func (c *diskCache) purge(ctx context.Context) {
 		// stat all cached file ranges and cacheDataFile.
 		cachedFiles := fiStatFn(meta.Ranges, cacheDataFile, pathJoin(c.dir, name))
 		objInfo := meta.ToObjectInfo("", "")
+		// prevent gc from clearing un-synced commits. This metadata is present when
+		// cache writeback commit setting is enabled.
+		status, ok := objInfo.UserDefined[writeBackStatusHeader]
+		if ok && status != CommitComplete.String() {
+			return nil
+		}
 		cc := cacheControlOpts(objInfo)
 		for fname, fi := range cachedFiles {
 			if cc != nil {
@@ -419,8 +436,8 @@ func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectI
 // if partial object is cached.
 func (c *diskCache) statCachedMeta(ctx context.Context, cacheObjPath string) (meta *cacheMeta, partial bool, numHits int, err error) {
 
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err = cLock.GetRLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cacheObjPath)
+	if err = cLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return
 	}
 
@@ -501,8 +518,8 @@ func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (meta *c
 // incHitsOnly is true if metadata update is incrementing only the hit counter
 func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
 	cachedPath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cachedPath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cachedPath)
+	if err := cLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		return err
 	}
 	defer cLock.Unlock()
@@ -524,7 +541,11 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 	}
 	defer f.Close()
 
-	m := &cacheMeta{Version: cacheMetaVersion}
+	m := &cacheMeta{
+		Version: cacheMetaVersion,
+		Bucket:  bucket,
+		Object:  object,
+	}
 	if err := jsonLoad(f, m); err != nil && err != io.EOF {
 		return err
 	}
@@ -538,12 +559,15 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 			}
 			m.Ranges[rs.String(actualSize)] = rsFileName
 		}
-	} else {
+	}
+	if rs == nil && !incHitsOnly {
 		// this is necessary cleanup of range files if entire object is cached.
-		for _, f := range m.Ranges {
-			removeAll(pathJoin(cachedPath, f))
+		if _, err := os.Stat(pathJoin(cachedPath, cacheDataFile)); err == nil {
+			for _, f := range m.Ranges {
+				removeAll(pathJoin(cachedPath, f))
+			}
+			m.Ranges = nil
 		}
-		m.Ranges = nil
 	}
 	m.Stat.Size = actualSize
 	m.Stat.ModTime = UTCNow()
@@ -558,7 +582,6 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 			m.Meta["etag"] = etag
 		}
 	}
-
 	m.Hits++
 
 	m.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
@@ -570,22 +593,22 @@ func getCacheSHADir(dir, bucket, object string) string {
 }
 
 // Cache data to disk with bitrot checksum added for each block of 1MB
-func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Reader, size uint64) (int64, error) {
+func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Reader, size uint64) (int64, string, error) {
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
-		return 0, err
+		return 0, "", err
 	}
 	filePath := pathJoin(cachePath, fileName)
 
 	if filePath == "" || reader == nil {
-		return 0, errInvalidArgument
+		return 0, "", errInvalidArgument
 	}
 
 	if err := checkPathLength(filePath); err != nil {
-		return 0, err
+		return 0, "", err
 	}
 	f, err := os.Create(filePath)
 	if err != nil {
-		return 0, osErrToFileErr(err)
+		return 0, "", osErrToFileErr(err)
 	}
 	defer f.Close()
 
@@ -595,12 +618,12 @@ func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Rea
 
 	bufp := c.pool.Get().(*[]byte)
 	defer c.pool.Put(bufp)
-
+	md5Hash := md5.New()
 	var n, n2 int
 	for {
 		n, err = io.ReadFull(reader, *bufp)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
-			return 0, err
+			return 0, "", err
 		}
 		eof := err == io.EOF || err == io.ErrUnexpectedEOF
 		if n == 0 && size != 0 {
@@ -609,21 +632,28 @@ func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Rea
 		}
 		h.Reset()
 		if _, err = h.Write((*bufp)[:n]); err != nil {
-			return 0, err
+			return 0, "", err
 		}
 		hashBytes := h.Sum(nil)
+		// compute md5Hash of original data stream if writeback commit to cache
+		if c.commitWriteback {
+			if _, err = md5Hash.Write((*bufp)[:n]); err != nil {
+				return 0, "", err
+			}
+		}
 		if _, err = f.Write(hashBytes); err != nil {
-			return 0, err
+			return 0, "", err
 		}
 		if n2, err = f.Write((*bufp)[:n]); err != nil {
-			return 0, err
+			return 0, "", err
 		}
 		bytesWritten += int64(n2)
 		if eof {
 			break
 		}
 	}
-	return bytesWritten, nil
+
+	return bytesWritten, base64.StdEncoding.EncodeToString(md5Hash.Sum(nil)), nil
 }
 
 func newCacheEncryptReader(content io.Reader, bucket, object string, metadata map[string]string) (r io.Reader, err error) {
@@ -660,41 +690,41 @@ func newCacheEncryptMetadata(bucket, object string, metadata map[string]string)
 }
 
 // Caches the object to disk
-func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) error {
+func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly bool) (oi ObjectInfo, err error) {
 	if !c.diskSpaceAvailable(size) {
 		io.Copy(ioutil.Discard, data)
-		return errDiskFull
+		return oi, errDiskFull
 	}
 	cachePath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cachePath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
-		return err
+	cLock := c.NewNSLockFn(cachePath)
+	if err := cLock.GetLock(ctx, globalOperationTimeout); err != nil {
+		return oi, err
 	}
 	defer cLock.Unlock()
 
 	meta, _, numHits, err := c.statCache(ctx, cachePath)
 	// Case where object not yet cached
 	if os.IsNotExist(err) && c.after >= 1 {
-		return c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
+		return oi, c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
 	}
 	// Case where object already has a cache metadata entry but not yet cached
 	if err == nil && numHits < c.after {
 		cETag := extractETag(meta.Meta)
 		bETag := extractETag(opts.UserDefined)
 		if cETag == bETag {
-			return c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
+			return oi, c.saveMetadata(ctx, bucket, object, opts.UserDefined, size, nil, "", false)
 		}
 		incHitsOnly = true
 	}
 
 	if rs != nil {
-		return c.putRange(ctx, bucket, object, data, size, rs, opts)
+		return oi, c.putRange(ctx, bucket, object, data, size, rs, opts)
 	}
 	if !c.diskSpaceAvailable(size) {
-		return errDiskFull
+		return oi, errDiskFull
 	}
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
-		return err
+		return oi, err
 	}
 	var metadata = cloneMSS(opts.UserDefined)
 	var reader = data
@@ -702,25 +732,39 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 	if globalCacheKMS != nil {
 		reader, err = newCacheEncryptReader(data, bucket, object, metadata)
 		if err != nil {
-			return err
+			return oi, err
 		}
 		actualSize, _ = sio.EncryptedSize(uint64(size))
 	}
-	n, err := c.bitrotWriteToCache(cachePath, cacheDataFile, reader, actualSize)
+	n, md5sum, err := c.bitrotWriteToCache(cachePath, cacheDataFile, reader, actualSize)
 	if IsErr(err, baseErrs...) {
 		// take the cache drive offline
 		c.setOffline()
 	}
 	if err != nil {
 		removeAll(cachePath)
-		return err
+		return oi, err
 	}
 
 	if actualSize != uint64(n) {
 		removeAll(cachePath)
-		return IncompleteBody{Bucket: bucket, Object: object}
+		return oi, IncompleteBody{Bucket: bucket, Object: object}
 	}
-	return c.saveMetadata(ctx, bucket, object, metadata, n, nil, "", incHitsOnly)
+	if c.commitWriteback {
+		metadata["content-md5"] = md5sum
+		if md5bytes, err := base64.StdEncoding.DecodeString(md5sum); err == nil {
+			metadata["etag"] = hex.EncodeToString(md5bytes)
+		}
+		metadata[writeBackStatusHeader] = CommitPending.String()
+	}
+	return ObjectInfo{
+			Bucket:      bucket,
+			Name:        object,
+			ETag:        metadata["etag"],
+			Size:        n,
+			UserDefined: metadata,
+		},
+		c.saveMetadata(ctx, bucket, object, metadata, n, nil, "", incHitsOnly)
 }
 
 // Caches the range to disk
@@ -751,7 +795,7 @@ func (c *diskCache) putRange(ctx context.Context, bucket, object string, data io
 
 	}
 	cacheFile := MustGetUUID()
-	n, err := c.bitrotWriteToCache(cachePath, cacheFile, reader, actualSize)
+	n, _, err := c.bitrotWriteToCache(cachePath, cacheFile, reader, actualSize)
 	if IsErr(err, baseErrs...) {
 		// take the cache drive offline
 		c.setOffline()
@@ -866,8 +910,8 @@ func (c *diskCache) bitrotReadFromCache(ctx context.Context, filePath string, of
 // Get returns ObjectInfo and reader for object from disk cache
 func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, numHits int, err error) {
 	cacheObjPath := getCacheSHADir(c.dir, bucket, object)
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err := cLock.GetRLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cacheObjPath)
+	if err := cLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return nil, numHits, err
 	}
 
@@ -930,8 +974,8 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 
 // Deletes the cached object
 func (c *diskCache) delete(ctx context.Context, cacheObjPath string) (err error) {
-	cLock := c.NewNSLockFn(ctx, cacheObjPath)
-	if err := cLock.GetLock(globalOperationTimeout); err != nil {
+	cLock := c.NewNSLockFn(cacheObjPath)
+	if err := cLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		return err
 	}
 	defer cLock.Unlock()
@@ -951,3 +995,36 @@ func (c *diskCache) Exists(ctx context.Context, bucket, object string) bool {
 	}
 	return true
 }
+
+// queues writeback upload failures on server startup
+func (c *diskCache) scanCacheWritebackFailures(ctx context.Context) {
+	defer close(c.retryWritebackCh)
+	filterFn := func(name string, typ os.FileMode) error {
+		if name == minioMetaBucket {
+			// Proceed to next file.
+			return nil
+		}
+		cacheDir := pathJoin(c.dir, name)
+		meta, _, _, err := c.statCachedMeta(ctx, cacheDir)
+		if err != nil {
+			return nil
+		}
+
+		objInfo := meta.ToObjectInfo("", "")
+		status, ok := objInfo.UserDefined[writeBackStatusHeader]
+		if !ok || status == CommitComplete.String() {
+			return nil
+		}
+		select {
+		case c.retryWritebackCh <- objInfo:
+		default:
+		}
+
+		return nil
+	}
+
+	if err := readDirFilterFn(c.dir, filterFn); err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+}

cmd/disk-cache.go

@@ -22,24 +22,47 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"github.com/minio/minio/cmd/config/cache"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/color"
+	"github.com/minio/minio/pkg/hash"
 	"github.com/minio/minio/pkg/sync/errgroup"
 	"github.com/minio/minio/pkg/wildcard"
 )
 
 const (
-	cacheBlkSize    = int64(1 * 1024 * 1024)
-	cacheGCInterval = time.Minute * 30
+	cacheBlkSize          = 1 << 20
+	cacheGCInterval       = time.Minute * 30
+	writeBackStatusHeader = ReservedMetadataPrefixLower + "write-back-status"
+	writeBackRetryHeader  = ReservedMetadataPrefixLower + "write-back-retry"
 )
 
+type cacheCommitStatus string
+
+const (
+	// CommitPending - cache writeback with backend is pending.
+	CommitPending cacheCommitStatus = "pending"
+
+	// CommitComplete - cache writeback completed ok.
+	CommitComplete cacheCommitStatus = "complete"
+
+	// CommitFailed - cache writeback needs a retry.
+	CommitFailed cacheCommitStatus = "failed"
+)
+
+// String returns string representation of status
+func (s cacheCommitStatus) String() string {
+	return string(s)
+}
+
 // CacheStorageInfo - represents total, free capacity of
 // underlying cache storage.
 type CacheStorageInfo struct {
@@ -69,19 +92,22 @@ type cacheObjects struct {
 	exclude []string
 	// number of accesses after which to cache an object
 	after int
+	// commit objects in async manner
+	commitWriteback bool
 	// if true migration is in progress from v1 to v2
 	migrating bool
 	// mutex to protect migration bool
 	migMutex sync.Mutex
-
+	// retry queue for writeback cache mode to reattempt upload to backend
+	wbRetryCh chan ObjectInfo
 	// Cache stats
 	cacheStats *CacheStats
 
-	GetObjectNInfoFn func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
-	GetObjectInfoFn  func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	DeleteObjectFn   func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	PutObjectFn      func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
-	CopyObjectFn     func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
+	InnerGetObjectNInfoFn func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
+	InnerGetObjectInfoFn  func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	InnerDeleteObjectFn   func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	InnerPutObjectFn      func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
+	InnerCopyObjectFn     func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error)
 }
 
 func (c *cacheObjects) incHitsToMeta(ctx context.Context, dcache *diskCache, bucket, object string, size int64, eTag string, rs *HTTPRangeSpec) error {
@@ -120,7 +146,7 @@ func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *disk
 
 // DeleteObject clears cache entry if backend delete operation succeeds
 func (c *cacheObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	if objInfo, err = c.DeleteObjectFn(ctx, bucket, object, opts); err != nil {
+	if objInfo, err = c.InnerDeleteObjectFn(ctx, bucket, object, opts); err != nil {
 		return
 	}
 	if c.isCacheExclude(bucket, object) || c.skipCache() {
@@ -188,14 +214,14 @@ func (c *cacheObjects) incCacheStats(size int64) {
 
 func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	if c.isCacheExclude(bucket, object) || c.skipCache() {
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
 	var cc *cacheControl
 	var cacheObjSize int64
 	// fetch diskCache if object is currently cached or nearest available cache drive
 	dcache, err := c.getCacheToLoc(ctx, bucket, object)
 	if err != nil {
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
 
 	cacheReader, numCacheHits, cacheErr := dcache.Get(ctx, bucket, object, rs, h, opts)
@@ -224,14 +250,14 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		if cc != nil && cc.noStore {
 			cacheReader.Close()
 			c.cacheStats.incMiss()
-			bReader, err := c.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+			bReader, err := c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 			bReader.ObjInfo.CacheLookupStatus = CacheHit
 			bReader.ObjInfo.CacheStatus = CacheMiss
 			return bReader, err
 		}
 	}
 
-	objInfo, err := c.GetObjectInfoFn(ctx, bucket, object, opts)
+	objInfo, err := c.InnerGetObjectInfoFn(ctx, bucket, object, opts)
 	if backendDownError(err) && cacheErr == nil {
 		c.incCacheStats(cacheObjSize)
 		return cacheReader, nil
@@ -255,7 +281,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			cacheReader.Close()
 		}
 		c.cacheStats.incMiss()
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
 	// skip cache for objects with locks
 	objRetention := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
@@ -265,7 +291,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			cacheReader.Close()
 		}
 		c.cacheStats.incMiss()
-		return c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 	}
 	if cacheErr == nil {
 		// if ETag matches for stale cache entry, serve from cache
@@ -283,7 +309,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	// Reaching here implies cache miss
 	c.cacheStats.incMiss()
 
-	bkReader, bkErr := c.GetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+	bkReader, bkErr := c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
 
 	if bkErr != nil {
 		return bkReader, bkErr
@@ -305,14 +331,12 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		return bkReader, bkErr
 	}
 
-	if rs != nil {
+	if rs != nil && !dcache.enableRange {
 		go func() {
 			// if range caching is disabled, download entire object.
-			if !dcache.enableRange {
-				rs = nil
-			}
+			rs = nil
 			// fill cache in the background for range GET requests
-			bReader, bErr := c.GetObjectNInfoFn(GlobalContext, bucket, object, rs, h, lockType, opts)
+			bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, rs, h, lockType, opts)
 			if bErr != nil {
 				return
 			}
@@ -335,9 +359,9 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	teeReader := io.TeeReader(bkReader, pipeWriter)
 	userDefined := getMetadata(bkReader.ObjInfo)
 	go func() {
-		putErr := dcache.Put(ctx, bucket, object,
+		_, putErr := dcache.Put(ctx, bucket, object,
 			io.LimitReader(pipeReader, bkReader.ObjInfo.Size),
-			bkReader.ObjInfo.Size, nil, ObjectOptions{
+			bkReader.ObjInfo.Size, rs, ObjectOptions{
 				UserDefined: userDefined,
 			}, false)
 		// close the write end of the pipe, so the error gets
@@ -351,7 +375,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 
 // Returns ObjectInfo from cache if available.
 func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
-	getObjectInfoFn := c.GetObjectInfoFn
+	getObjectInfoFn := c.InnerGetObjectInfoFn
 
 	if c.isCacheExclude(bucket, object) || c.skipCache() {
 		return getObjectInfoFn(ctx, bucket, object, opts)
@@ -409,7 +433,7 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 
 // CopyObject reverts to backend after evicting any stale cache entries
 func (c *cacheObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
-	copyObjectFn := c.CopyObjectFn
+	copyObjectFn := c.InnerCopyObjectFn
 	if c.isCacheExclude(srcBucket, srcObject) || c.skipCache() {
 		return copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts)
 	}
@@ -596,7 +620,7 @@ func (c *cacheObjects) migrateCacheFromV1toV2(ctx context.Context) {
 
 // PutObject - caches the uploaded object for single Put operations
 func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	putObjectFn := c.PutObjectFn
+	putObjectFn := c.InnerPutObjectFn
 	dcache, err := c.getCacheToLoc(ctx, bucket, object)
 	if err != nil {
 		// disk cache could not be located,execute backend call.
@@ -631,13 +655,20 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 		dcache.Delete(ctx, bucket, object)
 		return putObjectFn(ctx, bucket, object, r, opts)
 	}
-
+	if c.commitWriteback {
+		oi, err := dcache.Put(ctx, bucket, object, r, r.Size(), nil, opts, false)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		go c.uploadObject(GlobalContext, oi)
+		return oi, nil
+	}
 	objInfo, err = putObjectFn(ctx, bucket, object, r, opts)
 
 	if err == nil {
 		go func() {
 			// fill cache in the background
-			bReader, bErr := c.GetObjectNInfoFn(GlobalContext, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
+			bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
 			if bErr != nil {
 				return
 			}
@@ -649,9 +680,68 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 			}
 		}()
 	}
-
 	return objInfo, err
+}
 
+// upload cached object to backend in async commit mode.
+func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
+	dcache, err := c.getCacheToLoc(ctx, oi.Bucket, oi.Name)
+	if err != nil {
+		// disk cache could not be located.
+		logger.LogIf(ctx, fmt.Errorf("Could not upload %s/%s to backend: %w", oi.Bucket, oi.Name, err))
+		return
+	}
+	cReader, _, bErr := dcache.Get(ctx, oi.Bucket, oi.Name, nil, http.Header{}, ObjectOptions{})
+	if bErr != nil {
+		return
+	}
+	defer cReader.Close()
+
+	if cReader.ObjInfo.ETag != oi.ETag {
+		return
+	}
+	st := cacheCommitStatus(oi.UserDefined[writeBackStatusHeader])
+	if st == CommitComplete || st.String() == "" {
+		return
+	}
+	hashReader, err := hash.NewReader(cReader, oi.Size, "", "", oi.Size, globalCLIContext.StrictS3Compat)
+	if err != nil {
+		return
+	}
+	var opts ObjectOptions
+	opts.UserDefined = make(map[string]string)
+	opts.UserDefined[xhttp.ContentMD5] = oi.UserDefined["content-md5"]
+	objInfo, err := c.InnerPutObjectFn(ctx, oi.Bucket, oi.Name, NewPutObjReader(hashReader, nil, nil), opts)
+	wbCommitStatus := CommitComplete
+	if err != nil {
+		wbCommitStatus = CommitFailed
+	}
+
+	meta := cloneMSS(cReader.ObjInfo.UserDefined)
+	retryCnt := 0
+	if wbCommitStatus == CommitFailed {
+		retryCnt, _ = strconv.Atoi(meta[writeBackRetryHeader])
+		retryCnt++
+		meta[writeBackRetryHeader] = strconv.Itoa(retryCnt)
+	} else {
+		delete(meta, writeBackRetryHeader)
+	}
+	meta[writeBackStatusHeader] = wbCommitStatus.String()
+	meta["etag"] = oi.ETag
+	dcache.SaveMetadata(ctx, oi.Bucket, oi.Name, meta, objInfo.Size, nil, "", false)
+	if retryCnt > 0 {
+		// slow down retries
+		time.Sleep(time.Second * time.Duration(retryCnt%10+1))
+		c.queueWritebackRetry(oi)
+	}
+}
+
+func (c *cacheObjects) queueWritebackRetry(oi ObjectInfo) {
+	select {
+	case c.wbRetryCh <- oi:
+		c.uploadObject(GlobalContext, oi)
+	default:
+	}
 }
 
 // Returns cacheObjects for use by Server.
@@ -662,25 +752,26 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		return nil, err
 	}
 	c := &cacheObjects{
-		cache:      cache,
-		exclude:    config.Exclude,
-		after:      config.After,
-		migrating:  migrateSw,
-		migMutex:   sync.Mutex{},
-		cacheStats: newCacheStats(),
-		GetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+		cache:           cache,
+		exclude:         config.Exclude,
+		after:           config.After,
+		migrating:       migrateSw,
+		migMutex:        sync.Mutex{},
+		commitWriteback: config.CommitWriteback,
+		cacheStats:      newCacheStats(),
+		InnerGetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
 		},
-		GetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+		InnerGetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 		},
-		DeleteObjectFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
+		InnerDeleteObjectFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().DeleteObject(ctx, bucket, object, opts)
 		},
-		PutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
+		InnerPutObjectFn: func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().PutObject(ctx, bucket, object, data, opts)
 		},
-		CopyObjectFn: func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
+		InnerCopyObjectFn: func(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) {
 			return newObjectLayerFn().CopyObject(ctx, srcBucket, srcObject, destBucket, destObject, srcInfo, srcOpts, dstOpts)
 		},
 	}
@@ -701,6 +792,15 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		go c.migrateCacheFromV1toV2(ctx)
 	}
 	go c.gc(ctx)
+	if c.commitWriteback {
+		c.wbRetryCh = make(chan ObjectInfo, 10000)
+		go func() {
+			<-GlobalContext.Done()
+			close(c.wbRetryCh)
+		}()
+		go c.queuePendingWriteback(ctx)
+	}
+
 	return c, nil
 }
 
@@ -726,3 +826,25 @@ func (c *cacheObjects) gc(ctx context.Context) {
 		}
 	}
 }
+
+// queues any pending or failed async commits when server restarts
+func (c *cacheObjects) queuePendingWriteback(ctx context.Context) {
+	for _, dcache := range c.cache {
+		if dcache != nil {
+			for {
+				select {
+				case <-ctx.Done():
+					return
+				case oi, ok := <-dcache.retryWritebackCh:
+					if !ok {
+						goto next
+					}
+					c.queueWritebackRetry(oi)
+				default:
+					time.Sleep(time.Second * 1)
+				}
+			}
+		next:
+		}
+	}
+}

cmd/encryption-v1_test.go

@@ -19,10 +19,14 @@ package cmd
 import (
 	"bytes"
 	"encoding/base64"
+	"encoding/json"
+	"fmt"
 	"net/http"
+	"os"
 	"testing"
 
 	humanize "github.com/dustin/go-humanize"
+	"github.com/klauspost/compress/zstd"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio/cmd/crypto"
 	"github.com/minio/sio"
@@ -622,3 +626,89 @@ func TestGetDefaultOpts(t *testing.T) {
 		}
 	}
 }
+func Test_decryptObjectInfo(t *testing.T) {
+	var testSet []struct {
+		Bucket  string
+		Name    string
+		UserDef map[string]string
+	}
+	file, err := os.Open("testdata/decryptObjectInfo.json.zst")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer file.Close()
+	dec, err := zstd.NewReader(file)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer dec.Close()
+	js := json.NewDecoder(dec)
+	err = js.Decode(&testSet)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574")
+	defer os.Setenv("MINIO_KMS_MASTER_KEY", "")
+	GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var dst [32]byte
+	for i := range testSet {
+		t.Run(fmt.Sprint("case-", i), func(t *testing.T) {
+			test := &testSet[i]
+			_, err := decryptObjectInfo(dst[:], test.Bucket, test.Name, test.UserDef)
+			if err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}
+
+func Benchmark_decryptObjectInfo(b *testing.B) {
+	var testSet []struct {
+		Bucket  string
+		Name    string
+		UserDef map[string]string
+	}
+	file, err := os.Open("testdata/decryptObjectInfo.json.zst")
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer file.Close()
+	dec, err := zstd.NewReader(file)
+	if err != nil {
+		b.Fatal(err)
+	}
+	defer dec.Close()
+	js := json.NewDecoder(dec)
+	err = js.Decode(&testSet)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	os.Setenv("MINIO_KMS_MASTER_KEY", "my-minio-key:6368616e676520746869732070617373776f726420746f206120736563726574")
+	defer os.Setenv("MINIO_KMS_MASTER_KEY", "")
+	GlobalKMS, err = crypto.NewKMS(crypto.KMSConfig{})
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	b.SetBytes(int64(len(testSet)))
+	b.RunParallel(func(pb *testing.PB) {
+		for pb.Next() {
+			var dst [32]byte
+			for i := range testSet {
+				test := &testSet[i]
+				_, err := decryptObjectInfo(dst[:], test.Bucket, test.Name, test.UserDef)
+				if err != nil {
+					b.Fatal(err)
+				}
+			}
+		}
+	})
+}

cmd/endpoint-ellipses.go

@@ -329,7 +329,7 @@ var (
 // CreateServerEndpoints - validates and creates new endpoints from input args, supports
 // both ellipses and without ellipses transparently.
 func createServerEndpoints(serverAddr string, args ...string) (
-	endpointZones EndpointZones, setupType SetupType, err error) {
+	endpointServerSets EndpointServerSets, setupType SetupType, err error) {
 
 	if len(args) == 0 {
 		return nil, -1, errInvalidArgument
@@ -352,13 +352,13 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		if err != nil {
 			return nil, -1, err
 		}
-		endpointZones = append(endpointZones, ZoneEndpoints{
+		endpointServerSets = append(endpointServerSets, ZoneEndpoints{
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
 		})
 		setupType = newSetupType
-		return endpointZones, setupType, nil
+		return endpointServerSets, setupType, nil
 	}
 
 	var prevSetupType SetupType
@@ -374,12 +374,12 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			return nil, -1, err
 		}
 		if setDriveCount != 0 && setDriveCount != len(setArgs[0]) {
-			return nil, -1, fmt.Errorf("All zones should have same drive per set ratio - expected %d, got %d", setDriveCount, len(setArgs[0]))
+			return nil, -1, fmt.Errorf("All serverSets should have same drive per set ratio - expected %d, got %d", setDriveCount, len(setArgs[0]))
 		}
 		if prevSetupType != UnknownSetupType && prevSetupType != setupType {
-			return nil, -1, fmt.Errorf("All zones should be of the same setup-type to maintain the original SLA expectations - expected %s, got %s", prevSetupType, setupType)
+			return nil, -1, fmt.Errorf("All serverSets should be of the same setup-type to maintain the original SLA expectations - expected %s, got %s", prevSetupType, setupType)
 		}
-		if err = endpointZones.Add(ZoneEndpoints{
+		if err = endpointServerSets.Add(ZoneEndpoints{
 			SetCount:     len(setArgs),
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
@@ -393,5 +393,5 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		prevSetupType = setupType
 	}
 
-	return endpointZones, setupType, nil
+	return endpointServerSets, setupType, nil
 }

cmd/endpoint.go

@@ -17,7 +17,9 @@
 package cmd
 
 import (
+	"context"
 	"crypto/tls"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -26,17 +28,20 @@ import (
 	"path/filepath"
 	"reflect"
 	"runtime"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/cmd/config"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/mountinfo"
+	xnet "github.com/minio/minio/pkg/net"
 )
 
 // EndpointType - enum for endpoint type.
@@ -201,12 +206,12 @@ type ZoneEndpoints struct {
 	Endpoints    Endpoints
 }
 
-// EndpointZones - list of list of endpoints
-type EndpointZones []ZoneEndpoints
+// EndpointServerSets - list of list of endpoints
+type EndpointServerSets []ZoneEndpoints
 
 // GetLocalZoneIdx returns the zone which endpoint belongs to locally.
 // if ep is remote this code will return -1 zoneIndex
-func (l EndpointZones) GetLocalZoneIdx(ep Endpoint) int {
+func (l EndpointServerSets) GetLocalZoneIdx(ep Endpoint) int {
 	for i, zep := range l {
 		for _, cep := range zep.Endpoints {
 			if cep.IsLocal && ep.IsLocal {
@@ -220,14 +225,14 @@ func (l EndpointZones) GetLocalZoneIdx(ep Endpoint) int {
 }
 
 // Add add zone endpoints
-func (l *EndpointZones) Add(zeps ZoneEndpoints) error {
+func (l *EndpointServerSets) Add(zeps ZoneEndpoints) error {
 	existSet := set.NewStringSet()
 	for _, zep := range *l {
 		for _, ep := range zep.Endpoints {
 			existSet.Add(ep.String())
 		}
 	}
-	// Validate if there are duplicate endpoints across zones
+	// Validate if there are duplicate endpoints across serverSets
 	for _, ep := range zeps.Endpoints {
 		if existSet.Contains(ep.String()) {
 			return fmt.Errorf("duplicate endpoints found")
@@ -238,17 +243,17 @@ func (l *EndpointZones) Add(zeps ZoneEndpoints) error {
 }
 
 // FirstLocal returns true if the first endpoint is local.
-func (l EndpointZones) FirstLocal() bool {
+func (l EndpointServerSets) FirstLocal() bool {
 	return l[0].Endpoints[0].IsLocal
 }
 
 // HTTPS - returns true if secure for URLEndpointType.
-func (l EndpointZones) HTTPS() bool {
+func (l EndpointServerSets) HTTPS() bool {
 	return l[0].Endpoints.HTTPS()
 }
 
 // NEndpoints - returns all nodes count
-func (l EndpointZones) NEndpoints() (count int) {
+func (l EndpointServerSets) NEndpoints() (count int) {
 	for _, ep := range l {
 		count += len(ep.Endpoints)
 	}
@@ -256,7 +261,7 @@ func (l EndpointZones) NEndpoints() (count int) {
 }
 
 // Hostnames - returns list of unique hostnames
-func (l EndpointZones) Hostnames() []string {
+func (l EndpointServerSets) Hostnames() []string {
 	foundSet := set.NewStringSet()
 	for _, ep := range l {
 		for _, endpoint := range ep.Endpoints {
@@ -269,6 +274,52 @@ func (l EndpointZones) Hostnames() []string {
 	return foundSet.ToSlice()
 }
 
+// hostsSorted will return all hosts found.
+// The LOCAL host will be nil, but the indexes of all hosts should
+// remain consistent across the cluster.
+func (l EndpointServerSets) hostsSorted() []*xnet.Host {
+	peers, localPeer := l.peers()
+	sort.Strings(peers)
+	hosts := make([]*xnet.Host, len(peers))
+	for i, hostStr := range peers {
+		if hostStr == localPeer {
+			continue
+		}
+		host, err := xnet.ParseHost(hostStr)
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
+			continue
+		}
+		hosts[i] = host
+	}
+
+	return hosts
+}
+
+// peers will return all peers, including local.
+// The local peer is returned as a separate string.
+func (l EndpointServerSets) peers() (peers []string, local string) {
+	allSet := set.NewStringSet()
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.Type() != URLEndpointType {
+				continue
+			}
+
+			peer := endpoint.Host
+			if endpoint.IsLocal {
+				if _, port := mustSplitHostPort(peer); port == globalMinioPort {
+					local = peer
+				}
+			}
+
+			allSet.Add(peer)
+		}
+	}
+
+	return allSet.ToSlice(), local
+}
+
 // Endpoints - list of same type of endpoint.
 type Endpoints []Endpoint
 
@@ -688,9 +739,9 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 // the first element from the set of peers which indicate that
 // they are local. There is always one entry that is local
 // even with repeated server endpoints.
-func GetLocalPeer(endpointZones EndpointZones) (localPeer string) {
+func GetLocalPeer(endpointServerSets EndpointServerSets) (localPeer string) {
 	peerSet := set.NewStringSet()
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			if endpoint.Type() != URLEndpointType {
 				continue
@@ -712,28 +763,6 @@ func GetLocalPeer(endpointZones EndpointZones) (localPeer string) {
 	return peerSet.ToSlice()[0]
 }
 
-// GetRemotePeers - get hosts information other than this minio service.
-func GetRemotePeers(endpointZones EndpointZones) []string {
-	peerSet := set.NewStringSet()
-	for _, ep := range endpointZones {
-		for _, endpoint := range ep.Endpoints {
-			if endpoint.Type() != URLEndpointType {
-				continue
-			}
-
-			peer := endpoint.Host
-			if endpoint.IsLocal {
-				if _, port := mustSplitHostPort(peer); port == globalMinioPort {
-					continue
-				}
-			}
-
-			peerSet.Add(peer)
-		}
-	}
-	return peerSet.ToSlice()
-}
-
 // GetProxyEndpointLocalIndex returns index of the local proxy endpoint
 func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
 	for i, pep := range proxyEps {
@@ -744,13 +773,79 @@ func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int {
 	return -1
 }
 
+func httpDo(clnt *http.Client, req *http.Request, f func(*http.Response, error) error) error {
+	ctx, cancel := context.WithTimeout(GlobalContext, 200*time.Millisecond)
+	defer cancel()
+
+	// Run the HTTP request in a goroutine and pass the response to f.
+	c := make(chan error, 1)
+	req = req.WithContext(ctx)
+	go func() { c <- f(clnt.Do(req)) }()
+	select {
+	case <-ctx.Done():
+		<-c // Wait for f to return.
+		return ctx.Err()
+	case err := <-c:
+		return err
+	}
+}
+
+func getOnlineProxyEndpointIdx() int {
+	type reqIndex struct {
+		Request *http.Request
+		Idx     int
+	}
+
+	proxyRequests := make(map[*http.Client]reqIndex, len(globalProxyEndpoints))
+	for i, proxyEp := range globalProxyEndpoints {
+		proxyEp := proxyEp
+		serverURL := &url.URL{
+			Scheme: proxyEp.Scheme,
+			Host:   proxyEp.Host,
+			Path:   pathJoin(healthCheckPathPrefix, healthCheckLivenessPath),
+		}
+
+		req, err := http.NewRequest(http.MethodGet, serverURL.String(), nil)
+		if err != nil {
+			continue
+		}
+
+		proxyRequests[&http.Client{
+			Transport: proxyEp.Transport,
+		}] = reqIndex{
+			Request: req,
+			Idx:     i,
+		}
+	}
+
+	for c, r := range proxyRequests {
+		if err := httpDo(c, r.Request, func(resp *http.Response, err error) error {
+			if err != nil {
+				return err
+			}
+			xhttp.DrainBody(resp.Body)
+			if resp.StatusCode != http.StatusOK {
+				return errors.New(resp.Status)
+			}
+			if v := resp.Header.Get(xhttp.MinIOServerStatus); v == unavailable {
+				return errors.New(v)
+			}
+			return nil
+		}); err != nil {
+			continue
+		}
+		return r.Idx
+	}
+	return -1
+}
+
 // GetProxyEndpoints - get all endpoints that can be used to proxy list request.
-func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) {
+func GetProxyEndpoints(endpointServerSets EndpointServerSets) []ProxyEndpoint {
 	var proxyEps []ProxyEndpoint
 
 	proxyEpSet := set.NewStringSet()
 
-	for _, ep := range endpointZones {
+	for _, ep := range endpointServerSets {
 		for _, endpoint := range ep.Endpoints {
 			if endpoint.Type() != URLEndpointType {
 				continue
@@ -770,11 +865,8 @@ func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) {
 				}
 			}
 
-			tr := newCustomHTTPTransport(tlsConfig, rest.DefaultTimeout)()
-			// Allow more requests to be in flight with higher response header timeout.
-			tr.ResponseHeaderTimeout = 30 * time.Minute
-			tr.MaxIdleConns = 64
-			tr.MaxIdleConnsPerHost = 64
+			// allow transport to be HTTP/1.1 for proxying.
+			tr := newCustomHTTPProxyTransport(tlsConfig, rest.DefaultTimeout)()
 
 			proxyEps = append(proxyEps, ProxyEndpoint{
 				Endpoint:  endpoint,
@@ -782,7 +874,7 @@ func GetProxyEndpoints(endpointZones EndpointZones) ([]ProxyEndpoint, error) {
 			})
 		}
 	}
-	return proxyEps, nil
+	return proxyEps
 }
 
 func updateDomainIPs(endPoints set.StringSet) {

cmd/endpoint_test.go

@@ -380,24 +380,28 @@ func TestGetRemotePeers(t *testing.T) {
 	testCases := []struct {
 		endpointArgs   []string
 		expectedResult []string
+		expectedLocal  string
 	}{
-		{[]string{"/d1", "/d2", "d3", "d4"}, []string{}},
-		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000"}},
-		{[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000"}},
-		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000"}},
-		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9001", "localhost:9002", "localhost:9003"}},
+		{[]string{"/d1", "/d2", "d3", "d4"}, []string{}, ""},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:9000"}, "localhost:9000"},
+		{[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000", "localhost:9000"}, "localhost:9000"},
+		{[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000", "localhost:9000"}, "localhost:9000"},
+		{[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9000", "localhost:9001", "localhost:9002", "localhost:9003"}, "localhost:9000"},
 	}
 
 	for _, testCase := range testCases {
 		zendpoints := mustGetZoneEndpoints(testCase.endpointArgs...)
 		if !zendpoints[0].Endpoints[0].IsLocal {
 			if err := zendpoints[0].Endpoints.UpdateIsLocal(false); err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
+				t.Errorf("error: expected = <nil>, got = %v", err)
 			}
 		}
-		remotePeers := GetRemotePeers(zendpoints)
+		remotePeers, local := zendpoints.peers()
 		if !reflect.DeepEqual(remotePeers, testCase.expectedResult) {
-			t.Fatalf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
+			t.Errorf("expected: %v, got: %v", testCase.expectedResult, remotePeers)
+		}
+		if local != testCase.expectedLocal {
+			t.Errorf("expected: %v, got: %v", testCase.expectedLocal, local)
 		}
 	}
 }

cmd/erasure-bucket.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio/cmd/logger"
@@ -49,7 +50,7 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
 		g.Go(func() error {
 			if storageDisks[index] != nil {
 				if err := storageDisks[index].MakeVol(ctx, bucket); err != nil {
-					if err != errVolumeExists {
+					if !errors.Is(err, errVolumeExists) {
 						logger.LogIf(ctx, err)
 					}
 					return err
@@ -157,6 +158,7 @@ func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs
 // DeleteBucket - deletes a bucket.
 func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
 	// Collect if all disks report volume not found.
+	defer ObjectPathUpdated(bucket + slashSeparator)
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))

cmd/erasure-common.go

@@ -37,12 +37,48 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	return newDisks
 }
 
+func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
+	disks := er.getDisks()
+	var wg sync.WaitGroup
+	var mu sync.Mutex
+	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+		i := i
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			if disks[i-1] == nil {
+				return
+			}
+			di, err := disks[i-1].DiskInfo(context.Background())
+			if err != nil || di.Healing {
+
+				// - Do not consume disks which are not reachable
+				//   unformatted or simply not accessible for some reason.
+				//
+				// - Do not consume disks which are being healed
+				//
+				// - Future: skip busy disks
+				return
+			}
+
+			mu.Lock()
+			newDisks = append(newDisks, disks[i-1])
+			mu.Unlock()
+		}()
+	}
+	wg.Wait()
+	return newDisks
+}
+
 // getLoadBalancedNDisks - fetches load balanced (sufficiently randomized) disk slice
 // with N disks online. If ndisks is zero or negative, then it will returns all disks,
 // same if ndisks is greater than the number of all disks.
 func (er erasureObjects) getLoadBalancedNDisks(ndisks int) (newDisks []StorageAPI) {
-	disks := er.getLoadBalancedDisks()
+	disks := er.getLoadBalancedDisks(ndisks != -1)
 	for _, disk := range disks {
+		if disk == nil {
+			continue
+		}
 		newDisks = append(newDisks, disk)
 		ndisks--
 		if ndisks == 0 {
@@ -54,9 +90,17 @@ func (er erasureObjects) getLoadBalancedNDisks(ndisks int) (newDisks []StorageAP
 
 // getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
 // ensures to skip disks if they are not healing and online.
-func (er erasureObjects) getLoadBalancedDisks() []StorageAPI {
+func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 	disks := er.getDisks()
 
+	if !optimized {
+		var newDisks []StorageAPI
+		for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+			newDisks = append(newDisks, disks[i-1])
+		}
+		return newDisks
+	}
+
 	var wg sync.WaitGroup
 	var mu sync.Mutex
 	var newDisks = map[uint64][]StorageAPI{}
@@ -81,8 +125,8 @@ func (er erasureObjects) getLoadBalancedDisks() []StorageAPI {
 			}
 
 			mu.Lock()
-			// Capture disks usage wise
-			newDisks[di.Used] = append(newDisks[di.Used], disks[i-1])
+			// Capture disks usage wise upto resolution of MiB
+			newDisks[di.Used/1024/1024] = append(newDisks[di.Used/1024/1024], disks[i-1])
 			mu.Unlock()
 		}()
 	}

cmd/erasure-common_test.go

@@ -55,8 +55,8 @@ func TestErasureParentDirIsObject(t *testing.T) {
 		t.Fatalf("Unexpected object name returned got %s, expected %s", objInfo.Name, objectName)
 	}
 
-	z := obj.(*erasureZones)
-	xl := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	xl := z.serverSets[0].sets[0]
 	testCases := []struct {
 		parentIsObject bool
 		objectName     string

cmd/erasure-decode_test.go

@@ -203,8 +203,9 @@ func TestErasureDecode(t *testing.T) {
 // This test is t.Skip()ed as it a long time to run, hence should be run
 // explicitly after commenting out t.Skip()
 func TestErasureDecodeRandomOffsetLength(t *testing.T) {
-	// Comment the following line to run this test.
-	t.SkipNow()
+	if testing.Short() {
+		t.Skip()
+	}
 	// Initialize environment needed for the test.
 	dataBlocks := 7
 	parityBlocks := 7

cmd/erasure-encode_test.go

@@ -195,7 +195,7 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
 			if disk == OfflineDisk {
 				continue
 			}
-			disk.DeleteFile(context.Background(), "testbucket", "object")
+			disk.Delete(context.Background(), "testbucket", "object", false)
 			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)

cmd/erasure-healing-common.go

@@ -26,22 +26,25 @@ import (
 // commonTime returns a maximally occurring time from a list of time.
 func commonTime(modTimes []time.Time) (modTime time.Time, count int) {
 	var maxima int // Counter for remembering max occurrence of elements.
-	timeOccurenceMap := make(map[time.Time]int)
+	timeOccurenceMap := make(map[int64]int)
 	// Ignore the uuid sentinel and count the rest.
 	for _, time := range modTimes {
 		if time.Equal(timeSentinel) {
 			continue
 		}
-		timeOccurenceMap[time]++
+		timeOccurenceMap[time.UnixNano()]++
 	}
+
 	// Find the common cardinality from previously collected
 	// occurrences of elements.
-	for time, count := range timeOccurenceMap {
-		if count > maxima || (count == maxima && time.After(modTime)) {
+	for nano, count := range timeOccurenceMap {
+		t := time.Unix(0, nano)
+		if count > maxima || (count == maxima && t.After(modTime)) {
 			maxima = count
-			modTime = time
+			modTime = t
 		}
 	}
+
 	// Return the collected common uuid.
 	return modTime, maxima
 }
@@ -160,6 +163,32 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	dataErrs := make([]error, len(onlineDisks))
 
+	inconsistent := 0
+	for i, meta := range partsMetadata {
+		if !meta.IsValid() {
+			// Since for majority of the cases erasure.Index matches with erasure.Distribution we can
+			// consider the offline disks as consistent.
+			continue
+		}
+		if len(meta.Erasure.Distribution) != len(onlineDisks) {
+			// Erasure distribution seems to have lesser
+			// number of items than number of online disks.
+			inconsistent++
+			continue
+		}
+		if meta.Erasure.Distribution[i] != meta.Erasure.Index {
+			// Mismatch indexes with distribution order
+			inconsistent++
+		}
+	}
+
+	erasureDistributionReliable := true
+	if inconsistent > len(partsMetadata)/2 {
+		// If there are too many inconsistent files, then we can't trust erasure.Distribution (most likely
+		// because of bugs found in CopyObject/PutObjectTags) https://github.com/minio/minio/pull/10772
+		erasureDistributionReliable = false
+	}
+
 	for i, onlineDisk := range onlineDisks {
 		if errs[i] != nil {
 			dataErrs[i] = errs[i]
@@ -169,6 +198,28 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			dataErrs[i] = errDiskNotFound
 			continue
 		}
+		if erasureDistributionReliable {
+			meta := partsMetadata[i]
+			if !meta.IsValid() {
+				continue
+			}
+
+			if len(meta.Erasure.Distribution) != len(onlineDisks) {
+				// Erasure distribution is not the same as onlineDisks
+				// attempt a fix if possible, assuming other entries
+				// might have the right erasure distribution.
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
+				continue
+			}
+
+			// Since erasure.Distribution is trustable we can fix the mismatching erasure.Index
+			if meta.Erasure.Distribution[i] != meta.Erasure.Index {
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
+				continue
+			}
+		}
 
 		switch scanMode {
 		case madmin.HealDeepScan:

cmd/erasure-healing-common_test.go

@@ -178,8 +178,8 @@ func TestListOnlineDisks(t *testing.T) {
 
 	object := "object"
 	data := bytes.Repeat([]byte("a"), 1024)
-	z := obj.(*erasureZones)
-	erasureDisks := z.zones[0].sets[0].getDisks()
+	z := obj.(*erasureServerSets)
+	erasureDisks := z.serverSets[0].sets[0].getDisks()
 	for i, test := range testCases {
 		_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{})
 		if err != nil {
@@ -211,7 +211,7 @@ func TestListOnlineDisks(t *testing.T) {
 				// and check if that disk
 				// appears in outDatedDisks.
 				tamperedIndex = index
-				dErr := erasureDisks[index].DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+				dErr := erasureDisks[index].Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
 				if dErr != nil {
 					t.Fatalf("Test %d: Failed to delete %s - %v", i+1,
 						filepath.Join(object, "part.1"), dErr)
@@ -274,8 +274,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	// make data with more than one part
 	partCount := 3
 	data := bytes.Repeat([]byte("a"), 6*1024*1024*partCount)
-	z := obj.(*erasureZones)
-	s := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	s := z.serverSets[0].sets[0]
 	erasureDisks := s.getDisks()
 	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
 	if err != nil {

cmd/erasure-healing.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"path"
 	"sync"
 	"time"
 
@@ -304,6 +305,10 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		})
 	}
 
+	if isAllNotFound(errs) {
+		return defaultHealResult(latestFileInfo, storageDisks, storageEndpoints, errs, bucket, object), nil
+	}
+
 	// If less than read quorum number of disks have all the parts
 	// of the data, we can't reconstruct the erasure-coded data.
 	if numAvailableDisks < dataBlocks {
@@ -342,6 +347,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	if pErr != nil {
 		return result, toObjectErr(pErr, bucket, object)
 	}
+	defer ObjectPathUpdated(pathJoin(bucket, object))
 
 	cleanFileInfo := func(fi FileInfo) FileInfo {
 		// Returns a copy of the 'fi' with checksums and parts nil'ed.
@@ -449,7 +455,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		}
 	}
 
-	defer er.deleteObject(ctx, minioMetaTmpBucket, tmpID, len(storageDisks)/2+1)
+	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tmpID, len(storageDisks)/2+1)
 
 	// Generate and write `xl.meta` generated from other disks.
 	outDatedDisks, err = writeUniqueFileInfo(ctx, outDatedDisks, minioMetaTmpBucket, tmpID,
@@ -518,10 +524,11 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 				wg.Add(1)
 				go func(index int, disk StorageAPI) {
 					defer wg.Done()
-					_ = disk.DeleteFile(ctx, bucket, object)
+					_ = disk.Delete(ctx, bucket, object, false)
 				}(index, disk)
 			}
 			wg.Wait()
+			ObjectPathUpdated(path.Join(bucket, object))
 		}
 	}
 
@@ -544,7 +551,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 			hr.After.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateCorrupt}
 		}
 	}
-	if dryRun || danglingObject {
+	if dryRun || danglingObject || isAllNotFound(errs) {
 		return hr, nil
 	}
 	for i, err := range errs {
@@ -649,9 +656,23 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
 	return g.Wait()
 }
 
+// isAllNotFound will return if any element of the error slice is not
+// errFileNotFound, errFileVersionNotFound or errVolumeNotFound.
+// A 0 length slice will always return false.
+func isAllNotFound(errs []error) bool {
+	for _, err := range errs {
+		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) || errors.Is(err, errFileVersionNotFound) {
+			continue
+		}
+		return false
+	}
+	return len(errs) > 0
+}
+
 // ObjectDir is considered dangling/corrupted if any only
 // if total disks - a combination of corrupted and missing
 // files is lesser than N/2+1 number of disks.
+// If no files were found false will be returned.
 func isObjectDirDangling(errs []error) (ok bool) {
 	var found int
 	var notFound int
@@ -668,7 +689,8 @@ func isObjectDirDangling(errs []error) (ok bool) {
 			otherFound++
 		}
 	}
-	return found+foundNotEmpty+otherFound < notFound
+	found = found + foundNotEmpty + otherFound
+	return found < notFound && found > 0
 }
 
 // Object is considered dangling/corrupted if any only
@@ -711,7 +733,8 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	}
 
 	if validMeta.Deleted {
-		return validMeta, false
+		// notFoundParts is ignored since a delete marker does not have any parts
+		return validMeta, corruptedErasureMeta+notFoundErasureMeta > len(errs)/2
 	}
 
 	// We couldn't find any valid meta we are indeed corrupted, return true right away.
@@ -747,6 +770,10 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	// Read metadata files from all the disks
 	partsMetadata, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID)
 
+	if isAllNotFound(errs) {
+		// Nothing to do
+		return defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs, bucket, object), nil
+	}
 	// Check if the object is dangling, if yes and user requested
 	// remove we simply delete it from namespace.
 	if m, ok := isObjectDangling(partsMetadata, errs, []error{}); ok {

cmd/erasure-healing_test.go

@@ -42,8 +42,8 @@ func TestHealing(t *testing.T) {
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
-	z := obj.(*erasureZones)
-	er := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	er := z.serverSets[0].sets[0]
 
 	// Create "bucket"
 	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
@@ -67,7 +67,7 @@ func TestHealing(t *testing.T) {
 	}
 
 	disk := er.getDisks()[0]
-	fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "")
+	fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -84,7 +84,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "")
+	fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -113,7 +113,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "")
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -197,11 +197,11 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	// Test 1: Remove the object backend files from the first disk.
-	z := objLayer.(*erasureZones)
-	er := z.zones[0].sets[0]
+	z := objLayer.(*erasureServerSets)
+	er := z.serverSets[0].sets[0]
 	erasureDisks := er.getDisks()
 	firstDisk := erasureDisks[0]
-	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -221,12 +221,12 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
 	}
 
-	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
 	if err != nil {
 		t.Errorf("Failure during deleting part.1 - %v", err)
 	}
 
-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader([]byte{}))
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
 	if err != nil {
 		t.Errorf("Failure during creating part.1 - %v", err)
 	}
@@ -246,13 +246,13 @@ func TestHealObjectCorrupted(t *testing.T) {
 		t.Fatalf("FileInfo not equal after healing")
 	}
 
-	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"))
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
 	if err != nil {
 		t.Errorf("Failure during deleting part.1 - %v", err)
 	}
 
 	bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
-	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bytes.NewReader(bdata))
+	err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
 	if err != nil {
 		t.Errorf("Failure during creating part.1 - %v", err)
 	}
@@ -275,7 +275,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	// Test 4: checks if HealObject returns an error when xl.meta is not found
 	// in more than read quorum number of disks, to create a corrupted situation.
 	for i := 0; i <= len(er.getDisks())/2; i++ {
-		er.getDisks()[i].DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
+		er.getDisks()[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
 	}
 
 	// Try healing now, expect to receive errFileNotFound.
@@ -342,8 +342,8 @@ func TestHealObjectErasure(t *testing.T) {
 	}
 
 	// Remove the object backend files from the first disk.
-	z := obj.(*erasureZones)
-	er := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	er := z.serverSets[0].sets[0]
 	firstDisk := er.getDisks()[0]
 
 	_, err = obj.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
@@ -351,7 +351,7 @@ func TestHealObjectErasure(t *testing.T) {
 		t.Fatalf("Failed to complete multipart upload - %v", err)
 	}
 
-	err = firstDisk.DeleteFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile))
+	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
 	if err != nil {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
@@ -366,7 +366,7 @@ func TestHealObjectErasure(t *testing.T) {
 	}
 
 	erasureDisks := er.getDisks()
-	z.zones[0].erasureDisksMu.Lock()
+	z.serverSets[0].erasureDisksMu.Lock()
 	er.getDisks = func() []StorageAPI {
 		// Nil more than half the disks, to remove write quorum.
 		for i := 0; i <= len(erasureDisks)/2; i++ {
@@ -374,7 +374,7 @@ func TestHealObjectErasure(t *testing.T) {
 		}
 		return erasureDisks
 	}
-	z.zones[0].erasureDisksMu.Unlock()
+	z.serverSets[0].erasureDisksMu.Unlock()
 
 	// Try healing now, expect to receive errDiskNotFound.
 	_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan})
@@ -419,8 +419,8 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	}
 
 	// Remove the object backend files from the first disk.
-	z := obj.(*erasureZones)
-	er := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	er := z.serverSets[0].sets[0]
 	firstDisk := er.getDisks()[0]
 	err = firstDisk.DeleteVol(context.Background(), pathJoin(bucket, encodeDirObject(object)), true)
 	if err != nil {

cmd/erasure-metadata-utils.go

@@ -113,9 +113,21 @@ func hashOrder(key string, cardinality int) []int {
 	return nums
 }
 
+// Reads all `xl.meta` metadata as a FileInfo slice and checks if the data dir exists as well,
+// otherwise returns errFileNotFound (or errFileVersionNotFound)
+func getAllObjectFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string) ([]FileInfo, []error) {
+	return readVersionFromDisks(ctx, disks, bucket, object, versionID, true)
+}
+
 // Reads all `xl.meta` metadata as a FileInfo slice.
 // Returns error slice indicating the failed metadata reads.
 func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string) ([]FileInfo, []error) {
+	return readVersionFromDisks(ctx, disks, bucket, object, versionID, false)
+}
+
+// Reads all `xl.meta` metadata as a FileInfo slice and checks if the data dir
+// exists as well, if checkDataDir is set to true.
+func readVersionFromDisks(ctx context.Context, disks []StorageAPI, bucket, object, versionID string, checkDataDir bool) ([]FileInfo, []error) {
 	metadataArray := make([]FileInfo, len(disks))
 
 	g := errgroup.WithNErrs(len(disks))
@@ -126,7 +138,7 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID)
+			metadataArray[index], err = disks[index].ReadVersion(ctx, bucket, object, versionID, checkDataDir)
 			if err != nil {
 				if err != errFileNotFound && err != errVolumeNotFound && err != errFileVersionNotFound {
 					logger.GetReqInfo(ctx).AppendTags("disk", disks[index].String())
@@ -141,6 +153,56 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 	return metadataArray, g.Wait()
 }
 
+func shuffleDisksAndPartsMetadataByIndex(disks []StorageAPI, metaArr []FileInfo, distribution []int) (shuffledDisks []StorageAPI, shuffledPartsMetadata []FileInfo) {
+	shuffledDisks = make([]StorageAPI, len(disks))
+	shuffledPartsMetadata = make([]FileInfo, len(disks))
+	var inconsistent int
+	for i, meta := range metaArr {
+		if disks[i] == nil {
+			// Assuming offline drives as inconsistent,
+			// to be safe and fallback to original
+			// distribution order.
+			inconsistent++
+			continue
+		}
+		// check if erasure distribution order matches the index
+		// position if this is not correct we discard the disk
+		// and move to collect others
+		if distribution[i] != meta.Erasure.Index {
+			inconsistent++ // keep track of inconsistent entries
+			continue
+		}
+		shuffledDisks[meta.Erasure.Index-1] = disks[i]
+		shuffledPartsMetadata[meta.Erasure.Index-1] = metaArr[i]
+	}
+
+	// Inconsistent meta info is with in the limit of
+	// expected quorum, proceed with EcIndex based
+	// disk order.
+	if inconsistent < len(disks)/2 {
+		return shuffledDisks, shuffledPartsMetadata
+	}
+
+	// fall back to original distribution based order.
+	return shuffleDisksAndPartsMetadata(disks, metaArr, distribution)
+}
+
+// Return shuffled partsMetadata depending on distribution.
+func shuffleDisksAndPartsMetadata(disks []StorageAPI, partsMetadata []FileInfo, distribution []int) (shuffledDisks []StorageAPI, shuffledPartsMetadata []FileInfo) {
+	if distribution == nil {
+		return disks, partsMetadata
+	}
+	shuffledDisks = make([]StorageAPI, len(disks))
+	shuffledPartsMetadata = make([]FileInfo, len(partsMetadata))
+	// Shuffle slice xl metadata for expected distribution.
+	for index := range partsMetadata {
+		blockIndex := distribution[index]
+		shuffledPartsMetadata[blockIndex-1] = partsMetadata[index]
+		shuffledDisks[blockIndex-1] = disks[index]
+	}
+	return shuffledDisks, shuffledPartsMetadata
+}
+
 // Return shuffled partsMetadata depending on distribution.
 func shufflePartsMetadata(partsMetadata []FileInfo, distribution []int) (shuffledPartsMetadata []FileInfo) {
 	if distribution == nil {

cmd/erasure-metadata-utils_test.go

@@ -148,13 +148,13 @@ func TestShuffleDisks(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer removeRoots(disks)
-	z := objLayer.(*erasureZones)
+	z := objLayer.(*erasureServerSets)
 	testShuffleDisks(t, z)
 }
 
 // Test shuffleDisks which returns shuffled slice of disks for their actual distribution.
-func testShuffleDisks(t *testing.T, z *erasureZones) {
-	disks := z.zones[0].GetDisks(0)()
+func testShuffleDisks(t *testing.T, z *erasureServerSets) {
+	disks := z.serverSets[0].GetDisks(0)()
 	distribution := []int{16, 14, 12, 10, 8, 6, 4, 2, 1, 3, 5, 7, 9, 11, 13, 15}
 	shuffledDisks := shuffleDisks(disks, distribution)
 	// From the "distribution" above you can notice that:
@@ -196,6 +196,6 @@ func TestEvalDisks(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer removeRoots(disks)
-	z := objLayer.(*erasureZones)
+	z := objLayer.(*erasureServerSets)
 	testShuffleDisks(t, z)
 }

cmd/erasure-metadata.go

@@ -90,8 +90,12 @@ func (fi FileInfo) IsValid() bool {
 	}
 	dataBlocks := fi.Erasure.DataBlocks
 	parityBlocks := fi.Erasure.ParityBlocks
+	correctIndexes := (fi.Erasure.Index > 0 &&
+		fi.Erasure.Index <= dataBlocks+parityBlocks &&
+		len(fi.Erasure.Distribution) == (dataBlocks+parityBlocks))
 	return ((dataBlocks >= parityBlocks) &&
-		(dataBlocks != 0) && (parityBlocks != 0))
+		(dataBlocks != 0) && (parityBlocks != 0) &&
+		correctIndexes)
 }
 
 // ToObjectInfo - Converts metadata to object info.
@@ -213,13 +217,15 @@ func (fi FileInfo) ObjectToPartOffset(ctx context.Context, offset int64) (partIn
 
 func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.Time, quorum int) (xmv FileInfo, e error) {
 	metaHashes := make([]string, len(metaArr))
+	h := sha256.New()
 	for i, meta := range metaArr {
 		if meta.IsValid() && meta.ModTime.Equal(modTime) {
-			h := sha256.New()
 			for _, part := range meta.Parts {
 				h.Write([]byte(fmt.Sprintf("part.%d", part.Number)))
 			}
+			h.Write([]byte(fmt.Sprintf("%v", meta.Erasure.Distribution)))
 			metaHashes[i] = hex.EncodeToString(h.Sum(nil))
+			h.Reset()
 		}
 	}
 

cmd/erasure-metadata_test.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"strconv"
 	"testing"
+	"time"
 
 	humanize "github.com/dustin/go-humanize"
 )
@@ -47,6 +48,7 @@ func TestAddObjectPart(t *testing.T) {
 
 	// Setup.
 	fi := newFileInfo("test-object", 8, 8)
+	fi.Erasure.Index = 1
 	if !fi.IsValid() {
 		t.Fatalf("unable to get xl meta")
 	}
@@ -80,6 +82,7 @@ func TestObjectPartIndex(t *testing.T) {
 
 	// Setup.
 	fi := newFileInfo("test-object", 8, 8)
+	fi.Erasure.Index = 1
 	if !fi.IsValid() {
 		t.Fatalf("unable to get xl meta")
 	}
@@ -108,6 +111,7 @@ func TestObjectPartIndex(t *testing.T) {
 func TestObjectToPartOffset(t *testing.T) {
 	// Setup.
 	fi := newFileInfo("test-object", 8, 8)
+	fi.Erasure.Index = 1
 	if !fi.IsValid() {
 		t.Fatalf("unable to get xl meta")
 	}
@@ -151,3 +155,48 @@ func TestObjectToPartOffset(t *testing.T) {
 		}
 	}
 }
+
+func TestFindFileInfoInQuorum(t *testing.T) {
+	getNFInfo := func(n int, quorum int, t int64) []FileInfo {
+		fi := newFileInfo("test", 8, 8)
+		fi.AddObjectPart(1, "etag", 100, 100)
+		fi.ModTime = time.Unix(t, 0)
+		fis := make([]FileInfo, n)
+		for i := range fis {
+			fis[i] = fi
+			fis[i].Erasure.Index = i + 1
+			quorum--
+			if quorum == 0 {
+				break
+			}
+		}
+		return fis
+	}
+
+	tests := []struct {
+		fis         []FileInfo
+		modTime     time.Time
+		expectedErr error
+	}{
+		{
+			fis:         getNFInfo(16, 16, 1603863445),
+			modTime:     time.Unix(1603863445, 0),
+			expectedErr: nil,
+		},
+		{
+			fis:         getNFInfo(16, 7, 1603863445),
+			modTime:     time.Unix(1603863445, 0),
+			expectedErr: errErasureReadQuorum,
+		},
+	}
+
+	for _, test := range tests {
+		test := test
+		t.Run("", func(t *testing.T) {
+			_, err := findFileInfoInQuorum(context.Background(), test.fis, test.modTime, 8)
+			if err != test.expectedErr {
+				t.Errorf("Expected %s, got %s", test.expectedErr, err)
+			}
+		})
+	}
+}

cmd/erasure-multipart.go

@@ -43,7 +43,25 @@ func (er erasureObjects) getMultipartSHADir(bucket, object string) string {
 
 // checkUploadIDExists - verify if a given uploadID exists and is valid.
 func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string) error {
-	_, _, _, err := er.getObjectFileInfo(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID), ObjectOptions{})
+	disks := er.getDisks()
+
+	// Read metadata associated with the object from all disks.
+	metaArr, errs := readAllFileInfo(ctx, disks, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID), "")
+
+	readQuorum, _, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
+	if err != nil {
+		return err
+	}
+
+	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
+		return toObjectErr(reducedErr, bucket, object)
+	}
+
+	// List all online disks.
+	_, modTime := listOnlineDisks(disks, metaArr, errs)
+
+	// Pick latest valid metadata.
+	_, err = pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
 	return err
 }
 
@@ -63,7 +81,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
 			// requests. xl.meta is the authoritative source of truth on which parts constitute
 			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
-			_ = storageDisks[index].DeleteFile(context.TODO(), minioMetaMultipartBucket, curpartPath)
+			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, false)
 			return nil
 		}, index)
 	}
@@ -110,7 +128,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		}
 		for _, uploadIDDir := range uploadIDDirs {
 			uploadIDPath := pathJoin(shaDir, uploadIDDir)
-			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "")
+			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "", false)
 			if err != nil {
 				continue
 			}
@@ -124,7 +142,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		return
 	}
 	for _, tmpDir := range tmpDirs {
-		fi, err := disk.ReadVersion(ctx, minioMetaTmpBucket, tmpDir, "")
+		fi, err := disk.ReadVersion(ctx, minioMetaTmpBucket, tmpDir, "", false)
 		if err != nil {
 			continue
 		}
@@ -149,7 +167,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 
 	var uploadIDs []string
 	var disk StorageAPI
-	for _, disk = range er.getLoadBalancedDisks() {
+	for _, disk = range er.getLoadBalancedDisks(true) {
 		uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
 		if err != nil {
 			if err == errDiskNotFound {
@@ -178,7 +196,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 		if populatedUploadIds.Contains(uploadID) {
 			continue
 		}
-		fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "")
+		fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "", false)
 		if err != nil {
 			return result, toObjectErr(err, bucket, object)
 		}
@@ -277,13 +295,15 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	// Delete the tmp path later in case we fail to commit (ignore
 	// returned errors) - this will be a no-op in case of a commit
 	// success.
-	defer er.deleteObject(ctx, minioMetaTmpBucket, tempUploadIDPath, writeQuorum)
+	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tempUploadIDPath, writeQuorum)
 
 	var partsMetadata = make([]FileInfo, len(onlineDisks))
 	for i := range onlineDisks {
 		partsMetadata[i] = fi
 	}
 
+	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(onlineDisks, partsMetadata, fi.Erasure.Distribution)
+
 	var err error
 	// Write updated `xl.meta` to all disks.
 	onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaTmpBucket, tempUploadIDPath, partsMetadata, writeQuorum)
@@ -335,8 +355,8 @@ func (er erasureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObjec
 //
 // Implements S3 compatible Upload Part API.
 func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	if err = uploadIDLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return PartInfo{}, err
 	}
 	readLocked := true
@@ -396,7 +416,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	tmpPartPath := pathJoin(tmpPart, partSuffix)
 
 	// Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete.
-	defer er.deleteObject(ctx, minioMetaTmpBucket, tmpPart, writeQuorum)
+	defer er.deleteObject(context.Background(), minioMetaTmpBucket, tmpPart, writeQuorum)
 
 	erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
 	if err != nil {
@@ -449,7 +469,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	// PutObjectParts would serialize here updating `xl.meta`
 	uploadIDLock.RUnlock()
 	readLocked = false
-	if err = uploadIDLock.GetLock(globalOperationTimeout); err != nil {
+	if err = uploadIDLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		return PartInfo{}, err
 	}
 	defer uploadIDLock.Unlock()
@@ -530,8 +550,8 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 		UploadID: uploadID,
 	}
 
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	if err := uploadIDLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return MultipartInfo{}, err
 	}
 	defer uploadIDLock.RUnlock()
@@ -578,8 +598,8 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 // ListPartsInfo structure is marshaled directly into XML and
 // replied back to the client.
 func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, e error) {
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	if err := uploadIDLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return ListPartsInfo{}, err
 	}
 	defer uploadIDLock.RUnlock()
@@ -671,8 +691,8 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
 	// Hold read-locks to verify uploaded parts, also disallows
 	// parallel part uploads as well.
-	uploadIDLock := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err = uploadIDLock.GetRLock(globalOperationTimeout); err != nil {
+	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	if err = uploadIDLock.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer uploadIDLock.RUnlock()
@@ -725,10 +745,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	// Order online disks in accordance with distribution order.
-	onlineDisks = shuffleDisks(onlineDisks, fi.Erasure.Distribution)
-
 	// Order parts metadata in accordance with distribution order.
-	partsMetadata = shufflePartsMetadata(partsMetadata, fi.Erasure.Distribution)
+	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadataByIndex(onlineDisks, partsMetadata, fi.Erasure.Distribution)
 
 	// Save current erasure metadata for validation.
 	var currentFI = fi
@@ -826,8 +844,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	// Hold namespace to complete the transaction
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	if err = lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer lk.Unlock()
@@ -868,8 +886,8 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 // would be removed from the system, rollback is not possible on this
 // operation.
 func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
-	lk := er.NewNSLock(ctx, bucket, pathJoin(object, uploadID))
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, pathJoin(object, uploadID))
+	if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		return err
 	}
 	defer lk.Unlock()

cmd/erasure-object.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -47,9 +46,10 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	if !srcInfo.metadataOnly {
 		return oi, NotImplemented{}
 	}
+
 	defer ObjectPathUpdated(path.Join(dstBucket, dstObject))
-	lk := er.NewNSLock(ctx, dstBucket, dstObject)
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(dstBucket, dstObject)
+	if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer lk.Unlock()
@@ -73,6 +73,8 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
 
+	onlineDisks, metaArr = shuffleDisksAndPartsMetadataByIndex(onlineDisks, metaArr, fi.Erasure.Distribution)
+
 	if fi.Deleted {
 		if srcOpts.VersionID == "" {
 			return oi, toObjectErr(errFileNotFound, srcBucket, srcObject)
@@ -123,10 +125,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 // GetObjectNInfo - returns object info and an object
 // Read(Closer). When err != nil, the returned reader is always nil.
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
-		return nil, err
-	}
-
 	var unlockOnDefer bool
 	var nsUnlocker = func() {}
 	defer func() {
@@ -137,15 +135,15 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 
 	// Acquire lock
 	if lockType != noLock {
-		lock := er.NewNSLock(ctx, bucket, object)
+		lock := er.NewNSLock(bucket, object)
 		switch lockType {
 		case writeLock:
-			if err = lock.GetLock(globalOperationTimeout); err != nil {
+			if err = lock.GetLock(ctx, globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.Unlock
 		case readLock:
-			if err = lock.GetRLock(globalOperationTimeout); err != nil {
+			if err = lock.GetRLock(ctx, globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.RUnlock
@@ -153,17 +151,6 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		unlockOnDefer = true
 	}
 
-	// Handler directory request by returning a reader that
-	// returns no bytes.
-	if HasSuffix(object, SlashSeparator) {
-		var objInfo ObjectInfo
-		if objInfo, err = er.getObjectInfoDir(ctx, bucket, object); err != nil {
-			return nil, toObjectErr(err, bucket, object)
-		}
-		unlockOnDefer = false
-		return NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts, nsUnlocker)
-	}
-
 	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
 	if err != nil {
 		return nil, toObjectErr(err, bucket, object)
@@ -190,7 +177,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 
 	pr, pw := io.Pipe()
 	go func() {
-		err := er.getObjectWithFileInfo(ctx, bucket, object, off, length, pw, "", opts, fi, metaArr, onlineDisks)
+		err := er.getObjectWithFileInfo(ctx, bucket, object, off, length, pw, fi, metaArr, onlineDisks)
 		pw.CloseWithError(err)
 	}()
 
@@ -208,13 +195,9 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 // startOffset indicates the starting read location of the object.
 // length indicates the total length of the object.
 func (er erasureObjects) GetObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
-	if err := checkGetObjArgs(ctx, bucket, object); err != nil {
-		return err
-	}
-
 	// Lock the object before reading.
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return err
 	}
 	defer lk.RUnlock()
@@ -231,22 +214,13 @@ func (er erasureObjects) GetObject(ctx context.Context, bucket, object string, s
 		return errUnexpected
 	}
 
-	// If its a directory request, we return an empty body.
-	if HasSuffix(object, SlashSeparator) {
-		_, err := writer.Write([]byte(""))
-		logger.LogIf(ctx, err)
-		return toObjectErr(err, bucket, object)
-	}
-
-	return er.getObject(ctx, bucket, object, startOffset, length, writer, etag, opts)
+	return er.getObject(ctx, bucket, object, startOffset, length, writer, opts)
 }
 
-func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions, fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI) error {
+func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI) error {
 	// Reorder online disks based on erasure distribution order.
-	onlineDisks = shuffleDisks(onlineDisks, fi.Erasure.Distribution)
-
 	// Reorder parts metadata based on erasure distribution order.
-	metaArr = shufflePartsMetadata(metaArr, fi.Erasure.Distribution)
+	onlineDisks, metaArr = shuffleDisksAndPartsMetadataByIndex(onlineDisks, metaArr, fi.Erasure.Distribution)
 
 	// For negative length read everything.
 	if length < 0 {
@@ -350,7 +324,7 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 }
 
 // getObject wrapper for erasure GetObject
-func (er erasureObjects) getObject(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) error {
+func (er erasureObjects) getObject(ctx context.Context, bucket, object string, startOffset, length int64, writer io.Writer, opts ObjectOptions) error {
 	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts)
 	if err != nil {
 		return toObjectErr(err, bucket, object)
@@ -363,61 +337,18 @@ func (er erasureObjects) getObject(ctx context.Context, bucket, object string, s
 		return toObjectErr(errMethodNotAllowed, bucket, object)
 	}
 
-	return er.getObjectWithFileInfo(ctx, bucket, object, startOffset, length, writer, etag, opts, fi, metaArr, onlineDisks)
-}
-
-// getObjectInfoDir - This getObjectInfo is specific to object directory lookup.
-func (er erasureObjects) getObjectInfoDir(ctx context.Context, bucket, object string) (ObjectInfo, error) {
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	// Prepare object creation in a all disks
-	for index, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			// Check if 'prefix' is an object on this 'disk'.
-			entries, err := storageDisks[index].ListDir(ctx, bucket, object, 1)
-			if err != nil {
-				return err
-			}
-			if len(entries) > 0 {
-				// Not a directory if not empty.
-				return errFileNotFound
-			}
-			return nil
-		}, index)
-	}
-
-	readQuorum := getReadQuorum(len(storageDisks))
-	err := reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum)
-	return dirObjectInfo(bucket, object, 0, map[string]string{}), err
+	return er.getObjectWithFileInfo(ctx, bucket, object, startOffset, length, writer, fi, metaArr, onlineDisks)
 }
 
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
-	if err = checkGetObjArgs(ctx, bucket, object); err != nil {
-		return info, err
-	}
-
 	// Lock the object before reading.
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return ObjectInfo{}, err
 	}
 	defer lk.RUnlock()
 
-	if HasSuffix(object, SlashSeparator) {
-		info, err = er.getObjectInfoDir(ctx, bucket, object)
-		if err != nil {
-			return info, toObjectErr(err, bucket, object)
-		}
-		return info, nil
-	}
-
 	return er.getObjectInfo(ctx, bucket, object, opts)
 }
 
@@ -425,7 +356,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	disks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID)
+	metaArr, errs := getAllObjectFileInfo(ctx, disks, bucket, object, opts.VersionID)
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
 	if err != nil {
@@ -433,6 +364,24 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	}
 
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
+		if reducedErr == errErasureReadQuorum && bucket != minioMetaBucket {
+			if _, ok := isObjectDangling(metaArr, errs, nil); ok {
+				reducedErr = errFileNotFound
+				if opts.VersionID != "" {
+					reducedErr = errFileVersionNotFound
+				}
+				// Remove the dangling object only when:
+				//  - This is a non versioned bucket
+				//  - This is a versioned bucket and the version ID is passed, the reason
+				//    is that we cannot fetch the ID of the latest version when we don't trust xl.meta
+				if !opts.Versioned || opts.VersionID != "" {
+					er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
+						Name:      object,
+						VersionID: opts.VersionID,
+					})
+				}
+			}
+		}
 		return fi, nil, nil, toObjectErr(reducedErr, bucket, object)
 	}
 
@@ -444,7 +393,6 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	if err != nil {
 		return fi, nil, nil, err
 	}
-
 	return fi, metaArr, onlineDisks, nil
 }
 
@@ -495,6 +443,9 @@ func undoRename(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry str
 // Similar to rename but renames data from srcEntry to dstEntry at dataDir
 func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dataDir, dstBucket, dstEntry string, writeQuorum int, ignoredErr []error) ([]StorageAPI, error) {
 	dataDir = retainSlash(dataDir)
+	defer ObjectPathUpdated(path.Join(srcBucket, srcEntry))
+	defer ObjectPathUpdated(path.Join(dstBucket, dstEntry))
+
 	g := errgroup.WithNErrs(len(disks))
 
 	// Rename file on all underlying storage disks.
@@ -542,11 +493,12 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, da
 // rename - common function that renamePart and renameObject use to rename
 // the respective underlying storage layer representations.
 func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, isDir bool, writeQuorum int, ignoredErr []error) ([]StorageAPI, error) {
-
 	if isDir {
 		dstEntry = retainSlash(dstEntry)
 		srcEntry = retainSlash(srcEntry)
 	}
+	defer ObjectPathUpdated(path.Join(srcBucket, srcEntry))
+	defer ObjectPathUpdated(path.Join(dstBucket, dstEntry))
 
 	g := errgroup.WithNErrs(len(disks))
 
@@ -584,11 +536,6 @@ func rename(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBuc
 // writes `xl.meta` which carries the necessary metadata for future
 // object operations.
 func (er erasureObjects) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	// Validate put object input args.
-	if err = checkPutObjectArgs(ctx, bucket, object, er, data.Size()); err != nil {
-		return ObjectInfo{}, err
-	}
-
 	return er.putObject(ctx, bucket, object, data, opts)
 }
 
@@ -658,7 +605,8 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	// Order disks according to erasure distribution
-	onlineDisks := shuffleDisks(storageDisks, fi.Erasure.Distribution)
+	var onlineDisks []StorageAPI
+	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi.Erasure.Distribution)
 
 	erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize)
 	if err != nil {
@@ -705,8 +653,8 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object}
 	}
 
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		return ObjectInfo{}, err
 	}
 	defer lk.Unlock()
@@ -778,21 +726,16 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 }
 
 func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, writeQuorum int, fi FileInfo) error {
+	defer ObjectPathUpdated(pathJoin(bucket, object))
 	disks := er.getDisks()
-
 	g := errgroup.WithNErrs(len(disks))
-
 	for index := range disks {
 		index := index
 		g.Go(func() error {
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			err := disks[index].DeleteVersion(ctx, bucket, object, fi)
-			if err != nil && err != errVolumeNotFound {
-				return err
-			}
-			return nil
+			return disks[index].DeleteVersion(ctx, bucket, object, fi)
 		}, index)
 	}
 
@@ -853,16 +796,9 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 	dobjects := make([]DeletedObject, len(objects))
 	writeQuorums := make([]int, len(objects))
 
-	for i, object := range objects {
-		errs[i] = checkDelObjArgs(ctx, bucket, object.ObjectName)
-	}
-
 	storageDisks := er.getDisks()
 
 	for i := range objects {
-		if errs[i] != nil {
-			continue
-		}
 		// Assume (N/2 + 1) quorums for all objects
 		// this is a theoretical assumption such that
 		// for delete's we do not need to honor storage
@@ -876,21 +812,19 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 	for i := range objects {
 		if objects[i].VersionID == "" {
 			if opts.Versioned || opts.VersionSuspended {
-				if !HasSuffix(objects[i].ObjectName, SlashSeparator) {
-					fi := FileInfo{
-						Name:    objects[i].ObjectName,
-						ModTime: UTCNow(),
-						Deleted: true, // delete marker
-					}
-					if opts.Versioned {
-						fi.VersionID = mustGetUUID()
-					}
-					// versioning suspended means we add `null`
-					// version as delete marker
-
-					versions[i] = fi
-					continue
+				fi := FileInfo{
+					Name:    objects[i].ObjectName,
+					ModTime: UTCNow(),
+					Deleted: true, // delete marker
 				}
+				if opts.Versioned {
+					fi.VersionID = mustGetUUID()
+				}
+				// versioning suspended means we add `null`
+				// version as delete marker
+
+				versions[i] = fi
+				continue
 			}
 		}
 		versions[i] = FileInfo{
@@ -939,15 +873,16 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		}
 		errs[objIndex] = reduceWriteQuorumErrs(ctx, diskErrs, objectOpIgnoredErrs, writeQuorums[objIndex])
 		if errs[objIndex] == nil {
+			ObjectPathUpdated(pathJoin(bucket, objects[objIndex].ObjectName))
 			if versions[objIndex].Deleted {
 				dobjects[objIndex] = DeletedObject{
 					DeleteMarker:          versions[objIndex].Deleted,
 					DeleteMarkerVersionID: versions[objIndex].VersionID,
-					ObjectName:            versions[objIndex].Name,
+					ObjectName:            decodeDirObject(versions[objIndex].Name),
 				}
 			} else {
 				dobjects[objIndex] = DeletedObject{
-					ObjectName: versions[objIndex].Name,
+					ObjectName: decodeDirObject(versions[objIndex].Name),
 					VersionID:  versions[objIndex].VersionID,
 				}
 			}
@@ -978,13 +913,19 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 // any error as it is not necessary for the handler to reply back a
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	if err = checkDelObjArgs(ctx, bucket, object); err != nil {
-		return objInfo, err
+	defer ObjectPathUpdated(path.Join(bucket, object))
+	goi, gerr := er.GetObjectInfo(ctx, bucket, object, opts)
+	if gerr != nil && goi.Name == "" {
+		switch gerr.(type) {
+		case InsufficientReadQuorum:
+			return objInfo, InsufficientWriteQuorum{}
+		}
+		return objInfo, gerr
 	}
 
 	// Acquire a write lock before deleting the object.
-	lk := er.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
+	lk := er.NewNSLock(bucket, object)
+	if err = lk.GetLock(ctx, globalDeleteOperationTimeout); err != nil {
 		return ObjectInfo{}, err
 	}
 	defer lk.Unlock()
@@ -994,26 +935,24 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 
 	if opts.VersionID == "" {
 		if opts.Versioned || opts.VersionSuspended {
-			if !HasSuffix(object, SlashSeparator) {
-				fi := FileInfo{
-					Name:    object,
-					Deleted: true,
-					ModTime: UTCNow(),
-				}
-
-				if opts.Versioned {
-					fi.VersionID = mustGetUUID()
-				}
-
-				// versioning suspended means we add `null`
-				// version as delete marker
-
-				// Add delete marker, since we don't have any version specified explicitly.
-				if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi); err != nil {
-					return objInfo, toObjectErr(err, bucket, object)
-				}
-				return fi.ToObjectInfo(bucket, object), nil
+			fi := FileInfo{
+				Name:    object,
+				Deleted: true,
+				ModTime: UTCNow(),
 			}
+
+			if opts.Versioned {
+				fi.VersionID = mustGetUUID()
+			}
+
+			// versioning suspended means we add `null`
+			// version as delete marker
+
+			// Add delete marker, since we don't have any version specified explicitly.
+			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi); err != nil {
+				return objInfo, toObjectErr(err, bucket, object)
+			}
+			return fi.ToObjectInfo(bucket, object), nil
 		}
 	}
 
@@ -1032,7 +971,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
-	return ObjectInfo{Bucket: bucket, Name: object, VersionID: opts.VersionID}, nil
+	return ObjectInfo{Bucket: bucket, Name: decodeDirObject(object), VersionID: opts.VersionID}, nil
 }
 
 // Send the successful but partial upload/delete, however ignore
@@ -1057,7 +996,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	}
 
 	// List all online disks.
-	_, modTime := listOnlineDisks(disks, metaArr, errs)
+	onlineDisks, modTime := listOnlineDisks(disks, metaArr, errs)
 
 	// Pick latest valid metadata.
 	fi, err := pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
@@ -1065,6 +1004,8 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 		return toObjectErr(err, bucket, object)
 	}
 
+	onlineDisks, metaArr = shuffleDisksAndPartsMetadataByIndex(onlineDisks, metaArr, fi.Erasure.Distribution)
+
 	if fi.Deleted {
 		if opts.VersionID == "" {
 			return toObjectErr(errFileNotFound, bucket, object)
@@ -1089,6 +1030,66 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 
 	tempObj := mustGetUUID()
 
+	// Write unique `xl.meta` for each disk.
+	if onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	// Atomically rename metadata from tmp location to destination for each disk.
+	if _, err = renameFileInfo(ctx, onlineDisks, minioMetaTmpBucket, tempObj, bucket, object, writeQuorum); err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	return nil
+}
+
+// updateObjectMeta will update the metadata of a file.
+func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object string, meta map[string]string, opts ObjectOptions) error {
+	if len(meta) == 0 {
+		return nil
+	}
+	disks := er.getDisks()
+
+	// Read metadata associated with the object from all disks.
+	metaArr, errs := readAllFileInfo(ctx, disks, bucket, object, opts.VersionID)
+
+	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, er, metaArr, errs)
+	if err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	// List all online disks.
+	_, modTime := listOnlineDisks(disks, metaArr, errs)
+
+	// Pick latest valid metadata.
+	fi, err := pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
+	if err != nil {
+		return toObjectErr(err, bucket, object)
+	}
+
+	// Update metadata
+	for k, v := range meta {
+		fi.Metadata[k] = v
+	}
+
+	if fi.Deleted {
+		if opts.VersionID == "" {
+			return toObjectErr(errFileNotFound, bucket, object)
+		}
+		return toObjectErr(errMethodNotAllowed, bucket, object)
+	}
+
+	for i := range metaArr {
+		if errs[i] != nil {
+			// Avoid disks where loading metadata fail
+			continue
+		}
+
+		metaArr[i].Metadata = fi.Metadata
+	}
+
+	tempObj := mustGetUUID()
+
 	// Write unique `xl.meta` for each disk.
 	if disks, err = writeUniqueFileInfo(ctx, disks, minioMetaTmpBucket, tempObj, metaArr, writeQuorum); err != nil {
 		return toObjectErr(err, bucket, object)

cmd/erasure-object_test.go

@@ -132,8 +132,8 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 		for _, dir := range fsDirs {
 			defer os.RemoveAll(dir)
 		}
-		z := obj.(*erasureZones)
-		xl := z.zones[0].sets[0]
+		z := obj.(*erasureServerSets)
+		xl := z.serverSets[0].sets[0]
 		objs = append(objs, xl)
 	}
 
@@ -205,8 +205,8 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
-	z := obj.(*erasureZones)
-	xl := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	xl := z.serverSets[0].sets[0]
 
 	// Create "bucket"
 	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
@@ -225,7 +225,7 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 	// for a 16 disk setup, quorum is 9. To simulate disks not found yet
 	// quorum is available, we remove disks leaving quorum disks behind.
 	erasureDisks := xl.getDisks()
-	z.zones[0].erasureDisksMu.Lock()
+	z.serverSets[0].erasureDisksMu.Lock()
 	xl.getDisks = func() []StorageAPI {
 		for i := range erasureDisks[:7] {
 			erasureDisks[i] = newNaughtyDisk(erasureDisks[i], nil, errFaultyDisk)
@@ -233,7 +233,7 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 		return erasureDisks
 	}
 
-	z.zones[0].erasureDisksMu.Unlock()
+	z.serverSets[0].erasureDisksMu.Unlock()
 	_, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{})
 	if err != nil {
 		t.Fatal(err)
@@ -247,14 +247,14 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 
 	// Remove one more disk to 'lose' quorum, by setting it to nil.
 	erasureDisks = xl.getDisks()
-	z.zones[0].erasureDisksMu.Lock()
+	z.serverSets[0].erasureDisksMu.Lock()
 	xl.getDisks = func() []StorageAPI {
 		erasureDisks[7] = nil
 		erasureDisks[8] = nil
 		return erasureDisks
 	}
 
-	z.zones[0].erasureDisksMu.Unlock()
+	z.serverSets[0].erasureDisksMu.Unlock()
 	_, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{})
 	// since majority of disks are not available, metaquorum is not achieved and hence errErasureWriteQuorum error
 	if err != toObjectErr(errErasureWriteQuorum, bucket, object) {
@@ -275,8 +275,8 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
-	z := obj.(*erasureZones)
-	xl := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	xl := z.serverSets[0].sets[0]
 
 	// Create "bucket"
 	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
@@ -311,11 +311,11 @@ func TestGetObjectNoQuorum(t *testing.T) {
 				erasureDisks[i] = newNaughtyDisk(erasureDisks[i], diskErrors, errFaultyDisk)
 			}
 		}
-		z.zones[0].erasureDisksMu.Lock()
+		z.serverSets[0].erasureDisksMu.Lock()
 		xl.getDisks = func() []StorageAPI {
 			return erasureDisks
 		}
-		z.zones[0].erasureDisksMu.Unlock()
+		z.serverSets[0].erasureDisksMu.Unlock()
 		// Fetch object from store.
 		err = xl.GetObject(ctx, bucket, object, 0, int64(len("abcd")), ioutil.Discard, "", opts)
 		if err != toObjectErr(errErasureReadQuorum, bucket, object) {
@@ -338,8 +338,8 @@ func TestPutObjectNoQuorum(t *testing.T) {
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(fsDirs)
 
-	z := obj.(*erasureZones)
-	xl := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	xl := z.serverSets[0].sets[0]
 
 	// Create "bucket"
 	err = obj.MakeBucketWithLocation(ctx, "bucket", BucketOptions{})
@@ -374,11 +374,11 @@ func TestPutObjectNoQuorum(t *testing.T) {
 				erasureDisks[i] = newNaughtyDisk(erasureDisks[i], diskErrors, errFaultyDisk)
 			}
 		}
-		z.zones[0].erasureDisksMu.Lock()
+		z.serverSets[0].erasureDisksMu.Lock()
 		xl.getDisks = func() []StorageAPI {
 			return erasureDisks
 		}
-		z.zones[0].erasureDisksMu.Unlock()
+		z.serverSets[0].erasureDisksMu.Unlock()
 		// Upload new content to same object "object"
 		_, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts)
 		if err != toObjectErr(errErasureWriteQuorum, bucket, object) {
@@ -404,8 +404,8 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	partCount := 3
 	data := bytes.Repeat([]byte("a"), 6*1024*1024*partCount)
 
-	z := obj.(*erasureZones)
-	xl := z.zones[0].sets[0]
+	z := obj.(*erasureServerSets)
+	xl := z.serverSets[0].sets[0]
 	erasureDisks := xl.getDisks()
 
 	ctx, cancel := context.WithCancel(GlobalContext)

cmd/erasure-sets.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"hash/crc32"
 	"io"
+	"math/rand"
 	"net/http"
 	"sort"
 	"sync"
@@ -30,7 +31,6 @@ import (
 	"github.com/dchest/siphash"
 	"github.com/google/uuid"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/bpool"
 	"github.com/minio/minio/pkg/dsync"
@@ -79,7 +79,6 @@ type erasureSets struct {
 
 	// Total number of sets and the number of disks per set.
 	setCount, setDriveCount int
-	listTolerancePerSet     int
 
 	disksConnectEvent chan diskConnectInfo
 
@@ -89,11 +88,6 @@ type erasureSets struct {
 
 	disksStorageInfoCache timedValue
 
-	// Merge tree walk
-	pool         *MergeWalkPool
-	poolSplunk   *MergeWalkPool
-	poolVersions *MergeWalkVersionsPool
-
 	mrfMU         sync.Mutex
 	mrfOperations map[healSource]int
 }
@@ -130,7 +124,7 @@ func (s *erasureSets) getDiskMap() map[string]StorageAPI {
 // Initializes a new StorageAPI from the endpoint argument, returns
 // StorageAPI and also `format` which exists on the disk.
 func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
-	disk, err := newStorageAPI(endpoint)
+	disk, err := newStorageAPIWithoutHealthCheck(endpoint)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -218,29 +212,35 @@ func (s *erasureSets) connectDisks() {
 				}
 				return
 			}
+			if disk.IsLocal() && disk.Healing() {
+				globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint())
+				logger.Info(fmt.Sprintf("Found the drive %s that needs healing, attempting to heal...", disk))
+			}
 			s.erasureDisksMu.RLock()
 			setIndex, diskIndex, err := findDiskIndex(s.format, format)
 			s.erasureDisksMu.RUnlock()
 			if err != nil {
-				if endpoint.IsLocal {
-					globalBackgroundHealState.pushHealLocalDisks(endpoint)
-					logger.Info(fmt.Sprintf("Found inconsistent drive %s with format.json, attempting to heal...", endpoint))
-				} else {
-					printEndpointError(endpoint, err, false)
-				}
+				printEndpointError(endpoint, err, false)
 				return
 			}
-			disk.SetDiskID(format.Erasure.This)
-			if endpoint.IsLocal && disk.Healing() {
-				globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint())
-				logger.Info(fmt.Sprintf("Found the drive %s which needs healing, attempting to heal...", disk))
-			}
 
 			s.erasureDisksMu.Lock()
 			if s.erasureDisks[setIndex][diskIndex] != nil {
 				s.erasureDisks[setIndex][diskIndex].Close()
 			}
-			s.erasureDisks[setIndex][diskIndex] = disk
+			if disk.IsLocal() {
+				disk.SetDiskID(format.Erasure.This)
+				s.erasureDisks[setIndex][diskIndex] = disk
+			} else {
+				// Enable healthcheck disk for remote endpoint.
+				disk, err = newStorageAPI(endpoint)
+				if err != nil {
+					printEndpointError(endpoint, err, false)
+					return
+				}
+				disk.SetDiskID(format.Erasure.This)
+				s.erasureDisks[setIndex][diskIndex] = disk
+			}
 			s.endpointStrings[setIndex*s.setDriveCount+diskIndex] = disk.String()
 			s.erasureDisksMu.Unlock()
 			go func(setIndex int) {
@@ -259,6 +259,13 @@ func (s *erasureSets) connectDisks() {
 // endpoints by reconnecting them and making sure to place them into right position in
 // the set topology, this monitoring happens at a given monitoring interval.
 func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInterval time.Duration) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
+
+	// Pre-emptively connect the disks if possible.
+	s.connectDisks()
+
 	for {
 		select {
 		case <-ctx.Done():
@@ -269,10 +276,28 @@ func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInt
 	}
 }
 
+// GetAllLockers return a list of all lockers for all sets.
+func (s *erasureSets) GetAllLockers() []dsync.NetLocker {
+	allLockers := make([]dsync.NetLocker, s.setDriveCount*s.setCount)
+	for i, lockers := range s.erasureLockers {
+		for j, locker := range lockers {
+			allLockers[i*s.setDriveCount+j] = locker
+		}
+	}
+	return allLockers
+}
+
 func (s *erasureSets) GetLockers(setIndex int) func() ([]dsync.NetLocker, string) {
 	return func() ([]dsync.NetLocker, string) {
 		lockers := make([]dsync.NetLocker, s.setDriveCount)
 		copy(lockers, s.erasureLockers[setIndex])
+		sort.Slice(lockers, func(i, j int) bool {
+			// re-order lockers with affinity for
+			// - non-local lockers
+			// - online lockers
+			// are used first
+			return !lockers[i].IsLocal() && lockers[i].IsOnline()
+		})
 		return lockers, s.erasureLockOwner
 	}
 }
@@ -311,25 +336,22 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	setDriveCount := len(format.Erasure.Sets[0])
 
 	endpointStrings := make([]string, len(endpoints))
+
 	// Initialize the erasure sets instance.
 	s := &erasureSets{
-		sets:                make([]*erasureObjects, setCount),
-		erasureDisks:        make([][]StorageAPI, setCount),
-		erasureLockers:      make([][]dsync.NetLocker, setCount),
-		erasureLockOwner:    mustGetUUID(),
-		endpoints:           endpoints,
-		endpointStrings:     endpointStrings,
-		setCount:            setCount,
-		setDriveCount:       setDriveCount,
-		listTolerancePerSet: 3, // Expect 3 good entries across disks.
-		format:              format,
-		disksConnectEvent:   make(chan diskConnectInfo),
-		distributionAlgo:    format.Erasure.DistributionAlgo,
-		deploymentID:        uuid.MustParse(format.ID),
-		pool:                NewMergeWalkPool(globalMergeLookupTimeout),
-		poolSplunk:          NewMergeWalkPool(globalMergeLookupTimeout),
-		poolVersions:        NewMergeWalkVersionsPool(globalMergeLookupTimeout),
-		mrfOperations:       make(map[healSource]int),
+		sets:              make([]*erasureObjects, setCount),
+		erasureDisks:      make([][]StorageAPI, setCount),
+		erasureLockers:    make([][]dsync.NetLocker, setCount),
+		erasureLockOwner:  GetLocalPeer(globalEndpoints),
+		endpoints:         endpoints,
+		endpointStrings:   endpointStrings,
+		setCount:          setCount,
+		setDriveCount:     setDriveCount,
+		format:            format,
+		disksConnectEvent: make(chan diskConnectInfo),
+		distributionAlgo:  format.Erasure.DistributionAlgo,
+		deploymentID:      uuid.MustParse(format.ID),
+		mrfOperations:     make(map[healSource]int),
 	}
 
 	mutex := newNSLock(globalIsDistErasure)
@@ -387,11 +409,11 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (s *erasureSets) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
+func (s *erasureSets) NewNSLock(bucket string, objects ...string) RWLocker {
 	if len(objects) == 1 {
-		return s.getHashedSet(objects[0]).NewNSLock(ctx, bucket, objects...)
+		return s.getHashedSet(objects[0]).NewNSLock(bucket, objects...)
 	}
-	return s.getHashedSet("").NewNSLock(ctx, bucket, objects...)
+	return s.getHashedSet("").NewNSLock(bucket, objects...)
 }
 
 // SetDriveCount returns the current drives per set.
@@ -400,7 +422,7 @@ func (s *erasureSets) SetDriveCount() int {
 }
 
 // StorageUsageInfo - combines output of StorageInfo across all erasure coded object sets.
-// This only returns disk usage info for Zones to perform placement decision, this call
+// This only returns disk usage info for ServerSets to perform placement decision, this call
 // is not implemented in Object interface and is not meant to be used by other object
 // layer implementations.
 func (s *erasureSets) StorageUsageInfo(ctx context.Context) StorageInfo {
@@ -448,7 +470,6 @@ func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo,
 
 	storageInfos := make([]StorageInfo, len(s.sets))
 	storageInfoErrs := make([][]error, len(s.sets))
-	storageInfo.Backend.Type = BackendErasure
 
 	g := errgroup.WithNErrs(len(s.sets))
 	for index := range s.sets {
@@ -468,17 +489,6 @@ func (s *erasureSets) StorageInfo(ctx context.Context, local bool) (StorageInfo,
 		storageInfo.Backend.OfflineDisks = storageInfo.Backend.OfflineDisks.Merge(lstorageInfo.Backend.OfflineDisks)
 	}
 
-	scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD)
-	if scParity == 0 {
-		scParity = s.setDriveCount / 2
-	}
-	storageInfo.Backend.StandardSCData = s.setDriveCount - scParity
-	storageInfo.Backend.StandardSCParity = scParity
-
-	rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS)
-	storageInfo.Backend.RRSCData = s.setDriveCount - rrSCParity
-	storageInfo.Backend.RRSCParity = rrSCParity
-
 	if local {
 		// if local is true, we are not interested in the drive UUID info.
 		// this is called primarily by prometheus
@@ -767,6 +777,7 @@ func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstB
 
 	// Check if this request is only metadata update.
 	if cpSrcDstSame && srcInfo.metadataOnly {
+
 		// Version ID is set for the destination and source == destination version ID.
 		// perform an in-place update.
 		if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID {
@@ -843,70 +854,6 @@ func (f *FileInfoCh) Push(fi FileInfo) {
 	f.Valid = true
 }
 
-// Calculate lexically least entry across multiple FileInfo channels,
-// returns the lexically common entry and the total number of times
-// we found this entry. Additionally also returns a boolean
-// to indicate if the caller needs to call this function
-// again to list the next entry. It is callers responsibility
-// if the caller wishes to list N entries to call lexicallySortedEntry
-// N times until this boolean is 'false'.
-func lexicallySortedEntry(entryChs []FileInfoCh, entries []FileInfo, entriesValid []bool) (FileInfo, int, bool) {
-	for j := range entryChs {
-		entries[j], entriesValid[j] = entryChs[j].Pop()
-	}
-
-	var isTruncated = false
-	for _, valid := range entriesValid {
-		if !valid {
-			continue
-		}
-		isTruncated = true
-		break
-	}
-
-	var lentry FileInfo
-	var found bool
-	for i, valid := range entriesValid {
-		if !valid {
-			continue
-		}
-		if !found {
-			lentry = entries[i]
-			found = true
-			continue
-		}
-		if entries[i].Name < lentry.Name {
-			lentry = entries[i]
-		}
-	}
-
-	// We haven't been able to find any lexically least entry,
-	// this would mean that we don't have valid entry.
-	if !found {
-		return lentry, 0, isTruncated
-	}
-
-	lexicallySortedEntryCount := 0
-	for i, valid := range entriesValid {
-		if !valid {
-			continue
-		}
-
-		// Entries are duplicated across disks,
-		// we should simply skip such entries.
-		if lentry.Name == entries[i].Name && lentry.ModTime.Equal(entries[i].ModTime) {
-			lexicallySortedEntryCount++
-			continue
-		}
-
-		// Push all entries which are lexically higher
-		// and will be returned later in Pop()
-		entryChs[i].Push(entries[i])
-	}
-
-	return lentry, lexicallySortedEntryCount, isTruncated
-}
-
 // Calculate lexically least entry across multiple FileInfo channels,
 // returns the lexically common entry and the total number of times
 // we found this entry. Additionally also returns a boolean
@@ -971,10 +918,6 @@ func lexicallySortedEntryVersions(entryChs []FileInfoVersionsCh, entries []FileI
 	return lentry, lexicallySortedEntryCount, isTruncated
 }
 
-func (s *erasureSets) startMergeWalks(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh <-chan struct{}) []FileInfoCh {
-	return s.startMergeWalksN(ctx, bucket, prefix, marker, recursive, endWalkCh, -1, false)
-}
-
 func (s *erasureSets) startMergeWalksVersions(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh <-chan struct{}) []FileInfoVersionsCh {
 	return s.startMergeWalksVersionsN(ctx, bucket, prefix, marker, recursive, endWalkCh, -1)
 }
@@ -991,6 +934,7 @@ func (s *erasureSets) startMergeWalksVersionsN(ctx context.Context, bucket, pref
 			wg.Add(1)
 			go func(disk StorageAPI) {
 				defer wg.Done()
+
 				entryCh, err := disk.WalkVersions(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
 				if err != nil {
 					return
@@ -1008,42 +952,6 @@ func (s *erasureSets) startMergeWalksVersionsN(ctx context.Context, bucket, pref
 	return entryChs
 }
 
-// Starts a walk channel across n number of disks and returns a slice of
-// FileInfoCh which can be read from.
-func (s *erasureSets) startMergeWalksN(ctx context.Context, bucket, prefix, marker string, recursive bool, endWalkCh <-chan struct{}, ndisks int, splunk bool) []FileInfoCh {
-	var entryChs []FileInfoCh
-	var wg sync.WaitGroup
-	var mutex sync.Mutex
-	for _, set := range s.sets {
-		// Reset for the next erasure set.
-		for _, disk := range set.getLoadBalancedNDisks(ndisks) {
-			wg.Add(1)
-			go func(disk StorageAPI) {
-				defer wg.Done()
-
-				var entryCh chan FileInfo
-				var err error
-				if splunk {
-					entryCh, err = disk.WalkSplunk(GlobalContext, bucket, prefix, marker, endWalkCh)
-				} else {
-					entryCh, err = disk.Walk(GlobalContext, bucket, prefix, marker, recursive, endWalkCh)
-				}
-				if err != nil {
-					// Disk walk returned error, ignore it.
-					return
-				}
-				mutex.Lock()
-				entryChs = append(entryChs, FileInfoCh{
-					Ch: entryCh,
-				})
-				mutex.Unlock()
-			}(disk)
-		}
-	}
-	wg.Wait()
-	return entryChs
-}
-
 func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
 	// In list multipart uploads we are going to treat input prefix as the object,
 	// this means that we are not supporting directory navigation.
@@ -1171,65 +1079,6 @@ func formatsToDrivesInfo(endpoints Endpoints, formats []*formatErasureV3, sErrs
 	return beforeDrives
 }
 
-// Reloads the format from the disk, usually called by a remote peer notifier while
-// healing in a distributed setup.
-func (s *erasureSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
-	storageDisks, errs := initStorageDisksWithErrors(s.endpoints)
-	for i, err := range errs {
-		if err != nil && err != errDiskNotFound {
-			return fmt.Errorf("Disk %s: %w", s.endpoints[i], err)
-		}
-	}
-	defer func(storageDisks []StorageAPI) {
-		if err != nil {
-			closeStorageDisks(storageDisks)
-		}
-	}(storageDisks)
-
-	formats, _ := loadFormatErasureAll(storageDisks, false)
-	if err = checkFormatErasureValues(formats, s.setDriveCount); err != nil {
-		return err
-	}
-
-	refFormat, err := getFormatErasureInQuorum(formats)
-	if err != nil {
-		return err
-	}
-
-	s.erasureDisksMu.Lock()
-
-	// Replace with new reference format.
-	s.format = refFormat
-
-	// Close all existing disks and reconnect all the disks.
-	for _, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-
-		diskID, err := disk.GetDiskID()
-		if err != nil {
-			continue
-		}
-
-		m, n, err := findDiskIndexByDiskID(refFormat, diskID)
-		if err != nil {
-			continue
-		}
-
-		if s.erasureDisks[m][n] != nil {
-			s.erasureDisks[m][n].Close()
-		}
-
-		s.endpointStrings[m*s.setDriveCount+n] = disk.String()
-		s.erasureDisks[m][n] = disk
-	}
-
-	s.erasureDisksMu.Unlock()
-
-	return nil
-}
-
 // If it is a single node Erasure and all disks are root disks, it is most likely a test setup, else it is a production setup.
 // On a test setup we allow creation of format.json on root disks to help with dev/testing.
 func isTestSetup(infos []DiskInfo, errs []error) bool {
@@ -1284,7 +1133,7 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 
 // HealFormat - heals missing `format.json` on fresh unformatted disks.
 func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
-	storageDisks, errs := initStorageDisksWithErrors(s.endpoints)
+	storageDisks, errs := initStorageDisksWithErrorsWithoutHealthCheck(s.endpoints)
 	for i, derr := range errs {
 		if derr != nil && derr != errDiskNotFound {
 			return madmin.HealResultItem{}, fmt.Errorf("Disk %s: %w", s.endpoints[i], derr)
@@ -1310,14 +1159,6 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 		return res, err
 	}
 
-	for i, format := range formats {
-		if format != nil {
-			if ferr := formatErasureV3Check(refFormat, format); ferr != nil {
-				sErrs[i] = errUnformattedDisk
-			}
-		}
-	}
-
 	// Prepare heal-result
 	res = madmin.HealResultItem{
 		Type:      madmin.HealItemMetadata,
@@ -1338,45 +1179,12 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 	}
 
 	if countErrs(sErrs, errUnformattedDisk) == 0 {
-		// No unformatted disks found disks are either offline
-		// or online, no healing is required.
 		return res, errNoHealRequired
 	}
 
-	// Mark all UUIDs which might be offline, use list
-	// of formats to mark them appropriately.
-	markUUIDsOffline(refFormat, formats)
-
 	// Initialize a new set of set formats which will be written to disk.
 	newFormatSets := newHealFormatSets(refFormat, s.setCount, s.setDriveCount, formats, sErrs)
 
-	// Look for all offline/unformatted disks in our reference format,
-	// such that we can fill them up with new UUIDs, this looping also
-	// ensures that the replaced disks allocated evenly across all sets.
-	// Making sure that the redundancy is not lost.
-	for i := range refFormat.Erasure.Sets {
-		for j := range refFormat.Erasure.Sets[i] {
-			if refFormat.Erasure.Sets[i][j] == offlineDiskUUID {
-				for l := range newFormatSets[i] {
-					if newFormatSets[i][l] == nil {
-						continue
-					}
-					if newFormatSets[i][l].Erasure.This == "" {
-						newFormatSets[i][l].Erasure.This = mustGetUUID()
-						refFormat.Erasure.Sets[i][j] = newFormatSets[i][l].Erasure.This
-						for m, v := range res.After.Drives {
-							if v.Endpoint == s.endpoints.GetString(i*s.setDriveCount+l) {
-								res.After.Drives[m].UUID = newFormatSets[i][l].Erasure.This
-								res.After.Drives[m].State = madmin.DriveStateOk
-							}
-						}
-						break
-					}
-				}
-			}
-		}
-	}
-
 	if !dryRun {
 		var tmpNewFormats = make([]*formatErasureV3, s.setCount*s.setDriveCount)
 		for i := range newFormatSets {
@@ -1384,38 +1192,25 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 				if newFormatSets[i][j] == nil {
 					continue
 				}
+				res.After.Drives[i*s.setDriveCount+j].UUID = newFormatSets[i][j].Erasure.This
+				res.After.Drives[i*s.setDriveCount+j].State = madmin.DriveStateOk
 				tmpNewFormats[i*s.setDriveCount+j] = newFormatSets[i][j]
-				tmpNewFormats[i*s.setDriveCount+j].Erasure.Sets = refFormat.Erasure.Sets
 			}
 		}
 
-		// Save formats `format.json` across all disks.
-		if err = saveFormatErasureAllWithErrs(ctx, storageDisks, sErrs, tmpNewFormats); err != nil {
-			return madmin.HealResultItem{}, err
-		}
-
-		refFormat, err = getFormatErasureInQuorum(tmpNewFormats)
-		if err != nil {
+		// Save new formats `format.json` on unformatted disks.
+		if err = saveUnformattedFormat(ctx, storageDisks, tmpNewFormats); err != nil {
 			return madmin.HealResultItem{}, err
 		}
 
 		s.erasureDisksMu.Lock()
 
-		// Replace with new reference format.
-		s.format = refFormat
-
-		// Disconnect/relinquish all existing disks, lockers and reconnect the disks, lockers.
-		for _, disk := range storageDisks {
-			if disk == nil {
+		for index, format := range tmpNewFormats {
+			if format == nil {
 				continue
 			}
 
-			diskID, err := disk.GetDiskID()
-			if err != nil {
-				continue
-			}
-
-			m, n, err := findDiskIndexByDiskID(refFormat, diskID)
+			m, n, err := findDiskIndexByDiskID(refFormat, format.Erasure.This)
 			if err != nil {
 				continue
 			}
@@ -1424,10 +1219,13 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 				s.erasureDisks[m][n].Close()
 			}
 
-			s.endpointStrings[m*s.setDriveCount+n] = disk.String()
-			s.erasureDisks[m][n] = disk
+			s.erasureDisks[m][n] = storageDisks[index]
+			s.endpointStrings[m*s.setDriveCount+n] = storageDisks[index].String()
 		}
 
+		// Replace reference format with what was loaded from disks.
+		s.format = refFormat
+
 		s.erasureDisksMu.Unlock()
 	}
 
@@ -1485,105 +1283,6 @@ func (s *erasureSets) ListBucketsHeal(ctx context.Context) ([]BucketInfo, error)
 	return listBuckets, nil
 }
 
-// Walk a bucket, optionally prefix recursively, until we have returned
-// all the content to objectInfo channel, it is callers responsibility
-// to allocate a receive channel for ObjectInfo, upon any unhandled
-// error walker returns error. Optionally if context.Done() is received
-// then Walk() stops the walker.
-func (s *erasureSets) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo, opts ObjectOptions) error {
-	if err := checkListObjsArgs(ctx, bucket, prefix, "", s); err != nil {
-		// Upon error close the channel.
-		close(results)
-		return err
-	}
-
-	if opts.WalkVersions {
-		entryChs := s.startMergeWalksVersions(ctx, bucket, prefix, "", true, ctx.Done())
-
-		entriesValid := make([]bool, len(entryChs))
-		entries := make([]FileInfoVersions, len(entryChs))
-
-		go func() {
-			defer close(results)
-
-			for {
-				entry, quorumCount, ok := lexicallySortedEntryVersions(entryChs, entries, entriesValid)
-				if !ok {
-					return
-				}
-
-				if quorumCount >= s.setDriveCount/2 {
-					// Read quorum exists proceed
-					for _, version := range entry.Versions {
-						results <- version.ToObjectInfo(bucket, version.Name)
-					}
-				}
-				// skip entries which do not have quorum
-			}
-		}()
-
-		return nil
-	}
-
-	entryChs := s.startMergeWalks(ctx, bucket, prefix, "", true, ctx.Done())
-
-	entriesValid := make([]bool, len(entryChs))
-	entries := make([]FileInfo, len(entryChs))
-
-	go func() {
-		defer close(results)
-
-		for {
-			entry, quorumCount, ok := lexicallySortedEntry(entryChs, entries, entriesValid)
-			if !ok {
-				return
-			}
-
-			if quorumCount >= s.setDriveCount/2 {
-				// Read quorum exists proceed
-				results <- entry.ToObjectInfo(bucket, entry.Name)
-			}
-			// skip entries which do not have quorum
-		}
-	}()
-
-	return nil
-}
-
-// HealObjects - Heal all objects recursively at a specified prefix, any
-// dangling objects deleted as well automatically.
-func (s *erasureSets) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObject HealObjectFn) error {
-	endWalkCh := make(chan struct{})
-	defer close(endWalkCh)
-
-	entryChs := s.startMergeWalksVersions(ctx, bucket, prefix, "", true, endWalkCh)
-
-	entriesValid := make([]bool, len(entryChs))
-	entries := make([]FileInfoVersions, len(entryChs))
-	for {
-		entry, quorumCount, ok := lexicallySortedEntryVersions(entryChs, entries, entriesValid)
-		if !ok {
-			break
-		}
-
-		if quorumCount == s.setDriveCount && opts.ScanMode == madmin.HealNormalScan {
-			// Skip good entries.
-			continue
-		}
-
-		for _, version := range entry.Versions {
-			// Wait and proceed if there are active requests
-			waitForLowHTTPReq(int32(s.setDriveCount), time.Second)
-
-			if err := healObject(bucket, version.Name, version.VersionID); err != nil {
-				return toObjectErr(err, bucket, version.Name)
-			}
-		}
-	}
-
-	return nil
-}
-
 // PutObjectTags - replace or add tags to an existing object
 func (s *erasureSets) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) error {
 	return s.getHashedSet(object).PutObjectTags(ctx, bucket, object, tags, opts)

cmd/erasure-sets_test.go

@@ -197,27 +197,13 @@ func TestNewErasureSets(t *testing.T) {
 // TestHashedLayer - tests the hashed layer which will be returned
 // consistently for a given object name.
 func TestHashedLayer(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	var objs []*erasureObjects
-	for i := 0; i < 16; i++ {
-		obj, fsDirs, err := prepareErasure16(ctx)
-		if err != nil {
-			t.Fatal("Unable to initialize 'Erasure' object layer.", err)
-		}
-		defer obj.Shutdown(ctx)
-
-		// Remove all dirs.
-		for _, dir := range fsDirs {
-			defer os.RemoveAll(dir)
-		}
-
-		z := obj.(*erasureZones)
-		objs = append(objs, z.zones[0].sets[0])
+	// Test distribution with 16 sets.
+	var objs [16]*erasureObjects
+	for i := range objs {
+		objs[i] = &erasureObjects{}
 	}
 
-	sets := &erasureSets{sets: objs, distributionAlgo: "CRCMOD"}
+	sets := &erasureSets{sets: objs[:], distributionAlgo: "CRCMOD"}
 
 	testCases := []struct {
 		objectName  string

cmd/erasure.go

@@ -68,8 +68,8 @@ type erasureObjects struct {
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (er erasureObjects) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	return er.nsMutex.NewNSLock(ctx, er.getLockers, bucket, objects...)
+func (er erasureObjects) NewNSLock(bucket string, objects ...string) RWLocker {
+	return er.nsMutex.NewNSLock(er.getLockers, bucket, objects...)
 }
 
 // SetDriveCount returns the current drives per set.
@@ -81,14 +81,6 @@ func (er erasureObjects) SetDriveCount() int {
 func (er erasureObjects) Shutdown(ctx context.Context) error {
 	// Add any object layer shutdown activities here.
 	closeStorageDisks(er.getDisks())
-	select {
-	case _, ok := <-er.mrfOpCh:
-		if ok {
-			close(er.mrfOpCh)
-		}
-	default:
-		close(er.mrfOpCh)
-	}
 	return nil
 }
 
@@ -186,7 +178,9 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 		}
 	}
 
-	if len(disksInfo) == rootDiskCount {
+	// Count offline disks as well to ensure consistent
+	// reportability of offline drives on local setups.
+	if len(disksInfo) == (rootDiskCount + offlineDisks.Sum()) {
 		// Success.
 		return disksInfo, errs, onlineDisks, offlineDisks
 	}
@@ -252,7 +246,7 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 	}
 
 	// Collect disks we can use.
-	disks := er.getLoadBalancedDisks()
+	disks := er.getOnlineDisks()
 	if len(disks) == 0 {
 		logger.Info(color.Green("data-crawl:") + " all disks are offline or being healed, skipping crawl")
 		return nil

cmd/etcd.go

@@ -38,9 +38,23 @@ func etcdErrToErr(err error, etcdEndpoints []string) error {
 	}
 }
 
-func saveKeyEtcd(ctx context.Context, client *etcd.Client, key string, data []byte) error {
+func saveKeyEtcdWithTTL(ctx context.Context, client *etcd.Client, key string, data []byte, ttl int64) error {
 	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
+	lease, err := client.Grant(timeoutCtx, ttl)
+	if err != nil {
+		return etcdErrToErr(err, client.Endpoints())
+	}
+	_, err = client.Put(timeoutCtx, key, string(data), etcd.WithLease(lease.ID))
+	return etcdErrToErr(err, client.Endpoints())
+}
+
+func saveKeyEtcd(ctx context.Context, client *etcd.Client, key string, data []byte, opts ...options) error {
+	timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
+	defer cancel()
+	if len(opts) > 0 {
+		return saveKeyEtcdWithTTL(ctx, client, key, data, opts[0].ttl)
+	}
 	_, err := client.Put(timeoutCtx, key, string(data))
 	return etcdErrToErr(err, client.Endpoints())
 }

cmd/format-disk-cache.go

@@ -365,7 +365,7 @@ func migrateCacheData(ctx context.Context, c *diskCache, bucket, object, oldfile
 		}
 		actualSize, _ = sio.EncryptedSize(uint64(st.Size()))
 	}
-	_, err = c.bitrotWriteToCache(destDir, cacheDataFile, reader, actualSize)
+	_, _, err = c.bitrotWriteToCache(destDir, cacheDataFile, reader, actualSize)
 	return err
 }
 

cmd/format-erasure.go

@@ -17,7 +17,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"encoding/hex"
 	"encoding/json"
@@ -131,13 +130,17 @@ func (f *formatErasureV3) Clone() *formatErasureV3 {
 }
 
 // Returns formatErasure.Erasure.Version
-func newFormatErasureV3(numSets int, setLen int) *formatErasureV3 {
+func newFormatErasureV3(numSets int, setLen int, distributionAlgo string) *formatErasureV3 {
 	format := &formatErasureV3{}
 	format.Version = formatMetaVersionV1
 	format.Format = formatBackendErasure
 	format.ID = mustGetUUID()
 	format.Erasure.Version = formatErasureVersionV3
-	format.Erasure.DistributionAlgo = formatErasureVersionV3DistributionAlgo
+	if distributionAlgo == "" {
+		format.Erasure.DistributionAlgo = formatErasureVersionV3DistributionAlgo
+	} else {
+		format.Erasure.DistributionAlgo = distributionAlgo
+	}
 	format.Erasure.Sets = make([][]string, numSets)
 
 	for i := 0; i < numSets; i++ {
@@ -356,10 +359,10 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 	tmpFormat := mustGetUUID()
 
 	// Purge any existing temporary file, okay to ignore errors here.
-	defer disk.DeleteFile(context.TODO(), minioMetaBucket, tmpFormat)
+	defer disk.Delete(context.TODO(), minioMetaBucket, tmpFormat, false)
 
 	// write to unique file.
-	if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, bytes.NewReader(formatBytes)); err != nil {
+	if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, formatBytes); err != nil {
 		return err
 	}
 
@@ -379,7 +382,7 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 		}
 		return disk.WriteAll(context.TODO(), minioMetaBucket,
 			pathJoin(bucketMetaPrefix, slashSeparator, healingTrackerFilename),
-			bytes.NewReader(htrackerBytes))
+			htrackerBytes)
 	}
 	return nil
 }
@@ -645,8 +648,8 @@ func formatErasureV3Check(reference *formatErasureV3, format *formatErasureV3) e
 		}
 		for j := range reference.Erasure.Sets[i] {
 			if reference.Erasure.Sets[i][j] != format.Erasure.Sets[i][j] {
-				return fmt.Errorf("UUID on positions %d:%d do not match with, expected %s got %s",
-					i, j, reference.Erasure.Sets[i][j], format.Erasure.Sets[i][j])
+				return fmt.Errorf("UUID on positions %d:%d do not match with, expected %s got %s: (%w)",
+					i, j, reference.Erasure.Sets[i][j], format.Erasure.Sets[i][j], errInconsistentDisk)
 			}
 		}
 	}
@@ -700,25 +703,18 @@ func initErasureMetaVolumesInLocalDisks(storageDisks []StorageAPI, formats []*fo
 	return nil
 }
 
-// saveFormatErasureAllWithErrs - populates `format.json` on disks in its order.
+// saveUnformattedFormat - populates `format.json` on unformatted disks.
 // also adds `.healing.bin` on the disks which are being actively healed.
-func saveFormatErasureAllWithErrs(ctx context.Context, storageDisks []StorageAPI, fErrs []error, formats []*formatErasureV3) error {
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	// Write `format.json` to all disks.
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if formats[index] == nil {
-				return errDiskNotFound
-			}
-			return saveFormatErasure(storageDisks[index], formats[index], errors.Is(fErrs[index], errUnformattedDisk))
-		}, index)
+func saveUnformattedFormat(ctx context.Context, storageDisks []StorageAPI, formats []*formatErasureV3) error {
+	for index, format := range formats {
+		if format == nil {
+			continue
+		}
+		if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
+			return err
+		}
 	}
-
-	writeQuorum := getWriteQuorum(len(storageDisks))
-	// Wait for the routines to finish.
-	return reduceWriteQuorumErrs(ctx, g.Wait(), nil, writeQuorum)
+	return nil
 }
 
 // saveFormatErasureAll - populates `format.json` on disks in its order.
@@ -751,6 +747,20 @@ func closeStorageDisks(storageDisks []StorageAPI) {
 	}
 }
 
+func initStorageDisksWithErrorsWithoutHealthCheck(endpoints Endpoints) ([]StorageAPI, []error) {
+	// Bootstrap disks.
+	storageDisks := make([]StorageAPI, len(endpoints))
+	g := errgroup.WithNErrs(len(endpoints))
+	for index := range endpoints {
+		index := index
+		g.Go(func() (err error) {
+			storageDisks[index], err = newStorageAPIWithoutHealthCheck(endpoints[index])
+			return err
+		}, index)
+	}
+	return storageDisks, g.Wait()
+}
+
 // Initialize storage disks for each endpoint.
 // Errors are returned for each endpoint with matching index.
 func initStorageDisksWithErrors(endpoints Endpoints) ([]StorageAPI, []error) {
@@ -824,8 +834,8 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats
 }
 
 // initFormatErasure - save Erasure format configuration on all disks.
-func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, setDriveCount int, deploymentID string, sErrs []error) (*formatErasureV3, error) {
-	format := newFormatErasureV3(setCount, setDriveCount)
+func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, setDriveCount int, distributionAlgo string, deploymentID string, sErrs []error) (*formatErasureV3, error) {
+	format := newFormatErasureV3(setCount, setDriveCount, distributionAlgo)
 	formats := make([]*formatErasureV3, len(storageDisks))
 	wantAtMost := ecDrivesNoConfig(setDriveCount)
 
@@ -901,60 +911,6 @@ func makeFormatErasureMetaVolumes(disk StorageAPI) error {
 	return disk.MakeVolBulk(context.TODO(), minioMetaBucket, minioMetaTmpBucket, minioMetaMultipartBucket, dataUsageBucket)
 }
 
-// Get all UUIDs which are present in reference format should
-// be present in the list of formats provided, those are considered
-// as online UUIDs.
-func getOnlineUUIDs(refFormat *formatErasureV3, formats []*formatErasureV3) (onlineUUIDs []string) {
-	for _, format := range formats {
-		if format == nil {
-			continue
-		}
-		for _, set := range refFormat.Erasure.Sets {
-			for _, uuid := range set {
-				if format.Erasure.This == uuid {
-					onlineUUIDs = append(onlineUUIDs, uuid)
-				}
-			}
-		}
-	}
-	return onlineUUIDs
-}
-
-// Look for all UUIDs which are not present in reference format
-// but are present in the onlineUUIDs list, construct of list such
-// offline UUIDs.
-func getOfflineUUIDs(refFormat *formatErasureV3, formats []*formatErasureV3) (offlineUUIDs []string) {
-	onlineUUIDs := getOnlineUUIDs(refFormat, formats)
-	for i, set := range refFormat.Erasure.Sets {
-		for j, uuid := range set {
-			var found bool
-			for _, onlineUUID := range onlineUUIDs {
-				if refFormat.Erasure.Sets[i][j] == onlineUUID {
-					found = true
-				}
-			}
-			if !found {
-				offlineUUIDs = append(offlineUUIDs, uuid)
-			}
-		}
-	}
-	return offlineUUIDs
-}
-
-// Mark all UUIDs that are offline.
-func markUUIDsOffline(refFormat *formatErasureV3, formats []*formatErasureV3) {
-	offlineUUIDs := getOfflineUUIDs(refFormat, formats)
-	for i, set := range refFormat.Erasure.Sets {
-		for j := range set {
-			for _, offlineUUID := range offlineUUIDs {
-				if refFormat.Erasure.Sets[i][j] == offlineUUID {
-					refFormat.Erasure.Sets[i][j] = offlineDiskUUID
-				}
-			}
-		}
-	}
-}
-
 // Initialize a new set of set formats which will be written to all disks.
 func newHealFormatSets(refFormat *formatErasureV3, setCount, setDriveCount int, formats []*formatErasureV3, errs []error) [][]*formatErasureV3 {
 	newFormats := make([][]*formatErasureV3, setCount)
@@ -963,23 +919,16 @@ func newHealFormatSets(refFormat *formatErasureV3, setCount, setDriveCount int,
 	}
 	for i := range refFormat.Erasure.Sets {
 		for j := range refFormat.Erasure.Sets[i] {
-			if errs[i*setDriveCount+j] == errUnformattedDisk || errs[i*setDriveCount+j] == nil {
+			if errors.Is(errs[i*setDriveCount+j], errUnformattedDisk) {
 				newFormats[i][j] = &formatErasureV3{}
-				newFormats[i][j].Version = refFormat.Version
 				newFormats[i][j].ID = refFormat.ID
 				newFormats[i][j].Format = refFormat.Format
+				newFormats[i][j].Version = refFormat.Version
+				newFormats[i][j].Erasure.This = refFormat.Erasure.Sets[i][j]
+				newFormats[i][j].Erasure.Sets = refFormat.Erasure.Sets
 				newFormats[i][j].Erasure.Version = refFormat.Erasure.Version
 				newFormats[i][j].Erasure.DistributionAlgo = refFormat.Erasure.DistributionAlgo
 			}
-			if errs[i*setDriveCount+j] == errUnformattedDisk {
-				newFormats[i][j].Erasure.This = ""
-				newFormats[i][j].Erasure.Sets = nil
-				continue
-			}
-			if errs[i*setDriveCount+j] == nil {
-				newFormats[i][j].Erasure.This = formats[i*setDriveCount+j].Erasure.This
-				newFormats[i][j].Erasure.Sets = nil
-			}
 		}
 	}
 	return newFormats

cmd/format-erasure_test.go

@@ -25,56 +25,6 @@ import (
 	"testing"
 )
 
-// Test get offline/online uuids.
-func TestGetUUIDs(t *testing.T) {
-	fmtV2 := newFormatErasureV3(4, 16)
-	formats := make([]*formatErasureV3, 64)
-
-	for i := 0; i < 4; i++ {
-		for j := 0; j < 16; j++ {
-			newFormat := *fmtV2
-			newFormat.Erasure.This = fmtV2.Erasure.Sets[i][j]
-			formats[i*16+j] = &newFormat
-		}
-	}
-
-	gotCount := len(getOnlineUUIDs(fmtV2, formats))
-	if gotCount != 64 {
-		t.Errorf("Expected online count '64', got '%d'", gotCount)
-	}
-
-	for i := 0; i < 4; i++ {
-		for j := 0; j < 16; j++ {
-			if j < 4 {
-				formats[i*16+j] = nil
-			}
-		}
-	}
-
-	gotCount = len(getOnlineUUIDs(fmtV2, formats))
-	if gotCount != 48 {
-		t.Errorf("Expected online count '48', got '%d'", gotCount)
-	}
-
-	gotCount = len(getOfflineUUIDs(fmtV2, formats))
-	if gotCount != 16 {
-		t.Errorf("Expected offline count '16', got '%d'", gotCount)
-	}
-
-	markUUIDsOffline(fmtV2, formats)
-	gotCount = 0
-	for i := range fmtV2.Erasure.Sets {
-		for j := range fmtV2.Erasure.Sets[i] {
-			if fmtV2.Erasure.Sets[i][j] == offlineDiskUUID {
-				gotCount++
-			}
-		}
-	}
-	if gotCount != 16 {
-		t.Errorf("Expected offline count '16', got '%d'", gotCount)
-	}
-}
-
 // tests fixFormatErasureV3 - fix format.json on all disks.
 func TestFixFormatV3(t *testing.T) {
 	erasureDirs, err := getRandomDisks(8)
@@ -93,7 +43,7 @@ func TestFixFormatV3(t *testing.T) {
 		}
 	}
 
-	format := newFormatErasureV3(1, 8)
+	format := newFormatErasureV3(1, 8, "CRCMOD")
 	formats := make([]*formatErasureV3, 8)
 
 	for j := 0; j < 8; j++ {
@@ -127,7 +77,7 @@ func TestFixFormatV3(t *testing.T) {
 
 // tests formatErasureV3ThisEmpty conditions.
 func TestFormatErasureEmpty(t *testing.T) {
-	format := newFormatErasureV3(1, 16)
+	format := newFormatErasureV3(1, 16, "CRCMOD")
 	formats := make([]*formatErasureV3, 16)
 
 	for j := 0; j < 16; j++ {
@@ -326,7 +276,7 @@ func TestGetFormatErasureInQuorumCheck(t *testing.T) {
 	setCount := 2
 	setDriveCount := 16
 
-	format := newFormatErasureV3(setCount, setDriveCount)
+	format := newFormatErasureV3(setCount, setDriveCount, "CRCMOD")
 	formats := make([]*formatErasureV3, 32)
 
 	for i := 0; i < setCount; i++ {
@@ -392,7 +342,7 @@ func TestGetErasureID(t *testing.T) {
 	setCount := 2
 	setDriveCount := 8
 
-	format := newFormatErasureV3(setCount, setDriveCount)
+	format := newFormatErasureV3(setCount, setDriveCount, "CRCMOD")
 	formats := make([]*formatErasureV3, 16)
 
 	for i := 0; i < setCount; i++ {
@@ -447,7 +397,7 @@ func TestNewFormatSets(t *testing.T) {
 	setCount := 2
 	setDriveCount := 16
 
-	format := newFormatErasureV3(setCount, setDriveCount)
+	format := newFormatErasureV3(setCount, setDriveCount, "CRCMOD")
 	formats := make([]*formatErasureV3, 32)
 	errs := make([]error, 32)
 
@@ -475,6 +425,9 @@ func TestNewFormatSets(t *testing.T) {
 	// Check if deployment IDs are preserved.
 	for i := range newFormats {
 		for j := range newFormats[i] {
+			if newFormats[i][j] == nil {
+				continue
+			}
 			if newFormats[i][j].ID != quorumFormat.ID {
 				t.Fatal("Deployment id in the new format is lost")
 			}

cmd/format-fs.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math/rand"
 	"os"
 	"path"
 	"time"
@@ -27,7 +28,6 @@ import (
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/lock"
-	"github.com/minio/minio/pkg/retry"
 )
 
 // FS format version strings.
@@ -340,31 +340,28 @@ func formatFSFixDeploymentID(ctx context.Context, fsFormatPath string) error {
 		return time.Now().Round(time.Second).Sub(formatStartTime).String()
 	}
 
-	retryCtx, cancel := context.WithCancel(ctx)
-	// Indicate to our routine to exit cleanly upon return.
-	defer cancel()
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	var wlk *lock.LockedFile
-	retryCh := retry.NewTimerWithJitter(retryCtx, time.Second, 30*time.Second, retry.MaxJitter)
 	var stop bool
 	for !stop {
 		select {
-		case <-retryCh:
+		case <-ctx.Done():
+			return fmt.Errorf("Initializing FS format stopped gracefully")
+		default:
 			wlk, err = lock.TryLockedOpenFile(fsFormatPath, os.O_RDWR, 0)
 			if err == lock.ErrAlreadyLocked {
 				// Lock already present, sleep and attempt again
 				logger.Info("Another minio process(es) might be holding a lock to the file %s. Please kill that minio process(es) (elapsed %s)\n", fsFormatPath, getElapsedTime())
+				time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
 				continue
 			}
 			if err != nil {
 				return err
 			}
-			stop = true
-		case <-ctx.Done():
-			return fmt.Errorf("Initializing FS format stopped gracefully")
 		}
+		stop = true
 	}
-
 	defer wlk.Close()
 
 	if err = jsonLoad(wlk, format); err != nil {

cmd/fs-v1-helpers_test.go

@@ -297,7 +297,7 @@ func TestFSDeletes(t *testing.T) {
 			srcPath:     "success-file",
 			expectedErr: nil,
 		},
-		// The file was deleted in the last case, so DeleteFile should fail.
+		// The file was deleted in the last case, so Delete should fail.
 		{
 			basePath:    path,
 			srcVol:      "success-vol",
@@ -423,7 +423,7 @@ func TestFSRemoves(t *testing.T) {
 			expectedErr: nil,
 		},
 		// Test case - 2.
-		// The file was deleted in the last case, so DeleteFile should fail.
+		// The file was deleted in the last case, so Delete should fail.
 		{
 			srcFSPath:   path,
 			srcVol:      "success-vol",

cmd/fs-v1-multipart.go

@@ -714,8 +714,8 @@ func (fs *FSObjects) CompleteMultipartUpload(ctx context.Context, bucket string,
 	}
 
 	// Hold write lock on the object.
-	destLock := fs.NewNSLock(ctx, bucket, object)
-	if err = destLock.GetLock(globalOperationTimeout); err != nil {
+	destLock := fs.NewNSLock(bucket, object)
+	if err = destLock.GetLock(ctx, globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer destLock.Unlock()

cmd/fs-v1-multipart_test.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sync"
 	"testing"
 	"time"
@@ -195,6 +196,11 @@ func TestCompleteMultipartUpload(t *testing.T) {
 
 // TestCompleteMultipartUpload - test CompleteMultipartUpload
 func TestAbortMultipartUpload(t *testing.T) {
+	if runtime.GOOS == globalWindowsOSName {
+		// Concurrent AbortMultipartUpload() fails on windows
+		t.Skip()
+	}
+
 	// Prepare for tests
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)
@@ -219,7 +225,6 @@ func TestAbortMultipartUpload(t *testing.T) {
 	if _, err := obj.PutObjectPart(GlobalContext, bucketName, objectName, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(data), 5, md5Hex, ""), opts); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}
-	time.Sleep(time.Second) // Without Sleep on windows, the fs.AbortMultipartUpload() fails with "The process cannot access the file because it is being used by another process."
 	if err := obj.AbortMultipartUpload(GlobalContext, bucketName, objectName, uploadID, opts); err != nil {
 		t.Fatal("Unexpected error ", err)
 	}

cmd/fs-v1.go

@@ -186,9 +186,9 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
-func (fs *FSObjects) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
+func (fs *FSObjects) NewNSLock(bucket string, objects ...string) RWLocker {
 	// lockers are explicitly 'nil' for FS mode since there are only local lockers
-	return fs.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
+	return fs.nsMutex.NewNSLock(nil, bucket, objects...)
 }
 
 // SetDriveCount no-op
@@ -327,7 +327,7 @@ func (fs *FSObjects) crawlBucket(ctx context.Context, bucket string, cache dataU
 	}
 
 	// Load bucket info.
-	cache, err = crawlDataFolder(ctx, fs.fsPath, cache, fs.waitForLowActiveIO, func(item crawlItem) (int64, error) {
+	cache, err = crawlDataFolder(ctx, fs.fsPath, cache, func(item crawlItem) (int64, error) {
 		bucket, object := item.bucket, item.objectPath()
 		fsMetaBytes, err := ioutil.ReadFile(pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile))
 		if err != nil && !os.IsNotExist(err) {
@@ -402,6 +402,7 @@ func (fs *FSObjects) MakeBucketWithLocation(ctx context.Context, bucket string,
 		return BucketNameInvalid{Bucket: bucket}
 	}
 
+	defer ObjectPathUpdated(bucket + slashSeparator)
 	atomic.AddInt64(&fs.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt64(&fs.activeIOCount, -1)
@@ -601,8 +602,8 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 	defer ObjectPathUpdated(path.Join(dstBucket, dstObject))
 
 	if !cpSrcDstSame {
-		objectDWLock := fs.NewNSLock(ctx, dstBucket, dstObject)
-		if err := objectDWLock.GetLock(globalOperationTimeout); err != nil {
+		objectDWLock := fs.NewNSLock(dstBucket, dstObject)
+		if err := objectDWLock.GetLock(ctx, globalOperationTimeout); err != nil {
 			return oi, err
 		}
 		defer objectDWLock.Unlock()
@@ -653,7 +654,7 @@ func (fs *FSObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBu
 		return fsMeta.ToObjectInfo(srcBucket, srcObject, fi), nil
 	}
 
-	if err := checkPutObjectArgs(ctx, dstBucket, dstObject, fs, srcInfo.PutObjReader.Size()); err != nil {
+	if err := checkPutObjectArgs(ctx, dstBucket, dstObject, fs); err != nil {
 		return ObjectInfo{}, err
 	}
 
@@ -692,15 +693,15 @@ func (fs *FSObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 
 	if lockType != noLock {
 		// Lock the object before reading.
-		lock := fs.NewNSLock(ctx, bucket, object)
+		lock := fs.NewNSLock(bucket, object)
 		switch lockType {
 		case writeLock:
-			if err = lock.GetLock(globalOperationTimeout); err != nil {
+			if err = lock.GetLock(ctx, globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.Unlock
 		case readLock:
-			if err = lock.GetRLock(globalOperationTimeout); err != nil {
+			if err = lock.GetRLock(ctx, globalOperationTimeout); err != nil {
 				return nil, err
 			}
 			nsUnlocker = lock.RUnlock
@@ -785,8 +786,8 @@ func (fs *FSObjects) GetObject(ctx context.Context, bucket, object string, offse
 	}
 
 	// Lock the object before reading.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return err
 	}
@@ -1013,8 +1014,8 @@ func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (
 // getObjectInfoWithLock - reads object metadata and replies back ObjectInfo.
 func (fs *FSObjects) getObjectInfoWithLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
 	// Lock the object before reading.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetRLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
 		return oi, err
 	}
 	defer lk.RUnlock()
@@ -1051,8 +1052,8 @@ func (fs *FSObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
 
 	oi, err := fs.getObjectInfoWithLock(ctx, bucket, object)
 	if err == errCorruptedFormat || err == io.EOF {
-		lk := fs.NewNSLock(ctx, bucket, object)
-		if err = lk.GetLock(globalOperationTimeout); err != nil {
+		lk := fs.NewNSLock(bucket, object)
+		if err = lk.GetLock(ctx, globalOperationTimeout); err != nil {
 			return oi, toObjectErr(err, bucket, object)
 		}
 
@@ -1097,13 +1098,13 @@ func (fs *FSObjects) PutObject(ctx context.Context, bucket string, object string
 		return objInfo, NotImplemented{}
 	}
 
-	if err := checkPutObjectArgs(ctx, bucket, object, fs, r.Size()); err != nil {
+	if err := checkPutObjectArgs(ctx, bucket, object, fs); err != nil {
 		return ObjectInfo{}, err
 	}
 
 	// Lock the object.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err := lk.GetLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	if err := lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		logger.LogIf(ctx, err)
 		return objInfo, err
 	}
@@ -1284,8 +1285,8 @@ func (fs *FSObjects) DeleteObject(ctx context.Context, bucket, object string, op
 	}
 
 	// Acquire a write lock before deleting the object.
-	lk := fs.NewNSLock(ctx, bucket, object)
-	if err = lk.GetLock(globalOperationTimeout); err != nil {
+	lk := fs.NewNSLock(bucket, object)
+	if err = lk.GetLock(ctx, globalOperationTimeout); err != nil {
 		return objInfo, err
 	}
 	defer lk.Unlock()
@@ -1535,12 +1536,6 @@ func (fs *FSObjects) DeleteObjectTags(ctx context.Context, bucket, object string
 	return fs.PutObjectTags(ctx, bucket, object, "", opts)
 }
 
-// ReloadFormat - no-op for fs, Valid only for Erasure.
-func (fs *FSObjects) ReloadFormat(ctx context.Context, dryRun bool) error {
-	logger.LogIf(ctx, NotImplemented{})
-	return NotImplemented{}
-}
-
 // HealFormat - no-op for fs, Valid only for Erasure.
 func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
 	logger.LogIf(ctx, NotImplemented{})

cmd/gateway-common.go

@@ -270,7 +270,7 @@ func IsBackendOnline(ctx context.Context, clnt *http.Client, urlStr string) bool
 	resp, err := clnt.Do(req)
 	if err != nil {
 		clnt.CloseIdleConnections()
-		return !xnet.IsNetworkOrHostDown(err)
+		return !xnet.IsNetworkOrHostDown(err, false)
 	}
 	xhttp.DrainBody(resp.Body)
 	return true
@@ -291,7 +291,7 @@ func ErrorRespToObjectError(err error, params ...string) error {
 		object = params[1]
 	}
 
-	if xnet.IsNetworkOrHostDown(err) {
+	if xnet.IsNetworkOrHostDown(err, false) {
 		return BackendDown{}
 	}
 

cmd/gateway-main.go

@@ -51,8 +51,8 @@ type GatewayLocker struct {
 }
 
 // NewNSLock - implements gateway level locker
-func (l *GatewayLocker) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	return l.nsMutex.NewNSLock(ctx, nil, bucket, objects...)
+func (l *GatewayLocker) NewNSLock(bucket string, objects ...string) RWLocker {
+	return l.nsMutex.NewNSLock(nil, bucket, objects...)
 }
 
 // Walk - implements common gateway level Walker, to walk on all objects recursively at a prefix
@@ -153,6 +153,8 @@ func ValidateGatewayArguments(serverAddr, endpointAddr string) error {
 
 // StartGateway - handler for 'minio gateway <name>'.
 func StartGateway(ctx *cli.Context, gw Gateway) {
+	defer globalDNSCache.Stop()
+
 	// This is only to uniquely identify each gateway deployments.
 	globalDeploymentID = env.Get("MINIO_GATEWAY_DEPLOYMENT_ID", mustGetUUID())
 	logger.SetDeploymentID(globalDeploymentID)
@@ -289,7 +291,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	globalHTTPServer = httpServer
 	globalObjLayerMutex.Unlock()
 
-	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM)
+	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
 
 	newObject, err := gw.NewGatewayLayer(globalActiveCred)
 	if err != nil {
@@ -298,15 +300,14 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	}
 	newObject = NewGatewayLayerWithLocker(newObject)
 
-	// Once endpoints are finalized, initialize the new object api in safe mode.
-	globalObjLayerMutex.Lock()
-	globalSafeMode = true
-	globalObjectAPI = newObject
-	globalObjLayerMutex.Unlock()
-
 	// Calls all New() for all sub-systems.
 	newAllSubsystems()
 
+	// Once endpoints are finalized, initialize the new object api in safe mode.
+	globalObjLayerMutex.Lock()
+	globalObjectAPI = newObject
+	globalObjLayerMutex.Unlock()
+
 	if gatewayName == NASBackendGateway {
 		buckets, err := newObject.ListBuckets(GlobalContext)
 		if err != nil {
@@ -324,8 +325,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	}
 
 	if enableIAMOps {
-		// Initialize IAM sys.
-		startBackgroundIAMLoad(GlobalContext)
+		// Initialize users credentials and policies in background.
+		globalIAMSys.InitStore(newObject)
+
+		go globalIAMSys.Init(GlobalContext, newObject)
 	}
 
 	if globalCacheConfig.Enabled {
@@ -353,11 +356,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// - compression
 	verifyObjectLayerFeatures("gateway "+gatewayName, newObject)
 
-	// Disable safe mode operation, after all initialization is over.
-	globalObjLayerMutex.Lock()
-	globalSafeMode = false
-	globalObjLayerMutex.Unlock()
-
 	// Prints the formatted startup message once object layer is initialized.
 	if !globalCLIContext.Quiet {
 		mode := globalMinioModeGatewayPrefix + gatewayName

cmd/gateway-unsupported.go

@@ -41,8 +41,8 @@ func (a GatewayUnsupported) CrawlAndGetDataUsage(ctx context.Context, bf *bloomF
 }
 
 // NewNSLock is a dummy stub for gateway.
-func (a GatewayUnsupported) NewNSLock(ctx context.Context, bucket string, objects ...string) RWLocker {
-	logger.CriticalIf(ctx, errors.New("not implemented"))
+func (a GatewayUnsupported) NewNSLock(bucket string, objects ...string) RWLocker {
+	logger.CriticalIf(context.Background(), errors.New("not implemented"))
 	return nil
 }
 
@@ -160,11 +160,6 @@ func (a GatewayUnsupported) DeleteBucketSSEConfig(ctx context.Context, bucket st
 	return NotImplemented{}
 }
 
-// ReloadFormat - Not implemented stub.
-func (a GatewayUnsupported) ReloadFormat(ctx context.Context, dryRun bool) error {
-	return NotImplemented{}
-}
-
 // HealFormat - Not implemented stub
 func (a GatewayUnsupported) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) {
 	return madmin.HealResultItem{}, NotImplemented{}

cmd/gateway/azure/gateway-azure.go

@@ -559,7 +559,8 @@ func (a *azureObjects) StorageInfo(ctx context.Context, _ bool) (si minio.Storag
 
 // MakeBucketWithLocation - Create a new container on azure backend.
 func (a *azureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts minio.BucketOptions) error {
-	if opts.LockEnabled || opts.VersioningEnabled {
+	// Filter out unsupported features in Azure and return immediately with NotImplemented error
+	if opts.LockEnabled || opts.VersioningEnabled || strings.ContainsAny(bucket, ".") {
 		return minio.NotImplemented{}
 	}
 

cmd/gateway/nas/gateway-nas.go

@@ -52,7 +52,6 @@ EXAMPLES:
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_AFTER{{.AssignmentOperator}}3
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_LOW{{.AssignmentOperator}}75
      {{.Prompt}} {{.EnvVarSetCommand}} MINIO_CACHE_WATERMARK_HIGH{{.AssignmentOperator}}85
-
      {{.Prompt}} {{.HelpName}} /shared/nasvol
 `
 

cmd/gateway/s3/gateway-s3-sse.go

@@ -65,8 +65,8 @@ type s3EncObjects struct {
 
 // ListObjects lists all blobs in S3 bucket filtered by prefix
 func (l *s3EncObjects) ListObjects(ctx context.Context, bucket string, prefix string, marker string, delimiter string, maxKeys int) (loi minio.ListObjectsInfo, e error) {
-	var continuationToken, startAfter string
-	res, err := l.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, maxKeys, false, startAfter)
+	var startAfter string
+	res, err := l.ListObjectsV2(ctx, bucket, prefix, marker, delimiter, maxKeys, false, startAfter)
 	if err != nil {
 		return loi, err
 	}
@@ -91,10 +91,12 @@ func (l *s3EncObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 		if e != nil {
 			return loi, minio.ErrorRespToObjectError(e, bucket)
 		}
+
+		continuationToken = loi.NextContinuationToken
+		isTruncated = loi.IsTruncated
+
 		for _, obj := range loi.Objects {
 			startAfter = obj.Name
-			continuationToken = loi.NextContinuationToken
-			isTruncated = loi.IsTruncated
 
 			if !isGWObject(obj.Name) {
 				continue
@@ -708,16 +710,15 @@ func (l *s3EncObjects) cleanupStaleEncMultipartUploads(ctx context.Context, clea
 
 // cleanupStaleUploads removes old custom encryption multipart uploads on backend
 func (l *s3EncObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
-	for {
-		buckets, err := l.s3Objects.ListBuckets(ctx)
-		if err != nil {
-			break
-		}
-		for _, b := range buckets {
-			expParts := l.getStalePartsForBucket(ctx, b.Name, expiry)
-			for k := range expParts {
-				l.s3Objects.DeleteObject(ctx, b.Name, k, minio.ObjectOptions{})
-			}
+	buckets, err := l.s3Objects.ListBuckets(ctx)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
+	for _, b := range buckets {
+		expParts := l.getStalePartsForBucket(ctx, b.Name, expiry)
+		for k := range expParts {
+			l.s3Objects.DeleteObject(ctx, b.Name, k, minio.ObjectOptions{})
 		}
 	}
 }
@@ -729,6 +730,7 @@ func (l *s3EncObjects) getStalePartsForBucket(ctx context.Context, bucket string
 	for {
 		loi, err := l.s3Objects.ListObjectsV2(ctx, bucket, prefix, continuationToken, delimiter, 1000, false, startAfter)
 		if err != nil {
+			logger.LogIf(ctx, err)
 			break
 		}
 		for _, obj := range loi.Objects {