fix: delete marker permanent delete replication (#11581)

A small fix, use healDeleteDangling constant instead of 'true' in the
self-healing code.

.github/workflows/go.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.14.x, 1.15.x]
+        go-version: [1.15.x, 1.16.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2

Dockerfile.release

@@ -5,8 +5,8 @@ ARG TARGETARCH
 LABEL name="MinIO" \
       vendor="MinIO Inc <dev@min.io>" \
       maintainer="MinIO Inc <dev@min.io>" \
-      version="RELEASE.2021-01-30T00-20-58Z" \
-      release="RELEASE.2021-01-30T00-20-58Z" \
+      version="RELEASE.2021-02-14T04-01-33Z" \
+      release="RELEASE.2021-02-14T04-01-33Z" \
       summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
       description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
 

Makefile

@@ -71,10 +71,11 @@ build: checks
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
-hotfix: LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
-    sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#'))
-	TAG := "minio/minio:$(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD)"
-hotfix: install
+hotfix-vars:
+	$(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
+    sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#')))
+	$(eval TAG := "minio/minio:$(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD)")
+hotfix: hotfix-vars install
 
 docker-hotfix: hotfix checks
 	@echo "Building minio docker image '$(TAG)'"

README_zh_CN.md

@@ -89,7 +89,7 @@ service minio start
 
 ## 使用源码安装
 
-采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install)。最低需要Golang版本为 [go1.14](https://golang.org/dl/#stable)
+采用源码安装仅供开发人员和高级用户使用,如果你还没有Golang环境， 请参考 [How to install Golang](https://golang.org/doc/install)。最低需要Golang版本为 [go1.15](https://golang.org/dl/#stable)
 
 ```sh
 GO111MODULE=on go get github.com/minio/minio

cmd/admin-handlers-users.go

@@ -23,8 +23,10 @@ import (
 	"io/ioutil"
 	"net/http"
 	"path"
+	"sort"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	iampolicy "github.com/minio/minio/pkg/iam/policy"
@@ -688,12 +690,6 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	buckets, err := objectAPI.ListBuckets(ctx)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
 	// Load the latest calculated data usage
 	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
 	if err != nil {
@@ -701,11 +697,34 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		logger.LogIf(ctx, err)
 	}
 
-	accountName := cred.AccessKey
-	if cred.ParentUser != "" {
-		accountName = cred.ParentUser
+	// If etcd, dns federation configured list buckets from etcd.
+	var buckets []BucketInfo
+	if globalDNSConfig != nil && globalBucketFederation {
+		dnsBuckets, err := globalDNSConfig.List()
+		if err != nil && !IsErrIgnored(err,
+			dns.ErrNoEntriesFound,
+			dns.ErrDomainMissing) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
+		for _, dnsRecords := range dnsBuckets {
+			buckets = append(buckets, BucketInfo{
+				Name:    dnsRecords[0].Key,
+				Created: dnsRecords[0].CreationDate,
+			})
+		}
+		sort.Slice(buckets, func(i, j int) bool {
+			return buckets[i].Name < buckets[j].Name
+		})
+	} else {
+		buckets, err = objectAPI.ListBuckets(ctx)
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
 	}
 
+	accountName := cred.AccessKey
 	policies, err := globalIAMSys.PolicyDBGet(accountName, false)
 	if err != nil {
 		logger.LogIf(ctx, err)

cmd/admin-handlers.go

@@ -1585,6 +1585,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
 			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
 			usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
+		} else {
+			buckets = madmin.Buckets{Error: err.Error()}
+			objects = madmin.Objects{Error: err.Error()}
+			usage = madmin.Usage{Error: err.Error()}
 		}
 
 		// Fetching the backend information

cmd/admin-heal-ops.go

@@ -662,6 +662,13 @@ func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
 	}
 }
 
+func (h *healSequence) logHeal(healType madmin.HealItemType) {
+	h.mutex.Lock()
+	h.scannedItemsMap[healType]++
+	h.lastHealActivity = UTCNow()
+	h.mutex.Unlock()
+}
+
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
 	globalHealConfigMu.Lock()
 	opts := globalHealConfig

cmd/admin-router.go

@@ -110,10 +110,9 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 			// -- IAM APIs --
 
 			// Add policy IAM
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(httpTraceAll(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
 
 			// Add user IAM
-
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(httpTraceAll(adminAPI.AccountInfoHandler))
 
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")

cmd/api-headers.go

@@ -133,18 +133,20 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		}
 
 		// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
-		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+		if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 			continue
 		}
+
 		var isSet bool
 		for _, userMetadataPrefix := range userMetadataKeyPrefixes {
-			if !strings.HasPrefix(k, userMetadataPrefix) {
+			if !strings.HasPrefix(strings.ToLower(k), strings.ToLower(userMetadataPrefix)) {
 				continue
 			}
 			w.Header()[strings.ToLower(k)] = []string{v}
 			isSet = true
 			break
 		}
+
 		if !isSet {
 			w.Header().Set(k, v)
 		}
@@ -182,12 +184,13 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 	if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil {
 		ruleID, expiryTime := lc.PredictExpiryTime(lifecycle.ObjectOpts{
-			Name:         objInfo.Name,
-			UserTags:     objInfo.UserTags,
-			VersionID:    objInfo.VersionID,
-			ModTime:      objInfo.ModTime,
-			IsLatest:     objInfo.IsLatest,
-			DeleteMarker: objInfo.DeleteMarker,
+			Name:             objInfo.Name,
+			UserTags:         objInfo.UserTags,
+			VersionID:        objInfo.VersionID,
+			ModTime:          objInfo.ModTime,
+			IsLatest:         objInfo.IsLatest,
+			DeleteMarker:     objInfo.DeleteMarker,
+			SuccessorModTime: objInfo.SuccessorModTime,
 		})
 		if !expiryTime.IsZero() {
 			w.Header()[xhttp.AmzExpiration] = []string{

cmd/api-response.go

@@ -565,7 +565,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 					continue
 				}
 				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
-				if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 					continue
 				}
 				content.UserMetadata[k] = v
@@ -639,8 +639,16 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
 	listPartsResponse.Key = s3EncodeName(partsInfo.Object, encodingType)
 	listPartsResponse.UploadID = partsInfo.UploadID
 	listPartsResponse.StorageClass = globalMinioDefaultStorageClass
-	listPartsResponse.Initiator.ID = globalMinioDefaultOwnerID
-	listPartsResponse.Owner.ID = globalMinioDefaultOwnerID
+
+	// Dumb values not meaningful
+	listPartsResponse.Initiator = Initiator{
+		ID:          globalMinioDefaultOwnerID,
+		DisplayName: globalMinioDefaultOwnerID,
+	}
+	listPartsResponse.Owner = Owner{
+		ID:          globalMinioDefaultOwnerID,
+		DisplayName: globalMinioDefaultOwnerID,
+	}
 
 	listPartsResponse.MaxParts = partsInfo.MaxParts
 	listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker

cmd/auth-handler.go

@@ -498,6 +498,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 			// Validate Authorization header if its valid for JWT request.
 			if _, _, authErr := webRequestAuthenticate(r); authErr != nil {
 				w.WriteHeader(http.StatusUnauthorized)
+				w.Write([]byte(authErr.Error()))
 				return
 			}
 			h.ServeHTTP(w, r)

cmd/background-newdisks-heal-ops.go

@@ -204,8 +204,8 @@ wait:
 							}
 						}
 
-						lbDisks := z.serverPools[i].sets[setIndex].getOnlineDisks()
-						if err := healErasureSet(ctx, setIndex, buckets, lbDisks); err != nil {
+						err := z.serverPools[i].sets[setIndex].healErasureSet(ctx, buckets)
+						if err != nil {
 							logger.LogIf(ctx, err)
 							continue
 						}

cmd/bucket-handlers.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/textproto"
 	"net/url"
 	"path"
 	"path/filepath"
@@ -125,7 +126,10 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 
 	// Add/update buckets that are not registered with the DNS
 	bucketsToBeUpdatedSlice := bucketsToBeUpdated.ToSlice()
-	g := errgroup.WithNErrs(len(bucketsToBeUpdatedSlice))
+	g := errgroup.WithNErrs(len(bucketsToBeUpdatedSlice)).WithConcurrency(50)
+	ctx, cancel := g.WithCancelOnError(GlobalContext)
+	defer cancel()
+
 	for index := range bucketsToBeUpdatedSlice {
 		index := index
 		g.Go(func() error {
@@ -133,14 +137,13 @@ func initFederatorBackend(buckets []BucketInfo, objLayer ObjectLayer) {
 		}, index)
 	}
 
-	for _, err := range g.Wait() {
-		if err != nil {
-			logger.LogIf(GlobalContext, err)
-		}
+	if err := g.WaitErr(); err != nil {
+		logger.LogIf(ctx, err)
+		return
 	}
 
 	for _, bucket := range bucketsInConflict.ToSlice() {
-		logger.LogIf(GlobalContext, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket by a different tenant. This local bucket will be ignored. Bucket names are globally unique in federated deployments. Use path style requests on following addresses '%v' to access this bucket.", bucket, globalDomainIPs.ToSlice()))
+		logger.LogIf(ctx, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket by a different tenant. This local bucket will be ignored. Bucket names are globally unique in federated deployments. Use path style requests on following addresses '%v' to access this bucket", bucket, globalDomainIPs.ToSlice()))
 	}
 
 	var wg sync.WaitGroup
@@ -424,6 +427,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		deleteObjectsFn = api.CacheAPI().DeleteObjects
 	}
 
+	// Return Malformed XML as S3 spec if the list of objects is empty
+	if len(deleteObjects.Objects) == 0 {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	var objectsToDelete = map[ObjectToDelete]int{}
 	getObjectInfoFn := objectAPI.GetObjectInfo
 	if api.CacheAPI() != nil {
@@ -544,7 +553,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		dindex := objectsToDelete[ObjectToDelete{
 			ObjectName:                    dObjects[i].ObjectName,
 			VersionID:                     dObjects[i].VersionID,
-			DeleteMarkerVersionID:         dObjects[i].DeleteMarkerVersionID,
 			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus,
 			DeleteMarkerReplicationStatus: dObjects[i].DeleteMarkerReplicationStatus,
 			PurgeTransitioned:             dObjects[i].PurgeTransitioned,
@@ -580,6 +588,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Write success response.
 	writeSuccessResponseXML(w, encodedSuccessResponse)
 	for _, dobj := range deletedObjects {
+		if dobj.ObjectName == "" {
+			continue
+		}
+
 		if replicateDeletes {
 			if dobj.DeleteMarkerReplicationStatus == string(replication.Pending) || dobj.VersionPurgeStatus == Pending {
 				dv := DeletedObjectVersionInfo{
@@ -591,18 +603,14 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 
 		if hasLifecycleConfig && dobj.PurgeTransitioned == lifecycle.TransitionComplete { // clean up transitioned tier
-			deleteTransitionedObject(ctx, newObjectLayerFn(), bucket, dobj.ObjectName, lifecycle.ObjectOpts{
+			deleteTransitionedObject(ctx, objectAPI, bucket, dobj.ObjectName, lifecycle.ObjectOpts{
 				Name:         dobj.ObjectName,
 				VersionID:    dobj.VersionID,
 				DeleteMarker: dobj.DeleteMarker,
 			}, false, true)
 		}
 
-	}
-	// Notify deleted event for objects.
-	for _, dobj := range deletedObjects {
 		eventName := event.ObjectRemovedDelete
-
 		objInfo := ObjectInfo{
 			Name:      dobj.ObjectName,
 			VersionID: dobj.VersionID,
@@ -903,7 +911,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 
 	// Extract metadata to be saved from received Form.
 	metadata := make(map[string]string)
-	err = extractMetadataFromMap(ctx, formValues, metadata)
+	err = extractMetadataFromMime(ctx, textproto.MIMEHeader(formValues), metadata)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
@@ -916,7 +924,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	rawReader := hashReader
-	pReader := NewPutObjReader(rawReader, nil, nil)
+	pReader := NewPutObjReader(rawReader)
 	var objectEncryptionKey crypto.ObjectKey
 
 	// Check if bucket encryption is enabled
@@ -961,7 +969,11 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
-			pReader = NewPutObjReader(rawReader, hashReader, &objectEncryptionKey)
+			pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
+			if err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+				return
+			}
 		}
 	}
 

cmd/bucket-lifecycle.go

@@ -65,13 +65,18 @@ func NewLifecycleSys() *LifecycleSys {
 	return &LifecycleSys{}
 }
 
-type expiryState struct {
-	expiryCh chan ObjectInfo
+type expiryTask struct {
+	objInfo       ObjectInfo
+	versionExpiry bool
 }
 
-func (es *expiryState) queueExpiryTask(oi ObjectInfo) {
+type expiryState struct {
+	expiryCh chan expiryTask
+}
+
+func (es *expiryState) queueExpiryTask(oi ObjectInfo, rmVersion bool) {
 	select {
-	case es.expiryCh <- oi:
+	case es.expiryCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion}:
 	default:
 	}
 }
@@ -82,7 +87,7 @@ var (
 
 func newExpiryState() *expiryState {
 	es := &expiryState{
-		expiryCh: make(chan ObjectInfo, 10000),
+		expiryCh: make(chan expiryTask, 10000),
 	}
 	go func() {
 		<-GlobalContext.Done()
@@ -94,8 +99,8 @@ func newExpiryState() *expiryState {
 func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 	globalExpiryState = newExpiryState()
 	go func() {
-		for oi := range globalExpiryState.expiryCh {
-			applyExpiryRule(ctx, objectAPI, oi, false)
+		for t := range globalExpiryState.expiryCh {
+			applyExpiryRule(ctx, objectAPI, t.objInfo, false, t.versionExpiry)
 		}
 	}()
 }
@@ -240,16 +245,11 @@ func transitionSCInUse(ctx context.Context, lfc *lifecycle.Lifecycle, bucket, ar
 }
 
 // set PutObjectOptions for PUT operation to transition data to target cluster
-func putTransitionOpts(objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
+func putTransitionOpts(objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions, err error) {
 	meta := make(map[string]string)
 
-	tag, err := tags.ParseObjectTags(objInfo.UserTags)
-	if err != nil {
-		return
-	}
 	putOpts = miniogo.PutObjectOptions{
 		UserMetadata:    meta,
-		UserTags:        tag.ToMap(),
 		ContentType:     objInfo.ContentType,
 		ContentEncoding: objInfo.ContentEncoding,
 		StorageClass:    objInfo.StorageClass,
@@ -259,22 +259,40 @@ func putTransitionOpts(objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
 			SourceETag:      objInfo.ETag,
 		},
 	}
-	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
+
+	if objInfo.UserTags != "" {
+		tag, _ := tags.ParseObjectTags(objInfo.UserTags)
+		if tag != nil {
+			putOpts.UserTags = tag.ToMap()
+		}
+	}
+
+	lkMap := caseInsensitiveMap(objInfo.UserDefined)
+	if lang, ok := lkMap.Lookup(xhttp.ContentLanguage); ok {
+		putOpts.ContentLanguage = lang
+	}
+	if disp, ok := lkMap.Lookup(xhttp.ContentDisposition); ok {
+		putOpts.ContentDisposition = disp
+	}
+	if cc, ok := lkMap.Lookup(xhttp.CacheControl); ok {
+		putOpts.CacheControl = cc
+	}
+	if mode, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); ok {
 		rmode := miniogo.RetentionMode(mode)
 		putOpts.Mode = rmode
 	}
-	if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
+	if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
 		rdate, err := time.Parse(time.RFC3339, retainDateStr)
 		if err != nil {
-			return
+			return putOpts, err
 		}
 		putOpts.RetainUntilDate = rdate
 	}
-	if lhold, ok := objInfo.UserDefined[xhttp.AmzObjectLockLegalHold]; ok {
+	if lhold, ok := lkMap.Lookup(xhttp.AmzObjectLockLegalHold); ok {
 		putOpts.LegalHold = miniogo.LegalHoldStatus(lhold)
 	}
 
-	return
+	return putOpts, nil
 }
 
 // handle deletes of transitioned objects or object versions when one of the following is true:
@@ -375,7 +393,12 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo Object
 		return nil
 	}
 
-	putOpts := putTransitionOpts(oi)
+	putOpts, err := putTransitionOpts(oi)
+	if err != nil {
+		gr.Close()
+		return err
+
+	}
 	if _, err = tgt.PutObject(ctx, arn.Bucket, oi.Name, gr, oi.Size, putOpts); err != nil {
 		gr.Close()
 		return err
@@ -400,6 +423,7 @@ func transitionObject(ctx context.Context, objectAPI ObjectLayer, objInfo Object
 		Object:     objInfo,
 		Host:       "Internal: [ILM-Transition]",
 	})
+
 	return err
 }
 
@@ -672,7 +696,7 @@ func restoreTransitionedObject(ctx context.Context, bucket, object string, objAP
 	if err != nil {
 		return err
 	}
-	pReader := NewPutObjReader(hashReader, nil, nil)
+	pReader := NewPutObjReader(hashReader)
 	opts := putRestoreOpts(bucket, object, rreq, objInfo)
 	opts.UserDefined[xhttp.AmzRestore] = fmt.Sprintf("ongoing-request=%t, expiry-date=%s", false, restoreExpiry.Format(http.TimeFormat))
 	if _, err := objAPI.PutObject(ctx, bucket, object, pReader, opts); err != nil {

cmd/bucket-listobjects-handlers.go

@@ -26,32 +26,25 @@ import (
 	"github.com/minio/minio/cmd/logger"
 
 	"github.com/minio/minio/pkg/bucket/policy"
-	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 func concurrentDecryptETag(ctx context.Context, objects []ObjectInfo) {
-	inParallel := func(objects []ObjectInfo) {
-		g := errgroup.WithNErrs(len(objects))
-		for index := range objects {
-			index := index
-			g.Go(func() error {
-				objects[index].ETag = objects[index].GetActualETag(nil)
-				objects[index].Size, _ = objects[index].GetActualSize()
-				return nil
-			}, index)
-		}
-		g.Wait()
-	}
-	const maxConcurrent = 500
-	for {
-		if len(objects) < maxConcurrent {
-			inParallel(objects)
-			return
-		}
-		inParallel(objects[:maxConcurrent])
-		objects = objects[maxConcurrent:]
+	g := errgroup.WithNErrs(len(objects)).WithConcurrency(500)
+	_, cancel := g.WithCancelOnError(ctx)
+	defer cancel()
+	for index := range objects {
+		index := index
+		g.Go(func() error {
+			size, err := objects[index].GetActualSize()
+			if err == nil {
+				objects[index].Size = size
+			}
+			objects[index].ETag = objects[index].GetActualETag(nil)
+			return nil
+		}, index)
 	}
+	g.WaitErr()
 }
 
 // Validate all the ListObjects query arguments, returns an APIErrorCode
@@ -301,10 +294,6 @@ func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http
 	return proxyRequest(ctx, w, r, ep)
 }
 
-func proxyRequestByStringHash(ctx context.Context, w http.ResponseWriter, r *http.Request, str string) (success bool) {
-	return proxyRequestByNodeIndex(ctx, w, r, crcHashMod(str, len(globalProxyEndpoints)))
-}
-
 // ListObjectsV1Handler - GET Bucket (List Objects) Version 1.
 // --------------------------
 // This implementation of the GET operation returns some or all (up to 10000)
@@ -343,15 +332,6 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Forward the request using Source IP or bucket
-	forwardStr := handlers.GetSourceIPFromHeaders(r)
-	if forwardStr == "" {
-		forwardStr = bucket
-	}
-	if proxyRequestByStringHash(ctx, w, r, forwardStr) {
-		return
-	}
-
 	listObjects := objectAPI.ListObjects
 
 	// Inititate a list objects operation based on the input params.

cmd/bucket-replication.go

@@ -20,7 +20,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
-	"runtime"
+	"reflect"
 	"strings"
 	"time"
 
@@ -169,14 +169,8 @@ func hasReplicationRules(ctx context.Context, bucket string, objects []ObjectToD
 }
 
 // isStandardHeader returns true if header is a supported header and not a custom header
-func isStandardHeader(headerKey string) bool {
-	key := strings.ToLower(headerKey)
-	for _, header := range standardHeaders {
-		if strings.ToLower(header) == key {
-			return true
-		}
-	}
-	return false
+func isStandardHeader(matchHeaderKey string) bool {
+	return equals(matchHeaderKey, standardHeaders...)
 }
 
 // returns whether object version is a deletemarker and if object qualifies for replication
@@ -185,6 +179,14 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 	if err != nil || rcfg == nil {
 		return false, false, sync
 	}
+	opts := replication.ObjectOpts{
+		Name:         dobj.ObjectName,
+		SSEC:         crypto.SSEC.IsEncrypted(oi.UserDefined),
+		UserTags:     oi.UserTags,
+		DeleteMarker: oi.DeleteMarker,
+		VersionID:    dobj.VersionID,
+	}
+	replicate = rcfg.Replicate(opts)
 	// when incoming delete is removal of a delete marker( a.k.a versioned delete),
 	// GetObjectInfo returns extra information even though it returns errFileNotFound
 	if gerr != nil {
@@ -193,10 +195,12 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 		case replication.Pending, replication.Completed, replication.Failed:
 			validReplStatus = true
 		}
-		if oi.DeleteMarker && validReplStatus {
+		if oi.DeleteMarker && (validReplStatus || replicate) {
 			return oi.DeleteMarker, true, sync
 		}
-		return oi.DeleteMarker, false, sync
+		// can be the case that other cluster is down and duplicate `mc rm --vid`
+		// is issued - this still needs to be replicated back to the other target
+		return oi.DeleteMarker, oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed, sync
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
 	// the target online status should not be used here while deciding
@@ -204,14 +208,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 	if tgt == nil {
 		return oi.DeleteMarker, false, false
 	}
-	opts := replication.ObjectOpts{
-		Name:         dobj.ObjectName,
-		SSEC:         crypto.SSEC.IsEncrypted(oi.UserDefined),
-		UserTags:     oi.UserTags,
-		DeleteMarker: oi.DeleteMarker,
-		VersionID:    dobj.VersionID,
-	}
-	return oi.DeleteMarker, rcfg.Replicate(opts), tgt.replicateSync
+	return oi.DeleteMarker, replicate, tgt.replicateSync
 }
 
 // replicate deletes to the designated replication target if replication configuration
@@ -226,20 +223,45 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 // on target.
 func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectAPI ObjectLayer) {
 	bucket := dobj.Bucket
-	rcfg, err := getReplicationConfig(ctx, bucket)
-	if err != nil || rcfg == nil {
-		logger.LogIf(ctx, err)
-		return
-	}
-	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
-	if tgt == nil {
-		logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, rcfg.RoleArn))
-		return
-	}
 	versionID := dobj.DeleteMarkerVersionID
 	if versionID == "" {
 		versionID = dobj.VersionID
 	}
+
+	rcfg, err := getReplicationConfig(ctx, bucket)
+	if err != nil || rcfg == nil {
+		logger.LogIf(ctx, err)
+		sendEvent(eventArgs{
+			BucketName: bucket,
+			Object: ObjectInfo{
+				Bucket:       bucket,
+				Name:         dobj.ObjectName,
+				VersionID:    versionID,
+				DeleteMarker: dobj.DeleteMarker,
+			},
+			Host:      "Internal: [Replication]",
+			EventName: event.ObjectReplicationNotTracked,
+		})
+		return
+	}
+
+	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rcfg.RoleArn)
+	if tgt == nil {
+		logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, rcfg.RoleArn))
+		sendEvent(eventArgs{
+			BucketName: bucket,
+			Object: ObjectInfo{
+				Bucket:       bucket,
+				Name:         dobj.ObjectName,
+				VersionID:    versionID,
+				DeleteMarker: dobj.DeleteMarker,
+			},
+			Host:      "Internal: [Replication]",
+			EventName: event.ObjectReplicationNotTracked,
+		})
+		return
+	}
+
 	rmErr := tgt.RemoveObject(ctx, rcfg.GetDestination().Bucket, dobj.ObjectName, miniogo.RemoveObjectOptions{
 		VersionID: versionID,
 		Internal: miniogo.AdvancedRemoveOptions{
@@ -258,6 +280,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectA
 		} else {
 			versionPurgeStatus = Failed
 		}
+		logger.LogIf(ctx, fmt.Errorf("Unable to replicate delete marker to %s/%s(%s): %s", rcfg.GetDestination().Bucket, dobj.ObjectName, versionID, rmErr))
 	} else {
 		if dobj.VersionID == "" {
 			replicationStatus = string(replication.Completed)
@@ -272,66 +295,102 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectVersionInfo, objectA
 	}
 
 	// Update metadata on the delete marker or purge permanent delete if replication success.
-	objInfo, err := objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{
+	dobjInfo, err := objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{
 		VersionID:                     versionID,
-		DeleteMarker:                  dobj.DeleteMarker,
 		DeleteMarkerReplicationStatus: replicationStatus,
-		Versioned:                     globalBucketVersioningSys.Enabled(bucket),
 		VersionPurgeStatus:            versionPurgeStatus,
+		Versioned:                     globalBucketVersioningSys.Enabled(bucket),
 		VersionSuspended:              globalBucketVersioningSys.Suspended(bucket),
 	})
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s %s: %w", bucket, dobj.ObjectName, dobj.VersionID, err))
+	if err != nil && !isErrVersionNotFound(err) { // VersionNotFound would be reported by pool that object version is missing on.
+		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %s", bucket, dobj.ObjectName, versionID, err))
+		sendEvent(eventArgs{
+			BucketName: bucket,
+			Object: ObjectInfo{
+				Bucket:       bucket,
+				Name:         dobj.ObjectName,
+				VersionID:    versionID,
+				DeleteMarker: dobj.DeleteMarker,
+			},
+			Host:      "Internal: [Replication]",
+			EventName: eventName,
+		})
+	} else {
+		sendEvent(eventArgs{
+			BucketName: bucket,
+			Object:     dobjInfo,
+			Host:       "Internal: [Replication]",
+			EventName:  eventName,
+		})
 	}
-
-	sendEvent(eventArgs{
-		BucketName: bucket,
-		Object:     objInfo,
-		Host:       "Internal: [Replication]",
-		EventName:  eventName,
-	})
 }
 
 func getCopyObjMetadata(oi ObjectInfo, dest replication.Destination) map[string]string {
 	meta := make(map[string]string, len(oi.UserDefined))
 	for k, v := range oi.UserDefined {
-		if k == xhttp.AmzBucketReplicationStatus {
-			continue
-		}
 		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			continue
 		}
+
+		if equals(k, xhttp.AmzBucketReplicationStatus) {
+			continue
+		}
+
+		// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
+			continue
+		}
+
 		meta[k] = v
 	}
+
 	if oi.ContentEncoding != "" {
 		meta[xhttp.ContentEncoding] = oi.ContentEncoding
 	}
+
 	if oi.ContentType != "" {
 		meta[xhttp.ContentType] = oi.ContentType
 	}
-	tag, err := tags.ParseObjectTags(oi.UserTags)
-	if err != nil {
-		return nil
-	}
-	if tag != nil {
-		meta[xhttp.AmzObjectTagging] = tag.String()
+
+	if oi.UserTags != "" {
+		meta[xhttp.AmzObjectTagging] = oi.UserTags
 		meta[xhttp.AmzTagDirective] = "REPLACE"
 	}
+
 	sc := dest.StorageClass
 	if sc == "" {
 		sc = oi.StorageClass
 	}
-	meta[xhttp.AmzStorageClass] = sc
-	if oi.UserTags != "" {
-		meta[xhttp.AmzObjectTagging] = oi.UserTags
+	if sc != "" {
+		meta[xhttp.AmzStorageClass] = sc
 	}
-	meta[xhttp.MinIOSourceMTime] = oi.ModTime.Format(time.RFC3339Nano)
 	meta[xhttp.MinIOSourceETag] = oi.ETag
+	meta[xhttp.MinIOSourceMTime] = oi.ModTime.Format(time.RFC3339Nano)
 	meta[xhttp.AmzBucketReplicationStatus] = replication.Replica.String()
 	return meta
 }
 
-func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions) {
+type caseInsensitiveMap map[string]string
+
+// Lookup map entry case insensitively.
+func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
+	if len(m) == 0 {
+		return "", false
+	}
+	for _, k := range []string{
+		key,
+		strings.ToLower(key),
+		http.CanonicalHeaderKey(key),
+	} {
+		v, ok := m[k]
+		if ok {
+			return v, ok
+		}
+	}
+	return "", false
+}
+
+func putReplicationOpts(ctx context.Context, dest replication.Destination, objInfo ObjectInfo) (putOpts miniogo.PutObjectOptions, err error) {
 	meta := make(map[string]string)
 	for k, v := range objInfo.UserDefined {
 		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
@@ -342,17 +401,13 @@ func putReplicationOpts(ctx context.Context, dest replication.Destination, objIn
 		}
 		meta[k] = v
 	}
-	tag, err := tags.ParseObjectTags(objInfo.UserTags)
-	if err != nil {
-		return
-	}
+
 	sc := dest.StorageClass
 	if sc == "" {
 		sc = objInfo.StorageClass
 	}
 	putOpts = miniogo.PutObjectOptions{
 		UserMetadata:    meta,
-		UserTags:        tag.ToMap(),
 		ContentType:     objInfo.ContentType,
 		ContentEncoding: objInfo.ContentEncoding,
 		StorageClass:    sc,
@@ -363,33 +418,40 @@ func putReplicationOpts(ctx context.Context, dest replication.Destination, objIn
 			SourceETag:        objInfo.ETag,
 		},
 	}
-	if lang, ok := objInfo.UserDefined[xhttp.ContentLanguage]; ok {
+	if objInfo.UserTags != "" {
+		tag, _ := tags.ParseObjectTags(objInfo.UserTags)
+		if tag != nil {
+			putOpts.UserTags = tag.ToMap()
+		}
+	}
+
+	lkMap := caseInsensitiveMap(objInfo.UserDefined)
+	if lang, ok := lkMap.Lookup(xhttp.ContentLanguage); ok {
 		putOpts.ContentLanguage = lang
 	}
-	if disp, ok := objInfo.UserDefined[xhttp.ContentDisposition]; ok {
+	if disp, ok := lkMap.Lookup(xhttp.ContentDisposition); ok {
 		putOpts.ContentDisposition = disp
 	}
-	if cc, ok := objInfo.UserDefined[xhttp.CacheControl]; ok {
+	if cc, ok := lkMap.Lookup(xhttp.CacheControl); ok {
 		putOpts.CacheControl = cc
 	}
-	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
+	if mode, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); ok {
 		rmode := miniogo.RetentionMode(mode)
 		putOpts.Mode = rmode
 	}
-	if retainDateStr, ok := objInfo.UserDefined[xhttp.AmzObjectLockRetainUntilDate]; ok {
-		rdate, err := time.Parse(time.RFC3339Nano, retainDateStr)
+	if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
+		rdate, err := time.Parse(time.RFC3339, retainDateStr)
 		if err != nil {
-			return
+			return putOpts, err
 		}
 		putOpts.RetainUntilDate = rdate
 	}
-	if lhold, ok := objInfo.UserDefined[xhttp.AmzObjectLockLegalHold]; ok {
+	if lhold, ok := lkMap.Lookup(xhttp.AmzObjectLockLegalHold); ok {
 		putOpts.LegalHold = miniogo.LegalHoldStatus(lhold)
 	}
 	if crypto.S3.IsEncrypted(objInfo.UserDefined) {
 		putOpts.ServerSideEncryption = encrypt.NewSSE()
 	}
-
 	return
 }
 
@@ -401,6 +463,16 @@ const (
 	replicateAll      replicationAction = "all"
 )
 
+// matches k1 with all keys, returns 'true' if one of them matches
+func equals(k1 string, keys ...string) bool {
+	for _, k2 := range keys {
+		if strings.ToLower(k1) == strings.ToLower(k2) {
+			return true
+		}
+	}
+	return false
+}
+
 // returns replicationAction by comparing metadata between source and target
 func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo) replicationAction {
 	// needs full replication
@@ -408,83 +480,153 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo) replicationActio
 		oi1.VersionID != oi2.VersionID ||
 		oi1.Size != oi2.Size ||
 		oi1.DeleteMarker != oi2.IsDeleteMarker ||
-		!oi1.ModTime.Equal(oi2.LastModified) {
+		oi1.ModTime.Unix() != oi2.LastModified.Unix() {
 		return replicateAll
 	}
+
 	if oi1.ContentType != oi2.ContentType {
 		return replicateMetadata
 	}
+
 	if oi1.ContentEncoding != "" {
 		enc, ok := oi2.Metadata[xhttp.ContentEncoding]
-		if !ok || strings.Join(enc, "") != oi1.ContentEncoding {
+		if !ok {
+			enc, ok = oi2.Metadata[strings.ToLower(xhttp.ContentEncoding)]
+			if !ok {
+				return replicateMetadata
+			}
+		}
+		if strings.Join(enc, ",") != oi1.ContentEncoding {
 			return replicateMetadata
 		}
 	}
+
+	t, _ := tags.ParseObjectTags(oi1.UserTags)
+	if !reflect.DeepEqual(oi2.UserTags, t.ToMap()) {
+		return replicateMetadata
+	}
+
+	// Compare only necessary headers
+	compareKeys := []string{
+		"Expires",
+		"Cache-Control",
+		"Content-Language",
+		"Content-Disposition",
+		"X-Amz-Object-Lock-Mode",
+		"X-Amz-Object-Lock-Retain-Until-Date",
+		"X-Amz-Object-Lock-Legal-Hold",
+		"X-Amz-Website-Redirect-Location",
+		"X-Amz-Meta-",
+	}
+
 	// compare metadata on both maps to see if meta is identical
-	for k1, v1 := range oi1.UserDefined {
-		if v2, ok := oi2.UserMetadata[k1]; ok && v1 == v2 {
-			continue
+	compareMeta1 := make(map[string]string)
+	for k, v := range oi1.UserDefined {
+		var found bool
+		for _, prefix := range compareKeys {
+			if !strings.HasPrefix(strings.ToLower(k), strings.ToLower(prefix)) {
+				continue
+			}
+			found = true
+			break
 		}
-		if v2, ok := oi2.Metadata[k1]; ok && v1 == strings.Join(v2, "") {
-			continue
+		if found {
+			compareMeta1[strings.ToLower(k)] = v
 		}
+	}
+
+	compareMeta2 := make(map[string]string)
+	for k, v := range oi2.Metadata {
+		var found bool
+		for _, prefix := range compareKeys {
+			if !strings.HasPrefix(strings.ToLower(k), strings.ToLower(prefix)) {
+				continue
+			}
+			found = true
+			break
+		}
+		if found {
+			compareMeta2[strings.ToLower(k)] = strings.Join(v, ",")
+		}
+	}
+
+	if !reflect.DeepEqual(compareMeta1, compareMeta2) {
 		return replicateMetadata
 	}
-	for k1, v1 := range oi2.UserMetadata {
-		if v2, ok := oi1.UserDefined[k1]; !ok || v1 != v2 {
-			return replicateMetadata
-		}
-	}
-	for k1, v1slc := range oi2.Metadata {
-		v1 := strings.Join(v1slc, "")
-		if k1 == xhttp.ContentEncoding { //already compared
-			continue
-		}
-		if v2, ok := oi1.UserDefined[k1]; !ok || v1 != v2 {
-			return replicateMetadata
-		}
-	}
-	t, _ := tags.MapToObjectTags(oi2.UserTags)
-	if t.String() != oi1.UserTags {
-		return replicateMetadata
-	}
+
 	return replicateNone
 }
 
 // replicateObject replicates the specified version of the object to destination bucket
 // The source object is then updated to reflect the replication status.
 func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLayer) {
+	z, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		return
+	}
+
 	bucket := objInfo.Bucket
 	object := objInfo.Name
 
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
 		logger.LogIf(ctx, err)
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
 		return
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, cfg.RoleArn)
 	if tgt == nil {
 		logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, cfg.RoleArn))
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
 		return
 	}
 	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
 		VersionID: objInfo.VersionID,
 	})
 	if err != nil {
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
 		logger.LogIf(ctx, err)
 		return
 	}
+	defer gr.Close() // hold read lock for entire transaction
+
 	objInfo = gr.ObjInfo
 	size, err := objInfo.GetActualSize()
 	if err != nil {
 		logger.LogIf(ctx, err)
-		gr.Close()
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
 		return
 	}
 
 	dest := cfg.GetDestination()
 	if dest.Bucket == "" {
-		gr.Close()
+		logger.LogIf(ctx, fmt.Errorf("Unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
+		sendEvent(eventArgs{
+			EventName:  event.ObjectReplicationNotTracked,
+			BucketName: bucket,
+			Object:     objInfo,
+			Host:       "Internal: [Replication]",
+		})
 		return
 	}
 
@@ -497,7 +639,6 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 	if err == nil {
 		rtype = getReplicationAction(objInfo, oi)
 		if rtype == replicateNone {
-			gr.Close()
 			// object with same VersionID already exists, replication kicked off by
 			// PutObject might have completed.
 			return
@@ -505,23 +646,43 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 	}
 	replicationStatus := replication.Completed
 	if rtype != replicateAll {
-		gr.Close()
 		// replicate metadata for object tagging/copy with metadata replacement
+		srcOpts := miniogo.CopySrcOptions{
+			Bucket:    dest.Bucket,
+			Object:    object,
+			VersionID: objInfo.VersionID}
 		dstOpts := miniogo.PutObjectOptions{Internal: miniogo.AdvancedPutOptions{SourceVersionID: objInfo.VersionID}}
 		c := &miniogo.Core{Client: tgt.Client}
-		_, err = c.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), dstOpts)
-		if err != nil {
+
+		if _, err = c.CopyObject(ctx, dest.Bucket, object, dest.Bucket, object, getCopyObjMetadata(objInfo, dest), srcOpts, dstOpts); err != nil {
 			replicationStatus = replication.Failed
+			logger.LogIf(ctx, fmt.Errorf("Unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 		}
 	} else {
 		target, err := globalBucketMetadataSys.GetBucketTarget(bucket, cfg.RoleArn)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%s", bucket, cfg.RoleArn, err))
-			gr.Close()
+			sendEvent(eventArgs{
+				EventName:  event.ObjectReplicationNotTracked,
+				BucketName: bucket,
+				Object:     objInfo,
+				Host:       "Internal: [Replication]",
+			})
+			return
+		}
+
+		putOpts, err := putReplicationOpts(ctx, dest, objInfo)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s cfg:%s err:%w", bucket, cfg.RoleArn, err))
+			sendEvent(eventArgs{
+				EventName:  event.ObjectReplicationNotTracked,
+				BucketName: bucket,
+				Object:     objInfo,
+				Host:       "Internal: [Replication]",
+			})
 			return
 		}
 
-		putOpts := putReplicationOpts(ctx, dest, objInfo)
 		// Setup bandwidth throttling
 		peers, _ := globalEndpoints.peers()
 		totalNodesCount := len(peers)
@@ -536,11 +697,11 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 
 		// r takes over closing gr.
 		r := bandwidth.NewMonitoredReader(ctx, globalBucketMonitor, objInfo.Bucket, objInfo.Name, gr, headerSize, b, target.BandwidthLimit)
-		_, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, putOpts)
-		if err != nil {
+		if _, err = tgt.PutObject(ctx, dest.Bucket, object, r, size, putOpts); err != nil {
 			replicationStatus = replication.Failed
+			logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %w", bucket, objInfo.Name, objInfo.VersionID, err))
 		}
-		r.Close()
+		defer r.Close()
 	}
 
 	objInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = replicationStatus.String()
@@ -549,7 +710,6 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 	}
 
 	// FIXME: add support for missing replication events
-	// - event.ObjectReplicationNotTracked
 	// - event.ObjectReplicationMissedThreshold
 	// - event.ObjectReplicationReplicatedAfterThreshold
 	var eventName = event.ObjectReplicationComplete
@@ -557,16 +717,17 @@ func replicateObject(ctx context.Context, objInfo ObjectInfo, objectAPI ObjectLa
 		eventName = event.ObjectReplicationFailed
 	}
 
-	objInfo.metadataOnly = true // Perform only metadata updates.
-	objInfo, err = objectAPI.CopyObject(ctx, bucket, object, bucket, object, objInfo, ObjectOptions{
-		VersionID: objInfo.VersionID,
-	}, ObjectOptions{
-		VersionID: objInfo.VersionID,
-	})
+	// This lower level implementation is necessary to avoid write locks from CopyObject.
+	poolIdx, err := z.getPoolIdx(ctx, bucket, object, objInfo.Size)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s: %s", objInfo.VersionID, err))
+		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w", bucket, objInfo.Name, objInfo.VersionID, err))
+	} else {
+		if err = z.serverPools[poolIdx].getHashedSet(object).updateObjectMeta(ctx, bucket, object, objInfo.UserDefined, ObjectOptions{
+			VersionID: objInfo.VersionID,
+		}); err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w", bucket, objInfo.Name, objInfo.VersionID, err))
+		}
 	}
-
 	sendEvent(eventArgs{
 		EventName:  eventName,
 		BucketName: bucket,
@@ -631,19 +792,9 @@ func (r *replicationState) queueReplicaDeleteTask(doi DeletedObjectVersionInfo)
 
 var (
 	globalReplicationState *replicationState
-	// TODO: currently keeping it conservative
-	// but eventually can be tuned in future,
-	// take only half the CPUs for replication
-	// conservatively.
-	globalReplicationConcurrent = runtime.GOMAXPROCS(0) / 2
 )
 
 func newReplicationState() *replicationState {
-
-	// fix minimum concurrent replication to 1 for single CPU setup
-	if globalReplicationConcurrent == 0 {
-		globalReplicationConcurrent = 1
-	}
 	rs := &replicationState{
 		replicaCh:       make(chan ObjectInfo, 10000),
 		replicaDeleteCh: make(chan DeletedObjectVersionInfo, 10000),
@@ -684,8 +835,8 @@ func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
 		return
 	}
 
-	// Start with globalReplicationConcurrent.
-	for i := 0; i < globalReplicationConcurrent; i++ {
+	// Start replication workers per count set in api config or MINIO_API_REPLICATION_WORKERS.
+	for i := 0; i < globalAPIConfig.getReplicationWorkers(); i++ {
 		globalReplicationState.addWorker(ctx, objectAPI)
 	}
 }
@@ -714,8 +865,11 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 			return nil, false
 		}
 	}
+	// Make sure to match ETag when proxying.
+	if err = gopts.SetMatchETag(oi.ETag); err != nil {
+		return nil, false
+	}
 	c := miniogo.Core{Client: tgt.Client}
-
 	obj, _, _, err := c.GetObject(ctx, bucket, object, gopts)
 	if err != nil {
 		return nil, false
@@ -726,6 +880,7 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 	if err != nil {
 		return nil, false
 	}
+	reader.ObjInfo = oi.Clone()
 	return reader, true
 }
 
@@ -781,6 +936,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 	if err != nil {
 		return nil, oi, false, err
 	}
+
 	tags, _ := tags.MapToObjectTags(objInfo.UserTags)
 	oi = ObjectInfo{
 		Bucket:            bucket,
@@ -795,12 +951,18 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, opts Objec
 		Expires:           objInfo.Expires,
 		StorageClass:      objInfo.StorageClass,
 		ReplicationStatus: replication.StatusType(objInfo.ReplicationStatus),
-		UserDefined:       cloneMSS(objInfo.UserMetadata),
 		UserTags:          tags.String(),
 	}
-	if ce, ok := oi.UserDefined[xhttp.ContentEncoding]; ok {
+	oi.UserDefined = make(map[string]string, len(objInfo.Metadata))
+	for k, v := range objInfo.Metadata {
+		oi.UserDefined[k] = v[0]
+	}
+	ce, ok := oi.UserDefined[xhttp.ContentEncoding]
+	if !ok {
+		ce, ok = oi.UserDefined[strings.ToLower(xhttp.ContentEncoding)]
+	}
+	if ok {
 		oi.ContentEncoding = ce
-		delete(oi.UserDefined, xhttp.ContentEncoding)
 	}
 	return tgt, oi, true, nil
 }

cmd/bucket-targets.go

@@ -362,6 +362,7 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 	api, err := minio.New(tcfg.Endpoint, &miniogo.Options{
 		Creds:     creds,
 		Secure:    tcfg.Secure,
+		Region:    tcfg.Region,
 		Transport: getRemoteTargetInstanceTransport,
 	})
 	if err != nil {

cmd/common-main.go

@@ -50,10 +50,14 @@ import (
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
 
 func init() {
+	rand.Seed(time.Now().UTC().UnixNano())
+
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	rand.Seed(time.Now().UTC().UnixNano())
+	// Inject into config package.
+	config.Logger.Info = logger.Info
+	config.Logger.LogIf = logger.LogIf
 
 	globalDNSCache = xhttp.NewDNSCache(10*time.Second, 10*time.Second, logger.LogOnceIf)
 

cmd/config-common.go

@@ -20,6 +20,8 @@ import (
 	"bytes"
 	"context"
 	"errors"
+	"io/ioutil"
+	"net/http"
 
 	"github.com/minio/minio/pkg/hash"
 )
@@ -27,9 +29,9 @@ import (
 var errConfigNotFound = errors.New("config file not found")
 
 func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
-	var buffer bytes.Buffer
 	// Read entire content by setting size to -1
-	if err := objAPI.GetObject(ctx, minioMetaBucket, configFile, 0, -1, &buffer, "", ObjectOptions{}); err != nil {
+	r, err := objAPI.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
+	if err != nil {
 		// Treat object not found as config not found.
 		if isErrObjectNotFound(err) {
 			return nil, errConfigNotFound
@@ -37,13 +39,16 @@ func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]b
 
 		return nil, err
 	}
+	defer r.Close()
 
-	// Return config not found on empty content.
-	if buffer.Len() == 0 {
+	buf, err := ioutil.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	if len(buf) == 0 {
 		return nil, errConfigNotFound
 	}
-
-	return buffer.Bytes(), nil
+	return buf, nil
 }
 
 type objectDeleter interface {
@@ -64,7 +69,7 @@ func saveConfig(ctx context.Context, objAPI ObjectLayer, configFile string, data
 		return err
 	}
 
-	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader, nil, nil), ObjectOptions{})
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), ObjectOptions{})
 	return err
 }
 

cmd/config-current.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"context"
+	"crypto/tls"
 	"fmt"
 	"strings"
 	"sync"
@@ -26,7 +27,6 @@ import (
 	"github.com/minio/minio/cmd/config/api"
 	"github.com/minio/minio/cmd/config/cache"
 	"github.com/minio/minio/cmd/config/compress"
-	"github.com/minio/minio/cmd/config/crawler"
 	"github.com/minio/minio/cmd/config/dns"
 	"github.com/minio/minio/cmd/config/etcd"
 	"github.com/minio/minio/cmd/config/heal"
@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/cmd/config/identity/openid"
 	"github.com/minio/minio/cmd/config/notify"
 	"github.com/minio/minio/cmd/config/policy/opa"
+	"github.com/minio/minio/cmd/config/scanner"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/crypto"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -59,7 +60,7 @@ func initHelp() {
 		config.LoggerWebhookSubSys:  logger.DefaultKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditKVS,
 		config.HealSubSys:           heal.DefaultKVS,
-		config.CrawlerSubSys:        crawler.DefaultKVS,
+		config.ScannerSubSys:        scanner.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -116,8 +117,8 @@ func initHelp() {
 			Description: "manage object healing frequency and bitrot verification checks",
 		},
 		config.HelpKV{
-			Key:         config.CrawlerSubSys,
-			Description: "manage crawling for usage calculation, lifecycle, healing and more",
+			Key:         config.ScannerSubSys,
+			Description: "manage scanner for usage calculation, lifecycle, healing and more",
 		},
 		config.HelpKV{
 			Key:             config.LoggerWebhookSubSys,
@@ -199,7 +200,7 @@ func initHelp() {
 		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
 		config.HealSubSys:           heal.Help,
-		config.CrawlerSubSys:        crawler.Help,
+		config.ScannerSubSys:        scanner.Help,
 		config.IdentityOpenIDSubSys: openid.Help,
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.PolicyOPASubSys:      opa.Help,
@@ -273,11 +274,11 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 		}
 	}
 
-	if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
+	if _, err = heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
 		return err
 	}
 
-	if _, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default]); err != nil {
+	if _, err = scanner.LookupConfig(s[config.ScannerSubSys][config.Default]); err != nil {
 		return err
 	}
 
@@ -295,7 +296,10 @@ func validateConfig(s config.Config, setDriveCounts []int) error {
 		}
 	}
 	{
-		kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
+		kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), newCustomHTTPTransportWithHTTP2(
+			&tls.Config{
+				RootCAs: globalRootCAs,
+			}, defaultDialTimeout)())
 		if err != nil {
 			return err
 		}
@@ -471,7 +475,10 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 		}
 	}
 
-	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), NewGatewayHTTPTransport())
+	kmsCfg, err := crypto.LookupConfig(s, globalCertsCADir.Get(), newCustomHTTPTransportWithHTTP2(
+		&tls.Config{
+			RootCAs: globalRootCAs,
+		}, defaultDialTimeout)())
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to setup KMS config: %w", err))
 	}
@@ -599,10 +606,10 @@ func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config
 		return fmt.Errorf("Unable to apply heal config: %w", err)
 	}
 
-	// Crawler
-	crawlerCfg, err := crawler.LookupConfig(s[config.CrawlerSubSys][config.Default])
+	// Scanner
+	scannerCfg, err := scanner.LookupConfig(s[config.ScannerSubSys][config.Default])
 	if err != nil {
-		return fmt.Errorf("Unable to apply crawler config: %w", err)
+		return fmt.Errorf("Unable to apply scanner config: %w", err)
 	}
 
 	// Apply configurations.
@@ -617,7 +624,7 @@ func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config
 	globalHealConfig = healCfg
 	globalHealConfigMu.Unlock()
 
-	logger.LogIf(ctx, crawlerSleeper.Update(crawlerCfg.Delay, crawlerCfg.MaxWait))
+	logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
 
 	// Update all dynamic config values in memory.
 	globalServerConfigMu.Lock()

cmd/config/api/api.go

@@ -29,14 +29,14 @@ import (
 
 // API sub-system constants
 const (
-	apiRequestsMax             = "requests_max"
-	apiRequestsDeadline        = "requests_deadline"
-	apiClusterDeadline         = "cluster_deadline"
-	apiCorsAllowOrigin         = "cors_allow_origin"
-	apiRemoteTransportDeadline = "remote_transport_deadline"
-	apiListQuorum              = "list_quorum"
-	apiExtendListCacheLife     = "extend_list_cache_life"
-
+	apiRequestsMax                = "requests_max"
+	apiRequestsDeadline           = "requests_deadline"
+	apiClusterDeadline            = "cluster_deadline"
+	apiCorsAllowOrigin            = "cors_allow_origin"
+	apiRemoteTransportDeadline    = "remote_transport_deadline"
+	apiListQuorum                 = "list_quorum"
+	apiExtendListCacheLife        = "extend_list_cache_life"
+	apiReplicationWorkers         = "replication_workers"
 	EnvAPIRequestsMax             = "MINIO_API_REQUESTS_MAX"
 	EnvAPIRequestsDeadline        = "MINIO_API_REQUESTS_DEADLINE"
 	EnvAPIClusterDeadline         = "MINIO_API_CLUSTER_DEADLINE"
@@ -45,6 +45,7 @@ const (
 	EnvAPIListQuorum              = "MINIO_API_LIST_QUORUM"
 	EnvAPIExtendListCacheLife     = "MINIO_API_EXTEND_LIST_CACHE_LIFE"
 	EnvAPISecureCiphers           = "MINIO_API_SECURE_CIPHERS"
+	EnvAPIReplicationWorkers      = "MINIO_API_REPLICATION_WORKERS"
 )
 
 // Deprecated key and ENVs
@@ -78,12 +79,16 @@ var (
 		},
 		config.KV{
 			Key:   apiListQuorum,
-			Value: "optimal",
+			Value: "strict",
 		},
 		config.KV{
 			Key:   apiExtendListCacheLife,
 			Value: "0s",
 		},
+		config.KV{
+			Key:   apiReplicationWorkers,
+			Value: "100",
+		},
 	}
 )
 
@@ -96,6 +101,7 @@ type Config struct {
 	RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"`
 	ListQuorum              string        `json:"list_strict_quorum"`
 	ExtendListLife          time.Duration `json:"extend_list_cache_life"`
+	ReplicationWorkers      int           `json:"replication_workers"`
 }
 
 // UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON.
@@ -173,6 +179,15 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		return cfg, err
 	}
 
+	replicationWorkers, err := strconv.Atoi(env.Get(EnvAPIReplicationWorkers, kvs.Get(apiReplicationWorkers)))
+	if err != nil {
+		return cfg, err
+	}
+
+	if replicationWorkers <= 0 {
+		return cfg, config.ErrInvalidReplicationWorkersValue(nil).Msg("Minimum number of replication workers should be 1")
+	}
+
 	return Config{
 		RequestsMax:             requestsMax,
 		RequestsDeadline:        requestsDeadline,
@@ -181,5 +196,6 @@ func LookupConfig(kvs config.KVS) (cfg Config, err error) {
 		RemoteTransportDeadline: remoteTransportDeadline,
 		ListQuorum:              listQuorum,
 		ExtendListLife:          listLife,
+		ReplicationWorkers:      replicationWorkers,
 	}, nil
 }

cmd/config/api/help.go

@@ -45,5 +45,11 @@ var (
 			Optional:    true,
 			Type:        "duration",
 		},
+		config.HelpKV{
+			Key:         apiReplicationWorkers,
+			Description: `set the number of replication workers, defaults to 100`,
+			Optional:    true,
+			Type:        "number",
+		},
 	}
 )

cmd/config/compress/compress.go

@@ -45,7 +45,7 @@ const (
 
 	// Include-list for compression.
 	DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin"
-	DefaultMimeTypes  = "text/*,application/json,application/xml"
+	DefaultMimeTypes  = "text/*,application/json,application/xml,binary/octet-stream"
 )
 
 // DefaultKVS - default KV config for compression settings

cmd/config/config.go

@@ -77,6 +77,7 @@ const (
 	LoggerWebhookSubSys  = "logger_webhook"
 	AuditWebhookSubSys   = "audit_webhook"
 	HealSubSys           = "heal"
+	ScannerSubSys        = "scanner"
 	CrawlerSubSys        = "crawler"
 
 	// Add new constants here if you add new fields to config.
@@ -114,7 +115,7 @@ var SubSystems = set.CreateStringSet(
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
-	CrawlerSubSys,
+	ScannerSubSys,
 	HealSubSys,
 	NotifyAMQPSubSys,
 	NotifyESSubSys,
@@ -132,7 +133,7 @@ var SubSystems = set.CreateStringSet(
 var SubSystemsDynamic = set.CreateStringSet(
 	APISubSys,
 	CompressionSubSys,
-	CrawlerSubSys,
+	ScannerSubSys,
 	HealSubSys,
 )
 
@@ -151,7 +152,7 @@ var SubSystemsSingleTargets = set.CreateStringSet([]string{
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
 	HealSubSys,
-	CrawlerSubSys,
+	ScannerSubSys,
 }...)
 
 // Constant separators
@@ -462,6 +463,13 @@ func LookupWorm() (bool, error) {
 	return ParseBool(env.Get(EnvWorm, EnableOff))
 }
 
+// Carries all the renamed sub-systems from their
+// previously known names
+var renamedSubsys = map[string]string{
+	CrawlerSubSys: ScannerSubSys,
+	// Add future sub-system renames
+}
+
 // Merge - merges a new config with all the
 // missing values for default configs,
 // returns a config.
@@ -477,9 +485,21 @@ func (c Config) Merge() Config {
 				}
 			}
 			if _, ok := cp[subSys]; !ok {
-				// A config subsystem was removed or server was downgraded.
-				Logger.Info("config: ignoring unknown subsystem config %q\n", subSys)
-				continue
+				rnSubSys, ok := renamedSubsys[subSys]
+				if !ok {
+					// A config subsystem was removed or server was downgraded.
+					Logger.Info("config: ignoring unknown subsystem config %q\n", subSys)
+					continue
+				}
+				// Copy over settings from previous sub-system
+				// to newly renamed sub-system
+				for _, kv := range cp[rnSubSys][Default] {
+					_, ok := c[subSys][tgt].Lookup(kv.Key)
+					if !ok {
+						ckvs.Set(kv.Key, kv.Value)
+					}
+				}
+				subSys = rnSubSys
 			}
 			cp[subSys][tgt] = ckvs
 		}

cmd/config/errors.go

@@ -281,4 +281,10 @@ Example 1:
 		"",
 		"Refer to https://docs.min.io/docs/minio-kms-quickstart-guide.html for setting up SSE",
 	)
+
+	ErrInvalidReplicationWorkersValue = newErrFn(
+		"Invalid value for replication workers",
+		"",
+		"MINIO_API_REPLICATION_WORKERS: should be > 0",
+	)
 )

cmd/config/identity/ldap/config.go

@@ -25,9 +25,9 @@ import (
 	"strings"
 	"time"
 
+	ldap "github.com/go-ldap/ldap/v3"
 	"github.com/minio/minio/cmd/config"
 	"github.com/minio/minio/pkg/env"
-	ldap "gopkg.in/ldap.v3"
 )
 
 const (
@@ -58,7 +58,6 @@ type Config struct {
 	GroupSearchBaseDistName  string   `json:"groupSearchBaseDN"`
 	GroupSearchBaseDistNames []string `json:"-"`
 	GroupSearchFilter        string   `json:"groupSearchFilter"`
-	GroupNameAttribute       string   `json:"groupNameAttribute"`
 
 	// Lookup bind LDAP service account
 	LookupBindDN       string `json:"lookupBindDN"`
@@ -82,7 +81,6 @@ const (
 	UserDNSearchFilter = "user_dn_search_filter"
 	UsernameFormat     = "username_format"
 	GroupSearchFilter  = "group_search_filter"
-	GroupNameAttribute = "group_name_attribute"
 	GroupSearchBaseDN  = "group_search_base_dn"
 	TLSSkipVerify      = "tls_skip_verify"
 	ServerInsecure     = "server_insecure"
@@ -97,7 +95,6 @@ const (
 	EnvUserDNSearchBaseDN = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN"
 	EnvUserDNSearchFilter = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER"
 	EnvGroupSearchFilter  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER"
-	EnvGroupNameAttribute = "MINIO_IDENTITY_LDAP_GROUP_NAME_ATTRIBUTE"
 	EnvGroupSearchBaseDN  = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN"
 	EnvLookupBindDN       = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN"
 	EnvLookupBindPassword = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD"
@@ -106,6 +103,7 @@ const (
 var removedKeys = []string{
 	"username_search_filter",
 	"username_search_base_dn",
+	"group_name_attribute",
 }
 
 // DefaultKVS - default config for LDAP config
@@ -131,10 +129,6 @@ var (
 			Key:   GroupSearchFilter,
 			Value: "",
 		},
-		config.KV{
-			Key:   GroupNameAttribute,
-			Value: "",
-		},
 		config.KV{
 			Key:   GroupSearchBaseDN,
 			Value: "",
@@ -180,7 +174,7 @@ func getGroups(conn *ldap.Conn, sreq *ldap.SearchRequest) ([]string, error) {
 	for _, entry := range sres.Entries {
 		// We only queried one attribute,
 		// so we only look up the first one.
-		groups = append(groups, entry.Attributes[0].Values...)
+		groups = append(groups, entry.DN)
 	}
 	return groups, nil
 }
@@ -312,7 +306,7 @@ func (l *Config) Bind(username, password string) (string, []string, error) {
 				groupSearchBase,
 				ldap.ScopeWholeSubtree, ldap.NeverDerefAliases, 0, 0, false,
 				filter,
-				[]string{l.GroupNameAttribute},
+				nil,
 				nil,
 			)
 
@@ -434,7 +428,7 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 
 		// User DN search configuration
 		userDNSearchBaseDN := env.Get(EnvUserDNSearchBaseDN, kvs.Get(UserDNSearchBaseDN))
-		userDNSearchFilter := env.Get(EnvUserDNSearchFilter, kvs.Get(EnvUserDNSearchFilter))
+		userDNSearchFilter := env.Get(EnvUserDNSearchFilter, kvs.Get(UserDNSearchFilter))
 		if userDNSearchFilter == "" || userDNSearchBaseDN == "" {
 			return l, errors.New("In lookup bind mode, userDN search base DN and userDN search filter are both required")
 		}
@@ -463,21 +457,15 @@ func Lookup(kvs config.KVS, rootCAs *x509.CertPool) (l Config, err error) {
 
 	// Group search params configuration
 	grpSearchFilter := env.Get(EnvGroupSearchFilter, kvs.Get(GroupSearchFilter))
-	grpSearchNameAttr := env.Get(EnvGroupNameAttribute, kvs.Get(GroupNameAttribute))
 	grpSearchBaseDN := env.Get(EnvGroupSearchBaseDN, kvs.Get(GroupSearchBaseDN))
 
 	// Either all group params must be set or none must be set.
-	var allSet bool
-	if grpSearchFilter != "" {
-		if grpSearchNameAttr == "" || grpSearchBaseDN == "" {
-			return l, errors.New("All group related parameters must be set")
-		}
-		allSet = true
+	if (grpSearchFilter != "" && grpSearchBaseDN == "") || (grpSearchFilter == "" && grpSearchBaseDN != "") {
+		return l, errors.New("All group related parameters must be set")
 	}
 
-	if allSet {
+	if grpSearchFilter != "" {
 		l.GroupSearchFilter = grpSearchFilter
-		l.GroupNameAttribute = grpSearchNameAttr
 		l.GroupSearchBaseDistName = grpSearchBaseDN
 		l.GroupSearchBaseDistNames = strings.Split(l.GroupSearchBaseDistName, dnDelimiter)
 	}

cmd/config/identity/ldap/help.go

@@ -68,12 +68,6 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
-		config.HelpKV{
-			Key:         GroupNameAttribute,
-			Description: `search attribute for group name e.g. "cn"`,
-			Optional:    true,
-			Type:        "string",
-		},
 		config.HelpKV{
 			Key:         GroupSearchBaseDN,
 			Description: `";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"`,

cmd/config/identity/ldap/legacy.go

@@ -41,10 +41,6 @@ func SetIdentityLDAP(s config.Config, ldapArgs Config) {
 			Key:   GroupSearchFilter,
 			Value: ldapArgs.GroupSearchFilter,
 		},
-		config.KV{
-			Key:   GroupNameAttribute,
-			Value: ldapArgs.GroupNameAttribute,
-		},
 		config.KV{
 			Key:   GroupSearchBaseDN,
 			Value: ldapArgs.GroupSearchBaseDistName,

cmd/config/scanner/scanner.go

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-package crawler
+package scanner
 
 import (
 	"strconv"
@@ -29,8 +29,10 @@ const (
 	Delay   = "delay"
 	MaxWait = "max_wait"
 
-	EnvDelay   = "MINIO_CRAWLER_DELAY"
-	EnvMaxWait = "MINIO_CRAWLER_MAX_WAIT"
+	EnvDelay         = "MINIO_SCANNER_DELAY"
+	EnvDelayLegacy   = "MINIO_CRAWLER_DELAY"
+	EnvMaxWait       = "MINIO_SCANNER_MAX_WAIT"
+	EnvMaxWaitLegacy = "MINIO_CRAWLER_MAX_WAIT"
 )
 
 // Config represents the heal settings.
@@ -58,7 +60,7 @@ var (
 	Help = config.HelpKVS{
 		config.HelpKV{
 			Key:         Delay,
-			Description: `crawler delay multiplier, defaults to '10.0'`,
+			Description: `scanner delay multiplier, defaults to '10.0'`,
 			Optional:    true,
 			Type:        "float",
 		},
@@ -73,14 +75,22 @@ var (
 
 // LookupConfig - lookup config and override with valid environment settings if any.
 func LookupConfig(kvs config.KVS) (cfg Config, err error) {
-	if err = config.CheckValidKeys(config.CrawlerSubSys, kvs, DefaultKVS); err != nil {
+	if err = config.CheckValidKeys(config.ScannerSubSys, kvs, DefaultKVS); err != nil {
 		return cfg, err
 	}
-	cfg.Delay, err = strconv.ParseFloat(env.Get(EnvDelay, kvs.Get(Delay)), 64)
+	delay := env.Get(EnvDelayLegacy, "")
+	if delay == "" {
+		delay = env.Get(EnvDelay, kvs.Get(Delay))
+	}
+	cfg.Delay, err = strconv.ParseFloat(delay, 64)
 	if err != nil {
 		return cfg, err
 	}
-	cfg.MaxWait, err = time.ParseDuration(env.Get(EnvMaxWait, kvs.Get(MaxWait)))
+	maxWait := env.Get(EnvMaxWaitLegacy, "")
+	if maxWait == "" {
+		maxWait = env.Get(EnvMaxWait, kvs.Get(MaxWait))
+	}
+	cfg.MaxWait, err = time.ParseDuration(maxWait)
 	if err != nil {
 		return cfg, err
 	}

cmd/crypto/metadata.go

@@ -44,6 +44,15 @@ const (
 	// MetaDataEncryptionKey is the sealed data encryption key (DEK) received from
 	// the KMS.
 	MetaDataEncryptionKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key"
+
+	// MetaContext is the KMS context provided by a client when encrypting an
+	// object with SSE-KMS. A client may not send a context in which case the
+	// MetaContext will not be present.
+	// MetaContext only contains the bucket/object name if the client explicitly
+	// added it. However, when decrypting an object the bucket/object name must
+	// be part of the object. Therefore, the bucket/object name must be added
+	// to the context, if not present, whenever a decryption is performed.
+	MetaContext = "X-Minio-Internal-Server-Side-Encryption-Context"
 )
 
 // IsMultiPart returns true if the object metadata indicates
@@ -109,26 +118,35 @@ func IsSourceEncrypted(metadata map[string]string) bool {
 //
 // IsEncrypted only checks whether the metadata contains at least
 // one entry indicating SSE-C or SSE-S3.
-func IsEncrypted(metadata map[string]string) bool {
-	if _, ok := metadata[MetaIV]; ok {
-		return true
-	}
-	if _, ok := metadata[MetaAlgorithm]; ok {
-		return true
-	}
-	if IsMultiPart(metadata) {
-		return true
+func IsEncrypted(metadata map[string]string) (Type, bool) {
+	if S3KMS.IsEncrypted(metadata) {
+		return S3KMS, true
 	}
 	if S3.IsEncrypted(metadata) {
-		return true
+		return S3, true
 	}
 	if SSEC.IsEncrypted(metadata) {
-		return true
+		return SSEC, true
 	}
-	if S3KMS.IsEncrypted(metadata) {
-		return true
+	if IsMultiPart(metadata) {
+		return nil, true
 	}
-	return false
+	if _, ok := metadata[MetaIV]; ok {
+		return nil, true
+	}
+	if _, ok := metadata[MetaAlgorithm]; ok {
+		return nil, true
+	}
+	if _, ok := metadata[MetaKeyID]; ok {
+		return nil, true
+	}
+	if _, ok := metadata[MetaDataEncryptionKey]; ok {
+		return nil, true
+	}
+	if _, ok := metadata[MetaContext]; ok {
+		return nil, true
+	}
+	return nil, false
 }
 
 // CreateMultipartMetadata adds the multipart flag entry to metadata

cmd/crypto/metadata_test.go

@@ -59,7 +59,7 @@ var isEncryptedTests = []struct {
 
 func TestIsEncrypted(t *testing.T) {
 	for i, test := range isEncryptedTests {
-		if isEncrypted := IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
+		if _, isEncrypted := IsEncrypted(test.Metadata); isEncrypted != test.Encrypted {
 			t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted)
 		}
 	}
@@ -74,8 +74,8 @@ var s3IsEncryptedTests = []struct {
 	{Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}},                             // 2
 	{Encrypted: false, Metadata: map[string]string{MetaSealedKeySSEC: ""}},                         // 3
 	{Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}},                            // 4
-	{Encrypted: true, Metadata: map[string]string{MetaKeyID: ""}},                                  // 5
-	{Encrypted: true, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                      // 6
+	{Encrypted: false, Metadata: map[string]string{MetaKeyID: ""}},                                 // 5
+	{Encrypted: false, Metadata: map[string]string{MetaDataEncryptionKey: ""}},                     // 6
 	{Encrypted: false, Metadata: map[string]string{"": ""}},                                        // 7
 	{Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8
 }

cmd/crypto/sse-kms.go

@@ -82,12 +82,6 @@ func (ssekms) IsEncrypted(metadata map[string]string) bool {
 	if _, ok := metadata[MetaSealedKeyKMS]; ok {
 		return true
 	}
-	if _, ok := metadata[MetaKeyID]; ok {
-		return true
-	}
-	if _, ok := metadata[MetaDataEncryptionKey]; ok {
-		return true
-	}
 	return false
 }
 
@@ -95,11 +89,14 @@ func (ssekms) IsEncrypted(metadata map[string]string) bool {
 // from the metadata using KMS and returns the decrypted object
 // key.
 func (s3 ssekms) UnsealObjectKey(kms KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) {
-	keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata)
+	keyID, kmsKey, sealedKey, ctx, err := s3.ParseMetadata(metadata)
 	if err != nil {
 		return key, err
 	}
-	unsealKey, err := kms.UnsealKey(keyID, kmsKey, Context{bucket: path.Join(bucket, object)})
+	if _, ok := ctx[bucket]; !ok {
+		ctx[bucket] = path.Join(bucket, object)
+	}
+	unsealKey, err := kms.UnsealKey(keyID, kmsKey, ctx)
 	if err != nil {
 		return key, err
 	}
@@ -147,19 +144,19 @@ func (ssekms) CreateMetadata(metadata map[string]string, keyID string, kmsKey []
 // KMS data key it returns both. If the metadata does not contain neither a
 // KMS master key ID nor a sealed KMS data key it returns an empty keyID and
 // KMS data key. Otherwise, it returns an error.
-func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) {
+func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, ctx Context, err error) {
 	// Extract all required values from object metadata
 	b64IV, ok := metadata[MetaIV]
 	if !ok {
-		return keyID, kmsKey, sealedKey, errMissingInternalIV
+		return keyID, kmsKey, sealedKey, ctx, errMissingInternalIV
 	}
 	algorithm, ok := metadata[MetaAlgorithm]
 	if !ok {
-		return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm
+		return keyID, kmsKey, sealedKey, ctx, errMissingInternalSealAlgorithm
 	}
 	b64SealedKey, ok := metadata[MetaSealedKeyKMS]
 	if !ok {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
+		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal sealed key for SSE-S3")
 	}
 
 	// There are two possibilites:
@@ -169,33 +166,44 @@ func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []
 	keyID, idPresent := metadata[MetaKeyID]
 	b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey]
 	if !idPresent && kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
+		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3")
 	}
 	if idPresent && !kmsKeyPresent {
-		return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
+		return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3")
 	}
 
 	// Check whether all extracted values are well-formed
 	iv, err := base64.StdEncoding.DecodeString(b64IV)
 	if err != nil || len(iv) != 32 {
-		return keyID, kmsKey, sealedKey, errInvalidInternalIV
+		return keyID, kmsKey, sealedKey, ctx, errInvalidInternalIV
 	}
 	if algorithm != SealAlgorithm {
-		return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm
+		return keyID, kmsKey, sealedKey, ctx, errInvalidInternalSealAlgorithm
 	}
 	encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey)
 	if err != nil || len(encryptedKey) != 64 {
-		return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid")
+		return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed key for SSE-KMS is invalid")
 	}
 	if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key.
 		kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey)
 		if err != nil {
-			return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid")
+			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS data key for SSE-KMS is invalid")
+		}
+	}
+	b64Ctx, ok := metadata[MetaContext]
+	if ok {
+		b, err := base64.StdEncoding.DecodeString(b64Ctx)
+		if err != nil {
+			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal KMS context is not base64-encoded")
+		}
+		var json = jsoniter.ConfigCompatibleWithStandardLibrary
+		if err = json.Unmarshal(b, ctx); err != nil {
+			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS context is invalid")
 		}
 	}
 
 	sealedKey.Algorithm = algorithm
 	copy(sealedKey.IV[:], iv)
 	copy(sealedKey.Key[:], encryptedKey)
-	return keyID, kmsKey, sealedKey, nil
+	return keyID, kmsKey, sealedKey, ctx, nil
 }

cmd/crypto/sse-s3.go

@@ -62,12 +62,6 @@ func (sses3) IsEncrypted(metadata map[string]string) bool {
 	if _, ok := metadata[MetaSealedKeyS3]; ok {
 		return true
 	}
-	if _, ok := metadata[MetaKeyID]; ok {
-		return true
-	}
-	if _, ok := metadata[MetaDataEncryptionKey]; ok {
-		return true
-	}
 	return false
 }
 

cmd/data-scanner.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"math"
 	"math/rand"
+	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -55,25 +56,25 @@ var (
 	globalHealConfig   heal.Config
 	globalHealConfigMu sync.Mutex
 
-	dataCrawlerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
+	dataScannerLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 	// Sleeper values are updated when config is loaded.
-	crawlerSleeper = newDynamicSleeper(10, 10*time.Second)
+	scannerSleeper = newDynamicSleeper(10, 10*time.Second)
 )
 
-// initDataCrawler will start the crawler in the background.
-func initDataCrawler(ctx context.Context, objAPI ObjectLayer) {
-	go runDataCrawler(ctx, objAPI)
+// initDataScanner will start the scanner in the background.
+func initDataScanner(ctx context.Context, objAPI ObjectLayer) {
+	go runDataScanner(ctx, objAPI)
 }
 
-// runDataCrawler will start a data crawler.
+// runDataScanner will start a data scanner.
 // The function will block until the context is canceled.
-// There should only ever be one crawler running per cluster.
-func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
-	// Make sure only 1 crawler is running on the cluster.
-	locker := objAPI.NewNSLock(minioMetaBucket, "runDataCrawler.lock")
+// There should only ever be one scanner running per cluster.
+func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
+	// Make sure only 1 scanner is running on the cluster.
+	locker := objAPI.NewNSLock(minioMetaBucket, "runDataScanner.lock")
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	for {
-		err := locker.GetLock(ctx, dataCrawlerLeaderLockTimeout)
+		err := locker.GetLock(ctx, dataScannerLeaderLockTimeout)
 		if err != nil {
 			time.Sleep(time.Duration(r.Float64() * float64(dataCrawlStartDelay)))
 			continue
@@ -84,16 +85,19 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 
 	// Load current bloom cycle
 	nextBloomCycle := intDataUpdateTracker.current() + 1
-	var buf bytes.Buffer
-	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageBloomName, 0, -1, &buf, "", ObjectOptions{})
+
+	br, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageBloomName, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
 		if !isErrObjectNotFound(err) && !isErrBucketNotFound(err) {
 			logger.LogIf(ctx, err)
 		}
 	} else {
-		if buf.Len() == 8 {
-			nextBloomCycle = binary.LittleEndian.Uint64(buf.Bytes())
+		if br.ObjInfo.Size == 8 {
+			if err = binary.Read(br, binary.LittleEndian, &nextBloomCycle); err != nil {
+				logger.LogIf(ctx, err)
+			}
 		}
+		br.Close()
 	}
 
 	crawlTimer := time.NewTimer(dataCrawlStartDelay)
@@ -108,7 +112,7 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 			crawlTimer.Reset(dataCrawlStartDelay)
 
 			if intDataUpdateTracker.debug {
-				console.Debugln("starting crawler cycle")
+				console.Debugln("starting scanner cycle")
 			}
 
 			// Wait before starting next cycle and wait on startup.
@@ -130,7 +134,7 @@ func runDataCrawler(ctx context.Context, objAPI ObjectLayer) {
 					continue
 				}
 
-				_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r, nil, nil), ObjectOptions{})
+				_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageBloomName, NewPutObjReader(r), ObjectOptions{})
 				if !isErrBucketNotFound(err) {
 					logger.LogIf(ctx, err)
 				}
@@ -163,7 +167,7 @@ type folderScanner struct {
 
 // crawlDataFolder will crawl the basepath+cache.Info.Name and return an updated cache.
 // The returned cache will always be valid, but may not be updated from the existing.
-// Before each operation sleepDuration is called which can be used to temporarily halt the crawler.
+// Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
 func crawlDataFolder(ctx context.Context, basePath string, cache dataUsageCache, getSize getSizeFn) (dataUsageCache, error) {
 	t := UTCNow()
@@ -386,12 +390,12 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 					if f.dataUsageCrawlDebug {
 						console.Debugf(scannerLogPrefix+" Adding non-updated folder to heal check: %v\n", folder.name)
 					}
-					// If probability was already crawlerHealFolderInclude, keep it.
+					// If probability was already scannerHealFolderInclude, keep it.
 					folder.objectHealProbDiv = f.healFolderInclude
 				}
 			}
 		}
-		crawlerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
+		scannerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
 
 		cache := dataUsageEntry{}
 
@@ -403,19 +407,19 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 				if f.dataUsageCrawlDebug {
 					console.Debugf(scannerLogPrefix+" no bucket (%s,%s)\n", f.root, entName)
 				}
-				return nil
+				return errDoneForNow
 			}
 
 			if isReservedOrInvalidBucket(bucket, false) {
 				if f.dataUsageCrawlDebug {
 					console.Debugf(scannerLogPrefix+" invalid bucket: %v, entry: %v\n", bucket, entName)
 				}
-				return nil
+				return errDoneForNow
 			}
 
 			select {
 			case <-done:
-				return ctx.Err()
+				return errDoneForNow
 			default:
 			}
 
@@ -443,7 +447,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			}
 
 			// Dynamic time delay.
-			wait := crawlerSleeper.Timer(ctx)
+			wait := scannerSleeper.Timer(ctx)
 
 			// Get file size, ignore errors.
 			item := crawlItem{
@@ -533,7 +537,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 			}
 
 			// Dynamic time delay.
-			wait := crawlerSleeper.Timer(ctx)
+			wait := scannerSleeper.Timer(ctx)
 			resolver.bucket = bucket
 
 			foundObjs := false
@@ -563,7 +567,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 
 					// Sleep and reset.
 					wait()
-					wait = crawlerSleeper.Timer(ctx)
+					wait = scannerSleeper.Timer(ctx)
 					entry, ok := entries.resolve(&resolver)
 					if !ok {
 						for _, err := range errs {
@@ -600,7 +604,7 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 					for _, ver := range fiv.Versions {
 						// Sleep and reset.
 						wait()
-						wait = crawlerSleeper.Timer(ctx)
+						wait = scannerSleeper.Timer(ctx)
 						err := bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
 							object:    fiv.Name,
@@ -633,12 +637,12 @@ func (f *folderScanner) scanQueuedLevels(ctx context.Context, folders []cachedFo
 
 				objAPI.HealObjects(ctx, bucket, prefix, madmin.HealOpts{
 					Recursive: true,
-					Remove:    true,
+					Remove:    healDeleteDangling,
 				},
 					func(bucket, object, versionID string) error {
-						// Wait for each heal as per crawler frequency.
+						// Wait for each heal as per scanner frequency.
 						wait()
-						wait = crawlerSleeper.Timer(ctx)
+						wait = scannerSleeper.Timer(ctx)
 						return bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
 							object:    object,
@@ -678,7 +682,7 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder,
 	addDir = func(entName string, typ os.FileMode) error {
 		select {
 		case <-done:
-			return ctx.Err()
+			return errDoneForNow
 		default:
 		}
 
@@ -686,12 +690,12 @@ func (f *folderScanner) deepScanFolder(ctx context.Context, folder cachedFolder,
 			dirStack = append(dirStack, entName)
 			err := readDirFn(path.Join(dirStack...), addDir)
 			dirStack = dirStack[:len(dirStack)-1]
-			crawlerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
+			scannerSleeper.Sleep(ctx, dataCrawlSleepPerFolder)
 			return err
 		}
 
 		// Dynamic time delay.
-		wait := crawlerSleeper.Timer(ctx)
+		wait := scannerSleeper.Timer(ctx)
 
 		// Get file size, ignore errors.
 		dirStack = append(dirStack, entName)
@@ -990,10 +994,12 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 	return true
 }
 
-func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo) bool {
+func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool {
 	opts := ObjectOptions{}
 
-	opts.VersionID = obj.VersionID
+	if applyOnVersion {
+		opts.VersionID = obj.VersionID
+	}
 	if opts.VersionID == "" {
 		opts.Versioned = globalBucketVersioningSys.Enabled(obj.Bucket)
 	}
@@ -1025,20 +1031,20 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 }
 
 // Apply object, object version, restored object or restored object version action on the given object
-func applyExpiryRule(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject bool) bool {
+func applyExpiryRule(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
 	if obj.TransitionStatus != "" {
 		return applyExpiryOnTransitionedObject(ctx, objLayer, obj, restoredObject)
 	}
-	return applyExpiryOnNonTransitionedObjects(ctx, objLayer, obj)
+	return applyExpiryOnNonTransitionedObjects(ctx, objLayer, obj, applyOnVersion)
 }
 
 // Perform actions (removal of transitioning of objects), return true the action is successfully performed
 func applyLifecycleAction(ctx context.Context, action lifecycle.Action, objLayer ObjectLayer, obj ObjectInfo) (success bool) {
 	switch action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-		success = applyExpiryRule(ctx, objLayer, obj, false)
+		success = applyExpiryRule(ctx, objLayer, obj, false, action == lifecycle.DeleteVersionAction)
 	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-		success = applyExpiryRule(ctx, objLayer, obj, true)
+		success = applyExpiryRule(ctx, objLayer, obj, true, action == lifecycle.DeleteRestoredVersionAction)
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
 		success = applyTransitionAction(ctx, action, objLayer, obj)
 	}

cmd/data-usage-cache.go

@@ -91,7 +91,7 @@ type dataUsageCacheInfo struct {
 	Name       string
 	LastUpdate time.Time
 	NextCycle  uint32
-	// indicates if the disk is being healed and crawler
+	// indicates if the disk is being healed and scanner
 	// should skip healing the disk
 	SkipHealing bool
 	BloomFilter []byte               `msg:"BloomFilter,omitempty"`
@@ -521,8 +521,8 @@ func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string)
 	_, err = store.PutObject(ctx,
 		dataUsageBucket,
 		name,
-		NewPutObjReader(r, nil, nil),
-		ObjectOptions{})
+		NewPutObjReader(r),
+		ObjectOptions{NoLock: true})
 	if isErrBucketNotFound(err) {
 		return nil
 	}

cmd/data-usage.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/json"
+	"net/http"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/cmd/logger"
@@ -51,7 +52,7 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 			logger.LogIf(ctx, err)
 			continue
 		}
-		_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r, nil, nil), ObjectOptions{})
+		_, err = objAPI.PutObject(ctx, dataUsageBucket, dataUsageObjName, NewPutObjReader(r), ObjectOptions{})
 		if !isErrBucketNotFound(err) {
 			logger.LogIf(ctx, err)
 		}
@@ -59,20 +60,18 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 }
 
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
-	var dataUsageInfoJSON bytes.Buffer
-
-	err := objAPI.GetObject(ctx, dataUsageBucket, dataUsageObjName, 0, -1, &dataUsageInfoJSON, "", ObjectOptions{})
+	r, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageObjName, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
 		if isErrObjectNotFound(err) || isErrBucketNotFound(err) {
 			return DataUsageInfo{}, nil
 		}
 		return DataUsageInfo{}, toObjectErr(err, dataUsageBucket, dataUsageObjName)
 	}
+	defer r.Close()
 
 	var dataUsageInfo DataUsageInfo
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary
-	err = json.Unmarshal(dataUsageInfoJSON.Bytes(), &dataUsageInfo)
-	if err != nil {
+	if err = json.NewDecoder(r).Decode(&dataUsageInfo); err != nil {
 		return DataUsageInfo{}, err
 	}
 

cmd/disk-cache-backend.go

@@ -23,7 +23,6 @@ import (
 	"crypto/rand"
 	"encoding/base64"
 	"encoding/hex"
-	"errors"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -103,7 +102,6 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 	}
 
 	// We set file info only if its valid.
-	o.ModTime = m.Stat.ModTime
 	o.Size = m.Stat.Size
 	o.ETag = extractETag(m.Meta)
 	o.ContentType = m.Meta["content-type"]
@@ -122,6 +120,12 @@ func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
 			o.Expires = t.UTC()
 		}
 	}
+	if mtime, ok := m.Meta["last-modified"]; ok {
+		if t, e = time.Parse(http.TimeFormat, mtime); e == nil {
+			o.ModTime = t.UTC()
+		}
+	}
+
 	// etag/md5Sum has already been extracted. We need to
 	// remove to avoid it from appearing as part of user-defined metadata
 	o.UserDefined = cleanMetadata(m.Meta)
@@ -264,10 +268,6 @@ func (c *diskCache) toClear() uint64 {
 	return bytesToClear(int64(di.Total), int64(di.Free), uint64(c.quotaPct), uint64(c.lowWatermark), uint64(c.highWatermark))
 }
 
-var (
-	errDoneForNow = errors.New("done for now")
-)
-
 func (c *diskCache) purgeWait(ctx context.Context) {
 	for {
 		select {
@@ -377,7 +377,7 @@ func (c *diskCache) purge(ctx context.Context) {
 		return nil
 	}
 
-	if err := readDirFilterFn(c.dir, filterFn); err != nil {
+	if err := readDirFn(c.dir, filterFn); err != nil {
 		logger.LogIf(ctx, err)
 		return
 	}
@@ -506,9 +506,7 @@ func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (meta *c
 	}
 	// get metadata of part.1 if full file has been cached.
 	partial = true
-	fi, err := os.Stat(pathJoin(cacheObjPath, cacheDataFile))
-	if err == nil {
-		meta.Stat.ModTime = atime.Get(fi)
+	if _, err := os.Stat(pathJoin(cacheObjPath, cacheDataFile)); err == nil {
 		partial = false
 	}
 	return meta, partial, meta.Hits, nil
@@ -570,7 +568,6 @@ func (c *diskCache) saveMetadata(ctx context.Context, bucket, object string, met
 		}
 	}
 	m.Stat.Size = actualSize
-	m.Stat.ModTime = UTCNow()
 	if !incHitsOnly {
 		// reset meta
 		m.Meta = meta
@@ -1023,7 +1020,7 @@ func (c *diskCache) scanCacheWritebackFailures(ctx context.Context) {
 		return nil
 	}
 
-	if err := readDirFilterFn(c.dir, filterFn); err != nil {
+	if err := readDirFn(c.dir, filterFn); err != nil {
 		logger.LogIf(ctx, err)
 		return
 	}

cmd/disk-cache-utils.go

@@ -173,7 +173,11 @@ func backendDownError(err error) bool {
 
 // IsCacheable returns if the object should be saved in the cache.
 func (o ObjectInfo) IsCacheable() bool {
-	return !crypto.IsEncrypted(o.UserDefined) || globalCacheKMS != nil
+	if globalCacheKMS != nil {
+		return true
+	}
+	_, ok := crypto.IsEncrypted(o.UserDefined)
+	return !ok
 }
 
 // reads file cached on disk from offset upto length

cmd/disk-cache.go

@@ -200,6 +200,7 @@ func getMetadata(objInfo ObjectInfo) map[string]string {
 	if !objInfo.Expires.Equal(timeSentinel) {
 		metadata["expires"] = objInfo.Expires.Format(http.TimeFormat)
 	}
+	metadata["last-modified"] = objInfo.ModTime.Format(http.TimeFormat)
 	for k, v := range objInfo.UserDefined {
 		metadata[k] = v
 	}
@@ -711,7 +712,7 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 	var opts ObjectOptions
 	opts.UserDefined = make(map[string]string)
 	opts.UserDefined[xhttp.ContentMD5] = oi.UserDefined["content-md5"]
-	objInfo, err := c.InnerPutObjectFn(ctx, oi.Bucket, oi.Name, NewPutObjReader(hashReader, nil, nil), opts)
+	objInfo, err := c.InnerPutObjectFn(ctx, oi.Bucket, oi.Name, NewPutObjReader(hashReader), opts)
 	wbCommitStatus := CommitComplete
 	if err != nil {
 		wbCommitStatus = CommitFailed

cmd/disk-cache_test.go

@@ -18,6 +18,7 @@ package cmd
 
 import (
 	"testing"
+	"time"
 )
 
 // Tests ToObjectInfo function.
@@ -27,7 +28,7 @@ func TestCacheMetadataObjInfo(t *testing.T) {
 	if objInfo.Size != 0 {
 		t.Fatal("Unexpected object info value for Size", objInfo.Size)
 	}
-	if !objInfo.ModTime.Equal(timeSentinel) {
+	if !objInfo.ModTime.Equal(time.Time{}) {
 		t.Fatal("Unexpected object info value for ModTime ", objInfo.ModTime)
 	}
 	if objInfo.IsDir {

cmd/encryption-v1.go

@@ -247,56 +247,45 @@ func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, m
 }
 
 func decryptObjectInfo(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
-	switch {
-	default:
-		return nil, errObjectTampered
-	case crypto.S3.IsEncrypted(metadata) && isCacheEncrypted(metadata):
-		if globalCacheKMS == nil {
+	switch kind, _ := crypto.IsEncrypted(metadata); kind {
+	case crypto.S3:
+		var KMS crypto.KMS = GlobalKMS
+		if isCacheEncrypted(metadata) {
+			KMS = globalCacheKMS
+		}
+		if KMS == nil {
 			return nil, errKMSNotConfigured
 		}
-		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
+		objectKey, err := crypto.S3.UnsealObjectKey(KMS, metadata, bucket, object)
 		if err != nil {
 			return nil, err
 		}
-		extKey, err := globalCacheKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
-		if err != nil {
-			return nil, err
-		}
-		var objectKey crypto.ObjectKey
-		if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
-			return nil, err
-		}
 		return objectKey[:], nil
-	case crypto.S3.IsEncrypted(metadata):
+	case crypto.S3KMS:
 		if GlobalKMS == nil {
 			return nil, errKMSNotConfigured
 		}
-		keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata)
-
+		objectKey, err := crypto.S3KMS.UnsealObjectKey(GlobalKMS, metadata, bucket, object)
 		if err != nil {
 			return nil, err
 		}
-		extKey, err := GlobalKMS.UnsealKey(keyID, kmsKey, crypto.Context{bucket: path.Join(bucket, object)})
-		if err != nil {
-			return nil, err
-		}
-		var objectKey crypto.ObjectKey
-		if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), bucket, object); err != nil {
-			return nil, err
-		}
 		return objectKey[:], nil
-	case crypto.SSEC.IsEncrypted(metadata):
-		var extKey [32]byte
-		copy(extKey[:], key)
+	case crypto.SSEC:
 		sealedKey, err := crypto.SSEC.ParseMetadata(metadata)
 		if err != nil {
 			return nil, err
 		}
-		var objectKey crypto.ObjectKey
+		var (
+			objectKey crypto.ObjectKey
+			extKey    [32]byte
+		)
+		copy(extKey[:], key)
 		if err = objectKey.Unseal(extKey, sealedKey, crypto.SSEC.String(), bucket, object); err != nil {
 			return nil, err
 		}
 		return objectKey[:], nil
+	default:
+		return nil, errObjectTampered
 	}
 }
 
@@ -516,7 +505,7 @@ func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
 // It returns an error if the object is not encrypted or marked as encrypted
 // but has an invalid size.
 func (o *ObjectInfo) DecryptedSize() (int64, error) {
-	if !crypto.IsEncrypted(o.UserDefined) {
+	if _, ok := crypto.IsEncrypted(o.UserDefined); !ok {
 		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
 	if !isEncryptedMultipart(*o) {
@@ -649,7 +638,7 @@ func tryDecryptETag(key []byte, encryptedETag string, ssec bool) string {
 // requested range starts, along with the DARE sequence number within
 // that part. For single part objects, the partStart will be 0.
 func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, skipLen int64, seqNumber uint32, partStart int, err error) {
-	if !crypto.IsEncrypted(o.UserDefined) {
+	if _, ok := crypto.IsEncrypted(o.UserDefined); !ok {
 		err = errors.New("Object is not encrypted")
 		return
 	}
@@ -796,7 +785,7 @@ func DecryptObjectInfo(info *ObjectInfo, r *http.Request) (encrypted bool, err e
 		}
 	}
 
-	encrypted = crypto.IsEncrypted(info.UserDefined)
+	_, encrypted = crypto.IsEncrypted(info.UserDefined)
 	if !encrypted && crypto.SSEC.IsRequested(headers) && r.Header.Get(xhttp.AmzCopySource) == "" {
 		return false, errInvalidEncryptionParameters
 	}
@@ -818,7 +807,7 @@ func DecryptObjectInfo(info *ObjectInfo, r *http.Request) (encrypted bool, err e
 			return encrypted, err
 		}
 
-		if crypto.IsEncrypted(info.UserDefined) && !crypto.IsMultiPart(info.UserDefined) {
+		if _, ok := crypto.IsEncrypted(info.UserDefined); ok && !crypto.IsMultiPart(info.UserDefined) {
 			info.ETag = getDecryptedETag(headers, *info, false)
 		}
 	}

cmd/encryption-v1_test.go

@@ -129,7 +129,7 @@ func TestDecryptObjectInfo(t *testing.T) {
 	for i, test := range decryptObjectInfoTests {
 		if encrypted, err := DecryptObjectInfo(&test.info, test.request); err != test.expErr {
 			t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr)
-		} else if enc := crypto.IsEncrypted(test.info.UserDefined); encrypted && enc != encrypted {
+		} else if _, enc := crypto.IsEncrypted(test.info.UserDefined); encrypted && enc != encrypted {
 			t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i)
 		} else if !encrypted && enc != encrypted {
 			t.Errorf("Test %d: Decryption thinks object is not encrypted but it is", i)

cmd/erasure-healing.go

@@ -709,7 +709,7 @@ func (er erasureObjects) purgeObjectDangling(ctx context.Context, bucket, object
 			if versionID == "" {
 				err = er.deleteObject(ctx, bucket, object, writeQuorum)
 			} else {
-				err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, FileInfo{VersionID: versionID})
+				err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, FileInfo{VersionID: versionID}, false)
 			}
 
 			// If Delete was successful, make sure to return the appropriate error

cmd/erasure-multipart.go

@@ -24,6 +24,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -91,12 +92,47 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 // Clean-up the old multipart uploads. Should be run in a Go routine.
 func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
 	// run multiple cleanup's local to this server.
+	var wg sync.WaitGroup
 	for _, disk := range er.getLoadBalancedLocalDisks() {
 		if disk != nil {
-			er.cleanupStaleUploadsOnDisk(ctx, disk, expiry)
-			return
+			wg.Add(1)
+			go func(disk StorageAPI) {
+				defer wg.Done()
+				er.cleanupStaleUploadsOnDisk(ctx, disk, expiry)
+			}(disk)
 		}
 	}
+	wg.Wait()
+}
+
+func (er erasureObjects) renameAll(ctx context.Context, bucket, prefix string) {
+	var wg sync.WaitGroup
+	for _, disk := range er.getDisks() {
+		if disk == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(disk StorageAPI) {
+			defer wg.Done()
+			disk.RenameFile(ctx, bucket, prefix, minioMetaTmpBucket, mustGetUUID())
+		}(disk)
+	}
+	wg.Wait()
+}
+
+func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) {
+	var wg sync.WaitGroup
+	for _, disk := range er.getDisks() {
+		if disk == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(disk StorageAPI) {
+			defer wg.Done()
+			disk.Delete(ctx, bucket, prefix, true)
+		}(disk)
+	}
+	wg.Wait()
 }
 
 // Remove the old multipart uploads on the given disk.
@@ -118,7 +154,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 				continue
 			}
 			if now.Sub(fi.ModTime) > expiry {
-				er.deleteObject(ctx, minioMetaMultipartBucket, uploadIDPath, fi.Erasure.DataBlocks+1)
+				er.renameAll(ctx, minioMetaMultipartBucket, uploadIDPath)
 			}
 		}
 	}
@@ -127,12 +163,12 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		return
 	}
 	for _, tmpDir := range tmpDirs {
-		fi, err := disk.ReadVersion(ctx, minioMetaTmpBucket, tmpDir, "", false)
+		vi, err := disk.StatVol(ctx, pathJoin(minioMetaTmpBucket, tmpDir))
 		if err != nil {
 			continue
 		}
-		if now.Sub(fi.ModTime) > expiry {
-			er.deleteObject(ctx, minioMetaTmpBucket, tmpDir, fi.Erasure.DataBlocks+1)
+		if now.Sub(vi.Created) > expiry {
+			er.deleteAll(ctx, minioMetaTmpBucket, tmpDir)
 		}
 	}
 }
@@ -325,7 +361,7 @@ func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object
 //
 // Implements S3 compatible Upload Part Copy API.
 func (er erasureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
-	partInfo, err := er.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, NewPutObjReader(srcInfo.Reader, nil, nil), dstOpts)
+	partInfo, err := er.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, NewPutObjReader(srcInfo.Reader), dstOpts)
 	if err != nil {
 		return pi, toObjectErr(err, dstBucket, dstObject)
 	}

cmd/erasure-object.go

@@ -162,14 +162,6 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 
 	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
 	if err != nil {
-		if isProxyable(ctx, bucket) && (errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound)) {
-			// proxy to replication target if active-active replication is in place.
-			reader, proxy := proxyGetToReplicationTarget(ctx, bucket, object, rs, h, opts)
-			if reader == nil || !proxy {
-				return nil, toObjectErr(err, bucket, object)
-			}
-			return reader, nil
-		}
 		return nil, toObjectErr(err, bucket, object)
 	}
 
@@ -187,7 +179,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 	}
 	if objInfo.TransitionStatus == lifecycle.TransitionComplete {
 		// If transitioned, stream from transition tier unless object is restored locally or restore date is past.
-		restoreHdr, ok := objInfo.UserDefined[xhttp.AmzRestore]
+		restoreHdr, ok := caseInsensitiveMap(objInfo.UserDefined).Lookup(xhttp.AmzRestore)
 		if !ok || !strings.HasPrefix(restoreHdr, "ongoing-request=false") || (!objInfo.RestoreExpires.IsZero() && time.Now().After(objInfo.RestoreExpires)) {
 			return getTransitionedObjectReader(ctx, bucket, object, rs, h, objInfo, opts)
 		}
@@ -336,7 +328,9 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 				}
 				if scan != madmin.HealUnknownScan {
 					healOnce.Do(func() {
-						go healObject(bucket, object, fi.VersionID, scan)
+						if _, healing := er.getOnlineDisksWithHealing(); !healing {
+							go healObject(bucket, object, fi.VersionID, scan)
+						}
 					})
 				}
 			}
@@ -378,12 +372,14 @@ func (er erasureObjects) getObject(ctx context.Context, bucket, object string, s
 
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
-	// Lock the object before reading.
-	lk := er.NewNSLock(bucket, object)
-	if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
-		return ObjectInfo{}, err
+	if !opts.NoLock {
+		// Lock the object before reading.
+		lk := er.NewNSLock(bucket, object)
+		if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
+			return ObjectInfo{}, err
+		}
+		defer lk.RUnlock()
 	}
-	defer lk.RUnlock()
 
 	return er.getObjectInfo(ctx, bucket, object, opts)
 }
@@ -414,7 +410,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 					er.deleteObjectVersion(ctx, bucket, object, 1, FileInfo{
 						Name:      object,
 						VersionID: opts.VersionID,
-					})
+					}, false)
 				}
 			}
 		}
@@ -444,7 +440,9 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 
 	// if missing metadata can be reconstructed, attempt to reconstruct.
 	if missingBlocks > 0 && missingBlocks < readQuorum {
-		go healObject(bucket, object, fi.VersionID, madmin.HealNormalScan)
+		if _, healing := er.getOnlineDisksWithHealing(); !healing {
+			go healObject(bucket, object, fi.VersionID, madmin.HealNormalScan)
+		}
 	}
 
 	return fi, metaArr, onlineDisks, nil
@@ -454,13 +452,6 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
 	if err != nil {
-		// proxy HEAD to replication target if active-active replication configured on bucket
-		if isProxyable(ctx, bucket) && (errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound)) {
-			oi, proxy, err := proxyHeadToReplicationTarget(ctx, bucket, object, opts)
-			if proxy {
-				return oi, err
-			}
-		}
 		return objInfo, toObjectErr(err, bucket, object)
 
 	}
@@ -471,10 +462,11 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 			objInfo.StorageClass = sc
 		}
 	}
-	if !fi.VersionPurgeStatus.Empty() {
+	if !fi.VersionPurgeStatus.Empty() && opts.VersionID != "" {
 		// Make sure to return object info to provide extra information.
 		return objInfo, toObjectErr(errMethodNotAllowed, bucket, object)
 	}
+
 	if fi.Deleted {
 		if opts.VersionID == "" || opts.DeleteMarker {
 			return objInfo, toObjectErr(errFileNotFound, bucket, object)
@@ -800,7 +792,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	return fi.ToObjectInfo(bucket, object), nil
 }
 
-func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, writeQuorum int, fi FileInfo) error {
+func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, writeQuorum int, fi FileInfo, forceDelMarker bool) error {
 	defer ObjectPathUpdated(pathJoin(bucket, object))
 	disks := er.getDisks()
 	g := errgroup.WithNErrs(len(disks))
@@ -810,7 +802,7 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			return disks[index].DeleteVersion(ctx, bucket, object, fi)
+			return disks[index].DeleteVersion(ctx, bucket, object, fi, forceDelMarker)
 		}, index)
 	}
 	// return errors if any during deletion
@@ -1049,11 +1041,10 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	if goi.VersionID != "" {
 		markDelete = true
 	}
+
 	// Default deleteMarker to true if object is under versioning
-	deleteMarker := true
-	if gerr == nil {
-		deleteMarker = goi.VersionID != ""
-	}
+	deleteMarker := opts.Versioned
+
 	if opts.VersionID != "" {
 		// case where replica version needs to be deleted on target cluster
 		if versionFound && opts.DeleteMarkerReplicationStatus == replication.Replica.String() {
@@ -1062,7 +1053,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		if opts.VersionPurgeStatus.Empty() && opts.DeleteMarkerReplicationStatus == "" {
 			markDelete = false
 		}
-		if opts.DeleteMarker && opts.VersionPurgeStatus == Complete {
+		if opts.VersionPurgeStatus == Complete {
 			markDelete = false
 		}
 		// determine if the version represents an object delete
@@ -1098,7 +1089,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 			// version as delete marker
 			// Add delete marker, since we don't have any version specified explicitly.
 			// Or if a particular version id needs to be replicated.
-			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi); err != nil {
+			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, opts.DeleteMarker); err != nil {
 				return objInfo, toObjectErr(err, bucket, object)
 			}
 			return fi.ToObjectInfo(bucket, object), nil
@@ -1115,7 +1106,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		DeleteMarkerReplicationStatus: opts.DeleteMarkerReplicationStatus,
 		VersionPurgeStatus:            opts.VersionPurgeStatus,
 		TransitionStatus:              opts.TransitionStatus,
-	}); err != nil {
+	}, opts.DeleteMarker); err != nil {
 		return objInfo, toObjectErr(err, bucket, object)
 	}
 

cmd/erasure-server-pool.go

@@ -33,7 +33,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/cmd/config/storageclass"
 	"github.com/minio/minio/cmd/logger"
-	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/sync/errgroup"
 )
@@ -107,11 +106,6 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			return nil, fmt.Errorf("All serverPools should have same deployment ID expected %s, got %s", deploymentID, formats[i].ID)
 		}
 
-		// Validate if users brought different different distribution algo pools.
-		if distributionAlgo != formats[i].Erasure.DistributionAlgo {
-			return nil, fmt.Errorf("All serverPools should have same distributionAlgo expected %s, got %s", distributionAlgo, formats[i].Erasure.DistributionAlgo)
-		}
-
 		z.serverPools[i], err = newErasureSets(ctx, ep.Endpoints, storageDisks[i], formats[i], commonParityDrives)
 		if err != nil {
 			return nil, err
@@ -252,29 +246,39 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, s
 
 // getPoolIdx returns the found previous object and its corresponding pool idx,
 // if none are found falls back to most available space pool.
-func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, opts ObjectOptions, size int64) (idx int, err error) {
+func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) {
 	if z.SinglePool() {
 		return 0, nil
 	}
+
+	errs := make([]error, len(z.serverPools))
+	objInfos := make([]ObjectInfo, len(z.serverPools))
+
+	var wg sync.WaitGroup
 	for i, pool := range z.serverPools {
-		objInfo, err := pool.GetObjectInfo(ctx, bucket, object, opts)
-		switch err.(type) {
-		case ObjectNotFound:
-			// VersionId was not specified but found delete marker or no versions exist.
-		case MethodNotAllowed:
-			// VersionId was specified but found delete marker
-		default:
-			if err != nil {
-				// any other un-handled errors return right here.
-				return -1, err
-			}
+		wg.Add(1)
+		go func(i int, pool *erasureSets) {
+			defer wg.Done()
+			objInfos[i], errs[i] = pool.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+		}(i, pool)
+	}
+	wg.Wait()
+
+	for i, err := range errs {
+		if err != nil && !isErrObjectNotFound(err) {
+			return -1, err
 		}
-		// delete marker not specified means no versions
-		// exist continue to next pool.
-		if !objInfo.DeleteMarker && err != nil {
+		if isErrObjectNotFound(err) {
+			// No object exists or its a delete marker,
+			// check objInfo to confirm.
+			if objInfos[i].DeleteMarker && objInfos[i].Name != "" {
+				return i, nil
+			}
+			// objInfo is not valid, truly the object doesn't
+			// exist proceed to next pool.
 			continue
 		}
-		// Success case and when DeleteMarker is true return.
+		// object exists at this pool.
 		return i, nil
 	}
 
@@ -371,7 +375,6 @@ func (z *erasureServerPools) CrawlAndGetDataUsage(ctx context.Context, bf *bloom
 	}
 
 	if len(allBuckets) == 0 {
-		logger.Info(color.Green("data-crawl:") + " No buckets found, skipping crawl")
 		updates <- DataUsageInfo{} // no buckets found update data usage to reflect latest state
 		return nil
 	}
@@ -398,7 +401,7 @@ func (z *erasureServerPools) CrawlAndGetDataUsage(ctx context.Context, bf *bloom
 						mu.Unlock()
 					}
 				}()
-				// Start crawler. Blocks until done.
+				// Start scanner. Blocks until done.
 				err := erObj.crawlAndGetDataUsage(ctx, allBuckets, bf, updates)
 				if err != nil {
 					logger.LogIf(ctx, err)
@@ -521,16 +524,71 @@ func (z *erasureServerPools) GetObjectNInfo(ctx context.Context, bucket, object
 
 	object = encodeDirObject(object)
 
-	for _, pool := range z.serverPools {
-		gr, err = pool.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
-		if err != nil {
-			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-				continue
+	if z.SinglePool() {
+		return z.serverPools[0].GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+	}
+
+	var unlockOnDefer bool
+	var nsUnlocker = func() {}
+	defer func() {
+		if unlockOnDefer {
+			nsUnlocker()
+		}
+	}()
+
+	// Acquire lock
+	if lockType != noLock {
+		lock := z.NewNSLock(bucket, object)
+		switch lockType {
+		case writeLock:
+			if err = lock.GetLock(ctx, globalOperationTimeout); err != nil {
+				return nil, err
+			}
+			nsUnlocker = lock.Unlock
+		case readLock:
+			if err = lock.GetRLock(ctx, globalOperationTimeout); err != nil {
+				return nil, err
+			}
+			nsUnlocker = lock.RUnlock
+		}
+		unlockOnDefer = true
+	}
+
+	errs := make([]error, len(z.serverPools))
+	grs := make([]*GetObjectReader, len(z.serverPools))
+
+	lockType = noLock // do not take locks at lower levels
+	var wg sync.WaitGroup
+	for i, pool := range z.serverPools {
+		wg.Add(1)
+		go func(i int, pool *erasureSets) {
+			defer wg.Done()
+			grs[i], errs[i] = pool.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+		}(i, pool)
+	}
+	wg.Wait()
+
+	var found int = -1
+	for i, err := range errs {
+		if err == nil {
+			found = i
+			break
+		}
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			for _, grr := range grs {
+				if grr != nil {
+					grr.Close()
+				}
 			}
 			return gr, err
 		}
-		return gr, nil
 	}
+
+	if found >= 0 {
+		return grs[found], nil
+	}
+
+	object = decodeDirObject(object)
 	if opts.VersionID != "" {
 		return gr, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
 	}
@@ -543,7 +601,6 @@ func (z *erasureServerPools) GetObject(ctx context.Context, bucket, object strin
 	}
 
 	object = encodeDirObject(object)
-
 	for _, pool := range z.serverPools {
 		if err := pool.GetObject(ctx, bucket, object, startOffset, length, writer, etag, opts); err != nil {
 			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
@@ -565,16 +622,49 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
 	}
 
 	object = encodeDirObject(object)
-	for _, pool := range z.serverPools {
-		objInfo, err = pool.GetObjectInfo(ctx, bucket, object, opts)
-		if err != nil {
-			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-				continue
-			}
-			return objInfo, err
-		}
-		return objInfo, nil
+
+	if z.SinglePool() {
+		return z.serverPools[0].GetObjectInfo(ctx, bucket, object, opts)
 	}
+
+	// Lock the object before reading.
+	lk := z.NewNSLock(bucket, object)
+	if err := lk.GetRLock(ctx, globalOperationTimeout); err != nil {
+		return ObjectInfo{}, err
+	}
+	defer lk.RUnlock()
+
+	errs := make([]error, len(z.serverPools))
+	objInfos := make([]ObjectInfo, len(z.serverPools))
+
+	opts.NoLock = true // avoid taking locks at lower levels for multi-pool setups.
+	var wg sync.WaitGroup
+	for i, pool := range z.serverPools {
+		wg.Add(1)
+		go func(i int, pool *erasureSets) {
+			defer wg.Done()
+			objInfos[i], errs[i] = pool.GetObjectInfo(ctx, bucket, object, opts)
+		}(i, pool)
+	}
+	wg.Wait()
+
+	var found int = -1
+	for i, err := range errs {
+		if err == nil {
+			found = i
+			break
+		}
+		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+			// some errors such as MethodNotAllowed for delete marker
+			// should be returned upwards.
+			return objInfos[i], err
+		}
+	}
+
+	if found >= 0 {
+		return objInfos[found], nil
+	}
+
 	object = decodeDirObject(object)
 	if opts.VersionID != "" {
 		return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID}
@@ -595,7 +685,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 		return z.serverPools[0].PutObject(ctx, bucket, object, data, opts)
 	}
 
-	idx, err := z.getPoolIdx(ctx, bucket, object, opts, data.Size())
+	idx, err := z.getPoolIdx(ctx, bucket, object, data.Size())
 	if err != nil {
 		return ObjectInfo{}, err
 	}
@@ -610,21 +700,17 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 	}
 
 	object = encodeDirObject(object)
-
 	if z.SinglePool() {
 		return z.serverPools[0].DeleteObject(ctx, bucket, object, opts)
 	}
-	for _, pool := range z.serverPools {
-		objInfo, err = pool.DeleteObject(ctx, bucket, object, opts)
-		if err == nil {
-			return objInfo, nil
-		}
-		if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-			break
-		}
+
+	// We don't know the size here set 1GiB atleast.
+	idx, err := z.getPoolIdx(ctx, bucket, object, 1<<30)
+	if err != nil {
+		return objInfo, err
 	}
 
-	return objInfo, err
+	return z.serverPools[idx].DeleteObject(ctx, bucket, object, opts)
 }
 
 func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
@@ -638,6 +724,23 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		objSets.Add(objects[i].ObjectName)
 	}
 
+	poolObjIdxMap := map[int][]ObjectToDelete{}
+	origIndexMap := map[int][]int{}
+	if !z.SinglePool() {
+		for j, obj := range objects {
+			idx, err := z.getPoolIdx(ctx, bucket, obj.ObjectName, 1<<30)
+			if err != nil {
+				// Unhandled errors return right here.
+				for i := range derrs {
+					derrs[i] = err
+				}
+				return dobjects, derrs
+			}
+			poolObjIdxMap[idx] = append(poolObjIdxMap[idx], obj)
+			origIndexMap[idx] = append(origIndexMap[idx], j)
+		}
+	}
+
 	// Acquire a bulk write lock across 'objects'
 	multiDeleteLock := z.NewNSLock(bucket, objSets.ToSlice()...)
 	if err := multiDeleteLock.GetLock(ctx, globalOperationTimeout); err != nil {
@@ -652,13 +755,15 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		return z.serverPools[0].DeleteObjects(ctx, bucket, objects, opts)
 	}
 
-	for _, pool := range z.serverPools {
-		deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objects, opts)
+	for idx, pool := range z.serverPools {
+		objs := poolObjIdxMap[idx]
+		orgIndexes := origIndexMap[idx]
+		deletedObjects, errs := pool.DeleteObjects(ctx, bucket, objs, opts)
 		for i, derr := range errs {
 			if derr != nil {
-				derrs[i] = derr
+				derrs[orgIndexes[i]] = derr
 			}
-			dobjects[i] = deletedObjects[i]
+			dobjects[orgIndexes[i]] = deletedObjects[i]
 		}
 	}
 	return dobjects, derrs
@@ -670,7 +775,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 
 	cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject))
 
-	poolIdx, err := z.getPoolIdx(ctx, dstBucket, dstObject, dstOpts, srcInfo.Size)
+	poolIdx, err := z.getPoolIdx(ctx, dstBucket, dstObject, srcInfo.Size)
 	if err != nil {
 		return objInfo, err
 	}
@@ -766,7 +871,7 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 		loi.IsTruncated = true
 	}
 	for _, obj := range objects {
-		if obj.IsDir && delimiter != "" {
+		if obj.IsDir && obj.ModTime.IsZero() && delimiter != "" {
 			loi.Prefixes = append(loi.Prefixes, obj.Name)
 		} else {
 			loi.Objects = append(loi.Objects, obj)
@@ -805,7 +910,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		loi.IsTruncated = true
 	}
 	for _, obj := range objects {
-		if obj.IsDir && delimiter != "" {
+		if obj.IsDir && obj.ModTime.IsZero() && delimiter != "" {
 			loi.Prefixes = append(loi.Prefixes, obj.Name)
 		} else {
 			loi.Objects = append(loi.Objects, obj)
@@ -854,7 +959,7 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 	}
 
 	// We don't know the exact size, so we ask for at least 1GiB file.
-	idx, err := z.getPoolIdx(ctx, bucket, object, opts, 1<<30)
+	idx, err := z.getPoolIdx(ctx, bucket, object, 1<<30)
 	if err != nil {
 		return "", err
 	}
@@ -869,7 +974,7 @@ func (z *erasureServerPools) CopyObjectPart(ctx context.Context, srcBucket, srcO
 	}
 
 	return z.PutObjectPart(ctx, destBucket, destObject, uploadID, partID,
-		NewPutObjReader(srcInfo.Reader, nil, nil), dstOpts)
+		NewPutObjReader(srcInfo.Reader), dstOpts)
 }
 
 // PutObjectPart - writes part of an object to hashedSet based on the object name.
@@ -1115,22 +1220,24 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, fo
 // data is not distributed across sets.
 // Errors are logged but individual disk failures are not returned.
 func (z *erasureServerPools) deleteAll(ctx context.Context, bucket, prefix string) {
-	var wg sync.WaitGroup
 	for _, servers := range z.serverPools {
 		for _, set := range servers.sets {
-			for _, disk := range set.getDisks() {
-				if disk == nil {
-					continue
-				}
-				wg.Add(1)
-				go func(disk StorageAPI) {
-					defer wg.Done()
-					disk.Delete(ctx, bucket, prefix, true)
-				}(disk)
-			}
+			set.deleteAll(ctx, bucket, prefix)
+		}
+	}
+}
+
+// renameAll will rename bucket+prefix unconditionally across all disks to
+// minioMetaTmpBucket + unique uuid,
+// Note that set distribution is ignored so it should only be used in cases where
+// data is not distributed across sets. Errors are logged but individual
+// disk failures are not returned.
+func (z *erasureServerPools) renameAll(ctx context.Context, bucket, prefix string) {
+	for _, servers := range z.serverPools {
+		for _, set := range servers.sets {
+			set.renameAll(ctx, bucket, prefix)
 		}
 	}
-	wg.Wait()
 }
 
 // This function is used to undo a successful DeleteBucket operation.
@@ -1455,6 +1562,40 @@ type HealthResult struct {
 	WriteQuorum   int
 }
 
+// ReadHealth returns if the cluster can serve read requests
+func (z *erasureServerPools) ReadHealth(ctx context.Context) bool {
+	erasureSetUpCount := make([][]int, len(z.serverPools))
+	for i := range z.serverPools {
+		erasureSetUpCount[i] = make([]int, len(z.serverPools[i].sets))
+	}
+
+	diskIDs := globalNotificationSys.GetLocalDiskIDs(ctx)
+	diskIDs = append(diskIDs, getLocalDiskIDs(z))
+
+	for _, localDiskIDs := range diskIDs {
+		for _, id := range localDiskIDs {
+			poolIdx, setIdx, err := z.getPoolAndSet(id)
+			if err != nil {
+				logger.LogIf(ctx, err)
+				continue
+			}
+			erasureSetUpCount[poolIdx][setIdx]++
+		}
+	}
+
+	b := z.BackendInfo()
+	readQuorum := b.StandardSCData[0]
+
+	for poolIdx := range erasureSetUpCount {
+		for setIdx := range erasureSetUpCount[poolIdx] {
+			if erasureSetUpCount[poolIdx][setIdx] < readQuorum {
+				return false
+			}
+		}
+	}
+	return true
+}
+
 // Health - returns current status of the object layer health,
 // provides if write access exists across sets, additionally
 // can be used to query scenarios if health may be lost
@@ -1548,27 +1689,13 @@ func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object s
 		return z.serverPools[0].PutObjectTags(ctx, bucket, object, tags, opts)
 	}
 
-	for _, pool := range z.serverPools {
-		objInfo, err := pool.PutObjectTags(ctx, bucket, object, tags, opts)
-		if err != nil {
-			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-				continue
-			}
-			return ObjectInfo{}, err
-		}
-		return objInfo, nil
-	}
-	if opts.VersionID != "" {
-		return ObjectInfo{}, VersionNotFound{
-			Bucket:    bucket,
-			Object:    object,
-			VersionID: opts.VersionID,
-		}
-	}
-	return ObjectInfo{}, ObjectNotFound{
-		Bucket: bucket,
-		Object: object,
+	// We don't know the size here set 1GiB atleast.
+	idx, err := z.getPoolIdx(ctx, bucket, object, 1<<30)
+	if err != nil {
+		return ObjectInfo{}, err
 	}
+
+	return z.serverPools[idx].PutObjectTags(ctx, bucket, object, tags, opts)
 }
 
 // DeleteObjectTags - delete object tags from an existing object
@@ -1577,27 +1704,14 @@ func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, objec
 	if z.SinglePool() {
 		return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts)
 	}
-	for _, pool := range z.serverPools {
-		objInfo, err := pool.DeleteObjectTags(ctx, bucket, object, opts)
-		if err != nil {
-			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-				continue
-			}
-			return ObjectInfo{}, err
-		}
-		return objInfo, nil
-	}
-	if opts.VersionID != "" {
-		return ObjectInfo{}, VersionNotFound{
-			Bucket:    bucket,
-			Object:    object,
-			VersionID: opts.VersionID,
-		}
-	}
-	return ObjectInfo{}, ObjectNotFound{
-		Bucket: bucket,
-		Object: object,
+
+	// We don't know the size here set 1GiB atleast.
+	idx, err := z.getPoolIdx(ctx, bucket, object, 1<<30)
+	if err != nil {
+		return ObjectInfo{}, err
 	}
+
+	return z.serverPools[idx].DeleteObjectTags(ctx, bucket, object, opts)
 }
 
 // GetObjectTags - get object tags from an existing object
@@ -1606,25 +1720,12 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
 	if z.SinglePool() {
 		return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
 	}
-	for _, pool := range z.serverPools {
-		tags, err := pool.GetObjectTags(ctx, bucket, object, opts)
-		if err != nil {
-			if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-				continue
-			}
-			return tags, err
-		}
-		return tags, nil
-	}
-	if opts.VersionID != "" {
-		return nil, VersionNotFound{
-			Bucket:    bucket,
-			Object:    object,
-			VersionID: opts.VersionID,
-		}
-	}
-	return nil, ObjectNotFound{
-		Bucket: bucket,
-		Object: object,
+
+	// We don't know the size here set 1GiB atleast.
+	idx, err := z.getPoolIdx(ctx, bucket, object, 1<<30)
+	if err != nil {
+		return nil, err
 	}
+
+	return z.serverPools[idx].GetObjectTags(ctx, bucket, object, opts)
 }

cmd/erasure-sets.go

@@ -357,9 +357,14 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 
 	mutex := newNSLock(globalIsDistErasure)
 
+	// Number of buffers, max 2GB.
+	n := setCount * setDriveCount
+	if n > 100 {
+		n = 100
+	}
 	// Initialize byte pool once for all sets, bpool size is set to
 	// setCount * setDriveCount with each memory upto blockSizeV1.
-	bp := bpool.NewBytePoolCap(setCount*setDriveCount, blockSizeV1, blockSizeV1*2)
+	bp := bpool.NewBytePoolCap(n, blockSizeV1, blockSizeV1*2)
 
 	for i := 0; i < setCount; i++ {
 		s.erasureDisks[i] = make([]StorageAPI, setDriveCount)
@@ -423,14 +428,17 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 }
 
 func (s *erasureSets) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
-	ticker := time.NewTicker(cleanupInterval)
-	defer ticker.Stop()
+	timer := time.NewTimer(cleanupInterval)
+	defer timer.Stop()
 
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-ticker.C:
+		case <-timer.C:
+			// Reset for the next interval
+			timer.Reset(cleanupInterval)
+
 			for _, set := range s.sets {
 				set.cleanupStaleUploads(ctx, expiry)
 			}
@@ -1073,7 +1081,7 @@ func (s *erasureSets) CopyObjectPart(ctx context.Context, srcBucket, srcObject,
 	startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (partInfo PartInfo, err error) {
 	destSet := s.getHashedSet(destObject)
 	auditObjectErasureSet(ctx, destObject, destSet, s.poolNumber)
-	return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, NewPutObjReader(srcInfo.Reader, nil, nil), dstOpts)
+	return destSet.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, NewPutObjReader(srcInfo.Reader), dstOpts)
 }
 
 // PutObjectPart - writes part of an object to hashedSet based on the object name.

cmd/erasure.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"math/rand"
 	"sort"
 	"sync"
 	"time"
@@ -238,6 +239,7 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 			disk := disks[i-1]
 
 			if disk == nil {
+				infos[i-1].Error = "nil disk"
 				return
 			}
 
@@ -248,6 +250,7 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 				//
 				//
 				// - Future: skip busy disks
+				infos[i-1].Error = err.Error()
 				return
 			}
 
@@ -258,9 +261,9 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 
 	for i, info := range infos {
 		// Check if one of the drives in the set is being healed.
-		// this information is used by crawler to skip healing
+		// this information is used by scanner to skip healing
 		// this erasure set while it calculates the usage.
-		if info.Healing {
+		if info.Healing || info.Error != "" {
 			healing = true
 			continue
 		}
@@ -274,7 +277,6 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 // Updates are sent on a regular basis and the caller *must* consume them.
 func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, updates chan<- dataUsageCache) error {
 	if len(buckets) == 0 {
-		logger.Info(color.Green("data-crawl:") + " No buckets found, skipping crawl")
 		return nil
 	}
 
@@ -344,7 +346,8 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 	var saverWg sync.WaitGroup
 	saverWg.Add(1)
 	go func() {
-		const updateTime = 30 * time.Second
+		// Add jitter to the update time so multiple sets don't sync up.
+		var updateTime = 30*time.Second + time.Duration(float64(10*time.Second)*rand.Float64())
 		t := time.NewTicker(updateTime)
 		defer t.Stop()
 		defer saverWg.Done()
@@ -377,7 +380,7 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 		}
 	}()
 
-	// Start one crawler per disk
+	// Start one scanner per disk
 	var wg sync.WaitGroup
 	wg.Add(len(disks))
 	for i := range disks {
@@ -428,11 +431,15 @@ func (er erasureObjects) crawlAndGetDataUsage(ctx context.Context, buckets []Buc
 				if r := cache.root(); r != nil {
 					root = cache.flatten(*r)
 				}
+				t := time.Now()
 				bucketResults <- dataUsageEntryInfo{
 					Name:   cache.Info.Name,
 					Parent: dataUsageRoot,
 					Entry:  root,
 				}
+				// We want to avoid synchronizing up all writes in case
+				// the results are piled up.
+				time.Sleep(time.Duration(float64(time.Since(t)) * rand.Float64()))
 				// Save cache
 				logger.LogIf(ctx, cache.save(ctx, er, cacheName))
 			}

cmd/fastwalk.go

@@ -1,38 +0,0 @@
-// Copyright 2016 The Go Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style
-// license that can be found in the LICENSE file.
-
-// This code is imported from "golang.org/x/tools/internal/fastwalk",
-// only fastwalk.go is imported since we already implement readDir()
-// with some little tweaks.
-
-package cmd
-
-import (
-	"errors"
-	"os"
-	"strings"
-)
-
-var errSkipFile = errors.New("fastwalk: skip this file")
-
-func readDirFn(dirName string, fn func(entName string, typ os.FileMode) error) error {
-	fis, err := readDir(dirName)
-	if err != nil {
-		if osIsNotExist(err) || err == errFileNotFound {
-			return nil
-		}
-		return err
-	}
-	for _, fi := range fis {
-		var mode os.FileMode
-		if strings.HasSuffix(fi, SlashSeparator) {
-			mode |= os.ModeDir
-		}
-
-		if err = fn(fi, mode); err != nil {
-			return err
-		}
-	}
-	return nil
-}

cmd/fs-v1-multipart.go

@@ -849,14 +849,17 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 // on all buckets for every `cleanupInterval`, this function is
 // blocking and should be run in a go-routine.
 func (fs *FSObjects) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
-	ticker := time.NewTicker(cleanupInterval)
-	defer ticker.Stop()
+	timer := time.NewTimer(cleanupInterval)
+	defer timer.Stop()
 
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-ticker.C:
+		case <-timer.C:
+			// Reset for the next interval
+			timer.Reset(cleanupInterval)
+
 			now := time.Now()
 			entries, err := readDir(pathJoin(fs.fsPath, minioMetaMultipartBucket))
 			if err != nil {

cmd/fs-v1.go

@@ -1623,3 +1623,9 @@ func (fs *FSObjects) Health(ctx context.Context, opts HealthOptions) HealthResul
 		Healthy: newObjectLayerFn() != nil,
 	}
 }
+
+// ReadHealth returns "read" health of the object layer
+func (fs *FSObjects) ReadHealth(ctx context.Context) bool {
+	_, err := os.Stat(fs.fsPath)
+	return err == nil
+}

cmd/gateway-unsupported.go

@@ -254,3 +254,8 @@ func (a GatewayUnsupported) IsCompressionSupported() bool {
 func (a GatewayUnsupported) Health(_ context.Context, _ HealthOptions) HealthResult {
 	return HealthResult{}
 }
+
+// ReadHealth - No Op.
+func (a GatewayUnsupported) ReadHealth(_ context.Context) bool {
+	return true
+}

cmd/gateway/hdfs/gateway-hdfs.go

@@ -33,6 +33,10 @@ import (
 
 	"github.com/colinmarc/hdfs/v2"
 	"github.com/colinmarc/hdfs/v2/hadoopconf"
+	krb "github.com/jcmturner/gokrb5/v8/client"
+	"github.com/jcmturner/gokrb5/v8/config"
+	"github.com/jcmturner/gokrb5/v8/credentials"
+	"github.com/jcmturner/gokrb5/v8/keytab"
 	"github.com/minio/cli"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	minio "github.com/minio/minio/cmd"
@@ -41,9 +45,6 @@ import (
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
 	xnet "github.com/minio/minio/pkg/net"
-	krb "gopkg.in/jcmturner/gokrb5.v7/client"
-	"gopkg.in/jcmturner/gokrb5.v7/config"
-	"gopkg.in/jcmturner/gokrb5.v7/credentials"
 )
 
 const (
@@ -121,6 +122,23 @@ func getKerberosClient() (*krb.Client, error) {
 		return nil, err
 	}
 
+	keytabPath := env.Get("KRB5KEYTAB", "")
+	if keytabPath != "" {
+		kt, err := keytab.Load(keytabPath)
+		if err != nil {
+			return nil, err
+		}
+
+		username := env.Get("KRB5USERNAME", "")
+		realm := env.Get("KRB5REALM", "")
+		if username == "" || realm == "" {
+			return nil, errors.New("empty KRB5USERNAME or KRB5REALM")
+
+		}
+
+		return krb.NewWithKeytab(username, realm, kt, cfg), nil
+	}
+
 	// Determine the ccache location from the environment, falling back to the default location.
 	ccachePath := env.Get("KRB5CCNAME", fmt.Sprintf("/tmp/krb5cc_%s", u.Uid))
 	if strings.Contains(ccachePath, ":") {
@@ -136,7 +154,7 @@ func getKerberosClient() (*krb.Client, error) {
 		return nil, err
 	}
 
-	return krb.NewClientFromCCache(ccache, cfg)
+	return krb.NewFromCCache(ccache, cfg)
 }
 
 // NewGatewayLayer returns hdfs gatewaylayer.
@@ -195,7 +213,7 @@ func (g *HDFS) NewGatewayLayer(creds auth.Credentials) (minio.ObjectLayer, error
 
 	clnt, err := hdfs.NewClient(opts)
 	if err != nil {
-		return nil, fmt.Errorf("unable to initialize hdfsClient")
+		return nil, fmt.Errorf("unable to initialize hdfsClient: %v", err)
 	}
 
 	if err = clnt.MkdirAll(minio.PathJoin(commonPath, hdfsSeparator, minioMetaTmpBucket), os.FileMode(0755)); err != nil {

cmd/gateway/s3/gateway-s3-metadata.go

@@ -172,5 +172,5 @@ func getGWMetadata(ctx context.Context, bucket, prefix string, gwMeta gwMetaV1)
 	if err != nil {
 		return nil, err
 	}
-	return minio.NewPutObjReader(hashReader, nil, nil), nil
+	return minio.NewPutObjReader(hashReader), nil
 }

cmd/gateway/s3/gateway-s3-sse.go

@@ -316,7 +316,7 @@ func (l *s3EncObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	if err != nil {
 		return l.s3Objects.GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
 	}
-	fn, off, length, err := minio.NewGetObjectReader(rs, objInfo, o)
+	fn, off, length, err := minio.NewGetObjectReader(rs, objInfo, opts)
 	if err != nil {
 		return nil, minio.ErrorRespToObjectError(err, bucket, object)
 	}
@@ -325,7 +325,20 @@ func (l *s3EncObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	}
 	pr, pw := io.Pipe()
 	go func() {
-		err := l.getObject(ctx, bucket, object, off, length, pw, objInfo.ETag, opts)
+		// Do not set an `If-Match` header for the ETag when
+		// the ETag is encrypted. The ETag at the backend never
+		// matches an encrypted ETag and there is in any case
+		// no way to make two consecutive S3 calls safe for concurrent
+		// access.
+		// However,  the encrypted object changes concurrently then the
+		// gateway will not be able to decrypt it since the key (obtained
+		// from dare.meta) will not work for any new created object. Therefore,
+		// we will in any case not return invalid data to the client.
+		etag := objInfo.ETag
+		if len(etag) > 32 && strings.Count(etag, "-") == 0 {
+			etag = ""
+		}
+		err := l.getObject(ctx, bucket, object, off, length, pw, etag, opts)
 		pw.CloseWithError(err)
 	}()
 
@@ -352,7 +365,7 @@ func (l *s3EncObjects) GetObjectInfo(ctx context.Context, bucket string, object
 
 // CopyObject copies an object from source bucket to a destination bucket.
 func (l *s3EncObjects) CopyObject(ctx context.Context, srcBucket string, srcObject string, dstBucket string, dstObject string, srcInfo minio.ObjectInfo, s, d minio.ObjectOptions) (objInfo minio.ObjectInfo, err error) {
-	cpSrcDstSame := strings.EqualFold(path.Join(srcBucket, srcObject), path.Join(dstBucket, dstObject))
+	cpSrcDstSame := path.Join(srcBucket, srcObject) == path.Join(dstBucket, dstObject)
 	if cpSrcDstSame {
 		var gwMeta gwMetaV1
 		if s.ServerSideEncryption != nil && d.ServerSideEncryption != nil &&

cmd/gateway/s3/gateway-s3.go

@@ -515,7 +515,7 @@ func (l *s3Objects) CopyObject(ctx context.Context, srcBucket string, srcObject
 		srcInfo.UserDefined[k] = v[0]
 	}
 
-	if _, err = l.Client.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined, miniogo.PutObjectOptions{}); err != nil {
+	if _, err = l.Client.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo.UserDefined, miniogo.CopySrcOptions{}, miniogo.PutObjectOptions{}); err != nil {
 		return objInfo, minio.ErrorRespToObjectError(err, srcBucket, srcObject)
 	}
 	return l.GetObjectInfo(ctx, dstBucket, dstObject, dstOpts)

cmd/generic-handlers.go

@@ -216,7 +216,8 @@ func guessIsHealthCheckReq(req *http.Request) bool {
 	return aType == authTypeAnonymous && (req.Method == http.MethodGet || req.Method == http.MethodHead) &&
 		(req.URL.Path == healthCheckPathPrefix+healthCheckLivenessPath ||
 			req.URL.Path == healthCheckPathPrefix+healthCheckReadinessPath ||
-			req.URL.Path == healthCheckPathPrefix+healthCheckClusterPath)
+			req.URL.Path == healthCheckPathPrefix+healthCheckClusterPath ||
+			req.URL.Path == healthCheckPathPrefix+healthCheckClusterReadPath)
 }
 
 // guessIsMetricsReq - returns true if incoming request looks

cmd/global-heal.go

@@ -18,7 +18,7 @@ package cmd
 
 import (
 	"context"
-	"sync"
+	"errors"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
@@ -120,18 +120,14 @@ func mustGetHealSequence(ctx context.Context) *healSequence {
 }
 
 // healErasureSet lists and heals all objects in a specific erasure set
-func healErasureSet(ctx context.Context, setIndex int, buckets []BucketInfo, disks []StorageAPI) error {
+func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []BucketInfo) error {
 	bgSeq := mustGetHealSequence(ctx)
-
 	buckets = append(buckets, BucketInfo{
 		Name: pathJoin(minioMetaBucket, minioConfigPrefix),
 	})
 
 	// Try to pro-actively heal backend-encrypted file.
-	if err := bgSeq.queueHealTask(healSource{
-		bucket: minioMetaBucket,
-		object: backendEncryptedFile,
-	}, madmin.HealItemMetadata); err != nil {
+	if _, err := er.HealObject(ctx, minioMetaBucket, backendEncryptedFile, "", madmin.HealOpts{}); err != nil {
 		if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 			logger.LogIf(ctx, err)
 		}
@@ -140,61 +136,60 @@ func healErasureSet(ctx context.Context, setIndex int, buckets []BucketInfo, dis
 	// Heal all buckets with all objects
 	for _, bucket := range buckets {
 		// Heal current bucket
-		if err := bgSeq.queueHealTask(healSource{
-			bucket: bucket.Name,
-		}, madmin.HealItemBucket); err != nil {
+		if _, err := er.HealBucket(ctx, bucket.Name, madmin.HealOpts{}); err != nil {
 			if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 				logger.LogIf(ctx, err)
 			}
 		}
 
 		if serverDebugLog {
-			console.Debugf(color.Green("healDisk:")+" healing bucket %s content on erasure set %d\n", bucket.Name, setIndex+1)
+			console.Debugf(color.Green("healDisk:")+" healing bucket %s content on erasure set %d\n", bucket.Name, er.setNumber+1)
 		}
 
-		var entryChs []FileInfoVersionsCh
-		var mu sync.Mutex
-		var wg sync.WaitGroup
-		for _, disk := range disks {
-			disk := disk
-			wg.Add(1)
-			go func() {
-				defer wg.Done()
-				entryCh, err := disk.WalkVersions(ctx, bucket.Name, "", "", true, ctx.Done())
-				if err != nil {
-					// Disk walk returned error, ignore it.
-					return
-				}
-				mu.Lock()
-				entryChs = append(entryChs, FileInfoVersionsCh{
-					Ch: entryCh,
-				})
-				mu.Unlock()
-			}()
+		disks, _ := er.getOnlineDisksWithHealing()
+		if len(disks) == 0 {
+			return errors.New("healErasureSet: No non-healing disks found")
 		}
-		wg.Wait()
-
-		entriesValid := make([]bool, len(entryChs))
-		entries := make([]FileInfoVersions, len(entryChs))
-
-		for {
-			entry, _, ok := lexicallySortedEntryVersions(entryChs, entries, entriesValid)
-			if !ok {
-				break
+		// Limit listing to 3 drives.
+		if len(disks) > 3 {
+			disks = disks[:3]
+		}
+		healEntry := func(entry metaCacheEntry) {
+			if entry.isDir() {
+				return
 			}
-
-			for _, version := range entry.Versions {
-				if err := bgSeq.queueHealTask(healSource{
-					bucket:    bucket.Name,
-					object:    version.Name,
-					versionID: version.VersionID,
-				}, madmin.HealItemObject); err != nil {
+			fivs, err := entry.fileInfoVersions(bucket.Name)
+			if err != nil {
+				logger.LogIf(ctx, err)
+				return
+			}
+			waitForLowHTTPReq(globalHealConfig.IOCount, globalHealConfig.Sleep)
+			for _, version := range fivs.Versions {
+				if _, err := er.HealObject(ctx, bucket.Name, version.Name, version.VersionID, madmin.HealOpts{ScanMode: madmin.HealNormalScan, Remove: healDeleteDangling}); err != nil {
 					if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 						logger.LogIf(ctx, err)
 					}
 				}
+				bgSeq.logHeal(madmin.HealItemObject)
 			}
 		}
+		err := listPathRaw(ctx, listPathRawOptions{
+			disks:          disks,
+			bucket:         bucket.Name,
+			recursive:      true,
+			forwardTo:      "", //TODO(klauspost): Set this to last known offset when resuming.
+			minDisks:       1,
+			reportNotFound: false,
+			agreed:         healEntry,
+			partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+				entry, _ := entries.firstFound()
+				if entry != nil && !entry.isDir() {
+					healEntry(*entry)
+				}
+			},
+			finished: nil,
+		})
+		logger.LogIf(ctx, err)
 	}
 
 	return nil

cmd/handler-api.go

@@ -36,7 +36,8 @@ type apiConfig struct {
 	extendListLife   time.Duration
 	corsAllowOrigins []string
 	// total drives per erasure set across pools.
-	totalDriveCount int
+	totalDriveCount    int
+	replicationWorkers int
 }
 
 func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
@@ -59,7 +60,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		}
 		// max requests per node is calculated as
 		// total_ram / ram_per_request
-		// ram_per_request is 1MiB * driveCount + 2 * 10MiB (default erasure block size)
+		// ram_per_request is (2MiB+128KiB) * driveCount + 2 * 10MiB (default erasure block size)
 		apiRequestsMaxPerNode = int(stats.TotalRAM / uint64(t.totalDriveCount*(blockSizeLarge+blockSizeSmall)+blockSizeV1*2))
 	} else {
 		apiRequestsMaxPerNode = cfg.RequestsMax
@@ -78,6 +79,7 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.requestsDeadline = cfg.RequestsDeadline
 	t.listQuorum = cfg.GetListQuorum()
 	t.extendListLife = cfg.ExtendListLife
+	t.replicationWorkers = cfg.ReplicationWorkers
 }
 
 func (t *apiConfig) getListQuorum() int {
@@ -152,3 +154,10 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 		}
 	}
 }
+
+func (t *apiConfig) getReplicationWorkers() int {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	return t.replicationWorkers
+}

cmd/handler-utils.go

@@ -26,6 +26,7 @@ import (
 	"mime/multipart"
 	"net"
 	"net/http"
+	"net/textproto"
 	"net/url"
 	"regexp"
 	"strings"
@@ -74,6 +75,7 @@ var supportedHeaders = []string{
 	"content-language",
 	"content-encoding",
 	"content-disposition",
+	"x-amz-storage-class",
 	xhttp.AmzStorageClass,
 	xhttp.AmzObjectTagging,
 	"expires",
@@ -103,8 +105,6 @@ func isDirectiveReplace(value string) bool {
 // All values stored with a key starting with one of the following prefixes
 // must be extracted from the header.
 var userMetadataKeyPrefixes = []string{
-	"X-Amz-Meta-",
-	"X-Minio-Meta-",
 	"x-amz-meta-",
 	"x-minio-meta-",
 }
@@ -115,25 +115,25 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]
 	header := r.Header
 	metadata = make(map[string]string)
 	// Extract all query values.
-	err = extractMetadataFromMap(ctx, query, metadata)
+	err = extractMetadataFromMime(ctx, textproto.MIMEHeader(query), metadata)
 	if err != nil {
 		return nil, err
 	}
 
 	// Extract all header values.
-	err = extractMetadataFromMap(ctx, header, metadata)
+	err = extractMetadataFromMime(ctx, textproto.MIMEHeader(header), metadata)
 	if err != nil {
 		return nil, err
 	}
 
 	// Set content-type to default value if it is not set.
 	if _, ok := metadata[strings.ToLower(xhttp.ContentType)]; !ok {
-		metadata[strings.ToLower(xhttp.ContentType)] = "application/octet-stream"
+		metadata[strings.ToLower(xhttp.ContentType)] = "binary/octet-stream"
 	}
 
 	// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
 	for k := range metadata {
-		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+		if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 			delete(metadata, k)
 		}
 	}
@@ -161,25 +161,32 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]
 }
 
 // extractMetadata extracts metadata from map values.
-func extractMetadataFromMap(ctx context.Context, v map[string][]string, m map[string]string) error {
+func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error {
 	if v == nil {
 		logger.LogIf(ctx, errInvalidArgument)
 		return errInvalidArgument
 	}
+
+	nv := make(textproto.MIMEHeader, len(v))
+	for k, kv := range v {
+		// Canonicalize all headers, to remove any duplicates.
+		nv[http.CanonicalHeaderKey(k)] = kv
+	}
+
 	// Save all supported headers.
 	for _, supportedHeader := range supportedHeaders {
-		if value, ok := v[http.CanonicalHeaderKey(supportedHeader)]; ok {
-			m[supportedHeader] = value[0]
-		} else if value, ok := v[supportedHeader]; ok {
-			m[supportedHeader] = value[0]
+		value, ok := nv[http.CanonicalHeaderKey(supportedHeader)]
+		if ok {
+			m[supportedHeader] = strings.Join(value, ",")
 		}
 	}
+
 	for key := range v {
 		for _, prefix := range userMetadataKeyPrefixes {
 			if !strings.HasPrefix(strings.ToLower(key), strings.ToLower(prefix)) {
 				continue
 			}
-			value, ok := v[key]
+			value, ok := nv[http.CanonicalHeaderKey(key)]
 			if ok {
 				m[key] = strings.Join(value, ",")
 				break
@@ -412,20 +419,65 @@ func getResource(path string, host string, domains []string) (string, error) {
 	return path, nil
 }
 
-var regexVersion = regexp.MustCompile(`(\w\d+)`)
+var regexVersion = regexp.MustCompile(`^/minio.*/(v\d+)/.*`)
 
 func extractAPIVersion(r *http.Request) string {
-	return regexVersion.FindString(r.URL.Path)
+	if matches := regexVersion.FindStringSubmatch(r.URL.Path); len(matches) > 1 {
+		return matches[1]
+	}
+	return "unknown"
 }
 
 func methodNotAllowedHandler(api string) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
-		code := "XMinio" + api + "VersionMismatch"
-		writeErrorResponseString(r.Context(), w, APIError{
-			Code:           code,
-			Description:    "Not allowed (" + r.Method + " " + r.URL.String() + " on " + api + " API)",
-			HTTPStatusCode: http.StatusMethodNotAllowed,
-		}, r.URL)
+		if r.Method == http.MethodOptions {
+			return
+		}
+		version := extractAPIVersion(r)
+		switch {
+		case strings.HasPrefix(r.URL.Path, peerRESTPrefix):
+			desc := fmt.Sprintf("Server expects 'peer' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", peerRESTVersion, version, ReleaseTag)
+			writeErrorResponseString(r.Context(), w, APIError{
+				Code:           "XMinioPeerVersionMismatch",
+				Description:    desc,
+				HTTPStatusCode: http.StatusUpgradeRequired,
+			}, r.URL)
+		case strings.HasPrefix(r.URL.Path, storageRESTPrefix):
+			desc := fmt.Sprintf("Server expects 'storage' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", storageRESTVersion, version, ReleaseTag)
+			writeErrorResponseString(r.Context(), w, APIError{
+				Code:           "XMinioStorageVersionMismatch",
+				Description:    desc,
+				HTTPStatusCode: http.StatusUpgradeRequired,
+			}, r.URL)
+		case strings.HasPrefix(r.URL.Path, lockRESTPrefix):
+			desc := fmt.Sprintf("Server expects 'lock' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", lockRESTVersion, version, ReleaseTag)
+			writeErrorResponseString(r.Context(), w, APIError{
+				Code:           "XMinioLockVersionMismatch",
+				Description:    desc,
+				HTTPStatusCode: http.StatusUpgradeRequired,
+			}, r.URL)
+		case strings.HasPrefix(r.URL.Path, adminPathPrefix):
+			var desc string
+			if version == "v1" {
+				desc = fmt.Sprintf("Server expects client requests with 'admin' API version '%s', found '%s', please upgrade the client to latest releases", madmin.AdminAPIVersion, version)
+			} else if version == madmin.AdminAPIVersion {
+				desc = fmt.Sprintf("This 'admin' API is not supported by server in '%s'", getMinioMode())
+			} else {
+				desc = fmt.Sprintf("Unexpected client 'admin' API version found '%s', expected '%s', please downgrade the client to older releases", version, madmin.AdminAPIVersion)
+			}
+			writeErrorResponseJSON(r.Context(), w, APIError{
+				Code:           "XMinioAdminVersionMismatch",
+				Description:    desc,
+				HTTPStatusCode: http.StatusUpgradeRequired,
+			}, r.URL)
+		default:
+			desc := fmt.Sprintf("Unknown API request at %s", r.URL.Path)
+			writeErrorResponse(r.Context(), w, APIError{
+				Code:           "XMinioUnknownAPIRequest",
+				Description:    desc,
+				HTTPStatusCode: http.StatusBadRequest,
+			}, r.URL, guessIsBrowserReq(r))
+		}
 	}
 }
 
@@ -437,24 +489,21 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 	version := extractAPIVersion(r)
 	switch {
 	case strings.HasPrefix(r.URL.Path, peerRESTPrefix):
-		desc := fmt.Sprintf("Expected 'peer' API version '%s', instead found '%s', please upgrade the servers",
-			peerRESTVersion, version)
+		desc := fmt.Sprintf("Server expects 'peer' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", peerRESTVersion, version, ReleaseTag)
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioPeerVersionMismatch",
 			Description:    desc,
 			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, storageRESTPrefix):
-		desc := fmt.Sprintf("Expected 'storage' API version '%s', instead found '%s', please upgrade the servers",
-			storageRESTVersion, version)
+		desc := fmt.Sprintf("Server expects 'storage' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", storageRESTVersion, version, ReleaseTag)
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioStorageVersionMismatch",
 			Description:    desc,
 			HTTPStatusCode: http.StatusUpgradeRequired,
 		}, r.URL)
 	case strings.HasPrefix(r.URL.Path, lockRESTPrefix):
-		desc := fmt.Sprintf("Expected 'lock' API version '%s', instead found '%s', please upgrade the servers",
-			lockRESTVersion, version)
+		desc := fmt.Sprintf("Server expects 'lock' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", lockRESTVersion, version, ReleaseTag)
 		writeErrorResponseString(r.Context(), w, APIError{
 			Code:           "XMinioLockVersionMismatch",
 			Description:    desc,
@@ -482,6 +531,7 @@ func errorResponseHandler(w http.ResponseWriter, r *http.Request) {
 			HTTPStatusCode: http.StatusBadRequest,
 		}, r.URL, guessIsBrowserReq(r))
 	}
+
 }
 
 // gets host name for current node

cmd/handler-utils_test.go

@@ -22,6 +22,7 @@ import (
 	"encoding/xml"
 	"io/ioutil"
 	"net/http"
+	"net/textproto"
 	"os"
 	"reflect"
 	"strings"
@@ -197,7 +198,7 @@ func TestExtractMetadataHeaders(t *testing.T) {
 	// Validate if the extracting headers.
 	for i, testCase := range testCases {
 		metadata := make(map[string]string)
-		err := extractMetadataFromMap(context.Background(), testCase.header, metadata)
+		err := extractMetadataFromMime(context.Background(), textproto.MIMEHeader(testCase.header), metadata)
 		if err != nil && !testCase.shouldFail {
 			t.Fatalf("Test %d failed to extract metadata: %v", i+1, err)
 		}

cmd/healthcheck-handler.go

@@ -64,6 +64,29 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
+// ClusterReadCheckHandler returns if the server is ready for requests.
+func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ClusterReadCheckHandler")
+
+	if shouldProxy() {
+		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return
+	}
+
+	objLayer := newObjectLayerFn()
+
+	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline())
+	defer cancel()
+
+	result := objLayer.ReadHealth(ctx)
+	if !result {
+		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
+		return
+	}
+	writeResponse(w, http.StatusOK, nil, mimeNone)
+}
+
 // ReadinessCheckHandler Checks if the process is up. Always returns success.
 func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 	if shouldProxy() {

cmd/healthcheck-router.go

@@ -23,11 +23,12 @@ import (
 )
 
 const (
-	healthCheckPath          = "/health"
-	healthCheckLivenessPath  = "/live"
-	healthCheckReadinessPath = "/ready"
-	healthCheckClusterPath   = "/cluster"
-	healthCheckPathPrefix    = minioReservedBucketPath + healthCheckPath
+	healthCheckPath            = "/health"
+	healthCheckLivenessPath    = "/live"
+	healthCheckReadinessPath   = "/ready"
+	healthCheckClusterPath     = "/cluster"
+	healthCheckClusterReadPath = "/cluster/read"
+	healthCheckPathPrefix      = minioReservedBucketPath + healthCheckPath
 )
 
 // registerHealthCheckRouter - add handler functions for liveness and readiness routes.
@@ -38,6 +39,7 @@ func registerHealthCheckRouter(router *mux.Router) {
 
 	// Cluster check handler to verify cluster is active
 	healthRouter.Methods(http.MethodGet).Path(healthCheckClusterPath).HandlerFunc(httpTraceAll(ClusterCheckHandler))
+	healthRouter.Methods(http.MethodGet).Path(healthCheckClusterReadPath).HandlerFunc(httpTraceAll(ClusterReadCheckHandler))
 
 	// Liveness handler
 	healthRouter.Methods(http.MethodGet).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler))

cmd/healthinfo.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"os"
 	"sync"
@@ -41,7 +42,7 @@ func getLocalCPUInfo(ctx context.Context, r *http.Request) madmin.ServerCPUInfo
 	if err != nil {
 		return madmin.ServerCPUInfo{
 			Addr:  addr,
-			Error: err.Error(),
+			Error: fmt.Sprintf("info: %v", err),
 		}
 	}
 
@@ -49,7 +50,7 @@ func getLocalCPUInfo(ctx context.Context, r *http.Request) madmin.ServerCPUInfo
 	if err != nil {
 		return madmin.ServerCPUInfo{
 			Addr:  addr,
-			Error: err.Error(),
+			Error: fmt.Sprintf("times: %v", err),
 		}
 	}
 
@@ -72,7 +73,7 @@ func getLocalDrives(ctx context.Context, parallel bool, endpointServerPools Endp
 					// Since this drive is not available, add relevant details and proceed
 					drivesPerfInfo = append(drivesPerfInfo, madmin.DrivePerfInfo{
 						Path:  endpoint.Path,
-						Error: err.Error(),
+						Error: fmt.Sprintf("stat: %v", err),
 					})
 					continue
 				}
@@ -84,7 +85,7 @@ func getLocalDrives(ctx context.Context, parallel bool, endpointServerPools Endp
 					}
 					latency, throughput, err := disk.GetHealthInfo(ctx, path, pathJoin(path, measurePath))
 					if err != nil {
-						driveInfo.Error = err.Error()
+						driveInfo.Error = fmt.Sprintf("health-info: %v", err)
 					} else {
 						driveInfo.Latency = latency
 						driveInfo.Throughput = throughput
@@ -129,7 +130,7 @@ func getLocalMemInfo(ctx context.Context, r *http.Request) madmin.ServerMemInfo
 	if err != nil {
 		return madmin.ServerMemInfo{
 			Addr:  addr,
-			Error: err.Error(),
+			Error: fmt.Sprintf("swap: %v", err),
 		}
 	}
 
@@ -137,7 +138,7 @@ func getLocalMemInfo(ctx context.Context, r *http.Request) madmin.ServerMemInfo
 	if err != nil {
 		return madmin.ServerMemInfo{
 			Addr:  addr,
-			Error: err.Error(),
+			Error: fmt.Sprintf("virtual-mem: %v", err),
 		}
 	}
 
@@ -154,23 +155,20 @@ func getLocalProcInfo(ctx context.Context, r *http.Request) madmin.ServerProcInf
 		addr = GetLocalPeer(globalEndpoints)
 	}
 
-	errProcInfo := func(err error) madmin.ServerProcInfo {
+	errProcInfo := func(tag string, err error) madmin.ServerProcInfo {
 		return madmin.ServerProcInfo{
 			Addr:  addr,
-			Error: err.Error(),
+			Error: fmt.Sprintf("%s: %v", tag, err),
 		}
 	}
 
 	selfPid := int32(syscall.Getpid())
 	self, err := process.NewProcess(selfPid)
 	if err != nil {
-		return errProcInfo(err)
+		return errProcInfo("new-process", err)
 	}
 
 	processes := []*process.Process{self}
-	if err != nil {
-		return errProcInfo(err)
-	}
 
 	sysProcs := []madmin.SysProcess{}
 	for _, proc := range processes {
@@ -179,13 +177,13 @@ func getLocalProcInfo(ctx context.Context, r *http.Request) madmin.ServerProcInf
 
 		bg, err := proc.BackgroundWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("background", err)
 		}
 		sysProc.Background = bg
 
 		cpuPercent, err := proc.CPUPercentWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("cpu-percent", err)
 		}
 		sysProc.CPUPercent = cpuPercent
 
@@ -196,109 +194,109 @@ func getLocalProcInfo(ctx context.Context, r *http.Request) madmin.ServerProcInf
 		}
 		cmdLine, err := proc.CmdlineWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("cmdline", err)
 		}
 		sysProc.CmdLine = cmdLine
 
 		conns, err := proc.ConnectionsWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("conns", err)
 		}
-		sysProc.Connections = conns
+		sysProc.ConnectionCount = len(conns)
 
 		createTime, err := proc.CreateTimeWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("create-time", err)
 		}
 		sysProc.CreateTime = createTime
 
 		cwd, err := proc.CwdWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("cwd", err)
 		}
 		sysProc.Cwd = cwd
 
 		exe, err := proc.ExeWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("exe", err)
 		}
 		sysProc.Exe = exe
 
 		gids, err := proc.GidsWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("gids", err)
 		}
 		sysProc.Gids = gids
 
 		ioCounters, err := proc.IOCountersWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("iocounters", err)
 		}
 		sysProc.IOCounters = ioCounters
 
 		isRunning, err := proc.IsRunningWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("is-running", err)
 		}
 		sysProc.IsRunning = isRunning
 
 		memInfo, err := proc.MemoryInfoWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("mem-info", err)
 		}
 		sysProc.MemInfo = memInfo
 
 		memMaps, err := proc.MemoryMapsWithContext(ctx, true)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("mem-maps", err)
 		}
 		sysProc.MemMaps = memMaps
 
 		memPercent, err := proc.MemoryPercentWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("mem-percent", err)
 		}
 		sysProc.MemPercent = memPercent
 
 		name, err := proc.NameWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("name", err)
 		}
 		sysProc.Name = name
 
 		netIOCounters, err := proc.NetIOCountersWithContext(ctx, false)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("netio-counters", err)
 		}
 		sysProc.NetIOCounters = netIOCounters
 
 		nice, err := proc.NiceWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("nice", err)
 		}
 		sysProc.Nice = nice
 
 		numCtxSwitches, err := proc.NumCtxSwitchesWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("num-ctx-switches", err)
 		}
 		sysProc.NumCtxSwitches = numCtxSwitches
 
 		numFds, err := proc.NumFDsWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("num-fds", err)
 		}
 		sysProc.NumFds = numFds
 
 		numThreads, err := proc.NumThreadsWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("num-threads", err)
 		}
 		sysProc.NumThreads = numThreads
 
 		pageFaults, err := proc.PageFaultsWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("page-faults", err)
 		}
 		sysProc.PageFaults = pageFaults
 
@@ -314,37 +312,37 @@ func getLocalProcInfo(ctx context.Context, r *http.Request) madmin.ServerProcInf
 
 		rlimit, err := proc.RlimitWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("rlimit", err)
 		}
 		sysProc.Rlimit = rlimit
 
 		status, err := proc.StatusWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("status", err)
 		}
 		sysProc.Status = status
 
 		tgid, err := proc.Tgid()
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("tgid", err)
 		}
 		sysProc.Tgid = tgid
 
 		times, err := proc.TimesWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("times", err)
 		}
 		sysProc.Times = times
 
 		uids, err := proc.UidsWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("uids", err)
 		}
 		sysProc.Uids = uids
 
 		username, err := proc.UsernameWithContext(ctx)
 		if err != nil {
-			return errProcInfo(err)
+			return errProcInfo("username", err)
 		}
 		sysProc.Username = username
 

cmd/healthinfo_linux.go

@@ -21,9 +21,9 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"strings"
-	"syscall"
 
 	"github.com/minio/minio/pkg/madmin"
 	"github.com/minio/minio/pkg/smart"
@@ -37,31 +37,29 @@ func getLocalOsInfo(ctx context.Context, r *http.Request) madmin.ServerOsInfo {
 		addr = GetLocalPeer(globalEndpoints)
 	}
 
-	info, err := host.InfoWithContext(ctx)
+	srvrOsInfo := madmin.ServerOsInfo{Addr: addr}
+	var err error
+
+	srvrOsInfo.Info, err = host.InfoWithContext(ctx)
 	if err != nil {
 		return madmin.ServerOsInfo{
 			Addr:  addr,
-			Error: err.Error(),
+			Error: fmt.Sprintf("info: %v", err),
 		}
 	}
 
-	sensors, err := host.SensorsTemperaturesWithContext(ctx)
+	srvrOsInfo.Sensors, err = host.SensorsTemperaturesWithContext(ctx)
 	if err != nil {
-		return madmin.ServerOsInfo{
-			Addr:  addr,
-			Error: err.Error(),
+		// Set error only when it's not of WARNINGS type
+		if _, isWarning := err.(*host.Warnings); !isWarning {
+			srvrOsInfo.Error = fmt.Sprintf("sensors-temp: %v", err)
 		}
 	}
 
 	// ignore user err, as it cannot be obtained reliably inside containers
-	users, _ := host.UsersWithContext(ctx)
+	srvrOsInfo.Users, _ = host.UsersWithContext(ctx)
 
-	return madmin.ServerOsInfo{
-		Addr:    addr,
-		Info:    info,
-		Sensors: sensors,
-		Users:   users,
-	}
+	return srvrOsInfo
 }
 
 func getLocalDiskHwInfo(ctx context.Context, r *http.Request) madmin.ServerDiskHwInfo {
@@ -74,7 +72,7 @@ func getLocalDiskHwInfo(ctx context.Context, r *http.Request) madmin.ServerDiskH
 	if err != nil {
 		return madmin.ServerDiskHwInfo{
 			Addr:  addr,
-			Error: err.Error(),
+			Error: fmt.Sprintf("partitions: %v", err),
 		}
 	}
 
@@ -98,14 +96,7 @@ func getLocalDiskHwInfo(ctx context.Context, r *http.Request) madmin.ServerDiskH
 			paths = append(paths, path)
 			smartInfo, err := smart.GetInfo(device)
 			if err != nil {
-				if syscall.EACCES == err {
-					smartInfo.Error = err.Error()
-				} else {
-					return madmin.ServerDiskHwInfo{
-						Addr:  addr,
-						Error: err.Error(),
-					}
-				}
+				smartInfo.Error = fmt.Sprintf("smart: %v", err)
 			}
 			partition := madmin.PartitionStat{
 				Device:     part.Device,
@@ -122,7 +113,7 @@ func getLocalDiskHwInfo(ctx context.Context, r *http.Request) madmin.ServerDiskH
 	if err != nil {
 		return madmin.ServerDiskHwInfo{
 			Addr:  addr,
-			Error: err.Error(),
+			Error: fmt.Sprintf("iocounters: %v", err),
 		}
 	}
 	usages := []*diskhw.UsageStat{}
@@ -131,7 +122,7 @@ func getLocalDiskHwInfo(ctx context.Context, r *http.Request) madmin.ServerDiskH
 		if err != nil {
 			return madmin.ServerDiskHwInfo{
 				Addr:  addr,
-				Error: err.Error(),
+				Error: fmt.Sprintf("usage: %v", err),
 			}
 		}
 		usages = append(usages, usage)

cmd/http/dial_dnscache.go

@@ -44,7 +44,6 @@ func DialContextWithDNSCache(cache *DNSCache, baseDialCtx DialContext) DialConte
 		baseDialCtx = (&net.Dialer{
 			Timeout:   30 * time.Second,
 			KeepAlive: 30 * time.Second,
-			DualStack: true,
 		}).DialContext
 	}
 	return func(ctx context.Context, network, host string) (net.Conn, error) {

cmd/http/dial_linux.go

@@ -38,6 +38,10 @@ func setInternalTCPParameters(c syscall.RawConn) error {
 		// since Linux 4.11.
 		_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_FASTOPEN_CONNECT, 1)
 
+		// Enable TCP quick ACK, John Nagle says
+		// "Set TCP_QUICKACK. If you find a case where that makes things worse, let me know."
+		_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_QUICKACK, 1)
+
 		// The time (in seconds) the connection needs to remain idle before
 		// TCP starts sending keepalive probes, set this to 5 secs
 		// system defaults to 7200 secs!!!
@@ -52,6 +56,7 @@ func setInternalTCPParameters(c syscall.RawConn) error {
 		// ~ cat /proc/sys/net/ipv4/tcp_keepalive_intvl (defaults to 75 secs, we reduce it to 3 secs)
 		// 75
 		_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 3)
+
 	})
 }
 
@@ -86,6 +91,10 @@ func NewCustomDialContext(dialTimeout time.Duration) DialContext {
 					// the TCP fast open connect. This feature is supported
 					// since Linux 4.11.
 					_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_FASTOPEN_CONNECT, 1)
+
+					// Enable TCP quick ACK, John Nagle says
+					// "Set TCP_QUICKACK. If you find a case where that makes things worse, let me know."
+					_ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_QUICKACK, 1)
 				})
 			},
 		}

cmd/http/server.go

@@ -191,7 +191,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 			// TLS hardening
 			PreferServerCipherSuites: true,
 			MinVersion:               tls.VersionTLS12,
-			NextProtos:               []string{"h2", "http/1.1"},
+			NextProtos:               []string{"http/1.1", "h2"},
 		}
 		tlsConfig.GetCertificate = getCert
 	}

cmd/iam.go

@@ -1703,27 +1703,41 @@ func (sys *IAMSys) PolicyDBGet(name string, isGroup bool) ([]string, error) {
 // This call assumes that caller has the sys.RLock()
 func (sys *IAMSys) policyDBGet(name string, isGroup bool) ([]string, error) {
 	if isGroup {
-		if _, ok := sys.iamGroupsMap[name]; !ok {
+		g, ok := sys.iamGroupsMap[name]
+		if !ok {
 			return nil, errNoSuchGroup
 		}
 
+		// Group is disabled, so we return no policy - this
+		// ensures the request is denied.
+		if g.Status == statusDisabled {
+			return nil, nil
+		}
+
 		mp := sys.iamGroupPolicyMap[name]
 		return mp.toSlice(), nil
 	}
 
 	// When looking for a user's policies, we also check if the
 	// user and the groups they are member of are enabled.
-	if u, ok := sys.iamUsersMap[name]; !ok {
+	u, ok := sys.iamUsersMap[name]
+	if !ok {
 		return nil, errNoSuchUser
-	} else if u.Status == statusDisabled {
-		// User is disabled, so we return no policy - this
-		// ensures the request is denied.
+	}
+
+	if !u.IsValid() {
 		return nil, nil
 	}
 
 	var policies []string
 
-	mp := sys.iamUserPolicyMap[name]
+	mp, ok := sys.iamUserPolicyMap[name]
+	if !ok {
+		if u.ParentUser != "" {
+			mp = sys.iamUserPolicyMap[u.ParentUser]
+		}
+	}
+
 	// returned policy could be empty
 	policies = append(policies, mp.toSlice()...)
 

cmd/local-locker.go

@@ -249,7 +249,10 @@ func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired
 		// Lock found, proceed to verify if belongs to given uid.
 		lri, ok := l.lockMap[resource]
 		if !ok {
-			return true, nil
+			// lock doesn't exist yet not reason to
+			// expire that doesn't exist yet - it may be
+			// racing with other active lock requests.
+			return false, nil
 		}
 
 		// Check whether uid is still active
@@ -258,7 +261,7 @@ func (l *localLocker) Expired(ctx context.Context, args dsync.LockArgs) (expired
 				ep := globalRemoteEndpoints[args.Owner]
 				if !ep.IsLocal {
 					// check if the owner is online
-					return isServerResolvable(ep, 250*time.Millisecond) != nil, nil
+					return isServerResolvable(ep, 3*time.Second) != nil, nil
 				}
 				return false, nil
 			}

cmd/lock-rest-server.go

@@ -33,7 +33,7 @@ import (
 
 const (
 	// Lock maintenance interval.
-	lockMaintenanceInterval = 10 * time.Second
+	lockMaintenanceInterval = 15 * time.Second
 
 	// Lock validity check interval.
 	lockValidityCheckInterval = 30 * time.Second
@@ -254,15 +254,15 @@ func getLongLivedLocks(interval time.Duration) []lockRequesterInfo {
 // - some network error (and server is up normally)
 //
 // We will ignore the error, and we will retry later to get a resolve on this lock
-func lockMaintenance(ctx context.Context, interval time.Duration) error {
+func lockMaintenance(ctx context.Context, interval time.Duration) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
-		return nil
+		return
 	}
 
 	z, ok := objAPI.(*erasureServerPools)
 	if !ok {
-		return nil
+		return
 	}
 
 	type nlock struct {
@@ -295,14 +295,15 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 		if lrip.Group {
 			lockers, _ = z.serverPools[0].getHashedSet("").getLockers()
 		} else {
-			lockers, _ = z.serverPools[0].getHashedSet(lrip.Name).getLockers()
+			_, objName := path2BucketObject(lrip.Name)
+			lockers, _ = z.serverPools[0].getHashedSet(objName).getLockers()
 		}
 		var wg sync.WaitGroup
 		wg.Add(len(lockers))
 		for _, c := range lockers {
 			go func(lrip lockRequesterInfo, c dsync.NetLocker) {
 				defer wg.Done()
-				ctx, cancel := context.WithTimeout(GlobalContext, 3*time.Second)
+				ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
 
 				// Call back to all participating servers, verify
 				// if each of those servers think lock is still
@@ -336,8 +337,6 @@ func lockMaintenance(ctx context.Context, interval time.Duration) error {
 			globalLockServer.removeEntryIfExists(lrip)
 		}
 	}
-
-	return nil
 }
 
 // Start lock maintenance from all lock servers.
@@ -354,27 +353,28 @@ func startLockMaintenance(ctx context.Context) {
 		break
 	}
 
-	// Initialize a new ticker with a minute between each ticks.
-	ticker := time.NewTicker(lockMaintenanceInterval)
-	// Stop the timer upon service closure and cleanup the go-routine.
-	defer ticker.Stop()
+	// Initialize a new ticker with 15secs between each ticks.
+	lkTimer := time.NewTimer(lockMaintenanceInterval)
+	// Stop the timer upon returning.
+	defer lkTimer.Stop()
 
 	r := rand.New(rand.NewSource(UTCNow().UnixNano()))
+
+	// Start with random sleep time, so as to avoid
+	// "synchronous checks" between servers
+	duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
+	time.Sleep(duration)
+
 	for {
 		// Verifies every minute for locks held more than 2 minutes.
 		select {
 		case <-ctx.Done():
 			return
-		case <-ticker.C:
-			// Start with random sleep time, so as to avoid
-			// "synchronous checks" between servers
-			duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
-			time.Sleep(duration)
-			if err := lockMaintenance(ctx, lockValidityCheckInterval); err != nil {
-				// Sleep right after an error.
-				duration := time.Duration(r.Float64() * float64(lockMaintenanceInterval))
-				time.Sleep(duration)
-			}
+		case <-lkTimer.C:
+			// Reset the timer for next cycle.
+			lkTimer.Reset(time.Duration(r.Float64() * float64(lockMaintenanceInterval)))
+
+			lockMaintenance(ctx, lockValidityCheckInterval)
 		}
 	}
 }

cmd/logger/config.go

@@ -56,12 +56,6 @@ const (
 	EnvAuditWebhookAuthToken = "MINIO_AUDIT_WEBHOOK_AUTH_TOKEN"
 )
 
-// Inject into config package.
-func init() {
-	config.Logger.Info = Info
-	config.Logger.LogIf = LogIf
-}
-
 // Default KVS for loggerHTTP and loggerAuditHTTP
 var (
 	DefaultKVS = config.KVS{

cmd/metacache-bucket.go

@@ -21,9 +21,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"io"
+	"net/http"
 	"path"
 	"runtime/debug"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -63,7 +64,7 @@ func newBucketMetacache(bucket string, cleanup bool) *bucketMetacache {
 		ez, ok := objAPI.(*erasureServerPools)
 		if ok {
 			ctx := context.Background()
-			ez.deleteAll(ctx, minioMetaBucket, metacachePrefixForID(bucket, slashSeparator))
+			ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(bucket, slashSeparator))
 		}
 	}
 	return &bucketMetacache{
@@ -95,25 +96,19 @@ func loadBucketMetaCache(ctx context.Context, bucket string) (*bucketMetacache,
 			logger.LogIf(ctx, fmt.Errorf("loadBucketMetaCache: object layer not ready. bucket: %q", bucket))
 		}
 	}
+
 	var meta bucketMetacache
 	var decErr error
-	var wg sync.WaitGroup
-	wg.Add(1)
-
-	r, w := io.Pipe()
-	go func() {
-		defer wg.Done()
+	// Use global context for this.
+	r, err := objAPI.GetObjectNInfo(GlobalContext, minioMetaBucket, pathJoin("buckets", bucket, ".metacache", "index.s2"), nil, http.Header{}, readLock, ObjectOptions{})
+	if err == nil {
 		dec := s2DecPool.Get().(*s2.Reader)
 		dec.Reset(r)
 		decErr = meta.DecodeMsg(msgp.NewReader(dec))
 		dec.Reset(nil)
+		r.Close()
 		s2DecPool.Put(dec)
-		r.CloseWithError(decErr)
-	}()
-	// Use global context for this.
-	err := objAPI.GetObject(GlobalContext, minioMetaBucket, pathJoin("buckets", bucket, ".metacache", "index.s2"), 0, -1, w, "", ObjectOptions{})
-	logger.LogIf(ctx, w.CloseWithError(err))
-	wg.Wait()
+	}
 	if err != nil {
 		switch err.(type) {
 		case ObjectNotFound:
@@ -185,7 +180,7 @@ func (b *bucketMetacache) save(ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	_, err = objAPI.PutObject(ctx, minioMetaBucket, pathJoin("buckets", b.bucket, ".metacache", "index.s2"), NewPutObjReader(hr, nil, nil), ObjectOptions{})
+	_, err = objAPI.PutObject(ctx, minioMetaBucket, pathJoin("buckets", b.bucket, ".metacache", "index.s2"), NewPutObjReader(hr), ObjectOptions{})
 	logger.LogIf(ctx, err)
 	return err
 }
@@ -297,7 +292,7 @@ func (b *bucketMetacache) cleanup() {
 	caches, rootIdx := b.cloneCaches()
 
 	for id, cache := range caches {
-		if b.transient && time.Since(cache.lastUpdate) > 15*time.Minute && time.Since(cache.lastHandout) > 15*time.Minute {
+		if b.transient && time.Since(cache.lastUpdate) > 10*time.Minute && time.Since(cache.lastHandout) > 10*time.Minute {
 			// Keep transient caches only for 15 minutes.
 			remove[id] = struct{}{}
 			continue
@@ -350,6 +345,29 @@ func (b *bucketMetacache) cleanup() {
 		}
 	}
 
+	// If above limit, remove the caches with the oldest handout time.
+	if len(caches)-len(remove) > metacacheMaxEntries {
+		remainCaches := make([]metacache, 0, len(caches)-len(remove))
+		for id, cache := range caches {
+			if _, ok := remove[id]; ok {
+				continue
+			}
+			remainCaches = append(remainCaches, cache)
+		}
+		if len(remainCaches) > metacacheMaxEntries {
+			// Sort oldest last...
+			sort.Slice(remainCaches, func(i, j int) bool {
+				return remainCaches[i].lastHandout.Before(remainCaches[j].lastHandout)
+			})
+			// Keep first metacacheMaxEntries...
+			for _, cache := range remainCaches[metacacheMaxEntries:] {
+				if time.Since(cache.lastHandout) > 30*time.Minute {
+					remove[cache.id] = struct{}{}
+				}
+			}
+		}
+	}
+
 	for id := range remove {
 		b.deleteCache(id)
 	}
@@ -391,7 +409,6 @@ func (b *bucketMetacache) updateCacheEntry(update metacache) (metacache, error)
 	defer b.mu.Unlock()
 	existing, ok := b.caches[update.id]
 	if !ok {
-		logger.Info("updateCacheEntry: bucket %s list id %v not found", b.bucket, update.id)
 		return update, errFileNotFound
 	}
 	existing.update(update)
@@ -447,7 +464,7 @@ func (b *bucketMetacache) deleteAll() {
 	b.updated = true
 	if !b.transient {
 		// Delete all.
-		ez.deleteAll(ctx, minioMetaBucket, metacachePrefixForID(b.bucket, slashSeparator))
+		ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(b.bucket, slashSeparator))
 		b.caches = make(map[string]metacache, 10)
 		b.cachesRoot = make(map[string][]string, 10)
 		return
@@ -459,7 +476,7 @@ func (b *bucketMetacache) deleteAll() {
 		wg.Add(1)
 		go func(cache metacache) {
 			defer wg.Done()
-			ez.deleteAll(ctx, minioMetaBucket, metacachePrefixForID(cache.bucket, cache.id))
+			ez.renameAll(ctx, minioMetaBucket, metacachePrefixForID(cache.bucket, cache.id))
 		}(b.caches[id])
 	}
 	wg.Wait()

cmd/metacache-entries.go

@@ -330,16 +330,23 @@ func (m *metaCacheEntriesSorted) fileInfoVersions(bucket, prefix, delimiter, aft
 			}
 
 			fiv, err := entry.fileInfoVersions(bucket)
+			if err != nil {
+				continue
+			}
+
+			fiVersions := fiv.Versions
 			if afterV != "" {
-				// Forward first entry to specified version
-				fiv.forwardPastVersion(afterV)
+				vidMarkerIdx := fiv.findVersionIndex(afterV)
+				if vidMarkerIdx >= 0 {
+					fiVersions = fiVersions[vidMarkerIdx+1:]
+				}
 				afterV = ""
 			}
-			if err == nil {
-				for _, version := range fiv.Versions {
-					versions = append(versions, version.ToObjectInfo(bucket, entry.name))
-				}
+
+			for _, version := range fiVersions {
+				versions = append(versions, version.ToObjectInfo(bucket, entry.name))
 			}
+
 			continue
 		}
 

cmd/metacache-manager.go

@@ -43,6 +43,7 @@ type metacacheManager struct {
 }
 
 const metacacheManagerTransientBucket = "**transient**"
+const metacacheMaxEntries = 5000
 
 // initManager will start async saving the cache.
 func (m *metacacheManager) initManager() {
@@ -64,6 +65,8 @@ func (m *metacacheManager) initManager() {
 		}
 
 		t := time.NewTicker(time.Minute)
+		defer t.Stop()
+
 		var exit bool
 		bg := context.Background()
 		for !exit {
@@ -89,7 +92,6 @@ func (m *metacacheManager) initManager() {
 			}
 			m.mu.Unlock()
 		}
-		m.getTransient().deleteAll()
 	}()
 }
 
@@ -121,11 +123,11 @@ func (m *metacacheManager) updateCacheEntry(update metacache) (metacache, error)
 	}
 
 	b, ok := m.buckets[update.bucket]
+	m.mu.RUnlock()
 	if ok {
-		m.mu.RUnlock()
 		return b.updateCacheEntry(update)
 	}
-	m.mu.RUnlock()
+
 	// We should have either a trashed bucket or this
 	return metacache{}, errVolumeNotFound
 }

cmd/metacache-server-pool.go

@@ -19,7 +19,9 @@ package cmd
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
+	"os"
 	"path"
 	"strings"
 	"sync"
@@ -28,6 +30,24 @@ import (
 	"github.com/minio/minio/cmd/logger"
 )
 
+func renameAllBucketMetacache(epPath string) error {
+	// Rename all previous `.minio.sys/buckets/<bucketname>/.metacache` to
+	// to `.minio.sys/tmp/` for deletion.
+	return readDirFn(pathJoin(epPath, minioMetaBucket, bucketMetaPrefix), func(name string, typ os.FileMode) error {
+		if typ == os.ModeDir {
+			tmpMetacacheOld := pathJoin(epPath, minioMetaTmpBucket+"-old", mustGetUUID())
+			if err := renameAll(pathJoin(epPath, minioMetaBucket, metacachePrefixForID(name, slashSeparator)),
+				tmpMetacacheOld); err != nil && err != errFileNotFound {
+				return fmt.Errorf("unable to rename (%s -> %s) %w",
+					pathJoin(epPath, minioMetaBucket+metacachePrefixForID(minioMetaBucket, slashSeparator)),
+					tmpMetacacheOld,
+					osErrToFileErr(err))
+			}
+		}
+		return nil
+	})
+}
+
 // listPath will return the requested entries.
 // If no more entries are in the listing io.EOF is returned,
 // otherwise nil or an unexpected error is returned.

cmd/metacache-set.go

@@ -675,7 +675,7 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions) (entr
 				r, err := hash.NewReader(bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)), false)
 				logger.LogIf(ctx, err)
 				custom := b.headerKV()
-				_, err = er.putObject(ctx, minioMetaBucket, o.objectPath(b.n), NewPutObjReader(r, nil, nil), ObjectOptions{
+				_, err = er.putObject(ctx, minioMetaBucket, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{
 					UserDefined:    custom,
 					NoLock:         true, // No need to hold namespace lock, each prefix caches uniquely.
 					ParentIsObject: nil,
@@ -782,7 +782,13 @@ type listPathRawOptions struct {
 	disks        []StorageAPI
 	bucket, path string
 	recursive    bool
+
+	// Only return results with this prefix.
 	filterPrefix string
+
+	// Forward to this prefix before returning results.
+	forwardTo string
+
 	// Minimum number of good disks to continue.
 	// An error will be returned if this many disks returned an error.
 	minDisks       int
@@ -836,7 +842,9 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 				BaseDir:        opts.path,
 				Recursive:      opts.recursive,
 				ReportNotFound: opts.reportNotFound,
-				FilterPrefix:   opts.filterPrefix}, w)
+				FilterPrefix:   opts.filterPrefix,
+				ForwardTo:      opts.forwardTo,
+			}, w)
 			w.CloseWithError(werr)
 			if werr != io.EOF && werr != nil && werr.Error() != errFileNotFound.Error() && werr.Error() != errVolumeNotFound.Error() {
 				logger.LogIf(ctx, werr)

cmd/metacache-walk.go

@@ -49,6 +49,9 @@ type WalkDirOptions struct {
 	// FilterPrefix will only return results with given prefix within folder.
 	// Should never contain a slash.
 	FilterPrefix string
+
+	// ForwardTo will forward to the given object path.
+	ForwardTo string
 }
 
 // WalkDir will traverse a directory and return all entries found.
@@ -66,7 +69,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	}
 
 	// Stat a volume entry.
-	_, err = os.Stat(volumeDir)
+	_, err = os.Lstat(volumeDir)
 	if err != nil {
 		if osIsNotExist(err) {
 			return errVolumeNotFound
@@ -76,13 +79,6 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		return err
 	}
 
-	// Fast exit track to check if we are listing an object with
-	// a trailing slash, this will avoid to list the object content.
-	if HasSuffix(opts.BaseDir, SlashSeparator) {
-		if st, err := os.Stat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile)); err == nil && st.Mode().IsRegular() {
-			return errFileNotFound
-		}
-	}
 	// Use a small block size to start sending quickly
 	w := newMetacacheWriter(wr, 16<<10)
 	defer w.Close()
@@ -92,7 +88,29 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	}
 	defer close(out)
 
+	// Fast exit track to check if we are listing an object with
+	// a trailing slash, this will avoid to list the object content.
+	if HasSuffix(opts.BaseDir, SlashSeparator) {
+		metadata, err := ioutil.ReadFile(pathJoin(volumeDir,
+			opts.BaseDir[:len(opts.BaseDir)-1]+globalDirSuffix,
+			xlStorageFormatFile))
+		if err == nil {
+			// if baseDir is already a directory object, consider it
+			// as part of the list call, this is a AWS S3 specific
+			// behavior.
+			out <- metaCacheEntry{
+				name:     opts.BaseDir,
+				metadata: metadata,
+			}
+		} else {
+			if st, err := os.Lstat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile)); err == nil && st.Mode().IsRegular() {
+				return errFileNotFound
+			}
+		}
+	}
+
 	prefix := opts.FilterPrefix
+	forward := opts.ForwardTo
 	var scanDir func(path string) error
 	scanDir = func(current string) error {
 		entries, err := s.ListDir(ctx, opts.Bucket, current, -1)
@@ -112,6 +130,9 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			if len(prefix) > 0 && !strings.HasPrefix(entry, prefix) {
 				continue
 			}
+			if len(forward) > 0 && entry < forward {
+				continue
+			}
 			if strings.HasSuffix(entry, slashSeparator) {
 				if strings.HasSuffix(entry, globalDirSuffixWithSlash) {
 					// Add without extension so it is sorted correctly.
@@ -163,6 +184,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		sort.Strings(entries)
 		dirStack := make([]string, 0, 5)
 		prefix = "" // Remove prefix after first level.
+
 		for _, entry := range entries {
 			if entry == "" {
 				continue
@@ -175,8 +197,11 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				out <- metaCacheEntry{name: pop}
 				if opts.Recursive {
 					// Scan folder we found. Should be in correct sort order where we are.
-					err := scanDir(pop)
-					logger.LogIf(ctx, err)
+					forward = ""
+					if len(opts.ForwardTo) > 0 && strings.HasPrefix(opts.ForwardTo, pop) {
+						forward = strings.TrimPrefix(opts.ForwardTo, pop)
+					}
+					logger.LogIf(ctx, scanDir(pop))
 				}
 				dirStack = dirStack[:len(dirStack)-1]
 			}
@@ -209,7 +234,9 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				// NOT an object, append to stack (with slash)
 				// If dirObject, but no metadata (which is unexpected) we skip it.
 				if !isDirObj {
-					dirStack = append(dirStack, meta.name+slashSeparator)
+					if !isDirEmpty(pathJoin(volumeDir, meta.name+slashSeparator)) {
+						dirStack = append(dirStack, meta.name+slashSeparator)
+					}
 				}
 			case isSysErrNotDir(err):
 				// skip
@@ -223,8 +250,11 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			out <- metaCacheEntry{name: pop}
 			if opts.Recursive {
 				// Scan folder we found. Should be in correct sort order where we are.
-				err := scanDir(pop)
-				logger.LogIf(ctx, err)
+				forward = ""
+				if len(opts.ForwardTo) > 0 && strings.HasPrefix(opts.ForwardTo, pop) {
+					forward = strings.TrimPrefix(opts.ForwardTo, pop)
+				}
+				logger.LogIf(ctx, scanDir(pop))
 			}
 			dirStack = dirStack[:len(dirStack)-1]
 		}
@@ -251,6 +281,7 @@ func (client *storageRESTClient) WalkDir(ctx context.Context, opts WalkDirOption
 	values.Set(storageRESTRecursive, strconv.FormatBool(opts.Recursive))
 	values.Set(storageRESTReportNotFound, strconv.FormatBool(opts.ReportNotFound))
 	values.Set(storageRESTPrefixFilter, opts.FilterPrefix)
+	values.Set(storageRESTForwardFilter, opts.ForwardTo)
 	respBody, err := client.call(ctx, storageRESTMethodWalkDir, values, nil, -1)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -283,6 +314,7 @@ func (s *storageRESTServer) WalkDirHandler(w http.ResponseWriter, r *http.Reques
 	}
 
 	prefix := r.URL.Query().Get(storageRESTPrefixFilter)
+	forward := r.URL.Query().Get(storageRESTForwardFilter)
 	writer := streamHTTPResponse(w)
 	writer.CloseWithError(s.storage.WalkDir(r.Context(), WalkDirOptions{
 		Bucket:         volume,
@@ -290,5 +322,6 @@ func (s *storageRESTServer) WalkDirHandler(w http.ResponseWriter, r *http.Reques
 		Recursive:      recursive,
 		ReportNotFound: reportNotFound,
 		FilterPrefix:   prefix,
+		ForwardTo:      forward,
 	}, writer))
 }

cmd/metacache.go

@@ -123,7 +123,7 @@ func (m *metacache) matches(o *listPathOptions, extend time.Duration) bool {
 		}
 		if time.Since(m.lastUpdate) > metacacheMaxRunningAge+extend {
 			// Cache ended within bloom cycle, but we can extend the life.
-			o.debugf("cache %s ended (%v) and beyond extended life (%v)", m.id, m.lastUpdate, extend+metacacheMaxRunningAge)
+			o.debugf("cache %s ended (%v) and beyond extended life (%v)", m.id, m.lastUpdate, metacacheMaxRunningAge+extend)
 			return false
 		}
 	}
@@ -145,14 +145,14 @@ func (m *metacache) worthKeeping(currentCycle uint64) bool {
 		// Cycle is somehow bigger.
 		return false
 	case cache.finished() && time.Since(cache.lastHandout) > 48*time.Hour:
-		// Keep only for 2 days. Fallback if crawler is clogged.
+		// Keep only for 2 days. Fallback if scanner is clogged.
 		return false
 	case cache.finished() && currentCycle >= dataUsageUpdateDirCycles && cache.startedCycle < currentCycle-dataUsageUpdateDirCycles:
 		// Cycle is too old to be valuable.
 		return false
 	case cache.status == scanStateError || cache.status == scanStateNone:
-		// Remove failed listings after 10 minutes.
-		return time.Since(cache.lastUpdate) < 10*time.Minute
+		// Remove failed listings after 5 minutes.
+		return time.Since(cache.lastUpdate) < 5*time.Minute
 	}
 	return true
 }
@@ -170,8 +170,9 @@ func (m *metacache) canBeReplacedBy(other *metacache) bool {
 	if m.status == scanStateStarted && time.Since(m.lastUpdate) < metacacheMaxRunningAge {
 		return false
 	}
+
 	// Keep it around a bit longer.
-	if time.Since(m.lastHandout) < time.Hour || time.Since(m.lastUpdate) < metacacheMaxRunningAge {
+	if time.Since(m.lastHandout) < 30*time.Minute || time.Since(m.lastUpdate) < metacacheMaxRunningAge {
 		return false
 	}
 

cmd/metrics-v2.go

@@ -53,18 +53,19 @@ const (
 	capacityRawSubsystem    MetricSubsystem = "capacity_raw"
 	capacityUsableSubsystem MetricSubsystem = "capacity_usable"
 	diskSubsystem           MetricSubsystem = "disk"
+	fileDescriptorSubsystem MetricSubsystem = "file_descriptor"
 	goRoutines              MetricSubsystem = "go_routine"
+	ioSubsystem             MetricSubsystem = "io"
 	nodesSubsystem          MetricSubsystem = "nodes"
 	objectsSubsystem        MetricSubsystem = "objects"
-	fileDescriptorSubsystem MetricSubsystem = "file_descriptor"
-	ioSubsystem             MetricSubsystem = "io"
+	processSubsystem        MetricSubsystem = "process"
 	replicationSubsystem    MetricSubsystem = "replication"
 	requestsSubsystem       MetricSubsystem = "requests"
 	timeSubsystem           MetricSubsystem = "time"
 	trafficSubsystem        MetricSubsystem = "traffic"
+	softwareSubsystem       MetricSubsystem = "software"
 	sysCallSubsystem        MetricSubsystem = "syscall"
 	usageSubsystem          MetricSubsystem = "usage"
-	softwareSubsystem       MetricSubsystem = "software"
 )
 
 // MetricName are the individual names for the metric.
@@ -107,6 +108,7 @@ const (
 	ttfbDistribution = "ttbf_seconds_distribution"
 
 	lastActivityTime = "last_activity_nano_seconds"
+	startTime        = "starttime_seconds"
 )
 
 const (
@@ -385,7 +387,7 @@ func getS3RequestsInFlightMD() MetricDescription {
 		Subsystem: requestsSubsystem,
 		Name:      inflightTotal,
 		Help:      "Total number of S3 requests currently in flight.",
-		Type:      counterMetric,
+		Type:      gaugeMetric,
 	}
 }
 func getS3RequestsTotalMD() MetricDescription {
@@ -631,6 +633,15 @@ func getMinIOGORoutineCountMD() MetricDescription {
 		Type:      gaugeMetric,
 	}
 }
+func getMinIOProcessStartTimeMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: processSubsystem,
+		Name:      startTime,
+		Help:      "Start time for MinIO process per node in seconds.",
+		Type:      gaugeMetric,
+	}
+}
 func getMinioProcMetrics() MetricsGroup {
 	return MetricsGroup{
 		Metrics: []Metric{},
@@ -656,6 +667,16 @@ func getMinioProcMetrics() MetricsGroup {
 				logger.LogOnceIf(ctx, err, ioSubsystem)
 				return
 			}
+			stat, err := p.Stat()
+			if err != nil {
+				logger.LogOnceIf(ctx, err, processSubsystem)
+				return
+			}
+			startTime, err := stat.StartTime()
+			if err != nil {
+				logger.LogOnceIf(ctx, err, startTime)
+				return
+			}
 
 			metrics.Metrics = append(metrics.Metrics,
 				Metric{
@@ -698,6 +719,11 @@ func getMinioProcMetrics() MetricsGroup {
 					Description: getMinioProcessIOWriteCachedBytesMD(),
 					Value:       float64(io.WChar),
 				})
+			metrics.Metrics = append(metrics.Metrics,
+				Metric{
+					Description: getMinIOProcessStartTimeMD(),
+					Value:       startTime,
+				})
 		},
 	}
 }
@@ -1080,23 +1106,23 @@ func getClusterStorageMetrics() MetricsGroup {
 
 			metrics.Metrics = append(metrics.Metrics, Metric{
 				Description: getClusterCapacityTotalBytesMD(),
-				Value:       float64(GetTotalCapacity(ctx)),
+				Value:       float64(GetTotalCapacity(storageInfo.Disks)),
 			})
 
 			metrics.Metrics = append(metrics.Metrics, Metric{
 				Description: getClusterCapacityFreeBytesMD(),
-				Value:       float64(GetTotalCapacityFree(ctx)),
+				Value:       float64(GetTotalCapacityFree(storageInfo.Disks)),
 			})
 
 			s, _ := objLayer.StorageInfo(GlobalContext)
 			metrics.Metrics = append(metrics.Metrics, Metric{
 				Description: getClusterCapacityUsageBytesMD(),
-				Value:       GetTotalUsableCapacity(ctx, s),
+				Value:       GetTotalUsableCapacity(storageInfo.Disks, s),
 			})
 
 			metrics.Metrics = append(metrics.Metrics, Metric{
 				Description: getClusterCapacityUsageFreeBytesMD(),
-				Value:       GetTotalUsableCapacityFree(ctx, s),
+				Value:       GetTotalUsableCapacityFree(storageInfo.Disks, s),
 			})
 
 			metrics.Metrics = append(metrics.Metrics, Metric{

cmd/metrics.go

@@ -419,7 +419,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 }
 
 // Populates prometheus with bucket usage metrics, this metrics
-// is only enabled if crawler is enabled.
+// is only enabled if scanner is enabled.
 func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
 	objLayer := newObjectLayerFn()
 	// Service not initialized yet
@@ -535,7 +535,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total capacity online in the cluster",
 			nil, nil),
 		prometheus.GaugeValue,
-		float64(GetTotalCapacity(GlobalContext)),
+		float64(GetTotalCapacity(server.Disks)),
 	)
 
 	// Report total capacity free
@@ -545,7 +545,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total free capacity online in the cluster",
 			nil, nil),
 		prometheus.GaugeValue,
-		float64(GetTotalCapacityFree(GlobalContext)),
+		float64(GetTotalCapacityFree(server.Disks)),
 	)
 
 	s, _ := objLayer.StorageInfo(GlobalContext)
@@ -556,7 +556,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total usable capacity online in the cluster",
 			nil, nil),
 		prometheus.GaugeValue,
-		GetTotalUsableCapacity(GlobalContext, s),
+		GetTotalUsableCapacity(server.Disks, s),
 	)
 	// Report total usable capacity free
 	ch <- prometheus.MustNewConstMetric(
@@ -565,7 +565,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total free usable capacity online in the cluster",
 			nil, nil),
 		prometheus.GaugeValue,
-		GetTotalUsableCapacityFree(GlobalContext, s),
+		GetTotalUsableCapacityFree(server.Disks, s),
 	)
 
 	// MinIO Offline Disks per node

cmd/naughty-disk_test.go

@@ -245,11 +245,11 @@ func (d *naughtyDisk) WriteMetadata(ctx context.Context, volume, path string, fi
 	return d.disk.WriteMetadata(ctx, volume, path, fi)
 }
 
-func (d *naughtyDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) (err error) {
+func (d *naughtyDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) (err error) {
 	if err := d.calcError(); err != nil {
 		return err
 	}
-	return d.disk.DeleteVersion(ctx, volume, path, fi)
+	return d.disk.DeleteVersion(ctx, volume, path, fi, forceDelMarker)
 }
 
 func (d *naughtyDisk) ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (fi FileInfo, err error) {

cmd/notification-summary.go

@@ -18,22 +18,22 @@
 package cmd
 
 import (
-	"context"
+	"github.com/minio/minio/pkg/madmin"
 )
 
 // GetTotalCapacity gets the total capacity in the cluster.
-func GetTotalCapacity(ctx context.Context) (capacity uint64) {
-	d := globalNotificationSys.DiskHwInfo(ctx)
-	for _, s := range d {
-		capacity += s.GetTotalCapacity()
+func GetTotalCapacity(diskInfo []madmin.Disk) (capacity uint64) {
+
+	for _, disk := range diskInfo {
+		capacity += disk.TotalSpace
 	}
 	return
 }
 
 // GetTotalUsableCapacity gets the total usable capacity in the cluster.
 // This value is not an accurate representation of total usable in a multi-tenant deployment.
-func GetTotalUsableCapacity(ctx context.Context, s StorageInfo) (capacity float64) {
-	raw := GetTotalCapacity(ctx)
+func GetTotalUsableCapacity(diskInfo []madmin.Disk, s StorageInfo) (capacity float64) {
+	raw := GetTotalCapacity(diskInfo)
 	var approxDataBlocks float64
 	var actualDisks float64
 	for _, scData := range s.Backend.StandardSCData {
@@ -45,18 +45,17 @@ func GetTotalUsableCapacity(ctx context.Context, s StorageInfo) (capacity float6
 }
 
 // GetTotalCapacityFree gets the total capacity free in the cluster.
-func GetTotalCapacityFree(ctx context.Context) (capacity uint64) {
-	d := globalNotificationSys.DiskHwInfo(ctx)
-	for _, s := range d {
-		capacity += s.GetTotalFreeCapacity()
+func GetTotalCapacityFree(diskInfo []madmin.Disk) (capacity uint64) {
+	for _, d := range diskInfo {
+		capacity += d.AvailableSpace
 	}
 	return
 }
 
 // GetTotalUsableCapacityFree gets the total usable capacity free in the cluster.
 // This value is not an accurate representation of total free in a multi-tenant deployment.
-func GetTotalUsableCapacityFree(ctx context.Context, s StorageInfo) (capacity float64) {
-	raw := GetTotalCapacityFree(ctx)
+func GetTotalUsableCapacityFree(diskInfo []madmin.Disk, s StorageInfo) (capacity float64) {
+	raw := GetTotalCapacityFree(diskInfo)
 	var approxDataBlocks float64
 	var actualDisks float64
 	for _, scData := range s.Backend.StandardSCData {

cmd/object-api-datatypes.go

@@ -244,6 +244,51 @@ type ObjectInfo struct {
 	SuccessorModTime time.Time
 }
 
+// Clone - Returns a cloned copy of current objectInfo
+func (o ObjectInfo) Clone() (cinfo ObjectInfo) {
+	cinfo = ObjectInfo{
+		Bucket:             o.Bucket,
+		Name:               o.Name,
+		ModTime:            o.ModTime,
+		Size:               o.Size,
+		IsDir:              o.IsDir,
+		ETag:               o.ETag,
+		InnerETag:          o.InnerETag,
+		VersionID:          o.VersionID,
+		IsLatest:           o.IsLatest,
+		DeleteMarker:       o.DeleteMarker,
+		TransitionStatus:   o.TransitionStatus,
+		RestoreExpires:     o.RestoreExpires,
+		RestoreOngoing:     o.RestoreOngoing,
+		ContentType:        o.ContentType,
+		ContentEncoding:    o.ContentEncoding,
+		Expires:            o.Expires,
+		CacheStatus:        o.CacheStatus,
+		CacheLookupStatus:  o.CacheLookupStatus,
+		StorageClass:       o.StorageClass,
+		ReplicationStatus:  o.ReplicationStatus,
+		UserTags:           o.UserTags,
+		Parts:              o.Parts,
+		Writer:             o.Writer,
+		Reader:             o.Reader,
+		PutObjReader:       o.PutObjReader,
+		metadataOnly:       o.metadataOnly,
+		versionOnly:        o.versionOnly,
+		keyRotation:        o.keyRotation,
+		backendType:        o.backendType,
+		AccTime:            o.AccTime,
+		Legacy:             o.Legacy,
+		VersionPurgeStatus: o.VersionPurgeStatus,
+		NumVersions:        o.NumVersions,
+		SuccessorModTime:   o.SuccessorModTime,
+	}
+	cinfo.UserDefined = make(map[string]string, len(o.UserDefined))
+	for k, v := range o.UserDefined {
+		cinfo.UserDefined[k] = v
+	}
+	return cinfo
+}
+
 // MultipartInfo captures metadata information about the uploadId
 // this data structure is used primarily for some internal purposes
 // for verifying upload type such as was the upload

cmd/object-api-interface.go

@@ -155,6 +155,7 @@ type ObjectLayer interface {
 
 	// Returns health of the backend
 	Health(ctx context.Context, opts HealthOptions) HealthResult
+	ReadHealth(ctx context.Context) bool
 
 	// ObjectTagging operations
 	PutObjectTags(context.Context, string, string, string, ObjectOptions) (ObjectInfo, error)

cmd/object-api-utils.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"context"
 	"encoding/hex"
+	"errors"
 	"fmt"
 	"io"
 	"math/rand"
@@ -257,7 +258,7 @@ func cleanMetadata(metadata map[string]string) map[string]string {
 	// Remove STANDARD StorageClass
 	metadata = removeStandardStorageClass(metadata)
 	// Clean meta etag keys 'md5Sum', 'etag', "expires", "x-amz-tagging".
-	return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires", xhttp.AmzObjectTagging)
+	return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires", xhttp.AmzObjectTagging, "last-modified")
 }
 
 // Filter X-Amz-Storage-Class field only if it is set to STANDARD.
@@ -404,7 +405,7 @@ func (o ObjectInfo) IsCompressedOK() (bool, error) {
 // GetActualETag - returns the actual etag of the stored object
 // decrypts SSE objects.
 func (o ObjectInfo) GetActualETag(h http.Header) string {
-	if !crypto.IsEncrypted(o.UserDefined) {
+	if _, ok := crypto.IsEncrypted(o.UserDefined); !ok {
 		return o.ETag
 	}
 	return getDecryptedETag(h, o, false)
@@ -423,7 +424,7 @@ func (o ObjectInfo) GetActualSize() (int64, error) {
 		}
 		return size, nil
 	}
-	if crypto.IsEncrypted(o.UserDefined) {
+	if _, ok := crypto.IsEncrypted(o.UserDefined); ok {
 		return o.DecryptedSize()
 	}
 
@@ -600,7 +601,7 @@ func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, cl
 		}
 	}()
 
-	isEncrypted := crypto.IsEncrypted(oi.UserDefined)
+	_, isEncrypted := crypto.IsEncrypted(oi.UserDefined)
 	isCompressed, err := oi.IsCompressedOK()
 	if err != nil {
 		return nil, 0, 0, err
@@ -865,17 +866,23 @@ func (p *PutObjReader) MD5CurrentHexString() string {
 	return hex.EncodeToString(md5sumCurr)
 }
 
-// NewPutObjReader returns a new PutObjReader and holds
-// reference to underlying data stream from client and the encrypted
-// data reader
-func NewPutObjReader(rawReader *hash.Reader, encReader *hash.Reader, key *crypto.ObjectKey) *PutObjReader {
-	p := PutObjReader{Reader: rawReader, rawReader: rawReader}
-
-	if key != nil && encReader != nil {
-		p.sealMD5Fn = sealETagFn(*key)
-		p.Reader = encReader
+// WithEncryption sets up encrypted reader and the sealing for content md5sum
+// using objEncKey. Unsealed md5sum is computed from the rawReader setup when
+// NewPutObjReader was called. It returns an error if called on an uninitialized
+// PutObjReader.
+func (p *PutObjReader) WithEncryption(encReader *hash.Reader, objEncKey *crypto.ObjectKey) (*PutObjReader, error) {
+	if p.Reader == nil {
+		return nil, errors.New("put-object reader uninitialized")
 	}
-	return &p
+	p.Reader = encReader
+	p.sealMD5Fn = sealETagFn(*objEncKey)
+	return p, nil
+}
+
+// NewPutObjReader returns a new PutObjReader. It uses given hash.Reader's
+// MD5Current method to construct md5sum when requested downstream.
+func NewPutObjReader(rawReader *hash.Reader) *PutObjReader {
+	return &PutObjReader{Reader: rawReader, rawReader: rawReader}
 }
 
 func sealETag(encKey crypto.ObjectKey, md5CurrSum []byte) []byte {

cmd/object-handlers-common.go

@@ -26,8 +26,6 @@ import (
 
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/pkg/bucket/lifecycle"
-	"github.com/minio/minio/pkg/event"
-	"github.com/minio/minio/pkg/handlers"
 )
 
 var (
@@ -261,7 +259,7 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool) {
 		}
 	}
 
-	if objInfo.Bucket != "" {
+	if objInfo.Bucket != "" && objInfo.Name != "" {
 		if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil && !delete {
 			ruleID, expiryTime := lc.PredictExpiryTime(lifecycle.ObjectOpts{
 				Name:         objInfo.Name,
@@ -279,32 +277,3 @@ func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, delete bool) {
 		}
 	}
 }
-
-// deleteObject is a convenient wrapper to delete an object, this
-// is a common function to be called from object handlers and
-// web handlers.
-func deleteObject(ctx context.Context, obj ObjectLayer, cache CacheObjectLayer, bucket, object string, w http.ResponseWriter, r *http.Request, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	deleteObject := obj.DeleteObject
-	if cache != nil {
-		deleteObject = cache.DeleteObject
-	}
-	// Proceed to delete the object.
-	objInfo, err = deleteObject(ctx, bucket, object, opts)
-	if objInfo.Name != "" {
-		eventName := event.ObjectRemovedDelete
-		if objInfo.DeleteMarker {
-			eventName = event.ObjectRemovedDeleteMarkerCreated
-		}
-		// Notify object deleted marker event.
-		sendEvent(eventArgs{
-			EventName:    eventName,
-			BucketName:   bucket,
-			Object:       objInfo,
-			ReqParams:    extractReqParams(r),
-			RespElements: extractRespElements(w),
-			UserAgent:    r.UserAgent(),
-			Host:         handlers.GetSourceIP(r),
-		})
-	}
-	return objInfo, err
-}

cmd/object-handlers.go

@@ -265,19 +265,17 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 
 	// Set encryption response headers
 	if objectAPI.IsEncryptionSupported() {
-		if crypto.IsEncrypted(objInfo.UserDefined) {
-			switch {
-			case crypto.S3.IsEncrypted(objInfo.UserDefined):
-				w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
-			case crypto.SSEC.IsEncrypted(objInfo.UserDefined):
-				// Validate the SSE-C Key set in the header.
-				if _, err = crypto.SSEC.UnsealObjectKey(r.Header, objInfo.UserDefined, bucket, object); err != nil {
-					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-					return
-				}
-				w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm))
-				w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
+		switch kind, _ := crypto.IsEncrypted(objInfo.UserDefined); kind {
+		case crypto.S3:
+			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
+		case crypto.SSEC:
+			// Validate the SSE-C Key set in the header.
+			if _, err = crypto.SSEC.UnsealObjectKey(r.Header, objInfo.UserDefined, bucket, object); err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+				return
 			}
+			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm))
+			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
 		}
 	}
 
@@ -413,19 +411,40 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 
 	gr, err := getObjectNInfo(ctx, bucket, object, rs, r.Header, readLock, opts)
 	if err != nil {
-		if isErrPreconditionFailed(err) {
-			return
-		}
-		if globalBucketVersioningSys.Enabled(bucket) && gr != nil {
-			// Versioning enabled quite possibly object is deleted might be delete-marker
-			// if present set the headers, no idea why AWS S3 sets these headers.
-			if gr.ObjInfo.VersionID != "" && gr.ObjInfo.DeleteMarker {
-				w.Header()[xhttp.AmzVersionID] = []string{gr.ObjInfo.VersionID}
-				w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(gr.ObjInfo.DeleteMarker)}
+		var (
+			reader *GetObjectReader
+			proxy  bool
+		)
+		if isProxyable(ctx, bucket) {
+			// proxy to replication target if active-active replication is in place.
+			reader, proxy = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts)
+			if reader != nil && proxy {
+				gr = reader
 			}
 		}
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
-		return
+		if reader == nil || !proxy {
+			if isErrPreconditionFailed(err) {
+				return
+			}
+			if globalBucketVersioningSys.Enabled(bucket) && gr != nil {
+				if !gr.ObjInfo.VersionPurgeStatus.Empty() {
+					// Shows the replication status of a permanent delete of a version
+					w.Header()[xhttp.MinIODeleteReplicationStatus] = []string{string(gr.ObjInfo.VersionPurgeStatus)}
+				}
+				if !gr.ObjInfo.ReplicationStatus.Empty() && gr.ObjInfo.DeleteMarker {
+					w.Header()[xhttp.MinIODeleteMarkerReplicationStatus] = []string{string(gr.ObjInfo.ReplicationStatus)}
+				}
+
+				// Versioning enabled quite possibly object is deleted might be delete-marker
+				// if present set the headers, no idea why AWS S3 sets these headers.
+				if gr.ObjInfo.VersionID != "" && gr.ObjInfo.DeleteMarker {
+					w.Header()[xhttp.AmzVersionID] = []string{gr.ObjInfo.VersionID}
+					w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(gr.ObjInfo.DeleteMarker)}
+				}
+			}
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
 	}
 	defer gr.Close()
 
@@ -435,7 +454,7 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 	if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 		action := evalActionFromLifecycle(ctx, *lc, objInfo, false)
 		if action == lifecycle.DeleteAction || action == lifecycle.DeleteVersionAction {
-			globalExpiryState.queueExpiryTask(objInfo)
+			globalExpiryState.queueExpiryTask(objInfo, action == lifecycle.DeleteVersionAction)
 			writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
 			return
 		}
@@ -450,14 +469,12 @@ func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Req
 
 	// Set encryption response headers
 	if objectAPI.IsEncryptionSupported() {
-		if crypto.IsEncrypted(objInfo.UserDefined) {
-			switch {
-			case crypto.S3.IsEncrypted(objInfo.UserDefined):
-				w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
-			case crypto.SSEC.IsEncrypted(objInfo.UserDefined):
-				w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm))
-				w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
-			}
+		switch kind, _ := crypto.IsEncrypted(objInfo.UserDefined); kind {
+		case crypto.S3:
+			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
+		case crypto.SSEC:
+			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm))
+			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
 		}
 	}
 
@@ -581,30 +598,44 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 
 	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
-		if globalBucketVersioningSys.Enabled(bucket) {
-			if !objInfo.VersionPurgeStatus.Empty() {
-				// Shows the replication status of a permanent delete of a version
-				w.Header()[xhttp.MinIODeleteReplicationStatus] = []string{string(objInfo.VersionPurgeStatus)}
-			}
-			if !objInfo.ReplicationStatus.Empty() && objInfo.DeleteMarker {
-				w.Header()[xhttp.MinIODeleteMarkerReplicationStatus] = []string{string(objInfo.ReplicationStatus)}
-			}
-			// Versioning enabled quite possibly object is deleted might be delete-marker
-			// if present set the headers, no idea why AWS S3 sets these headers.
-			if objInfo.VersionID != "" && objInfo.DeleteMarker {
-				w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
-				w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(objInfo.DeleteMarker)}
+		var (
+			proxy bool
+			perr  error
+			oi    ObjectInfo
+		)
+		// proxy HEAD to replication target if active-active replication configured on bucket
+		if isProxyable(ctx, bucket) {
+			oi, proxy, perr = proxyHeadToReplicationTarget(ctx, bucket, object, opts)
+			if proxy && perr == nil {
+				objInfo = oi
 			}
 		}
-		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
-		return
+		if !proxy || perr != nil {
+			if globalBucketVersioningSys.Enabled(bucket) {
+				if !objInfo.VersionPurgeStatus.Empty() {
+					// Shows the replication status of a permanent delete of a version
+					w.Header()[xhttp.MinIODeleteReplicationStatus] = []string{string(objInfo.VersionPurgeStatus)}
+				}
+				if !objInfo.ReplicationStatus.Empty() && objInfo.DeleteMarker {
+					w.Header()[xhttp.MinIODeleteMarkerReplicationStatus] = []string{string(objInfo.ReplicationStatus)}
+				}
+				// Versioning enabled quite possibly object is deleted might be delete-marker
+				// if present set the headers, no idea why AWS S3 sets these headers.
+				if objInfo.VersionID != "" && objInfo.DeleteMarker {
+					w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
+					w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(objInfo.DeleteMarker)}
+				}
+			}
+			writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
+			return
+		}
 	}
 
 	// Automatically remove the object/version is an expiry lifecycle rule can be applied
 	if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 		action := evalActionFromLifecycle(ctx, *lc, objInfo, false)
 		if action == lifecycle.DeleteAction || action == lifecycle.DeleteVersionAction {
-			globalExpiryState.queueExpiryTask(objInfo)
+			globalExpiryState.queueExpiryTask(objInfo, action == lifecycle.DeleteVersionAction)
 			writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey))
 			return
 		}
@@ -654,19 +685,17 @@ func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Re
 
 	// Set encryption response headers
 	if objectAPI.IsEncryptionSupported() {
-		if crypto.IsEncrypted(objInfo.UserDefined) {
-			switch {
-			case crypto.S3.IsEncrypted(objInfo.UserDefined):
-				w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
-			case crypto.SSEC.IsEncrypted(objInfo.UserDefined):
-				// Validate the SSE-C Key set in the header.
-				if _, err = crypto.SSEC.UnsealObjectKey(r.Header, objInfo.UserDefined, bucket, object); err != nil {
-					writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
-					return
-				}
-				w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm))
-				w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
+		switch kind, _ := crypto.IsEncrypted(objInfo.UserDefined); kind {
+		case crypto.S3:
+			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
+		case crypto.SSEC:
+			// Validate the SSE-C Key set in the header.
+			if _, err = crypto.SSEC.UnsealObjectKey(r.Header, objInfo.UserDefined, bucket, object); err != nil {
+				writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
+				return
 			}
+			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm))
+			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
 		}
 	}
 
@@ -1038,8 +1067,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	rawReader := srcInfo.Reader
-	pReader := NewPutObjReader(srcInfo.Reader, nil, nil)
+	pReader := NewPutObjReader(srcInfo.Reader)
 
 	// Check if either the source is encrypted or the destination will be encrypted.
 	_, objectEncryption := crypto.IsRequested(r.Header)
@@ -1047,7 +1075,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	var encMetadata = make(map[string]string)
 	if objectAPI.IsEncryptionSupported() {
 		// Encryption parameters not applicable for this object.
-		if !crypto.IsEncrypted(srcInfo.UserDefined) && crypto.SSECopy.IsRequested(r.Header) {
+		if _, ok := crypto.IsEncrypted(srcInfo.UserDefined); !ok && crypto.SSECopy.IsRequested(r.Header) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL, guessIsBrowserReq(r))
 			return
 		}
@@ -1151,7 +1179,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			}
 
 			if isTargetEncrypted {
-				pReader = NewPutObjReader(rawReader, srcInfo.Reader, &objEncKey)
+				pReader, err = pReader.WithEncryption(srcInfo.Reader, &objEncKey)
+				if err != nil {
+					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+					return
+				}
 			}
 		}
 	}
@@ -1290,7 +1322,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime)
 	encodedSuccessResponse := encodeResponse(response)
 	if replicate, sync := mustReplicate(ctx, r, dstBucket, dstObject, objInfo.UserDefined, objInfo.ReplicationStatus.String()); replicate {
-		scheduleReplication(ctx, objInfo, objectAPI, sync)
+		scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync)
 	}
 
 	setPutObjHeaders(w, objInfo, false)
@@ -1501,7 +1533,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	rawReader := hashReader
-	pReader := NewPutObjReader(rawReader, nil, nil)
+	pReader := NewPutObjReader(rawReader)
 
 	// get gateway encryption options
 	var opts ObjectOptions
@@ -1570,7 +1602,11 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 				return
 			}
-			pReader = NewPutObjReader(rawReader, hashReader, &objectEncryptionKey)
+			pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
+			if err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+				return
+			}
 		}
 	}
 
@@ -1584,13 +1620,13 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	switch {
-	case crypto.IsEncrypted(objInfo.UserDefined):
-		switch {
-		case crypto.S3.IsEncrypted(objInfo.UserDefined):
+	switch kind, encrypted := crypto.IsEncrypted(objInfo.UserDefined); {
+	case encrypted:
+		switch kind {
+		case crypto.S3:
 			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 			objInfo.ETag, _ = DecryptETag(objectEncryptionKey, ObjectInfo{ETag: objInfo.ETag})
-		case crypto.SSEC.IsEncrypted(objInfo.UserDefined):
+		case crypto.SSEC:
 			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm))
 			w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5))
 
@@ -1604,7 +1640,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 	if replicate, sync := mustReplicate(ctx, r, bucket, object, metadata, ""); replicate {
-		scheduleReplication(ctx, objInfo, objectAPI, sync)
+		scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync)
 	}
 	setPutObjHeaders(w, objInfo, false)
 
@@ -1912,7 +1948,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	srcInfo := gr.ObjInfo
 
 	actualPartSize := srcInfo.Size
-	if crypto.IsEncrypted(srcInfo.UserDefined) {
+	if _, ok := crypto.IsEncrypted(srcInfo.UserDefined); ok {
 		actualPartSize, err = srcInfo.GetActualSize()
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -2008,9 +2044,9 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	}
 
 	rawReader := srcInfo.Reader
-	pReader := NewPutObjReader(rawReader, nil, nil)
+	pReader := NewPutObjReader(rawReader)
 
-	isEncrypted := crypto.IsEncrypted(mi.UserDefined)
+	_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
 	var objectEncryptionKey crypto.ObjectKey
 	if objectAPI.IsEncryptionSupported() && isEncrypted {
 		if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) {
@@ -2054,7 +2090,11 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		pReader = NewPutObjReader(rawReader, srcInfo.Reader, &objectEncryptionKey)
+		pReader, err = pReader.WithEncryption(srcInfo.Reader, &objectEncryptionKey)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
 	}
 
 	srcInfo.PutObjReader = pReader
@@ -2248,9 +2288,9 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		return
 	}
 	rawReader := hashReader
-	pReader := NewPutObjReader(rawReader, nil, nil)
+	pReader := NewPutObjReader(rawReader)
 
-	isEncrypted := crypto.IsEncrypted(mi.UserDefined)
+	_, isEncrypted := crypto.IsEncrypted(mi.UserDefined)
 	var objectEncryptionKey crypto.ObjectKey
 	if objectAPI.IsEncryptionSupported() && isEncrypted {
 		if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) {
@@ -2304,7 +2344,11 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		pReader = NewPutObjReader(rawReader, hashReader, &objectEncryptionKey)
+		pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
+			return
+		}
 	}
 
 	putObjectPart := objectAPI.PutObjectPart
@@ -2416,7 +2460,7 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 	}
 
 	var ssec bool
-	if objectAPI.IsEncryptionSupported() && crypto.IsEncrypted(listPartsInfo.UserDefined) {
+	if _, ok := crypto.IsEncrypted(listPartsInfo.UserDefined); ok && objectAPI.IsEncryptionSupported() {
 		var key []byte
 		if crypto.SSEC.IsEncrypted(listPartsInfo.UserDefined) {
 			ssec = true
@@ -2580,7 +2624,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		if crypto.IsEncrypted(mi.UserDefined) {
+		if _, ok := crypto.IsEncrypted(mi.UserDefined); ok {
 			var key []byte
 			isEncrypted = true
 			ssec = crypto.SSEC.IsEncrypted(mi.UserDefined)
@@ -2683,7 +2727,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 	setPutObjHeaders(w, objInfo, false)
 	if replicate, sync := mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, objInfo.ReplicationStatus.String()); replicate {
-		scheduleReplication(ctx, objInfo, objectAPI, sync)
+		scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync)
 	}
 
 	// Write success response.
@@ -2804,8 +2848,13 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
+	deleteObject := objectAPI.DeleteObject
+	if api.CacheAPI() != nil {
+		deleteObject = api.CacheAPI().DeleteObject
+	}
+
 	// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html
-	objInfo, err := deleteObject(ctx, objectAPI, api.CacheAPI(), bucket, object, w, r, opts)
+	objInfo, err := deleteObject(ctx, bucket, object, opts)
 	if err != nil {
 		switch err.(type) {
 		case BucketNotFound:
@@ -2813,9 +2862,32 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 			return
 		}
-		// Ignore delete object errors while replying to client, since we are suppposed to reply only 204.
 	}
 
+	if objInfo.Name == "" {
+		writeSuccessNoContent(w)
+		return
+	}
+
+	setPutObjHeaders(w, objInfo, true)
+	writeSuccessNoContent(w)
+
+	eventName := event.ObjectRemovedDelete
+	if objInfo.DeleteMarker {
+		eventName = event.ObjectRemovedDeleteMarkerCreated
+	}
+
+	// Notify object deleted event.
+	sendEvent(eventArgs{
+		EventName:    eventName,
+		BucketName:   bucket,
+		Object:       objInfo,
+		ReqParams:    extractReqParams(r),
+		RespElements: extractRespElements(w),
+		UserAgent:    r.UserAgent(),
+		Host:         handlers.GetSourceIP(r),
+	})
+
 	if replicateDel {
 		dmVersionID := ""
 		versionID := ""
@@ -2840,7 +2912,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 	}
 
 	if goi.TransitionStatus == lifecycle.TransitionComplete { // clean up transitioned tier
-		deleteTransitionedObject(ctx, newObjectLayerFn(), bucket, object, lifecycle.ObjectOpts{
+		deleteTransitionedObject(ctx, objectAPI, bucket, object, lifecycle.ObjectOpts{
 			Name:             object,
 			UserTags:         goi.UserTags,
 			VersionID:        goi.VersionID,
@@ -2849,9 +2921,6 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			IsLatest:         goi.IsLatest,
 		}, false, true)
 	}
-
-	setPutObjHeaders(w, objInfo, true)
-	writeSuccessNoContent(w)
 }
 
 // PutObjectLegalHoldHandler - set legal hold configuration to object,
@@ -2936,10 +3005,11 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 		return
 	}
 	if replicate {
-		scheduleReplication(ctx, objInfo, objectAPI, sync)
+		scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync)
 	}
 	writeSuccessResponseHeadersOnly(w)
-	// Notify object  event.
+
+	// Notify object event.
 	sendEvent(eventArgs{
 		EventName:    event.ObjectCreatedPutLegalHold,
 		BucketName:   bucket,
@@ -3108,7 +3178,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		return
 	}
 	if replicate {
-		scheduleReplication(ctx, objInfo, objectAPI, sync)
+		scheduleReplication(ctx, objInfo.Clone(), objectAPI, sync)
 	}
 
 	writeSuccessNoContent(w)
@@ -3291,7 +3361,7 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
 	}
 
 	if replicate {
-		scheduleReplication(ctx, objInfo, objAPI, sync)
+		scheduleReplication(ctx, objInfo.Clone(), objAPI, sync)
 	}
 
 	if objInfo.VersionID != "" {
@@ -3346,6 +3416,7 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
+
 	oi, err := objAPI.GetObjectInfo(ctx, bucket, object, opts)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
@@ -3357,14 +3428,14 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 		opts.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Pending.String()
 	}
 
-	if _, err = objAPI.DeleteObjectTags(ctx, bucket, object, opts); err != nil {
+	oi, err = objAPI.DeleteObjectTags(ctx, bucket, object, opts)
+	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	oi.UserTags = ""
 
 	if replicate {
-		scheduleReplication(ctx, oi, objAPI, sync)
+		scheduleReplication(ctx, oi.Clone(), objAPI, sync)
 	}
 
 	if oi.VersionID != "" {

cmd/os-readdir_other.go

@@ -29,11 +29,15 @@ func readDir(dirPath string) (entries []string, err error) {
 	return readDirN(dirPath, -1)
 }
 
-// readDir applies the filter function on each entries at dirPath, doesn't recurse into
-// the directory itself.
-func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
+// readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
+// the directory itself, if the dirPath doesn't exist this function doesn't return
+// an error.
+func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
 	d, err := os.Open(dirPath)
 	if err != nil {
+		if osErrToFileErr(err) == errFileNotFound {
+			return nil
+		}
 		return osErrToFileErr(err)
 	}
 	defer d.Close()
@@ -46,6 +50,9 @@ func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) e
 			if err == io.EOF {
 				break
 			}
+			if osErrToFileErr(err) == errFileNotFound {
+				return nil
+			}
 			return osErrToFileErr(err)
 		}
 		for _, fi := range fis {

cmd/os-readdir_unix.go

@@ -84,11 +84,15 @@ func readDir(dirPath string) (entries []string, err error) {
 	return readDirN(dirPath, -1)
 }
 
-// readDir applies the filter function on each entries at dirPath, doesn't recurse into
-// the directory itself.
-func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
+// readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
+// the directory itself, if the dirPath doesn't exist this function doesn't return
+// an error.
+func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) error {
 	f, err := os.Open(dirPath)
 	if err != nil {
+		if osErrToFileErr(err) == errFileNotFound {
+			return nil
+		}
 		return osErrToFileErr(err)
 	}
 	defer f.Close()
@@ -103,7 +107,7 @@ func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) e
 			nbuf, err = syscall.ReadDirent(int(f.Fd()), buf)
 			if err != nil {
 				if isSysErrNotDir(err) {
-					return errFileNotFound
+					return nil
 				}
 				return err
 			}
@@ -122,8 +126,8 @@ func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) e
 		if typ&os.ModeSymlink == os.ModeSymlink {
 			continue
 		}
-		if err = filter(string(name), typ); err == errDoneForNow {
-			// filtering requested to return by caller.
+		if err = fn(string(name), typ); err == errDoneForNow {
+			// fn() requested to return by caller.
 			return nil
 		}
 	}

cmd/os-readdir_windows.go

@@ -29,11 +29,15 @@ func readDir(dirPath string) (entries []string, err error) {
 	return readDirN(dirPath, -1)
 }
 
-// readDir applies the filter function on each entries at dirPath, doesn't recurse into
-// the directory itself.
-func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
+// readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
+// the directory itself, if the dirPath doesn't exist this function doesn't return
+// an error.
+func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) error {
 	f, err := os.Open(dirPath)
 	if err != nil {
+		if osErrToFileErr(err) == errFileNotFound {
+			return nil
+		}
 		return osErrToFileErr(err)
 	}
 	defer f.Close()
@@ -45,6 +49,9 @@ func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) e
 			if e == syscall.ERROR_NO_MORE_FILES {
 				break
 			} else {
+				if isSysErrPathNotFound(e) {
+					return nil
+				}
 				return osErrToFileErr(&os.PathError{
 					Op:   "FindNextFile",
 					Path: dirPath,
@@ -69,7 +76,7 @@ func readDirFilterFn(dirPath string, filter func(name string, typ os.FileMode) e
 		}
 	}
 
-	return err
+	return nil
 }
 
 // Return N entries at the directory dirPath. If count is -1, return all entries

cmd/prepare-storage.go

@@ -125,16 +125,8 @@ func formatErasureCleanupTmpLocalEndpoints(endpoints Endpoints) error {
 					osErrToFileErr(err))
 			}
 
-			// Move .minio.sys/buckets/.minio.sys/metacache transient list cache
-			// folder to speed up startup routines.
-			tmpMetacacheOld := pathJoin(epPath, minioMetaTmpBucket+"-old", mustGetUUID())
-			if err := renameAll(pathJoin(epPath, minioMetaBucket, metacachePrefixForID(minioMetaBucket, "")),
-				tmpMetacacheOld); err != nil && err != errFileNotFound {
-				return fmt.Errorf("unable to rename (%s -> %s) %w",
-					pathJoin(epPath, minioMetaBucket+metacachePrefixForID(minioMetaBucket, "")),
-					tmpMetacacheOld,
-					osErrToFileErr(err))
-			}
+			// Renames and schedules for puring all bucket metacache.
+			renameAllBucketMetacache(epPath)
 
 			// Removal of tmp-old folder is backgrounded completely.
 			go removeAll(pathJoin(epPath, minioMetaTmpBucket+"-old"))

cmd/rest/client.go

@@ -60,6 +60,8 @@ func (n *NetworkError) Unwrap() error {
 
 // Client - http based RPC client.
 type Client struct {
+	connected int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG
+
 	// HealthCheckFn is the function set to test for health.
 	// If not set the client will not keep track of health.
 	// Calling this returns true or false if the target
@@ -84,7 +86,6 @@ type Client struct {
 	httpClient   *http.Client
 	url          *url.URL
 	newAuthToken func(audience string) string
-	connected    int32
 }
 
 // URL query separator constants

cmd/server-main.go

@@ -41,6 +41,7 @@ import (
 	"github.com/minio/minio/pkg/color"
 	"github.com/minio/minio/pkg/env"
 	"github.com/minio/minio/pkg/madmin"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 // ServerFlags - server command specific flags
@@ -346,10 +347,22 @@ func initAllSubsystems(ctx context.Context, newObject ObjectLayer) (err error) {
 				logger.Info(fmt.Sprintf("Verifying if %d buckets are consistent across drives...", len(buckets)))
 			}
 		}
-		for _, bucket := range buckets {
-			if _, err = newObject.HealBucket(ctx, bucket.Name, madmin.HealOpts{Recreate: true}); err != nil {
-				return fmt.Errorf("Unable to list buckets to heal: %w", err)
-			}
+
+		// Limit to no more than 50 concurrent buckets.
+		g := errgroup.WithNErrs(len(buckets)).WithConcurrency(50)
+		ctx, cancel := g.WithCancelOnError(ctx)
+		defer cancel()
+		for index := range buckets {
+			index := index
+			g.Go(func() error {
+				if _, berr := newObject.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{Recreate: true}); berr != nil {
+					return fmt.Errorf("Unable to list buckets to heal: %w", berr)
+				}
+				return nil
+			}, index)
+		}
+		if err := g.WaitErr(); err != nil {
+			return err
 		}
 	}
 
@@ -492,12 +505,11 @@ func serverMain(ctx *cli.Context) {
 	// Enable background operations for erasure coding
 	if globalIsErasure {
 		initAutoHeal(GlobalContext, newObject)
-		initBackgroundReplication(GlobalContext, newObject)
 		initBackgroundTransition(GlobalContext, newObject)
 		initBackgroundExpiry(GlobalContext, newObject)
 	}
 
-	initDataCrawler(GlobalContext, newObject)
+	initDataScanner(GlobalContext, newObject)
 
 	if err = initServer(GlobalContext, newObject); err != nil {
 		var cerr config.Err
@@ -513,6 +525,9 @@ func serverMain(ctx *cli.Context) {
 		}
 	}
 
+	if globalIsErasure { // to be done after config init
+		initBackgroundReplication(GlobalContext, newObject)
+	}
 	if globalCacheConfig.Enabled {
 		// initialize the new disk cache objects.
 		var cacheAPI CacheObjectLayer

cmd/storage-datatypes.go

@@ -85,18 +85,18 @@ type FileInfoVersions struct {
 	Versions []FileInfo
 }
 
-// forwardPastVersion will truncate the result to only contain versions after 'v'.
-// If v is empty or the version isn't found no changes will be made.
-func (f *FileInfoVersions) forwardPastVersion(v string) {
-	if v == "" {
-		return
+// findVersionIndex will return the version index where the version
+// was found. Returns -1 if not found.
+func (f *FileInfoVersions) findVersionIndex(v string) int {
+	if f == nil || v == "" {
+		return -1
 	}
 	for i, ver := range f.Versions {
 		if ver.VersionID == v {
-			f.Versions = f.Versions[i+1:]
-			return
+			return i
 		}
 	}
+	return -1
 }
 
 // FileInfo - represents file stat information.

cmd/storage-errors.go

@@ -16,6 +16,8 @@
 
 package cmd
 
+import "errors"
+
 // errUnexpected - unexpected error, requires manual intervention.
 var errUnexpected = StorageErr("unexpected error, please report this issue at https://github.com/minio/minio/issues")
 
@@ -104,6 +106,13 @@ var errLessData = StorageErr("less data available than what was requested")
 // errMoreData = returned when more data was sent by the caller than what it was supposed to.
 var errMoreData = StorageErr("more data was sent than what was advertised")
 
+// indicates readDirFn to return without further applying the fn()
+var errDoneForNow = errors.New("done for now")
+
+// errSkipFile returned by the fn() for readDirFn() when it needs
+// to proceed to next entry.
+var errSkipFile = errors.New("skip this file")
+
 // StorageErr represents error generated by xlStorage call.
 type StorageErr string
 

cmd/storage-interface.go

@@ -55,7 +55,7 @@ type StorageAPI interface {
 	WalkVersions(ctx context.Context, volume, dirPath, marker string, recursive bool, endWalkCh <-chan struct{}) (chan FileInfoVersions, error)
 
 	// Metadata operations
-	DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error
+	DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) error
 	DeleteVersions(ctx context.Context, volume string, versions []FileInfo) []error
 	WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) error
 	ReadVersion(ctx context.Context, volume, path, versionID string, readData bool) (FileInfo, error)

cmd/storage-rest-client.go

@@ -29,7 +29,6 @@ import (
 	"strconv"
 	"strings"
 	"sync"
-	"time"
 
 	"github.com/minio/minio/cmd/http"
 	xhttp "github.com/minio/minio/cmd/http"
@@ -120,8 +119,6 @@ type storageRESTClient struct {
 	endpoint   Endpoint
 	restClient *rest.Client
 	diskID     string
-
-	diskInfoCache timedValue
 }
 
 // Wrapper to restClient.Call to handle network errors, in case of network error the connection is makred disconnected
@@ -218,27 +215,18 @@ func (client *storageRESTClient) SetDiskID(id string) {
 
 // DiskInfo - fetch disk information for a remote disk.
 func (client *storageRESTClient) DiskInfo(ctx context.Context) (info DiskInfo, err error) {
-	client.diskInfoCache.Once.Do(func() {
-		client.diskInfoCache.TTL = time.Second
-		client.diskInfoCache.Update = func() (interface{}, error) {
-			var info DiskInfo
-			respBody, err := client.call(ctx, storageRESTMethodDiskInfo, nil, nil, -1)
-			if err != nil {
-				return info, err
-			}
-			defer http.DrainBody(respBody)
-			if err = msgp.Decode(respBody, &info); err != nil {
-				return info, err
-			}
-			if info.Error != "" {
-				return info, toStorageErr(errors.New(info.Error))
-			}
-			return info, nil
-		}
-	})
-	v, err := client.diskInfoCache.Get()
-	info = v.(DiskInfo)
-	return info, err
+	respBody, err := client.call(ctx, storageRESTMethodDiskInfo, nil, nil, -1)
+	if err != nil {
+		return info, err
+	}
+	defer http.DrainBody(respBody)
+	if err = msgp.Decode(respBody, &info); err != nil {
+		return info, err
+	}
+	if info.Error != "" {
+		return info, toStorageErr(errors.New(info.Error))
+	}
+	return info, nil
 }
 
 // MakeVolBulk - create multiple volumes in a bulk operation.
@@ -332,10 +320,11 @@ func (client *storageRESTClient) WriteMetadata(ctx context.Context, volume, path
 	return err
 }
 
-func (client *storageRESTClient) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo) error {
+func (client *storageRESTClient) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool) error {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
+	values.Set(storageRESTForceDelMarker, strconv.FormatBool(forceDelMarker))
 
 	var buffer bytes.Buffer
 	if err := msgp.Encode(&buffer, &fi); err != nil {

cmd/storage-rest-common.go

@@ -17,7 +17,7 @@
 package cmd
 
 const (
-	storageRESTVersion       = "v26" // Add NumVersions/SuccessorModTime fields in FileInfo
+	storageRESTVersion       = "v27" // Add force-delete-marker to DeleteVersion
 	storageRESTVersionPrefix = SlashSeparator + storageRESTVersion
 	storageRESTPrefix        = minioReservedBucketPath + "/storage"
 )
@@ -58,6 +58,7 @@ const (
 	storageRESTVolumes        = "volumes"
 	storageRESTDirPath        = "dir-path"
 	storageRESTFilePath       = "file-path"
+	storageRESTForceDelMarker = "force-delete-marker"
 	storageRESTVersionID      = "version-id"
 	storageRESTReadData       = "read-data"
 	storageRESTTotalVersions  = "total-versions"
@@ -71,6 +72,7 @@ const (
 	storageRESTCount          = "count"
 	storageRESTMarkerPath     = "marker"
 	storageRESTPrefixFilter   = "prefix"
+	storageRESTForwardFilter  = "forward"
 	storageRESTRecursive      = "recursive"
 	storageRESTReportNotFound = "report-notfound"
 	storageRESTBitrotAlgo     = "bitrot-algo"