allow more socket listeners per instance for multi-core setups (#13385)

with some broken clients allow non-strict validation
of sha256 when ContentLength > 0, it has been found in
the wild some applications that need this behavior. This
shall be only allowed if `--no-compat` is used.

Makefile

@@ -10,33 +10,35 @@ TAG ?= "minio/minio:$(VERSION)"
 
 all: build
 
-checks:
+checks: ## check dependencies
 	@echo "Checking dependencies"
 	@(env bash $(PWD)/buildscripts/checkdeps.sh)
 
-getdeps:
+help: ## print this help
+	@grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
+
+getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
 	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.40.1
 	@which msgp 1>/dev/null || (echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.3)
 	@which stringer 1>/dev/null || (echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer)
 
-crosscompile:
+crosscompile: ## cross compile minio
 	@(env bash $(PWD)/buildscripts/cross-compile.sh)
 
 verifiers: getdeps lint check-gen
 
-check-gen:
+check-gen: ## check for updated autogenerated files
 	@go generate ./... >/dev/null
 	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
 
-lint:
+lint: ## runs golangci-lint suite of linters
 	@echo "Running $@ check"
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --build-tags kqueue --timeout=10m --config ./.golangci.yml
 
-# Builds minio, runs the verifiers then runs the tests.
 check: test
-test: verifiers build
+test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
 	@GOGC=25 GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
 
@@ -44,20 +46,17 @@ test-race: verifiers build
 	@echo "Running unit tests under -race"
 	@(env bash $(PWD)/buildscripts/race.sh)
 
-# Verify minio binary
-verify:
+verify: ## verify minio various setups
 	@echo "Verifying build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-build.sh)
 
-# Verify healing of disks with minio binary
-verify-healing:
+verify-healing: ## verify healing and replacing disks with minio binary
 	@echo "Verify healing build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
 
-# Builds minio locally.
-build: checks
+build: checks ## builds minio to $(PWD)
 	@echo "Building minio binary to './minio'"
 	@GO111MODULE=on CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 
@@ -65,23 +64,22 @@ hotfix-vars:
 	$(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
     sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#')))
 	$(eval TAG := "minio/minio:$(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD)")
-hotfix: hotfix-vars install
+hotfix: hotfix-vars install ## builds minio binary with hotfix tags
 
-docker-hotfix: hotfix checks
+docker-hotfix: hotfix checks ## builds minio docker container with hotfix tags
 	@echo "Building minio docker image '$(TAG)'"
 	@docker build -t $(TAG) . -f Dockerfile.dev
 
-docker: build checks
+docker: build checks ## builds minio docker container
 	@echo "Building minio docker image '$(TAG)'"
 	@docker build -t $(TAG) . -f Dockerfile.dev
 
-# Builds minio and installs it to $GOPATH/bin.
-install: build
+install: build ## builds minio and installs it to $GOPATH/bin.
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
 	@echo "Installation successful. To learn more, try \"minio --help\"."
 
-clean:
+clean: ## cleanup all generated assets
 	@echo "Cleaning up all the generated files"
 	@find . -name '*.test' | xargs rm -fv
 	@find . -name '*~' | xargs rm -fv

cmd/admin-handlers-site-replication.go

@@ -0,0 +1,315 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
+
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/bucket/policy"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
+func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationAdd")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var sites []madmin.PeerSite
+	errCode := readJSONBody(ctx, r.Body, &sites, cred.SecretKey)
+	if errCode != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+		return
+	}
+
+	status, errInfo := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
+	if errInfo.Code != ErrNone {
+		logger.LogIf(ctx, errInfo)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(errInfo.Code, errInfo.Cause), r.URL)
+		return
+	}
+
+	body, err := json.Marshal(status)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, body)
+}
+
+// SRInternalJoin - PUT /minio/admin/v3/site-replication/join
+//
+// used internally to tell current cluster to enable SR with
+// the provided peer clusters and service account.
+func (a adminAPIHandlers) SRInternalJoin(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRInternalJoin")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var joinArg madmin.SRInternalJoinReq
+	errCode := readJSONBody(ctx, r.Body, &joinArg, cred.SecretKey)
+	if errCode != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+		return
+	}
+
+	errInfo := globalSiteReplicationSys.InternalJoinReq(ctx, joinArg)
+	if errInfo.Code != ErrNone {
+		logger.LogIf(ctx, errInfo)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(errInfo.Code, errInfo.Cause), r.URL)
+		return
+	}
+}
+
+// SRInternalBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
+func (a adminAPIHandlers) SRInternalBucketOps(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRInternalBucketOps")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	operation := madmin.BktOp(vars["operation"])
+
+	var err error
+	switch operation {
+	case madmin.MakeWithVersioningBktOp:
+		_, isLockEnabled := r.Form["lockEnabled"]
+		_, isVersioningEnabled := r.Form["versioningEnabled"]
+		opts := BucketOptions{
+			Location:          r.Form.Get("location"),
+			LockEnabled:       isLockEnabled,
+			VersioningEnabled: isVersioningEnabled,
+		}
+		err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
+	case madmin.ConfigureReplBktOp:
+		err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
+	case madmin.DeleteBucketBktOp:
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, false)
+	case madmin.ForceDeleteBucketBktOp:
+		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, true)
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+
+}
+
+// SRInternalReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
+func (a adminAPIHandlers) SRInternalReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRInternalReplicateIAMItem")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var item madmin.SRIAMItem
+	errCode := readJSONBody(ctx, r.Body, &item, "")
+	if errCode != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+		return
+	}
+
+	var err error
+	switch item.Type {
+	case madmin.SRIAMItemPolicy:
+		var policy *iampolicy.Policy
+		if len(item.Policy) > 0 {
+			policy, err = iampolicy.ParseConfig(bytes.NewReader(item.Policy))
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+		}
+		err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy)
+	case madmin.SRIAMItemSvcAcc:
+		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, *item.SvcAccChange)
+	case madmin.SRIAMItemPolicyMapping:
+		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, *item.PolicyMapping)
+	case madmin.SRIAMItemSTSAcc:
+		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, *item.STSCredential)
+
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+}
+
+// SRInternalReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
+func (a adminAPIHandlers) SRInternalReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRInternalReplicateIAMItem")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	var item madmin.SRBucketMeta
+	errCode := readJSONBody(ctx, r.Body, &item, "")
+	if errCode != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+		return
+	}
+
+	var err error
+	switch item.Type {
+	case madmin.SRBucketMetaTypePolicy:
+		var bktPolicy *policy.Policy
+		if len(item.Policy) > 0 {
+			bktPolicy, err = policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+		}
+		err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
+	case madmin.SRBucketMetaTypeTags:
+		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags)
+	case madmin.SRBucketMetaTypeObjectLockConfig:
+		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig)
+	case madmin.SRBucketMetaTypeSSEConfig:
+		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig)
+
+	default:
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+}
+
+// SiteReplicationDisable - PUT /minio/admin/v3/site-replication/disable
+func (a adminAPIHandlers) SiteReplicationDisable(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationDisable")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationDisableAction)
+	if objectAPI == nil {
+		return
+	}
+}
+
+// SiteReplicationInfo - GET /minio/admin/v3/site-replication/info
+func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationInfo")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
+	if objectAPI == nil {
+		return
+	}
+
+	info, err := globalSiteReplicationSys.GetClusterInfo(ctx)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err = json.NewEncoder(w).Encode(info); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+func (a adminAPIHandlers) SRInternalGetIDPSettings(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SiteReplicationGetIDPSettings")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
+	if objectAPI == nil {
+		return
+	}
+
+	idpSettings := globalSiteReplicationSys.GetIDPSettings(ctx)
+	if err := json.NewEncoder(w).Encode(idpSettings); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	w.(http.Flusher).Flush()
+}
+
+func readJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) APIErrorCode {
+	data, err := ioutil.ReadAll(body)
+	if err != nil {
+		return ErrInvalidRequest
+	}
+
+	if encryptionKey != "" {
+		data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
+		if err != nil {
+			logger.LogIf(ctx, err)
+			return ErrInvalidRequest
+		}
+	}
+
+	err = json.Unmarshal(data, v)
+	if err != nil {
+		return ErrAdminConfigBadJSON
+	}
+
+	return ErrNone
+}

cmd/admin-handlers-users.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -216,7 +217,6 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	defer r.Body.Close()
 	data, err := ioutil.ReadAll(r.Body)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
@@ -532,6 +532,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	opts := newServiceAccountOpts{
 		accessKey: createReq.AccessKey,
 		secretKey: createReq.SecretKey,
+		claims:    make(map[string]interface{}),
 	}
 
 	// Find the user for the request sender (as it may be sent via a service
@@ -565,16 +566,13 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		}
 		targetGroups = requestorGroups
 
-		// In case of LDAP we need to set `opts.ldapUsername` to ensure
-		// it is associated with the LDAP user properly. We _only_ do
-		// this if this user is in LDAP (the other possibility is the
-		// root user).
-		if globalLDAPConfig.Enabled {
-			v1, ok1 := cred.Claims[ldapUserN]
-			v2, ok2 := v1.(string)
-			if ok1 && ok2 {
-				opts.ldapUsername = v2
+		// In case of LDAP/OIDC we need to set `opts.claims` to ensure
+		// it is associated with the LDAP/OIDC user properly.
+		for k, v := range cred.Claims {
+			if k == expClaim {
+				continue
 			}
+			opts.claims[k] = v
 		}
 	} else {
 		// Need permission if we are creating a service acccount for a
@@ -593,12 +591,13 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		// In case of LDAP we need to resolve the targetUser to a DN and
 		// query their groups:
 		if globalLDAPConfig.Enabled {
-			opts.ldapUsername = targetUser
+			opts.claims[ldapUserN] = targetUser // simple username
 			targetUser, targetGroups, err = globalLDAPConfig.LookupUserDN(targetUser)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
 			}
+			opts.claims[ldapUser] = targetUser // username DN
 		}
 
 		// NOTE: if not using LDAP, then internal IDP or open ID is
@@ -631,6 +630,34 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
+	// Call hook for cluster-replication.
+	//
+	// FIXME: This wont work in an OpenID situation as the parent credential
+	// may not be present on peer clusters to provide inherited policies.
+	// Also, we should not be replicating root user's service account - as
+	// they are not authenticated by a common external IDP, so we skip when
+	// opts.ldapUser == "".
+	if _, isLDAPAccount := opts.claims[ldapUserN]; isLDAPAccount {
+		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Create: &madmin.SRSvcAccCreate{
+					Parent:        newCred.ParentUser,
+					AccessKey:     newCred.AccessKey,
+					SecretKey:     newCred.SecretKey,
+					Groups:        newCred.Groups,
+					Claims:        opts.claims,
+					SessionPolicy: createReq.Policy,
+					Status:        auth.AccountOn,
+				},
+			},
+		})
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
 	var createResp = madmin.AddServiceAccountResp{
 		Credentials: madmin.Credentials{
 			AccessKey: newCred.AccessKey,
@@ -742,6 +769,30 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		}
 	}
 
+	// Call site replication hook. Only LDAP accounts are supported for
+	// replication operations.
+	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, accessKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
+		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Update: &madmin.SRSvcAccUpdate{
+					AccessKey:     accessKey,
+					SecretKey:     opts.secretKey,
+					Status:        opts.status,
+					SessionPolicy: updateReq.NewPolicy,
+				},
+			},
+		})
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
 	writeSuccessNoContent(w)
 }
 
@@ -976,6 +1027,28 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		}
 	}
 
+	// Call site replication hook. Only LDAP accounts are supported for
+	// replication operations.
+	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, serviceAccount)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+	if _, isLDAPAccount := svcAccClaims[ldapUserN]; isLDAPAccount {
+		err = globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+			Type: madmin.SRIAMItemSvcAcc,
+			SvcAccChange: &madmin.SRSvcAccChange{
+				Delete: &madmin.SRSvcAccDelete{
+					AccessKey: serviceAccount,
+				},
+			},
+		})
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+	}
+
 	writeSuccessNoContent(w)
 }
 
@@ -1287,6 +1360,16 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 			logger.LogIf(ctx, nerr.Err)
 		}
 	}
+
+	// Call cluster-replication policy creation hook to replicate policy deletion to
+	// other minio clusters.
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemPolicy,
+		Name: policyName,
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }
 
 // AddCannedPolicy - PUT /minio/admin/v3/add-canned-policy?name=<policy_name>
@@ -1315,7 +1398,13 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	iamPolicy, err := iampolicy.ParseConfig(io.LimitReader(r.Body, r.ContentLength))
+	iamPolicyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	iamPolicy, err := iampolicy.ParseConfig(bytes.NewReader(iamPolicyBytes))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1339,6 +1428,17 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 			logger.LogIf(ctx, nerr.Err)
 		}
 	}
+
+	// Call cluster-replication policy creation hook to replicate policy to
+	// other minio clusters.
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type:   madmin.SRIAMItemPolicy,
+		Name:   policyName,
+		Policy: iamPolicyBytes,
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }
 
 // SetPolicyForUserOrGroup - PUT /minio/admin/v3/set-policy?policy=xxx&user-or-group=?[&is-group]
@@ -1381,4 +1481,16 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 			logger.LogIf(ctx, nerr.Err)
 		}
 	}
+
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemPolicyMapping,
+		PolicyMapping: &madmin.SRPolicyMapping{
+			UserOrGroup: entityName,
+			IsGroup:     isGroup,
+			Policy:      policyName,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
 }

cmd/admin-handlers.go

@@ -38,7 +38,7 @@ import (
 	"strings"
 	"time"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go"
@@ -149,9 +149,13 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	for _, nerr := range globalNotificationSys.ServerUpdate(ctx, u, sha256Sum, lrTime, releaseInfo) {
 		if nerr.Err != nil {
+			err := AdminError{
+				Code:       AdminUpdateApplyFailure,
+				Message:    nerr.Err.Error(),
+				StatusCode: http.StatusInternalServerError,
+			}
 			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-			err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", nerr.Err)
+			logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
@@ -159,7 +163,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 
 	updateStatus, err := updateServer(u, sha256Sum, lrTime, releaseInfo, mode)
 	if err != nil {
-		err = fmt.Errorf("Server update failed, please do not restart the servers yet: failed with %w", err)
+		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -992,7 +996,7 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	objectAPI.DeleteBucket(ctx, pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), true)
+	objectAPI.DeleteBucket(ctx, pathJoin(minioMetaSpeedTestBucket, minioMetaSpeedTestBucketPrefix), DeleteBucketOptions{Force: true, NoRecreate: true})
 
 	w.(http.Flusher).Flush()
 }
@@ -2140,7 +2144,7 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 
 // getRawDataer provides an interface for getting raw FS files.
 type getRawDataer interface {
-	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error
+	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error
 }
 
 // InspectDataHandler - GET /minio/admin/v3/inspect-data
@@ -2173,6 +2177,13 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
+	file = strings.ReplaceAll(file, string(os.PathSeparator), "/")
+
+	// Reject attempts to traverse parent or absolute paths.
+	if strings.Contains(file, "..") || strings.Contains(volume, "..") {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
 
 	var key [32]byte
 	// MUST use crypto/rand
@@ -2210,15 +2221,19 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	zipWriter := zip.NewWriter(encw)
 	defer zipWriter.Close()
 
-	err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time) error {
+	err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time, isDir bool) error {
 		// Prefix host+disk
 		filename = path.Join(host, disk, filename)
+		if isDir {
+			filename += "/"
+			size = 0
+		}
 		header, zerr := zip.FileInfoHeader(dummyFileInfo{
 			name:    filename,
 			size:    size,
 			mode:    0600,
 			modTime: modtime,
-			isDir:   false,
+			isDir:   isDir,
 			sys:     nil,
 		})
 		if zerr != nil {

cmd/admin-router.go

@@ -191,6 +191,16 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
 			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler)))
+
+			// Cluster Replication APIs
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/disable").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationDisable)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalJoin)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalReplicateIAMItem)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalReplicateBucketItem)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalGetIDPSettings)))
 		}
 
 		if globalIsDistErasure {

cmd/api-errors.go

@@ -267,6 +267,16 @@ const (
 	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
+
+	// Site-Replication errors
+	ErrSiteReplicationInvalidRequest
+	ErrSiteReplicationPeerResp
+	ErrSiteReplicationBackendIssue
+	ErrSiteReplicationServiceAccountError
+	ErrSiteReplicationBucketConfigError
+	ErrSiteReplicationBucketMetaError
+	ErrSiteReplicationIAMError
+
 	// Bucket Quota error codes
 	ErrAdminBucketQuotaExceeded
 	ErrAdminNoSuchQuotaConfiguration
@@ -444,7 +454,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrInvalidMaxParts: {
 		Code:           "InvalidArgument",
-		Description:    "Argument max-parts must be an integer between 0 and 2147483647",
+		Description:    "Part number must be an integer between 1 and 10000, inclusive",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartNumberMarker: {
@@ -1269,6 +1279,43 @@ var errorCodes = errorCodeMap{
 		Description:    errObjectTampered.Error(),
 		HTTPStatusCode: http.StatusPartialContent,
 	},
+
+	ErrSiteReplicationInvalidRequest: {
+		Code:           "XMinioSiteReplicationInvalidRequest",
+		Description:    "Invalid site-replication request",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrSiteReplicationPeerResp: {
+		Code:           "XMinioSiteReplicationPeerResp",
+		Description:    "Error received when contacting a peer site",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationBackendIssue: {
+		Code:           "XMinioSiteReplicationBackendIssue",
+		Description:    "Error when requesting object layer backend",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationServiceAccountError: {
+		Code:           "XMinioSiteReplicationServiceAccountError",
+		Description:    "Site replication related service account error",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationBucketConfigError: {
+		Code:           "XMinioSiteReplicationBucketConfigError",
+		Description:    "Error while configuring replication on a bucket",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationBucketMetaError: {
+		Code:           "XMinioSiteReplicationBucketMetaError",
+		Description:    "Error while replicating bucket metadata",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+	ErrSiteReplicationIAMError: {
+		Code:           "XMinioSiteReplicationIAMError",
+		Description:    "Error while replicating an IAM item",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
+
 	ErrMaximumExpires: {
 		Code:           "AuthorizationQueryParametersError",
 		Description:    "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds",

cmd/bitrot.go

@@ -27,8 +27,10 @@ import (
 	"io"
 
 	"github.com/minio/highwayhash"
-	"github.com/minio/minio/internal/logger"
 	"golang.org/x/crypto/blake2b"
+
+	xioutil "github.com/minio/minio/internal/ioutil"
+	"github.com/minio/minio/internal/logger"
 )
 
 // magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key.
@@ -172,8 +174,8 @@ func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, w
 		return errFileCorrupt
 	}
 
-	bufp := xlPoolSmall.Get().(*[]byte)
-	defer xlPoolSmall.Put(bufp)
+	bufp := xioutil.ODirectPoolSmall.Get().(*[]byte)
+	defer xioutil.ODirectPoolSmall.Put(bufp)
 
 	for left > 0 {
 		// Read expected hash...

cmd/bucket-encryption-handlers.go

@@ -18,12 +18,14 @@
 package cmd
 
 import (
+	"encoding/base64"
 	"encoding/xml"
 	"fmt"
 	"io"
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
 )
@@ -95,6 +97,20 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		return
 	}
 
+	// Call site replication hook.
+	//
+	// We encode the xml bytes as base64 to ensure there are no encoding
+	// errors.
+	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:      madmin.SRBucketMetaTypeSSEConfig,
+		Bucket:    bucket,
+		SSEConfig: &cfgStr,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	writeSuccessResponseHeadersOnly(w)
 }
 

cmd/bucket-handlers.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/base64"
 	"encoding/json"
 	"encoding/xml"
@@ -37,6 +38,7 @@ import (
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	sse "github.com/minio/minio/internal/bucket/encryption"
@@ -727,7 +729,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				}
 
 				if err = globalDNSConfig.Put(bucket); err != nil {
-					objectAPI.DeleteBucket(ctx, bucket, false)
+					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: false, NoRecreate: true})
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 					return
 				}
@@ -770,6 +772,17 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 
 	// Proceed to creating a bucket.
 	err := objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
+	if _, ok := err.(BucketExists); ok {
+		// Though bucket exists locally, we send the site-replication
+		// hook to ensure all sites have this bucket. If the hook
+		// succeeds, the client will still receive a bucket exists
+		// message.
+		err2 := globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
+		if err2 != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+	}
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -778,6 +791,13 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	// Load updated bucket metadata into memory.
 	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
+	// Call site replication hook
+	err = globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Make sure to add Location information here only for bucket
 	if cp := pathClean(r.URL.Path); cp != "" {
 		w.Header().Set(xhttp.Location, cp) // Clean any trailing slashes.
@@ -1240,7 +1260,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	deleteBucket := objectAPI.DeleteBucket
 
 	// Attempt to delete bucket.
-	if err := deleteBucket(ctx, bucket, forceDelete); err != nil {
+	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete}); err != nil {
 		apiErr := toAPIError(ctx, err)
 		if _, ok := err.(BucketNotEmpty); ok {
 			if globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket) {
@@ -1258,6 +1278,12 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
 
+	// Call site replication hook.
+	if err := globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessNoContent(w)
 
@@ -1323,6 +1349,20 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 		return
 	}
 
+	// Call site replication hook.
+	//
+	// We encode the xml bytes as base64 to ensure there are no encoding
+	// errors.
+	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:             madmin.SRBucketMetaTypeObjectLockConfig,
+		Bucket:           bucket,
+		ObjectLockConfig: &cfgStr,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -1414,6 +1454,20 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
+	// Call site replication hook.
+	//
+	// We encode the xml bytes as base64 to ensure there are no encoding
+	// errors.
+	cfgStr := base64.StdEncoding.EncodeToString(configData)
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypeTags,
+		Bucket: bucket,
+		Tags:   &cfgStr,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -1482,6 +1536,14 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
+	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypeTags,
+		Bucket: bucket,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }

cmd/bucket-lifecycle.go

@@ -220,16 +220,31 @@ var errInvalidStorageClass = errors.New("invalid storage class")
 
 func validateTransitionTier(lc *lifecycle.Lifecycle) error {
 	for _, rule := range lc.Rules {
-		if rule.Transition.StorageClass == "" {
-			continue
+		if rule.Transition.StorageClass != "" {
+			if valid := globalTierConfigMgr.IsTierValid(rule.Transition.StorageClass); !valid {
+				return errInvalidStorageClass
+			}
 		}
-		if valid := globalTierConfigMgr.IsTierValid(rule.Transition.StorageClass); !valid {
-			return errInvalidStorageClass
+		if rule.NoncurrentVersionTransition.StorageClass != "" {
+			if valid := globalTierConfigMgr.IsTierValid(rule.NoncurrentVersionTransition.StorageClass); !valid {
+				return errInvalidStorageClass
+			}
 		}
 	}
 	return nil
 }
 
+// enqueueTransitionImmediate enqueues obj for transition if eligible.
+// This is to be called after a successful upload of an object (version).
+func enqueueTransitionImmediate(obj ObjectInfo) {
+	if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
+		switch lc.ComputeAction(obj.ToLifecycleOpts()) {
+		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
+			globalTransitionState.queueTransitionTask(obj)
+		}
+	}
+}
+
 // expireAction represents different actions to be performed on expiry of a
 // restored/transitioned object
 type expireAction int
@@ -702,17 +717,16 @@ func isRestoredObjectOnDisk(meta map[string]string) (onDisk bool) {
 // ToLifecycleOpts returns lifecycle.ObjectOpts value for oi.
 func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts {
 	return lifecycle.ObjectOpts{
-		Name:                   oi.Name,
-		UserTags:               oi.UserTags,
-		VersionID:              oi.VersionID,
-		ModTime:                oi.ModTime,
-		IsLatest:               oi.IsLatest,
-		NumVersions:            oi.NumVersions,
-		DeleteMarker:           oi.DeleteMarker,
-		SuccessorModTime:       oi.SuccessorModTime,
-		RestoreOngoing:         oi.RestoreOngoing,
-		RestoreExpires:         oi.RestoreExpires,
-		TransitionStatus:       oi.TransitionedObject.Status,
-		RemoteTiersImmediately: globalDebugRemoteTiersImmediately,
+		Name:             oi.Name,
+		UserTags:         oi.UserTags,
+		VersionID:        oi.VersionID,
+		ModTime:          oi.ModTime,
+		IsLatest:         oi.IsLatest,
+		NumVersions:      oi.NumVersions,
+		DeleteMarker:     oi.DeleteMarker,
+		SuccessorModTime: oi.SuccessorModTime,
+		RestoreOngoing:   oi.RestoreOngoing,
+		RestoreExpires:   oi.RestoreExpires,
+		TransitionStatus: oi.TransitionedObject.Status,
 	}
 }

cmd/bucket-policy-handlers.go

@@ -18,12 +18,15 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"io"
+	"io/ioutil"
 	"net/http"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
 )
@@ -76,7 +79,13 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
-	bucketPolicy, err := policy.ParseConfig(io.LimitReader(r.Body, r.ContentLength), bucket)
+	bucketPolicyBytes, err := ioutil.ReadAll(io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -99,6 +108,16 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 		return
 	}
 
+	// Call site replication hook.
+	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypePolicy,
+		Bucket: bucket,
+		Policy: bucketPolicyBytes,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Success.
 	writeSuccessNoContent(w)
 }
@@ -134,6 +153,15 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 		return
 	}
 
+	// Call site replication hook.
+	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+		Type:   madmin.SRBucketMetaTypePolicy,
+		Bucket: bucket,
+	}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
 	// Success.
 	writeSuccessNoContent(w)
 }

cmd/bucket-replication-utils.go

@@ -499,7 +499,7 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 	}
 	var dsc ReplicateDecision
 	var tgtStatuses map[string]replication.StatusType
-	if oi.DeleteMarker {
+	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{
 			ObjectName: oi.Name,
 			VersionID:  oi.VersionID,

cmd/bucket-replication.go

@@ -732,7 +732,12 @@ func equals(k1 string, keys ...string) bool {
 }
 
 // returns replicationAction by comparing metadata between source and target
-func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo) replicationAction {
+func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type) replicationAction {
+	// Avoid resyncing null versions created prior to enabling replication if target has a newer copy
+	if opType == replication.ExistingObjectReplicationType &&
+		oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID {
+		return replicateNone
+	}
 	// needs full replication
 	if oi1.ETag != oi2.ETag ||
 		oi1.VersionID != oi2.VersionID ||
@@ -1053,9 +1058,19 @@ func replicateObjectToTarget(ctx context.Context, ri ReplicateObjectInfo, object
 			ReplicationProxyRequest: "false",
 		}})
 	if cerr == nil {
-		rAction = getReplicationAction(objInfo, oi)
+		rAction = getReplicationAction(objInfo, oi, ri.OpType)
 		rinfo.ReplicationStatus = replication.Completed
 		if rAction == replicateNone {
+			if ri.OpType == replication.ExistingObjectReplicationType &&
+				objInfo.ModTime.Unix() > oi.LastModified.Unix() && objInfo.VersionID == nullVersionID {
+				logger.LogIf(ctx, fmt.Errorf("Unable to replicate %s/%s (null). Newer version exists on target", bucket, object))
+				sendEvent(eventArgs{
+					EventName:  event.ObjectReplicationNotTracked,
+					BucketName: bucket,
+					Object:     objInfo,
+					Host:       "Internal: [Replication]",
+				})
+			}
 			// object with same VersionID already exists, replication kicked off by
 			// PutObject might have completed
 			if objInfo.TargetReplicationStatus(tgt.ARN) == replication.Pending || objInfo.TargetReplicationStatus(tgt.ARN) == replication.Failed || ri.OpType == replication.ExistingObjectReplicationType {
@@ -1651,7 +1666,7 @@ func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo, dsc *Repli
 		return
 	}
 	// existing object replication does not apply to un-versioned objects
-	if oi.VersionID == "" || oi.VersionID == nullVersionID {
+	if oi.VersionID == "" {
 		return
 	}
 

cmd/common-main.go

@@ -51,7 +51,6 @@ import (
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/handlers"
-	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/certs"
@@ -59,6 +58,7 @@ import (
 	"github.com/minio/pkg/ellipses"
 	"github.com/minio/pkg/env"
 	xnet "github.com/minio/pkg/net"
+	"github.com/rs/dnscache"
 )
 
 // serverDebugLog will enable debug printing
@@ -71,17 +71,34 @@ func init() {
 	logger.Init(GOPATH, GOROOT)
 	logger.RegisterError(config.FmtError)
 
-	if IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile() {
-		// 30 seconds matches the orchestrator DNS TTLs, have
-		// a 5 second timeout to lookup from DNS servers.
-		globalDNSCache = xhttp.NewDNSCache(30*time.Second, 5*time.Second, logger.LogOnceIf)
-	} else {
-		// On bare-metals DNS do not change often, so it is
-		// safe to assume a higher timeout upto 10 minutes.
-		globalDNSCache = xhttp.NewDNSCache(10*time.Minute, 5*time.Second, logger.LogOnceIf)
-	}
 	initGlobalContext()
 
+	options := dnscache.ResolverRefreshOptions{
+		ClearUnused:      true,
+		PersistOnFailure: false,
+	}
+
+	// Call to refresh will refresh names in cache. If you pass true, it will also
+	// remove cached names not looked up since the last call to Refresh. It is a good idea
+	// to call this method on a regular interval.
+	go func() {
+		var t *time.Ticker
+		if IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile() {
+			t = time.NewTicker(1 * time.Minute)
+		} else {
+			t = time.NewTicker(10 * time.Minute)
+		}
+		defer t.Stop()
+		for {
+			select {
+			case <-t.C:
+				globalDNSCache.RefreshWithOptions(options)
+			case <-GlobalContext.Done():
+				return
+			}
+		}
+	}()
+
 	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
 		PassHost:     true,
 		RoundTripper: newGatewayHTTPTransport(1 * time.Hour),
@@ -138,7 +155,7 @@ func minioConfigToConsoleFeatures() {
 	}
 	// if IDP is enabled, set IDP environment variables
 	if globalOpenIDConfig.URL != nil {
-		os.Setenv("CONSOLE_IDP_URL", globalOpenIDConfig.DiscoveryDoc.Issuer)
+		os.Setenv("CONSOLE_IDP_URL", globalOpenIDConfig.URL.String())
 		os.Setenv("CONSOLE_IDP_CLIENT_ID", globalOpenIDConfig.ClientID)
 		os.Setenv("CONSOLE_IDP_SECRET", globalOpenIDConfig.ClientSecret)
 		os.Setenv("CONSOLE_IDP_HMAC_SALT", globalDeploymentID)
@@ -598,10 +615,6 @@ func handleCommonEnvVars() {
 		}
 		GlobalKMS = KMS
 	}
-
-	if tiers := env.Get("_MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY", ""); tiers != "" {
-		globalDebugRemoteTiersImmediately = strings.Split(tiers, ",")
-	}
 }
 
 func logStartupMessage(msg string) {

cmd/data-scanner.go

@@ -29,6 +29,7 @@ import (
 	"path"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/bits-and-blooms/bloom/v3"
@@ -194,6 +195,22 @@ type folderScanner struct {
 	lastUpdate time.Time
 }
 
+type scannerStats struct {
+	// All fields must be accessed atomically and aligned.
+
+	accTotalObjects  uint64
+	accTotalVersions uint64
+	accFolders       uint64
+	bucketsStarted   uint64
+	bucketsFinished  uint64
+	ilmChecks        uint64
+
+	// actions records actions performed.
+	actions [lifecycle.ActionCount]uint64
+}
+
+var globalScannerStats scannerStats
+
 // Cache structure and compaction:
 //
 // A cache structure will be kept with a tree of usages.
@@ -241,6 +258,10 @@ func scanDataFolder(ctx context.Context, basePath string, cache dataUsageCache,
 
 	logPrefix := color.Green("data-usage: ")
 	logSuffix := color.Blue("- %v + %v", basePath, cache.Info.Name)
+	atomic.AddUint64(&globalScannerStats.bucketsStarted, 1)
+	defer func() {
+		atomic.AddUint64(&globalScannerStats.bucketsFinished, 1)
+	}()
 	if intDataUpdateTracker.debug {
 		defer func() {
 			console.Debugf(logPrefix+" Scanner time: %v %s\n", time.Since(t), logSuffix)
@@ -349,6 +370,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 	thisHash := hashPath(folder.name)
 	// Store initial compaction state.
 	wasCompacted := into.Compacted
+	atomic.AddUint64(&globalScannerStats.accFolders, 1)
 
 	for {
 		select {
@@ -876,21 +898,21 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 		return false, size
 	}
 
+	atomic.AddUint64(&globalScannerStats.ilmChecks, 1)
 	versionID := oi.VersionID
 	action := i.lifeCycle.ComputeAction(
 		lifecycle.ObjectOpts{
-			Name:                   i.objectPath(),
-			UserTags:               oi.UserTags,
-			ModTime:                oi.ModTime,
-			VersionID:              oi.VersionID,
-			DeleteMarker:           oi.DeleteMarker,
-			IsLatest:               oi.IsLatest,
-			NumVersions:            oi.NumVersions,
-			SuccessorModTime:       oi.SuccessorModTime,
-			RestoreOngoing:         oi.RestoreOngoing,
-			RestoreExpires:         oi.RestoreExpires,
-			TransitionStatus:       oi.TransitionedObject.Status,
-			RemoteTiersImmediately: globalDebugRemoteTiersImmediately,
+			Name:             i.objectPath(),
+			UserTags:         oi.UserTags,
+			ModTime:          oi.ModTime,
+			VersionID:        oi.VersionID,
+			DeleteMarker:     oi.DeleteMarker,
+			IsLatest:         oi.IsLatest,
+			NumVersions:      oi.NumVersions,
+			SuccessorModTime: oi.SuccessorModTime,
+			RestoreOngoing:   oi.RestoreOngoing,
+			RestoreExpires:   oi.RestoreExpires,
+			TransitionStatus: oi.TransitionedObject.Status,
 		})
 	if i.debug {
 		if versionID != "" {
@@ -899,6 +921,8 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), action)
 		}
 	}
+	atomic.AddUint64(&globalScannerStats.actions[action], 1)
+
 	switch action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
 		return applyLifecycleAction(action, oi), 0

cmd/disk-cache-backend.go

@@ -938,6 +938,11 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 		rs = nil
 	}
 
+	if objInfo.IsCompressed() {
+		// Cache isn't compressed.
+		delete(objInfo.UserDefined, ReservedMetadataPrefix+"compression")
+	}
+
 	// For a directory, we need to send an reader that returns no bytes.
 	if HasSuffix(object, SlashSeparator) {
 		// The lock taken above is released when
@@ -945,7 +950,6 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 		gr, gerr := NewGetObjectReaderFromReader(bytes.NewBuffer(nil), objInfo, opts)
 		return gr, numHits, gerr
 	}
-
 	fn, off, length, nErr := NewGetObjectReader(rs, objInfo, opts)
 	if nErr != nil {
 		return nil, numHits, nErr

cmd/erasure-bucket.go

@@ -134,32 +134,8 @@ func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string) (bi B
 	return bucketInfo, nil
 }
 
-// Dangling buckets should be handled appropriately, in this following situation
-// we actually have quorum error to be `nil` but we have some disks where
-// the bucket delete returned `errVolumeNotEmpty` but this is not correct
-// can only happen if there are dangling objects in a bucket. Under such
-// a situation we simply attempt a full delete of the bucket including
-// the dangling objects. All of this happens under a lock and there
-// is no way a user can create buckets and sneak in objects into namespace,
-// so it is safer to do.
-func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs []error, bucket string) {
-	for index, err := range dErrs {
-		if err == errVolumeNotEmpty {
-			// Attempt to delete bucket again.
-			if derr := storageDisks[index].DeleteVol(ctx, bucket, false); derr == errVolumeNotEmpty {
-				_ = storageDisks[index].Delete(ctx, bucket, "", true)
-
-				_ = storageDisks[index].DeleteVol(ctx, bucket, false)
-
-				// Cleanup all the previously incomplete multiparts.
-				_ = storageDisks[index].Delete(ctx, minioMetaMultipartBucket, bucket, true)
-			}
-		}
-	}
-}
-
 // DeleteBucket - deletes a bucket.
-func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	// Collect if all disks report volume not found.
 	defer NSUpdated(bucket, slashSeparator)
 
@@ -171,13 +147,7 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD
 		index := index
 		g.Go(func() error {
 			if storageDisks[index] != nil {
-				if err := storageDisks[index].DeleteVol(ctx, bucket, forceDelete); err != nil {
-					return err
-				}
-				if err := storageDisks[index].Delete(ctx, minioMetaMultipartBucket, bucket, true); err != errFileNotFound {
-					return err
-				}
-				return nil
+				return storageDisks[index].DeleteVol(ctx, bucket, opts.Force)
 			}
 			return errDiskNotFound
 		}, index)
@@ -186,7 +156,7 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD
 	// Wait for all the delete vols to finish.
 	dErrs := g.Wait()
 
-	if forceDelete {
+	if opts.Force {
 		for _, err := range dErrs {
 			if err != nil {
 				undoDeleteBucket(storageDisks, bucket)
@@ -199,20 +169,20 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, forceD
 
 	writeQuorum := getWriteQuorum(len(storageDisks))
 	err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, writeQuorum)
-	if err == errErasureWriteQuorum {
+	if err == errErasureWriteQuorum && !opts.NoRecreate {
 		undoDeleteBucket(storageDisks, bucket)
 	}
 	if err != nil {
 		return toObjectErr(err, bucket)
 	}
 
-	// If we reduce quorum to nil, means we have deleted buckets properly
-	// on some servers in quorum, we should look for volumeNotEmpty errors
-	// and delete those buckets as well.
-	//
-	// let this call succeed, even if client cancels the context
-	// this is to ensure that we don't leave any stale content
-	deleteDanglingBucket(context.Background(), storageDisks, dErrs, bucket)
+	// At this point we have `err == nil` but some errors might be `errVolumeNotEmpty`
+	// we should proceed to attempt a force delete of such buckets.
+	for index, err := range dErrs {
+		if err == errVolumeNotEmpty && storageDisks[index] != nil {
+			storageDisks[index].RenameFile(ctx, bucket, "", minioMetaTmpDeletedBucket, mustGetUUID())
+		}
+	}
 
 	return nil
 }

cmd/erasure-common.go

@@ -22,16 +22,6 @@ import (
 	"sync"
 )
 
-func (er erasureObjects) getLocalDisks() (localDisks []StorageAPI) {
-	disks := er.getDisks()
-	for _, disk := range disks {
-		if disk != nil && disk.IsLocal() {
-			localDisks = append(localDisks, disk)
-		}
-	}
-	return localDisks
-}
-
 func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	disks := er.getDisks()
 	// Based on the random shuffling return back randomized disks.

cmd/erasure-healing-common.go

@@ -40,7 +40,11 @@ func commonTime(modTimes []time.Time, dataDirs []string) (modTime time.Time, dat
 	}
 
 	for _, dataDir := range dataDirs {
-		if dataDir == "" {
+		if dataDir == errorDir {
+			continue
+		}
+		if dataDir == delMarkerDir {
+			dataDirOccurenceMap[delMarkerDir]++
 			continue
 		}
 		dataDirOccurenceMap[dataDir]++
@@ -97,6 +101,11 @@ func listObjectModtimes(partsMetadata []FileInfo, errs []error) (modTimes []time
 	return modTimes
 }
 
+const (
+	errorDir     = "error-dir"
+	delMarkerDir = ""
+)
+
 // Notes:
 // There are 5 possible states a disk could be in,
 // 1. __online__             - has the latest copy of xl.meta - returned by listOnlineDisks
@@ -130,6 +139,7 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error)
 	dataDirs := make([]string, len(partsMetadata))
 	for idx, fi := range partsMetadata {
 		if errs[idx] != nil {
+			dataDirs[idx] = errorDir
 			continue
 		}
 		dataDirs[idx] = fi.DataDir
@@ -151,9 +161,9 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error)
 }
 
 // Returns the latest updated FileInfo files and error in case of failure.
-func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []error) (FileInfo, error) {
+func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []error, quorum int) (FileInfo, error) {
 	// There should be atleast half correct entries, if not return failure
-	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, len(partsMetadata)/2); reducedErr != nil {
+	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, quorum); reducedErr != nil {
 		return FileInfo{}, reducedErr
 	}
 
@@ -182,7 +192,8 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []err
 			count++
 		}
 	}
-	if count < len(partsMetadata)/2 {
+
+	if count < quorum {
 		return FileInfo{}, errErasureReadQuorum
 	}
 

cmd/erasure-healing-common_test.go

@@ -190,7 +190,7 @@ func TestListOnlineDisks(t *testing.T) {
 			}
 
 			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-			fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
+			fi, err := getLatestFileInfo(ctx, partsMetadata, errs, getReadQuorum(len(disks)))
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
 			}
@@ -365,7 +365,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 			}
 
 			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", true)
-			fi, err := getLatestFileInfo(ctx, partsMetadata, errs)
+			fi, err := getLatestFileInfo(ctx, partsMetadata, errs, getReadQuorum(len(disks)))
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
 			}
@@ -421,7 +421,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 
 			}
 			partsMetadata, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", true)
-			_, err = getLatestFileInfo(ctx, partsMetadata, errs)
+			_, err = getLatestFileInfo(ctx, partsMetadata, errs, len(disks)/2)
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
 			}

cmd/erasure-healing.go

@@ -53,7 +53,7 @@ func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts mad
 }
 
 // Heal bucket - create buckets on disks where it does not exist.
-func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints []string, bucket string, writeQuorum int,
+func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints []Endpoint, bucket string, writeQuorum int,
 	opts madmin.HealOpts) (res madmin.HealResultItem, err error) {
 
 	// Initialize sync waitgroup.
@@ -114,7 +114,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 	for i := range beforeState {
 		res.Before.Drives = append(res.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[i],
+			Endpoint: storageEndpoints[i].String(),
 			State:    beforeState[i],
 		})
 	}
@@ -124,7 +124,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 		for i := range beforeState {
 			res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: storageEndpoints[i],
+				Endpoint: storageEndpoints[i].String(),
 				State:    madmin.DriveStateOk,
 			})
 		}
@@ -159,7 +159,7 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, storageEndpoints
 	for i := range afterState {
 		res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[i],
+			Endpoint: storageEndpoints[i].String(),
 			State:    afterState[i],
 		})
 	}
@@ -275,7 +275,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	// Re-read when we have lock...
 	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, object, versionID, true)
 
-	if _, err = getLatestFileInfo(ctx, partsMetadata, errs); err != nil {
+	if _, err = getLatestFileInfo(ctx, partsMetadata, errs, er.defaultParityCount); err != nil {
 		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, []error{}, opts)
 	}
 
@@ -345,24 +345,24 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 			disksToHealCount++
 			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: storageEndpoints[i],
+				Endpoint: storageEndpoints[i].String(),
 				State:    driveState,
 			})
 			result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: storageEndpoints[i],
+				Endpoint: storageEndpoints[i].String(),
 				State:    driveState,
 			})
 			continue
 		}
 		result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[i],
+			Endpoint: storageEndpoints[i].String(),
 			State:    driveState,
 		})
 		result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[i],
+			Endpoint: storageEndpoints[i].String(),
 			State:    driveState,
 		})
 	}
@@ -609,7 +609,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 
 	// Prepare object creation in all disks
 	for i, err := range errs {
-		drive := storageEndpoints[i]
+		drive := storageEndpoints[i].String()
 		switch err {
 		case nil:
 			hr.Before.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateOk}
@@ -650,7 +650,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 
 // Populates default heal result item entries with possible values when we are returning prematurely.
 // This is to ensure that in any circumstance we are not returning empty arrays with wrong values.
-func (er erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints []string, errs []error, bucket, object, versionID string) madmin.HealResultItem {
+func (er erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints []Endpoint, errs []error, bucket, object, versionID string) madmin.HealResultItem {
 	// Initialize heal result object
 	result := madmin.HealResultItem{
 		Type:       madmin.HealItemObject,
@@ -673,12 +673,12 @@ func (er erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageA
 		if disk == nil {
 			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: storageEndpoints[index],
+				Endpoint: storageEndpoints[index].String(),
 				State:    madmin.DriveStateOffline,
 			})
 			result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: storageEndpoints[index],
+				Endpoint: storageEndpoints[index].String(),
 				State:    madmin.DriveStateOffline,
 			})
 			continue
@@ -692,12 +692,12 @@ func (er erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageA
 		}
 		result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[index],
+			Endpoint: storageEndpoints[index].String(),
 			State:    driveState,
 		})
 		result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{
 			UUID:     "",
-			Endpoint: storageEndpoints[index],
+			Endpoint: storageEndpoints[index].String(),
 			State:    driveState,
 		})
 	}

cmd/erasure-healing_test.go

@@ -416,12 +416,12 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	fi, err := getLatestFileInfo(ctx, fileInfos, errs, er.defaultParityCount)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
+	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
 		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
 	}
 
@@ -441,7 +441,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+	nfi, err := getLatestFileInfo(ctx, fileInfos, errs, er.defaultParityCount)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
@@ -467,7 +467,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
-	nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+	nfi, err = getLatestFileInfo(ctx, fileInfos, errs, er.defaultParityCount)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
@@ -565,7 +565,7 @@ func TestHealObjectErasure(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile); err != nil {
+	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
 		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
 	}
 

cmd/erasure-metadata.go

@@ -388,7 +388,7 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
 // writeQuorum is the min required disks to write data.
 func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []error, defaultParityCount int) (objectReadQuorum, objectWriteQuorum int, err error) {
 	// get the latest updated Metadata and a count of all the latest updated FileInfo(s)
-	latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, errs)
+	latestFileInfo, err := getLatestFileInfo(ctx, partsMetaData, errs, defaultParityCount)
 	if err != nil {
 		return 0, 0, err
 	}

cmd/erasure-server-pool.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -150,33 +151,45 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
 // GetRawData will return all files with a given raw path to the callback.
 // Errors are ignored, only errors from the callback are returned.
 // For now only direct file paths are supported.
-func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
+func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error {
+	found := 0
 	for _, s := range z.serverPools {
 		for _, disks := range s.erasureDisks {
 			for i, disk := range disks {
 				if disk == OfflineDisk {
 					continue
 				}
-				si, err := disk.StatInfoFile(ctx, volume, file)
+				stats, err := disk.StatInfoFile(ctx, volume, file, true)
 				if err != nil {
 					continue
 				}
-				r, err := disk.ReadFileStream(ctx, volume, file, 0, si.Size)
-				if err != nil {
-					continue
-				}
-				defer r.Close()
 				did, err := disk.GetDiskID()
 				if err != nil {
 					did = fmt.Sprintf("disk-%d", i)
 				}
-				err = fn(r, disk.Hostname(), did, pathJoin(volume, file), si.Size, si.ModTime)
-				if err != nil {
-					return err
+				for _, si := range stats {
+					found++
+					var r io.ReadCloser
+					if !si.Dir {
+						r, err = disk.ReadFileStream(ctx, volume, si.Name, 0, si.Size)
+						if err != nil {
+							continue
+						}
+					} else {
+						r = io.NopCloser(bytes.NewBuffer([]byte{}))
+					}
+					err = fn(r, disk.Hostname(), did, pathJoin(volume, si.Name), si.Size, si.ModTime, si.Dir)
+					r.Close()
+					if err != nil {
+						return err
+					}
 				}
 			}
 		}
 	}
+	if found == 0 {
+		return errFileNotFound
+	}
 	return nil
 }
 
@@ -619,6 +632,10 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 	// Return the first encountered error
 	for _, err := range errs {
 		if err != nil {
+			if _, ok := err.(BucketExists); !ok {
+				// Delete created buckets, ignoring errors.
+				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{Force: true, NoRecreate: true})
+			}
 			return err
 		}
 	}
@@ -630,7 +647,7 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 		meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
 	}
 
-	if err := meta.Save(ctx, z); err != nil {
+	if err := meta.Save(context.Background(), z); err != nil {
 		return toObjectErr(err, bucket)
 	}
 
@@ -1411,14 +1428,14 @@ func (z *erasureServerPools) IsTaggingSupported() bool {
 // DeleteBucket - deletes a bucket on all serverPools simultaneously,
 // even if one of the serverPools fail to delete buckets, we proceed to
 // undo a successful operation.
-func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	g := errgroup.WithNErrs(len(z.serverPools))
 
 	// Delete buckets in parallel across all serverPools.
 	for index := range z.serverPools {
 		index := index
 		g.Go(func() error {
-			return z.serverPools[index].DeleteBucket(ctx, bucket, forceDelete)
+			return z.serverPools[index].DeleteBucket(ctx, bucket, opts)
 		}, index)
 	}
 
@@ -1428,14 +1445,15 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, fo
 	// buckets operation by creating all the buckets again.
 	for _, err := range errs {
 		if err != nil {
-			if !z.SinglePool() {
-				undoDeleteBucketServerPools(ctx, bucket, z.serverPools, errs)
+			if !z.SinglePool() && !opts.NoRecreate {
+				undoDeleteBucketServerPools(context.Background(), bucket, z.serverPools, errs)
 			}
 			return err
 		}
 	}
 
-	deleteBucketMetadata(ctx, z, bucket)
+	// Purge the entire bucket metadata entirely.
+	z.renameAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
 
 	// Success.
 	return nil

cmd/erasure-sets.go

@@ -41,14 +41,11 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/console"
-	"github.com/minio/pkg/env"
 )
 
 // setsDsyncLockers is encapsulated type for Close()
 type setsDsyncLockers [][]dsync.NetLocker
 
-const envMinioDeleteCleanupInterval = "MINIO_DELETE_CLEANUP_INTERVAL"
-
 // erasureSets implements ObjectLayer combining a static list of erasure coded
 // object sets. NOTE: There is no dynamic scaling allowed or intended in
 // current design.
@@ -257,7 +254,6 @@ func (s *erasureSets) connectDisks() {
 				s.erasureDisks[setIndex][diskIndex] = disk
 			}
 			disk.SetDiskLoc(s.poolIndex, setIndex, diskIndex)
-			s.endpointStrings[setIndex*s.setDriveCount+diskIndex] = disk.String()
 			setsJustConnected[setIndex] = true
 			s.erasureDisksMu.Unlock()
 		}(endpoint)
@@ -314,14 +310,14 @@ func (s *erasureSets) GetLockers(setIndex int) func() ([]dsync.NetLocker, string
 	}
 }
 
-func (s *erasureSets) GetEndpoints(setIndex int) func() []string {
-	return func() []string {
+func (s *erasureSets) GetEndpoints(setIndex int) func() []Endpoint {
+	return func() []Endpoint {
 		s.erasureDisksMu.RLock()
 		defer s.erasureDisksMu.RUnlock()
 
-		eps := make([]string, s.setDriveCount)
+		eps := make([]Endpoint, s.setDriveCount)
 		for i := 0; i < s.setDriveCount; i++ {
-			eps[i] = s.endpointStrings[setIndex*s.setDriveCount+i]
+			eps[i] = s.endpoints[setIndex*s.setDriveCount+i]
 		}
 		return eps
 	}
@@ -348,6 +344,9 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	setDriveCount := len(format.Erasure.Sets[0])
 
 	endpointStrings := make([]string, len(endpoints))
+	for i, endpoint := range endpoints {
+		endpointStrings[i] = endpoint.String()
+	}
 
 	// Initialize the erasure sets instance.
 	s := &erasureSets{
@@ -416,6 +415,9 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 			if err != nil {
 				continue
 			}
+			if m != i || n != j {
+				return nil, fmt.Errorf("found a disk in an unexpected location, pool: %d, found (set=%d, disk=%d) expected (set=%d, disk=%d): %s(%s)", poolIdx, m, n, i, j, disk, diskID)
+			}
 			disk.SetDiskLoc(s.poolIndex, m, n)
 			s.endpointStrings[m*setDriveCount+n] = disk.String()
 			s.erasureDisks[m][n] = disk
@@ -437,19 +439,11 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 		}
 	}
 
-	// cleanup ".trash/" folder every 5m minutes with sufficient sleep cycles, between each
-	// deletes a dynamic sleeper is used with a factor of 10 ratio with max delay between
-	// deletes to be 2 seconds.
-	deletedObjectsCleanupInterval, err := time.ParseDuration(env.Get(envMinioDeleteCleanupInterval, "5m"))
-	if err != nil {
-		return nil, err
-	}
-
 	// start cleanup stale uploads go-routine.
-	go s.cleanupStaleUploads(ctx, GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
+	go s.cleanupStaleUploads(ctx)
 
 	// start cleanup of deleted objects.
-	go s.cleanupDeletedObjects(ctx, deletedObjectsCleanupInterval)
+	go s.cleanupDeletedObjects(ctx)
 
 	// Start the disk monitoring and connect routine.
 	go s.monitorAndConnectEndpoints(ctx, defaultMonitorConnectEndpointInterval)
@@ -457,8 +451,11 @@ func newErasureSets(ctx context.Context, endpoints Endpoints, storageDisks []Sto
 	return s, nil
 }
 
-func (s *erasureSets) cleanupDeletedObjects(ctx context.Context, cleanupInterval time.Duration) {
-	timer := time.NewTimer(cleanupInterval)
+// cleanup ".trash/" folder every 5m minutes with sufficient sleep cycles, between each
+// deletes a dynamic sleeper is used with a factor of 10 ratio with max delay between
+// deletes to be 2 seconds.
+func (s *erasureSets) cleanupDeletedObjects(ctx context.Context) {
+	timer := time.NewTimer(globalAPIConfig.getDeleteCleanupInterval())
 	defer timer.Stop()
 
 	for {
@@ -467,7 +464,7 @@ func (s *erasureSets) cleanupDeletedObjects(ctx context.Context, cleanupInterval
 			return
 		case <-timer.C:
 			// Reset for the next interval
-			timer.Reset(cleanupInterval)
+			timer.Reset(globalAPIConfig.getDeleteCleanupInterval())
 
 			for _, set := range s.sets {
 				set.cleanupDeletedObjects(ctx)
@@ -476,8 +473,8 @@ func (s *erasureSets) cleanupDeletedObjects(ctx context.Context, cleanupInterval
 	}
 }
 
-func (s *erasureSets) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
-	timer := time.NewTimer(cleanupInterval)
+func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
+	timer := time.NewTimer(globalAPIConfig.getStaleUploadsCleanupInterval())
 	defer timer.Stop()
 
 	for {
@@ -486,10 +483,10 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context, cleanupInterval,
 			return
 		case <-timer.C:
 			// Reset for the next interval
-			timer.Reset(cleanupInterval)
+			timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
 
 			for _, set := range s.sets {
-				set.cleanupStaleUploads(ctx, expiry)
+				set.cleanupStaleUploads(ctx, globalAPIConfig.getStaleUploadsExpiry())
 			}
 		}
 	}
@@ -525,10 +522,15 @@ func auditObjectErasureSet(ctx context.Context, object string, set *erasureObjec
 
 	object = decodeDirObject(object)
 
+	var disksEndpoints []string
+	for _, endpoint := range set.getEndpoints() {
+		disksEndpoints = append(disksEndpoints, endpoint.String())
+	}
+
 	op := auditObjectOp{
 		Pool:  set.poolIndex + 1,
 		Set:   set.setIndex + 1,
-		Disks: set.getEndpoints(),
+		Disks: disksEndpoints,
 	}
 
 	var objectErasureSetTag *auditObjectErasureMap
@@ -802,14 +804,14 @@ func (s *erasureSets) IsTaggingSupported() bool {
 // DeleteBucket - deletes a bucket on all sets simultaneously,
 // even if one of the sets fail to delete buckets, we proceed to
 // undo a successful operation.
-func (s *erasureSets) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (s *erasureSets) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	g := errgroup.WithNErrs(len(s.sets))
 
 	// Delete buckets in parallel across all sets.
 	for index := range s.sets {
 		index := index
 		g.Go(func() error {
-			return s.sets[index].DeleteBucket(ctx, bucket, forceDelete)
+			return s.sets[index].DeleteBucket(ctx, bucket, opts)
 		}, index)
 	}
 
@@ -817,7 +819,7 @@ func (s *erasureSets) DeleteBucket(ctx context.Context, bucket string, forceDele
 	// For any failure, we attempt undo all the delete buckets operation
 	// by creating buckets again on all sets which were successfully deleted.
 	for _, err := range errs {
-		if err != nil {
+		if err != nil && !opts.NoRecreate {
 			undoDeleteBucketSets(ctx, bucket, s.sets, errs)
 			return err
 		}
@@ -1300,7 +1302,6 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			if storageDisks[index] != nil {
 				storageDisks[index].SetDiskLoc(s.poolIndex, m, n)
 				s.erasureDisks[m][n] = storageDisks[index]
-				s.endpointStrings[m*s.setDriveCount+n] = storageDisks[index].String()
 			}
 		}
 

cmd/erasure.go

@@ -57,7 +57,7 @@ type erasureObjects struct {
 
 	// getEndpoints returns list of endpoint strings belonging this set.
 	// some may be local and some remote.
-	getEndpoints func() []string
+	getEndpoints func() []Endpoint
 
 	// Locker mutex map.
 	nsMutex *nsLockMap
@@ -164,7 +164,7 @@ func getOnlineOfflineDisksStats(disksInfo []madmin.Disk) (onlineDisks, offlineDi
 }
 
 // getDisksInfo - fetch disks info across all other storage API.
-func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Disk, errs []error) {
+func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.Disk, errs []error) {
 	disksInfo = make([]madmin.Disk, len(disks))
 
 	g := errgroup.WithNErrs(len(disks))
@@ -175,7 +175,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 				logger.LogIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]))
 				disksInfo[index] = madmin.Disk{
 					State:    diskErrToDriveState(errDiskNotFound),
-					Endpoint: endpoints[index],
+					Endpoint: endpoints[index].String(),
 				}
 				// Storage disk is empty, perhaps ignored disk or not available.
 				return errDiskNotFound
@@ -222,7 +222,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 }
 
 // Get an aggregated storage info across all disks.
-func getStorageInfo(disks []StorageAPI, endpoints []string) (StorageInfo, []error) {
+func getStorageInfo(disks []StorageAPI, endpoints []Endpoint) (StorageInfo, []error) {
 	disksInfo, errs := getDisksInfo(disks, endpoints)
 
 	// Sort so that the first element is the smallest.
@@ -245,14 +245,20 @@ func (er erasureObjects) StorageInfo(ctx context.Context) (StorageInfo, []error)
 
 // LocalStorageInfo - returns underlying local storage statistics.
 func (er erasureObjects) LocalStorageInfo(ctx context.Context) (StorageInfo, []error) {
-	disks := er.getLocalDisks()
-	endpoints := make([]string, len(disks))
-	for i, disk := range disks {
-		if disk != nil {
-			endpoints[i] = disk.String()
+	disks := er.getDisks()
+	endpoints := er.getEndpoints()
+
+	var localDisks []StorageAPI
+	var localEndpoints []Endpoint
+
+	for i, endpoint := range endpoints {
+		if endpoint.IsLocal {
+			localDisks = append(localDisks, disks[i])
+			localEndpoints = append(localEndpoints, endpoint)
 		}
 	}
-	return getStorageInfo(disks, endpoints)
+
+	return getStorageInfo(localDisks, localEndpoints)
 }
 
 func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, healing bool) {

cmd/fs-v1-multipart.go

@@ -847,8 +847,8 @@ func (fs *FSObjects) AbortMultipartUpload(ctx context.Context, bucket, object, u
 // Removes multipart uploads if any older than `expiry` duration
 // on all buckets for every `cleanupInterval`, this function is
 // blocking and should be run in a go-routine.
-func (fs *FSObjects) cleanupStaleUploads(ctx context.Context, cleanupInterval, expiry time.Duration) {
-	timer := time.NewTimer(cleanupInterval)
+func (fs *FSObjects) cleanupStaleUploads(ctx context.Context) {
+	timer := time.NewTimer(globalAPIConfig.getStaleUploadsCleanupInterval())
 	defer timer.Stop()
 
 	for {
@@ -857,7 +857,9 @@ func (fs *FSObjects) cleanupStaleUploads(ctx context.Context, cleanupInterval, e
 			return
 		case <-timer.C:
 			// Reset for the next interval
-			timer.Reset(cleanupInterval)
+			timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval())
+
+			expiry := globalAPIConfig.getStaleUploadsExpiry()
 
 			now := time.Now()
 			entries, err := readDir(pathJoin(fs.fsPath, minioMetaMultipartBucket))

cmd/fs-v1-multipart_test.go

@@ -26,6 +26,8 @@ import (
 	"sync"
 	"testing"
 	"time"
+
+	"github.com/minio/minio/internal/config/api"
 )
 
 // Tests cleanup multipart uploads for filesystem backend.
@@ -49,11 +51,23 @@ func TestFSCleanupMultipartUploadsInRoutine(t *testing.T) {
 		t.Fatal("Unexpected err: ", err)
 	}
 
+	globalAPIConfig.init(api.Config{
+		ListQuorum:                  "optimal",
+		StaleUploadsExpiry:          time.Millisecond,
+		StaleUploadsCleanupInterval: time.Millisecond,
+	}, obj.SetDriveCounts())
+
+	defer func() {
+		globalAPIConfig.init(api.Config{
+			ListQuorum: "optimal",
+		}, obj.SetDriveCounts())
+	}()
+
 	var cleanupWg sync.WaitGroup
 	cleanupWg.Add(1)
 	go func() {
 		defer cleanupWg.Done()
-		fs.cleanupStaleUploads(ctx, time.Millisecond, 0)
+		fs.cleanupStaleUploads(ctx)
 	}()
 
 	// Wait for 100ms such that - we have given enough time for

cmd/fs-v1.go

@@ -177,7 +177,7 @@ func NewFSObjectLayer(fsPath string) (ObjectLayer, error) {
 	// or cause changes on backend format.
 	fs.fsFormatRlk = rlk
 
-	go fs.cleanupStaleUploads(ctx, GlobalStaleUploadsCleanupInterval, GlobalStaleUploadsExpiry)
+	go fs.cleanupStaleUploads(ctx)
 	go intDataUpdateTracker.start(ctx, fsPath)
 
 	// Return successfully initialized object layer.
@@ -393,6 +393,8 @@ func (fs *FSObjects) scanBucket(ctx context.Context, bucket string, cache dataUs
 		}
 
 		oi := fsMeta.ToObjectInfo(bucket, object, fi)
+		atomic.AddUint64(&globalScannerStats.accTotalVersions, 1)
+		atomic.AddUint64(&globalScannerStats.accTotalObjects, 1)
 		sz := item.applyActions(ctx, fs, oi, &sizeSummary{})
 		if sz >= 0 {
 			return sizeSummary{totalSize: sz, versions: 1}, nil
@@ -586,7 +588,7 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 
 // DeleteBucket - delete a bucket and all the metadata associated
 // with the bucket including pending multipart, object metadata.
-func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
 	defer NSUpdated(bucket, slashSeparator)
 
 	atomic.AddInt64(&fs.activeIOCount, 1)
@@ -599,7 +601,7 @@ func (fs *FSObjects) DeleteBucket(ctx context.Context, bucket string, forceDelet
 		return toObjectErr(err, bucket)
 	}
 
-	if !forceDelete {
+	if !opts.Force {
 		// Attempt to delete regular bucket.
 		if err = fsRemoveDir(ctx, bucketDir); err != nil {
 			return toObjectErr(err, bucket)
@@ -1508,7 +1510,7 @@ func (fs *FSObjects) RestoreTransitionedObject(ctx context.Context, bucket, obje
 // GetRawData returns raw file data to the callback.
 // Errors are ignored, only errors from the callback are returned.
 // For now only direct file paths are supported.
-func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
+func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time, isDir bool) error) error {
 	f, err := os.Open(filepath.Join(fs.fsPath, volume, file))
 	if err != nil {
 		return nil
@@ -1518,5 +1520,5 @@ func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn fun
 	if err != nil || st.IsDir() {
 		return nil
 	}
-	return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime())
+	return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime(), st.IsDir())
 }

cmd/fs-v1_test.go

@@ -224,16 +224,16 @@ func TestFSDeleteBucket(t *testing.T) {
 	}
 
 	// Test with an invalid bucket name
-	if err = fs.DeleteBucket(GlobalContext, "fo", false); !isSameType(err, BucketNotFound{}) {
+	if err = fs.DeleteBucket(GlobalContext, "fo", DeleteBucketOptions{}); !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 
 	// Test with an inexistant bucket
-	if err = fs.DeleteBucket(GlobalContext, "foobucket", false); !isSameType(err, BucketNotFound{}) {
+	if err = fs.DeleteBucket(GlobalContext, "foobucket", DeleteBucketOptions{}); !isSameType(err, BucketNotFound{}) {
 		t.Fatal("Unexpected error: ", err)
 	}
 	// Test with a valid case
-	if err = fs.DeleteBucket(GlobalContext, bucketName, false); err != nil {
+	if err = fs.DeleteBucket(GlobalContext, bucketName, DeleteBucketOptions{}); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}
 
@@ -241,7 +241,7 @@ func TestFSDeleteBucket(t *testing.T) {
 
 	// Delete bucket should get error disk not found.
 	os.RemoveAll(disk)
-	if err = fs.DeleteBucket(GlobalContext, bucketName, false); err != nil {
+	if err = fs.DeleteBucket(GlobalContext, bucketName, DeleteBucketOptions{}); err != nil {
 		if !isSameType(err, BucketNotFound{}) {
 			t.Fatal("Unexpected error: ", err)
 		}

cmd/gateway-main.go

@@ -154,8 +154,6 @@ func ValidateGatewayArguments(serverAddr, endpointAddr string) error {
 
 // StartGateway - handler for 'minio gateway <name>'.
 func StartGateway(ctx *cli.Context, gw Gateway) {
-	defer globalDNSCache.Stop()
-
 	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
 
 	go handleSignals()
@@ -262,13 +260,21 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
-	httpServer := xhttp.NewServer([]string{globalMinioAddr},
-		criticalErrorHandler{corsHandler(router)}, getCert)
+	listeners := ctx.Int("listeners")
+	if listeners == 0 {
+		listeners = 1
+	}
+	addrs := make([]string, 0, listeners)
+	for i := 0; i < listeners; i++ {
+		addrs = append(addrs, globalMinioAddr)
+	}
+
+	httpServer := xhttp.NewServer(addrs, criticalErrorHandler{corsHandler(router)}, getCert)
 	httpServer.BaseContext = func(listener net.Listener) context.Context {
 		return GlobalContext
 	}
 	go func() {
-		globalHTTPServerErrorCh <- httpServer.Start()
+		globalHTTPServerErrorCh <- httpServer.Start(GlobalContext)
 	}()
 
 	globalObjLayerMutex.Lock()

cmd/gateway/azure/gateway-azure.go

@@ -38,7 +38,7 @@ import (
 
 	"github.com/Azure/azure-pipeline-go/pipeline"
 	"github.com/Azure/azure-storage-blob-go/azblob"
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 	"github.com/minio/cli"
 	"github.com/minio/madmin-go"
 	miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
@@ -623,8 +623,8 @@ func (a *azureObjects) ListBuckets(ctx context.Context) (buckets []minio.BucketI
 }
 
 // DeleteBucket - delete a container on azure, uses Azure equivalent `ContainerURL.Delete`.
-func (a *azureObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
-	if !forceDelete {
+func (a *azureObjects) DeleteBucket(ctx context.Context, bucket string, opts minio.DeleteBucketOptions) error {
+	if !opts.Force {
 		// Check if the container is empty before deleting it.
 		result, err := a.ListObjects(ctx, bucket, "", "", "", 1)
 		if err != nil {

cmd/gateway/gcs/gateway-gcs.go

@@ -470,7 +470,7 @@ func (l *gcsGateway) ListBuckets(ctx context.Context) (buckets []minio.BucketInf
 }
 
 // DeleteBucket delete a bucket on GCS.
-func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (l *gcsGateway) DeleteBucket(ctx context.Context, bucket string, opts minio.DeleteBucketOptions) error {
 	itObject := l.client.Bucket(bucket).Objects(ctx, &storage.Query{
 		Delimiter: minio.SlashSeparator,
 		Versions:  false,

cmd/gateway/hdfs/gateway-hdfs.go

@@ -305,11 +305,11 @@ func (n *hdfsObjects) hdfsPathJoin(args ...string) string {
 	return minio.PathJoin(append([]string{n.subPath, hdfsSeparator}, args...)...)
 }
 
-func (n *hdfsObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (n *hdfsObjects) DeleteBucket(ctx context.Context, bucket string, opts minio.DeleteBucketOptions) error {
 	if !hdfsIsValidBucketName(bucket) {
 		return minio.BucketNameInvalid{Bucket: bucket}
 	}
-	if forceDelete {
+	if opts.Force {
 		return hdfsToObjectErr(ctx, n.clnt.RemoveAll(n.hdfsPathJoin(bucket)), bucket)
 	}
 	return hdfsToObjectErr(ctx, n.clnt.Remove(n.hdfsPathJoin(bucket)), bucket)

cmd/gateway/s3/gateway-s3-chain.go

@@ -0,0 +1,103 @@
+/*
+ * MinIO Object Storage (c) 2021 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package s3
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/minio/minio-go/v7/pkg/credentials"
+)
+
+// A Chain will search for a provider which returns credentials
+// and cache that provider until Retrieve is called again.
+//
+// The Chain provides a way of chaining multiple providers together
+// which will pick the first available using priority order of the
+// Providers in the list.
+//
+// If none of the Providers retrieve valid credentials Value, ChainProvider's
+// Retrieve() will return the no credentials value.
+//
+// If a Provider is found which returns valid credentials Value ChainProvider
+// will cache that Provider for all calls to IsExpired(), until Retrieve is
+// called again after IsExpired() is true.
+//
+//     creds := credentials.NewChainCredentials(
+//         []credentials.Provider{
+//             &credentials.EnvAWSS3{},
+//             &credentials.EnvMinio{},
+//         })
+//
+//     // Usage of ChainCredentials.
+//     mc, err := minio.NewWithCredentials(endpoint, creds, secure, "us-east-1")
+//     if err != nil {
+//          log.Fatalln(err)
+//     }
+//
+type Chain struct {
+	Providers []credentials.Provider
+	curr      credentials.Provider
+}
+
+// NewChainCredentials returns a pointer to a new Credentials object
+// wrapping a chain of providers.
+func NewChainCredentials(providers []credentials.Provider) *credentials.Credentials {
+	for _, p := range providers {
+		if p == nil {
+			panic("providers cannot be uninitialized")
+		}
+	}
+	return credentials.New(&Chain{
+		Providers: append([]credentials.Provider{}, providers...),
+	})
+}
+
+// Retrieve returns the credentials value, returns no credentials(anonymous)
+// if no credentials provider returned any value.
+//
+// If a provider is found with credentials, it will be cached and any calls
+// to IsExpired() will return the expired state of the cached provider.
+func (c *Chain) Retrieve() (credentials.Value, error) {
+	for _, p := range c.Providers {
+		creds, _ := p.Retrieve()
+		if creds.AccessKeyID != "" && !p.IsExpired() {
+			// Only return credentials that are
+			// available and not expired.
+			c.curr = p
+			return creds, nil
+		}
+	}
+
+	providers := make([]string, 0, len(c.Providers))
+	for _, p := range c.Providers {
+		providers = append(providers, reflect.TypeOf(p).String())
+	}
+
+	return credentials.Value{}, fmt.Errorf("no credentials found in %s cannot proceed", providers)
+
+}
+
+// IsExpired will returned the expired state of the currently cached provider
+// if there is one. If there is no current provider, true will be returned.
+func (c *Chain) IsExpired() bool {
+	if c.curr != nil {
+		return c.curr.IsExpired()
+	}
+
+	return true
+}

cmd/gateway/s3/gateway-s3-sse.go

@@ -113,7 +113,7 @@ func (l *s3EncObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 			} else {
 				objects = append(objects, obj)
 			}
-			if len(objects) > maxKeys {
+			if maxKeys > 0 && len(objects) > maxKeys {
 				break
 			}
 		}
@@ -130,7 +130,7 @@ func (l *s3EncObjects) ListObjectsV2(ctx context.Context, bucket, prefix, contin
 				prefixes = append(prefixes, p)
 			}
 		}
-		if (len(objects) > maxKeys) || !loi.IsTruncated {
+		if (maxKeys > 0 && len(objects) > maxKeys) || !loi.IsTruncated {
 			break
 		}
 	}
@@ -784,7 +784,7 @@ func (l *s3EncObjects) getStalePartsForBucket(ctx context.Context, bucket string
 	return
 }
 
-func (l *s3EncObjects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (l *s3EncObjects) DeleteBucket(ctx context.Context, bucket string, opts minio.DeleteBucketOptions) error {
 	var prefix, continuationToken, delimiter, startAfter string
 	expParts := make(map[string]string)
 

cmd/gateway/s3/gateway-s3.go

@@ -146,7 +146,6 @@ func randString(n int, src rand.Source, prefix string) string {
 var defaultProviders = []credentials.Provider{
 	&credentials.EnvAWS{},
 	&credentials.FileAWSCredentials{},
-	&credentials.EnvMinio{},
 }
 
 // Chains all credential types, in the following order:
@@ -160,15 +159,17 @@ var defaultAWSCredProviders = []credentials.Provider{
 	&credentials.EnvAWS{},
 	&credentials.FileAWSCredentials{},
 	&credentials.IAM{
+		// you can specify a custom STS endpoint.
+		Endpoint: env.Get("MINIO_GATEWAY_S3_STS_ENDPOINT", ""),
 		Client: &http.Client{
 			Transport: minio.NewGatewayHTTPTransport(),
 		},
 	},
-	&credentials.EnvMinio{},
 }
 
-// newS3 - Initializes a new client by auto probing S3 server signature.
-func newS3(urlStr string, tripper http.RoundTripper) (*miniogo.Core, error) {
+// new - Initializes a new client by auto probing S3 server signature.
+func (g *S3) new(creds madmin.Credentials, transport http.RoundTripper) (*miniogo.Core, error) {
+	urlStr := g.host
 	if urlStr == "" {
 		urlStr = "https://s3.amazonaws.com"
 	}
@@ -184,30 +185,70 @@ func newS3(urlStr string, tripper http.RoundTripper) (*miniogo.Core, error) {
 		return nil, err
 	}
 
-	var creds *credentials.Credentials
+	var chainCreds *credentials.Credentials
 	if s3utils.IsAmazonEndpoint(*u) {
 		// If we see an Amazon S3 endpoint, then we use more ways to fetch backend credentials.
 		// Specifically IAM style rotating credentials are only supported with AWS S3 endpoint.
-		creds = credentials.NewChainCredentials(defaultAWSCredProviders)
-
+		chainCreds = NewChainCredentials(defaultAWSCredProviders)
 	} else {
-		creds = credentials.NewChainCredentials(defaultProviders)
+		chainCreds = NewChainCredentials(defaultProviders)
 	}
 
-	options := &miniogo.Options{
-		Creds:        creds,
+	optionsStaticCreds := &miniogo.Options{
+		Creds:        credentials.NewStaticV4(creds.AccessKey, creds.SecretKey, creds.SessionToken),
 		Secure:       secure,
 		Region:       s3utils.GetRegionFromURL(*u),
 		BucketLookup: miniogo.BucketLookupAuto,
-		Transport:    tripper,
+		Transport:    transport,
 	}
 
-	clnt, err := miniogo.New(endpoint, options)
+	optionsChainCreds := &miniogo.Options{
+		Creds:        chainCreds,
+		Secure:       secure,
+		Region:       s3utils.GetRegionFromURL(*u),
+		BucketLookup: miniogo.BucketLookupAuto,
+		Transport:    transport,
+	}
+
+	clntChain, err := miniogo.New(endpoint, optionsChainCreds)
 	if err != nil {
 		return nil, err
 	}
 
-	return &miniogo.Core{Client: clnt}, nil
+	clntStatic, err := miniogo.New(endpoint, optionsStaticCreds)
+	if err != nil {
+		return nil, err
+	}
+
+	if g.debug {
+		clntChain.TraceOn(os.Stderr)
+		clntStatic.TraceOn(os.Stderr)
+	}
+
+	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+
+	if _, err = clntStatic.BucketExists(context.Background(), probeBucketName); err != nil {
+		switch miniogo.ToErrorResponse(err).Code {
+		case "InvalidAccessKeyId":
+			// Check if the provided keys are valid for chain.
+			if _, err = clntChain.BucketExists(context.Background(), probeBucketName); err != nil {
+				if miniogo.ToErrorResponse(err).Code != "AccessDenied" {
+					return nil, err
+				}
+			}
+			return &miniogo.Core{Client: clntChain}, nil
+		case "AccessDenied":
+			// this is a good error means backend is reachable
+			// and credentials are valid but credentials don't
+			// have access to 'probeBucketName' which is harmless.
+			return &miniogo.Core{Client: clntStatic}, nil
+		default:
+			return nil, err
+		}
+	}
+
+	// if static keys are valid always use static keys.
+	return &miniogo.Core{Client: clntStatic}, nil
 }
 
 // NewGatewayLayer returns s3 ObjectLayer.
@@ -221,24 +262,11 @@ func (g *S3) NewGatewayLayer(creds madmin.Credentials) (minio.ObjectLayer, error
 
 	// creds are ignored here, since S3 gateway implements chaining
 	// all credentials.
-	clnt, err := newS3(g.host, t)
+	clnt, err := g.new(creds, t)
 	if err != nil {
 		return nil, err
 	}
 
-	if g.debug {
-		clnt.Client.TraceOn(os.Stderr)
-	}
-
-	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
-
-	// Check if the provided keys are valid.
-	if _, err = clnt.BucketExists(context.Background(), probeBucketName); err != nil {
-		if miniogo.ToErrorResponse(err).Code != "AccessDenied" {
-			return nil, err
-		}
-	}
-
 	s := s3Objects{
 		Client:  clnt,
 		Metrics: metrics,
@@ -282,11 +310,17 @@ func (l *s3Objects) Shutdown(ctx context.Context) error {
 // StorageInfo is not relevant to S3 backend.
 func (l *s3Objects) StorageInfo(ctx context.Context) (si minio.StorageInfo, _ []error) {
 	si.Backend.Type = madmin.Gateway
-	host := l.Client.EndpointURL().Host
-	if l.Client.EndpointURL().Port() == "" {
-		host = l.Client.EndpointURL().Host + ":" + l.Client.EndpointURL().Scheme
+	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
+
+	// check if bucket exists.
+	_, err := l.Client.BucketExists(ctx, probeBucketName)
+	switch miniogo.ToErrorResponse(err).Code {
+	case "", "AccessDenied":
+		si.Backend.GatewayOnline = true
+	default:
+		logger.LogIf(ctx, err)
+		si.Backend.GatewayOnline = false
 	}
-	si.Backend.GatewayOnline = minio.IsBackendOnline(ctx, host)
 	return si, nil
 }
 
@@ -365,7 +399,7 @@ func (l *s3Objects) ListBuckets(ctx context.Context) ([]minio.BucketInfo, error)
 }
 
 // DeleteBucket deletes a bucket on S3
-func (l *s3Objects) DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error {
+func (l *s3Objects) DeleteBucket(ctx context.Context, bucket string, opts minio.DeleteBucketOptions) error {
 	err := l.Client.RemoveBucket(ctx, bucket)
 	if err != nil {
 		return minio.ErrorRespToObjectError(err, bucket)

cmd/globals.go

@@ -30,6 +30,7 @@ import (
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/kms"
+	"github.com/rs/dnscache"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/auth"
@@ -91,14 +92,12 @@ const (
 	// date and server date during signature verification.
 	globalMaxSkewTime = 15 * time.Minute // 15 minutes skew allowed.
 
-	// GlobalStaleUploadsExpiry - Expiry duration after which the uploads in multipart, tmp directory are deemed stale.
+	// GlobalStaleUploadsExpiry - Expiry duration after which the uploads in multipart,
+	// tmp directory are deemed stale.
 	GlobalStaleUploadsExpiry = time.Hour * 24 // 24 hrs.
 
 	// GlobalStaleUploadsCleanupInterval - Cleanup interval when the stale uploads cleanup is initiated.
-	GlobalStaleUploadsCleanupInterval = time.Hour * 12 // 12 hrs.
-
-	// GlobalServiceExecutionInterval - Executes the Lifecycle events.
-	GlobalServiceExecutionInterval = time.Hour * 24 // 24 hrs.
+	GlobalStaleUploadsCleanupInterval = time.Hour * 6 // 6 hrs.
 
 	// Refresh interval to update in-memory iam config cache.
 	globalRefreshIAMInterval = 5 * time.Minute
@@ -257,6 +256,9 @@ var (
 	// Allocated etcd endpoint for config and bucket DNS.
 	globalEtcdClient *etcd.Client
 
+	// Cluster replication manager.
+	globalSiteReplicationSys SiteReplicationSys
+
 	// Is set to true when Bucket federation is requested
 	// and is 'true' when etcdConfig.PathPrefix is empty
 	globalBucketFederation bool
@@ -311,7 +313,9 @@ var (
 
 	globalProxyTransport http.RoundTripper
 
-	globalDNSCache *xhttp.DNSCache
+	globalDNSCache = &dnscache.Resolver{
+		Timeout: 5 * time.Second,
+	}
 
 	globalForwarder *handlers.Forwarder
 
@@ -321,7 +325,6 @@ var (
 
 	globalConsoleSrv *restapi.Server
 
-	globalDebugRemoteTiersImmediately []string
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -22,9 +22,11 @@ import (
 	"sync"
 	"time"
 
-	"github.com/minio/minio/internal/config/api"
-	"github.com/minio/minio/internal/logger"
 	mem "github.com/shirou/gopsutil/v3/mem"
+
+	"github.com/minio/minio/internal/config/api"
+	xioutil "github.com/minio/minio/internal/ioutil"
+	"github.com/minio/minio/internal/logger"
 )
 
 type apiConfig struct {
@@ -40,6 +42,10 @@ type apiConfig struct {
 	replicationWorkers       int
 	replicationFailedWorkers int
 	transitionWorkers        int
+
+	staleUploadsExpiry          time.Duration
+	staleUploadsCleanupInterval time.Duration
+	deleteCleanupInterval       time.Duration
 }
 
 func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
@@ -71,7 +77,8 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		// total_ram / ram_per_request
 		// ram_per_request is (2MiB+128KiB) * driveCount \
 		//    + 2 * 10MiB (default erasure block size v1) + 2 * 1MiB (default erasure block size v2)
-		apiRequestsMaxPerNode = int(maxMem / uint64(maxSetDrives*(blockSizeLarge+blockSizeSmall)+int(blockSizeV1*2+blockSizeV2*2)))
+		blockSize := xioutil.BlockSizeLarge + xioutil.BlockSizeSmall
+		apiRequestsMaxPerNode = int(maxMem / uint64(maxSetDrives*blockSize+int(blockSizeV1*2+blockSizeV2*2)))
 
 		if globalIsErasure {
 			logger.Info("Automatically configured API requests per node based on available memory on the system: %d", apiRequestsMaxPerNode)
@@ -104,6 +111,10 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		globalTransitionState.UpdateWorkers(cfg.TransitionWorkers)
 	}
 	t.transitionWorkers = cfg.TransitionWorkers
+
+	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
+	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
+	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
 }
 
 func (t *apiConfig) getListQuorum() int {
@@ -122,6 +133,39 @@ func (t *apiConfig) getCorsAllowOrigins() []string {
 	return corsAllowOrigins
 }
 
+func (t *apiConfig) getStaleUploadsCleanupInterval() time.Duration {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.staleUploadsCleanupInterval == 0 {
+		return 6 * time.Hour // default 6 hours
+	}
+
+	return t.staleUploadsCleanupInterval
+}
+
+func (t *apiConfig) getStaleUploadsExpiry() time.Duration {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.staleUploadsExpiry == 0 {
+		return 24 * time.Hour // default 24 hours
+	}
+
+	return t.staleUploadsExpiry
+}
+
+func (t *apiConfig) getDeleteCleanupInterval() time.Duration {
+	t.mu.RLock()
+	defer t.mu.RUnlock()
+
+	if t.deleteCleanupInterval == 0 {
+		return 5 * time.Minute // every 5 minutes
+	}
+
+	return t.deleteCleanupInterval
+}
+
 func (t *apiConfig) getClusterDeadline() time.Duration {
 	t.mu.RLock()
 	defer t.mu.RUnlock()

cmd/healthcheck-handler.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"strconv"
 
@@ -29,6 +30,11 @@ const unavailable = "offline"
 
 // ClusterCheckHandler returns if the server is ready for requests.
 func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
+	if globalIsGateway {
+		writeResponse(w, http.StatusOK, nil, mimeNone)
+		return
+	}
+
 	ctx := newContext(r, w, "ClusterCheckHandler")
 
 	if shouldProxy() {
@@ -67,6 +73,11 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 
 // ClusterReadCheckHandler returns if the server is ready for requests.
 func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
+	if globalIsGateway {
+		writeResponse(w, http.StatusOK, nil, mimeNone)
+		return
+	}
+
 	ctx := newContext(r, w, "ClusterReadCheckHandler")
 
 	if shouldProxy() {
@@ -85,28 +96,13 @@ func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
+
 	writeResponse(w, http.StatusOK, nil, mimeNone)
 }
 
 // ReadinessCheckHandler Checks if the process is up. Always returns success.
 func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	if shouldProxy() {
-		// Service not initialized yet
-		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
-	}
-
-	if globalIsGateway && globalEtcdClient != nil {
-		// Borrowed from https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118
-		ctx, cancel := context.WithTimeout(r.Context(), defaultContextTimeout)
-		defer cancel()
-		// etcd unreachable throw an error for readiness.
-		if _, err := globalEtcdClient.Get(ctx, "health"); err != nil {
-			writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
-			return
-		}
-	}
-
-	writeResponse(w, http.StatusOK, nil, mimeNone)
+	LivenessCheckHandler(w, r)
 }
 
 // LivenessCheckHandler - Checks if the process is up. Always returns success.
@@ -116,15 +112,49 @@ func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 	}
 
-	if globalIsGateway && globalEtcdClient != nil {
-		// Borrowed from https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118
-		ctx, cancel := context.WithTimeout(r.Context(), defaultContextTimeout)
-		defer cancel()
-		// etcd unreachable throw an error for readiness.
-		if _, err := globalEtcdClient.Get(ctx, "health"); err != nil {
-			writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
+	if globalIsGateway {
+		objLayer := newObjectLayerFn()
+		if objLayer == nil {
+			apiErr := toAPIError(r.Context(), errServerNotInitialized)
+			switch r.Method {
+			case http.MethodHead:
+				writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+			case http.MethodGet:
+				writeErrorResponse(r.Context(), w, apiErr, r.URL)
+			}
 			return
 		}
+
+		storageInfo, _ := objLayer.StorageInfo(r.Context())
+		if !storageInfo.Backend.GatewayOnline {
+			err := errors.New("gateway backend is not reachable")
+			apiErr := toAPIError(r.Context(), err)
+			switch r.Method {
+			case http.MethodHead:
+				writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+			case http.MethodGet:
+				writeErrorResponse(r.Context(), w, apiErr, r.URL)
+			}
+			return
+		}
+
+		if globalEtcdClient != nil {
+			// Borrowed from
+			// https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118
+			ctx, cancel := context.WithTimeout(r.Context(), defaultContextTimeout)
+			defer cancel()
+			if _, err := globalEtcdClient.Get(ctx, "health"); err != nil {
+				// etcd unreachable throw an error..
+				switch r.Method {
+				case http.MethodHead:
+					apiErr := toAPIError(r.Context(), err)
+					writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone)
+				case http.MethodGet:
+					writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL)
+				}
+				return
+			}
+		}
 	}
 
 	writeResponse(w, http.StatusOK, nil, mimeNone)

cmd/http-tracer.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"bytes"
 	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"reflect"
@@ -50,6 +49,11 @@ type recordRequest struct {
 	bytesRead int
 }
 
+func (r *recordRequest) Close() error {
+	// no-op
+	return nil
+}
+
 func (r *recordRequest) Read(p []byte) (n int, err error) {
 	n, err = r.Reader.Read(p)
 	r.bytesRead += n
@@ -62,6 +66,7 @@ func (r *recordRequest) Read(p []byte) (n int, err error) {
 	}
 	return n, err
 }
+
 func (r *recordRequest) Size() int {
 	sz := r.bytesRead
 	for k, v := range r.headers {
@@ -122,7 +127,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 	}
 
 	reqBodyRecorder := &recordRequest{Reader: r.Body, logBody: logBody, headers: reqHeaders}
-	r.Body = ioutil.NopCloser(reqBodyRecorder)
+	r.Body = reqBodyRecorder
 
 	now := time.Now().UTC()
 	t := madmin.TraceInfo{TraceType: madmin.TraceHTTP, FuncName: name, Time: now}

cmd/iam.go

@@ -514,6 +514,7 @@ func (sys *IAMSys) Load(ctx context.Context, store IAMStorageAPI) error {
 		return err
 	}
 
+	// load service accounts
 	if err := store.loadUsers(ctx, svcUser, iamUsersMap); err != nil {
 		return err
 	}
@@ -1171,8 +1172,7 @@ type newServiceAccountOpts struct {
 	accessKey     string
 	secretKey     string
 
-	// LDAP username
-	ldapUsername string
+	claims map[string]interface{}
 }
 
 // NewServiceAccount - create a new service account
@@ -1260,9 +1260,12 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 		m[iamPolicyClaimNameSA()] = "inherited-policy"
 	}
 
-	// For LDAP service account, save the ldap username in the metadata.
-	if opts.ldapUsername != "" {
-		m[ldapUserN] = opts.ldapUsername
+	// Add all the necessary claims for the service accounts.
+	for k, v := range opts.claims {
+		_, ok := m[k]
+		if !ok {
+			m[k] = v
+		}
 	}
 
 	var (
@@ -1425,6 +1428,31 @@ func (sys *IAMSys) GetServiceAccount(ctx context.Context, accessKey string) (aut
 	return sa, embeddedPolicy, nil
 }
 
+// GetClaimsForSvcAcc - gets the claims associated with the service account.
+func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (map[string]interface{}, error) {
+	if !sys.Initialized() {
+		return nil, errServerNotInitialized
+	}
+
+	if sys.usersSysType != LDAPUsersSysType {
+		return nil, nil
+	}
+
+	sys.store.rlock()
+	defer sys.store.runlock()
+
+	sa, ok := sys.iamUsersMap[accessKey]
+	if !ok || !sa.IsServiceAccount() {
+		return nil, errNoSuchServiceAccount
+	}
+
+	jwtClaims, err := auth.ExtractClaims(sa.SessionToken, globalActiveCred.SecretKey)
+	if err != nil {
+		return nil, err
+	}
+	return jwtClaims.Map(), nil
+}
+
 // DeleteServiceAccount - delete a service account
 func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) error {
 	if !sys.Initialized() {

cmd/lock-rest-client.go

@@ -22,7 +22,6 @@ import (
 	"context"
 	"io"
 	"net/url"
-	"strconv"
 
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/http"
@@ -89,17 +88,13 @@ func (client *lockRESTClient) Close() error {
 
 // restCall makes a call to the lock REST server.
 func (client *lockRESTClient) restCall(ctx context.Context, call string, args dsync.LockArgs) (reply bool, err error) {
-	values := url.Values{}
-	values.Set(lockRESTUID, args.UID)
-	values.Set(lockRESTOwner, args.Owner)
-	values.Set(lockRESTSource, args.Source)
-	values.Set(lockRESTQuorum, strconv.Itoa(args.Quorum))
-	var buffer bytes.Buffer
-	for _, resource := range args.Resources {
-		buffer.WriteString(resource)
-		buffer.WriteString("\n")
+	argsBytes, err := args.MarshalMsg(metaDataPoolGet()[:0])
+	if err != nil {
+		return false, err
 	}
-	respBody, err := client.callWithContext(ctx, call, values, &buffer, -1)
+	defer metaDataPoolPut(argsBytes)
+	body := bytes.NewReader(argsBytes)
+	respBody, err := client.callWithContext(ctx, call, nil, body, body.Size())
 	defer http.DrainBody(respBody)
 	switch err {
 	case nil:

cmd/lock-rest-server-common.go

@@ -22,7 +22,7 @@ import (
 )
 
 const (
-	lockRESTVersion       = "v6" // Add Refresh API
+	lockRESTVersion       = "v7" // Add msgp for lockArgs
 	lockRESTVersionPrefix = SlashSeparator + lockRESTVersion
 	lockRESTPrefix        = minioReservedBucketPath + "/lock"
 )
@@ -35,20 +35,6 @@ const (
 	lockRESTMethodUnlock      = "/unlock"
 	lockRESTMethodRUnlock     = "/runlock"
 	lockRESTMethodForceUnlock = "/force-unlock"
-
-	// lockRESTOwner represents owner UUID
-	lockRESTOwner = "owner"
-
-	// Unique ID of lock/unlock request.
-	lockRESTUID = "uid"
-
-	// Source contains the line number, function and file name of the code
-	// on the client node that requested the lock.
-	lockRESTSource = "source"
-
-	// Quroum value to be saved along lock requester info, useful
-	// in verifying stale locks
-	lockRESTQuorum = "quorum"
 )
 
 var (

cmd/lock-rest-server.go

@@ -18,14 +18,13 @@
 package cmd
 
 import (
-	"bufio"
 	"context"
 	"errors"
+	"io"
 	"net/http"
-	"sort"
-	"strconv"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/dsync"
 )
@@ -63,32 +62,10 @@ func (l *lockRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
 }
 
 func getLockArgs(r *http.Request) (args dsync.LockArgs, err error) {
-	values := r.Form
-	quorum, err := strconv.Atoi(values.Get(lockRESTQuorum))
-	if err != nil {
-		return args, err
-	}
-
-	args = dsync.LockArgs{
-		Owner:  values.Get(lockRESTOwner),
-		UID:    values.Get(lockRESTUID),
-		Source: values.Get(lockRESTSource),
-		Quorum: quorum,
-	}
-
-	var resources []string
-	bio := bufio.NewScanner(r.Body)
-	for bio.Scan() {
-		resources = append(resources, bio.Text())
-	}
-
-	if err := bio.Err(); err != nil {
-		return args, err
-	}
-
-	sort.Strings(resources)
-	args.Resources = resources
-	return args, nil
+	dec := msgpNewReader(io.LimitReader(r.Body, 1000*humanize.KiByte))
+	defer readMsgpReaderPool.Put(dec)
+	err = args.DecodeMsg(dec)
+	return args, err
 }
 
 // HealthHandler returns success if request is authenticated.

cmd/lock-rest-server_test.go

@@ -0,0 +1,104 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"bufio"
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"sort"
+	"strconv"
+	"testing"
+
+	"github.com/minio/minio/internal/dsync"
+)
+
+func BenchmarkLockArgs(b *testing.B) {
+	args := dsync.LockArgs{
+		Owner:     "minio",
+		UID:       "uid",
+		Source:    "lockArgs.go",
+		Quorum:    3,
+		Resources: []string{"obj.txt"},
+	}
+
+	argBytes, err := args.MarshalMsg(nil)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	req := &http.Request{
+		Body: ioutil.NopCloser(bytes.NewReader(argBytes)),
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		getLockArgs(req)
+	}
+}
+
+func BenchmarkLockArgsOld(b *testing.B) {
+	values := url.Values{}
+	values.Set("owner", "minio")
+	values.Set("uid", "uid")
+	values.Set("source", "lockArgs.go")
+	values.Set("quorum", "3")
+
+	req := &http.Request{
+		Form: values,
+		Body: ioutil.NopCloser(bytes.NewReader([]byte(`obj.txt`))),
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		getLockArgsOld(req)
+	}
+}
+
+func getLockArgsOld(r *http.Request) (args dsync.LockArgs, err error) {
+	values := r.Form
+	quorum, err := strconv.Atoi(values.Get("quorum"))
+	if err != nil {
+		return args, err
+	}
+
+	args = dsync.LockArgs{
+		Owner:  values.Get("onwer"),
+		UID:    values.Get("uid"),
+		Source: values.Get("source"),
+		Quorum: quorum,
+	}
+
+	var resources []string
+	bio := bufio.NewScanner(r.Body)
+	for bio.Scan() {
+		resources = append(resources, bio.Text())
+	}
+
+	if err := bio.Err(); err != nil {
+		return args, err
+	}
+
+	sort.Strings(resources)
+	args.Resources = resources
+	return args, nil
+}

cmd/metacache-server-pool.go

@@ -209,6 +209,10 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 			}
 		}()
 		o.ID = ""
+
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Resuming listing from drives failed %w, proceeding to do raw listing", err))
+		}
 	}
 
 	// Do listing in-place.

cmd/metacache-set.go

@@ -364,8 +364,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 			retries = 1
 		}
 
-		const retryDelay = 500 * time.Millisecond
-		// Load first part metadata...
+		const retryDelay = 250 * time.Millisecond
 		// All operations are performed without locks, so we must be careful and allow for failures.
 		// Read metadata associated with the object from a disk.
 		if retries > 0 {
@@ -373,6 +372,9 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				if disk == nil {
 					continue
 				}
+				if !disk.IsOnline() {
+					continue
+				}
 				_, err := disk.ReadVersion(ctx, minioMetaBucket,
 					o.objectPath(0), "", false)
 				if err != nil {
@@ -384,6 +386,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 			}
 		}
 
+		// Load first part metadata...
 		// Read metadata associated with the object from all disks.
 		fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(0), ObjectOptions{}, true)
 		if err != nil {
@@ -441,6 +444,9 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 						if disk == nil {
 							continue
 						}
+						if !disk.IsOnline() {
+							continue
+						}
 						_, err := disk.ReadVersion(ctx, minioMetaBucket,
 							o.objectPath(partN), "", false)
 						if err != nil {
@@ -452,7 +458,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 					}
 				}
 
-				// Load first part metadata...
+				// Load partN metadata...
 				fi, metaArr, onlineDisks, err = er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(partN), ObjectOptions{}, true)
 				if err != nil {
 					time.Sleep(retryDelay)
@@ -785,11 +791,13 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 	defer cancel()
 
 	fallback := func(err error) bool {
-		if err == nil {
-			return false
+		switch err.(type) {
+		case StorageErr:
+			// all supported disk errors
+			// attempt a fallback.
+			return true
 		}
-		return err.Error() == errUnformattedDisk.Error() ||
-			err.Error() == errVolumeNotFound.Error()
+		return false
 	}
 	askDisks := len(disks)
 	readers := make([]*metacacheReader, askDisks)
@@ -844,6 +852,8 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 			if werr != io.EOF && werr != nil &&
 				werr.Error() != errFileNotFound.Error() &&
 				werr.Error() != errVolumeNotFound.Error() &&
+				werr.Error() != errDiskNotFound.Error() &&
+				werr.Error() != errUnformattedDisk.Error() &&
 				!errors.Is(werr, context.Canceled) {
 				logger.LogIf(ctx, werr)
 			}
@@ -874,12 +884,11 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 				continue
 			case nil:
 			default:
-				if err.Error() == errFileNotFound.Error() {
-					atEOF++
-					fnf++
-					continue
-				}
-				if err.Error() == errVolumeNotFound.Error() {
+				switch err.Error() {
+				case errFileNotFound.Error(),
+					errVolumeNotFound.Error(),
+					errUnformattedDisk.Error(),
+					errDiskNotFound.Error():
 					atEOF++
 					fnf++
 					continue

cmd/metrics-v2.go

@@ -24,8 +24,10 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/logger"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -69,6 +71,7 @@ const (
 	sysCallSubsystem          MetricSubsystem = "syscall"
 	usageSubsystem            MetricSubsystem = "usage"
 	ilmSubsystem              MetricSubsystem = "ilm"
+	scannerSubsystem          MetricSubsystem = "scanner"
 )
 
 // MetricName are the individual names for the metric.
@@ -255,6 +258,7 @@ func GetGeneratorsForPeer() []MetricsGenerator {
 		getNetworkMetrics,
 		getS3TTFBMetric,
 		getILMNodeMetrics,
+		getScannerNodeMetrics,
 	}
 	return g
 }
@@ -1066,6 +1070,95 @@ func getILMNodeMetrics() MetricsGroup {
 	}
 }
 
+func getScannerNodeMetrics() MetricsGroup {
+	return MetricsGroup{
+		id:         "ScannerNodeMetrics",
+		cachedRead: cachedRead,
+		read: func(_ context.Context) []Metric {
+			metrics := []Metric{
+				{
+					Description: MetricDescription{
+						Namespace: nodeMetricNamespace,
+						Subsystem: scannerSubsystem,
+						Name:      "objects_scanned",
+						Help:      "Total number of unique objects scanned since server start.",
+						Type:      counterMetric,
+					},
+					Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalObjects)),
+				},
+				{
+					Description: MetricDescription{
+						Namespace: nodeMetricNamespace,
+						Subsystem: scannerSubsystem,
+						Name:      "versions_scanned",
+						Help:      "Total number of object versions scanned since server start.",
+						Type:      counterMetric,
+					},
+					Value: float64(atomic.LoadUint64(&globalScannerStats.accTotalVersions)),
+				},
+				{
+					Description: MetricDescription{
+						Namespace: nodeMetricNamespace,
+						Subsystem: scannerSubsystem,
+						Name:      "directories_scanned",
+						Help:      "Total number of directories scanned since server start.",
+						Type:      counterMetric,
+					},
+					Value: float64(atomic.LoadUint64(&globalScannerStats.accFolders)),
+				},
+				{
+					Description: MetricDescription{
+						Namespace: nodeMetricNamespace,
+						Subsystem: scannerSubsystem,
+						Name:      "bucket_scans_started",
+						Help:      "Total number of bucket scans started since server start",
+						Type:      counterMetric,
+					},
+					Value: float64(atomic.LoadUint64(&globalScannerStats.bucketsStarted)),
+				},
+				{
+					Description: MetricDescription{
+						Namespace: nodeMetricNamespace,
+						Subsystem: scannerSubsystem,
+						Name:      "bucket_scans_finished",
+						Help:      "Total number of bucket scans finished since server start",
+						Type:      counterMetric,
+					},
+					Value: float64(atomic.LoadUint64(&globalScannerStats.bucketsFinished)),
+				},
+				{
+					Description: MetricDescription{
+						Namespace: nodeMetricNamespace,
+						Subsystem: ilmSubsystem,
+						Name:      "versions_scanned",
+						Help:      "Total number of object versions checked for ilm actions since server start",
+						Type:      counterMetric,
+					},
+					Value: float64(atomic.LoadUint64(&globalScannerStats.ilmChecks)),
+				},
+			}
+			for i := range globalScannerStats.actions {
+				action := lifecycle.Action(i)
+				v := atomic.LoadUint64(&globalScannerStats.actions[action])
+				if v == 0 {
+					continue
+				}
+				metrics = append(metrics, Metric{
+					Description: MetricDescription{
+						Namespace: nodeMetricNamespace,
+						Subsystem: ilmSubsystem,
+						Name:      MetricName("action_count_" + toSnake(action.String())),
+						Help:      "Total action outcome of lifecycle checks since server start",
+						Type:      counterMetric,
+					},
+					Value: float64(v),
+				})
+			}
+			return metrics
+		},
+	}
+}
+
 func getMinioVersionMetrics() MetricsGroup {
 	return MetricsGroup{
 		id:         "MinioVersionMetrics",
@@ -1753,3 +1846,26 @@ func metricsNodeHandler() http.Handler {
 			}),
 	)
 }
+
+func toSnake(camel string) (snake string) {
+	var b strings.Builder
+	l := len(camel)
+	for i, v := range camel {
+		// A is 65, a is 97
+		if v >= 'a' {
+			b.WriteRune(v)
+			continue
+		}
+		// v is capital letter here
+		// disregard first letter
+		// add underscore if last letter is capital letter
+		// add underscore when previous letter is lowercase
+		// add underscore when next letter is lowercase
+		if (i != 0 || i == l-1) && ((i > 0 && rune(camel[i-1]) >= 'a') ||
+			(i < l-1 && rune(camel[i+1]) >= 'a')) {
+			b.WriteRune('_')
+		}
+		b.WriteRune(v + 'a' - 'A')
+	}
+	return b.String()
+}

cmd/naughty-disk_test.go

@@ -285,9 +285,9 @@ func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi Fi
 	return d.disk.VerifyFile(ctx, volume, path, fi)
 }
 
-func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
 	if err := d.calcError(); err != nil {
 		return stat, err
 	}
-	return d.disk.StatInfoFile(ctx, volume, path)
+	return d.disk.StatInfoFile(ctx, volume, path, glob)
 }

cmd/notification.go

@@ -1557,3 +1557,23 @@ func (sys *NotificationSys) Speedtest(ctx context.Context, size int, concurrent
 
 	return results
 }
+
+// ReloadSiteReplicationConfig - tells all peer minio nodes to reload the
+// site-replication configuration.
+func (sys *NotificationSys) ReloadSiteReplicationConfig(ctx context.Context) []error {
+	errs := make([]error, len(sys.allPeerClients))
+	var wg sync.WaitGroup
+	for index := range sys.peerClients {
+		if sys.peerClients[index] == nil {
+			continue
+		}
+		wg.Add(1)
+		go func(index int) {
+			defer wg.Done()
+			errs[index] = sys.peerClients[index].ReloadSiteReplicationConfig(ctx)
+		}(index)
+	}
+
+	wg.Wait()
+	return errs
+}

cmd/object-api-interface.go

@@ -89,6 +89,12 @@ type BucketOptions struct {
 	VersioningEnabled bool
 }
 
+// DeleteBucketOptions provides options for DeleteBucket calls.
+type DeleteBucketOptions struct {
+	Force      bool // Force deletion
+	NoRecreate bool // Do not recreate on delete failures
+}
+
 // SetReplicaStatus sets replica status and timestamp for delete operations in ObjectOptions
 func (o *ObjectOptions) SetReplicaStatus(st replication.StatusType) {
 	o.DeleteReplication.ReplicaStatus = st
@@ -164,7 +170,7 @@ type ObjectLayer interface {
 	MakeBucketWithLocation(ctx context.Context, bucket string, opts BucketOptions) error
 	GetBucketInfo(ctx context.Context, bucket string) (bucketInfo BucketInfo, err error)
 	ListBuckets(ctx context.Context) (buckets []BucketInfo, err error)
-	DeleteBucket(ctx context.Context, bucket string, forceDelete bool) error
+	DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error
 	ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error)
 	ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error)
 	ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionMarker, delimiter string, maxKeys int) (result ListObjectVersionsInfo, err error)

cmd/object-api-listobjects_test.go

@@ -1385,6 +1385,11 @@ func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) {
 	if err != nil {
 		t.Fatal("Unexpected err: ", err)
 	}
+
+	newTestConfig(globalMinioDefaultRegion, obj)
+
+	newAllSubsystems()
+
 	return obj
 }
 

cmd/object-handlers.go

@@ -1507,6 +1507,11 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		UserAgent:    r.UserAgent(),
 		Host:         handlers.GetSourceIP(r),
 	})
+	if !globalTierConfigMgr.Empty() {
+		// Schedule object for immediate transition if eligible.
+		enqueueTransitionImmediate(objInfo)
+	}
+
 }
 
 // PutObjectHandler - PUT Object
@@ -1851,6 +1856,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 
 	// Remove the transitioned object whose object version is being overwritten.
 	if !globalTierConfigMgr.Empty() {
+		// Schedule object for immediate transition if eligible.
+		enqueueTransitionImmediate(objInfo)
 		logger.LogIf(ctx, os.Sweep())
 	}
 }
@@ -3292,6 +3299,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 	// Remove the transitioned object whose object version is being overwritten.
 	if !globalTierConfigMgr.Empty() {
+		// Schedule object for immediate transition if eligible.
+		enqueueTransitionImmediate(objInfo)
 		logger.LogIf(ctx, os.Sweep())
 	}
 }

cmd/peer-rest-client.go

@@ -1029,3 +1029,12 @@ func (client *peerRESTClient) Speedtest(ctx context.Context, size, concurrent in
 	}
 	return result, nil
 }
+
+func (client *peerRESTClient) ReloadSiteReplicationConfig(ctx context.Context) error {
+	respBody, err := client.callWithContext(context.Background(), peerRESTMethodReloadSiteReplicationConfig, nil, nil, -1)
+	if err != nil {
+		return err
+	}
+	defer http.DrainBody(respBody)
+	return nil
+}

cmd/peer-rest-common.go

@@ -25,47 +25,48 @@ const (
 )
 
 const (
-	peerRESTMethodHealth                   = "/health"
-	peerRESTMethodServerInfo               = "/serverinfo"
-	peerRESTMethodDriveInfo                = "/driveinfo"
-	peerRESTMethodNetInfo                  = "/netinfo"
-	peerRESTMethodCPUInfo                  = "/cpuinfo"
-	peerRESTMethodDiskHwInfo               = "/diskhwinfo"
-	peerRESTMethodOsInfo                   = "/osinfo"
-	peerRESTMethodMemInfo                  = "/meminfo"
-	peerRESTMethodProcInfo                 = "/procinfo"
-	peerRESTMethodSysErrors                = "/syserrors"
-	peerRESTMethodSysServices              = "/sysservices"
-	peerRESTMethodSysConfig                = "/sysconfig"
-	peerRESTMethodDispatchNetInfo          = "/dispatchnetinfo"
-	peerRESTMethodDeleteBucketMetadata     = "/deletebucketmetadata"
-	peerRESTMethodLoadBucketMetadata       = "/loadbucketmetadata"
-	peerRESTMethodGetBucketStats           = "/getbucketstats"
-	peerRESTMethodServerUpdate             = "/serverupdate"
-	peerRESTMethodSignalService            = "/signalservice"
-	peerRESTMethodBackgroundHealStatus     = "/backgroundhealstatus"
-	peerRESTMethodGetLocks                 = "/getlocks"
-	peerRESTMethodLoadUser                 = "/loaduser"
-	peerRESTMethodLoadServiceAccount       = "/loadserviceaccount"
-	peerRESTMethodDeleteUser               = "/deleteuser"
-	peerRESTMethodDeleteServiceAccount     = "/deleteserviceaccount"
-	peerRESTMethodLoadPolicy               = "/loadpolicy"
-	peerRESTMethodLoadPolicyMapping        = "/loadpolicymapping"
-	peerRESTMethodDeletePolicy             = "/deletepolicy"
-	peerRESTMethodLoadGroup                = "/loadgroup"
-	peerRESTMethodStartProfiling           = "/startprofiling"
-	peerRESTMethodDownloadProfilingData    = "/downloadprofilingdata"
-	peerRESTMethodCycleBloom               = "/cyclebloom"
-	peerRESTMethodTrace                    = "/trace"
-	peerRESTMethodListen                   = "/listen"
-	peerRESTMethodLog                      = "/log"
-	peerRESTMethodGetLocalDiskIDs          = "/getlocaldiskids"
-	peerRESTMethodGetBandwidth             = "/bandwidth"
-	peerRESTMethodGetMetacacheListing      = "/getmetacache"
-	peerRESTMethodUpdateMetacacheListing   = "/updatemetacache"
-	peerRESTMethodGetPeerMetrics           = "/peermetrics"
-	peerRESTMethodLoadTransitionTierConfig = "/loadtransitiontierconfig"
-	peerRESTMethodSpeedtest                = "/speedtest"
+	peerRESTMethodHealth                      = "/health"
+	peerRESTMethodServerInfo                  = "/serverinfo"
+	peerRESTMethodDriveInfo                   = "/driveinfo"
+	peerRESTMethodNetInfo                     = "/netinfo"
+	peerRESTMethodCPUInfo                     = "/cpuinfo"
+	peerRESTMethodDiskHwInfo                  = "/diskhwinfo"
+	peerRESTMethodOsInfo                      = "/osinfo"
+	peerRESTMethodMemInfo                     = "/meminfo"
+	peerRESTMethodProcInfo                    = "/procinfo"
+	peerRESTMethodSysErrors                   = "/syserrors"
+	peerRESTMethodSysServices                 = "/sysservices"
+	peerRESTMethodSysConfig                   = "/sysconfig"
+	peerRESTMethodDispatchNetInfo             = "/dispatchnetinfo"
+	peerRESTMethodDeleteBucketMetadata        = "/deletebucketmetadata"
+	peerRESTMethodLoadBucketMetadata          = "/loadbucketmetadata"
+	peerRESTMethodGetBucketStats              = "/getbucketstats"
+	peerRESTMethodServerUpdate                = "/serverupdate"
+	peerRESTMethodSignalService               = "/signalservice"
+	peerRESTMethodBackgroundHealStatus        = "/backgroundhealstatus"
+	peerRESTMethodGetLocks                    = "/getlocks"
+	peerRESTMethodLoadUser                    = "/loaduser"
+	peerRESTMethodLoadServiceAccount          = "/loadserviceaccount"
+	peerRESTMethodDeleteUser                  = "/deleteuser"
+	peerRESTMethodDeleteServiceAccount        = "/deleteserviceaccount"
+	peerRESTMethodLoadPolicy                  = "/loadpolicy"
+	peerRESTMethodLoadPolicyMapping           = "/loadpolicymapping"
+	peerRESTMethodDeletePolicy                = "/deletepolicy"
+	peerRESTMethodLoadGroup                   = "/loadgroup"
+	peerRESTMethodStartProfiling              = "/startprofiling"
+	peerRESTMethodDownloadProfilingData       = "/downloadprofilingdata"
+	peerRESTMethodCycleBloom                  = "/cyclebloom"
+	peerRESTMethodTrace                       = "/trace"
+	peerRESTMethodListen                      = "/listen"
+	peerRESTMethodLog                         = "/log"
+	peerRESTMethodGetLocalDiskIDs             = "/getlocaldiskids"
+	peerRESTMethodGetBandwidth                = "/bandwidth"
+	peerRESTMethodGetMetacacheListing         = "/getmetacache"
+	peerRESTMethodUpdateMetacacheListing      = "/updatemetacache"
+	peerRESTMethodGetPeerMetrics              = "/peermetrics"
+	peerRESTMethodLoadTransitionTierConfig    = "/loadtransitiontierconfig"
+	peerRESTMethodSpeedtest                   = "/speedtest"
+	peerRESTMethodReloadSiteReplicationConfig = "/reloadsitereplicationconfig"
 )
 
 const (

cmd/peer-rest-server.go

@@ -514,7 +514,6 @@ func (s *peerRESTServer) GetMemInfoHandler(w http.ResponseWriter, r *http.Reques
 // (only the config that are of concern to minio)
 func (s *peerRESTServer) GetSysConfigHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
-		fmt.Println("Invalid request")
 		s.writeErrorResponse(w, errors.New("Invalid request"))
 		return
 	}
@@ -532,7 +531,6 @@ func (s *peerRESTServer) GetSysConfigHandler(w http.ResponseWriter, r *http.Requ
 // (only the services that are of concern to minio)
 func (s *peerRESTServer) GetSysServicesHandler(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
-		fmt.Println("Invalid request")
 		s.writeErrorResponse(w, errors.New("Invalid request"))
 		return
 	}
@@ -584,6 +582,24 @@ func (s *peerRESTServer) DeleteBucketMetadataHandler(w http.ResponseWriter, r *h
 	}
 }
 
+// ReloadSiteReplicationConfigHandler - reloads site replication configuration from the disks
+func (s *peerRESTServer) ReloadSiteReplicationConfigHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("Invalid request"))
+		return
+	}
+
+	ctx := newContext(r, w, "LoadSiteReplication")
+
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		s.writeErrorResponse(w, errServerNotInitialized)
+		return
+	}
+
+	logger.LogIf(r.Context(), globalSiteReplicationSys.Init(ctx, objAPI))
+}
+
 // GetBucketStatsHandler - fetches current in-memory bucket stats, currently only
 // returns BucketReplicationStatus
 func (s *peerRESTServer) GetBucketStatsHandler(w http.ResponseWriter, r *http.Request) {
@@ -1385,4 +1401,5 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetPeerMetrics).HandlerFunc(httpTraceHdrs(server.GetPeerMetrics))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedtest).HandlerFunc(httpTraceHdrs(server.SpeedtestHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadSiteReplicationConfig).HandlerFunc(httpTraceHdrs(server.ReloadSiteReplicationConfigHandler))
 }

cmd/server-main.go

@@ -55,6 +55,11 @@ var ServerFlags = []cli.Flag{
 		Value: ":" + GlobalMinioDefaultPort,
 		Usage: "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname",
 	},
+	cli.IntFlag{
+		Name:  "listeners",
+		Value: 1,
+		Usage: "bind N number of listeners per ADDRESS:PORT",
+	},
 	cli.StringFlag{
 		Name:  "console-address",
 		Usage: "bind to a specific ADDRESS:PORT for embedded Console UI, ADDRESS can be an IP or hostname",
@@ -411,6 +416,12 @@ func initAllSubsystems(ctx context.Context, newObject ObjectLayer) (err error) {
 	// Initialize bucket metadata sub-system.
 	globalBucketMetadataSys.Init(ctx, buckets, newObject)
 
+	// Initialize bucket notification sub-system.
+	globalNotificationSys.Init(ctx, buckets, newObject)
+
+	// Initialize site replication manager.
+	globalSiteReplicationSys.Init(ctx, newObject)
+
 	if globalIsErasure {
 		// Initialize transition tier configuration manager
 		if err = globalTierConfigMgr.Init(ctx, newObject); err != nil {
@@ -428,8 +439,6 @@ func (lw nullWriter) Write(b []byte) (int, error) {
 
 // serverMain handler called for 'minio server' command.
 func serverMain(ctx *cli.Context) {
-	defer globalDNSCache.Stop()
-
 	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
 
 	go handleSignals()
@@ -493,14 +502,23 @@ func serverMain(ctx *cli.Context) {
 		getCert = globalTLSCerts.GetCertificate
 	}
 
-	httpServer := xhttp.NewServer([]string{globalMinioAddr}, criticalErrorHandler{corsHandler(handler)}, getCert)
+	listeners := ctx.Int("listeners")
+	if listeners == 0 {
+		listeners = 1
+	}
+	addrs := make([]string, 0, listeners)
+	for i := 0; i < listeners; i++ {
+		addrs = append(addrs, globalMinioAddr)
+	}
+
+	httpServer := xhttp.NewServer(addrs, criticalErrorHandler{corsHandler(handler)}, getCert)
 	httpServer.BaseContext = func(listener net.Listener) context.Context {
 		return GlobalContext
 	}
 	// Turn-off random logging by Go internally
 	httpServer.ErrorLog = log.New(&nullWriter{}, "", 0)
 	go func() {
-		globalHTTPServerErrorCh <- httpServer.Start()
+		globalHTTPServerErrorCh <- httpServer.Start(GlobalContext)
 	}()
 
 	setHTTPServer(httpServer)
@@ -582,6 +600,10 @@ func serverMain(ctx *cli.Context) {
 		logStartupMessage(color.RedBold(msg))
 	}
 
+	if !globalCLIContext.StrictS3Compat {
+		logStartupMessage(color.RedBold("WARNING: Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production"))
+	}
+
 	if globalBrowserEnabled {
 		globalConsoleSrv, err = initConsoleServer()
 		if err != nil {
@@ -593,6 +615,14 @@ func serverMain(ctx *cli.Context) {
 		}()
 	}
 
+	if serverDebugLog {
+		logger.Info("== DEBUG Mode enabled ==")
+		logger.Info("Currently set environment settings:")
+		for _, v := range os.Environ() {
+			logger.Info(v)
+		}
+		logger.Info("======")
+	}
 	<-globalOSSignalCh
 }
 

cmd/server_test.go

@@ -2428,7 +2428,7 @@ func (s *TestSuiteCommon) TestObjectMultipartListError(c *check) {
 	c.Assert(err, nil)
 	// Since max-keys parameter in the ListMultipart request set to invalid value of -2,
 	// its expected to fail with error message "InvalidArgument".
-	verifyError(c, response4, "InvalidArgument", "Argument max-parts must be an integer between 0 and 2147483647", http.StatusBadRequest)
+	verifyError(c, response4, "InvalidArgument", "Part number must be an integer between 1 and 10000, inclusive", http.StatusBadRequest)
 }
 
 // TestObjectValidMD5 - First uploads an object with a valid Content-Md5 header and verifies the status,

cmd/signature-v4-utils.go

@@ -55,9 +55,30 @@ func skipContentSha256Cksum(r *http.Request) bool {
 		v, ok = r.Header[xhttp.AmzContentSha256]
 	}
 
+	// Skip if no header was set.
+	if !ok {
+		return true
+	}
+
 	// If x-amz-content-sha256 is set and the value is not
 	// 'UNSIGNED-PAYLOAD' we should validate the content sha256.
-	return !(ok && v[0] != unsignedPayload)
+	switch v[0] {
+	case unsignedPayload:
+		return true
+	case emptySHA256:
+		// some broken clients set empty-sha256
+		// with > 0 content-length in the body,
+		// we should skip such clients and allow
+		// blindly such insecure clients only if
+		// S3 strict compatibility is disabled.
+		if r.ContentLength > 0 && !globalCLIContext.StrictS3Compat {
+			// We return true only in situations when
+			// deployment has asked MinIO to allow for
+			// such broken clients and content-length > 0.
+			return true
+		}
+	}
+	return false
 }
 
 // Returns SHA256 for calculating canonical-request.

cmd/storage-errors.go

@@ -17,7 +17,12 @@
 
 package cmd
 
-import "errors"
+import (
+	"context"
+	"errors"
+
+	"github.com/minio/minio/internal/logger"
+)
 
 // errUnexpected - unexpected error, requires manual intervention.
 var errUnexpected = StorageErr("unexpected error, please report this issue at https://github.com/minio/minio/issues")
@@ -157,7 +162,11 @@ func osErrToFileErr(err error) error {
 		return errFaultyDisk
 	}
 	if isSysErrInvalidArg(err) {
-		return errUnsupportedDisk
+		logger.LogIf(context.Background(), err)
+		// For some odd calls with O_DIRECT reads
+		// filesystems can return EINVAL, handle
+		// these as FileNotFound instead.
+		return errFileNotFound
 	}
 	if isSysErrNoSpace(err) {
 		return errDiskFull

cmd/storage-interface.go

@@ -73,7 +73,7 @@ type StorageAPI interface {
 	CheckParts(ctx context.Context, volume string, path string, fi FileInfo) error
 	Delete(ctx context.Context, volume string, path string, recursive bool) (err error)
 	VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
-	StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error)
+	StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error)
 
 	// Write all data, syncs the data to disk.
 	// Should be used for smaller payloads.

cmd/storage-rest-client.go

@@ -23,6 +23,7 @@ import (
 	"encoding/gob"
 	"encoding/hex"
 	"errors"
+	"fmt"
 	"io"
 	"io/ioutil"
 	"net/url"
@@ -680,10 +681,11 @@ func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path st
 	return toStorageErr(verifyResp.Err)
 }
 
-func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
+	values.Set(storageRESTGlob, fmt.Sprint(glob))
 	respBody, err := client.call(ctx, storageRESTMethodStatInfoFile, values, nil, -1)
 	if err != nil {
 		return stat, err
@@ -693,7 +695,19 @@ func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path
 	if err != nil {
 		return stat, err
 	}
-	err = stat.DecodeMsg(msgpNewReader(respReader))
+	rd := msgpNewReader(respReader)
+	for {
+		var st StatInfo
+		err = st.DecodeMsg(rd)
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				err = nil
+			}
+			break
+		}
+		stat = append(stat, st)
+	}
+
 	return stat, err
 }
 

cmd/storage-rest-common.go

@@ -78,4 +78,5 @@ const (
 	storageRESTBitrotHash     = "bitrot-hash"
 	storageRESTDiskID         = "disk-id"
 	storageRESTForceDelete    = "force-delete"
+	storageRESTGlob           = "glob"
 )

cmd/storage-rest-server.go

@@ -1122,13 +1122,16 @@ func (s *storageRESTServer) StatInfoFile(w http.ResponseWriter, r *http.Request)
 	vars := mux.Vars(r)
 	volume := vars[storageRESTVolume]
 	filePath := vars[storageRESTFilePath]
+	glob := vars[storageRESTGlob]
 	done := keepHTTPResponseAlive(w)
-	si, err := s.storage.StatInfoFile(r.Context(), volume, filePath)
+	stats, err := s.storage.StatInfoFile(r.Context(), volume, filePath, glob == "true")
 	done(err)
 	if err != nil {
 		return
 	}
-	msgp.Encode(w, &si)
+	for _, si := range stats {
+		msgp.Encode(w, &si)
+	}
 }
 
 // registerStorageRPCRouter - register storage rpc router.
@@ -1221,7 +1224,7 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(httpTraceHdrs(server.WalkDirHandler)).
 				Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTRecursive)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(httpTraceHdrs(server.StatInfoFile)).
-				Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
+				Queries(restQueries(storageRESTVolume, storageRESTFilePath, storageRESTGlob)...)
 		}
 	}
 }

cmd/storage-rest_test.go

@@ -186,11 +186,11 @@ func testStorageAPIStatInfoFile(t *testing.T, storage StorageAPI) {
 	}
 
 	for i, testCase := range testCases {
-		_, err := storage.StatInfoFile(context.Background(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile)
+		_, err := storage.StatInfoFile(context.Background(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile, false)
 		expectErr := (err != nil)
 
 		if expectErr != testCase.expectErr {
-			t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr)
+			t.Fatalf("case %v: error: expected: %v, got: %v, err: %v", i+1, expectErr, testCase.expectErr, err)
 		}
 	}
 }

cmd/sts-handlers.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/identity/openid"
 	xhttp "github.com/minio/minio/internal/http"
@@ -651,6 +652,19 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 		}
 	}
 
+	// Call hook for cluster-replication.
+	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+		Type: madmin.SRIAMItemSTSAcc,
+		STSCredential: &madmin.SRSTSCredential{
+			AccessKey:    cred.AccessKey,
+			SecretKey:    cred.SecretKey,
+			SessionToken: cred.SessionToken,
+		},
+	}); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
 	ldapIdentityResponse := &AssumeRoleWithLDAPResponse{
 		Result: LDAPIdentityResult{
 			Credentials: cred,

cmd/test-utils_test.go

@@ -63,7 +63,6 @@ import (
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
-	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/rest"
 	"github.com/minio/pkg/bucket/policy"
@@ -106,8 +105,6 @@ func TestMain(m *testing.M) {
 	// Initialize globalConsoleSys system
 	globalConsoleSys = NewConsoleLogger(context.Background())
 
-	globalDNSCache = xhttp.NewDNSCache(3*time.Second, 10*time.Second, logger.LogOnceIf)
-
 	globalInternodeTransport = newInternodeHTTPTransport(nil, rest.DefaultTimeout)()
 
 	initHelp()
@@ -223,7 +220,10 @@ func initFSObjects(disk string, t *testing.T) (obj ObjectLayer) {
 	if err != nil {
 		t.Fatal(err)
 	}
+
 	newTestConfig(globalMinioDefaultRegion, obj)
+
+	newAllSubsystems()
 	return obj
 }
 

cmd/tier.go

@@ -146,7 +146,7 @@ func (config *TierConfigMgr) Edit(ctx context.Context, tierName string, creds ma
 		return errTierNotFound
 	}
 
-	newCfg := config.Tiers[tierName]
+	cfg := config.Tiers[tierName]
 	switch tierType {
 	case madmin.S3:
 		if (creds.AccessKey == "" || creds.SecretKey == "") && !creds.AWSRole {
@@ -154,30 +154,29 @@ func (config *TierConfigMgr) Edit(ctx context.Context, tierName string, creds ma
 		}
 		switch {
 		case creds.AWSRole:
-			newCfg.S3.AWSRole = true
+			cfg.S3.AWSRole = true
 		default:
-			newCfg.S3.AccessKey = creds.AccessKey
-			newCfg.S3.SecretKey = creds.SecretKey
+			cfg.S3.AccessKey = creds.AccessKey
+			cfg.S3.SecretKey = creds.SecretKey
 		}
 	case madmin.Azure:
-		if creds.AccessKey == "" || creds.SecretKey == "" {
+		if creds.SecretKey == "" {
 			return errTierInsufficientCreds
 		}
-		newCfg.Azure.AccountName = creds.AccessKey
-		newCfg.Azure.AccountKey = creds.SecretKey
+		cfg.Azure.AccountKey = creds.SecretKey
 
 	case madmin.GCS:
 		if creds.CredsJSON == nil {
 			return errTierInsufficientCreds
 		}
-		newCfg.GCS.Creds = base64.URLEncoding.EncodeToString(creds.CredsJSON)
+		cfg.GCS.Creds = base64.URLEncoding.EncodeToString(creds.CredsJSON)
 	}
 
-	d, err := newWarmBackend(ctx, newCfg)
+	d, err := newWarmBackend(ctx, cfg)
 	if err != nil {
 		return err
 	}
-	config.Tiers[tierName] = newCfg
+	config.Tiers[tierName] = cfg
 	config.drivercache[tierName] = d
 	return nil
 }

cmd/update.go

@@ -537,6 +537,14 @@ func doUpdate(u *url.URL, lrTime time.Time, sha256Sum []byte, releaseInfo string
 		Checksum: sha256Sum,
 	}
 
+	if err := opts.CheckPermissions(); err != nil {
+		return AdminError{
+			Code:       AdminUpdateApplyFailure,
+			Message:    fmt.Sprintf("server update failed with: %s, do not restart the servers yet", err),
+			StatusCode: http.StatusInternalServerError,
+		}
+	}
+
 	minisignPubkey := env.Get(envMinisignPubKey, "")
 	if minisignPubkey != "" {
 		v := selfupdate.NewVerifier()

cmd/xl-storage-disk-id-check.go

@@ -547,18 +547,18 @@ func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path
 	return p.storage.ReadAll(ctx, volume, path)
 }
 
-func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
 	defer p.updateStorageMetrics(storageStatInfoFile, volume, path)()
 
 	if contextCanceled(ctx) {
-		return StatInfo{}, ctx.Err()
+		return nil, ctx.Err()
 	}
 
 	if err = p.checkDiskStale(); err != nil {
-		return StatInfo{}, err
+		return nil, err
 	}
 
-	return p.storage.StatInfoFile(ctx, volume, path)
+	return p.storage.StatInfoFile(ctx, volume, path, glob)
 }
 
 func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string) madmin.TraceInfo {

cmd/xl-storage-format-v1.go

@@ -81,6 +81,8 @@ type xlMetaV1Object struct {
 type StatInfo struct {
 	Size    int64     `json:"size"`    // Size of the object `xl.meta`.
 	ModTime time.Time `json:"modTime"` // ModTime of the object `xl.meta`.
+	Name    string    `json:"name"`
+	Dir     bool      `json:"dir"`
 }
 
 // ErasureInfo holds erasure coding and bitrot related information.

cmd/xl-storage-format-v1_gen.go

@@ -759,6 +759,18 @@ func (z *StatInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ModTime")
 				return
 			}
+		case "Name":
+			z.Name, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "Dir":
+			z.Dir, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "Dir")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -771,10 +783,10 @@ func (z *StatInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 }
 
 // EncodeMsg implements msgp.Encodable
-func (z StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
+func (z *StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 4
 	// write "Size"
-	err = en.Append(0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
+	err = en.Append(0x84, 0xa4, 0x53, 0x69, 0x7a, 0x65)
 	if err != nil {
 		return
 	}
@@ -793,19 +805,45 @@ func (z StatInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ModTime")
 		return
 	}
+	// write "Name"
+	err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Name)
+	if err != nil {
+		err = msgp.WrapError(err, "Name")
+		return
+	}
+	// write "Dir"
+	err = en.Append(0xa3, 0x44, 0x69, 0x72)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.Dir)
+	if err != nil {
+		err = msgp.WrapError(err, "Dir")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
-func (z StatInfo) MarshalMsg(b []byte) (o []byte, err error) {
+func (z *StatInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
+	// map header, size 4
 	// string "Size"
-	o = append(o, 0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
+	o = append(o, 0x84, 0xa4, 0x53, 0x69, 0x7a, 0x65)
 	o = msgp.AppendInt64(o, z.Size)
 	// string "ModTime"
 	o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
 	o = msgp.AppendTime(o, z.ModTime)
+	// string "Name"
+	o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = msgp.AppendString(o, z.Name)
+	// string "Dir"
+	o = append(o, 0xa3, 0x44, 0x69, 0x72)
+	o = msgp.AppendBool(o, z.Dir)
 	return
 }
 
@@ -839,6 +877,18 @@ func (z *StatInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ModTime")
 				return
 			}
+		case "Name":
+			z.Name, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Name")
+				return
+			}
+		case "Dir":
+			z.Dir, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Dir")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -852,8 +902,8 @@ func (z *StatInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z StatInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize
+func (z *StatInfo) Msgsize() (s int) {
+	s = 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize + 5 + msgp.StringPrefixSize + len(z.Name) + 4 + msgp.BoolSize
 	return
 }
 
@@ -1041,40 +1091,11 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Stat":
-			var zb0002 uint32
-			zb0002, err = dc.ReadMapHeader()
+			err = z.Stat.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "Stat")
 				return
 			}
-			for zb0002 > 0 {
-				zb0002--
-				field, err = dc.ReadMapKeyPtr()
-				if err != nil {
-					err = msgp.WrapError(err, "Stat")
-					return
-				}
-				switch msgp.UnsafeString(field) {
-				case "Size":
-					z.Stat.Size, err = dc.ReadInt64()
-					if err != nil {
-						err = msgp.WrapError(err, "Stat", "Size")
-						return
-					}
-				case "ModTime":
-					z.Stat.ModTime, err = dc.ReadTime()
-					if err != nil {
-						err = msgp.WrapError(err, "Stat", "ModTime")
-						return
-					}
-				default:
-					err = dc.Skip()
-					if err != nil {
-						err = msgp.WrapError(err, "Stat")
-						return
-					}
-				}
-			}
 		case "Erasure":
 			err = z.Erasure.DecodeMsg(dc)
 			if err != nil {
@@ -1082,14 +1103,14 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Minio":
-			var zb0003 uint32
-			zb0003, err = dc.ReadMapHeader()
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Minio")
 				return
 			}
-			for zb0003 > 0 {
-				zb0003--
+			for zb0002 > 0 {
+				zb0002--
 				field, err = dc.ReadMapKeyPtr()
 				if err != nil {
 					err = msgp.WrapError(err, "Minio")
@@ -1111,21 +1132,21 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
 				}
 			}
 		case "Meta":
-			var zb0004 uint32
-			zb0004, err = dc.ReadMapHeader()
+			var zb0003 uint32
+			zb0003, err = dc.ReadMapHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Meta")
 				return
 			}
 			if z.Meta == nil {
-				z.Meta = make(map[string]string, zb0004)
+				z.Meta = make(map[string]string, zb0003)
 			} else if len(z.Meta) > 0 {
 				for key := range z.Meta {
 					delete(z.Meta, key)
 				}
 			}
-			for zb0004 > 0 {
-				zb0004--
+			for zb0003 > 0 {
+				zb0003--
 				var za0001 string
 				var za0002 string
 				za0001, err = dc.ReadString()
@@ -1141,16 +1162,16 @@ func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) {
 				z.Meta[za0001] = za0002
 			}
 		case "Parts":
-			var zb0005 uint32
-			zb0005, err = dc.ReadArrayHeader()
+			var zb0004 uint32
+			zb0004, err = dc.ReadArrayHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Parts")
 				return
 			}
-			if cap(z.Parts) >= int(zb0005) {
-				z.Parts = (z.Parts)[:zb0005]
+			if cap(z.Parts) >= int(zb0004) {
+				z.Parts = (z.Parts)[:zb0004]
 			} else {
-				z.Parts = make([]ObjectPartInfo, zb0005)
+				z.Parts = make([]ObjectPartInfo, zb0004)
 			}
 			for za0003 := range z.Parts {
 				err = z.Parts[za0003].DecodeMsg(dc)
@@ -1210,25 +1231,9 @@ func (z *xlMetaV1Object) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	// map header, size 2
-	// write "Size"
-	err = en.Append(0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
+	err = z.Stat.EncodeMsg(en)
 	if err != nil {
-		return
-	}
-	err = en.WriteInt64(z.Stat.Size)
-	if err != nil {
-		err = msgp.WrapError(err, "Stat", "Size")
-		return
-	}
-	// write "ModTime"
-	err = en.Append(0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
-	if err != nil {
-		return
-	}
-	err = en.WriteTime(z.Stat.ModTime)
-	if err != nil {
-		err = msgp.WrapError(err, "Stat", "ModTime")
+		err = msgp.WrapError(err, "Stat")
 		return
 	}
 	// write "Erasure"
@@ -1331,13 +1336,11 @@ func (z *xlMetaV1Object) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendString(o, z.Format)
 	// string "Stat"
 	o = append(o, 0xa4, 0x53, 0x74, 0x61, 0x74)
-	// map header, size 2
-	// string "Size"
-	o = append(o, 0x82, 0xa4, 0x53, 0x69, 0x7a, 0x65)
-	o = msgp.AppendInt64(o, z.Stat.Size)
-	// string "ModTime"
-	o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65)
-	o = msgp.AppendTime(o, z.Stat.ModTime)
+	o, err = z.Stat.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Stat")
+		return
+	}
 	// string "Erasure"
 	o = append(o, 0xa7, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65)
 	o, err = z.Erasure.MarshalMsg(o)
@@ -1408,40 +1411,11 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "Stat":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			bts, err = z.Stat.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Stat")
 				return
 			}
-			for zb0002 > 0 {
-				zb0002--
-				field, bts, err = msgp.ReadMapKeyZC(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Stat")
-					return
-				}
-				switch msgp.UnsafeString(field) {
-				case "Size":
-					z.Stat.Size, bts, err = msgp.ReadInt64Bytes(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Stat", "Size")
-						return
-					}
-				case "ModTime":
-					z.Stat.ModTime, bts, err = msgp.ReadTimeBytes(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Stat", "ModTime")
-						return
-					}
-				default:
-					bts, err = msgp.Skip(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Stat")
-						return
-					}
-				}
-			}
 		case "Erasure":
 			bts, err = z.Erasure.UnmarshalMsg(bts)
 			if err != nil {
@@ -1449,14 +1423,14 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "Minio":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Minio")
 				return
 			}
-			for zb0003 > 0 {
-				zb0003--
+			for zb0002 > 0 {
+				zb0002--
 				field, bts, err = msgp.ReadMapKeyZC(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Minio")
@@ -1478,23 +1452,23 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				}
 			}
 		case "Meta":
-			var zb0004 uint32
-			zb0004, bts, err = msgp.ReadMapHeaderBytes(bts)
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Meta")
 				return
 			}
 			if z.Meta == nil {
-				z.Meta = make(map[string]string, zb0004)
+				z.Meta = make(map[string]string, zb0003)
 			} else if len(z.Meta) > 0 {
 				for key := range z.Meta {
 					delete(z.Meta, key)
 				}
 			}
-			for zb0004 > 0 {
+			for zb0003 > 0 {
 				var za0001 string
 				var za0002 string
-				zb0004--
+				zb0003--
 				za0001, bts, err = msgp.ReadStringBytes(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Meta")
@@ -1508,16 +1482,16 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				z.Meta[za0001] = za0002
 			}
 		case "Parts":
-			var zb0005 uint32
-			zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			var zb0004 uint32
+			zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Parts")
 				return
 			}
-			if cap(z.Parts) >= int(zb0005) {
-				z.Parts = (z.Parts)[:zb0005]
+			if cap(z.Parts) >= int(zb0004) {
+				z.Parts = (z.Parts)[:zb0004]
 			} else {
-				z.Parts = make([]ObjectPartInfo, zb0005)
+				z.Parts = make([]ObjectPartInfo, zb0004)
 			}
 			for za0003 := range z.Parts {
 				bts, err = z.Parts[za0003].UnmarshalMsg(bts)
@@ -1552,7 +1526,7 @@ func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *xlMetaV1Object) Msgsize() (s int) {
-	s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 7 + msgp.StringPrefixSize + len(z.Format) + 5 + 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize + 8 + z.Erasure.Msgsize() + 6 + 1 + 8 + msgp.StringPrefixSize + len(z.Minio.Release) + 5 + msgp.MapHeaderSize
+	s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 7 + msgp.StringPrefixSize + len(z.Format) + 5 + z.Stat.Msgsize() + 8 + z.Erasure.Msgsize() + 6 + 1 + 8 + msgp.StringPrefixSize + len(z.Minio.Release) + 5 + msgp.MapHeaderSize
 	if z.Meta != nil {
 		for za0001, za0002 := range z.Meta {
 			_ = za0002

cmd/xl-storage.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"bufio"
 	"bytes"
 	"context"
 	"crypto/rand"
@@ -34,12 +33,12 @@ import (
 	"runtime"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"syscall"
 	"time"
 
 	"github.com/dustin/go-humanize"
 	jsoniter "github.com/json-iterator/go"
-	"github.com/klauspost/readahead"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
@@ -48,21 +47,11 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
 	"github.com/minio/pkg/env"
+	"github.com/yargevad/filepathx"
 )
 
 const (
-	nullVersionID        = "null"
-	blockSizeSmall       = 128 * humanize.KiByte // Default r/w block size for smaller objects.
-	blockSizeLarge       = 2 * humanize.MiByte   // Default r/w block size for larger objects.
-	blockSizeReallyLarge = 4 * humanize.MiByte   // Default write block size for objects per shard >= 64MiB
-
-	// On regular files bigger than this;
-	readAheadSize = 16 << 20
-	// Read this many buffers ahead.
-	readAheadBuffers = 4
-	// Size of each buffer.
-	readAheadBufSize = 1 << 20
-
+	nullVersionID = "null"
 	// Really large streams threshold per shard.
 	reallyLargeFileThreshold = 64 * humanize.MiByte // Optimized for HDDs
 
@@ -78,7 +67,7 @@ const (
 var alignedBuf []byte
 
 func init() {
-	alignedBuf = disk.AlignedBlock(4096)
+	alignedBuf = disk.AlignedBlock(xioutil.DirectioAlignSize)
 	_, _ = rand.Read(alignedBuf)
 }
 
@@ -97,27 +86,6 @@ func isValidVolname(volname string) bool {
 	return true
 }
 
-var (
-	xlPoolReallyLarge = sync.Pool{
-		New: func() interface{} {
-			b := disk.AlignedBlock(blockSizeReallyLarge)
-			return &b
-		},
-	}
-	xlPoolLarge = sync.Pool{
-		New: func() interface{} {
-			b := disk.AlignedBlock(blockSizeLarge)
-			return &b
-		},
-	}
-	xlPoolSmall = sync.Pool{
-		New: func() interface{} {
-			b := disk.AlignedBlock(blockSizeSmall)
-			return &b
-		},
-	}
-)
-
 // xlStorage - implements StorageAPI interface.
 type xlStorage struct {
 	diskPath string
@@ -387,7 +355,7 @@ func (s *xlStorage) SetDiskLoc(poolIdx, setIdx, diskIdx int) {
 func (s *xlStorage) Healing() *healingTracker {
 	healingFile := pathJoin(s.diskPath, minioMetaBucket,
 		bucketMetaPrefix, healingTrackerFilename)
-	b, err := ioutil.ReadFile(healingFile)
+	b, err := xioutil.ReadFile(healingFile)
 	if err != nil {
 		return nil
 	}
@@ -495,7 +463,9 @@ func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates
 			return sizeSummary{}, errSkipFile
 		}
 		sizeS := sizeSummary{}
+		atomic.AddUint64(&globalScannerStats.accTotalObjects, 1)
 		for _, version := range fivs.Versions {
+			atomic.AddUint64(&globalScannerStats.accTotalVersions, 1)
 			oi := version.ToObjectInfo(item.bucket, item.objectPath())
 			sz := item.applyActions(ctx, objAPI, oi, &sizeS)
 			if !oi.DeleteMarker && sz == oi.Size {
@@ -1166,7 +1136,7 @@ func (s *xlStorage) ReadVersion(ctx context.Context, volume, path, versionID str
 			// Check the data path if there is a part with data.
 			partPath := fmt.Sprintf("part.%d", fi.Parts[0].Number)
 			dataPath := pathJoin(path, fi.DataDir, partPath)
-			_, err = s.StatInfoFile(ctx, volume, dataPath)
+			_, err = s.StatInfoFile(ctx, volume, dataPath, false)
 			if err != nil {
 				// Set the inline header, our inlined data is fine.
 				fi.SetInlineData()
@@ -1228,7 +1198,10 @@ func (s *xlStorage) readAllData(volumeDir string, filePath string) (buf []byte,
 		}
 		return nil, err
 	}
-	r := &odirectReader{f, nil, nil, true, true, s, nil}
+	r := &xioutil.ODirectReader{
+		File:      f,
+		SmallFile: true,
+	}
 	defer r.Close()
 	buf, err = ioutil.ReadAll(r)
 	if err != nil {
@@ -1418,75 +1391,6 @@ func (s *xlStorage) openFileNoSync(filePath string, mode int) (f *os.File, err e
 	return w, nil
 }
 
-// To support O_DIRECT reads for erasure backends.
-type odirectReader struct {
-	f         *os.File
-	buf       []byte
-	bufp      *[]byte
-	freshRead bool
-	smallFile bool
-	s         *xlStorage
-	err       error
-}
-
-// Read - Implements Reader interface.
-func (o *odirectReader) Read(buf []byte) (n int, err error) {
-	if o.err != nil && (len(o.buf) == 0 || o.freshRead) {
-		return 0, o.err
-	}
-	if o.buf == nil {
-		if o.smallFile {
-			o.bufp = xlPoolSmall.Get().(*[]byte)
-		} else {
-			o.bufp = xlPoolLarge.Get().(*[]byte)
-		}
-	}
-	if o.freshRead {
-		o.buf = *o.bufp
-		n, err = o.f.Read(o.buf)
-		if err != nil && err != io.EOF {
-			if isSysErrInvalidArg(err) {
-				if err = disk.DisableDirectIO(o.f); err != nil {
-					o.err = err
-					return n, err
-				}
-				n, err = o.f.Read(o.buf)
-			}
-			if err != nil && err != io.EOF {
-				o.err = err
-				return n, err
-			}
-		}
-		if n == 0 {
-			// err is likely io.EOF
-			o.err = err
-			return n, err
-		}
-		o.err = err
-		o.buf = o.buf[:n]
-		o.freshRead = false
-	}
-	if len(buf) >= len(o.buf) {
-		n = copy(buf, o.buf)
-		o.freshRead = true
-		return n, o.err
-	}
-	n = copy(buf, o.buf)
-	o.buf = o.buf[n:]
-	// There is more left in buffer, do not return any EOF yet.
-	return n, nil
-}
-
-// Close - Release the buffer and close the file.
-func (o *odirectReader) Close() error {
-	if o.smallFile {
-		xlPoolSmall.Put(o.bufp)
-	} else {
-		xlPoolLarge.Put(o.bufp)
-	}
-	return o.f.Close()
-}
-
 // ReadFileStream - Returns the read stream of the file.
 func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) {
 	if offset < 0 {
@@ -1504,14 +1408,7 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 		return nil, err
 	}
 
-	var file *os.File
-	// O_DIRECT only supported if offset is zero
-	if offset == 0 {
-		file, err = OpenFileDirectIO(filePath, readMode, 0666)
-	} else {
-		// Open the file for reading.
-		file, err = OpenFile(filePath, readMode, 0666)
-	}
+	file, err := OpenFileDirectIO(filePath, readMode, 0666)
 	if err != nil {
 		switch {
 		case osIsNotExist(err):
@@ -1547,46 +1444,44 @@ func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, off
 		return nil, errIsNotRegular
 	}
 
-	if offset == 0 {
-		or := &odirectReader{file, nil, nil, true, false, s, nil}
-		if length <= smallFileThreshold {
-			or = &odirectReader{file, nil, nil, true, true, s, nil}
+	alignment := offset%xioutil.DirectioAlignSize == 0
+	if !alignment {
+		if err = disk.DisableDirectIO(file); err != nil {
+			file.Close()
+			return nil, err
+		}
+	}
+
+	if offset > 0 {
+		if _, err = file.Seek(offset, io.SeekStart); err != nil {
+			file.Close()
+			return nil, err
+		}
+	}
+
+	or := &xioutil.ODirectReader{
+		File:      file,
+		SmallFile: false,
+	}
+
+	if length <= smallFileThreshold {
+		or = &xioutil.ODirectReader{
+			File:      file,
+			SmallFile: true,
 		}
-		r := struct {
-			io.Reader
-			io.Closer
-		}{Reader: io.LimitReader(or, length), Closer: closeWrapper(func() error {
-			return or.Close()
-		})}
-		return r, nil
 	}
 
 	r := struct {
 		io.Reader
 		io.Closer
-	}{Reader: io.LimitReader(file, length), Closer: closeWrapper(func() error {
-		return file.Close()
+	}{Reader: io.LimitReader(or, length), Closer: closeWrapper(func() error {
+		if !alignment || offset+length%xioutil.DirectioAlignSize != 0 {
+			// invalidate page-cache for unaligned reads.
+			disk.FadviseDontNeed(file)
+		}
+		return or.Close()
 	})}
 
-	if offset > 0 {
-		if _, err = file.Seek(offset, io.SeekStart); err != nil {
-			r.Close()
-			return nil, err
-		}
-	}
-
-	// Add readahead to big reads
-	if length >= readAheadSize {
-		rc, err := readahead.NewReadCloserSize(r, readAheadBuffers, readAheadBufSize)
-		if err != nil {
-			r.Close()
-			return nil, err
-		}
-		return rc, nil
-	}
-
-	// Just add a small 64k buffer.
-	r.Reader = bufio.NewReaderSize(r.Reader, 64<<10)
 	return r, nil
 }
 
@@ -1667,11 +1562,11 @@ func (s *xlStorage) CreateFile(ctx context.Context, volume, path string, fileSiz
 	var bufp *[]byte
 	if fileSize > 0 && fileSize >= reallyLargeFileThreshold {
 		// use a larger 4MiB buffer for really large streams.
-		bufp = xlPoolReallyLarge.Get().(*[]byte)
-		defer xlPoolReallyLarge.Put(bufp)
+		bufp = xioutil.ODirectPoolXLarge.Get().(*[]byte)
+		defer xioutil.ODirectPoolXLarge.Put(bufp)
 	} else {
-		bufp = xlPoolLarge.Get().(*[]byte)
-		defer xlPoolLarge.Put(bufp)
+		bufp = xioutil.ODirectPoolLarge.Get().(*[]byte)
+		defer xioutil.ODirectPoolLarge.Put(bufp)
 	}
 
 	written, err := xioutil.CopyAligned(w, r, *bufp, fileSize)
@@ -2313,7 +2208,7 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File
 	return nil
 }
 
-func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) {
 	volumeDir, err := s.getVolDir(volume)
 	if err != nil {
 		return stat, err
@@ -2330,14 +2225,29 @@ func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat
 		}
 		return stat, err
 	}
-	filePath := pathJoin(volumeDir, path)
-	if err := checkPathLength(filePath); err != nil {
-		return stat, err
+	var files = []string{pathJoin(volumeDir, path)}
+	if glob {
+		files, err = filepathx.Glob(pathJoin(volumeDir, path))
+		if err != nil {
+			return nil, err
+		}
 	}
-	st, _ := Lstat(filePath)
-	if st == nil {
-		return stat, errPathNotFound
+	for _, filePath := range files {
+		if err := checkPathLength(filePath); err != nil {
+			return stat, err
+		}
+		st, _ := Lstat(filePath)
+		if st == nil {
+			return stat, errPathNotFound
+		}
+		name, err := filepath.Rel(volumeDir, filePath)
+		if err != nil {
+			name = filePath
+		}
+		if os.PathSeparator != '/' {
+			name = strings.Replace(name, string(os.PathSeparator), "/", -1)
+		}
+		stat = append(stat, StatInfo{ModTime: st.ModTime(), Size: st.Size(), Name: name, Dir: st.IsDir()})
 	}
-
-	return StatInfo{ModTime: st.ModTime(), Size: st.Size()}, nil
+	return stat, nil
 }

cmd/xl-storage_test.go

@@ -1711,7 +1711,7 @@ func TestXLStorageStatInfoFile(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		_, err := xlStorage.StatInfoFile(context.Background(), testCase.srcVol, testCase.srcPath+"/"+xlStorageFormatFile)
+		_, err := xlStorage.StatInfoFile(context.Background(), testCase.srcVol, testCase.srcPath+"/"+xlStorageFormatFile, false)
 		if err != testCase.expectedErr {
 			t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err)
 		}

cmd/xl-storage_unix_test.go

@@ -114,7 +114,7 @@ func TestIsValidUmaskFile(t *testing.T) {
 	}
 
 	// CheckFile - stat the file.
-	if _, err := disk.StatInfoFile(context.Background(), testCase.volName, "hello-world.txt/"+xlStorageFormatFile); err != nil {
+	if _, err := disk.StatInfoFile(context.Background(), testCase.volName, "hello-world.txt/"+xlStorageFormatFile, false); err != nil {
 		t.Fatalf("Stat failed with %s expected to pass.", err)
 	}
 }

docs/bucket/replication/DESIGN.md

@@ -39,7 +39,7 @@ Note that synchronous replication, i.e. when remote target is configured with --
 Existing object replication works similar to regular replication. Objects qualifying for existing object replication are detected when scanner runs, and will be replicated if existing object replication is enabled and applicable replication rules are satisfied. Because replication depends on the immutability of versions, only pre-existing objects created while versioning was enabled can be replicated. Even if replication rules are disabled and re-enabled later, the objects created during the interim will be synced as the scanner queues them. For saving iops, objects qualifying for
 existing object replication are not marked as `PENDING` prior to replication.
 
-Note that objects with `null` versions, i.e. objects created prior to enabling versioning cannot be replicated as this would break the immutability guarantees provided by versioning. For replicating such objects, `mc cp alias/bucket/object alias/bucket/object` can be performed to create a server side copy of the object as a versioned object - this versioned object will replicate if replication is enabled and the previously present `null` version can then be deleted.
+Note that objects with `null` versions, i.e. objects created prior to enabling versioning break the immutability guarantees provided by versioning. When existing object replication is enabled, these objects will be replicated as `null` versions to the remote targets provided they are not present on the target or if `null` version of object on source is newer than the `null` version of object on target.
 
 If the remote site is fully lost and objects previously replicated need to be re-synced, the `mc replicate resync` command with optional flag of `--older-than` needs to be used to trigger re-syncing of previously replicated objects. This command generates a ResetID which is a unique UUID saved to the remote target config along with the applicable date(defaults to time of initiating the reset). All objects created prior to this date are eligible for re-replication if existing object replication is enabled for the replication rule the object satisfies. At the time of completion of replication, `X-Minio-Replication-Reset-Status` is set in the metadata with the timestamp of replication and ResetID. For saving iops, the objects which are re-replicated are not first set to `PENDING` state.
 

docs/bucket/replication/README.md

@@ -213,8 +213,6 @@ Existing object replication as detailed [here](https://aws.amazon.com/blogs/stor
 
 Once existing object replication is enabled, all objects or object prefixes that satisfy the replication rules and were created prior to adding replication configuration OR while replication rules were disabled will be synced to the target cluster. Depending on the number of previously existing objects, the existing objects that are now eligible to be replicated will eventually be synced to the target cluster as the scanner schedules them. This may be slower depending on the load on the cluster, latency and size of the namespace.
 
-Note that existing object replication requires that the objects being replicated were created while versioning was enabled. Objects with null version (i.e. those created when versioning is not enabled or versioning is suspended) will not be replicated.
-
 In the rare event that target DR site is entirely lost and previously replicated objects to the DR cluster need to be re-replicated, `mc replicate resync alias/bucket` can be used to initiate a reset. This would initiate a re-sync between the two clusters on a lower priority as the scanner picks up these objects to re-sync.
 
 This is an expensive operation and should be initiated only once - progress of the syncing can be monitored by looking at Prometheus metrics. If object version has been re-replicated, `mc stat --vid --debug` on this version shows an additional header `X-Minio-Replication-Reset-Status` with the replication timestamp and ResetID generated at the time of issuing the `mc replicate resync` command.

docs/bucket/versioning/xl-meta.go

@@ -27,10 +27,12 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"path/filepath"
+	"strings"
 
+	"github.com/klauspost/compress/zip"
 	"github.com/minio/cli"
 	"github.com/tinylib/msgp/msgp"
+	"github.com/yargevad/filepathx"
 )
 
 func main() {
@@ -44,9 +46,10 @@ func main() {
 USAGE:
   {{.Name}} {{if .VisibleFlags}}[FLAGS]{{end}} METAFILES...
 
-Multiple files can be added.
+Multiple files can be added. Files ending in ".zip" will be searched for 'xl.meta' files.  
 Wildcards are accepted: testdir/*.txt will compress all files in testdir ending with .txt
 Directories can be wildcards as well. testdir/*/*.txt will match testdir/subdir/b.txt
+Double stars means full recursive. testdir/**/xl.meta will search for all xl.meta recursively.
 
 {{if .VisibleFlags}}
 GLOBAL FLAGS:
@@ -72,43 +75,15 @@ GLOBAL FLAGS:
 	}
 
 	app.Action = func(c *cli.Context) error {
-		args := c.Args()
-		if len(args) == 0 {
-			// If no args, assume xl.meta
-			args = []string{"xl.meta"}
-		}
-		var files []string
-		for _, pattern := range args {
-			found, err := filepath.Glob(pattern)
-			if err != nil {
-				return err
-			}
-			if len(found) == 0 {
-				return fmt.Errorf("unable to find file %v", pattern)
-			}
-			files = append(files, found...)
-		}
-		for _, file := range files {
-			var r io.Reader
-			switch file {
-			case "-":
-				r = os.Stdin
-			default:
-				f, err := os.Open(file)
-				if err != nil {
-					return err
-				}
-				defer f.Close()
-				r = f
-			}
-
+		ndjson := c.Bool("ndjson")
+		decode := func(r io.Reader, file string) ([]byte, error) {
 			b, err := ioutil.ReadAll(r)
 			if err != nil {
-				return err
+				return nil, err
 			}
 			b, _, minor, err := checkXL2V1(b)
 			if err != nil {
-				return err
+				return nil, err
 			}
 
 			buf := bytes.NewBuffer(nil)
@@ -117,12 +92,12 @@ GLOBAL FLAGS:
 			case 0:
 				_, err = msgp.CopyToJSON(buf, bytes.NewBuffer(b))
 				if err != nil {
-					return err
+					return nil, err
 				}
 			case 1, 2:
 				v, b, err := msgp.ReadBytesZC(b)
 				if err != nil {
-					return err
+					return nil, err
 				}
 				if _, nbuf, err := msgp.ReadUint32Bytes(b); err == nil {
 					// Read metadata CRC (added in v2, ignore if not found)
@@ -131,31 +106,47 @@ GLOBAL FLAGS:
 
 				_, err = msgp.CopyToJSON(buf, bytes.NewBuffer(v))
 				if err != nil {
-					return err
+					return nil, err
 				}
 				data = b
 			default:
-				return fmt.Errorf("unknown metadata version %d", minor)
+				return nil, fmt.Errorf("unknown metadata version %d", minor)
 			}
 
 			if c.Bool("data") {
 				b, err := data.json()
 				if err != nil {
-					return err
+					return nil, err
 				}
 				buf = bytes.NewBuffer(b)
 			}
 			if c.Bool("export") {
+				file := strings.Map(func(r rune) rune {
+					switch {
+					case r >= 'a' && r <= 'z':
+						return r
+					case r >= 'A' && r <= 'Z':
+						return r
+					case r >= '0' && r <= '9':
+						return r
+					case strings.ContainsAny(string(r), "+=-_()!@."):
+						return r
+					default:
+						return '_'
+					}
+				}, file)
 				err := data.files(func(name string, data []byte) {
-					ioutil.WriteFile(fmt.Sprintf("%s-%s.data", file, name), data, os.ModePerm)
+					err = ioutil.WriteFile(fmt.Sprintf("%s-%s.data", file, name), data, os.ModePerm)
+					if err != nil {
+						fmt.Println(err)
+					}
 				})
 				if err != nil {
-					return err
+					return nil, err
 				}
 			}
-			if c.Bool("ndjson") {
-				fmt.Println(buf.String())
-				continue
+			if ndjson {
+				return buf.Bytes(), nil
 			}
 			var msi map[string]interface{}
 			dec := json.NewDecoder(buf)
@@ -163,14 +154,113 @@ GLOBAL FLAGS:
 			dec.UseNumber()
 			err = dec.Decode(&msi)
 			if err != nil {
-				return err
+				return nil, err
 			}
 			b, err = json.MarshalIndent(msi, "", "  ")
+			if err != nil {
+				return nil, err
+			}
+			return b, nil
+		}
+
+		args := c.Args()
+		if len(args) == 0 {
+			// If no args, assume xl.meta
+			args = []string{"xl.meta"}
+		}
+		var files []string
+
+		for _, pattern := range args {
+			if pattern == "-" {
+				files = append(files, pattern)
+				continue
+			}
+			found, err := filepathx.Glob(pattern)
 			if err != nil {
 				return err
 			}
-			fmt.Println(string(b))
+			if len(found) == 0 {
+				return fmt.Errorf("unable to find file %v", pattern)
+			}
+			files = append(files, found...)
 		}
+		if len(files) == 0 {
+			return fmt.Errorf("no files found")
+		}
+		multiple := len(files) > 1 || strings.HasSuffix(files[0], ".zip")
+		if multiple {
+			ndjson = true
+			fmt.Println("{")
+		}
+
+		hasWritten := false
+		for _, file := range files {
+			var r io.Reader
+			var sz int64
+			switch file {
+			case "-":
+				r = os.Stdin
+			default:
+				f, err := os.Open(file)
+				if err != nil {
+					return err
+				}
+				if st, err := f.Stat(); err == nil {
+					sz = st.Size()
+				}
+				defer f.Close()
+				r = f
+			}
+			if strings.HasSuffix(file, ".zip") {
+				zr, err := zip.NewReader(r.(io.ReaderAt), sz)
+				if err != nil {
+					return err
+				}
+				for _, file := range zr.File {
+					if !file.FileInfo().IsDir() && strings.HasSuffix(file.Name, "xl.meta") {
+						r, err := file.Open()
+						if err != nil {
+							return err
+						}
+						// Quote string...
+						b, _ := json.Marshal(file.Name)
+						if hasWritten {
+							fmt.Print(",\n")
+						}
+						fmt.Printf("\t%s: ", string(b))
+
+						b, err = decode(r, file.Name)
+						if err != nil {
+							return err
+						}
+						fmt.Print(string(b))
+						hasWritten = true
+					}
+				}
+			} else {
+				if multiple {
+					// Quote string...
+					b, _ := json.Marshal(file)
+					if hasWritten {
+						fmt.Print(",\n")
+					}
+					fmt.Printf("\t%s: ", string(b))
+				}
+
+				b, err := decode(r, file)
+				if err != nil {
+					return err
+				}
+
+				hasWritten = true
+				fmt.Print(string(b))
+			}
+		}
+		fmt.Println("")
+		if multiple {
+			fmt.Println("}")
+		}
+
 		return nil
 	}
 	err := app.Run(os.Args)

docs/debugging/inspect/main.go

@@ -21,10 +21,13 @@ import (
 	"bufio"
 	"encoding/binary"
 	"encoding/hex"
+	"encoding/json"
+	"errors"
 	"flag"
 	"fmt"
 	"hash/crc32"
 	"io"
+	"io/ioutil"
 	"log"
 	"os"
 	"strings"
@@ -34,25 +37,48 @@ import (
 
 var (
 	key = flag.String("key", "", "decryption string")
-	//js = flag.Bool("json", false, "expect json input")
+	js  = flag.Bool("json", false, "expect json input from stdin")
 )
 
 func main() {
 	flag.Parse()
+
+	if *js {
+		// Match struct in https://github.com/minio/mc/blob/b3ce21fb72a914f50522358668e6464eb97de8d1/cmd/admin-inspect.go#L135
+		input := struct {
+			File string `json:"file"`
+			Key  string `json:"key"`
+		}{}
+		got, err := ioutil.ReadAll(os.Stdin)
+		if err != nil {
+			fatalErr(err)
+		}
+		fatalErr(json.Unmarshal(got, &input))
+		r, err := os.Open(input.File)
+		fatalErr(err)
+		defer r.Close()
+		dstName := strings.TrimSuffix(input.File, ".enc") + ".zip"
+		w, err := os.Create(dstName)
+		fatalErr(err)
+		defer w.Close()
+		decrypt(input.Key, r, w)
+		fmt.Println("Output decrypted to", dstName)
+		return
+	}
 	args := flag.Args()
 	switch len(flag.Args()) {
 	case 0:
 		// Read from stdin, write to stdout.
+		if *key == "" {
+			flag.Usage()
+			fatalErr(errors.New("no key supplied"))
+		}
 		decrypt(*key, os.Stdin, os.Stdout)
 		return
 	case 1:
 		r, err := os.Open(args[0])
 		fatalErr(err)
 		defer r.Close()
-		dstName := strings.TrimSuffix(args[0], ".enc") + ".zip"
-		w, err := os.Create(dstName)
-		fatalErr(err)
-		defer w.Close()
 		if len(*key) == 0 {
 			reader := bufio.NewReader(os.Stdin)
 			fmt.Print("Enter Decryption Key: ")
@@ -61,18 +87,25 @@ func main() {
 			// convert CRLF to LF
 			*key = strings.Replace(text, "\n", "", -1)
 		}
+		*key = strings.TrimSpace(*key)
+		fatalIf(len(*key) != 72, "Unexpected key length: %d, want 72", len(*key))
+
+		dstName := strings.TrimSuffix(args[0], ".enc") + ".zip"
+		w, err := os.Create(dstName)
+		fatalErr(err)
+		defer w.Close()
+
 		decrypt(*key, r, w)
 		fmt.Println("Output decrypted to", dstName)
 		return
 	default:
+		flag.Usage()
 		fatalIf(true, "Only 1 file can be decrypted")
 		os.Exit(1)
 	}
 }
 
 func decrypt(keyHex string, r io.Reader, w io.Writer) {
-	keyHex = strings.TrimSpace(keyHex)
-	fatalIf(len(keyHex) != 72, "Unexpected key length: %d, want 72", len(keyHex))
 	id, err := hex.DecodeString(keyHex[:8])
 	fatalErr(err)
 	key, err := hex.DecodeString(keyHex[8:])

docs/logging/README.md

@@ -72,51 +72,60 @@ NOTE:
 ```json
 {
   "version": "1",
-  "deploymentid": "bc0e4d1e-bacc-42eb-91ad-2d7f3eacfa8d",
-  "time": "2019-08-12T21:34:37.187817748Z",
+  "deploymentid": "51bcc7b9-a447-4251-a940-d9d0aab9af69",
+  "time": "2021-10-08T00:46:36.801714978Z",
+  "trigger": "incoming",
   "api": {
     "name": "PutObject",
     "bucket": "testbucket",
     "object": "hosts",
     "status": "OK",
     "statusCode": 200,
-    "timeToFirstByte": "366333ns",
-    "timeToResponse": "16438202ns"
+    "rx": 380,
+    "tx": 476,
+    "timeToResponse": "257694819ns"
   },
   "remotehost": "127.0.0.1",
-  "requestID": "15BA4A72C0C70AFC",
-  "userAgent": "MinIO (linux; amd64) minio-go/v6.0.32 mc/2019-08-12T18:27:13Z",
+  "requestID": "16ABE7A785E7AC2C",
+  "userAgent": "MinIO (linux; amd64) minio-go/v7.0.15 mc/DEVELOPMENT.2021-10-06T23-39-34Z",
   "requestHeader": {
-    "Authorization": "AWS4-HMAC-SHA256 Credential=minio/20190812/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=d3f02a6aeddeb29b06e1773b6a8422112890981269f2463a26f307b60423177c",
-    "Content-Length": "686",
+    "Authorization": "AWS4-HMAC-SHA256 Credential=minio/20211008/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=4c60a59e5eb3b0a68693c7fee9dbb5a8a509e0717668669194d37bf182fde031",
+    "Content-Length": "380",
     "Content-Type": "application/octet-stream",
-    "User-Agent": "MinIO (linux; amd64) minio-go/v6.0.32 mc/2019-08-12T18:27:13Z",
+    "User-Agent": "MinIO (linux; amd64) minio-go/v7.0.15 mc/DEVELOPMENT.2021-10-06T23-39-34Z",
     "X-Amz-Content-Sha256": "STREAMING-AWS4-HMAC-SHA256-PAYLOAD",
-    "X-Amz-Date": "20190812T213437Z",
-    "X-Amz-Decoded-Content-Length": "512"
+    "X-Amz-Date": "20211008T004636Z",
+    "X-Amz-Decoded-Content-Length": "207",
+    "X-Amz-Server-Side-Encryption": "aws:kms"
   },
   "responseHeader": {
     "Accept-Ranges": "bytes",
     "Content-Length": "0",
     "Content-Security-Policy": "block-all-mixed-content",
-    "ETag": "a414c889dc276457bd7175f974332cb0-1",
-    "Server": "MinIO/DEVELOPMENT.2019-08-12T21-28-07Z",
-    "Vary": "Origin",
-    "X-Amz-Request-Id": "15BA4A72C0C70AFC",
-    "X-Xss-Protection": "1; mode=block"
+    "ETag": "4939450d1beec11e10a91ee7700bb593",
+    "Server": "MinIO",
+    "Strict-Transport-Security": "max-age=31536000; includeSubDomains",
+    "Vary": "Origin,Accept-Encoding",
+    "X-Amz-Request-Id": "16ABE7A785E7AC2C",
+    "X-Amz-Server-Side-Encryption": "aws:kms",
+    "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": "my-minio-key",
+    "X-Content-Type-Options": "nosniff",
+    "X-Xss-Protection": "1; mode=block",
+    "x-amz-version-id": "ac4639f6-c544-4f3f-af1e-b4c0736f67f9"
   },
   "tags": {
     "objectErasureMap": {
-      "object": {
+      "hosts": {
         "poolId": 1,
-        "setId": 10,
+        "setId": 1,
         "disks": [
-          "http://server01/mnt/pool1/disk01",
-          "http://server02/mnt/pool1/disk02",
-          "http://server03/mnt/pool1/disk03",
-          "http://server04/mnt/pool1/disk04"
+          "/mnt/data1",
+          "/mnt/data2",
+          "/mnt/data3",
+          "/mnt/data4"
         ]
-     }
+      }
+    }
   }
 }
 ```

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2021-09-18T18-09-59Z
+  image: quay.io/minio/minio:RELEASE.2021-10-06T23-36-31Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/site-replication/README.md

@@ -0,0 +1,47 @@
+# Site Replication Guide #
+
+This feature allows multiple independent MinIO sites (or clusters) that are using the same external IDentity Provider (IDP) to be configured as replicas. In this situation the set of replica sites are referred to as peer sites or just sites. When site-replication is enabled on a set of sites, the following changes are replicated to all other sites:
+
+- creation and deletion of buckets and objects
+- creation and deletion of all IAM policies
+- creation of STS credentials and creation and deletion of service accounts (for users authenticated by the external IDP)
+- changes to bucket policies, bucket tags, bucket object-lock configurations (including retention and legal hold configuration) and bucket encryption configuration
+
+The following bucket-scoped items are **not replicated**, and can differ between sites:
+
+- bucket notification configuration
+- ILM configuration
+
+This feature is built on top of multi-site bucket replication feature. It enables bucket versioning automatically for all new and existing buckets in the replicated sites.
+
+## Pre-requisites
+
+1. Initially, only **one** of the sites being added for replication may have data. After site-replication is successfully configured, this data is replicated to the other (initially empty) sites. Subsequently, objects may be written to any of the sites, and they will be replicated to all other sites.
+2. Only the **LDAP IDP** is currently supported.
+3. At present, all sites are **required** to have the same root credentials.
+4. At present it is not possible to **add a new site** to an existing set of replicated sites or to **remove a site** from a set of replicated sites.
+5. If using [SSE-S3 or SSE-KMS encryption via KMS](https://docs.min.io/docs/minio-kms-quickstart-guide.html "MinIO KMS Guide"), all sites are required to have access to the same KES keys. This can be achieved via a central KES server or multiple KES servers (say one per site) connected to a central KMS server.
+
+## Configuring Site Replication ##
+
+To configure site replication, ensure that all MinIO sites are using the same external IDP. 
+
+1. Configure an alias in `mc` for each of the sites. For example if you have three MinIO sites, you may run:
+
+```shell
+$ mc alias set minio1 https://minio1.example.com:9000 minio1 minio1123
+$ mc alias set minio2 https://minio2.example.com:9000 minio2 minio2123
+$ mc alias set minio3 https://minio3.example.com:9000 minio3 minio3123
+```
+
+2. Add site replication configuration with:
+
+```shell
+$ mc admin replicate add minio1 minio2 minio3
+```
+
+3. Once the above command returns success, you may query site replication configuration with:
+
+```shell
+$ mc admin replicate info minio1
+```

go.mod

@@ -33,22 +33,22 @@ require (
 	github.com/jcmturner/gokrb5/v8 v8.4.2
 	github.com/json-iterator/go v1.1.11
 	github.com/klauspost/compress v1.13.5
-	github.com/klauspost/cpuid/v2 v2.0.4
+	github.com/klauspost/cpuid/v2 v2.0.6
 	github.com/klauspost/pgzip v1.2.5
 	github.com/klauspost/readahead v1.3.1
-	github.com/klauspost/reedsolomon v1.9.11
+	github.com/klauspost/reedsolomon v1.9.13
 	github.com/lib/pq v1.9.0
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/miekg/dns v1.1.35
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.10.1
+	github.com/minio/console v0.10.3
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.14.0
-	github.com/minio/madmin-go v1.1.6
-	github.com/minio/minio-go/v7 v7.0.15-0.20210921183434-174b4c070788
+	github.com/minio/madmin-go v1.1.7
+	github.com/minio/minio-go/v7 v7.0.15-0.20210928020726-a58653d41dd8
 	github.com/minio/parquet-go v1.0.0
-	github.com/minio/pkg v1.1.3
+	github.com/minio/pkg v1.1.4
 	github.com/minio/selfupdate v0.3.1
 	github.com/minio/sha256-simd v1.0.0
 	github.com/minio/simdjson-go v0.2.1
@@ -69,13 +69,14 @@ require (
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/procfs v0.7.3
 	github.com/rs/cors v1.7.0
+	github.com/rs/dnscache v0.0.0-20210201191234-295bba877686
 	github.com/secure-io/sio-go v0.3.1
 	github.com/shirou/gopsutil/v3 v3.21.7
 	github.com/streadway/amqp v1.0.0
 	github.com/tinylib/msgp v1.1.6-0.20210521143832-0becd170c402
 	github.com/valyala/bytebufferpool v1.0.0
-	github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
+	github.com/yargevad/filepathx v1.0.0
 	go.etcd.io/etcd/api/v3 v3.5.0-beta.4
 	go.etcd.io/etcd/client/v3 v3.5.0-beta.4
 	go.opencensus.io v0.22.5 // indirect

go.sum

@@ -885,16 +885,16 @@ github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgo
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
 github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
-github.com/klauspost/cpuid/v2 v2.0.2/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.3/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
-github.com/klauspost/cpuid/v2 v2.0.4 h1:g0I61F2K2DjRHz1cnxlkNSBIaePVoJIjjnHui8QHbiw=
 github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/RarI=
+github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
 github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/klauspost/readahead v1.3.1 h1:QqXNYvm+VvqYcbrRT4LojUciM0XrznFRIDrbHiJtu/0=
 github.com/klauspost/readahead v1.3.1/go.mod h1:AH9juHzNH7xqdqFHrMRSHeH2Ps+vFf+kblDqzPFiLJg=
-github.com/klauspost/reedsolomon v1.9.11 h1:n2kipJFo+CPqg7fH988XJXjqEyj14RJ8BYj7UayxPNg=
-github.com/klauspost/reedsolomon v1.9.11/go.mod h1:nLvuzNvy1ZDNQW30IuMc2ZWCbiqrJgdLoUS2X8HAUVg=
+github.com/klauspost/reedsolomon v1.9.13 h1:Xr0COKf7F0ACTXUNnz2ZFCWlUKlUTAUX3y7BODdUxqU=
+github.com/klauspost/reedsolomon v1.9.13/go.mod h1:eqPAcE7xar5CIzcdfwydOEdcmchAKAP/qs14y4GCBOk=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
@@ -1004,8 +1004,8 @@ github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68=
 github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.1 h1:+hvfP8C1iMB95AT+ZFDRE+Knn9QPd9lg0CRJY9DRpos=
 github.com/minio/colorjson v1.0.1/go.mod h1:oPM3zQQY8Gz9NGtgvuBEjQ+gPZLKAGc7T+kjMlwtOgs=
-github.com/minio/console v0.10.1 h1:UqY0MWRC41107g9CkF68bSKMaZQmXcO0bGTF43lkhhk=
-github.com/minio/console v0.10.1/go.mod h1:FDVVU8Na/x4tuU12V8RS13FVqex/ilwmrhytVJZ3bfg=
+github.com/minio/console v0.10.3 h1:GofiA2d7x38uELZhChQ2Tg638SVnTHWWMozK8eQdpPM=
+github.com/minio/console v0.10.3/go.mod h1:FDVVU8Na/x4tuU12V8RS13FVqex/ilwmrhytVJZ3bfg=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/direct-csi v1.3.5-0.20210601185811-f7776f7961bf h1:wylCc/PdvdTIqYqVNEU9LJAZBanvfGY1TwTnjM3zQaA=
@@ -1019,8 +1019,9 @@ github.com/minio/kes v0.11.0/go.mod h1:mTF1Bv8YVEtQqF/B7Felp4tLee44Pp+dgI0rhCvgN
 github.com/minio/kes v0.14.0 h1:plCGm4LwR++T1P1sXsJbyFRX54CE1WRuo9PAPj6MC3Q=
 github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQcfA=
 github.com/minio/madmin-go v1.0.12/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
-github.com/minio/madmin-go v1.1.6 h1:L53ALIbAilaEvuvMMT4XkJpd6mtaorkMBwCQ+zraYBA=
 github.com/minio/madmin-go v1.1.6/go.mod h1:vw+c3/u+DeVKqReEavo///Cl2OO8nt5s4ee843hJeLs=
+github.com/minio/madmin-go v1.1.7 h1:vZCnIfPlb40sBap+bmwvxG4/dSfkwF8QCRUHZL16Ylg=
+github.com/minio/madmin-go v1.1.7/go.mod h1:Iu0OnrMWNBYx1lqJTW+BFjBMx0Hi0wjw8VmqhiOs2Jo=
 github.com/minio/mc v0.0.0-20210626002108-cebf3318546f h1:hyFvo5hSFw2K417YvDr/vAKlgCG69uTuhZW/5LNdL0U=
 github.com/minio/mc v0.0.0-20210626002108-cebf3318546f/go.mod h1:tuaonkPjVApCXkbtKENHBtsqUf7YTV33qmFrC+Pgp5g=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
@@ -1030,8 +1031,8 @@ github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8
 github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
 github.com/minio/minio-go/v7 v7.0.11-0.20210607181445-e162fdb8e584/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA=
 github.com/minio/minio-go/v7 v7.0.14/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
-github.com/minio/minio-go/v7 v7.0.15-0.20210921183434-174b4c070788 h1:O+/N9vxhoObjuCuQczycuzdG240SoLrrdnyipJ5JJc0=
-github.com/minio/minio-go/v7 v7.0.15-0.20210921183434-174b4c070788/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
+github.com/minio/minio-go/v7 v7.0.15-0.20210928020726-a58653d41dd8 h1:+8oaaj9Gm/yJFvR09Mz4iefX7pSYcFw6d6fMfvr1Mow=
+github.com/minio/minio-go/v7 v7.0.15-0.20210928020726-a58653d41dd8/go.mod h1:pUV0Pc+hPd1nccgmzQF/EXh48l/Z/yps6QPF1aaie4g=
 github.com/minio/operator v0.0.0-20210812082324-26350f153661 h1:dGAJHpfmhNukFg0M0wDqH+G1OB2YPgZCcT6uv4n9YQk=
 github.com/minio/operator v0.0.0-20210812082324-26350f153661/go.mod h1:zQqn6VGT46xlSpVXh1I/VZRv+eSgHtVu6URdg71YKX8=
 github.com/minio/operator/logsearchapi v0.0.0-20210812082324-26350f153661 h1:tJw15hS3b1dVTf5PwA4roXZ/oRNnHyZ/8Y+yNTmQ5rA=
@@ -1041,8 +1042,9 @@ github.com/minio/parquet-go v1.0.0/go.mod h1:aQlkSOfOq2AtQKkuou3mosNVMwNokd+faTa
 github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/pkg v1.0.8/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
-github.com/minio/pkg v1.1.3 h1:J4vGnlNSxc/o9gDOQMZ3k0L3koA7ZgBQ7GRMrUpt/OY=
 github.com/minio/pkg v1.1.3/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
+github.com/minio/pkg v1.1.4 h1:VxdPYpXAi1xseavq2WAJLr9BaKn5OP0TC1/2R4Ad9Oc=
+github.com/minio/pkg v1.1.4/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
 github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs=
 github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
@@ -1268,6 +1270,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/rs/dnscache v0.0.0-20210201191234-295bba877686 h1:IJ6Df0uxPDtNoByV0KkzVKNseWvZFCNM/S9UoyOMCSI=
+github.com/rs/dnscache v0.0.0-20210201191234-295bba877686/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA=
 github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
@@ -1423,7 +1427,6 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
 github.com/valyala/fasthttp v1.2.0/go.mod h1:4vX61m6KN+xDduDNwXrhIAVZaZaZiQ1luJk8LWSxF3s=
 github.com/valyala/quicktemplate v1.2.0/go.mod h1:EH+4AkTd43SvgIbQHYu59/cJyxDoOVRUAfrukLPuGJ4=
-github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
 github.com/vdemeester/k8s-pkg-credentialprovider v1.17.4/go.mod h1:inCTmtUdr5KJbreVojo06krnTgaeAz/Z7lynpPk/Q2c=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
@@ -1438,6 +1441,8 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0
 github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yargevad/filepathx v1.0.0 h1:SYcT+N3tYGi+NvazubCNlvgIPbzAk7i7y2dwg3I5FYc=
+github.com/yargevad/filepathx v1.0.0/go.mod h1:BprfX/gpYNJHJfc35GjRRpVcwWXS89gGulUIU5tK3tA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 3.1.5
-appVersion: RELEASE.2021-09-18T18-09-59Z
+version: 3.1.8
+appVersion: RELEASE.2021-10-06T23-36-31Z
 keywords:
   - minio
   - storage

helm/minio/templates/_helpers.tpl

@@ -70,8 +70,10 @@ Return the appropriate apiVersion for ingress.
 {{- define "minio.ingress.apiVersion" -}}
 {{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}}
 {{- print "extensions/v1beta1" -}}
-{{- else -}}
+{{- else if semverCompare "<1.19-0" .Capabilities.KubeVersion.GitVersion -}}
 {{- print "networking.k8s.io/v1beta1" -}}
+{{- else -}}
+{{- print "networking.k8s.io/v1" -}}
 {{- end -}}
 {{- end -}}
 
@@ -81,8 +83,10 @@ Return the appropriate apiVersion for console ingress.
 {{- define "minio.consoleIngress.apiVersion" -}}
 {{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}}
 {{- print "extensions/v1beta1" -}}
-{{- else -}}
+{{- else if semverCompare "<1.19-0" .Capabilities.KubeVersion.GitVersion -}}
 {{- print "networking.k8s.io/v1beta1" -}}
+{{- else -}}
+{{- print "networking.k8s.io/v1" -}}
 {{- end -}}
 {{- end -}}
 

helm/minio/templates/configmap.yaml

@@ -2,6 +2,7 @@ apiVersion: v1
 kind: ConfigMap
 metadata:
   name: {{ template "minio.fullname" . }}
+  namespace: {{ .Release.Namespace | quote }}
   labels:
     app: {{ template "minio.name" . }}
     chart: {{ template "minio.chart" . }}

helm/minio/templates/console-ingress.yaml

@@ -6,6 +6,7 @@ apiVersion: {{ template "minio.consoleIngress.apiVersion" . }}
 kind: Ingress
 metadata:
   name: {{ $fullName }}
+  namespace: {{ .Release.Namespace | quote }}
   labels:
     app: {{ template "minio.name" . }}
     chart: {{ template "minio.chart" . }}
@@ -20,6 +21,9 @@ metadata:
 {{ toYaml . | indent 4 }}
 {{- end }}
 spec:
+{{- if .Values.consoleIngress.ingressClassName }}
+  ingressClassName: {{ .Values.consoleIngress.ingressClassName }}
+{{- end }}
 {{- if .Values.consoleIngress.tls }}
   tls:
   {{- range .Values.consoleIngress.tls }}
@@ -35,9 +39,18 @@ spec:
     - http:
         paths:
           - path: {{ $ingressPath }}
+          {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ $fullName }}
+                port: 
+                  number: {{ $servicePort }}
+          {{- else }}
             backend:
               serviceName: {{ $fullName }}
               servicePort: {{ $servicePort }}
+          {{- end }}
       {{- if . }}
       host: {{ . | quote }}
       {{- end }}

helm/minio/templates/console-service.yaml

@@ -6,6 +6,7 @@ apiVersion: v1
 kind: Service
 metadata:
   name: {{ template "minio.fullname" . }}-console
+  namespace: {{ .Release.Namespace | quote }}
   labels:
     app: {{ template "minio.name" . }}
     chart: {{ template "minio.chart" . }}

helm/minio/templates/deployment.yaml

@@ -8,6 +8,7 @@ apiVersion: {{ template "minio.deployment.apiVersion" . }}
 kind: Deployment
 metadata:
   name: {{ template "minio.fullname" . }}
+  namespace: {{ .Release.Namespace | quote }}
   labels:
     app: {{ template "minio.name" . }}
     chart: {{ template "minio.chart" . }}

helm/minio/templates/gateway-deployment.yaml

@@ -8,6 +8,7 @@ apiVersion: {{ template "minio.deployment.apiVersion" . }}
 kind: Deployment
 metadata:
   name: {{ template "minio.fullname" . }}
+  namespace: {{ .Release.Namespace | quote }}
   labels:
     app: {{ template "minio.name" . }}
     chart: {{ template "minio.chart" . }}

helm/minio/templates/ingress.yaml

@@ -6,6 +6,7 @@ apiVersion: {{ template "minio.ingress.apiVersion" . }}
 kind: Ingress
 metadata:
   name: {{ $fullName }}
+  namespace: {{ .Release.Namespace | quote }}
   labels:
     app: {{ template "minio.name" . }}
     chart: {{ template "minio.chart" . }}
@@ -20,6 +21,9 @@ metadata:
 {{ toYaml . | indent 4 }}
 {{- end }}
 spec:
+{{- if .Values.ingress.ingressClassName }}
+  ingressClassName: {{ .Values.ingress.ingressClassName }}
+{{- end }}
 {{- if .Values.ingress.tls }}
   tls:
   {{- range .Values.ingress.tls }}
@@ -35,9 +39,18 @@ spec:
     - http:
         paths:
           - path: {{ $ingressPath }}
+          {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }}
+            pathType: Prefix
+            backend:
+              service:
+                name: {{ $fullName }}
+                port: 
+                  number: {{ $servicePort }}
+          {{- else }}
             backend:
               serviceName: {{ $fullName }}
               servicePort: {{ $servicePort }}
+          {{- end }}
       {{- if . }}
       host: {{ . | quote }}
       {{- end }}

helm/minio/templates/networkpolicy.yaml

@@ -3,6 +3,7 @@ kind: NetworkPolicy
 apiVersion: {{ template "minio.networkPolicy.apiVersion" . }}
 metadata:
   name: {{ template "minio.fullname" . }}
+  namespace: {{ .Release.Namespace | quote }}
   labels:
     app: {{ template "minio.name" . }}
     chart: {{ template "minio.chart" . }}

helm/minio/templates/poddisruptionbudget.yaml

@@ -3,6 +3,7 @@ apiVersion: policy/v1beta1
 kind: PodDisruptionBudget
 metadata:
   name: minio
+  namespace: {{ .Release.Namespace | quote }}
   labels:
     app: {{ template "minio.name" . }}
 spec: