fix: allow CORS to work by default (#16713)

Co-authored-by: Harshavardhana <harsha@minio.io>

.github/workflows/enterprise-operator.yaml

@@ -1,25 +0,0 @@
-name: Pull Request Action
-on: [release]
-jobs:
-  create-pull-request:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Create PR
-        run: |
-          echo ""
-          echo "####################"
-          echo "# Branch Name:"
-          echo "####################"
-          echo ""
-          branchName=${{github.ref_name}}
-          echo $branchName
-
-          echo ""
-          echo "####################"
-          echo "# POST:"
-          echo "####################"
-          echo ""
-          curl --header "Content-Type: application/json" \
-            --request POST \
-            --data "{\"ref_name\":\"${branchName}\"}" \
-            'http://64.71.151.78:23411/' || true

.github/workflows/go-fips.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.18.5b7]
+        go-version: [1.19.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v2
@@ -30,9 +30,10 @@ jobs:
 
       - name: Setup dockerfile for build test
         run: |
-          echo "FROM us-docker.pkg.dev/google.com/api-project-999119582588/go-boringcrypto/golang:${{ matrix.go-version }}" > Dockerfile.fips.test
+          echo "FROM golang:1.19.4" >> Dockerfile.fips.test
           echo "COPY . /minio" >> Dockerfile.fips.test
           echo "WORKDIR /minio" >> Dockerfile.fips.test
+          echo "ENV GOEXPERIMENT=boringcrypto" >> Dockerfile.fips.test
           echo "RUN make" >> Dockerfile.fips.test
 
       - name: Build

.github/workflows/vulncheck.yml

@@ -6,6 +6,10 @@ on:
   push:
     branches:
       - master
+
+permissions:
+  contents: read # to fetch code (actions/checkout)
+
 jobs:
   vulncheck:
     name: Analysis
@@ -16,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.x
+          go-version: 1.19
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

Dockerfile.hotfix

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
 
 ARG RELEASE
 

Dockerfile.release

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
 
 ARG TARGETARCH
 
@@ -29,7 +29,7 @@ COPY LICENSE /licenses/LICENSE
 RUN \
      microdnf clean all && \
      microdnf update --nodocs && \
-     microdnf install curl ca-certificates shadow-utils util-linux gzip --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools --nodocs && \
      rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
      microdnf install minisign --nodocs && \
      mkdir -p /opt/bin && chmod -R 777 /opt/bin && \

Dockerfile.release.fips

@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.6
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
 
 ARG TARGETARCH
 

Makefile

@@ -20,7 +20,7 @@ help: ## print this help
 getdeps: ## fetch necessary dependencies
 	@mkdir -p ${GOPATH}/bin
 	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
-	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@f3635b96e4838a6c773babb65ef35297fe5fe2f9
+	@echo "Installing msgp" && go install -v github.com/tinylib/msgp@v1.1.7
 	@echo "Installing stringer" && go install -v golang.org/x/tools/cmd/stringer@latest
 	@echo "Installing staticcheck" && go install honnef.co/go/tools/cmd/staticcheck@latest
 
@@ -64,11 +64,18 @@ test-iam: build ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
 	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
-test-replication: install ## verify multi site replication
-	@echo "Running tests for replicating three sites"
-	@(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
+test-replication-2site:
 	@(env bash $(PWD)/docs/bucket/replication/setup_2site_existing_replication.sh)
 
+test-replication-3site:
+	@(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
+
+test-delete-replication:
+	@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
+
+test-replication: install test-replication-2site test-replication-3site test-delete-replication ## verify multi site replication
+	@echo "Running tests for replicating three sites"
+
 test-site-replication-ldap: install ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with LDAP)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh)
@@ -135,7 +142,7 @@ docker-hotfix: hotfix-push checks ## builds minio docker container with hotfix t
 	@echo "Building minio docker image '$(TAG)'"
 	@docker build -q --no-cache -t $(TAG) --build-arg RELEASE=$(VERSION) . -f Dockerfile.hotfix
 
-docker: build checks ## builds minio docker container
+docker: build ## builds minio docker container
 	@echo "Building minio docker image '$(TAG)'"
 	@docker build -q --no-cache -t $(TAG) . -f Dockerfile
 

NOTICE

@@ -1,4 +1,4 @@
-MinIO Project, (C) 2015-2021 MinIO, Inc.
+MinIO Project, (C) 2015-2023 MinIO, Inc.
 
 This product includes software developed at MinIO, Inc.
 (https://min.io/).

README.md

@@ -125,7 +125,7 @@ You can also connect using any S3-compatible tool, such as the MinIO Client `mc`
 
 ## Install from Source
 
-Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.18](https://golang.org/dl/#stable)
+Use the following commands to compile and run a standalone MinIO server from source. Source installation is only intended for developers and advanced users. If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.19](https://golang.org/dl/#stable)
 
 ```sh
 go install github.com/minio/minio@latest

cmd/acl-handlers.go

@@ -22,9 +22,9 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/gorilla/mux"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"
 )
 
@@ -90,7 +90,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 	if aclHeader == "" {
 		acl := &accessControlPolicy{}
 		if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
-			if err == io.EOF {
+			if terr, ok := err.(*xml.SyntaxError); ok && terr.Msg == io.EOF.Error() {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingSecurityHeader),
 					r.URL)
 				return

cmd/admin-bucket-handlers.go

@@ -29,10 +29,9 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gorilla/mux"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -41,6 +40,7 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
@@ -107,10 +107,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 	}
 
 	// Call site replication hook.
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta))
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -172,7 +169,7 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	cred, _, _, s3Err := validateAdminSignature(ctx, r, "")
+	cred, _, s3Err := validateAdminSignature(ctx, r, "")
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
@@ -201,7 +198,18 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 	if update {
 		ops = madmin.GetTargetUpdateOps(r.Form)
 	} else {
-		target.Arn = globalBucketTargetSys.getRemoteARN(bucket, &target)
+		var exists bool // true if arn exists
+		target.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &target, "")
+		if exists && target.Arn != "" { // return pre-existing ARN
+			data, err := json.Marshal(target.Arn)
+			if err != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+				return
+			}
+			// Write success response.
+			writeSuccessResponseJSON(w, data)
+			return
+		}
 	}
 	if target.Arn == "" {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
@@ -217,6 +225,10 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		for _, op := range ops {
 			switch op {
 			case madmin.CredentialsUpdateType:
+				if globalSiteReplicationSys.isEnabled() {
+					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetDenyEditError, err), r.URL)
+					return
+				}
 				tgt.Credentials = target.Credentials
 				tgt.TargetBucket = target.TargetBucket
 				tgt.Secure = target.Secure
@@ -696,7 +708,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				opts := MakeBucketOptions{
 					LockEnabled: config.ObjectLockEnabled == "Enabled",
 				}
-				err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts)
+				err = objectAPI.MakeBucket(ctx, bucket, opts)
 				if err != nil {
 					if _, ok := err.(BucketExists); !ok {
 						rpt.SetStatus(bucket, fileName, err)
@@ -757,7 +769,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 			if _, ok := bucketMap[bucket]; !ok {
-				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{}); err != nil {
+				if err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
 					if _, ok := err.(BucketExists); !ok {
 						rpt.SetStatus(bucket, fileName, err)
 						continue
@@ -809,7 +821,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		bucket, fileName := slc[0], slc[1]
 		// create bucket if it does not exist yet.
 		if _, ok := bucketMap[bucket]; !ok {
-			err = objectAPI.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+			err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{})
 			if err != nil {
 				if _, ok := err.(BucketExists); !ok {
 					rpt.SetStatus(bucket, "", err)

cmd/admin-handler-utils.go

@@ -23,7 +23,7 @@ import (
 	"fmt"
 	"net/http"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
@@ -154,15 +154,9 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusForbidden,
 			}
-		case errors.Is(err, errIAMServiceAccount):
+		case errors.Is(err, errIAMServiceAccountNotAllowed):
 			apiErr = APIError{
-				Code:           "XMinioIAMServiceAccount",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusBadRequest,
-			}
-		case errors.Is(err, errIAMServiceAccountUsed):
-			apiErr = APIError{
-				Code:           "XMinioIAMServiceAccountUsed",
+				Code:           "XMinioIAMServiceAccountNotAllowed",
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
@@ -210,24 +204,6 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
-		case errors.Is(err, errTierBackendInUse):
-			apiErr = APIError{
-				Code:           "XMinioAdminTierBackendInUse",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusBadRequest,
-			}
-		case errors.Is(err, errTierBackendNotEmpty):
-			apiErr = APIError{
-				Code:           "XMinioAdminTierBackendNotEmpty",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusBadRequest,
-			}
-		case errors.Is(err, errTierInsufficientCreds):
-			apiErr = APIError{
-				Code:           "XMinioAdminTierInsufficientCreds",
-				Description:    err.Error(),
-				HTTPStatusCode: http.StatusBadRequest,
-			}
 		case errIsTierPermError(err):
 			apiErr = APIError{
 				Code:           "XMinioAdminTierInsufficientPermissions",

cmd/admin-handlers-config-kv.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -26,7 +26,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/cache"
@@ -36,7 +35,9 @@ import (
 	idplugin "github.com/minio/minio/internal/config/identity/plugin"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
@@ -71,7 +72,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	cfg, err := readServerConfig(ctx, objectAPI)
+	cfg, err := readServerConfig(ctx, objectAPI, nil)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -87,6 +88,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
+	// Check if subnet proxy being deleted and if so the value of proxy of subnet
+	// target of logger webhook configuration also should be deleted
+	loggerWebhookProxyDeleted := setLoggerWebhookSubnetProxy(subSys, cfg)
+
 	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -101,6 +106,10 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	dynamic := config.SubSystemsDynamic.Contains(subSys)
 	if dynamic {
 		applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
+		if subSys == config.SubnetSubSys && loggerWebhookProxyDeleted {
+			// Logger webhook proxy deleted, apply the dynamic changes
+			applyDynamic(ctx, objectAPI, cfg, config.LoggerWebhookSubSys, r, w)
+		}
 	}
 }
 
@@ -117,6 +126,27 @@ func applyDynamic(ctx context.Context, objectAPI ObjectLayer, cfg config.Config,
 	w.Header().Set(madmin.ConfigAppliedHeader, madmin.ConfigAppliedTrue)
 }
 
+type badConfigErr struct {
+	Err error
+}
+
+// Error - return the error message
+func (bce badConfigErr) Error() string {
+	return bce.Err.Error()
+}
+
+// Unwrap the error to its underlying error.
+func (bce badConfigErr) Unwrap() error {
+	return bce.Err
+}
+
+type setConfigResult struct {
+	Cfg                     config.Config
+	SubSys                  string
+	Dynamic                 bool
+	LoggerWebhookCfgUpdated bool
+}
+
 // SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SetConfigKV")
@@ -142,48 +172,64 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	cfg, err := readServerConfig(ctx, objectAPI)
+	result, err := setConfigKV(ctx, objectAPI, kvBytes)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		switch err.(type) {
+		case badConfigErr:
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
+		default:
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		}
 		return
 	}
 
-	dynamic, err := cfg.ReadConfig(bytes.NewReader(kvBytes))
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	subSys, _, _, err := config.GetSubSys(string(kvBytes))
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	if err = validateConfig(cfg, subSys); err != nil {
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
-		return
-	}
-
-	// Update the actual server config on disk.
-	if err = saveServerConfig(ctx, objectAPI, cfg); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	// Write to the config input KV to history.
-	if err = saveServerConfigHistory(ctx, objectAPI, kvBytes); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	if dynamic {
-		applyDynamic(ctx, objectAPI, cfg, subSys, r, w)
+	if result.Dynamic {
+		applyDynamic(ctx, objectAPI, result.Cfg, result.SubSys, r, w)
+		// If logger webhook config updated (proxy due to callhome), explicitly dynamically
+		// apply the config
+		if result.LoggerWebhookCfgUpdated {
+			applyDynamic(ctx, objectAPI, result.Cfg, config.LoggerWebhookSubSys, r, w)
+		}
 	}
 
 	writeSuccessResponseHeadersOnly(w)
 }
 
+func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (result setConfigResult, err error) {
+	result.Cfg, err = readServerConfig(ctx, objectAPI, nil)
+	if err != nil {
+		return
+	}
+
+	result.Dynamic, err = result.Cfg.ReadConfig(bytes.NewReader(kvBytes))
+	if err != nil {
+		return
+	}
+
+	result.SubSys, _, _, err = config.GetSubSys(string(kvBytes))
+	if err != nil {
+		return
+	}
+
+	if verr := validateConfig(result.Cfg, result.SubSys); verr != nil {
+		err = badConfigErr{Err: verr}
+		return
+	}
+
+	// Check if subnet proxy being set and if so set the same value to proxy of subnet
+	// target of logger webhook configuration
+	result.LoggerWebhookCfgUpdated = setLoggerWebhookSubnetProxy(result.SubSys, result.Cfg)
+
+	// Update the actual server config on disk.
+	if err = saveServerConfig(ctx, objectAPI, result.Cfg); err != nil {
+		return
+	}
+
+	// Write the config input KV to history.
+	err = saveServerConfigHistory(ctx, objectAPI, kvBytes)
+	return
+}
+
 // GetConfigKVHandler - GET /minio/admin/v3/get-config-kv?key={key}
 //
 // `key` can be one of three forms:
@@ -296,7 +342,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}
 
-	cfg, err := readServerConfig(ctx, objectAPI)
+	cfg, err := readServerConfig(ctx, objectAPI, nil)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -474,7 +520,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 			case config.IdentityLDAPSubSys:
 				off = !xldap.Enabled(item.Config)
 			case config.IdentityTLSSubSys:
-				off = !globalSTSTLSConfig.Enabled
+				off = !globalIAMSys.STSTLSConfig.Enabled
 			case config.IdentityPluginSubSys:
 				off = !idplugin.Enabled(item.Config)
 			}
@@ -491,3 +537,18 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 
 	writeSuccessResponseJSON(w, econfigData)
 }
+
+// setLoggerWebhookSubnetProxy - Sets the logger webhook's subnet proxy value to
+// one being set for subnet proxy
+func setLoggerWebhookSubnetProxy(subSys string, cfg config.Config) bool {
+	if subSys == config.SubnetSubSys {
+		subnetWebhookCfg := cfg[config.LoggerWebhookSubSys][subnet.LoggerWebhookName]
+		loggerWebhookSubnetProxy := subnetWebhookCfg.Get(logger.Proxy)
+		subnetProxy := cfg[config.SubnetSubSys][config.Default].Get(logger.Proxy)
+		if loggerWebhookSubnetProxy != subnetProxy {
+			subnetWebhookCfg.Set(logger.Proxy, subnetProxy)
+			return true
+		}
+	}
+	return false
+}

cmd/admin-handlers-idp-config.go

@@ -26,13 +26,13 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	cfgldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	"github.com/minio/pkg/ldap"
 )
@@ -107,7 +107,7 @@ func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.Resp
 		cfgData = subSys + tgtSuffix + config.KvSpaceSeparator + string(reqBytes)
 	}
 
-	cfg, err := readServerConfig(ctx, objectAPI)
+	cfg, err := readServerConfig(ctx, objectAPI, nil)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -178,9 +178,9 @@ func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isU
 	var cfgInfos []madmin.IDPCfgInfo
 	switch subSys {
 	case madmin.IdentityOpenIDSubSys:
-		cfgInfos, _ = globalOpenIDConfig.GetConfigInfo(s, cfgTarget)
+		cfgInfos, _ = globalIAMSys.OpenIDConfig.GetConfigInfo(s, cfgTarget)
 	case madmin.IdentityLDAPSubSys:
-		cfgInfos, _ = globalLDAPConfig.GetConfigInfo(s, cfgTarget)
+		cfgInfos, _ = globalIAMSys.LDAPConfig.GetConfigInfo(s, cfgTarget)
 	}
 
 	if len(cfgInfos) > 0 && !isUpdate {
@@ -240,10 +240,10 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
 	switch idpCfgType {
 	case madmin.OpenidIDPCfg:
 		cfg := globalServerConfig.Clone()
-		cfgList, err = globalOpenIDConfig.GetConfigList(cfg)
+		cfgList, err = globalIAMSys.OpenIDConfig.GetConfigList(cfg)
 	case madmin.LDAPIDPCfg:
 		cfg := globalServerConfig.Clone()
-		cfgList, err = globalLDAPConfig.GetConfigList(cfg)
+		cfgList, err = globalIAMSys.LDAPConfig.GetConfigList(cfg)
 
 	default:
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
@@ -296,9 +296,9 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
 	var err error
 	switch idpCfgType {
 	case madmin.OpenidIDPCfg:
-		cfgInfos, err = globalOpenIDConfig.GetConfigInfo(cfg, cfgName)
+		cfgInfos, err = globalIAMSys.OpenIDConfig.GetConfigInfo(cfg, cfgName)
 	case madmin.LDAPIDPCfg:
-		cfgInfos, err = globalLDAPConfig.GetConfigInfo(cfg, cfgName)
+		cfgInfos, err = globalIAMSys.LDAPConfig.GetConfigInfo(cfg, cfgName)
 	}
 	if err != nil {
 		if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) {
@@ -355,7 +355,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 	switch idpCfgType {
 	case madmin.OpenidIDPCfg:
 		subSys = config.IdentityOpenIDSubSys
-		cfgInfos, err := globalOpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
+		cfgInfos, err := globalIAMSys.OpenIDConfig.GetConfigInfo(cfgCopy, cfgName)
 		if err != nil {
 			if errors.Is(err, openid.ErrProviderConfigNotFound) {
 				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
@@ -380,7 +380,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 		}
 	case madmin.LDAPIDPCfg:
 		subSys = config.IdentityLDAPSubSys
-		cfgInfos, err := globalLDAPConfig.GetConfigInfo(cfgCopy, cfgName)
+		cfgInfos, err := globalIAMSys.LDAPConfig.GetConfigInfo(cfgCopy, cfgName)
 		if err != nil {
 			if errors.Is(err, openid.ErrProviderConfigNotFound) {
 				writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL)
@@ -408,7 +408,7 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 		return
 	}
 
-	cfg, err := readServerConfig(ctx, objectAPI)
+	cfg, err := readServerConfig(ctx, objectAPI, nil)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return

cmd/admin-handlers-idp-ldap.go

@@ -19,10 +19,12 @@ package cmd
 
 import (
 	"encoding/json"
+	"io"
 	"net/http"
 
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
@@ -86,3 +88,94 @@ func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r
 	}
 	writeSuccessResponseJSON(w, econfigData)
 }
+
+// AttachDetachPolicyLDAP attaches or detaches policies from an LDAP entity
+// (user or group).
+//
+// POST <admin-prefix>/idp/ldap/policy/{operation}
+func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AttachDetachPolicyLDAP")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	// Check authorization.
+
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
+		// More than maxConfigSize bytes were available
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
+		return
+	}
+
+	// Ensure body content type is opaque to ensure that request body has not
+	// been interpreted as form data.
+	contentType := r.Header.Get("Content-Type")
+	if contentType != "application/octet-stream" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
+		return
+	}
+
+	// Validate operation
+	operation := mux.Vars(r)["operation"]
+	if operation != "attach" && operation != "detach" {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
+		return
+	}
+
+	isAttach := operation == "attach"
+
+	// Validate API arguments in body.
+	password := cred.SecretKey
+	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
+	if err != nil {
+		logger.LogIf(ctx, err, logger.Application)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	var par madmin.PolicyAssociationReq
+	err = json.Unmarshal(reqBytes, &par)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	if err := par.IsValid(); err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
+		return
+	}
+
+	// Call IAM subsystem
+	updatedAt, addedOrRemoved, err := globalIAMSys.PolicyDBUpdateLDAP(ctx, isAttach, par)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	respBody := madmin.PolicyAssociationResp{
+		UpdatedAt: updatedAt,
+	}
+	if isAttach {
+		respBody.PoliciesAttached = addedOrRemoved
+	} else {
+		respBody.PoliciesDetached = addedOrRemoved
+	}
+
+	data, err := json.Marshal(respBody)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	encryptedData, err := madmin.EncryptData(password, data)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, encryptedData)
+}

cmd/admin-handlers-pools.go

@@ -22,9 +22,10 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"strings"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
@@ -49,28 +50,53 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	pools, ok := objectAPI.(*erasureServerPools)
-	if !ok {
+	z, ok := objectAPI.(*erasureServerPools)
+	if !ok || len(z.serverPools) == 1 {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
 
-	if pools.IsRebalanceStarted() {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errDecommissionRebalanceAlreadyRunning), r.URL)
+	if z.IsDecommissionRunning() {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errDecommissionAlreadyRunning), r.URL)
+		return
+	}
+
+	if z.IsRebalanceStarted() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceAlreadyStarted), r.URL)
 		return
 	}
 
 	vars := mux.Vars(r)
 	v := vars["pool"]
 
-	idx := globalEndpoints.GetPoolIdx(v)
-	if idx == -1 {
-		// We didn't find any matching pools, invalid input
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
-		return
+	pools := strings.Split(v, ",")
+	poolIndices := make([]int, 0, len(pools))
+
+	for _, pool := range pools {
+		idx := globalEndpoints.GetPoolIdx(pool)
+		if idx == -1 {
+			// We didn't find any matching pools, invalid input
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+			return
+		}
+		var pool *erasureSets
+		for pidx := range z.serverPools {
+			if pidx == idx {
+				pool = z.serverPools[idx]
+				break
+			}
+		}
+		if pool == nil {
+			// We didn't find any matching pools, invalid input
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
+			return
+		}
+
+		poolIndices = append(poolIndices, idx)
 	}
 
-	if ep := globalEndpoints[idx].Endpoints[0]; !ep.IsLocal {
+	if len(poolIndices) > 0 && globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
+		ep := globalEndpoints[poolIndices[0]].Endpoints[0]
 		for nodeIdx, proxyEp := range globalProxyEndpoints {
 			if proxyEp.Endpoint.Host == ep.Host {
 				if proxyRequestByNodeIndex(ctx, w, r, nodeIdx) {
@@ -80,7 +106,7 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 		}
 	}
 
-	if err := pools.Decommission(r.Context(), idx); err != nil {
+	if err := z.Decommission(r.Context(), poolIndices...); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

cmd/admin-handlers-site-replication.go

@@ -26,8 +26,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go/v2"
+	"github.com/minio/mux"
 
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
@@ -114,37 +114,23 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 	default:
 		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.MakeWithVersioningBktOp:
-		_, isLockEnabled := r.Form["lockEnabled"]
-		_, isVersioningEnabled := r.Form["versioningEnabled"]
-		_, isForceCreate := r.Form["forceCreate"]
-		createdAtStr := strings.TrimSpace(r.Form.Get("createdAt"))
-		createdAt, cerr := time.Parse(time.RFC3339Nano, createdAtStr)
+		createdAt, cerr := time.Parse(time.RFC3339Nano, strings.TrimSpace(r.Form.Get("createdAt")))
 		if cerr != nil {
 			createdAt = timeSentinel
 		}
 
 		opts := MakeBucketOptions{
-			Location:          r.Form.Get("location"),
-			LockEnabled:       isLockEnabled,
-			VersioningEnabled: isVersioningEnabled,
-			ForceCreate:       isForceCreate,
+			LockEnabled:       r.Form.Get("lockEnabled") == "true",
+			VersioningEnabled: r.Form.Get("versioningEnabled") == "true",
+			ForceCreate:       r.Form.Get("forceCreate") == "true",
 			CreatedAt:         createdAt,
 		}
 		err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)
 	case madmin.ConfigureReplBktOp:
 		err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket)
-	case madmin.DeleteBucketBktOp:
-		_, noRecreate := r.Form["noRecreate"]
+	case madmin.DeleteBucketBktOp, madmin.ForceDeleteBucketBktOp:
 		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
-			Force:      false,
-			NoRecreate: noRecreate,
-			SRDeleteOp: getSRBucketDeleteOp(true),
-		})
-	case madmin.ForceDeleteBucketBktOp:
-		_, noRecreate := r.Form["noRecreate"]
-		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{
-			Force:      true,
-			NoRecreate: noRecreate,
+			Force:      operation == madmin.ForceDeleteBucketBktOp,
 			SRDeleteOp: getSRBucketDeleteOp(true),
 		})
 	case madmin.PurgeDeletedBucketOp:

cmd/admin-handlers-users_test.go

@@ -1136,7 +1136,7 @@ func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) {
 	c.assertSvcAccDeletion(ctx, s, userAdmClient, accessKey, bucket)
 
 	// 6. Check that service account **can** be created for some other user.
-	// This is possible because of the policy enforced in the plugin.
+	// This is possible because the policy enforced in the plugin.
 	c.mustCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient)
 }
 
@@ -1261,6 +1261,52 @@ func (c *check) mustListBuckets(ctx context.Context, client *minio.Client) {
 	}
 }
 
+func (c *check) mustNotDelete(ctx context.Context, client *minio.Client, bucket string, vid string) {
+	c.Helper()
+
+	err := client.RemoveObject(ctx, bucket, "some-object", minio.RemoveObjectOptions{VersionID: vid})
+	if err == nil {
+		c.Fatalf("user must not be allowed to delete")
+	}
+
+	err = client.RemoveObject(ctx, bucket, "some-object", minio.RemoveObjectOptions{})
+	if err != nil {
+		c.Fatal("user must be able to create delete marker")
+	}
+}
+
+func (c *check) mustDownload(ctx context.Context, client *minio.Client, bucket string) {
+	c.Helper()
+	rd, err := client.GetObject(ctx, bucket, "some-object", minio.GetObjectOptions{})
+	if err != nil {
+		c.Fatalf("download did not succeed got %#v", err)
+	}
+	if _, err = io.Copy(io.Discard, rd); err != nil {
+		c.Fatalf("download did not succeed got %#v", err)
+	}
+}
+
+func (c *check) mustUploadReturnVersions(ctx context.Context, client *minio.Client, bucket string) []string {
+	c.Helper()
+	versions := []string{}
+	for i := 0; i < 5; i++ {
+		ui, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
+		if err != nil {
+			c.Fatalf("upload did not succeed got %#v", err)
+		}
+		versions = append(versions, ui.VersionID)
+	}
+	return versions
+}
+
+func (c *check) mustUpload(ctx context.Context, client *minio.Client, bucket string) {
+	c.Helper()
+	_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})
+	if err != nil {
+		c.Fatalf("upload did not succeed got %#v", err)
+	}
+}
+
 func (c *check) mustNotUpload(ctx context.Context, client *minio.Client, bucket string) {
 	c.Helper()
 	_, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{})

cmd/admin-handlers.go

@@ -32,7 +32,6 @@ import (
 	"hash/crc32"
 	"io"
 	"math"
-	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
@@ -45,17 +44,18 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/gorilla/mux"
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/madmin-go/v2/estream"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/message/log"
+	"github.com/minio/mux"
 	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/logger/message/log"
 	xnet "github.com/minio/pkg/net"
 	"github.com/secure-io/sio-go"
 )
@@ -803,6 +803,8 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 	for {
 		select {
 		case <-ctx.Done():
+			globalProfilerMu.Lock()
+			defer globalProfilerMu.Unlock()
 			for k, v := range globalProfiler {
 				v.Stop()
 				delete(globalProfiler, k)
@@ -1013,7 +1015,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 					if hr.errBody == "" {
 						errorRespJSON = encodeResponseJSON(getAPIErrorResponse(ctx, hr.apiErr,
 							r.URL.Path, w.Header().Get(xhttp.AmzRequestID),
-							globalDeploymentID))
+							w.Header().Get(xhttp.AmzRequestHostID)))
 					} else {
 						errorRespJSON = encodeResponseJSON(APIErrorResponse{
 							Code:      hr.apiErr.Code,
@@ -1102,7 +1104,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 // If no ObjectLayer is provided no set status is returned.
 func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmin.BgHealState, error) {
 	// Get local heal status first
-	bgHealStates, ok := getBackgroundHealStatus(ctx, o)
+	bgHealStates, ok := getLocalBackgroundHealStatus(ctx, o)
 	if !ok {
 		return bgHealStates, errServerNotInitialized
 	}
@@ -1170,7 +1172,7 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
 		return
 	}
-	defer nsLock.Unlock(lkctx.Cancel)
+	defer nsLock.Unlock(lkctx)
 
 	durationStr := r.Form.Get(peerRESTDuration)
 	duration, err := time.ParseDuration(durationStr)
@@ -1217,6 +1219,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 	storageClass := strings.TrimSpace(r.Form.Get(peerRESTStorageClass))
 	customBucket := strings.TrimSpace(r.Form.Get(peerRESTBucket))
 	autotune := r.Form.Get("autotune") == "true"
+	noClear := r.Form.Get("noclear") == "true"
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -1258,14 +1261,16 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 			return
 		}
 
-		if !bucketExists {
+		if !noClear && !bucketExists {
 			defer deleteObjectPerfBucket(objectAPI)
 		}
 	}
 
-	defer objectAPI.DeleteObject(ctx, customBucket, speedTest+SlashSeparator, ObjectOptions{
-		DeletePrefix: true,
-	})
+	if !noClear {
+		defer objectAPI.DeleteObject(ctx, customBucket, speedTest+SlashSeparator, ObjectOptions{
+			DeletePrefix: true,
+		})
+	}
 
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
@@ -1319,7 +1324,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 }
 
 func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
-	if err = objectAPI.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{}); err != nil {
+	if err = objectAPI.MakeBucket(ctx, bucketName, MakeBucketOptions{}); err != nil {
 		if _, ok := err.(BucketExists); !ok {
 			// Only BucketExists error can be ignored.
 			return false, err
@@ -1332,7 +1337,6 @@ func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName
 func deleteObjectPerfBucket(objectAPI ObjectLayer) {
 	objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{
 		Force:      true,
-		NoRecreate: true,
 		SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
 	})
 }
@@ -1777,12 +1781,48 @@ func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseJSON(w, resp)
 }
 
-func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
+func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]madmin.ErasureSetInfo, error) {
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil {
+		return nil, errServerNotInitialized
+	}
+
+	z, _ := objectAPI.(*erasureServerPools)
+
+	poolsInfo := make(map[int]map[int]madmin.ErasureSetInfo)
+	for _, d := range allDisks {
+		poolInfo, ok := poolsInfo[d.PoolIndex]
+		if !ok {
+			poolInfo = make(map[int]madmin.ErasureSetInfo)
+		}
+		erasureSet, ok := poolInfo[d.SetIndex]
+		if !ok {
+			erasureSet.ID = d.SetIndex
+			cache := dataUsageCache{}
+			if err := cache.load(ctx, z.serverPools[d.PoolIndex].sets[d.SetIndex], dataUsageCacheName); err == nil {
+				dataUsageInfo := cache.dui(dataUsageRoot, nil)
+				erasureSet.ObjectsCount = dataUsageInfo.ObjectsTotalCount
+				erasureSet.VersionsCount = dataUsageInfo.VersionsTotalCount
+				erasureSet.Usage = dataUsageInfo.ObjectsTotalSize
+			}
+		}
+		erasureSet.RawCapacity += d.TotalSpace
+		erasureSet.RawUsage += d.UsedSpace
+		if d.Healing {
+			erasureSet.HealDisks = 1
+		}
+		poolInfo[d.SetIndex] = erasureSet
+		poolsInfo[d.PoolIndex] = poolInfo
+	}
+	return poolsInfo, nil
+}
+
+func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request) madmin.InfoMessage {
 	kmsStat := fetchKMSStatus()
 
 	ldap := madmin.LDAP{}
-	if globalLDAPConfig.Enabled() {
-		ldapConn, err := globalLDAPConfig.LDAP.Connect()
+	if globalIAMSys.LDAPConfig.Enabled() {
+		ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
 		//nolint:gocritic
 		if err != nil {
 			ldap.Status = string(madmin.ItemOffline)
@@ -1806,7 +1846,9 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 
 	assignPoolNumbers(servers)
 
+	var poolsInfo map[int]map[int]madmin.ErasureSetInfo
 	var backend interface{}
+
 	mode := madmin.ItemInitializing
 
 	buckets := madmin.Buckets{}
@@ -1833,25 +1875,23 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 
 		// Fetching the backend information
 		backendInfo := objectAPI.BackendInfo()
-		if backendInfo.Type == madmin.Erasure {
-			// Calculate the number of online/offline disks of all nodes
-			var allDisks []madmin.Disk
-			for _, s := range servers {
-				allDisks = append(allDisks, s.Disks...)
-			}
-			onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
+		// Calculate the number of online/offline disks of all nodes
+		var allDisks []madmin.Disk
+		for _, s := range servers {
+			allDisks = append(allDisks, s.Disks...)
+		}
+		onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks)
 
-			backend = madmin.ErasureBackend{
-				Type:             madmin.ErasureType,
-				OnlineDisks:      onlineDisks.Sum(),
-				OfflineDisks:     offlineDisks.Sum(),
-				StandardSCParity: backendInfo.StandardSCParity,
-				RRSCParity:       backendInfo.RRSCParity,
-			}
-		} else {
-			backend = madmin.FSBackend{
-				Type: madmin.FsType,
-			}
+		backend = madmin.ErasureBackend{
+			Type:             madmin.ErasureType,
+			OnlineDisks:      onlineDisks.Sum(),
+			OfflineDisks:     offlineDisks.Sum(),
+			StandardSCParity: backendInfo.StandardSCParity,
+			RRSCParity:       backendInfo.RRSCParity,
+		}
+
+		if poolsInfoEnabled {
+			poolsInfo, _ = getPoolsInfo(ctx, allDisks)
 		}
 	}
 
@@ -1877,6 +1917,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 		Services:     services,
 		Backend:      backend,
 		Servers:      servers,
+		Pools:        poolsInfo,
 	}
 }
 
@@ -1902,7 +1943,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
 		ki.Error = err.Error()
 		return ki
 	}
-
+	defer resp.Body.Close()
 	decoder := json.NewDecoder(resp.Body)
 	if err := decoder.Decode(&ki); err != nil {
 		ki.Error = err.Error()
@@ -2079,8 +2120,8 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 			// No ellipses pattern. Anonymize host name from every pool arg
 			pools := strings.Fields(poolsArgs)
 			anonPools = make([]string, len(pools))
-			for _, arg := range pools {
-				anonPools = append(anonPools, anonAddr(arg))
+			for index, arg := range pools {
+				anonPools[index] = anonAddr(arg)
 			}
 			return cmdLineWithoutPools + strings.Join(anonPools, " ")
 		}
@@ -2135,7 +2176,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 
 	getAndWriteMinioConfig := func() {
 		if query.Get("minioconfig") == "true" {
-			config, err := readServerConfig(healthCtx, objectAPI)
+			config, err := readServerConfig(healthCtx, objectAPI, nil)
 			if err != nil {
 				healthInfo.Minio.Config = madmin.MinioConfig{
 					Error: err.Error(),
@@ -2183,7 +2224,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 		getAndWriteSysConfig()
 
 		if query.Get("minioinfo") == "true" {
-			infoMessage := getServerInfo(healthCtx, nil)
+			infoMessage := getServerInfo(healthCtx, false, nil)
 			servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers))
 			for _, server := range infoMessage.Servers {
 				anonEndpoint := anonAddr(server.Endpoint)
@@ -2263,7 +2304,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 
 	errResp := func(err error) {
 		errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(),
-			w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+			w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
 		encodedErrorResponse := encodeResponse(errorResponse)
 		healthInfo.Error = string(encodedErrorResponse)
 		logger.LogIf(ctx, enc.Encode(healthInfo))
@@ -2286,7 +2327,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	defer nsLock.Unlock(lkctx.Cancel)
+	defer nsLock.Unlock(lkctx)
 	healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline)
 	defer healthCancel()
 
@@ -2342,66 +2383,6 @@ func getTLSInfo() madmin.TLSInfo {
 	return tlsInfo
 }
 
-// BandwidthMonitorHandler - GET /minio/admin/v3/bandwidth
-// ----------
-// Get bandwidth consumption information
-func (a adminAPIHandlers) BandwidthMonitorHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "BandwidthMonitor")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.BandwidthMonitorAction, "")
-	if adminAPIErr != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
-		return
-	}
-
-	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
-
-	setEventStreamHeaders(w)
-	reportCh := make(chan madmin.BucketBandwidthReport)
-	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
-	defer keepAliveTicker.Stop()
-	bucketsRequestedString := r.Form.Get("buckets")
-	bucketsRequested := strings.Split(bucketsRequestedString, ",")
-	go func() {
-		defer close(reportCh)
-		for {
-			select {
-			case <-ctx.Done():
-				return
-			case reportCh <- globalNotificationSys.GetBandwidthReports(ctx, bucketsRequested...):
-				time.Sleep(time.Duration(rnd.Float64() * float64(2*time.Second)))
-			}
-		}
-	}()
-
-	enc := json.NewEncoder(w)
-	for {
-		select {
-		case report, ok := <-reportCh:
-			if !ok {
-				return
-			}
-			if err := enc.Encode(report); err != nil {
-				return
-			}
-			if len(reportCh) == 0 {
-				// Flush if nothing is queued
-				w.(http.Flusher).Flush()
-			}
-		case <-keepAliveTicker.C:
-			if _, err := w.Write([]byte(" ")); err != nil {
-				return
-			}
-			w.(http.Flusher).Flush()
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
 // ServerInfoHandler - GET /minio/admin/v3/info
 // ----------
 // Get server information
@@ -2418,7 +2399,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 
 	// Marshal API response
-	jsonBytes, err := json.Marshal(getServerInfo(ctx, r))
+	jsonBytes, err := json.Marshal(getServerInfo(ctx, true, r))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -2702,7 +2683,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		stream := estream.NewWriter(w)
 		defer stream.Close()
 
-		clusterKey, err := bytesToPublicKey(subnetAdminPublicKey)
+		clusterKey, err := bytesToPublicKey(getSubnetAdminPublicKey())
 		if err != nil {
 			logger.LogIf(ctx, stream.AddError(err.Error()))
 			return
@@ -2837,6 +2818,13 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes()))
 }
 
+func getSubnetAdminPublicKey() []byte {
+	if globalIsCICD {
+		return subnetAdminPublicKeyDev
+	}
+	return subnetAdminPublicKey
+}
+
 func createHostAnonymizerForFSMode() map[string]string {
 	hostAnonymizer := map[string]string{
 		globalLocalNodeName: "server1",
@@ -2914,10 +2902,16 @@ func createHostAnonymizer() map[string]string {
 	}
 
 	hostAnonymizer := map[string]string{}
+	hosts := set.NewStringSet()
+	srvrIdx := 0
 
 	for poolIdx, pool := range globalEndpoints {
-		for srvrIdx, endpoint := range pool.Endpoints {
-			anonymizeHost(hostAnonymizer, endpoint, poolIdx+1, srvrIdx+1)
+		for _, endpoint := range pool.Endpoints {
+			if !hosts.Contains(endpoint.Host) {
+				hosts.Add(endpoint.Host)
+				srvrIdx++
+			}
+			anonymizeHost(hostAnonymizer, endpoint, poolIdx+1, srvrIdx)
 		}
 	}
 	return hostAnonymizer

cmd/admin-handlers_test.go

@@ -29,9 +29,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/mux"
 )
 
 // adminErasureTestBed - encapsulates subsystems that need to be setup for

cmd/admin-heal-ops.go

@@ -176,11 +176,12 @@ func (ahs *allHealState) getHealLocalDiskEndpoints() Endpoints {
 	return endpoints
 }
 
-func (ahs *allHealState) markDiskForHealing(ep Endpoint) {
+// Set, in the memory, the state of the disk as currently healing or not
+func (ahs *allHealState) setDiskHealingStatus(ep Endpoint, healing bool) {
 	ahs.Lock()
 	defer ahs.Unlock()
 
-	ahs.healLocalDisks[ep] = true
+	ahs.healLocalDisks[ep] = healing
 }
 
 func (ahs *allHealState) pushHealLocalDisks(healLocalDisks ...Endpoint) {
@@ -396,6 +397,7 @@ type healSource struct {
 	bucket    string
 	object    string
 	versionID string
+	noWait    bool             // a non blocking call, if task queue is full return right away.
 	opts      *madmin.HealOpts // optional heal option overrides default setting
 }
 
@@ -405,9 +407,6 @@ type healSequence struct {
 	// bucket, and object on which heal seq. was initiated
 	bucket, object string
 
-	// A channel of entities with heal result
-	respCh chan healResult
-
 	// Report healing progress
 	reportProgress bool
 
@@ -470,7 +469,6 @@ func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string,
 	clientToken := mustGetUUID()
 
 	return &healSequence{
-		respCh:         make(chan healResult),
 		bucket:         bucket,
 		object:         objPrefix,
 		reportProgress: true,
@@ -699,7 +697,6 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		object:    source.object,
 		versionID: source.versionID,
 		opts:      h.settings,
-		respCh:    h.respCh,
 	}
 	if source.opts != nil {
 		task.opts = *source.opts
@@ -712,6 +709,24 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 	h.lastHealActivity = UTCNow()
 	h.mutex.Unlock()
 
+	if source.noWait {
+		select {
+		case globalBackgroundHealRoutine.tasks <- task:
+			if serverDebugLog {
+				logger.Info("Task in the queue: %#v", task)
+			}
+		default:
+			// task queue is full, no more workers, we shall move on and heal later.
+			return nil
+		}
+		// Don't wait for result
+		return nil
+	}
+
+	// respCh must be set to wait for result.
+	// We make it size 1, so a result can always be written
+	// even if we aren't listening.
+	task.respCh = make(chan healResult, 1)
 	select {
 	case globalBackgroundHealRoutine.tasks <- task:
 		if serverDebugLog {
@@ -721,8 +736,9 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		return nil
 	}
 
+	// task queued, now wait for the response.
 	select {
-	case res := <-h.respCh:
+	case res := <-task.respCh:
 		if !h.reportProgress {
 			if errors.Is(res.err, errSkipFile) { // this is only sent usually by nopHeal
 				return nil

cmd/admin-router.go

@@ -20,11 +20,11 @@ package cmd
 import (
 	"net/http"
 
-	"github.com/gorilla/mux"
 	"github.com/klauspost/compress/gzhttp"
 	"github.com/klauspost/compress/gzip"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 )
 
 const (
@@ -142,12 +142,18 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListServiceAccounts)))
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteServiceAccount))).Queries("accessKey", "{accessKey:.*}")
 
+		// STS accounts ops
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/temporary-account-info").HandlerFunc(gz(httpTraceHdrs(adminAPI.TemporaryAccountInfo))).Queries("accessKey", "{accessKey:.*}")
+
 		// Info policy IAM latest
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(gz(httpTraceHdrs(adminAPI.InfoCannedPolicy))).Queries("name", "{name:.*}")
 		// List policies latest
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListBucketPolicies))).Queries("bucket", "{bucket:.*}")
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListCannedPolicies)))
 
+		// Builtin IAM policy associations
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/builtin/policy-entities").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListPolicyMappingEntities)))
+
 		// Remove policy IAM
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveCannedPolicy))).Queries("name", "{name:.*}")
 
@@ -156,6 +162,12 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			HandlerFunc(gz(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup))).
 			Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
 
+		// Attach policies to user or group
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/attach").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachPolicyBuiltin)))
+
+		// Detach policies from user or group
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/detach").HandlerFunc(gz(httpTraceHdrs(adminAPI.DetachPolicyBuiltin)))
+
 		// Remove user IAM
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveUser))).Queries("accessKey", "{accessKey:.*}")
 
@@ -192,7 +204,7 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 		// LDAP IAM operations
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListLDAPPolicyMappingEntities)))
-
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachDetachPolicyLDAP)))
 		// -- END IAM APIs --
 
 		// GetBucketQuotaConfig
@@ -291,8 +303,6 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		// -- Health API --
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
 			HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
-			HandlerFunc(gz(httpTraceHdrs(adminAPI.BandwidthMonitorHandler)))
 	}
 
 	// If none of the routes match add default error handler routes

cmd/admin-server-info.go

@@ -28,6 +28,7 @@ import (
 
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -118,7 +119,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		config.EnvRootUser:          {},
 		config.EnvRootPassword:      {},
 		config.EnvMinIOSubnetAPIKey: {},
-		config.EnvKMSSecretKey:      {},
+		kms.EnvKMSSecretKey:         {},
 	}
 	for _, v := range os.Environ() {
 		if !strings.HasPrefix(v, "MINIO") && !strings.HasPrefix(v, "_MINIO") {

cmd/api-errors.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -132,6 +132,7 @@ const (
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
 	ErrReplicationDenyEditError
+	ErrRemoteTargetDenyEditError
 	ErrReplicationNoExistingObjects
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
@@ -164,7 +165,6 @@ const (
 	ErrMalformedDate
 	ErrMalformedPresignedDate
 	ErrMalformedCredentialDate
-	ErrMalformedCredentialRegion
 	ErrMalformedExpires
 	ErrNegativeExpires
 	ErrAuthHeaderEmpty
@@ -177,7 +177,6 @@ const (
 	ErrBucketAlreadyOwnedByYou
 	ErrInvalidDuration
 	ErrBucketAlreadyExists
-	ErrTooManyBuckets
 	ErrMetadataTooLarge
 	ErrUnsupportedMetadata
 	ErrMaximumExpires
@@ -196,6 +195,8 @@ const (
 	ErrBucketTaggingNotFound
 	ErrObjectLockInvalidHeaders
 	ErrInvalidTagDirective
+	ErrPolicyAlreadyAttached
+	ErrPolicyNotAttached
 	// Add new error codes here.
 
 	// SSE-S3/SSE-KMS related API errors
@@ -207,6 +208,8 @@ const (
 	ErrSSEMultipartEncrypted
 	ErrSSEEncryptedObject
 	ErrInvalidEncryptionParameters
+	ErrInvalidEncryptionParametersSSEC
+
 	ErrInvalidSSECustomerAlgorithm
 	ErrInvalidSSECustomerKey
 	ErrMissingSSECustomerKey
@@ -216,6 +219,7 @@ const (
 	ErrIncompatibleEncryptionMethod
 	ErrKMSNotConfigured
 	ErrKMSKeyNotFoundException
+	ErrKMSDefaultKeyAlreadyConfigured
 
 	ErrNoAccessKey
 	ErrInvalidToken
@@ -239,8 +243,6 @@ const (
 	// Add new extended error codes here.
 
 	// MinIO extended errors.
-	ErrReadQuorum
-	ErrWriteQuorum
 	ErrStorageFull
 	ErrRequestBodyParse
 	ErrObjectExistsAsDirectory
@@ -266,6 +268,7 @@ const (
 	ErrAdminGroupNotEmpty
 	ErrAdminNoSuchJob
 	ErrAdminNoSuchPolicy
+	ErrAdminPolicyChangeAlreadyApplied
 	ErrAdminInvalidArgument
 	ErrAdminInvalidAccessKey
 	ErrAdminInvalidSecretKey
@@ -406,6 +409,8 @@ const (
 	ErrPostPolicyConditionInvalidFormat
 
 	ErrInvalidChecksum
+
+	apiErrCodeEnd // This is used only for the testing code
 )
 
 type errorCodeMap map[APIErrorCode]APIError
@@ -695,11 +700,6 @@ var errorCodes = errorCodeMap{
 		Description:    "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrTooManyBuckets: {
-		Code:           "TooManyBuckets",
-		Description:    "You have attempted to create more buckets than allowed",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrBucketNotEmpty: {
 		Code:           "BucketNotEmpty",
 		Description:    "The bucket you tried to delete is not empty",
@@ -920,6 +920,11 @@ var errorCodes = errorCodeMap{
 		Description:    "No matching ExistingsObjects rule enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrRemoteTargetDenyEditError: {
+		Code:           "XMinioAdminRemoteTargetDenyEdit",
+		Description:    "Cannot alter remote target endpoint since this server is in a cluster replication setup. use `mc admin replicate update`",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrReplicationDenyEditError: {
 		Code:           "XMinioReplicationDenyEdit",
 		Description:    "Cannot alter local replication config since this server is in a cluster replication setup",
@@ -1117,6 +1122,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The encryption parameters are not applicable to this object.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidEncryptionParametersSSEC: {
+		Code:           "InvalidRequest",
+		Description:    "SSE-C encryption parameters are not supported on replicated bucket.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidSSECustomerAlgorithm: {
 		Code:           "InvalidArgument",
 		Description:    "Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm.",
@@ -1162,6 +1172,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Invalid keyId",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrKMSDefaultKeyAlreadyConfigured: {
+		Code:           "KMS.DefaultKeyAlreadyConfiguredException",
+		Description:    "A default encryption already exists and cannot be changed on KMS",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrNoAccessKey: {
 		Code:           "AccessDenied",
 		Description:    "No AWSAccessKey was presented",
@@ -1251,6 +1266,12 @@ var errorCodes = errorCodeMap{
 		Description:    "The canned policy does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrAdminPolicyChangeAlreadyApplied: {
+		Code:           "XMinioAdminPolicyChangeAlreadyApplied",
+		Description:    "The specified policy change is already in effect.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+
 	ErrAdminInvalidArgument: {
 		Code:           "XMinioAdminInvalidArgument",
 		Description:    "Invalid arguments specified.",
@@ -1381,7 +1402,7 @@ var errorCodes = errorCodeMap{
 	ErrSiteReplicationPeerResp: {
 		Code:           "XMinioSiteReplicationPeerResp",
 		Description:    "Error received when contacting a peer site",
-		HTTPStatusCode: http.StatusServiceUnavailable,
+		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrSiteReplicationBackendIssue: {
 		Code:           "XMinioSiteReplicationBackendIssue",
@@ -1499,7 +1520,7 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrBusy: {
-		Code:           "Busy",
+		Code:           "ServerBusy",
 		Description:    "The service is unavailable. Please retry.",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
@@ -1938,6 +1959,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Invalid checksum provided.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrPolicyAlreadyAttached: {
+		Code:           "XMinioPolicyAlreadyAttached",
+		Description:    "The specified policy is already attached.",
+		HTTPStatusCode: http.StatusConflict,
+	},
+	ErrPolicyNotAttached: {
+		Code:           "XMinioPolicyNotAttached",
+		Description:    "The specified policy is not found.",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	// Add your error structure here.
 }
 
@@ -1970,8 +2001,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminGroupNotEmpty
 	case errNoSuchJob:
 		apiErr = ErrAdminNoSuchJob
-	case errNoSuchPolicy:
-		apiErr = ErrAdminNoSuchPolicy
+	case errNoPolicyToAttachOrDetach:
+		apiErr = ErrAdminPolicyChangeAlreadyApplied
 	case errSignatureMismatch:
 		apiErr = ErrSignatureDoesNotMatch
 	case errInvalidRange:
@@ -1991,6 +2022,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	// SSE errors
 	case errInvalidEncryptionParameters:
 		apiErr = ErrInvalidEncryptionParameters
+	case errInvalidEncryptionParametersSSEC:
+		apiErr = ErrInvalidEncryptionParametersSSEC
 	case crypto.ErrInvalidEncryptionMethod:
 		apiErr = ErrInvalidEncryptionMethod
 	case crypto.ErrInvalidEncryptionKeyID:
@@ -2017,7 +2050,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSNotConfigured
 	case errKMSKeyNotFound:
 		apiErr = ErrKMSKeyNotFoundException
-
+	case errKMSDefaultKeyAlreadyConfigured:
+		apiErr = ErrKMSDefaultKeyAlreadyConfigured
 	case context.Canceled, context.DeadlineExceeded:
 		apiErr = ErrOperationTimedOut
 	case errDiskNotFound:
@@ -2032,6 +2066,11 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrObjectLockInvalidHeaders
 	case objectlock.ErrMalformedXML:
 		apiErr = ErrMalformedXML
+	default:
+		switch {
+		case errors.Is(err, errNoSuchPolicy):
+			apiErr = ErrAdminNoSuchPolicy
+		}
 	}
 
 	// Compression errors
@@ -2162,7 +2201,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrTransitionStorageClassNotFoundError
 	case InvalidObjectState:
 		apiErr = ErrInvalidObjectState
-
+	case PreConditionFailed:
+		apiErr = ErrPreconditionFailed
 	case BucketQuotaExceeded:
 		apiErr = ErrAdminBucketQuotaExceeded
 	case *event.ErrInvalidEventName:

cmd/api-errors_test.go

@@ -80,3 +80,19 @@ func TestAPIErrCode(t *testing.T) {
 		}
 	}
 }
+
+// Check if an API error is properly defined
+func TestAPIErrCodeDefinition(t *testing.T) {
+	for errAPI := ErrNone + 1; errAPI < apiErrCodeEnd; errAPI++ {
+		errCode, ok := errorCodes[errAPI]
+		if !ok {
+			t.Fatal(errAPI, "error code is not defined in the API error code table")
+		}
+		if errCode.Code == "" {
+			t.Fatal(errAPI, "error code has an empty XML code")
+		}
+		if errCode.HTTPStatusCode == 0 {
+			t.Fatal(errAPI, "error code has a zero HTTP status code")
+		}
+	}
+}

cmd/api-headers.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"encoding/json"
+	"encoding/xml"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -64,15 +65,31 @@ func setCommonHeaders(w http.ResponseWriter) {
 
 // Encodes the response headers into XML format.
 func encodeResponse(response interface{}) []byte {
-	var bytesBuffer bytes.Buffer
-	bytesBuffer.WriteString(xxml.Header)
-	buf, err := xxml.Marshal(response)
-	if err != nil {
+	var buf bytes.Buffer
+	buf.WriteString(xml.Header)
+	if err := xml.NewEncoder(&buf).Encode(response); err != nil {
 		logger.LogIf(GlobalContext, err)
 		return nil
 	}
-	bytesBuffer.Write(buf)
-	return bytesBuffer.Bytes()
+	return buf.Bytes()
+}
+
+// Use this encodeResponseList() to support control characters
+// this function must be used by only ListObjects() for objects
+// with control characters, this is a specialized extension
+// to support AWS S3 compatible behavior.
+//
+// Do not use this function for anything other than ListObjects()
+// variants, please open a github discussion if you wish to use
+// this in other places.
+func encodeResponseList(response interface{}) []byte {
+	var buf bytes.Buffer
+	buf.WriteString(xxml.Header)
+	if err := xxml.NewEncoder(&buf).Encode(response); err != nil {
+		logger.LogIf(GlobalContext, err)
+		return nil
+	}
+	return buf.Bytes()
 }
 
 // Encodes the response headers into JSON format.

cmd/api-response.go

@@ -29,6 +29,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/hash"
@@ -38,12 +39,10 @@ import (
 )
 
 const (
-	// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z
-	iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with nanosecond precision.
-	maxObjectList     = 1000                       // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
-	maxDeleteList     = 1000                       // Limit number of objects deleted in a delete call.
-	maxUploadsList    = 10000                      // Limit number of uploads in a listUploadsResponse.
-	maxPartsList      = 10000                      // Limit number of parts in a listPartsResponse.
+	maxObjectList  = 1000  // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse.
+	maxDeleteList  = 1000  // Limit number of objects deleted in a delete call.
+	maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse.
+	maxPartsList   = 10000 // Limit number of parts in a listPartsResponse.
 )
 
 // LocationResponse - format for location response.
@@ -315,12 +314,12 @@ func (s *Metadata) Set(k, v string) {
 }
 
 type xmlKeyEntry struct {
-	XMLName xml.Name
+	XMLName xxml.Name
 	Value   string `xml:",chardata"`
 }
 
 // MarshalXML - StringMap marshals into XML.
-func (s *Metadata) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
+func (s *Metadata) MarshalXML(e *xxml.Encoder, start xxml.StartElement) error {
 	if s == nil {
 		return nil
 	}
@@ -335,7 +334,7 @@ func (s *Metadata) MarshalXML(e *xml.Encoder, start xml.StartElement) error {
 
 	for _, item := range s.Items {
 		if err := e.Encode(xmlKeyEntry{
-			XMLName: xml.Name{Local: item.Key},
+			XMLName: xxml.Name{Local: item.Key},
 			Value:   item.Value,
 		}); err != nil {
 			return err
@@ -482,7 +481,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 	for _, bucket := range buckets {
 		listbuckets = append(listbuckets, Bucket{
 			Name:         bucket.Name,
-			CreationDate: bucket.Created.UTC().Format(iso8601TimeFormat),
+			CreationDate: amztime.ISO8601Format(bucket.Created.UTC()),
 		})
 	}
 
@@ -508,7 +507,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 		}
 		content := ObjectVersion{}
 		content.Key = s3EncodeName(object.Name, encodingType)
-		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
+		content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
 		}
@@ -566,7 +565,7 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 			continue
 		}
 		content.Key = s3EncodeName(object.Name, encodingType)
-		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
+		content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
 		}
@@ -615,7 +614,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 			continue
 		}
 		content.Key = s3EncodeName(object.Name, encodingType)
-		content.LastModified = object.ModTime.UTC().Format(iso8601TimeFormat)
+		content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
 		if object.ETag != "" {
 			content.ETag = "\"" + object.ETag + "\""
 		}
@@ -678,7 +677,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
 	return CopyObjectResponse{
 		ETag:         "\"" + etag + "\"",
-		LastModified: lastModified.UTC().Format(iso8601TimeFormat),
+		LastModified: amztime.ISO8601Format(lastModified.UTC()),
 	}
 }
 
@@ -686,7 +685,7 @@ func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectR
 func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse {
 	return CopyObjectPartResponse{
 		ETag:         "\"" + etag + "\"",
-		LastModified: lastModified.UTC().Format(iso8601TimeFormat),
+		LastModified: amztime.ISO8601Format(lastModified.UTC()),
 	}
 }
 
@@ -746,7 +745,7 @@ func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) Lis
 		newPart.PartNumber = part.PartNumber
 		newPart.ETag = "\"" + part.ETag + "\""
 		newPart.Size = part.Size
-		newPart.LastModified = part.LastModified.UTC().Format(iso8601TimeFormat)
+		newPart.LastModified = amztime.ISO8601Format(part.LastModified.UTC())
 		newPart.ChecksumCRC32 = part.ChecksumCRC32
 		newPart.ChecksumCRC32C = part.ChecksumCRC32C
 		newPart.ChecksumSHA1 = part.ChecksumSHA1
@@ -780,7 +779,7 @@ func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMult
 		newUpload := Upload{}
 		newUpload.UploadID = upload.UploadID
 		newUpload.Key = s3EncodeName(upload.Object, encodingType)
-		newUpload.Initiated = upload.Initiated.UTC().Format(iso8601TimeFormat)
+		newUpload.Initiated = amztime.ISO8601Format(upload.Initiated.UTC())
 		listMultipartUploadsResponse.Uploads[index] = newUpload
 	}
 	return listMultipartUploadsResponse
@@ -876,7 +875,7 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 
 	// Generate error response.
 	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path,
-		w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+		w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
 	encodedErrorResponse := encodeResponse(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML)
 }
@@ -894,7 +893,7 @@ func writeErrorResponseString(ctx context.Context, w http.ResponseWriter, err AP
 // useful for admin APIs.
 func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
 	// Generate error response.
-	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), globalDeploymentID)
+	errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID))
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
 }

cmd/api-router.go

@@ -22,11 +22,11 @@ import (
 	"net"
 	"net/http"
 
-	"github.com/gorilla/mux"
 	"github.com/klauspost/compress/gzhttp"
 	"github.com/minio/console/restapi"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/wildcard"
 	"github.com/rs/cors"
 )
@@ -516,7 +516,11 @@ func corsHandler(handler http.Handler) http.Handler {
 
 	return cors.New(cors.Options{
 		AllowOriginFunc: func(origin string) bool {
-			for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
+			allowedOrigins := globalAPIConfig.getCorsAllowOrigins()
+			if len(allowedOrigins) == 0 {
+				allowedOrigins = []string{"*"}
+			}
+			for _, allowedOrigin := range allowedOrigins {
 				if wildcard.MatchSimple(allowedOrigin, origin) {
 					return true
 				}

cmd/auth-handler.go

@@ -135,7 +135,7 @@ func getRequestAuthType(r *http.Request) authType {
 	return authTypeUnknown
 }
 
-func validateAdminSignature(ctx context.Context, r *http.Request, region string) (auth.Credentials, map[string]interface{}, bool, APIErrorCode) {
+func validateAdminSignature(ctx context.Context, r *http.Request, region string) (auth.Credentials, bool, APIErrorCode) {
 	var cred auth.Credentials
 	var owner bool
 	s3Err := ErrAccessDenied
@@ -144,27 +144,24 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 		// We only support admin credentials to access admin APIs.
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {
-			return cred, nil, owner, s3Err
+			return cred, owner, s3Err
 		}
 
 		// we only support V4 (no presign) with auth body
 		s3Err = isReqAuthenticated(ctx, r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
-		reqInfo := (&logger.ReqInfo{}).AppendTags("requestHeaders", dumpRequest(r))
-		ctx := logger.SetReqInfo(ctx, reqInfo)
-		logger.LogIf(ctx, errors.New(getAPIError(s3Err).Description), logger.Application)
-		return cred, nil, owner, s3Err
+		return cred, owner, s3Err
 	}
 
-	return cred, cred.Claims, owner, ErrNone
+	return cred, owner, ErrNone
 }
 
 // checkAdminRequestAuth checks for authentication and authorization for the incoming
 // request. It only accepts V2 and V4 requests. Presigned, JWT and anonymous requests
 // are automatically rejected.
 func checkAdminRequestAuth(ctx context.Context, r *http.Request, action iampolicy.AdminAction, region string) (auth.Credentials, APIErrorCode) {
-	cred, claims, owner, s3Err := validateAdminSignature(ctx, r, region)
+	cred, owner, s3Err := validateAdminSignature(ctx, r, region)
 	if s3Err != ErrNone {
 		return cred, s3Err
 	}
@@ -172,9 +169,9 @@ func checkAdminRequestAuth(ctx context.Context, r *http.Request, action iampolic
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.Action(action),
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
-		Claims:          claims,
+		Claims:          cred.Claims,
 	}) {
 		// Request is allowed return the appropriate access key.
 		return cred, ErrNone
@@ -256,7 +253,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 		return nil, ErrNoAccessKey
 	}
 
-	if token == "" && cred.IsTemp() {
+	if token == "" && cred.IsTemp() && !cred.IsServiceAccount() {
 		// Temporary credentials should always have x-amz-security-token
 		return nil, ErrInvalidToken
 	}
@@ -266,7 +263,7 @@ func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]in
 		return nil, ErrInvalidToken
 	}
 
-	if cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
+	if !cred.IsServiceAccount() && cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 {
 		// validate token for temporary credentials only.
 		return nil, ErrInvalidToken
 	}
@@ -303,6 +300,17 @@ func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Ac
 	return s3Err
 }
 
+// checkRequestAuthTypeWithVID is similar to checkRequestAuthType
+// passes versionID additionally.
+func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, versionID string) (s3Err APIErrorCode) {
+	logger.GetReqInfo(ctx).BucketName = bucketName
+	logger.GetReqInfo(ctx).ObjectName = objectName
+	logger.GetReqInfo(ctx).VersionID = versionID
+
+	_, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action)
+	return s3Err
+}
+
 func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
 	if logger.GetReqInfo(ctx) == nil {
 		logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.Minio)
@@ -374,14 +382,16 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 	region := reqInfo.Region
 	bucket := reqInfo.BucketName
 	object := reqInfo.ObjectName
+	versionID := reqInfo.VersionID
 
 	if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" {
 		// Anonymous checks are not meant for ListAllBuckets action
 		if globalPolicySys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
 			Action:          action,
 			BucketName:      bucket,
-			ConditionValues: getConditionValues(r, region, "", nil),
+			ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials),
 			IsOwner:         false,
 			ObjectName:      object,
 		}) {
@@ -394,9 +404,10 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 			// verify as a fallback.
 			if globalPolicySys.IsAllowed(policy.Args{
 				AccountName:     cred.AccessKey,
+				Groups:          cred.Groups,
 				Action:          policy.ListBucketAction,
 				BucketName:      bucket,
-				ConditionValues: getConditionValues(r, region, "", nil),
+				ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials),
 				IsOwner:         false,
 				ObjectName:      object,
 			}) {
@@ -407,13 +418,27 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 
 		return ErrAccessDenied
 	}
-
+	if action == policy.DeleteObjectAction && versionID != "" {
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          iampolicy.Action(policy.DeleteObjectVersionAction),
+			BucketName:      bucket,
+			ConditionValues: getConditionValues(r, "", cred),
+			ObjectName:      object,
+			IsOwner:         owner,
+			Claims:          cred.Claims,
+			DenyOnly:        true,
+		}) { // Request is not allowed if Deny action on DeleteObjectVersionAction
+			return ErrAccessDenied
+		}
+	}
 	if globalIAMSys.IsAllowed(iampolicy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.Action(action),
 		BucketName:      bucket,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		ObjectName:      object,
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -430,7 +455,7 @@ func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action
 			Groups:          cred.Groups,
 			Action:          iampolicy.ListBucketAction,
 			BucketName:      bucket,
-			ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+			ConditionValues: getConditionValues(r, "", cred),
 			ObjectName:      object,
 			IsOwner:         owner,
 			Claims:          cred.Claims,
@@ -625,7 +650,7 @@ func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate t
 		return ErrAccessDenied
 	}
 
-	conditions := getConditionValues(r, "", cred.AccessKey, cred.Claims)
+	conditions := getConditionValues(r, "", cred)
 	conditions["object-lock-mode"] = []string{string(retMode)}
 	conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)}
 	if retDays > 0 {
@@ -699,7 +724,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 			Groups:          cred.Groups,
 			Action:          policy.Action(action),
 			BucketName:      bucketName,
-			ConditionValues: getConditionValues(r, "", "", nil),
+			ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 			IsOwner:         false,
 			ObjectName:      objectName,
 		}) {
@@ -713,7 +738,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 		Groups:          cred.Groups,
 		Action:          action,
 		BucketName:      bucketName,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		ObjectName:      objectName,
 		IsOwner:         owner,
 		Claims:          cred.Claims,

cmd/auth-handler_test.go

@@ -492,7 +492,7 @@ func TestValidateAdminSignature(t *testing.T) {
 		if err := signRequestV4(req, testCase.AccessKey, testCase.SecretKey); err != nil {
 			t.Fatalf("Unable to inititalized new signed http request %s", err)
 		}
-		_, _, _, s3Error := validateAdminSignature(ctx, req, globalMinioDefaultRegion)
+		_, _, s3Error := validateAdminSignature(ctx, req, globalMinioDefaultRegion)
 		if s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i+1, testCase.ErrCode, s3Error)
 		}

cmd/background-heal-ops.go

@@ -19,9 +19,14 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"runtime"
+	"strconv"
+	"time"
 
 	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/env"
 )
 
 // healTask represents what to heal along with options
@@ -56,13 +61,43 @@ func activeListeners() int {
 	return int(globalHTTPListen.Subscribers()) + int(globalTrace.Subscribers())
 }
 
-func waitForLowHTTPReq() {
-	var currentIO func() int
-	if httpServer := newHTTPServerFn(); httpServer != nil {
-		currentIO = httpServer.GetRequestCount
+func waitForLowIO(maxIO int, maxWait time.Duration, currentIO func() int) {
+	// No need to wait run at full speed.
+	if maxIO <= 0 {
+		return
 	}
 
-	globalHealConfig.Wait(currentIO, activeListeners)
+	const waitTick = 100 * time.Millisecond
+
+	tmpMaxWait := maxWait
+
+	for currentIO() >= maxIO {
+		if tmpMaxWait > 0 {
+			if tmpMaxWait < waitTick {
+				time.Sleep(tmpMaxWait)
+			} else {
+				time.Sleep(waitTick)
+			}
+			tmpMaxWait -= waitTick
+		}
+		if tmpMaxWait <= 0 {
+			return
+		}
+	}
+}
+
+func currentHTTPIO() int {
+	httpServer := newHTTPServerFn()
+	if httpServer == nil {
+		return 0
+	}
+
+	return httpServer.GetRequestCount() - activeListeners()
+}
+
+func waitForLowHTTPReq() {
+	maxIO, maxWait, _ := globalHealConfig.Clone()
+	waitForLowIO(maxIO, maxWait, currentHTTPIO)
 }
 
 func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) {
@@ -88,8 +123,7 @@ func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
 			var err error
 			switch task.bucket {
 			case nopHeal:
-				task.respCh <- healResult{err: errSkipFile}
-				continue
+				err = errSkipFile
 			case SlashSeparator:
 				res, err = healDiskFormat(ctx, objAPI, task.opts)
 			default:
@@ -100,7 +134,10 @@ func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
 				}
 			}
 
-			task.respCh <- healResult{result: res, err: err}
+			if task.respCh != nil {
+				task.respCh <- healResult{result: res, err: err}
+			}
+
 		case <-ctx.Done():
 			return
 		}
@@ -109,9 +146,19 @@ func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer) {
 
 func newHealRoutine() *healRoutine {
 	workers := runtime.GOMAXPROCS(0) / 2
+
+	if envHealWorkers := env.Get("_MINIO_HEAL_WORKERS", ""); envHealWorkers != "" {
+		if numHealers, err := strconv.Atoi(envHealWorkers); err != nil {
+			logger.LogIf(context.Background(), fmt.Errorf("invalid _MINIO_HEAL_WORKERS value: %w", err))
+		} else {
+			workers = numHealers
+		}
+	}
+
 	if workers == 0 {
 		workers = 4
 	}
+
 	return &healRoutine{
 		tasks:   make(chan healTask),
 		workers: workers,

cmd/background-newdisks-heal-ops.go

@@ -79,6 +79,10 @@ type healingTracker struct {
 
 	// Filled during heal.
 	HealedBuckets []string
+
+	// ID of the current healing operation
+	HealID string
+
 	// Add future tracking capabilities
 	// Be sure that they are included in toHealingDisk
 }
@@ -112,11 +116,12 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
 }
 
 // newHealingTracker will create a new healing tracker for the disk.
-func newHealingTracker(disk StorageAPI) *healingTracker {
+func newHealingTracker(disk StorageAPI, healID string) *healingTracker {
 	diskID, _ := disk.GetDiskID()
 	h := healingTracker{
 		disk:     disk,
 		ID:       diskID,
+		HealID:   healID,
 		Path:     disk.String(),
 		Endpoint: disk.Endpoint().String(),
 		Started:  time.Now().UTC(),
@@ -227,6 +232,7 @@ func (h *healingTracker) printTo(writer io.Writer) {
 func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
 	return madmin.HealingDisk{
 		ID:                h.ID,
+		HealID:            h.HealID,
 		Endpoint:          h.Endpoint,
 		PoolIndex:         h.PoolIndex,
 		SetIndex:          h.SetIndex,
@@ -286,8 +292,6 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 
 func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
-	logger.Info(fmt.Sprintf("Proceeding to heal '%s' - 'mc admin heal alias/ --verbose' to check the status.", endpoint))
-
 	disk, format, err := connectEndpoint(endpoint)
 	if err != nil {
 		return fmt.Errorf("Error: %w, %s", err, endpoint)
@@ -295,7 +299,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	defer disk.Close()
 	poolIdx := globalEndpoints.GetLocalPoolIdx(disk.Endpoint())
 	if poolIdx < 0 {
-		return fmt.Errorf("unexpected pool index (%d) found in %s", poolIdx, disk.Endpoint())
+		return fmt.Errorf("unexpected pool index (%d) found for %s", poolIdx, disk.Endpoint())
 	}
 
 	// Calculate the set index where the current endpoint belongs
@@ -306,20 +310,36 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		return err
 	}
 	if setIdx < 0 {
-		return fmt.Errorf("unexpected set index (%d) found in %s", setIdx, disk.Endpoint())
+		return fmt.Errorf("unexpected set index (%d) found for  %s", setIdx, disk.Endpoint())
 	}
 
 	// Prevent parallel erasure set healing
 	locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx))
 	lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout)
 	if err != nil {
-		return err
+		return fmt.Errorf("Healing of drive '%v' on %s pool, belonging to %s erasure set already in progress: %w",
+			disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1), err)
 	}
 	ctx = lkctx.Context()
-	defer locker.Unlock(lkctx.Cancel)
+	defer locker.Unlock(lkctx)
+
+	// Load healing tracker in this disk
+	tracker, err := loadHealingTracker(ctx, disk)
+	if err != nil {
+		// A healing tracker may be deleted if another disk in the
+		// same erasure set with same healing-id successfully finished
+		// healing.
+		if errors.Is(err, errFileNotFound) {
+			return nil
+		}
+		logger.LogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
+		tracker = newHealingTracker(disk, mustGetUUID())
+	}
+
+	logger.Info(fmt.Sprintf("Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint))
 
 	buckets, _ := z.ListBuckets(ctx, BucketOptions{})
-	// Buckets data are dispersed in multiple zones/sets, make
+	// Buckets data are dispersed in multiple pools/sets, make
 	// sure to heal all bucket metadata configuration.
 	buckets = append(buckets, BucketInfo{
 		Name: pathJoin(minioMetaBucket, minioConfigPrefix),
@@ -337,16 +357,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	})
 
 	if serverDebugLog {
-		logger.Info("Healing drive '%v' on %s pool", disk, humanize.Ordinal(poolIdx+1))
-	}
-
-	// Load healing tracker in this disk
-	tracker, err := loadHealingTracker(ctx, disk)
-	if err != nil {
-		// So someone changed the drives underneath, healing tracker missing.
-		logger.LogIf(ctx, fmt.Errorf("Healing tracker missing on '%s', drive was swapped again on %s pool: %w",
-			disk, humanize.Ordinal(poolIdx+1), err))
-		tracker = newHealingTracker(disk)
+		logger.Info("Healing drive '%v' on %s pool, belonging to %s erasure set", disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1))
 	}
 
 	// Load bucket totals
@@ -369,9 +380,13 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	if tracker.ItemsFailed > 0 {
-		logger.Info("Healing drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		logger.Info("Healing of drive '%s' failed (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
 	} else {
-		logger.Info("Healing drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+		logger.Info("Healing of drive '%s' complete (healed: %d, failed: %d).", disk, tracker.ItemsHealed, tracker.ItemsFailed)
+	}
+
+	if len(tracker.QueuedBuckets) > 0 {
+		return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets)
 	}
 
 	if serverDebugLog {
@@ -379,7 +394,24 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 		logger.Info("\n")
 	}
 
-	logger.LogIf(ctx, tracker.delete(ctx))
+	if tracker.HealID == "" { // HealID was empty only before Feb 2023
+		logger.LogIf(ctx, tracker.delete(ctx))
+		return nil
+	}
+
+	// Remove .healing.bin from all disks with similar heal-id
+	for _, disk := range z.serverPools[poolIdx].sets[setIdx].getDisks() {
+		t, err := loadHealingTracker(ctx, disk)
+		if err != nil {
+			if !errors.Is(err, errFileNotFound) {
+				logger.LogIf(ctx, err)
+			}
+			continue
+		}
+		if t.HealID == tracker.HealID {
+			t.delete(ctx)
+		}
+	}
 
 	return nil
 }
@@ -415,10 +447,12 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 
 			for _, disk := range healDisks {
 				go func(disk Endpoint) {
-					globalBackgroundHealState.markDiskForHealing(disk)
-					err := healFreshDisk(ctx, z, disk)
-					if err != nil {
-						printEndpointError(disk, err, false)
+					globalBackgroundHealState.setDiskHealingStatus(disk, true)
+					if err := healFreshDisk(ctx, z, disk); err != nil {
+						globalBackgroundHealState.setDiskHealingStatus(disk, false)
+						if !errors.Is(err, context.Canceled) {
+							printEndpointError(disk, err, false)
+						}
 						return
 					}
 					// Only upon success pop the healed disk.

cmd/background-newdisks-heal-ops_gen.go

@@ -182,6 +182,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
+		case "HealID":
+			z.HealID, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "HealID")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -195,9 +201,9 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 22
+	// map header, size 23
 	// write "ID"
-	err = en.Append(0xde, 0x0, 0x16, 0xa2, 0x49, 0x44)
+	err = en.Append(0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -430,15 +436,25 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
+	// write "HealID"
+	err = en.Append(0xa6, 0x48, 0x65, 0x61, 0x6c, 0x49, 0x44)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.HealID)
+	if err != nil {
+		err = msgp.WrapError(err, "HealID")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 22
+	// map header, size 23
 	// string "ID"
-	o = append(o, 0xde, 0x0, 0x16, 0xa2, 0x49, 0x44)
+	o = append(o, 0xde, 0x0, 0x17, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "PoolIndex"
 	o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
@@ -509,6 +525,9 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	for za0002 := range z.HealedBuckets {
 		o = msgp.AppendString(o, z.HealedBuckets[za0002])
 	}
+	// string "HealID"
+	o = append(o, 0xa6, 0x48, 0x65, 0x61, 0x6c, 0x49, 0x44)
+	o = msgp.AppendString(o, z.HealID)
 	return
 }
 
@@ -688,6 +707,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
+		case "HealID":
+			z.HealID, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "HealID")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -710,5 +735,6 @@ func (z *healingTracker) Msgsize() (s int) {
 	for za0002 := range z.HealedBuckets {
 		s += msgp.StringPrefixSize + len(z.HealedBuckets[za0002])
 	}
+	s += 7 + msgp.StringPrefixSize + len(z.HealID)
 	return
 }

cmd/batch-handlers.go

@@ -28,6 +28,8 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"runtime"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -41,7 +43,9 @@ import (
 	"github.com/minio/minio/internal/auth"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/workers"
 	"github.com/minio/pkg/console"
+	"github.com/minio/pkg/env"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	"github.com/minio/pkg/wildcard"
 	"gopkg.in/yaml.v2"
@@ -356,6 +360,8 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
 
 // batchJobInfo current batch replication information
 type batchJobInfo struct {
+	mu sync.RWMutex `json:"-" msg:"-"`
+
 	Version       int       `json:"-" msg:"v"`
 	JobID         string    `json:"jobID" msg:"jid"`
 	JobType       string    `json:"jobType" msg:"jt"`
@@ -425,6 +431,9 @@ func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobR
 		return fmt.Errorf("batchRepl: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
 	}
 
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+
 	// OK, parse data.
 	if _, err = ri.UnmarshalMsg(data[4:]); err != nil {
 		return err
@@ -439,8 +448,11 @@ func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobR
 	return nil
 }
 
-func (ri batchJobInfo) clone() batchJobInfo {
-	return batchJobInfo{
+func (ri *batchJobInfo) clone() *batchJobInfo {
+	ri.mu.RLock()
+	defer ri.mu.RUnlock()
+
+	return &batchJobInfo{
 		Version:          ri.Version,
 		JobID:            ri.JobID,
 		JobType:          ri.JobType,
@@ -458,21 +470,6 @@ func (ri batchJobInfo) clone() batchJobInfo {
 	}
 }
 
-func (ri batchJobInfo) save(ctx context.Context, api ObjectLayer, jobLocation string) error {
-	data := make([]byte, 4, ri.Msgsize()+4)
-
-	// Initialize the header.
-	binary.LittleEndian.PutUint16(data[0:2], batchReplFormat)
-	binary.LittleEndian.PutUint16(data[2:4], batchReplVersion)
-
-	buf, err := ri.MarshalMsg(data)
-	if err != nil {
-		return err
-	}
-
-	return saveConfig(ctx, api, pathJoin(jobLocation, batchReplName), buf)
-}
-
 func (ri *batchJobInfo) countItem(size int64, dmarker, success bool) {
 	if ri == nil {
 		return
@@ -499,14 +496,28 @@ func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, durati
 		return errInvalidArgument
 	}
 	now := UTCNow()
+	ri.mu.Lock()
 	if now.Sub(ri.LastUpdate) >= duration {
 		if serverDebugLog {
 			console.Debugf("batchReplicate: persisting batchReplication info on drive: threshold:%s, batchRepl:%#v\n", now.Sub(ri.LastUpdate), ri)
 		}
 		ri.LastUpdate = now
 		ri.Version = batchReplVersionV1
-		return ri.save(ctx, api, jobLocation)
+
+		data := make([]byte, 4, ri.Msgsize()+4)
+
+		// Initialize the header.
+		binary.LittleEndian.PutUint16(data[0:2], batchReplFormat)
+		binary.LittleEndian.PutUint16(data[2:4], batchReplVersion)
+
+		buf, err := ri.MarshalMsg(data)
+		ri.mu.Unlock()
+		if err != nil {
+			return err
+		}
+		return saveConfig(ctx, api, pathJoin(jobLocation, batchReplName), buf)
 	}
+	ri.mu.Unlock()
 	return nil
 }
 
@@ -514,6 +525,10 @@ func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo,
 	if ri == nil {
 		return
 	}
+
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+
 	ri.Bucket = bucket
 	ri.Object = info.Name
 	ri.countItem(info.Size, info.DeleteMarker, failed)
@@ -529,7 +544,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	if err := ri.load(ctx, api, job); err != nil {
 		return err
 	}
-	globalBatchJobsMetrics.save(job.ID, ri.clone())
+	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
 
 	delay := job.Replicate.Flags.Retry.Delay
@@ -618,10 +633,24 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	if err != nil {
 		return err
 	}
+	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
+
+	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
+	if err != nil {
+		return err
+	}
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		// invalid worker size.
+		return err
+	}
 
 	retryAttempts := ri.RetryAttempts
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
+		attempts := attempts
+
 		ctx, cancel := context.WithCancel(ctx)
 
 		results := make(chan ObjectInfo, 100)
@@ -635,31 +664,43 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		}
 
 		for result := range results {
-			stopFn := globalBatchJobsMetrics.trace(batchReplicationMetricObject, job.ID, attempts, result)
-			success := true
-			if err := r.ReplicateToTarget(ctx, api, c, result, retry); err != nil {
-				if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
-					// object must be deleted concurrently, allow
-					// these failures but do not count them
-					continue
+			result := result
+			wk.Take()
+			go func() {
+				defer wk.Give()
+
+				stopFn := globalBatchJobsMetrics.trace(batchReplicationMetricObject, job.ID, attempts, result)
+				success := true
+				if err := r.ReplicateToTarget(ctx, api, c, result, retry); err != nil {
+					if miniogo.ToErrorResponse(err).Code == "PreconditionFailed" {
+						// pre-condition failed means we already have the object copied over.
+						return
+					}
+					// object must be deleted concurrently, allow these failures but do not count them
+					if isErrVersionNotFound(err) || isErrObjectNotFound(err) {
+						return
+					}
+					stopFn(err)
+					logger.LogIf(ctx, err)
+					success = false
+				} else {
+					stopFn(nil)
 				}
-				stopFn(err)
-				logger.LogIf(ctx, err)
-				success = false
-			} else {
-				stopFn(nil)
-			}
-			ri.trackCurrentBucketObject(r.Source.Bucket, result, success)
-			globalBatchJobsMetrics.save(job.ID, ri.clone())
-			// persist in-memory state to disk after every 10secs.
-			logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job.Location))
+				ri.trackCurrentBucketObject(r.Source.Bucket, result, success)
+				globalBatchJobsMetrics.save(job.ID, ri)
+				// persist in-memory state to disk after every 10secs.
+				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job.Location))
+			}()
 		}
+		wk.Wait()
 
 		ri.RetryAttempts = attempts
 		ri.Complete = ri.ObjectsFailed == 0
 		ri.Failed = ri.ObjectsFailed > 0
 
-		globalBatchJobsMetrics.save(job.ID, ri.clone())
+		globalBatchJobsMetrics.save(job.ID, ri)
+		// persist in-memory state to disk.
+		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job.Location))
 
 		buf, _ := json.Marshal(ri)
 		if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
@@ -697,7 +738,7 @@ func (e batchReplicationJobError) Error() string {
 }
 
 // Validate validates the job definition input
-func (r *BatchJobReplicateV1) Validate(ctx context.Context, o ObjectLayer) error {
+func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error {
 	if r == nil {
 		return nil
 	}
@@ -773,6 +814,7 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, o ObjectLayer) error
 	if err != nil {
 		return err
 	}
+	c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID)
 
 	vcfg, err := c.GetBucketVersioning(ctx, r.Target.Bucket)
 	if err != nil {
@@ -811,7 +853,7 @@ func (j BatchJobRequest) Type() madmin.BatchJobType {
 // persisting the job request
 func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error {
 	if j.Replicate != nil {
-		return j.Replicate.Validate(ctx, o)
+		return j.Replicate.Validate(ctx, j, o)
 	}
 	return errInvalidArgument
 }
@@ -1029,7 +1071,6 @@ type BatchJobPool struct {
 	mu           sync.Mutex
 	jobCh        chan *BatchJobRequest
 	workerKillCh chan struct{}
-	workerWg     sync.WaitGroup
 	workerSize   int
 }
 
@@ -1074,7 +1115,6 @@ func (j *BatchJobPool) AddWorker() {
 	if j == nil {
 		return
 	}
-	defer j.workerWg.Done()
 	for {
 		select {
 		case <-j.ctx.Done():
@@ -1110,7 +1150,6 @@ func (j *BatchJobPool) ResizeWorkers(n int) {
 
 	for j.workerSize < n {
 		j.workerSize++
-		j.workerWg.Add(1)
 		go j.AddWorker()
 	}
 	for j.workerSize > n {
@@ -1138,11 +1177,11 @@ func (j *BatchJobPool) queueJob(req *BatchJobRequest) error {
 //msgp:ignore batchJobMetrics
 type batchJobMetrics struct {
 	sync.RWMutex
-	metrics map[string]batchJobInfo
+	metrics map[string]*batchJobInfo
 }
 
 var globalBatchJobsMetrics = batchJobMetrics{
-	metrics: make(map[string]batchJobInfo),
+	metrics: make(map[string]*batchJobInfo),
 }
 
 //msgp:ignore batchReplicationMetric
@@ -1210,11 +1249,11 @@ func (m *batchJobMetrics) delete(jobID string) {
 	delete(m.metrics, jobID)
 }
 
-func (m *batchJobMetrics) save(jobID string, ri batchJobInfo) {
+func (m *batchJobMetrics) save(jobID string, ri *batchJobInfo) {
 	m.Lock()
 	defer m.Unlock()
 
-	m.metrics[jobID] = ri
+	m.metrics[jobID] = ri.clone()
 }
 
 func (m *batchJobMetrics) trace(d batchReplicationMetric, job string, attempts int, info ObjectInfo) func(err error) {

cmd/benchmark-utils_test.go

@@ -35,7 +35,7 @@ func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
+	err = obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -76,7 +76,7 @@ func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) {
 	object := getRandomObjectName()
 
 	// create bucket.
-	err = obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
+	err = obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -196,7 +196,7 @@ func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) {
 	// obtains random bucket name.
 	bucket := getRandomBucketName()
 	// create bucket.
-	err := obj.MakeBucketWithLocation(context.Background(), bucket, MakeBucketOptions{})
+	err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{})
 	if err != nil {
 		b.Fatal(err)
 	}

cmd/bootstrap-peer-server.go

@@ -25,14 +25,13 @@ import (
 	"net/http"
 	"net/url"
 	"reflect"
-	"runtime"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/minio-go/v7/pkg/set"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/rest"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/env"
 )
 
@@ -53,24 +52,22 @@ type bootstrapRESTServer struct{}
 
 // ServerSystemConfig - captures information about server configuration.
 type ServerSystemConfig struct {
-	MinioPlatform  string
 	MinioEndpoints EndpointServerPools
 	MinioEnv       map[string]string
 }
 
 // Diff - returns error on first difference found in two configs.
 func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
-	if s1.MinioPlatform != s2.MinioPlatform {
-		return fmt.Errorf("Expected platform '%s', found to be running '%s'",
-			s1.MinioPlatform, s2.MinioPlatform)
-	}
-
 	if s1.MinioEndpoints.NEndpoints() != s2.MinioEndpoints.NEndpoints() {
 		return fmt.Errorf("Expected number of endpoints %d, seen %d", s1.MinioEndpoints.NEndpoints(),
 			s2.MinioEndpoints.NEndpoints())
 	}
 
 	for i, ep := range s1.MinioEndpoints {
+		if ep.CmdLine != s2.MinioEndpoints[i].CmdLine {
+			return fmt.Errorf("Expected command line argument %s, seen %s", ep.CmdLine,
+				s2.MinioEndpoints[i].CmdLine)
+		}
 		if ep.SetCount != s2.MinioEndpoints[i].SetCount {
 			return fmt.Errorf("Expected set count %d, seen %d", ep.SetCount,
 				s2.MinioEndpoints[i].SetCount)
@@ -79,11 +76,9 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 			return fmt.Errorf("Expected drives pet set %d, seen %d", ep.DrivesPerSet,
 				s2.MinioEndpoints[i].DrivesPerSet)
 		}
-		for j, endpoint := range ep.Endpoints {
-			if endpoint.String() != s2.MinioEndpoints[i].Endpoints[j].String() {
-				return fmt.Errorf("Expected endpoint %s, seen %s", endpoint,
-					s2.MinioEndpoints[i].Endpoints[j])
-			}
+		if ep.Platform != s2.MinioEndpoints[i].Platform {
+			return fmt.Errorf("Expected platform '%s', found to be on '%s'",
+				ep.Platform, s2.MinioEndpoints[i].Platform)
 		}
 	}
 	if !reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
@@ -126,7 +121,6 @@ func getServerSystemCfg() ServerSystemConfig {
 		envValues[envK] = env.Get(envK, "")
 	}
 	return ServerSystemConfig{
-		MinioPlatform:  fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH),
 		MinioEndpoints: globalEndpoints,
 		MinioEnv:       envValues,
 	}
@@ -232,7 +226,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 					logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints))
 				}
 				if len(incorrectConfigs) > 0 {
-					logger.Info(fmt.Sprintf("Following servers mismatch in their configuration %s", incorrectConfigs))
+					logger.Info(fmt.Sprintf("Following servers have mismatching configuration %s", incorrectConfigs))
 				}
 				retries = 0 // reset to log again after 5 retries.
 			}

cmd/bucket-encryption-handlers.go

@@ -25,11 +25,11 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/gorilla/mux"
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"
 )
 
@@ -51,11 +51,6 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if !objAPI.IsEncryptionSupported() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 
@@ -119,15 +114,12 @@ func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: &cfgStr,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -209,16 +201,14 @@ func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+
 	// Call site replication hook.
-	//
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeSSEConfig,
 		Bucket:    bucket,
 		SSEConfig: nil,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
+
 	writeSuccessNoContent(w)
 }

cmd/bucket-handlers.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2022 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -34,11 +34,12 @@ import (
 	"sync"
 
 	"github.com/google/uuid"
-	"github.com/gorilla/mux"
+	"github.com/minio/mux"
 
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/auth"
 	sse "github.com/minio/minio/internal/bucket/encryption"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
@@ -359,7 +360,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 				Groups:          cred.Groups,
 				Action:          iampolicy.ListBucketAction,
 				BucketName:      bucketInfo.Name,
-				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+				ConditionValues: getConditionValues(r, "", cred),
 				IsOwner:         owner,
 				ObjectName:      "",
 				Claims:          cred.Claims,
@@ -371,7 +372,7 @@ func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.R
 				Groups:          cred.Groups,
 				Action:          iampolicy.GetBucketLocationAction,
 				BucketName:      bucketInfo.Name,
-				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+				ConditionValues: getConditionValues(r, "", cred),
 				IsOwner:         owner,
 				ObjectName:      "",
 				Claims:          cred.Claims,
@@ -431,7 +432,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Unmarshal list of keys to be deleted.
 	deleteObjectsReq := &DeleteObjectsRequest{}
 	if err := xmlDecoder(r.Body, deleteObjectsReq, maxBodySize); err != nil {
-		logger.LogIf(ctx, err, logger.Application)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -495,7 +495,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	vc, _ := globalBucketVersioningSys.Get(bucket)
 	oss := make([]*objSweeper, len(deleteObjectsReq.Objects))
 	for index, object := range deleteObjectsReq.Objects {
-		if apiErrCode := checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName); apiErrCode != ErrNone {
+		if apiErrCode := checkRequestAuthTypeWithVID(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName, object.VersionID); apiErrCode != ErrNone {
 			if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID {
 				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL)
 				return
@@ -511,11 +511,10 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 		if object.VersionID != "" && object.VersionID != nullVersionID {
 			if _, err := uuid.Parse(object.VersionID); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("invalid version-id specified %w", err))
 				apiErr := errorCodes.ToAPIErr(ErrNoSuchVersion)
 				deleteResults[index].errInfo = DeleteError{
 					Code:      apiErr.Code,
-					Message:   apiErr.Description,
+					Message:   fmt.Sprintf("%s (%s)", apiErr.Description, err),
 					Key:       object.ObjectName,
 					VersionID: object.VersionID,
 				}
@@ -744,7 +743,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 				AccountName:     cred.AccessKey,
 				Groups:          cred.Groups,
 				Action:          action,
-				ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+				ConditionValues: getConditionValues(r, "", cred),
 				BucketName:      bucket,
 				IsOwner:         owner,
 				Claims:          cred.Claims,
@@ -756,29 +755,18 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Parse incoming location constraint.
-	location, s3Error := parseLocationConstraint(r)
+	_, s3Error = parseLocationConstraint(r)
 	if s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
 
-	// Validate if location sent by the client is valid, reject
-	// requests which do not follow valid region requirements.
-	if !isValidLocation(location) {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRegion), r.URL)
-		return
-	}
-
-	// check if client is attempting to create more buckets than allowed maximum.
+	// check if client is attempting to create more buckets, complain about it.
 	if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
-		apiErr := errorCodes.ToAPIErr(ErrTooManyBuckets)
-		apiErr.Description = fmt.Sprintf("You have attempted to create %d buckets than allowed %d", currBuckets+1, maxBuckets)
-		writeErrorResponse(ctx, w, apiErr, r.URL)
-		return
+		logger.LogIf(ctx, fmt.Errorf("An attempt to create %d buckets beyond recommended %d", currBuckets+1, maxBuckets))
 	}
 
 	opts := MakeBucketOptions{
-		Location:    location,
 		LockEnabled: objectLockEnabled,
 		ForceCreate: forceCreate,
 	}
@@ -790,15 +778,14 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 			// exists elsewhere
 			if err == dns.ErrNoEntriesFound || err == dns.ErrNotImplemented {
 				// Proceed to creating a bucket.
-				if err = objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
+				if err = objectAPI.MakeBucket(ctx, bucket, opts); err != nil {
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 					return
 				}
 
 				if err = globalDNSConfig.Put(bucket); err != nil {
 					objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
-						Force:      false,
-						NoRecreate: true,
+						Force:      true,
 						SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
 					})
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -842,7 +829,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Proceed to creating a bucket.
-	if err := objectAPI.MakeBucketWithLocation(ctx, bucket, opts); err != nil {
+	if err := objectAPI.MakeBucket(ctx, bucket, opts); err != nil {
 		if _, ok := err.(BucketExists); ok {
 			// Though bucket exists locally, we send the site-replication
 			// hook to ensure all sites have this bucket. If the hook
@@ -858,7 +845,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req
 	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 
 	// Call site replication hook
-	globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)
+	logger.LogIf(ctx, globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts))
 
 	// Make sure to add Location information here only for bucket
 	if cp := pathClean(r.URL.Path); cp != "" {
@@ -897,11 +884,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if crypto.Requested(r.Header) && !objectAPI.IsEncryptionSupported() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-
 	bucket := mux.Vars(r)["bucket"]
 
 	// Require Content-Length to be set in the request
@@ -927,16 +909,18 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// be loaded in memory, the remaining being put in temporary files.
 	reader, err := r.MultipartReader()
 	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
+		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
+		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
 
 	// Read multipart data and save in memory and in the disk if needed
 	form, err := reader.ReadForm(maxFormMemory)
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
+		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
+		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
 
@@ -946,8 +930,9 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Extract all form fields
 	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
+		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
+		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
 
@@ -992,7 +977,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.PutObjectAction,
-		ConditionValues: getConditionValues(r, "", cred.AccessKey, cred.Claims),
+		ConditionValues: getConditionValues(r, "", cred),
 		BucketName:      bucket,
 		ObjectName:      object,
 		IsOwner:         globalActiveCred.AccessKey == cred.AccessKey,
@@ -1070,51 +1055,56 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		writeErrorResponseHeadersOnly(w, toAPIError(ctx, err))
 		return
 	}
-	if objectAPI.IsEncryptionSupported() {
-		if crypto.Requested(formValues) && !HasSuffix(object, SlashSeparator) { // handle SSE requests
-			if crypto.SSECopy.IsRequested(r.Header) {
-				writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
-				return
-			}
-			var (
-				reader io.Reader
-				keyID  string
-				key    []byte
-				kmsCtx kms.Context
-			)
-			kind, _ := crypto.IsRequested(formValues)
-			switch kind {
-			case crypto.SSEC:
-				key, err = ParseSSECustomerHeader(formValues)
-				if err != nil {
-					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-					return
-				}
-			case crypto.S3KMS:
-				keyID, kmsCtx, err = crypto.S3KMS.ParseHTTP(formValues)
-				if err != nil {
-					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-					return
-				}
-			}
-			reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
+
+	if crypto.Requested(formValues) {
+		if crypto.SSECopy.IsRequested(r.Header) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
+			return
+		}
+
+		if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
+			return
+		}
+
+		var (
+			reader io.Reader
+			keyID  string
+			key    []byte
+			kmsCtx kms.Context
+		)
+		kind, _ := crypto.IsRequested(formValues)
+		switch kind {
+		case crypto.SSEC:
+			key, err = ParseSSECustomerHeader(formValues)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
-			info := ObjectInfo{Size: fileSize}
-			// do not try to verify encrypted content
-			hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize)
-			if err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-				return
-			}
-			pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
+		case crypto.S3KMS:
+			keyID, kmsCtx, err = crypto.S3KMS.ParseHTTP(formValues)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
 		}
+		reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+		info := ObjectInfo{Size: fileSize}
+		// do not try to verify encrypted content
+		hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+		pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
+		if err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
 	}
 
 	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, opts)
@@ -1145,6 +1135,17 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		UserAgent:    r.UserAgent(),
 		Host:         handlers.GetSourceIP(r),
 	})
+	if objInfo.NumVersions > dataScannerExcessiveVersionsThreshold {
+		defer sendEvent(eventArgs{
+			EventName:    event.ObjectManyVersions,
+			BucketName:   objInfo.Bucket,
+			Object:       objInfo,
+			ReqParams:    extractReqParams(r),
+			RespElements: extractRespElements(w),
+			UserAgent:    r.UserAgent(),
+			Host:         handlers.GetSourceIP(r),
+		})
+	}
 
 	if redirectURL != nil { // success_action_redirect is valid and set.
 		v := redirectURL.Query()
@@ -1204,7 +1205,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
 	readable := globalPolicySys.IsAllowed(policy.Args{
 		Action:          policy.ListBucketAction,
 		BucketName:      bucket,
-		ConditionValues: getConditionValues(r, "", "", nil),
+		ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 		IsOwner:         false,
 	})
 
@@ -1212,7 +1213,7 @@ func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter,
 	writable := globalPolicySys.IsAllowed(policy.Args{
 		Action:          policy.PutObjectAction,
 		BucketName:      bucket,
-		ConditionValues: getConditionValues(r, "", "", nil),
+		ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials),
 		IsOwner:         false,
 	})
 
@@ -1356,11 +1357,9 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 
 	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
 	globalReplicationPool.deleteResyncMetadata(ctx, bucket)
+
 	// Call site replication hook.
-	if err := globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	logger.LogIf(ctx, globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete))
 
 	// Write success response.
 	writeSuccessNoContent(w)
@@ -1429,15 +1428,12 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:             madmin.SRBucketMetaTypeObjectLockConfig,
 		Bucket:           bucket,
 		ObjectLockConfig: &cfgStr,
 		UpdatedAt:        updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -1536,15 +1532,12 @@ func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *h
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeTags,
 		Bucket:    bucket,
 		Tags:      &cfgStr,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
@@ -1615,14 +1608,11 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypeTags,
 		Bucket:    bucket,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)

cmd/bucket-handlers_test.go

@@ -662,6 +662,9 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	var objectNames []string
 	for i := 0; i < 10; i++ {
 		objectName := "test-object-" + strconv.Itoa(i)
+		if i == 0 {
+			objectName += "/"
+		}
 		// uploading the object.
 		_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
 		// if object upload fails stop the test.
@@ -793,9 +796,9 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		expectedContent    []byte
 		expectedRespStatus int
 	}{
-		// Test case - 1.
+		// Test case - 0.
 		// Delete objects with invalid access key.
-		{
+		0: {
 			bucket:             bucketName,
 			objects:            successRequest0,
 			accessKey:          "Invalid-AccessID",
@@ -803,9 +806,19 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 			expectedContent:    nil,
 			expectedRespStatus: http.StatusForbidden,
 		},
-		// Test case - 2.
+		// Test case - 1.
 		// Delete valid objects with quiet flag off.
-		{
+		1: {
+			bucket:             bucketName,
+			objects:            successRequest0,
+			accessKey:          credentials.AccessKey,
+			secretKey:          credentials.SecretKey,
+			expectedContent:    encodedSuccessResponse0,
+			expectedRespStatus: http.StatusOK,
+		},
+		// Test case - 2.
+		// Delete deleted objects with quiet flag off.
+		2: {
 			bucket:             bucketName,
 			objects:            successRequest0,
 			accessKey:          credentials.AccessKey,
@@ -815,7 +828,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		},
 		// Test case - 3.
 		// Delete valid objects with quiet flag on.
-		{
+		3: {
 			bucket:             bucketName,
 			objects:            successRequest1,
 			accessKey:          credentials.AccessKey,
@@ -825,7 +838,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		},
 		// Test case - 4.
 		// Delete previously deleted objects.
-		{
+		4: {
 			bucket:             bucketName,
 			objects:            successRequest1,
 			accessKey:          credentials.AccessKey,
@@ -836,7 +849,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		// Test case - 5.
 		// Anonymous user access denied response
 		// Currently anonymous users cannot delete multiple objects in MinIO server
-		{
+		5: {
 			bucket:             bucketName,
 			objects:            anonRequest,
 			accessKey:          "",
@@ -847,7 +860,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		// Test case - 6.
 		// Anonymous user has access to some public folder, issue removing with
 		// another private object as well
-		{
+		6: {
 			bucket:             bucketName,
 			objects:            anonRequestWithPartialPublicAccess,
 			accessKey:          "",
@@ -881,19 +894,19 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		apiRouter.ServeHTTP(rec, req)
 		// Assert the response code with the expected status.
 		if rec.Code != testCase.expectedRespStatus {
-			t.Errorf("Test %d: MinIO %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code)
+			t.Errorf("Test %d: MinIO %s: Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, rec.Code)
 		}
 
 		// read the response body.
 		actualContent, err = io.ReadAll(rec.Body)
 		if err != nil {
-			t.Fatalf("Test %d : MinIO %s: Failed parsing response body: <ERROR> %v", i+1, instanceType, err)
+			t.Fatalf("Test %d : MinIO %s: Failed parsing response body: <ERROR> %v", i, instanceType, err)
 		}
 
 		// Verify whether the bucket obtained object is same as the one created.
 		if testCase.expectedContent != nil && !bytes.Equal(testCase.expectedContent, actualContent) {
 			t.Log(string(testCase.expectedContent), string(actualContent))
-			t.Errorf("Test %d : MinIO %s: Object content differs from expected value.", i+1, instanceType)
+			t.Errorf("Test %d : MinIO %s: Object content differs from expected value.", i, instanceType)
 		}
 	}
 

cmd/bucket-lifecycle-handlers.go

@@ -22,10 +22,10 @@ import (
 	"io"
 	"net/http"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"
 )
 

cmd/bucket-lifecycle.go

@@ -152,7 +152,6 @@ type transitionTask struct {
 }
 
 type transitionState struct {
-	once         sync.Once
 	transitionCh chan transitionTask
 
 	ctx        context.Context
@@ -169,10 +168,7 @@ type transitionState struct {
 
 func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
 	select {
-	case <-GlobalContext.Done():
-		t.once.Do(func() {
-			close(t.transitionCh)
-		})
+	case <-t.ctx.Done():
 	case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
 	default:
 	}
@@ -180,20 +176,32 @@ func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
 
 var globalTransitionState *transitionState
 
-func newTransitionState(ctx context.Context, objAPI ObjectLayer) *transitionState {
+// newTransitionState returns a transitionState object ready to be initialized
+// via its Init method.
+func newTransitionState(ctx context.Context) *transitionState {
 	return &transitionState{
 		transitionCh: make(chan transitionTask, 10000),
 		ctx:          ctx,
-		objAPI:       objAPI,
 		killCh:       make(chan struct{}),
 		lastDayStats: make(map[string]*lastDayTierStats),
 	}
 }
 
+// Init initializes t with given objAPI and instantiates the configured number
+// of transition workers.
+func (t *transitionState) Init(objAPI ObjectLayer) {
+	n := globalAPIConfig.getTransitionWorkers()
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	t.objAPI = objAPI
+	t.updateWorkers(n)
+}
+
 // PendingTasks returns the number of ILM transition tasks waiting for a worker
 // goroutine.
 func (t *transitionState) PendingTasks() int {
-	return len(globalTransitionState.transitionCh)
+	return len(t.transitionCh)
 }
 
 // ActiveTasks returns the number of active (ongoing) ILM transition tasks.
@@ -202,20 +210,20 @@ func (t *transitionState) ActiveTasks() int {
 }
 
 // worker waits for transition tasks
-func (t *transitionState) worker(ctx context.Context, objectAPI ObjectLayer) {
+func (t *transitionState) worker(objectAPI ObjectLayer) {
 	for {
 		select {
 		case <-t.killCh:
 			return
-		case <-ctx.Done():
+		case <-t.ctx.Done():
 			return
 		case task, ok := <-t.transitionCh:
 			if !ok {
 				return
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
-			if err := transitionObject(ctx, objectAPI, task.objInfo, task.tier); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
+			if err := transitionObject(t.ctx, objectAPI, task.objInfo, task.tier); err != nil {
+				logger.LogIf(t.ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
 					task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 			} else {
 				ts := tierStats{
@@ -258,9 +266,15 @@ func (t *transitionState) getDailyAllTierStats() DailyAllTierStats {
 func (t *transitionState) UpdateWorkers(n int) {
 	t.mu.Lock()
 	defer t.mu.Unlock()
+	if t.objAPI == nil { // Init hasn't been called yet.
+		return
+	}
+	t.updateWorkers(n)
+}
 
+func (t *transitionState) updateWorkers(n int) {
 	for t.numWorkers < n {
-		go t.worker(t.ctx, t.objAPI)
+		go t.worker(t.objAPI)
 		t.numWorkers++
 	}
 
@@ -270,12 +284,6 @@ func (t *transitionState) UpdateWorkers(n int) {
 	}
 }
 
-func initBackgroundTransition(ctx context.Context, objectAPI ObjectLayer) {
-	globalTransitionState = newTransitionState(ctx, objectAPI)
-	n := globalAPIConfig.getTransitionWorkers()
-	globalTransitionState.UpdateWorkers(n)
-}
-
 var errInvalidStorageClass = errors.New("invalid storage class")
 
 func validateTransitionTier(lc *lifecycle.Lifecycle) error {
@@ -366,7 +374,7 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 			EventName:  eventName,
 			BucketName: oi.Bucket,
 			Object:     objInfo,
-			Host:       "Internal: [ILM-EXPIRY]",
+			Host:       "Internal: [ILM-Expiry]",
 		})
 
 	case expireRestoredObj:
@@ -660,7 +668,9 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 	if len(objInfo.UserTags) != 0 {
 		meta[xhttp.AmzObjectTagging] = objInfo.UserTags
 	}
-
+	// Set restore object status
+	restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days)
+	meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String()
 	return ObjectOptions{
 		Versioned:        globalBucketVersioningSys.PrefixEnabled(bucket, object),
 		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object),

cmd/bucket-listobjects-handlers.go

@@ -23,8 +23,8 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 
 	"github.com/minio/pkg/bucket/policy"
 )
@@ -115,7 +115,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
 
 	// Write success response.
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, encodeResponseList(response))
 }
 
 // ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata.
@@ -185,7 +185,7 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, true)
 
 	// Write success response.
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, encodeResponseList(response))
 }
 
 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
@@ -260,7 +260,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
 
 	// Write success response.
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, encodeResponseList(response))
 }
 
 func parseRequestToken(token string) (subToken string, nodeIndex int) {
@@ -357,5 +357,5 @@ func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http
 	response := generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo)
 
 	// Write success response.
-	writeSuccessResponseXML(w, encodeResponse(response))
+	writeSuccessResponseXML(w, encodeResponseList(response))
 }

cmd/bucket-metadata-sys.go

@@ -21,10 +21,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"runtime"
 	"sync"
 	"time"
 
 	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	bucketsse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -40,6 +42,8 @@ import (
 
 // BucketMetadataSys captures all bucket metadata for a given cluster.
 type BucketMetadataSys struct {
+	objAPI ObjectLayer
+
 	sync.RWMutex
 	metadataMap map[string]BucketMetadata
 }
@@ -53,20 +57,36 @@ func (sys *BucketMetadataSys) Count() int {
 }
 
 // Remove bucket metadata from memory.
-func (sys *BucketMetadataSys) Remove(bucket string) {
+func (sys *BucketMetadataSys) Remove(buckets ...string) {
 	sys.Lock()
-	delete(sys.metadataMap, bucket)
-	globalBucketMonitor.DeleteBucket(bucket)
+	for _, bucket := range buckets {
+		delete(sys.metadataMap, bucket)
+		globalBucketMonitor.DeleteBucket(bucket)
+	}
 	sys.Unlock()
 }
 
+// RemoveStaleBuckets removes all stale buckets in memory that are not on disk.
+func (sys *BucketMetadataSys) RemoveStaleBuckets(diskBuckets set.StringSet) {
+	sys.Lock()
+	defer sys.Unlock()
+
+	for bucket := range sys.metadataMap {
+		if diskBuckets.Contains(bucket) {
+			continue
+		} // doesn't exist on disk remove from memory.
+		delete(sys.metadataMap, bucket)
+		globalBucketMonitor.DeleteBucket(bucket)
+	}
+}
+
 // Set - sets a new metadata in-memory.
 // Only a shallow copy is saved and fields with references
 // cannot be modified without causing a race condition,
 // so they should be replaced atomically and not appended to, etc.
 // Data is not persisted to disk.
 func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) {
-	if bucket != minioMetaBucket {
+	if !isMinioMetaBucketName(bucket) {
 		sys.Lock()
 		sys.metadataMap[bucket] = meta
 		sys.Unlock()
@@ -79,7 +99,7 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 		return updatedAt, errServerNotInitialized
 	}
 
-	if bucket == minioMetaBucket {
+	if isMinioMetaBucketName(bucket) {
 		return updatedAt, errInvalidArgument
 	}
 
@@ -164,7 +184,7 @@ func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configF
 // For all other bucket specific metadata, use the relevant
 // calls implemented specifically for each of those features.
 func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
-	if bucket == minioMetaBucket {
+	if isMinioMetaBucketName(bucket) {
 		return newBucketMetadata(bucket), errConfigNotFound
 	}
 
@@ -182,7 +202,7 @@ func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) {
 // GetVersioningConfig returns configured versioning config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, meta.Created, nil
@@ -195,7 +215,7 @@ func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Ve
 // GetTaggingConfig returns configured tagging config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketTaggingNotFound{Bucket: bucket}
@@ -211,7 +231,7 @@ func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, time.
 // GetObjectLockConfig returns configured object lock config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketObjectLockConfigNotFound{Bucket: bucket}
@@ -227,7 +247,7 @@ func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Co
 // GetLifecycleConfig returns configured lifecycle config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketLifecycleNotFound{Bucket: bucket}
@@ -243,7 +263,7 @@ func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Life
 // GetNotificationConfig returns configured notification config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		return nil, err
 	}
@@ -253,7 +273,7 @@ func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Confi
 // GetSSEConfig returns configured SSE config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketSSEConfigNotFound{Bucket: bucket}
@@ -268,7 +288,7 @@ func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEC
 
 // CreatedAt returns the time of creation of bucket
 func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		return time.Time{}, err
 	}
@@ -278,7 +298,7 @@ func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) {
 // GetPolicyConfig returns configured bucket policy
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, time.Time, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket}
@@ -294,7 +314,7 @@ func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.Policy, ti
 // GetQuotaConfig returns configured bucket quota
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string) (*madmin.BucketQuota, time.Time, error) {
-	meta, err := sys.GetConfig(ctx, bucket)
+	meta, _, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketQuotaConfigNotFound{Bucket: bucket}
@@ -307,7 +327,7 @@ func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string)
 // GetReplicationConfig returns configured bucket replication config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, time.Time, error) {
-	meta, err := sys.GetConfig(ctx, bucket)
+	meta, reloaded, err := sys.GetConfig(ctx, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket}
@@ -318,13 +338,18 @@ func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket s
 	if meta.replicationConfig == nil {
 		return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket}
 	}
+	if reloaded {
+		globalBucketTargetSys.set(BucketInfo{
+			Name: bucket,
+		}, meta)
+	}
 	return meta.replicationConfig, meta.ReplicationConfigUpdatedAt, nil
 }
 
 // GetBucketTargetsConfig returns configured bucket targets for this bucket
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.BucketTargets, error) {
-	meta, err := sys.GetConfig(GlobalContext, bucket)
+	meta, reloaded, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
 			return nil, BucketRemoteTargetNotFound{Bucket: bucket}
@@ -334,36 +359,56 @@ func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.Buc
 	if meta.bucketTargetConfig == nil {
 		return nil, BucketRemoteTargetNotFound{Bucket: bucket}
 	}
+	if reloaded {
+		globalBucketTargetSys.set(BucketInfo{
+			Name: bucket,
+		}, meta)
+	}
 	return meta.bucketTargetConfig, nil
 }
 
-// GetConfig returns a specific configuration from the bucket metadata.
-// The returned object may not be modified.
-func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (BucketMetadata, error) {
+// GetConfigFromDisk read bucket metadata config from disk.
+func (sys *BucketMetadataSys) GetConfigFromDisk(ctx context.Context, bucket string) (BucketMetadata, error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return newBucketMetadata(bucket), errServerNotInitialized
 	}
 
-	if bucket == minioMetaBucket {
+	if isMinioMetaBucketName(bucket) {
 		return newBucketMetadata(bucket), errInvalidArgument
 	}
 
+	return loadBucketMetadata(ctx, objAPI, bucket)
+}
+
+// GetConfig returns a specific configuration from the bucket metadata.
+// The returned object may not be modified.
+// reloaded will be true if metadata refreshed from disk
+func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (meta BucketMetadata, reloaded bool, err error) {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return newBucketMetadata(bucket), reloaded, errServerNotInitialized
+	}
+
+	if isMinioMetaBucketName(bucket) {
+		return newBucketMetadata(bucket), reloaded, errInvalidArgument
+	}
+
 	sys.RLock()
 	meta, ok := sys.metadataMap[bucket]
 	sys.RUnlock()
 	if ok {
-		return meta, nil
+		return meta, reloaded, nil
 	}
-	meta, err := loadBucketMetadata(ctx, objAPI, bucket)
+	meta, err = loadBucketMetadata(ctx, objAPI, bucket)
 	if err != nil {
-		return meta, err
+		return meta, reloaded, err
 	}
 	sys.Lock()
 	sys.metadataMap[bucket] = meta
 	sys.Unlock()
 
-	return meta, nil
+	return meta, true, nil
 }
 
 // Init - initializes bucket metadata system for all buckets.
@@ -372,59 +417,129 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 		return errServerNotInitialized
 	}
 
+	sys.objAPI = objAPI
+
 	// Load bucket metadata sys in background
-	go sys.load(ctx, buckets, objAPI)
+	go sys.init(ctx, buckets)
+	return nil
+}
+
+func (sys *BucketMetadataSys) loadBucketMetadata(ctx context.Context, bucket BucketInfo) error {
+	meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket.Name)
+	if err != nil {
+		return err
+	}
+
+	sys.Lock()
+	sys.metadataMap[bucket.Name] = meta
+	sys.Unlock()
+
 	return nil
 }
 
 // concurrently load bucket metadata to speed up loading bucket metadata.
-func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []BucketInfo) {
 	g := errgroup.WithNErrs(len(buckets))
+	bucketMetas := make([]BucketMetadata, len(buckets))
 	for index := range buckets {
 		index := index
 		g.Go(func() error {
-			_, _ = objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
+			_, _ = sys.objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
 				// Ensure heal opts for bucket metadata be deep healed all the time.
 				ScanMode: madmin.HealDeepScan,
 				Recreate: true,
 			})
-			meta, err := loadBucketMetadata(ctx, objAPI, buckets[index].Name)
+			meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[index].Name)
 			if err != nil {
-				if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) {
-					meta = newBucketMetadata(buckets[index].Name)
-				} else {
-					return err
-				}
+				return err
 			}
-			sys.Lock()
-			sys.metadataMap[buckets[index].Name] = meta
-			sys.Unlock()
-
-			globalEventNotifier.set(buckets[index], meta) // set notification targets
-
-			globalBucketTargetSys.set(buckets[index], meta) // set remote replication targets
-
+			bucketMetas[index] = meta
 			return nil
 		}, index)
 	}
-	for _, err := range g.Wait() {
+
+	errs := g.Wait()
+	for _, err := range errs {
 		if err != nil {
 			logger.LogIf(ctx, err)
 		}
 	}
+
+	// Hold lock here to update in-memory map at once,
+	// instead of serializing the Go routines.
+	sys.Lock()
+	for i, meta := range bucketMetas {
+		if errs[i] != nil {
+			continue
+		}
+		sys.metadataMap[buckets[i].Name] = meta
+	}
+	sys.Unlock()
+
+	for i, meta := range bucketMetas {
+		if errs[i] != nil {
+			continue
+		}
+		globalEventNotifier.set(buckets[i], meta)   // set notification targets
+		globalBucketTargetSys.set(buckets[i], meta) // set remote replication targets
+	}
+}
+
+func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
+	const bucketMetadataRefresh = 15 * time.Minute
+
+	t := time.NewTimer(bucketMetadataRefresh)
+	defer t.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-t.C:
+			buckets, err := sys.objAPI.ListBuckets(ctx, BucketOptions{})
+			if err != nil {
+				logger.LogIf(ctx, err)
+				continue
+			}
+
+			// Handle if we have some buckets in-memory those are stale.
+			// first delete them and then replace the newer state()
+			// from disk.
+			diskBuckets := set.CreateStringSet()
+			for _, bucket := range buckets {
+				diskBuckets.Add(bucket.Name)
+			}
+			sys.RemoveStaleBuckets(diskBuckets)
+
+			for _, bucket := range buckets {
+				err := sys.loadBucketMetadata(ctx, bucket)
+				if err != nil {
+					logger.LogIf(ctx, err)
+					continue
+				}
+				// Check if there is a spare procs, wait 100ms instead
+				waitForLowIO(runtime.GOMAXPROCS(0), 100*time.Millisecond, currentHTTPIO)
+			}
+
+			t.Reset(bucketMetadataRefresh)
+		}
+	}
 }
 
 // Loads bucket metadata for all buckets into BucketMetadataSys.
-func (sys *BucketMetadataSys) load(ctx context.Context, buckets []BucketInfo, objAPI ObjectLayer) {
+func (sys *BucketMetadataSys) init(ctx context.Context, buckets []BucketInfo) {
 	count := 100 // load 100 bucket metadata at a time.
 	for {
 		if len(buckets) < count {
-			sys.concurrentLoad(ctx, buckets, objAPI)
-			return
+			sys.concurrentLoad(ctx, buckets)
+			break
 		}
-		sys.concurrentLoad(ctx, buckets[:count], objAPI)
+		sys.concurrentLoad(ctx, buckets[:count])
 		buckets = buckets[count:]
 	}
+
+	if globalIsDistErasure {
+		go sys.refreshBucketsMetadataLoop(ctx)
+	}
 }
 
 // Reset the state of the BucketMetadataSys.

cmd/bucket-metadata.go

@@ -135,7 +135,7 @@ func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
 func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
 	if name == "" {
 		logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
-		return errors.New("bucket name cannot be empty")
+		return errInvalidArgument
 	}
 	configFile := path.Join(bucketMetaPrefix, name, bucketMetadataFile)
 	data, err := readConfig(ctx, api, configFile)

cmd/bucket-notification-handlers.go

@@ -23,9 +23,9 @@ import (
 	"net/http"
 	"reflect"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"
 )
 
@@ -50,11 +50,6 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if !objAPI.IsNotificationSupported() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -119,11 +114,6 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	if !objectAPI.IsNotificationSupported() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-
 	vars := mux.Vars(r)
 	bucketName := vars["bucket"]
 

cmd/bucket-object-lock.go

@@ -156,11 +156,8 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 				return ErrNone
 			}
 			// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
-			// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention
-			// or s3:GetBucketObjectLockConfiguration permissions, the operation will succeed.
-			govBypassPerms1 := checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName)
-			govBypassPerms2 := checkRequestAuthType(ctx, r, policy.GetBucketObjectLockConfigurationAction, bucket, object.ObjectName)
-			if govBypassPerms1 != ErrNone && govBypassPerms2 != ErrNone {
+			// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention, the operation will succeed.
+			if checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName) != ErrNone {
 				return ErrAccessDenied
 			}
 		}

cmd/bucket-policy-handlers.go

@@ -24,9 +24,9 @@ import (
 	"net/http"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"
 )
 
@@ -86,7 +86,11 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 
 	bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		writeErrorResponse(ctx, w, APIError{
+			Code:           "ErrMalformedXML",
+			HTTPStatusCode: http.StatusBadRequest,
+			Description:    err.Error(),
+		}, r.URL)
 		return
 	}
 
@@ -109,15 +113,12 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	}
 
 	// Call site replication hook.
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypePolicy,
 		Bucket:    bucket,
 		Policy:    bucketPolicyBytes,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	// Success.
 	writeSuccessNoContent(w)
@@ -156,14 +157,11 @@ func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r
 	}
 
 	// Call site replication hook.
-	if err := globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:      madmin.SRBucketMetaTypePolicy,
 		Bucket:    bucket,
 		UpdatedAt: updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	// Success.
 	writeSuccessNoContent(w)

cmd/bucket-policy-handlers_test.go

@@ -107,7 +107,7 @@ func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.Policy {
 
 // Wrapper for calling Create Bucket and ensure we get one and only one success.
 func TestCreateBucket(t *testing.T) {
-	ExecObjectLayerAPITest(t, testCreateBucket, []string{"MakeBucketWithLocation"})
+	ExecObjectLayerAPITest(t, testCreateBucket, []string{"MakeBucket"})
 }
 
 // testCreateBucket - Test for calling Create Bucket and ensure we get one and only one success.
@@ -127,7 +127,7 @@ func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRoute
 			defer wg.Done()
 			// Sync start.
 			<-start
-			if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
+			if err := obj.MakeBucket(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
 				if _, ok := err.(BucketExists); !ok {
 					t.Logf("unexpected error: %T: %v", err, err)
 					return
@@ -162,7 +162,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 	credentials auth.Credentials, t *testing.T,
 ) {
 	bucketName1 := fmt.Sprintf("%s-1", bucketName)
-	if err := obj.MakeBucketWithLocation(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
+	if err := obj.MakeBucket(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
 

cmd/bucket-policy.go

@@ -28,6 +28,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	miniogopolicy "github.com/minio/minio-go/v7/pkg/policy"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -65,9 +66,20 @@ func NewPolicySys() *PolicySys {
 	return &PolicySys{}
 }
 
-func getConditionValues(r *http.Request, lc string, username string, claims map[string]interface{}) map[string][]string {
+func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[string][]string {
 	currTime := UTCNow()
 
+	var (
+		username = cred.AccessKey
+		claims   = cred.Claims
+		groups   = cred.Groups
+	)
+
+	if cred.IsTemp() || cred.IsServiceAccount() {
+		// For derived credentials, check the parent user's permissions.
+		username = cred.ParentUser
+	}
+
 	principalType := "Anonymous"
 	if username != "" {
 		principalType = "User"
@@ -192,17 +204,11 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 	for k, v := range claims {
 		vStr, ok := v.(string)
 		if ok {
-			// Special case for AD/LDAP STS users
-			switch k {
-			case ldapUser:
-				args["user"] = []string{vStr}
-			case ldapUserN:
-				args["username"] = []string{vStr}
-			default:
-				args[k] = []string{vStr}
-			}
+			// Trim any LDAP specific prefix
+			args[strings.ToLower(strings.TrimPrefix(k, "ldap"))] = []string{vStr}
 		}
 	}
+
 	// Add groups claim which could be a list. This will ensure that the claim
 	// `jwt:groups` works.
 	if grpsVal, ok := claims["groups"]; ok {
@@ -219,6 +225,13 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 		}
 	}
 
+	// if not claim groups are available use the one with auth.Credentials
+	if _, ok := args["groups"]; !ok {
+		if len(groups) > 0 {
+			args["groups"] = groups
+		}
+	}
+
 	return args
 }
 

cmd/bucket-replication-handlers.go

@@ -25,10 +25,10 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"
 )
 
@@ -169,6 +169,21 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		return
 	}
 
+	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	for _, tgt := range targets.Targets {
+		if err := globalBucketTargetSys.RemoveTarget(ctx, bucket, tgt.Arn); err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+	}
+	if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketTargetsFile); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 	// Write success response.
 	writeSuccessResponseHeadersOnly(w)
 }
@@ -216,7 +231,19 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	w.Header().Set(xhttp.ContentType, string(mimeJSON))
 
 	enc := json.NewEncoder(w)
-	if err = enc.Encode(globalReplicationStats.getLatestReplicationStats(bucket, usageInfo)); err != nil {
+	stats := globalReplicationStats.getLatestReplicationStats(bucket, usageInfo)
+	bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
+	bwMap := bwRpt.BucketStats[bucket]
+	for arn, st := range stats.Stats {
+		if bwMap != nil {
+			if bw, ok := bwMap[arn]; ok {
+				st.BandWidthLimitInBytesPerSecond = bw.LimitInBytesPerSecond
+				st.CurrentBandwidthInBytesPerSecond = bw.CurrentBandwidthInBytesPerSecond
+				stats.Stats[arn] = st
+			}
+		}
+	}
+	if err = enc.Encode(stats); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

cmd/bucket-replication-stats.go

@@ -197,7 +197,8 @@ func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
 	m := make(map[string]*BucketReplicationStats)
 	if stats, err := globalReplicationPool.loadStatsFromDisk(); err == nil {
 		for b, st := range stats {
-			m[b] = &st
+			c := st.Clone()
+			m[b] = &c
 		}
 		r.ulock.Lock()
 		r.UsageCache = m
@@ -254,11 +255,10 @@ func (r *ReplicationStats) serializeStats() ([]byte, error) {
 	}
 	r.mostRecentStatsMu.Lock()
 	defer r.mostRecentStatsMu.Unlock()
-	bsm := r.mostRecentStats
-	if len(bsm.Stats) == 0 {
+	if len(r.mostRecentStats.Stats) == 0 {
 		return nil, nil
 	}
-	data := make([]byte, 4, 4+bsm.Msgsize())
+	data := make([]byte, 4, 4+r.mostRecentStats.Msgsize())
 	// Add the replication stats meta header.
 	binary.LittleEndian.PutUint16(data[0:2], replStatsMetaFormat)
 	binary.LittleEndian.PutUint16(data[2:4], replStatsVersion)
@@ -333,6 +333,7 @@ func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, u Buck
 	s = BucketReplicationStats{
 		Stats: make(map[string]*BucketReplicationStat, len(stats)),
 	}
+
 	var latestTotReplicatedSize int64
 	for _, st := range u.ReplicationInfo {
 		latestTotReplicatedSize += int64(st.ReplicatedSize)
@@ -372,7 +373,9 @@ func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, u Buck
 	if len(r.mostRecentStats.Stats) == 0 {
 		r.mostRecentStats = BucketStatsMap{Stats: make(map[string]BucketStats, 1), Timestamp: UTCNow()}
 	}
-	r.mostRecentStats.Stats[bucket] = BucketStats{ReplicationStats: s}
+	if len(s.Stats) > 0 {
+		r.mostRecentStats.Stats[bucket] = BucketStats{ReplicationStats: s}
+	}
 	r.mostRecentStats.Timestamp = UTCNow()
 	r.mostRecentStatsMu.Unlock()
 	return s

cmd/bucket-replication-utils.go

@@ -638,7 +638,7 @@ type replicationResyncer struct {
 }
 
 const (
-	replicationDir      = "replication"
+	replicationDir      = ".replication"
 	resyncFileName      = "resync.bin"
 	resyncMetaFormat    = 1
 	resyncMetaVersionV1 = 1
@@ -797,9 +797,8 @@ func (ri ReplicateObjectInfo) ToMRFEntry() MRFReplicateEntry {
 	}
 }
 
-func getReplicationStatsPath(nodeName string) string {
-	nodeStr := strings.ReplaceAll(nodeName, ":", "_")
-	return bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + nodeStr + ".stats"
+func getReplicationStatsPath() string {
+	return bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + "replication.stats"
 }
 
 const (

cmd/bucket-replication.go

@@ -38,6 +38,7 @@ import (
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
+	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
@@ -73,6 +74,11 @@ const (
 	ReplicationWorkerMultiplier = 1.5
 )
 
+func isReplicationEnabled(ctx context.Context, bucketName string) bool {
+	rc, _ := getReplicationConfig(ctx, bucketName)
+	return rc != nil
+}
+
 // gets replication config associated to a given bucket name.
 func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) {
 	rCfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName)
@@ -130,8 +136,8 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 			}
 		}
 		// validate replication ARN against target endpoint
-		c, ok := globalBucketTargetSys.arnRemotesMap[arnStr]
-		if ok {
+		c := globalBucketTargetSys.GetRemoteTargetClient(ctx, arnStr)
+		if c != nil {
 			if c.EndpointURL().String() == clnt.EndpointURL().String() {
 				selfTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
 				if !sameTarget {
@@ -308,7 +314,7 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 		for _, tgtArn := range tgtArns {
 			opts.TargetArn = tgtArn
 			replicate = rcfg.Replicate(opts)
-			// when incoming delete is removal of a delete marker( a.k.a versioned delete),
+			// when incoming delete is removal of a delete marker(a.k.a versioned delete),
 			// GetObjectInfo returns extra information even though it returns errFileNotFound
 			if gerr != nil {
 				validReplStatus := false
@@ -409,7 +415,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
-		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", dobj.ObjectName, bucket, rcfg.RoleArn))
+		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", dobj.ObjectName, bucket, dobj.TargetArn))
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -424,7 +430,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		return
 	}
 	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
+	defer lk.Unlock(lkctx)
 
 	var wg sync.WaitGroup
 	var rinfos replicatedInfos
@@ -738,12 +744,9 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (put
 		putOpts.Mode = rmode
 	}
 	if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok {
-		rdate, err := time.Parse(iso8601TimeFormat, retainDateStr)
+		rdate, err := amztime.ISO8601Parse(retainDateStr)
 		if err != nil {
-			rdate, err = time.Parse(time.RFC3339, retainDateStr)
-			if err != nil {
-				return putOpts, err
-			}
+			return putOpts, err
 		}
 		putOpts.RetainUntilDate = rdate
 		// set retention timestamp in opts
@@ -935,11 +938,11 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			Host:       "Internal: [Replication]",
 		})
 		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
-		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, cfg.RoleArn))
+		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, ri.TargetArn))
 		return
 	}
 	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
+	defer lk.Unlock(lkctx)
 
 	var wg sync.WaitGroup
 	var rinfos replicatedInfos
@@ -1146,10 +1149,11 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 
 	opts := &bandwidth.MonitorReaderOptions{
 		Bucket:     objInfo.Bucket,
+		TargetARN:  tgt.ARN,
 		HeaderSize: headerSize,
 	}
 	newCtx := ctx
-	if globalBucketMonitor.IsThrottled(bucket) {
+	if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) {
 		var cancel context.CancelFunc
 		newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
 		defer cancel()
@@ -1158,14 +1162,14 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	if objInfo.isMultipart() {
 		if err := replicateObjectWithMultipart(ctx, c, tgt.Bucket, object,
 			r, objInfo, putOpts); err != nil {
-			if minio.ToErrorResponse(err).Code != "PreConditionFailed" {
+			if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
 				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 			}
 		}
 	} else {
 		if _, err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); err != nil {
-			if minio.ToErrorResponse(err).Code != "PreConditionFailed" {
+			if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
 				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 			}
@@ -1346,10 +1350,11 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 
 		opts := &bandwidth.MonitorReaderOptions{
 			Bucket:     objInfo.Bucket,
+			TargetARN:  tgt.ARN,
 			HeaderSize: headerSize,
 		}
 		newCtx := ctx
-		if globalBucketMonitor.IsThrottled(bucket) {
+		if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) {
 			var cancel context.CancelFunc
 			newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
 			defer cancel()
@@ -1358,13 +1363,21 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		if objInfo.isMultipart() {
 			if err := replicateObjectWithMultipart(ctx, c, tgt.Bucket, object,
 				r, objInfo, putOpts); err != nil {
-				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+				if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
+					rinfo.ReplicationStatus = replication.Failed
+					logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+				} else {
+					rinfo.ReplicationStatus = replication.Completed
+				}
 			}
 		} else {
 			if _, err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); err != nil {
-				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+				if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
+					rinfo.ReplicationStatus = replication.Failed
+					logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+				} else {
+					rinfo.ReplicationStatus = replication.Completed
+				}
 			}
 		}
 	}
@@ -1522,15 +1535,14 @@ type ReplicationPool struct {
 	// workers:
 	workers         []chan ReplicationWorkerOperation
 	existingWorkers chan ReplicationWorkerOperation
-	workerWg        sync.WaitGroup
 
 	// mrf:
 	mrfWorkerKillCh chan struct{}
 	mrfReplicaCh    chan ReplicationWorkerOperation
 	mrfSaveCh       chan MRFReplicateEntry
+	mrfStopCh       chan struct{}
 	mrfWorkerSize   int
 	saveStateCh     chan struct{}
-	mrfWorkerWg     sync.WaitGroup
 }
 
 // ReplicationWorkerOperation is a shared interface of replication operations.
@@ -1584,6 +1596,7 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		mrfWorkerKillCh: make(chan struct{}, failedWorkers),
 		resyncer:        newresyncer(),
 		mrfSaveCh:       make(chan MRFReplicateEntry, 100000),
+		mrfStopCh:       make(chan struct{}, 1),
 		saveStateCh:     make(chan struct{}, 1),
 		ctx:             ctx,
 		objLayer:        o,
@@ -1592,7 +1605,6 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 
 	pool.ResizeWorkers(workers, 0)
 	pool.ResizeFailedWorkers(failedWorkers)
-	pool.workerWg.Add(1)
 	go pool.AddWorker(pool.existingWorkers, nil)
 	go pool.resyncer.PersistToDisk(ctx, o)
 	go pool.processMRF()
@@ -1604,7 +1616,6 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 // AddMRFWorker adds a pending/failed replication worker to handle requests that could not be queued
 // to the other workers
 func (p *ReplicationPool) AddMRFWorker() {
-	defer p.mrfWorkerWg.Done()
 	for {
 		select {
 		case <-p.ctx.Done():
@@ -1631,7 +1642,6 @@ func (p *ReplicationPool) AddMRFWorker() {
 // An optional pointer to a tracker that will be atomically
 // incremented when operations are running can be provided.
 func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opTracker *int32) {
-	defer p.workerWg.Done()
 	for {
 		select {
 		case <-p.ctx.Done():
@@ -1689,7 +1699,6 @@ func (p *ReplicationPool) ResizeWorkers(n, checkOld int) {
 		input := make(chan ReplicationWorkerOperation, 10000)
 		p.workers = append(p.workers, input)
 
-		p.workerWg.Add(1)
 		go p.AddWorker(input, &p.activeWorkers)
 	}
 	for len(p.workers) > n {
@@ -1733,7 +1742,6 @@ func (p *ReplicationPool) ResizeFailedWorkers(n int) {
 
 	for p.mrfWorkerSize < n {
 		p.mrfWorkerSize++
-		p.mrfWorkerWg.Add(1)
 		go p.AddMRFWorker()
 	}
 	for p.mrfWorkerSize > n {
@@ -1903,7 +1911,7 @@ func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs
 		return nil, proxy, err
 	}
 	c := minio.Core{Client: tgt.Client}
-	obj, _, h, err := c.GetObject(ctx, bucket, object, gopts)
+	obj, _, h, err := c.GetObject(ctx, tgt.Bucket, object, gopts)
 	if err != nil {
 		return nil, proxy, err
 	}
@@ -1928,7 +1936,7 @@ func getProxyTargets(ctx context.Context, bucket, object string, opts ObjectOpti
 	if opts.VersionSuspended {
 		return &madmin.BucketTargets{}
 	}
-	if !opts.ProxyRequest {
+	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) {
 		return &madmin.BucketTargets{}
 	}
 	cfg, err := getReplicationConfig(ctx, bucket)
@@ -2833,6 +2841,7 @@ func (p *ReplicationPool) persistMRF() {
 			saveMRFToDisk(false)
 			mTimer.Reset(mrfTimeInterval)
 		case <-p.ctx.Done():
+			p.mrfStopCh <- struct{}{}
 			close(p.mrfSaveCh)
 			saveMRFToDisk(true)
 			return
@@ -2862,8 +2871,13 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
 	select {
 	case <-GlobalContext.Done():
 		return
-	case p.mrfSaveCh <- entry:
+	case <-p.mrfStopCh:
+		return
 	default:
+		select {
+		case p.mrfSaveCh <- entry:
+		default:
+		}
 	}
 }
 
@@ -3007,7 +3021,7 @@ func (p *ReplicationPool) loadStatsFromDisk() (rs map[string]BucketReplicationSt
 		return map[string]BucketReplicationStats{}, nil
 	}
 
-	data, err := readConfig(p.ctx, p.objLayer, getReplicationStatsPath(globalLocalNodeName))
+	data, err := readConfig(p.ctx, p.objLayer, getReplicationStatsPath())
 	if err != nil {
 		if !errors.Is(err, errConfigNotFound) {
 			return rs, nil
@@ -3050,6 +3064,8 @@ func (p *ReplicationPool) saveStatsToDisk() {
 	if !p.initialized() {
 		return
 	}
+	ctx, cancel := globalLeaderLock.GetLock(p.ctx)
+	defer cancel()
 	sTimer := time.NewTimer(replStatsSaveInterval)
 	defer sTimer.Stop()
 	for {
@@ -3061,7 +3077,7 @@ func (p *ReplicationPool) saveStatsToDisk() {
 			}
 			p.saveStats(p.ctx)
 			sTimer.Reset(replStatsSaveInterval)
-		case <-p.ctx.Done():
+		case <-ctx.Done():
 			return
 		}
 	}
@@ -3077,20 +3093,5 @@ func (p *ReplicationPool) saveStats(ctx context.Context) error {
 	if data == nil {
 		return err
 	}
-	return saveConfig(ctx, p.objLayer, getReplicationStatsPath(globalLocalNodeName), data)
-}
-
-// SaveState saves replication stats and mrf data before server restart
-func (p *ReplicationPool) SaveState(ctx context.Context) error {
-	if !p.initialized() {
-		return nil
-	}
-	go func() {
-		select {
-		case p.saveStateCh <- struct{}{}:
-		case <-p.ctx.Done():
-			return
-		}
-	}()
-	return p.saveStats(ctx)
+	return saveConfig(ctx, p.objLayer, getReplicationStatsPath(), data)
 }

cmd/bucket-stats.go

@@ -120,6 +120,10 @@ type BucketReplicationStat struct {
 	FailedCount int64 `json:"failedReplicationCount"`
 	// Replication latency information
 	Latency ReplicationLatency `json:"replicationLatency"`
+	// bandwidth limit for target
+	BandWidthLimitInBytesPerSecond int64 `json:"limitInBits"`
+	// current bandwidth reported
+	CurrentBandwidthInBytesPerSecond float64 `json:"currentBandwidth"`
 }
 
 func (bs *BucketReplicationStat) hasReplicationUsage() bool {

cmd/bucket-stats_gen.go

@@ -89,6 +89,18 @@ func (z *BucketReplicationStat) DecodeMsg(dc *msgp.Reader) (err error) {
 					}
 				}
 			}
+		case "BandWidthLimitInBytesPerSecond":
+			z.BandWidthLimitInBytesPerSecond, err = dc.ReadInt64()
+			if err != nil {
+				err = msgp.WrapError(err, "BandWidthLimitInBytesPerSecond")
+				return
+			}
+		case "CurrentBandwidthInBytesPerSecond":
+			z.CurrentBandwidthInBytesPerSecond, err = dc.ReadFloat64()
+			if err != nil {
+				err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -102,9 +114,9 @@ func (z *BucketReplicationStat) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketReplicationStat) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 7
+	// map header, size 9
 	// write "PendingSize"
-	err = en.Append(0x87, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	err = en.Append(0x89, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
 	if err != nil {
 		return
 	}
@@ -179,15 +191,35 @@ func (z *BucketReplicationStat) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Latency", "UploadHistogram")
 		return
 	}
+	// write "BandWidthLimitInBytesPerSecond"
+	err = en.Append(0xbe, 0x42, 0x61, 0x6e, 0x64, 0x57, 0x69, 0x64, 0x74, 0x68, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt64(z.BandWidthLimitInBytesPerSecond)
+	if err != nil {
+		err = msgp.WrapError(err, "BandWidthLimitInBytesPerSecond")
+		return
+	}
+	// write "CurrentBandwidthInBytesPerSecond"
+	err = en.Append(0xd9, 0x20, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteFloat64(z.CurrentBandwidthInBytesPerSecond)
+	if err != nil {
+		err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketReplicationStat) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 7
+	// map header, size 9
 	// string "PendingSize"
-	o = append(o, 0x87, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
+	o = append(o, 0x89, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65)
 	o = msgp.AppendInt64(o, z.PendingSize)
 	// string "ReplicatedSize"
 	o = append(o, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65)
@@ -214,6 +246,12 @@ func (z *BucketReplicationStat) MarshalMsg(b []byte) (o []byte, err error) {
 		err = msgp.WrapError(err, "Latency", "UploadHistogram")
 		return
 	}
+	// string "BandWidthLimitInBytesPerSecond"
+	o = append(o, 0xbe, 0x42, 0x61, 0x6e, 0x64, 0x57, 0x69, 0x64, 0x74, 0x68, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
+	o = msgp.AppendInt64(o, z.BandWidthLimitInBytesPerSecond)
+	// string "CurrentBandwidthInBytesPerSecond"
+	o = append(o, 0xd9, 0x20, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64)
+	o = msgp.AppendFloat64(o, z.CurrentBandwidthInBytesPerSecond)
 	return
 }
 
@@ -300,6 +338,18 @@ func (z *BucketReplicationStat) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					}
 				}
 			}
+		case "BandWidthLimitInBytesPerSecond":
+			z.BandWidthLimitInBytesPerSecond, bts, err = msgp.ReadInt64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BandWidthLimitInBytesPerSecond")
+				return
+			}
+		case "CurrentBandwidthInBytesPerSecond":
+			z.CurrentBandwidthInBytesPerSecond, bts, err = msgp.ReadFloat64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -314,7 +364,7 @@ func (z *BucketReplicationStat) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketReplicationStat) Msgsize() (s int) {
-	s = 1 + 12 + msgp.Int64Size + 15 + msgp.Int64Size + 12 + msgp.Int64Size + 11 + msgp.Int64Size + 13 + msgp.Int64Size + 12 + msgp.Int64Size + 8 + 1 + 16 + z.Latency.UploadHistogram.Msgsize()
+	s = 1 + 12 + msgp.Int64Size + 15 + msgp.Int64Size + 12 + msgp.Int64Size + 11 + msgp.Int64Size + 13 + msgp.Int64Size + 12 + msgp.Int64Size + 8 + 1 + 16 + z.Latency.UploadHistogram.Msgsize() + 31 + msgp.Int64Size + 34 + msgp.Float64Size
 	return
 }
 

cmd/bucket-targets.go

@@ -98,7 +98,7 @@ func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
 		select {
 		case <-hcTimer.C:
 			sys.hMutex.RLock()
-			var eps []madmin.ServerProperties
+			eps := make([]madmin.ServerProperties, 0, len(sys.hc))
 			for _, ep := range sys.hc {
 				eps = append(eps, madmin.ServerProperties{Endpoint: ep.Endpoint, Scheme: ep.Scheme})
 			}
@@ -106,16 +106,12 @@ func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
 
 			if len(eps) > 0 {
 				cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
-				m := map[string]epHealth{}
+				m := make(map[string]epHealth, len(eps))
 				for result := range sys.hcClient.Alive(cctx, madmin.AliveOpts{}, eps...) {
-					var online bool
-					if result.Error == nil {
-						online = result.Online
-					}
 					m[result.Endpoint.Host] = epHealth{
 						Endpoint: result.Endpoint.Host,
 						Scheme:   result.Endpoint.Scheme,
-						Online:   online,
+						Online:   result.Online,
 					}
 				}
 				cancel()
@@ -217,10 +213,13 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	// validate if target credentials are ok
 	exists, err := clnt.BucketExists(ctx, tgt.TargetBucket)
 	if err != nil {
-		if minio.ToErrorResponse(err).Code == "NoSuchBucket" {
+		switch minio.ToErrorResponse(err).Code {
+		case "NoSuchBucket":
 			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+		case "AccessDenied":
+			return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, AccessKey: tgt.Credentials.AccessKey, Err: err}
 		}
-		return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
+		return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, AccessKey: tgt.Credentials.AccessKey, Err: err}
 	}
 	if !exists {
 		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
@@ -231,7 +230,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 		}
 		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
 		if err != nil {
-			return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err}
+			return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err, AccessKey: tgt.Credentials.AccessKey}
 		}
 		if !vcfg.Enabled() {
 			return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket}
@@ -253,6 +252,10 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 				found = true
 				continue
 			}
+			// fail if endpoint is already present in list of targets and not a matching ARN
+			if t.Endpoint == tgt.Endpoint {
+				return BucketRemoteAlreadyExists{Bucket: t.TargetBucket}
+			}
 		}
 		newtgts[idx] = t
 	}
@@ -262,18 +265,18 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 
 	sys.targetsMap[bucket] = newtgts
 	sys.arnRemotesMap[tgt.Arn] = clnt
-	sys.updateBandwidthLimit(bucket, tgt.BandwidthLimit)
+	sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
 	return nil
 }
 
-func (sys *BucketTargetSys) updateBandwidthLimit(bucket string, limit int64) {
+func (sys *BucketTargetSys) updateBandwidthLimit(bucket, arn string, limit int64) {
 	if limit == 0 {
-		globalBucketMonitor.DeleteBucket(bucket)
+		globalBucketMonitor.DeleteBucketThrottle(bucket, arn)
 		return
 	}
 	// Setup bandwidth throttling
 
-	globalBucketMonitor.SetBandwidthLimit(bucket, limit)
+	globalBucketMonitor.SetBandwidthLimit(bucket, arn, limit)
 }
 
 // RemoveTarget - removes a remote bucket target for this source bucket.
@@ -325,7 +328,7 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	}
 	sys.targetsMap[bucket] = targets
 	delete(sys.arnRemotesMap, arnStr)
-	sys.updateBandwidthLimit(bucket, 0)
+	sys.updateBandwidthLimit(bucket, arnStr, 0)
 	return nil
 }
 
@@ -395,7 +398,7 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 
 	// No need for more if not adding anything
 	if tgts == nil || tgts.Empty() {
-		sys.updateBandwidthLimit(bucket, 0)
+		globalBucketMonitor.DeleteBucket(bucket)
 		return
 	}
 
@@ -408,7 +411,7 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 			continue
 		}
 		sys.arnRemotesMap[tgt.Arn] = tgtClient
-		sys.updateBandwidthLimit(bucket, tgt.BandwidthLimit)
+		sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
 	}
 	sys.targetsMap[bucket] = tgts.Targets
 }
@@ -431,7 +434,7 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
 			continue
 		}
 		sys.arnRemotesMap[tgt.Arn] = tgtClient
-		sys.updateBandwidthLimit(bucket.Name, tgt.BandwidthLimit)
+		sys.updateBandwidthLimit(bucket.Name, tgt.Arn, tgt.BandwidthLimit)
 	}
 	sys.targetsMap[bucket.Name] = cfg.Targets
 }
@@ -450,6 +453,8 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 	if err != nil {
 		return nil, err
 	}
+	api.SetAppInfo("minio-replication-target", ReleaseTag+" "+tcfg.Arn)
+
 	hcDuration := defaultHealthCheckDuration
 	if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec.
 		hcDuration = tcfg.HealthCheckDuration
@@ -470,20 +475,23 @@ func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*T
 }
 
 // getRemoteARN gets existing ARN for an endpoint or generates a new one.
-func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget) string {
+func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget, deplID string) (arn string, exists bool) {
 	if target == nil {
-		return ""
+		return
 	}
 	tgts := sys.targetsMap[bucket]
 	for _, tgt := range tgts {
-		if tgt.Type == target.Type && tgt.TargetBucket == target.TargetBucket && target.URL().String() == tgt.URL().String() {
-			return tgt.Arn
+		if tgt.Type == target.Type &&
+			tgt.TargetBucket == target.TargetBucket &&
+			target.URL().String() == tgt.URL().String() &&
+			tgt.Credentials.AccessKey == target.Credentials.AccessKey {
+			return tgt.Arn, true
 		}
 	}
 	if !target.Type.IsValid() {
-		return ""
+		return
 	}
-	return generateARN(target)
+	return generateARN(target, deplID), false
 }
 
 // getRemoteARNForPeer returns the remote target for a peer site in site replication
@@ -503,10 +511,14 @@ func (sys *BucketTargetSys) getRemoteARNForPeer(bucket string, peer madmin.PeerI
 }
 
 // generate ARN that is unique to this target type
-func generateARN(t *madmin.BucketTarget) string {
+func generateARN(t *madmin.BucketTarget, deplID string) string {
+	uuid := deplID
+	if uuid == "" {
+		uuid = mustGetUUID()
+	}
 	arn := madmin.ARN{
 		Type:   t.Type,
-		ID:     mustGetUUID(),
+		ID:     uuid,
 		Region: t.Region,
 		Bucket: t.TargetBucket,
 	}

cmd/bucket-versioning-handler.go

@@ -24,10 +24,10 @@ import (
 	"net/http"
 
 	humanize "github.com/dustin/go-humanize"
-	"github.com/gorilla/mux"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"
 )
 
@@ -108,15 +108,12 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 	// We encode the xml bytes as base64 to ensure there are no encoding
 	// errors.
 	cfgStr := base64.StdEncoding.EncodeToString(configData)
-	if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+	logger.LogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
 		Type:       madmin.SRBucketMetaTypeVersionConfig,
 		Bucket:     bucket,
 		Versioning: &cfgStr,
 		UpdatedAt:  updatedAt,
-	}); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
+	}))
 
 	writeSuccessResponseHeadersOnly(w)
 }

cmd/callhome.go

@@ -79,7 +79,7 @@ func runCallhome(ctx context.Context, objAPI ObjectLayer) bool {
 	}
 
 	ctx = lkctx.Context()
-	defer locker.Unlock(lkctx.Cancel)
+	defer locker.Unlock(lkctx)
 
 	callhomeTimer := time.NewTimer(globalCallhomeConfig.FrequencyDur())
 	defer callhomeTimer.Stop()

cmd/common-main.go

@@ -51,7 +51,7 @@ import (
 	consoleCerts "github.com/minio/console/pkg/certs"
 	"github.com/minio/console/restapi"
 	"github.com/minio/console/restapi/operations"
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
@@ -117,7 +117,8 @@ func init() {
 	go func() {
 		var t *time.Ticker
 		if containers {
-			t = time.NewTicker(1 * time.Minute)
+			// k8s DNS TTL is 30s (Attempt a refresh only after)
+			t = time.NewTicker(30 * time.Second)
 		} else {
 			t = time.NewTicker(10 * time.Minute)
 		}
@@ -134,7 +135,7 @@ func init() {
 
 	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
 		PassHost:     true,
-		RoundTripper: newHTTPTransport(1 * time.Hour),
+		RoundTripper: NewHTTPTransportWithTimeout(1 * time.Hour),
 		Logger: func(err error) {
 			if err != nil && !errors.Is(err, context.Canceled) {
 				logger.LogIf(GlobalContext, err)
@@ -204,7 +205,7 @@ func minioConfigToConsoleFeatures() {
 		}
 	}
 	// Enable if LDAP is enabled.
-	if globalLDAPConfig.Enabled() {
+	if globalIAMSys.LDAPConfig.Enabled() {
 		os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
 	}
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
@@ -214,8 +215,9 @@ func minioConfigToConsoleFeatures() {
 }
 
 func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
-	m := make(map[string]consoleoauth2.ProviderConfig, len(globalOpenIDConfig.ProviderCfgs))
-	for name, cfg := range globalOpenIDConfig.ProviderCfgs {
+	pcfgs := globalIAMSys.OpenIDConfig.ProviderCfgs
+	m := make(map[string]consoleoauth2.ProviderConfig, len(pcfgs))
+	for name, cfg := range pcfgs {
 		callback := getConsoleEndpoints()[0] + "/oauth_callback"
 		if cfg.RedirectURI != "" {
 			callback = cfg.RedirectURI
@@ -599,13 +601,13 @@ func loadEnvVarsFromFiles() {
 		}
 	}
 
-	if env.IsSet(config.EnvKMSSecretKeyFile) {
-		kmsSecret, err := readFromSecret(env.Get(config.EnvKMSSecretKeyFile, ""))
+	if env.IsSet(kms.EnvKMSSecretKeyFile) {
+		kmsSecret, err := readFromSecret(env.Get(kms.EnvKMSSecretKeyFile, ""))
 		if err != nil {
 			logger.Fatal(err, "Unable to read the KMS secret key inherited from secret file")
 		}
 		if kmsSecret != "" {
-			os.Setenv(config.EnvKMSSecretKey, kmsSecret)
+			os.Setenv(kms.EnvKMSSecretKey, kmsSecret)
 		}
 	}
 
@@ -638,7 +640,7 @@ func handleCommonEnvVars() {
 			if !((u.Scheme == "http" || u.Scheme == "https") &&
 				u.Opaque == "" &&
 				!u.ForceQuery && u.RawQuery == "" && u.Fragment == "") {
-				err := fmt.Errorf("URL contains unexpected resources, expected URL to be of http(s)://minio.example.com format: %v", u)
+				err := fmt.Errorf("URL contains unexpected resources, expected URL to be one of http(s)://console.example.com or as a subpath via API endpoint http(s)://minio.example.com/minio format: %v", u)
 				logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value is environment variable")
 			}
 			globalBrowserRedirectURL = u
@@ -773,6 +775,8 @@ func handleCommonEnvVars() {
 			logger.Info(color.RedBold(msg))
 		}
 		globalActiveCred = cred
+	} else {
+		globalActiveCred = auth.DefaultCredentials
 	}
 }
 
@@ -780,20 +784,29 @@ func handleCommonEnvVars() {
 // It depends on KMS env variables and global cli flags.
 func handleKMSConfig() {
 	switch {
-	case env.IsSet(config.EnvKMSSecretKey) && env.IsSet(config.EnvKESEndpoint):
-		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", config.EnvKMSSecretKey, config.EnvKESEndpoint))
+	case env.IsSet(kms.EnvKMSSecretKey) && env.IsSet(kms.EnvKESEndpoint):
+		logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKMSSecretKey, kms.EnvKESEndpoint))
 	}
 
-	if env.IsSet(config.EnvKMSSecretKey) {
-		KMS, err := kms.Parse(env.Get(config.EnvKMSSecretKey, ""))
+	if env.IsSet(kms.EnvKMSSecretKey) {
+		KMS, err := kms.Parse(env.Get(kms.EnvKMSSecretKey, ""))
 		if err != nil {
 			logger.Fatal(err, "Unable to parse the KMS secret key inherited from the shell environment")
 		}
 		GlobalKMS = KMS
 	}
-	if env.IsSet(config.EnvKESEndpoint) {
+	if env.IsSet(kms.EnvKESEndpoint) {
+		if env.IsSet(kms.EnvKESAPIKey) {
+			if env.IsSet(kms.EnvKESClientKey) {
+				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientKey))
+			}
+			if env.IsSet(kms.EnvKESClientCert) {
+				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientCert))
+			}
+		}
+
 		var endpoints []string
-		for _, endpoint := range strings.Split(env.Get(config.EnvKESEndpoint, ""), ",") {
+		for _, endpoint := range strings.Split(env.Get(kms.EnvKESEndpoint, ""), ",") {
 			if strings.TrimSpace(endpoint) == "" {
 				continue
 			}
@@ -809,67 +822,82 @@ func handleKMSConfig() {
 				endpoints = append(endpoints, strings.Join(lbls, ""))
 			}
 		}
-		rootCAs, err := certs.GetRootCAs(env.Get(config.EnvKESServerCA, globalCertsCADir.Get()))
+		rootCAs, err := certs.GetRootCAs(env.Get(kms.EnvKESServerCA, globalCertsCADir.Get()))
 		if err != nil {
-			logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(config.EnvKESServerCA, globalCertsCADir.Get())))
+			logger.Fatal(err, fmt.Sprintf("Unable to load X.509 root CAs for KES from %q", env.Get(kms.EnvKESServerCA, globalCertsCADir.Get())))
 		}
 
-		loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
-			// Manually load the certificate and private key into memory.
-			// We need to check whether the private key is encrypted, and
-			// if so, decrypt it using the user-provided password.
-			certBytes, err := os.ReadFile(certFile)
+		var kmsConf kms.Config
+		if env.IsSet(kms.EnvKESAPIKey) {
+			key, err := kes.ParseAPIKey(env.Get(kms.EnvKESAPIKey, ""))
 			if err != nil {
-				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+				logger.Fatal(err, fmt.Sprintf("Failed to parse KES API key from %q", env.Get(kms.EnvKESAPIKey, "")))
 			}
-			keyBytes, err := os.ReadFile(keyFile)
-			if err != nil {
-				return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
+			kmsConf = kms.Config{
+				Endpoints:    endpoints,
+				Enclave:      env.Get(kms.EnvKESEnclave, ""),
+				DefaultKeyID: env.Get(kms.EnvKESKeyName, ""),
+				APIKey:       key,
+				RootCAs:      rootCAs,
 			}
-			privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
-			if len(rest) != 0 {
-				return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
-			}
-			if x509.IsEncryptedPEMBlock(privateKeyPEM) {
-				keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(config.EnvKESClientPassword, "")))
+		} else {
+			loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) {
+				// Manually load the certificate and private key into memory.
+				// We need to check whether the private key is encrypted, and
+				// if so, decrypt it using the user-provided password.
+				certBytes, err := os.ReadFile(certFile)
 				if err != nil {
-					return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
+					return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
 				}
-				keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
+				keyBytes, err := os.ReadFile(keyFile)
+				if err != nil {
+					return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err)
+				}
+				privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes))
+				if len(rest) != 0 {
+					return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data")
+				}
+				if x509.IsEncryptedPEMBlock(privateKeyPEM) {
+					keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(kms.EnvKESClientPassword, "")))
+					if err != nil {
+						return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err)
+					}
+					keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes})
+				}
+				certificate, err := tls.X509KeyPair(certBytes, keyBytes)
+				if err != nil {
+					return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+				}
+				return certificate, nil
 			}
-			certificate, err := tls.X509KeyPair(certBytes, keyBytes)
+
+			reloadCertEvents := make(chan tls.Certificate, 1)
+			certificate, err := certs.NewCertificate(env.Get(kms.EnvKESClientCert, ""), env.Get(kms.EnvKESClientKey, ""), loadX509KeyPair)
 			if err != nil {
-				return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err)
+				logger.Fatal(err, "Failed to load KES client certificate")
+			}
+			certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
+			certificate.Notify(reloadCertEvents)
+
+			kmsConf = kms.Config{
+				Endpoints:        endpoints,
+				Enclave:          env.Get(kms.EnvKESEnclave, ""),
+				DefaultKeyID:     env.Get(kms.EnvKESKeyName, ""),
+				Certificate:      certificate,
+				ReloadCertEvents: reloadCertEvents,
+				RootCAs:          rootCAs,
 			}
-			return certificate, nil
 		}
 
-		reloadCertEvents := make(chan tls.Certificate, 1)
-		certificate, err := certs.NewCertificate(env.Get(config.EnvKESClientCert, ""), env.Get(config.EnvKESClientKey, ""), loadX509KeyPair)
-		if err != nil {
-			logger.Fatal(err, "Failed to load KES client certificate")
-		}
-		certificate.Watch(context.Background(), 15*time.Minute, syscall.SIGHUP)
-		certificate.Notify(reloadCertEvents)
-
-		defaultKeyID := env.Get(config.EnvKESKeyName, "")
-		KMS, err := kms.NewWithConfig(kms.Config{
-			Endpoints:        endpoints,
-			Enclave:          env.Get(config.EnvKESEnclave, ""),
-			DefaultKeyID:     defaultKeyID,
-			Certificate:      certificate,
-			ReloadCertEvents: reloadCertEvents,
-			RootCAs:          rootCAs,
-		})
+		KMS, err := kms.NewWithConfig(kmsConf)
 		if err != nil {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
-
 		// We check that the default key ID exists or try to create it otherwise.
 		// This implicitly checks that we can communicate to KES. We don't treat
 		// a policy error as failure condition since MinIO may not have the permission
 		// to create keys - just to generate/decrypt data encryption keys.
-		if err = KMS.CreateKey(context.Background(), defaultKeyID); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
+		if err = KMS.CreateKey(context.Background(), env.Get(kms.EnvKESKeyName, "")); err != nil && !errors.Is(err, kes.ErrKeyExists) && !errors.Is(err, kes.ErrNotAllowed) {
 			logger.Fatal(err, "Unable to initialize a connection to KES as specified by the shell environment")
 		}
 		GlobalKMS = KMS

cmd/config-current.go

@@ -25,7 +25,6 @@ import (
 	"sync"
 
 	"github.com/minio/madmin-go/v2"
-	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/api"
 	"github.com/minio/minio/internal/config/cache"
@@ -65,7 +64,6 @@ func initHelp() {
 		config.SiteSubSys:           config.DefaultSiteKVS,
 		config.RegionSubSys:         config.DefaultRegionKVS,
 		config.APISubSys:            api.DefaultKVS,
-		config.CredentialsSubSys:    config.DefaultCredentialKVS,
 		config.LoggerWebhookSubSys:  logger.DefaultLoggerWebhookKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditWebhookKVS,
 		config.AuditKafkaSubSys:     logger.DefaultAuditKafkaKVS,
@@ -299,16 +297,9 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 			return err
 		}
 	case config.CompressionSubSys:
-		compCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
-		if err != nil {
+		if _, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default]); err != nil {
 			return err
 		}
-
-		if objAPI != nil {
-			if compCfg.Enabled && !objAPI.IsCompressionSupported() {
-				return fmt.Errorf("Backend does not support compression")
-			}
-		}
 	case config.HealSubSys:
 		if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil {
 			return err
@@ -374,8 +365,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 		subSys = config.PolicyPluginSubSys
 		fallthrough
 	case config.PolicyPluginSubSys:
-		if ppargs, err := polplugin.LookupConfig(s[config.PolicyPluginSubSys][config.Default],
-			NewHTTPTransport(), xhttp.DrainBody); err != nil {
+		if ppargs, err := polplugin.LookupConfig(s, GetDefaultConnSettings(), xhttp.DrainBody); err != nil {
 			return err
 		} else if ppargs.URL == nil {
 			// Check if legacy opa is configured.
@@ -428,15 +418,6 @@ func validateConfig(s config.Config, subSys string) error {
 func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	ctx := GlobalContext
 
-	var err error
-	if !globalActiveCred.IsValid() {
-		// Env doesn't seem to be set, we fallback to lookup creds from the config.
-		globalActiveCred, err = config.LookupCreds(s[config.CredentialsSubSys][config.Default])
-		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Invalid credentials configuration: %w", err))
-		}
-	}
-
 	dnsURL, dnsUser, dnsPass, err := env.LookupEnv(config.EnvDNSWebhook)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
@@ -512,22 +493,15 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.Fatal(errors.New("no KMS configured"), "MINIO_KMS_AUTO_ENCRYPTION requires a valid KMS configuration")
 	}
 
-	globalSTSTLSConfig, err = xtls.Lookup(s[config.IdentityTLSSubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err))
-	}
-
-	if globalSTSTLSConfig.InsecureSkipVerify {
-		logger.LogIf(ctx, fmt.Errorf("CRITICAL: enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify))
-	}
-
 	transport := NewHTTPTransport()
 
+	bootstrapTrace("lookup the event notification targets")
 	globalConfigTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
+	bootstrapTrace("applying the dynamic configuration")
 	// Apply dynamic config values
 	if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
 		logger.LogIf(ctx, err)
@@ -551,17 +525,13 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 
 		// Initialize remote instance transport once.
 		getRemoteInstanceTransportOnce.Do(func() {
-			getRemoteInstanceTransport = newHTTPTransport(apiConfig.RemoteTransportDeadline)
+			getRemoteInstanceTransport = NewHTTPTransportWithTimeout(apiConfig.RemoteTransportDeadline)
 		})
 	case config.CompressionSubSys:
 		cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default])
 		if err != nil {
 			return fmt.Errorf("Unable to setup Compression: %w", err)
 		}
-		// Validate if the object layer supports compression.
-		if cmpCfg.Enabled && !objAPI.IsCompressionSupported() {
-			return fmt.Errorf("Backend does not support compression")
-		}
 		globalCompressConfigMu.Lock()
 		globalCompressConfig = cmpCfg
 		globalCompressConfigMu.Unlock()
@@ -722,37 +692,42 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
 		h = config.HelpKVS{value}
 	}
 
-	envHelp := config.HelpKVS{}
-	if envOnly {
-		// Only for multiple targets, make sure
-		// to list the ENV, for regular k/v EnableKey is
-		// implicit, for ENVs we cannot make it implicit.
-		if subSysHelp.MultipleTargets {
-			envK := config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(madmin.EnableKey)
-			envHelp = append(envHelp, config.HelpKV{
-				Key:         envK,
-				Description: fmt.Sprintf("enable %s target, default is 'off'", subSys),
-				Optional:    false,
-				Type:        "on|off",
-			})
+	help := config.HelpKVS{}
+
+	// Only for multiple targets, make sure
+	// to list the ENV, for regular k/v EnableKey is
+	// implicit, for ENVs we cannot make it implicit.
+	if subSysHelp.MultipleTargets {
+		key := madmin.EnableKey
+		if envOnly {
+			key = config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(madmin.EnableKey)
 		}
-		for _, hkv := range h {
-			envK := config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(hkv.Key)
-			envHelp = append(envHelp, config.HelpKV{
-				Key:         envK,
-				Description: hkv.Description,
-				Optional:    hkv.Optional,
-				Type:        hkv.Type,
-			})
+		help = append(help, config.HelpKV{
+			Key:         key,
+			Description: fmt.Sprintf("enable %s target, default is 'off'", subSys),
+			Optional:    false,
+			Type:        "on|off",
+		})
+	}
+
+	for _, hkv := range h {
+		key := hkv.Key
+		if envOnly {
+			key = config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(hkv.Key)
 		}
-		h = envHelp
+		help = append(help, config.HelpKV{
+			Key:         key,
+			Description: hkv.Description,
+			Optional:    hkv.Optional,
+			Type:        hkv.Type,
+		})
 	}
 
 	return Help{
 		SubSys:          subSys,
 		Description:     subSysHelp.Description,
 		MultipleTargets: subSysHelp.MultipleTargets,
-		KeysHelp:        h,
+		KeysHelp:        help,
 	}, nil
 }
 
@@ -766,13 +741,6 @@ func newSrvConfig(objAPI ObjectLayer) error {
 	// Initialize server config.
 	srvCfg := newServerConfig()
 
-	if globalActiveCred.IsValid() && !globalActiveCred.Equal(auth.DefaultCredentials) {
-		kvs := srvCfg[config.CredentialsSubSys][config.Default]
-		kvs.Set(config.AccessKey, globalActiveCred.AccessKey)
-		kvs.Set(config.SecretKey, globalActiveCred.SecretKey)
-		srvCfg[config.CredentialsSubSys][config.Default] = kvs
-	}
-
 	// hold the mutex lock before a new config is assigned.
 	globalServerConfigMu.Lock()
 	globalServerConfig = srvCfg
@@ -783,17 +751,20 @@ func newSrvConfig(objAPI ObjectLayer) error {
 }
 
 func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
-	return readServerConfig(GlobalContext, objAPI)
+	return readServerConfig(GlobalContext, objAPI, nil)
 }
 
 // loadConfig - loads a new config from disk, overrides params
 // from env if found and valid
-func loadConfig(objAPI ObjectLayer) error {
-	srvCfg, err := getValidConfig(objAPI)
+// data is optional. If nil it will be loaded from backend.
+func loadConfig(objAPI ObjectLayer, data []byte) error {
+	bootstrapTrace("load the configuration")
+	srvCfg, err := readServerConfig(GlobalContext, objAPI, data)
 	if err != nil {
 		return err
 	}
 
+	bootstrapTrace("lookup the configuration")
 	// Override any values from ENVs.
 	lookupConfigs(srvCfg, objAPI)
 

cmd/config-current_test.go

@@ -61,7 +61,7 @@ func TestServerConfig(t *testing.T) {
 	}
 
 	// Initialize server config.
-	if err := loadConfig(objLayer); err != nil {
+	if err := loadConfig(objLayer, nil); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 }

cmd/config-encrypted.go

@@ -37,7 +37,10 @@ func handleEncryptedConfigBackend(objAPI ObjectLayer) error {
 	if err != nil {
 		return fmt.Errorf("Unable to encrypt config %w", err)
 	}
-	if err = migrateConfigPrefixToEncrypted(objAPI, encrypted); err != nil {
+	if !encrypted {
+		return nil
+	}
+	if err = migrateConfigPrefixToEncrypted(objAPI); err != nil {
 		return fmt.Errorf("Unable to migrate all config at .minio.sys/config/: %w", err)
 	}
 	return nil
@@ -48,6 +51,7 @@ const backendEncryptedFile = "backend-encrypted"
 var backendEncryptedMigrationComplete = []byte("encrypted")
 
 func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool, error) {
+	bootstrapTrace("check if etcd backend is encrypted")
 	data, err := readKeyEtcd(ctx, client, backendEncryptedFile)
 	if err != nil && err != errConfigNotFound {
 		return false, err
@@ -56,6 +60,7 @@ func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool,
 }
 
 func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) {
+	bootstrapTrace("check if the config backend is encrypted")
 	data, err := readConfig(GlobalContext, objAPI, backendEncryptedFile)
 	if err != nil && err != errConfigNotFound {
 		return false, err
@@ -75,7 +80,9 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 		return nil
 	}
 
-	if encrypted && GlobalKMS != nil {
+	bootstrapTrace("encrypt etcd config")
+
+	if GlobalKMS != nil {
 		stat, err := GlobalKMS.Stat(ctx)
 		if err != nil {
 			return err
@@ -136,18 +143,16 @@ func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client)
 		}
 	}
 
-	if encrypted && GlobalKMS != nil {
+	if GlobalKMS != nil {
 		logger.Info("Migration of encrypted IAM config data completed. All data is now encrypted on etcd.")
 	}
 
 	return deleteKeyEtcd(ctx, client, backendEncryptedFile)
 }
 
-func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
-	if !encrypted {
-		return nil
-	}
-	if encrypted && GlobalKMS != nil {
+func migrateConfigPrefixToEncrypted(objAPI ObjectLayer) error {
+	bootstrapTrace("migrating config prefix to encrypted")
+	if GlobalKMS != nil {
 		stat, err := GlobalKMS.Stat(context.Background())
 		if err != nil {
 			return err
@@ -202,7 +207,7 @@ func migrateConfigPrefixToEncrypted(objAPI ObjectLayer, encrypted bool) error {
 		}
 	}
 
-	if encrypted && GlobalKMS != nil {
+	if GlobalKMS != nil {
 		logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
 	}
 

cmd/config-migrate.go

@@ -46,7 +46,7 @@ import (
 	"github.com/minio/pkg/quick"
 )
 
-// DO NOT EDIT following message template, please open a GitHub issue to discuss instead.
+// Do not edit following message template, please open a GitHub issue to discuss instead.
 var configMigrateMSGTemplate = "Configuration file %s migrated from version '%s' to '%s' successfully."
 
 // Save config file to corresponding backend
@@ -2425,6 +2425,7 @@ func migrateV27ToV28() error {
 // Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
 // if etcd is configured then migrates /config/config.json to '<export_path>/.minio.sys/config/config.json'
 func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
+	bootstrapTrace("migrate config to .minio.sys/config/config.json")
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 
@@ -2472,6 +2473,7 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 
 // Migrates '.minio.sys/config.json' to v33.
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
+	bootstrapTrace("migrate .minio.sys/config/config.json to latest version")
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 
@@ -2513,26 +2515,34 @@ func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (
 			data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
 				minioMetaBucket: path.Join(minioMetaBucket, configFile),
 			})
-			if err != nil {
-				data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
-				if err != nil {
-					if err == madmin.ErrMaliciousData {
-						return false, nil, config.ErrInvalidCredentialsBackendEncrypted(nil)
-					}
-					return false, nil, err
-				}
-			}
-		} else {
+		}
+
+		if GlobalKMS == nil && err != nil {
 			data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
 			if err != nil {
-				if err == madmin.ErrMaliciousData {
-					return false, nil, config.ErrInvalidCredentialsBackendEncrypted(nil)
+				if err == madmin.ErrMaliciousData || err == madmin.ErrUnexpectedHeader {
+					return false, nil, config.ErrInvalidConfigDecryptionKey(nil)
 				}
 				return false, nil, err
 			}
 		}
 	}
 
+	if version == "kvs" {
+		// Check api config values are present.
+		var vcfg struct {
+			API struct {
+				E []struct {
+					Key   string `json:"key"`
+					Value string `json:"value"`
+				} `json:"_"`
+			} `json:"api"`
+		}
+		if err = json.Unmarshal(data, &vcfg); err != nil {
+			return false, nil, nil
+		}
+		return len(vcfg.API.E) > 0, data, nil
+	}
 	var versionConfig struct {
 		Version string `json:"version"`
 	}
@@ -2722,6 +2732,7 @@ func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
 }
 
 func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
+	bootstrapTrace("migrate config to KV style")
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 
 	// Check if the config version is latest, if not migrate.
@@ -2740,9 +2751,7 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
 
 	newCfg := newServerConfig()
 
-	config.SetCredentials(newCfg, cfg.Credential)
 	config.SetRegion(newCfg, cfg.Region)
-
 	storageclass.SetStorageClass(newCfg, cfg.StorageClass)
 
 	for k, loggerArgs := range cfg.Logger.HTTP {

cmd/config-migrate_test.go

@@ -22,8 +22,6 @@ import (
 	"fmt"
 	"os"
 	"testing"
-
-	"github.com/minio/minio/internal/config"
 )
 
 // Test if config v1 is purged
@@ -65,7 +63,7 @@ func TestServerConfigMigrateV1(t *testing.T) {
 	}
 
 	// Initialize server config and check again if everything is fine
-	if err := loadConfig(objLayer); err != nil {
+	if err := loadConfig(objLayer, nil); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
 }
@@ -207,20 +205,9 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 	}
 
 	// Initialize server config and check again if everything is fine
-	if err := loadConfig(objLayer); err != nil {
+	if err := loadConfig(objLayer, nil); err != nil {
 		t.Fatalf("Unable to initialize from updated config file %s", err)
 	}
-
-	// Check if accessKey and secretKey are not altered during migration
-	caccessKey := globalServerConfig[config.CredentialsSubSys][config.Default].Get(config.AccessKey)
-	if caccessKey != accessKey {
-		t.Fatalf("Access key lost during migration, expected: %v, found:%v", accessKey, caccessKey)
-	}
-
-	csecretKey := globalServerConfig[config.CredentialsSubSys][config.Default].Get(config.SecretKey)
-	if csecretKey != secretKey {
-		t.Fatalf("Secret key lost during migration, expected: %v, found: %v", secretKey, csecretKey)
-	}
 }
 
 // Test if all migrate code returns error with corrupted config files

cmd/config.go

@@ -152,30 +152,34 @@ func saveServerConfig(ctx context.Context, objAPI ObjectLayer, cfg interface{})
 	return saveConfig(ctx, objAPI, configFile, data)
 }
 
-func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, error) {
+// data is optional. If nil it will be loaded from backend.
+func readServerConfig(ctx context.Context, objAPI ObjectLayer, data []byte) (config.Config, error) {
 	srvCfg := config.New()
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	data, err := readConfig(ctx, objAPI, configFile)
-	if err != nil {
-		if errors.Is(err, errConfigNotFound) {
-			lookupConfigs(srvCfg, objAPI)
-			return srvCfg, nil
-		}
-		return nil, err
-	}
-
-	if GlobalKMS != nil && !utf8.Valid(data) {
-		data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-			minioMetaBucket: path.Join(minioMetaBucket, configFile),
-		})
+	var err error
+	if len(data) == 0 {
+		configFile := path.Join(minioConfigPrefix, minioConfigFile)
+		data, err = readConfig(ctx, objAPI, configFile)
 		if err != nil {
-			lookupConfigs(srvCfg, objAPI)
+			if errors.Is(err, errConfigNotFound) {
+				lookupConfigs(srvCfg, objAPI)
+				return srvCfg, nil
+			}
 			return nil, err
 		}
+
+		if GlobalKMS != nil && !utf8.Valid(data) {
+			data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
+				minioMetaBucket: path.Join(minioMetaBucket, configFile),
+			})
+			if err != nil {
+				lookupConfigs(srvCfg, objAPI)
+				return nil, err
+			}
+		}
 	}
 
 	json := jsoniter.ConfigCompatibleWithStandardLibrary
-	if err = json.Unmarshal(data, &srvCfg); err != nil {
+	if err := json.Unmarshal(data, &srvCfg); err != nil {
 		return nil, err
 	}
 
@@ -202,6 +206,8 @@ func NewConfigSys() *ConfigSys {
 
 // Initialize and load config from remote etcd or local config directory
 func initConfig(objAPI ObjectLayer) error {
+	bootstrapTrace("load the configuration")
+
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
@@ -212,6 +218,15 @@ func initConfig(objAPI ObjectLayer) error {
 		}
 	}
 
+	// Check if the config version is latest (kvs), if not migrate.
+	ok, data, err := checkConfigVersion(objAPI, path.Join(minioConfigPrefix, minioConfigFile), "kvs")
+	if err != nil && !errors.Is(err, errConfigNotFound) {
+		return err
+	}
+	if ok {
+		return loadConfig(objAPI, data)
+	}
+
 	// Migrates ${HOME}/.minio/config.json or config.json.deprecated
 	// to '<export_path>/.minio.sys/config/config.json'
 	// ignore if the file doesn't exist.
@@ -232,5 +247,5 @@ func initConfig(objAPI ObjectLayer) error {
 		return fmt.Errorf("migrateMinioSysConfigToKV: %w", err)
 	}
 
-	return loadConfig(objAPI)
+	return loadConfig(objAPI, nil)
 }

cmd/consolelogger.go

@@ -25,10 +25,10 @@ import (
 
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/message/log"
 	"github.com/minio/minio/internal/logger/target/console"
 	"github.com/minio/minio/internal/logger/target/types"
 	"github.com/minio/minio/internal/pubsub"
+	"github.com/minio/pkg/logger/message/log"
 	xnet "github.com/minio/pkg/net"
 )
 

cmd/crossdomain-xml-handler_test.go

@@ -22,7 +22,7 @@ import (
 	"net/http/httptest"
 	"testing"
 
-	"github.com/gorilla/mux"
+	"github.com/minio/mux"
 )
 
 // Test cross domain xml handler.

cmd/data-scanner-metric.go

@@ -69,9 +69,9 @@ const (
 
 // log scanner action.
 // Use for s > scannerMetricStartTrace
-func (p *scannerMetrics) log(s scannerMetric, paths ...string) func() {
+func (p *scannerMetrics) log(s scannerMetric, paths ...string) func(custom map[string]string) {
 	startTime := time.Now()
-	return func() {
+	return func(custom map[string]string) {
 		duration := time.Since(startTime)
 
 		atomic.AddUint64(&p.operations[s], 1)
@@ -80,7 +80,7 @@ func (p *scannerMetrics) log(s scannerMetric, paths ...string) func() {
 		}
 
 		if s > scannerMetricStartTrace && globalTrace.NumSubscribers(madmin.TraceScanner) > 0 {
-			globalTrace.Publish(scannerTrace(s, startTime, duration, strings.Join(paths, " ")))
+			globalTrace.Publish(scannerTrace(s, startTime, duration, strings.Join(paths, " "), custom))
 		}
 	}
 }

cmd/data-scanner.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"encoding/binary"
 	"encoding/json"
@@ -29,11 +28,11 @@ import (
 	"math/rand"
 	"os"
 	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/bits-and-blooms/bloom/v3"
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -59,6 +58,9 @@ const (
 	healDeleteDangling    = true
 	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
 	healObjectSelectProb  = 512 // Overall probability of a file being scanned; one in n.
+
+	dataScannerExcessiveVersionsThreshold = 1000  // Issue a warning when a single object has more versions than this
+	dataScannerExcessiveFoldersThreshold  = 50000 // Issue a warning when a folder has more subfolders than this in a *set*
 )
 
 var (
@@ -158,7 +160,6 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 
 	// Load current bloom cycle
 	var cycleInfo currentScannerCycle
-	cycleInfo.next = intDataUpdateTracker.current() + 1
 
 	buf, _ := readConfig(ctx, objAPI, dataUsageBloomNamePath)
 	if len(buf) == 8 {
@@ -203,11 +204,13 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 			// Wait before starting next cycle and wait on startup.
 			results := make(chan DataUsageInfo, 1)
 			go storeDataUsageInBackend(ctx, objAPI, results)
-			bf, err := globalNotificationSys.updateBloomFilter(ctx, cycleInfo.current)
+			err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode)
 			logger.LogIf(ctx, err)
-			err = objAPI.NSScanner(ctx, bf, results, uint32(cycleInfo.current), scanMode)
-			logger.LogIf(ctx, err)
-			stopFn()
+			res := map[string]string{"cycle": fmt.Sprint(cycleInfo.current)}
+			if err != nil {
+				res["error"] = err.Error()
+			}
+			stopFn(res)
 			if err == nil {
 				// Store new cycle...
 				cycleInfo.next++
@@ -240,10 +243,8 @@ type folderScanner struct {
 	oldCache    dataUsageCache
 	newCache    dataUsageCache
 	updateCache dataUsageCache
-	withFilter  *bloomFilter
 
 	dataUsageScannerDebug bool
-	healFolderInclude     uint32 // Include a clean folder one in n cycles.
 	healObjectSelect      uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 	scanMode              madmin.HealScanMode
 
@@ -302,8 +303,6 @@ type folderScanner struct {
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
 func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
-	logPrefix := color.Green("data-usage: ")
-
 	switch cache.Info.Name {
 	case "", dataUsageRoot:
 		return cache, errors.New("internal error: root scan attempted")
@@ -317,8 +316,7 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		oldCache:              cache,
 		newCache:              dataUsageCache{Info: cache.Info},
 		updateCache:           dataUsageCache{Info: cache.Info},
-		dataUsageScannerDebug: intDataUpdateTracker.debug,
-		healFolderInclude:     0,
+		dataUsageScannerDebug: false,
 		healObjectSelect:      0,
 		scanMode:              scanMode,
 		updates:               cache.Info.updates,
@@ -341,19 +339,9 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 
 	// Enable healing in XL mode.
 	if globalIsErasure && !cache.Info.SkipHealing {
-		// Include a clean folder one in n cycles.
-		s.healFolderInclude = healFolderIncludeProb
 		// Do a heal check on an object once every n cycles. Must divide into healFolderInclude
 		s.healObjectSelect = healObjectSelectProb
 	}
-	if len(cache.Info.BloomFilter) > 0 {
-		s.withFilter = &bloomFilter{BloomFilter: &bloom.BloomFilter{}}
-		_, err := s.withFilter.ReadFrom(bytes.NewReader(cache.Info.BloomFilter))
-		if err != nil {
-			logger.LogIf(ctx, err, logPrefix+"Error reading bloom filter")
-			s.withFilter = nil
-		}
-	}
 
 	done := ctx.Done()
 
@@ -410,14 +398,12 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			return ctx.Err()
 		default:
 		}
-		existing, ok := f.oldCache.Cache[thisHash.Key()]
 		var abandonedChildren dataUsageHashMap
 		if !into.Compacted {
 			abandonedChildren = f.oldCache.findChildrenCopy(thisHash)
 		}
 
 		// If there are lifecycle rules for the prefix, remove the filter.
-		filter := f.withFilter
 		_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 		var activeLifeCycle *lifecycle.Lifecycle
 		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix) {
@@ -425,33 +411,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix)
 			}
 			activeLifeCycle = f.oldCache.Info.lifeCycle
-			filter = nil
 		}
 		// If there are replication rules for the prefix, remove the filter.
 		var replicationCfg replicationConfig
 		if !f.oldCache.Info.replication.Empty() && f.oldCache.Info.replication.Config.HasActiveRules(prefix, true) {
 			replicationCfg = f.oldCache.Info.replication
-			filter = nil
 		}
 		// Check if we can skip it due to bloom filter...
-		if filter != nil && ok && existing.Compacted {
-			// If folder isn't in filter and we have data, skip it completely.
-			if folder.name != dataUsageRoot && !filter.containsDir(folder.name) {
-				if f.healObjectSelect == 0 || !thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
-					f.newCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
-					f.updateCache.copyWithChildren(&f.oldCache, thisHash, folder.parent)
-					if f.dataUsageScannerDebug {
-						console.Debugf(scannerLogPrefix+" Skipping non-updated folder: %v\n", folder.name)
-					}
-					return nil
-				}
-				if f.dataUsageScannerDebug {
-					console.Debugf(scannerLogPrefix+" Adding non-updated folder to heal check: %v\n", folder.name)
-				}
-				// If probability was already scannerHealFolderInclude, keep it.
-				folder.objectHealProbDiv = f.healFolderInclude
-			}
-		}
 		scannerSleeper.Sleep(ctx, dataScannerSleepPerFolder)
 
 		var existingFolders, newFolders []cachedFolder
@@ -566,6 +532,19 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders ||
 			len(existingFolders)+len(newFolders) >= dataScannerForceCompactAtFolders
 
+		if len(existingFolders)+len(newFolders) > dataScannerExcessiveFoldersThreshold {
+			// Notify object accessed via a GET request.
+			sendEvent(eventArgs{
+				EventName:  event.PrefixManyFolders,
+				BucketName: f.root,
+				Object: ObjectInfo{
+					Name: strings.TrimSuffix(folder.name, "/") + "/",
+					Size: int64(len(existingFolders) + len(newFolders)),
+				},
+				UserAgent: "scanner",
+				Host:      globalMinioHost,
+			})
+		}
 		if !into.Compacted && shouldCompact {
 			into.Compacted = true
 			newFolders = append(newFolders, existingFolders...)
@@ -632,7 +611,8 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			f.updateCurrentPath(folder.name)
 			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name)
 			scanFolder(folder)
-			stopFn()
+			stopFn(map[string]string{"type": "new"})
+
 			// Add new folders if this is new and we don't have existing.
 			if !into.Compacted {
 				parent := f.updateCache.find(thisHash.Key())
@@ -651,19 +631,16 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// and the entry itself is compacted.
 			if !into.Compacted && f.oldCache.isCompacted(h) {
 				if !h.mod(f.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) {
-					if f.healObjectSelect == 0 || !h.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healFolderInclude/folder.objectHealProbDiv) {
-						// Transfer and add as child...
-						f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
-						into.addChild(h)
-						continue
-					}
-					folder.objectHealProbDiv = f.healFolderInclude
+					// Transfer and add as child...
+					f.newCache.copyWithChildren(&f.oldCache, h, folder.parent)
+					into.addChild(h)
+					continue
 				}
 			}
 			f.updateCurrentPath(folder.name)
-			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name, "EXISTING")
+			stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name)
 			scanFolder(folder)
-			stopFn()
+			stopFn(map[string]string{"type": "existing"})
 		}
 
 		// Scan for healing
@@ -761,7 +738,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 							object:    entry.name,
 							versionID: "",
 						}, madmin.HealItemObject)
-						logger.LogIf(ctx, err)
+						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+							logger.LogIf(ctx, err)
+						}
 						foundObjs = foundObjs || err == nil
 						return
 					}
@@ -776,7 +755,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 							object:    fiv.Name,
 							versionID: ver.VersionID,
 						}, madmin.HealItemObject)
-						logger.LogIf(ctx, err)
+						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
+							logger.LogIf(ctx, err)
+						}
 						foundObjs = foundObjs || err == nil
 					}
 				},
@@ -799,7 +780,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: 1}
 				stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, this.name, "HEALED")
 				scanFolder(this)
-				stopFn()
+				stopFn(map[string]string{"type": "healed"})
 			}
 		}
 		break
@@ -813,9 +794,6 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 		flat.Compacted = true
 		var compact bool
 		if flat.Objects < dataScannerCompactLeastObject {
-			if flat.Objects > 1 {
-				globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
-			}
 			compact = true
 		} else {
 			// Compact if we only have objects as children...
@@ -828,13 +806,20 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 					}
 				}
 			}
-			if compact {
-				globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
-			}
+
 		}
 		if compact {
+			stop := globalScannerMetrics.log(scannerMetricCompactFolder, folder.name)
 			f.newCache.deleteRecursive(thisHash)
 			f.newCache.replaceHashed(thisHash, folder.parent, *flat)
+			total := map[string]string{
+				"objects": fmt.Sprint(flat.Objects),
+				"size":    fmt.Sprint(flat.Size),
+			}
+			if flat.Versions > 0 {
+				total["versions"] = fmt.Sprint(flat.Versions)
+			}
+			stop(total)
 		}
 
 	}
@@ -993,7 +978,8 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 
 	// Remove this free version
 	_, err = o.DeleteObject(ctx, oi.Bucket, oi.Name, ObjectOptions{
-		VersionID: oi.VersionID,
+		VersionID:        oi.VersionID,
+		InclFreeVersions: true,
 	})
 	if err == nil {
 		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete)
@@ -1077,8 +1063,27 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			}
 		}
 	}
+	fivs, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
+	if err != nil {
+		return fivs, err
+	}
 
-	return i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
+	// Check if we have many versions after applyNewerNoncurrentVersionLimit.
+	if len(fivs) > dataScannerExcessiveVersionsThreshold {
+		// Notify object accessed via a GET request.
+		sendEvent(eventArgs{
+			EventName:  event.ObjectManyVersions,
+			BucketName: i.bucket,
+			Object: ObjectInfo{
+				Name: i.objectPath(),
+			},
+			UserAgent:    "scanner",
+			Host:         globalMinioHost,
+			RespElements: map[string]string{"x-minio-versions": strconv.Itoa(len(fivs))},
+		})
+	}
+
+	return fivs, nil
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -1198,7 +1203,7 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		EventName:  eventName,
 		BucketName: obj.Bucket,
 		Object:     obj,
-		Host:       "Internal: [ILM-EXPIRY]",
+		Host:       "Internal: [ILM-Expiry]",
 	})
 
 	return true

cmd/data-update-tracker.go

@@ -1,682 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"encoding/binary"
-	"errors"
-	"io"
-	"os"
-	"path"
-	"sort"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/bits-and-blooms/bloom/v3"
-	"github.com/minio/minio/internal/color"
-	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/console"
-)
-
-const (
-	// Estimate bloom filter size. With this many items
-	dataUpdateTrackerEstItems = 200000
-	// ... we want this false positive rate:
-	dataUpdateTrackerFP        = 0.1
-	dataUpdateTrackerQueueSize = 0
-
-	dataUpdateTrackerFilename     = dataUsageBucket + SlashSeparator + ".tracker.bin"
-	dataUpdateTrackerVersion      = 7
-	dataUpdateTrackerSaveInterval = 5 * time.Minute
-)
-
-var intDataUpdateTracker *dataUpdateTracker
-
-func init() {
-	intDataUpdateTracker = newDataUpdateTracker()
-}
-
-type dataUpdateTracker struct {
-	mu         sync.Mutex
-	input      chan string
-	save       chan struct{}
-	debug      bool
-	saveExited chan struct{}
-	dirty      bool
-
-	Current dataUpdateFilter
-	History dataUpdateTrackerHistory
-	Saved   time.Time
-}
-
-// newDataUpdateTracker returns a dataUpdateTracker with default settings.
-func newDataUpdateTracker() *dataUpdateTracker {
-	d := &dataUpdateTracker{
-		Current: dataUpdateFilter{
-			idx: 1,
-		},
-		debug:      serverDebugLog,
-		input:      make(chan string, dataUpdateTrackerQueueSize),
-		save:       make(chan struct{}, 1),
-		saveExited: make(chan struct{}),
-	}
-	d.Current.bf = d.newBloomFilter()
-	d.dirty = true
-	return d
-}
-
-type dataUpdateTrackerHistory []dataUpdateFilter
-
-type dataUpdateFilter struct {
-	idx uint64
-	bf  bloomFilter
-}
-
-type bloomFilter struct {
-	*bloom.BloomFilter
-}
-
-// emptyBloomFilter returns an empty bloom filter.
-func emptyBloomFilter() bloomFilter {
-	return bloomFilter{BloomFilter: &bloom.BloomFilter{}}
-}
-
-// containsDir returns whether the bloom filter contains a directory.
-// Note that objects in XL mode are also considered directories.
-func (b bloomFilter) containsDir(in string) bool {
-	split := splitPathDeterministic(path.Clean(in))
-
-	if len(split) == 0 {
-		return false
-	}
-	return b.TestString(hashPath(path.Join(split...)).String())
-}
-
-// bytes returns the bloom filter serialized as a byte slice.
-func (b *bloomFilter) bytes() []byte {
-	if b == nil || b.BloomFilter == nil {
-		return nil
-	}
-	var buf bytes.Buffer
-	_, err := b.WriteTo(&buf)
-	if err != nil {
-		logger.LogIf(GlobalContext, err)
-		return nil
-	}
-	return buf.Bytes()
-}
-
-// sort the dataUpdateTrackerHistory, newest first.
-// Returns whether the history is complete.
-func (d dataUpdateTrackerHistory) sort() bool {
-	if len(d) == 0 {
-		return true
-	}
-	sort.Slice(d, func(i, j int) bool {
-		return d[i].idx > d[j].idx
-	})
-	return d[0].idx-d[len(d)-1].idx == uint64(len(d))
-}
-
-// removeOlderThan will remove entries older than index 'n'.
-func (d *dataUpdateTrackerHistory) removeOlderThan(n uint64) {
-	d.sort()
-	dd := *d
-	end := len(dd)
-	for i := end - 1; i >= 0; i-- {
-		if dd[i].idx < n {
-			end = i
-		}
-	}
-	dd = dd[:end]
-	*d = dd
-}
-
-// newBloomFilter returns a new bloom filter with default settings.
-func (d *dataUpdateTracker) newBloomFilter() bloomFilter {
-	return bloomFilter{bloom.NewWithEstimates(dataUpdateTrackerEstItems, dataUpdateTrackerFP)}
-}
-
-// current returns the current index.
-func (d *dataUpdateTracker) current() uint64 {
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	return d.Current.idx
-}
-
-// latestWithDir returns the highest index that contains the directory.
-// This means that any cycle higher than this does NOT contain the entry.
-func (d *dataUpdateTracker) latestWithDir(dir string) uint64 {
-	dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
-	bucket, _ := path2BucketObjectWithBasePath("", dir)
-	if bucket == "" {
-		if d.debug && len(dir) > 0 {
-			console.Debugf(dateUpdateTrackerLogPrefix+" no bucket (%s)\n", dir)
-		}
-		return d.current()
-	}
-	if isReservedOrInvalidBucket(bucket, false) {
-		return d.current()
-	}
-
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	if d.Current.bf.containsDir(dir) || d.Current.idx == 0 {
-		return d.Current.idx
-	}
-	if d.debug {
-		console.Debugf(dateUpdateTrackerLogPrefix+" current bloom does NOT contains dir %s\n", dir)
-	}
-
-	idx := d.Current.idx - 1
-	for {
-		f := d.History.find(idx)
-		if f == nil || f.bf.containsDir(dir) || idx == 0 {
-			break
-		}
-		idx--
-	}
-	return idx
-}
-
-// start will load the current data from the drives start collecting information and
-// start a saver goroutine.
-// All of these will exit when the context is canceled.
-func (d *dataUpdateTracker) start(ctx context.Context, drives ...string) {
-	if len(drives) == 0 {
-		logger.LogIf(ctx, errors.New("dataUpdateTracker.start: No local drives specified"))
-		return
-	}
-	d.load(ctx, drives...)
-	go d.startCollector(ctx)
-	// startSaver will unlock.
-	d.mu.Lock()
-	go d.startSaver(ctx, dataUpdateTrackerSaveInterval, drives...)
-}
-
-// load will attempt to load data tracking information from the supplied drives.
-// The data will only be loaded if d.Saved is older than the one found on disk.
-// The newest working cache will be kept in d.
-// If no valid data usage tracker can be found d will remain unchanged.
-// If object is shared the caller should lock it.
-func (d *dataUpdateTracker) load(ctx context.Context, drives ...string) {
-	if len(drives) == 0 {
-		logger.LogIf(ctx, errors.New("dataUpdateTracker.load: No local drives specified"))
-		return
-	}
-	for _, drive := range drives {
-
-		cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
-		f, err := OpenFile(cacheFormatPath, readMode, 0o666)
-		if err != nil {
-			if osIsNotExist(err) {
-				continue
-			}
-			logger.LogIf(ctx, err)
-			continue
-		}
-		err = d.deserialize(f, d.Saved)
-		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
-			logger.LogIf(ctx, err)
-		}
-		f.Close()
-	}
-}
-
-// startSaver will start a saver that will write d to all supplied drives at specific intervals.
-// 'd' must be write locked when started and will be unlocked.
-// The saver will save and exit when supplied context is closed.
-func (d *dataUpdateTracker) startSaver(ctx context.Context, interval time.Duration, drives ...string) {
-	if len(drives) == 0 {
-		return
-	}
-
-	saveNow := d.save
-	exited := make(chan struct{})
-	d.saveExited = exited
-	d.mu.Unlock()
-	t := time.NewTicker(interval)
-	defer t.Stop()
-	defer close(exited)
-	var buf bytes.Buffer
-	for {
-		var exit bool
-		select {
-		case <-ctx.Done():
-			exit = true
-		case <-t.C:
-		case <-saveNow:
-		}
-		buf.Reset()
-		d.mu.Lock()
-		if !d.dirty {
-			d.mu.Unlock()
-			if exit {
-				return
-			}
-			continue
-		}
-		d.Saved = UTCNow()
-		err := d.serialize(&buf)
-		if d.debug {
-			console.Debugf(color.Green("dataUpdateTracker:")+" Saving: %v bytes, Current idx: %v\n", buf.Len(), d.Current.idx)
-		}
-		d.dirty = false
-		d.mu.Unlock()
-		if err != nil {
-			logger.LogIf(ctx, err, "Error serializing usage tracker data")
-			if exit {
-				return
-			}
-			continue
-		}
-		if buf.Len() == 0 {
-			logger.LogIf(ctx, errors.New("zero sized output, skipping save"))
-			continue
-		}
-		for _, drive := range drives {
-			cacheFormatPath := pathJoin(drive, dataUpdateTrackerFilename)
-			err := os.WriteFile(cacheFormatPath, buf.Bytes(), os.ModePerm)
-			if err != nil {
-				if osIsNotExist(err) {
-					continue
-				}
-				logger.LogIf(ctx, err)
-				continue
-			}
-		}
-		if exit {
-			return
-		}
-	}
-}
-
-// serialize all data in d to dst.
-// Caller should hold lock if d is expected to be shared.
-// If an error is returned, there will likely be partial data written to dst.
-func (d *dataUpdateTracker) serialize(dst io.Writer) (err error) {
-	ctx := GlobalContext
-	var tmp [8]byte
-	o := bufio.NewWriter(dst)
-	defer func() {
-		if err == nil {
-			err = o.Flush()
-		}
-	}()
-
-	// Version
-	if err := o.WriteByte(dataUpdateTrackerVersion); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	// Timestamp.
-	binary.LittleEndian.PutUint64(tmp[:], uint64(d.Saved.Unix()))
-	if _, err := o.Write(tmp[:]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	// Current
-	binary.LittleEndian.PutUint64(tmp[:], d.Current.idx)
-	if _, err := o.Write(tmp[:]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	if _, err := d.Current.bf.WriteTo(o); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	// History
-	binary.LittleEndian.PutUint64(tmp[:], uint64(len(d.History)))
-	if _, err := o.Write(tmp[:]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	for _, bf := range d.History {
-		// Current
-		binary.LittleEndian.PutUint64(tmp[:], bf.idx)
-		if _, err := o.Write(tmp[:]); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-
-		if _, err := bf.bf.WriteTo(o); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-	}
-	return nil
-}
-
-// deserialize will deserialize the supplied input if the input is newer than the supplied time.
-func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) error {
-	ctx := GlobalContext
-	var dst dataUpdateTracker
-	var tmp [8]byte
-
-	// Version
-	if _, err := io.ReadFull(src, tmp[:1]); err != nil {
-		if d.debug {
-			if err != io.EOF {
-				logger.LogIf(ctx, err)
-			}
-		}
-		return err
-	}
-	switch tmp[0] {
-	case 1, 2, 3, 4, 5, 6:
-		if intDataUpdateTracker.debug {
-			console.Debugln(color.Green("dataUpdateTracker: ") + "deprecated data version, updating.")
-		}
-		return nil
-	case dataUpdateTrackerVersion:
-	default:
-		return errors.New("dataUpdateTracker: Unknown data version")
-	}
-	// Timestamp.
-	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	t := time.Unix(int64(binary.LittleEndian.Uint64(tmp[:])), 0)
-	if !t.After(newerThan) {
-		return nil
-	}
-
-	// Current
-	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	dst.Current.idx = binary.LittleEndian.Uint64(tmp[:])
-	dst.Current.bf = emptyBloomFilter()
-	if _, err := dst.Current.bf.ReadFrom(src); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-
-	// History
-	if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-		if d.debug {
-			logger.LogIf(ctx, err)
-		}
-		return err
-	}
-	n := binary.LittleEndian.Uint64(tmp[:])
-	dst.History = make(dataUpdateTrackerHistory, int(n))
-	for i, e := range dst.History {
-		if _, err := io.ReadFull(src, tmp[:8]); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-		e.idx = binary.LittleEndian.Uint64(tmp[:])
-		e.bf = emptyBloomFilter()
-		if _, err := e.bf.ReadFrom(src); err != nil {
-			if d.debug {
-				logger.LogIf(ctx, err)
-			}
-			return err
-		}
-		dst.History[i] = e
-	}
-	// Ignore what remains on the stream.
-	// Update d:
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	d.Current = dst.Current
-	d.History = dst.History
-	d.Saved = dst.Saved
-	return nil
-}
-
-// start a collector that picks up entries from objectUpdatedCh
-// and adds them  to the current bloom filter.
-func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case in := <-d.input:
-			bucket, _ := path2BucketObjectWithBasePath("", in)
-			if bucket == "" {
-				if d.debug && len(in) > 0 {
-					console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
-				}
-				continue
-			}
-
-			if isReservedOrInvalidBucket(bucket, false) {
-				continue
-			}
-			split := splitPathDeterministic(in)
-
-			// Add all paths until done.
-			d.mu.Lock()
-			for i := range split {
-				d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
-			}
-			d.dirty = d.dirty || len(split) > 0
-			d.mu.Unlock()
-		}
-	}
-}
-
-// markDirty adds the supplied path to the current bloom filter.
-func (d *dataUpdateTracker) markDirty(bucket, prefix string) {
-	dateUpdateTrackerLogPrefix := color.Green("dataUpdateTracker:")
-	if bucket == "" && d.debug {
-		console.Debugf(dateUpdateTrackerLogPrefix + " no bucket specified\n")
-		return
-	}
-
-	if isReservedOrInvalidBucket(bucket, false) && d.debug {
-		return
-	}
-	split := splitPathDeterministic(pathJoin(bucket, prefix))
-
-	// Add all paths until done.
-	d.mu.Lock()
-	for i := range split {
-		d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
-	}
-	d.dirty = d.dirty || len(split) > 0
-	d.mu.Unlock()
-}
-
-// find entry with specified index.
-// Returns nil if not found.
-func (d dataUpdateTrackerHistory) find(idx uint64) *dataUpdateFilter {
-	for _, f := range d {
-		if f.idx == idx {
-			return &f
-		}
-	}
-	return nil
-}
-
-// filterFrom will return a combined bloom filter.
-func (d *dataUpdateTracker) filterFrom(ctx context.Context, oldest, newest uint64) *bloomFilterResponse {
-	bf := d.newBloomFilter()
-	bfr := bloomFilterResponse{
-		OldestIdx:  oldest,
-		CurrentIdx: d.Current.idx,
-		Complete:   true,
-	}
-	// Loop through each index requested.
-	for idx := oldest; idx <= newest; idx++ {
-		v := d.History.find(idx)
-		if v == nil {
-			if d.Current.idx == idx {
-				// Merge current.
-				err := bf.Merge(d.Current.bf.BloomFilter)
-				logger.LogIf(ctx, err)
-				if err != nil {
-					bfr.Complete = false
-				}
-				continue
-			}
-			bfr.Complete = false
-			bfr.OldestIdx = idx + 1
-			continue
-		}
-
-		err := bf.Merge(v.bf.BloomFilter)
-		if err != nil {
-			bfr.Complete = false
-			logger.LogIf(ctx, err)
-			continue
-		}
-		bfr.NewestIdx = idx
-	}
-	var dst bytes.Buffer
-	_, err := bf.WriteTo(&dst)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return nil
-	}
-	bfr.Filter = dst.Bytes()
-
-	return &bfr
-}
-
-// cycleFilter will cycle the bloom filter to start recording to index y if not already.
-// The response will contain a bloom filter starting at index x up to, but not including index y.
-// If y is 0, the response will not update y, but return the currently recorded information
-// from the oldest (unless 0, then it will be all) until and including current y.
-func (d *dataUpdateTracker) cycleFilter(ctx context.Context, req bloomFilterRequest) (*bloomFilterResponse, error) {
-	if req.OldestClean != "" {
-		return &bloomFilterResponse{OldestIdx: d.latestWithDir(req.OldestClean)}, nil
-	}
-	current := req.Current
-	oldest := req.Oldest
-	d.mu.Lock()
-	defer d.mu.Unlock()
-	if current == 0 {
-		if len(d.History) == 0 {
-			return d.filterFrom(ctx, d.Current.idx, d.Current.idx), nil
-		}
-		d.History.sort()
-		if oldest == 0 {
-			oldest = d.History[len(d.History)-1].idx
-		}
-		return d.filterFrom(ctx, oldest, d.Current.idx), nil
-	}
-
-	// Move current to history if new one requested
-	if d.Current.idx != current {
-		d.dirty = true
-		if d.debug {
-			console.Debugf(color.Green("dataUpdateTracker:")+" cycle bloom filter: %v -> %v\n", d.Current.idx, current)
-		}
-
-		d.History = append(d.History, d.Current)
-		d.Current.idx = current
-		d.Current.bf = d.newBloomFilter()
-		select {
-		case d.save <- struct{}{}:
-		default:
-		}
-	}
-	d.History.removeOlderThan(oldest)
-	return d.filterFrom(ctx, oldest, current), nil
-}
-
-// splitPathDeterministic will split the provided relative path
-// deterministically and return up to the first 3 elements of the path.
-// slash and dot prefixes are removed.
-// Trailing slashes are removed.
-// Returns 0 length if no parts are found after trimming.
-func splitPathDeterministic(in string) []string {
-	split := strings.Split(decodeDirObject(in), SlashSeparator)
-
-	// Trim empty start/end
-	for len(split) > 0 {
-		if len(split[0]) > 0 && split[0] != "." {
-			break
-		}
-		split = split[1:]
-	}
-	for len(split) > 0 {
-		if len(split[len(split)-1]) > 0 {
-			break
-		}
-		split = split[:len(split)-1]
-	}
-
-	return split
-}
-
-// bloomFilterRequest request bloom filters.
-// Current index will be updated to current and entries back to Oldest is returned.
-type bloomFilterRequest struct {
-	Oldest  uint64
-	Current uint64
-	// If set the oldest clean version will be returned in OldestIdx
-	// and the rest of the request will be ignored.
-	OldestClean string
-}
-
-type bloomFilterResponse struct {
-	// Current index being written to.
-	CurrentIdx uint64
-	// Oldest index in the returned bloom filter.
-	OldestIdx uint64
-	// Newest Index in the returned bloom filter.
-	NewestIdx uint64
-	// Are all indexes between oldest and newest filled?
-	Complete bool
-	// Binary data of the bloom filter.
-	Filter []byte
-}
-
-// NSUpdated indicates namespace has been updated.
-// The function will block until the entry has been picked up.
-func NSUpdated(bucket, prefix string) {
-	if intDataUpdateTracker != nil {
-		intDataUpdateTracker.markDirty(bucket, prefix)
-	}
-}

cmd/data-update-tracker_test.go

@@ -1,299 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"math/rand"
-	"os"
-	"path"
-	"path/filepath"
-	"sync"
-	"testing"
-
-	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/logger/message/log"
-	"github.com/minio/minio/internal/logger/target/types"
-)
-
-type testLoggerI interface {
-	Helper()
-	Log(args ...interface{})
-}
-
-type testingLogger struct {
-	mu sync.Mutex
-	t  testLoggerI
-}
-
-func (t *testingLogger) Endpoint() string {
-	return ""
-}
-
-func (t *testingLogger) String() string {
-	return ""
-}
-
-func (t *testingLogger) Init() error {
-	return nil
-}
-
-func (t *testingLogger) Cancel() {
-}
-
-func (t *testingLogger) Type() types.TargetType {
-	return types.TargetHTTP
-}
-
-func (t *testingLogger) IsOnline() bool {
-	return true
-}
-
-// Stats returns the target statistics.
-func (t *testingLogger) Stats() types.TargetStats {
-	return types.TargetStats{}
-}
-
-func (t *testingLogger) Send(entry interface{}) error {
-	t.mu.Lock()
-	defer t.mu.Unlock()
-	if t.t == nil {
-		return nil
-	}
-	e, ok := entry.(log.Entry)
-	if !ok {
-		return fmt.Errorf("unexpected log entry structure %#v", entry)
-	}
-
-	t.t.Helper()
-	t.t.Log(e.Level, ":", e.Message)
-	return nil
-}
-
-func addTestingLogging(t testLoggerI) func() {
-	tl := &testingLogger{t: t}
-	logger.AddSystemTarget(tl)
-	return func() {
-		tl.mu.Lock()
-		defer tl.mu.Unlock()
-		tl.t = nil
-	}
-}
-
-func TestDataUpdateTracker(t *testing.T) {
-	dut := newDataUpdateTracker()
-	// Change some defaults.
-	dut.debug = testing.Verbose()
-	dut.input = make(chan string)
-	dut.save = make(chan struct{})
-
-	defer addTestingLogging(t)()
-
-	dut.Current.bf = dut.newBloomFilter()
-
-	tmpDir := t.TempDir()
-	err := os.MkdirAll(filepath.Dir(filepath.Join(tmpDir, dataUpdateTrackerFilename)), os.ModePerm)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	dut.start(ctx, tmpDir)
-
-	tests := []struct {
-		in    string
-		check []string // if not empty, check against these instead.
-		exist bool
-	}{
-		{
-			in:    "bucket/directory/file.txt",
-			check: []string{"bucket", "bucket/", "/bucket", "bucket/directory", "bucket/directory/", "bucket/directory/file.txt", "/bucket/directory/file.txt"},
-			exist: true,
-		},
-		{
-			// System bucket
-			in:    ".minio.sys/ignoreme/pls",
-			exist: false,
-		},
-		{
-			// Not a valid bucket
-			in:    "./bucket/okfile.txt",
-			check: []string{"./bucket/okfile.txt", "/bucket/okfile.txt", "bucket/okfile.txt"},
-			exist: false,
-		},
-		{
-			// Not a valid bucket
-			in:    "æ/okfile.txt",
-			check: []string{"æ/okfile.txt", "æ/okfile.txt", "æ"},
-			exist: false,
-		},
-		{
-			in:    "/bucket2/okfile2.txt",
-			check: []string{"./bucket2/okfile2.txt", "/bucket2/okfile2.txt", "bucket2/okfile2.txt", "bucket2"},
-			exist: true,
-		},
-		{
-			in:    "/bucket3/prefix/okfile2.txt",
-			check: []string{"./bucket3/prefix/okfile2.txt", "/bucket3/prefix/okfile2.txt", "bucket3/prefix/okfile2.txt", "bucket3/prefix", "bucket3"},
-			exist: true,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.in, func(t *testing.T) {
-			dut.input <- tt.in
-			dut.input <- "" // Sending empty string ensures the previous is added to filter.
-			dut.mu.Lock()
-			defer dut.mu.Unlock()
-			if len(tt.check) == 0 {
-				got := dut.Current.bf.containsDir(tt.in)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
-				}
-				return
-			}
-			for _, check := range tt.check {
-				got := dut.Current.bf.containsDir(check)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
-				}
-				continue
-			}
-		})
-	}
-	// Cycle to history
-	req := bloomFilterRequest{
-		Oldest:  1,
-		Current: 2,
-	}
-
-	_, err = dut.cycleFilter(ctx, req)
-	if err != nil {
-		t.Fatal(err)
-	}
-	dut.input <- "cycle2/file.txt"
-	dut.input <- "" // Sending empty string ensures the previous is added to filter.
-
-	tests = append(tests, struct {
-		in    string
-		check []string
-		exist bool
-	}{in: "cycle2/file.txt", exist: true})
-
-	// Shut down
-	cancel()
-	<-dut.saveExited
-
-	if dut.current() != 2 {
-		t.Fatal("wrong current idx after save. want 2, got:", dut.current())
-	}
-
-	ctx, cancel = context.WithCancel(context.Background())
-	// Reload...
-	dut = newDataUpdateTracker()
-	dut.start(ctx, tmpDir)
-	defer func() {
-		cancel()
-		<-dut.saveExited
-	}()
-
-	if dut.current() != 2 {
-		t.Fatal("current idx after load not preserved. want 2, got:", dut.current())
-	}
-	req = bloomFilterRequest{
-		Oldest:  1,
-		Current: 3,
-	}
-	bfr2, err := dut.cycleFilter(ctx, req)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bfr2.Complete {
-		t.Fatal("Wanted complete, didn't get it")
-	}
-	if bfr2.CurrentIdx != 3 {
-		t.Fatal("wanted index 3, got", bfr2.CurrentIdx)
-	}
-	if bfr2.OldestIdx != 1 {
-		t.Fatal("wanted oldest index 3, got", bfr2.OldestIdx)
-	}
-
-	t.Logf("Size of filter %d bytes, M: %d, K:%d", len(bfr2.Filter), dut.Current.bf.Cap(), dut.Current.bf.K())
-	// Rerun test with returned bfr2
-	bf := dut.newBloomFilter()
-	_, err = bf.ReadFrom(bytes.NewReader(bfr2.Filter))
-	if err != nil {
-		t.Fatal(err)
-	}
-	for _, tt := range tests {
-		t.Run(tt.in+"-reloaded", func(t *testing.T) {
-			if len(tt.check) == 0 {
-				got := bf.containsDir(tt.in)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, got: %v, want %v", tt.in, got, tt.exist)
-				}
-				return
-			}
-			for _, check := range tt.check {
-				got := bf.containsDir(check)
-				if got != tt.exist {
-					// For unlimited tests this could lead to false positives,
-					// but it should be deterministic.
-					t.Errorf("entry %q, check: %q, got: %v, want %v", tt.in, check, got, tt.exist)
-				}
-				continue
-			}
-		})
-	}
-}
-
-func BenchmarkDataUpdateTracker(b *testing.B) {
-	dut := newDataUpdateTracker()
-	// Change some defaults.
-	dut.debug = false
-	dut.input = make(chan string)
-	dut.save = make(chan struct{})
-
-	defer addTestingLogging(b)()
-
-	dut.Current.bf = dut.newBloomFilter()
-	// We do this unbuffered. This will very significantly reduce throughput, so this is a worst case.
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	go dut.startCollector(ctx)
-	input := make([]string, 1000)
-	rng := rand.New(rand.NewSource(0xabad1dea))
-	tmp := []string{"bucket", "aprefix", "nextprefixlevel", "maybeobjname", "evendeeper", "ok-one-morelevel", "final.object"}
-	for i := range input {
-		tmp := tmp[:1+rng.Intn(cap(tmp)-1)]
-		input[i] = path.Join(tmp...)
-	}
-	b.SetBytes(1)
-	b.ResetTimer()
-	b.ReportAllocs()
-	for i := 0; i < b.N; i++ {
-		dut.input <- input[rng.Intn(len(input))]
-	}
-}

cmd/data-usage-cache.go

@@ -276,7 +276,6 @@ type dataUsageCacheInfo struct {
 	// indicates if the disk is being healed and scanner
 	// should skip healing the disk
 	SkipHealing bool
-	BloomFilter []byte `msg:"BloomFilter,omitempty"`
 
 	// Active lifecycle, if any on the bucket
 	lifeCycle *lifecycle.Lifecycle `msg:"-"`
@@ -649,10 +648,7 @@ func (d *dataUsageCache) reduceChildrenOf(path dataUsageHash, limit int, compact
 // StringAll returns a detailed string representation of all entries in the cache.
 func (d *dataUsageCache) StringAll() string {
 	// Remove bloom filter from print.
-	bf := d.Info.BloomFilter
-	d.Info.BloomFilter = nil
 	s := fmt.Sprintf("info:%+v\n", d.Info)
-	d.Info.BloomFilter = bf
 	for k, v := range d.Cache {
 		s += fmt.Sprintf("\t%v: %+v\n", k, v)
 	}

cmd/data-usage-cache_gen.go

@@ -629,12 +629,6 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "SkipHealing")
 				return
 			}
-		case "BloomFilter":
-			z.BloomFilter, err = dc.ReadBytes(z.BloomFilter)
-			if err != nil {
-				err = msgp.WrapError(err, "BloomFilter")
-				return
-			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -648,24 +642,9 @@ func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
-	// omitempty: check for empty values
-	zb0001Len := uint32(5)
-	var zb0001Mask uint8 /* 5 bits */
-	_ = zb0001Mask
-	if z.BloomFilter == nil {
-		zb0001Len--
-		zb0001Mask |= 0x10
-	}
-	// variable map header, size zb0001Len
-	err = en.Append(0x80 | uint8(zb0001Len))
-	if err != nil {
-		return
-	}
-	if zb0001Len == 0 {
-		return
-	}
+	// map header, size 4
 	// write "Name"
-	err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -704,39 +683,15 @@ func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "SkipHealing")
 		return
 	}
-	if (zb0001Mask & 0x10) == 0 { // if not empty
-		// write "BloomFilter"
-		err = en.Append(0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
-		if err != nil {
-			return
-		}
-		err = en.WriteBytes(z.BloomFilter)
-		if err != nil {
-			err = msgp.WrapError(err, "BloomFilter")
-			return
-		}
-	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// omitempty: check for empty values
-	zb0001Len := uint32(5)
-	var zb0001Mask uint8 /* 5 bits */
-	_ = zb0001Mask
-	if z.BloomFilter == nil {
-		zb0001Len--
-		zb0001Mask |= 0x10
-	}
-	// variable map header, size zb0001Len
-	o = append(o, 0x80|uint8(zb0001Len))
-	if zb0001Len == 0 {
-		return
-	}
+	// map header, size 4
 	// string "Name"
-	o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "NextCycle"
 	o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65)
@@ -747,11 +702,6 @@ func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "SkipHealing"
 	o = append(o, 0xab, 0x53, 0x6b, 0x69, 0x70, 0x48, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67)
 	o = msgp.AppendBool(o, z.SkipHealing)
-	if (zb0001Mask & 0x10) == 0 { // if not empty
-		// string "BloomFilter"
-		o = append(o, 0xab, 0x42, 0x6c, 0x6f, 0x6f, 0x6d, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
-		o = msgp.AppendBytes(o, z.BloomFilter)
-	}
 	return
 }
 
@@ -797,12 +747,6 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "SkipHealing")
 				return
 			}
-		case "BloomFilter":
-			z.BloomFilter, bts, err = msgp.ReadBytesBytes(bts, z.BloomFilter)
-			if err != nil {
-				err = msgp.WrapError(err, "BloomFilter")
-				return
-			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -817,7 +761,7 @@ func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageCacheInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.Uint32Size + 11 + msgp.TimeSize + 12 + msgp.BoolSize + 12 + msgp.BytesPrefixSize + len(z.BloomFilter)
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.Uint32Size + 11 + msgp.TimeSize + 12 + msgp.BoolSize
 	return
 }
 

cmd/disk-cache-backend.go

@@ -512,7 +512,7 @@ func (c *diskCache) statCachedMeta(ctx context.Context, cacheObjPath string) (me
 		return
 	}
 	ctx = lkctx.Context()
-	defer cLock.RUnlock(lkctx.Cancel)
+	defer cLock.RUnlock(lkctx)
 	return c.statCache(ctx, cacheObjPath)
 }
 
@@ -597,7 +597,7 @@ func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, met
 		return err
 	}
 	ctx = lkctx.Context()
-	defer cLock.Unlock(lkctx.Cancel)
+	defer cLock.Unlock(lkctx)
 	if err = c.saveMetadata(ctx, bucket, object, meta, actualSize, rs, rsFileName, incHitsOnly); err != nil {
 		return err
 	}
@@ -842,7 +842,7 @@ func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Read
 		return oi, err
 	}
 	ctx = lkctx.Context()
-	defer cLock.Unlock(lkctx.Cancel)
+	defer cLock.Unlock(lkctx)
 
 	return c.put(ctx, bucket, object, data, size, rs, opts, incHitsOnly, writeback)
 }
@@ -1076,7 +1076,7 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 		return nil, numHits, err
 	}
 	ctx = lkctx.Context()
-	defer cLock.RUnlock(lkctx.Cancel)
+	defer cLock.RUnlock(lkctx)
 
 	var objInfo ObjectInfo
 	var rngInfo RangeInfo
@@ -1213,7 +1213,7 @@ func (c *diskCache) Delete(ctx context.Context, bucket, object string) (err erro
 	if err != nil {
 		return err
 	}
-	defer cLock.Unlock(lkctx.Cancel)
+	defer cLock.Unlock(lkctx)
 	return removeAll(cacheObjPath)
 }
 
@@ -1324,7 +1324,7 @@ func (c *diskCache) PutObjectPart(ctx context.Context, bucket, object, uploadID
 	}
 
 	ctx = lkctx.Context()
-	defer partIDLock.Unlock(lkctx.Cancel)
+	defer partIDLock.Unlock(lkctx)
 	meta, _, _, err := c.statCache(ctx, uploadIDDir)
 	// Case where object not yet cached
 	if err != nil {
@@ -1381,7 +1381,7 @@ func (c *diskCache) SavePartMetadata(ctx context.Context, bucket, object, upload
 	if err != nil {
 		return err
 	}
-	defer uploadLock.Unlock(ulkctx.Cancel)
+	defer uploadLock.Unlock(ulkctx)
 
 	metaPath := pathJoin(uploadDir, cacheMetaJSONFile)
 	f, err := OpenFile(metaPath, os.O_RDWR|writeMode, 0o666)
@@ -1480,7 +1480,7 @@ func (c *diskCache) CompleteMultipartUpload(ctx context.Context, bucket, object,
 	}
 
 	ctx = lkctx.Context()
-	defer cLock.Unlock(lkctx.Cancel)
+	defer cLock.Unlock(lkctx)
 	mpartCachePath := getMultipartCacheSHADir(c.dir, bucket, object)
 	uploadIDDir := path.Join(mpartCachePath, uploadID)
 

cmd/disk-cache.go

@@ -728,7 +728,7 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 	if cerr != nil {
 		return putObjectFn(ctx, bucket, object, r, opts)
 	}
-	defer cLock.Unlock(lkctx.Cancel)
+	defer cLock.Unlock(lkctx)
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
 	hashReader, err := hash.NewReader(pipeReader, size, "", "", r.ActualSize())

cmd/dummy-handlers.go

@@ -20,8 +20,8 @@ package cmd
 import (
 	"net/http"
 
-	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"
 )
 

cmd/encryption-v1.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -34,7 +34,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/minio/kes"
+	"github.com/minio/kes-go"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/etag"
 	"github.com/minio/minio/internal/fips"
@@ -48,14 +48,16 @@ import (
 
 var (
 	// AWS errors for invalid SSE-C requests.
-	errEncryptedObject      = errors.New("The object was stored using a form of SSE")
-	errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
-	errKMSNotConfigured     = errors.New("KMS not configured for a server side encrypted object")
-	errKMSKeyNotFound       = errors.New("Invalid KMS keyId")
+	errEncryptedObject                = errors.New("The object was stored using a form of SSE")
+	errInvalidSSEParameters           = errors.New("The SSE-C key for key-rotation is not correct") // special access denied
+	errKMSNotConfigured               = errors.New("KMS not configured for a server side encrypted objects")
+	errKMSKeyNotFound                 = errors.New("Unknown KMS key ID")
+	errKMSDefaultKeyAlreadyConfigured = errors.New("A default encryption already exists on KMS")
 	// Additional MinIO errors for SSE-C requests.
 	errObjectTampered = errors.New("The requested object was modified and may be compromised")
 	// error returned when invalid encryption parameters are specified
-	errInvalidEncryptionParameters = errors.New("The encryption parameters are not applicable to this object")
+	errInvalidEncryptionParameters     = errors.New("The encryption parameters are not applicable to this object")
+	errInvalidEncryptionParametersSSEC = errors.New("SSE-C encryption parameters are not supported on this bucket")
 )
 
 const (
@@ -1083,6 +1085,44 @@ func (o *ObjectInfo) metadataDecrypter() objectMetaDecryptFn {
 	}
 }
 
+// metadataEncryptFn provides an encryption function for metadata.
+// Will return nil, nil if unencrypted.
+func (o *ObjectInfo) metadataEncryptFn(headers http.Header) (objectMetaEncryptFn, error) {
+	kind, _ := crypto.IsEncrypted(o.UserDefined)
+	switch kind {
+	case crypto.SSEC:
+		if crypto.SSECopy.IsRequested(headers) {
+			key, err := crypto.SSECopy.ParseHTTP(headers)
+			if err != nil {
+				return nil, err
+			}
+			objectEncryptionKey, err := decryptObjectMeta(key[:], o.Bucket, o.Name, o.UserDefined)
+			if err != nil {
+				return nil, err
+			}
+			if len(objectEncryptionKey) == 32 {
+				var key crypto.ObjectKey
+				copy(key[:], objectEncryptionKey)
+				return metadataEncrypter(key), nil
+			}
+			return nil, errors.New("metadataEncryptFn: unexpected key size")
+		}
+	case crypto.S3, crypto.S3KMS:
+		objectEncryptionKey, err := decryptObjectMeta(nil, o.Bucket, o.Name, o.UserDefined)
+		if err != nil {
+			return nil, err
+		}
+		if len(objectEncryptionKey) == 32 {
+			var key crypto.ObjectKey
+			copy(key[:], objectEncryptionKey)
+			return metadataEncrypter(key), nil
+		}
+		return nil, errors.New("metadataEncryptFn: unexpected key size")
+	}
+
+	return nil, nil
+}
+
 // decryptChecksums will attempt to decode checksums and return it/them if set.
 func (o *ObjectInfo) decryptChecksums() map[string]string {
 	data := o.Checksum

cmd/endpoint-ellipses.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"fmt"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -364,6 +365,7 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
 			CmdLine:      strings.Join(args, " "),
+			Platform:     fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH),
 		})
 		setupType = newSetupType
 		return endpointServerPools, setupType, nil
@@ -389,6 +391,7 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			DrivesPerSet: len(setArgs[0]),
 			Endpoints:    endpointList,
 			CmdLine:      arg,
+			Platform:     fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH),
 		}); err != nil {
 			return nil, -1, err
 		}

cmd/endpoint.go

@@ -203,6 +203,7 @@ type PoolEndpoints struct {
 	DrivesPerSet int
 	Endpoints    Endpoints
 	CmdLine      string
+	Platform     string
 }
 
 // EndpointServerPools - list of list of endpoints
@@ -417,6 +418,27 @@ func (endpoints Endpoints) atleastOneEndpointLocal() bool {
 	return false
 }
 
+func hostResolveToLocalhost(endpoint Endpoint) bool {
+	hostIPs, err := getHostIP(endpoint.Hostname())
+	if err != nil {
+		// Log the message to console about the host resolving
+		reqInfo := (&logger.ReqInfo{}).AppendTags(
+			"host",
+			endpoint.Hostname(),
+		)
+		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
+		logger.LogOnceIf(ctx, err, endpoint.Hostname(), logger.Application)
+		return false
+	}
+	var loopback int
+	for _, hostIP := range hostIPs.ToSlice() {
+		if net.ParseIP(hostIP).IsLoopback() {
+			loopback++
+		}
+	}
+	return loopback == len(hostIPs)
+}
+
 // UpdateIsLocal - resolves the host and discovers the local host.
 func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 	orchestrated := IsDocker() || IsKubernetes()
@@ -452,6 +474,26 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 					endpoints[i].Hostname(),
 				)
 
+				if orchestrated && hostResolveToLocalhost(endpoints[i]) {
+					// time elapsed
+					timeElapsed := time.Since(startTime)
+					// log error only if more than a second has elapsed
+					if timeElapsed > time.Second {
+						reqInfo.AppendTags("elapsedTime",
+							humanize.RelTime(startTime,
+								startTime.Add(timeElapsed),
+								"elapsed",
+								"",
+							))
+						ctx := logger.SetReqInfo(GlobalContext,
+							reqInfo)
+						logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
+							endpoints[i].Hostname()), endpoints[i].Hostname(), logger.Application)
+					}
+
+					continue
+				}
+
 				// return err if not Docker or Kubernetes
 				// We use IsDocker() to check for Docker environment
 				// We use IsKubernetes() to check for Kubernetes environment
@@ -465,7 +507,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 				if err != nil {
 					// time elapsed
 					timeElapsed := time.Since(startTime)
-					// log error only if more than 1s elapsed
+					// log error only if more than a second has elapsed
 					if timeElapsed > time.Second {
 						reqInfo.AppendTags("elapsedTime",
 							humanize.RelTime(startTime,
@@ -475,7 +517,7 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 							))
 						ctx := logger.SetReqInfo(GlobalContext,
 							reqInfo)
-						logger.LogIf(ctx, err, logger.Application)
+						logger.LogOnceIf(ctx, err, endpoints[i].Hostname(), logger.Application)
 					}
 				} else {
 					resolvedList[i] = true

cmd/erasure-bucket.go

@@ -1,277 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"errors"
-
-	"github.com/minio/minio-go/v7/pkg/s3utils"
-	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
-)
-
-// list all errors that can be ignore in a bucket operation.
-var bucketOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errUnformattedDisk)
-
-// list all errors that can be ignored in a bucket metadata operation.
-var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound)
-
-// Bucket operations
-
-// MakeBucket - make a bucket.
-func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
-	defer NSUpdated(bucket, slashSeparator)
-
-	// Verify if bucket is valid.
-	if !isMinioMetaBucketName(bucket) {
-		if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
-			return BucketNameInvalid{Bucket: bucket}
-		}
-	}
-
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	// Make a volume entry on all underlying storage disks.
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if storageDisks[index] != nil {
-				if err := storageDisks[index].MakeVol(ctx, bucket); err != nil {
-					if opts.ForceCreate && errors.Is(err, errVolumeExists) {
-						// No need to return error when force create was
-						// requested.
-						return nil
-					}
-					if !errors.Is(err, errVolumeExists) {
-						logger.LogIf(ctx, err)
-					}
-					return err
-				}
-				return nil
-			}
-			return errDiskNotFound
-		}, index)
-	}
-
-	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
-	return toObjectErr(err, bucket)
-}
-
-func undoDeleteBucket(storageDisks []StorageAPI, bucket string) {
-	g := errgroup.WithNErrs(len(storageDisks))
-	// Undo previous make bucket entry on all underlying storage disks.
-	for index := range storageDisks {
-		if storageDisks[index] == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			_ = storageDisks[index].MakeVol(context.Background(), bucket)
-			return nil
-		}, index)
-	}
-
-	// Wait for all make vol to finish.
-	g.Wait()
-}
-
-// getBucketInfo - returns the BucketInfo from one of the load balanced disks.
-func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-	bucketsInfo := make([]BucketInfo, len(storageDisks))
-	// Undo previous make bucket entry on all underlying storage disks.
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if storageDisks[index] == nil {
-				return errDiskNotFound
-			}
-			volInfo, err := storageDisks[index].StatVol(ctx, bucketName)
-			if err != nil {
-				if opts.Deleted {
-					dvi, derr := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucketName))
-					if derr != nil {
-						return err
-					}
-					bucketsInfo[index] = BucketInfo{Name: bucketName, Deleted: dvi.Created}
-					return nil
-				}
-				return err
-			}
-			bucketsInfo[index] = BucketInfo{Name: volInfo.Name, Created: volInfo.Created}
-			return nil
-		}, index)
-	}
-
-	errs := g.Wait()
-
-	for i, err := range errs {
-		if err == nil {
-			return bucketsInfo[i], nil
-		}
-	}
-
-	// If all our errors were ignored, then we try to
-	// reduce to one error based on read quorum.
-	// `nil` is deliberately passed for ignoredErrs
-	// because these errors were already ignored.
-	return BucketInfo{}, reduceReadQuorumErrs(ctx, errs, nil, er.defaultRQuorum())
-}
-
-// GetBucketInfo - returns BucketInfo for a bucket.
-func (er erasureObjects) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bi BucketInfo, e error) {
-	bucketInfo, err := er.getBucketInfo(ctx, bucket, opts)
-	if err != nil {
-		return bi, toObjectErr(err, bucket)
-	}
-	return bucketInfo, nil
-}
-
-// DeleteBucket - deletes a bucket.
-func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
-	// Collect if all disks report volume not found.
-	defer NSUpdated(bucket, slashSeparator)
-
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if storageDisks[index] != nil {
-				return storageDisks[index].DeleteVol(ctx, bucket, opts.Force)
-			}
-			return errDiskNotFound
-		}, index)
-	}
-
-	// Wait for all the delete vols to finish.
-	dErrs := g.Wait()
-
-	if opts.Force {
-		for _, err := range dErrs {
-			if err != nil {
-				undoDeleteBucket(storageDisks, bucket)
-				return toObjectErr(err, bucket)
-			}
-		}
-
-		return nil
-	}
-
-	err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, er.defaultWQuorum())
-	if err == errErasureWriteQuorum && !opts.NoRecreate {
-		undoDeleteBucket(storageDisks, bucket)
-	}
-
-	if err == nil || errors.Is(err, errVolumeNotFound) {
-		var purgedDangling bool
-		// At this point we have `err == nil` but some errors might be `errVolumeNotEmpty`
-		// we should proceed to attempt a force delete of such buckets.
-		for index, err := range dErrs {
-			if err == errVolumeNotEmpty && storageDisks[index] != nil {
-				storageDisks[index].RenameFile(ctx, bucket, "", minioMetaTmpDeletedBucket, mustGetUUID())
-				purgedDangling = true
-			}
-		}
-		// if we purged dangling buckets, ignore errVolumeNotFound error.
-		if purgedDangling {
-			err = nil
-		}
-		if opts.SRDeleteOp == MarkDelete {
-			er.markDelete(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
-		}
-	}
-
-	return toObjectErr(err, bucket)
-}
-
-// markDelete creates a vol entry in .minio.sys/buckets/.deleted until site replication
-// syncs the delete to peers
-func (er erasureObjects) markDelete(ctx context.Context, bucket, prefix string) error {
-	storageDisks := er.getDisks()
-	g := errgroup.WithNErrs(len(storageDisks))
-	// Make a volume entry on all underlying storage disks.
-	for index := range storageDisks {
-		index := index
-		if storageDisks[index] == nil {
-			continue
-		}
-		g.Go(func() error {
-			if err := storageDisks[index].MakeVol(ctx, pathJoin(bucket, prefix)); err != nil {
-				if errors.Is(err, errVolumeExists) {
-					return nil
-				}
-				return err
-			}
-			return nil
-		}, index)
-	}
-	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
-	return toObjectErr(err, bucket)
-}
-
-// purgeDelete deletes vol entry in .minio.sys/buckets/.deleted after site replication
-// syncs the delete to peers OR on a new MakeBucket call.
-func (er erasureObjects) purgeDelete(ctx context.Context, bucket, prefix string) error {
-	storageDisks := er.getDisks()
-	g := errgroup.WithNErrs(len(storageDisks))
-	// Make a volume entry on all underlying storage disks.
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if storageDisks[index] != nil {
-				return storageDisks[index].DeleteVol(ctx, pathJoin(bucket, prefix), true)
-			}
-			return errDiskNotFound
-		}, index)
-	}
-	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
-	return toObjectErr(err, bucket)
-}
-
-// IsNotificationSupported returns whether bucket notification is applicable for this layer.
-func (er erasureObjects) IsNotificationSupported() bool {
-	return true
-}
-
-// IsListenSupported returns whether listen bucket notification is applicable for this layer.
-func (er erasureObjects) IsListenSupported() bool {
-	return true
-}
-
-// IsEncryptionSupported returns whether server side encryption is implemented for this layer.
-func (er erasureObjects) IsEncryptionSupported() bool {
-	return true
-}
-
-// IsCompressionSupported returns whether compression is applicable for this layer.
-func (er erasureObjects) IsCompressionSupported() bool {
-	return true
-}
-
-// IsTaggingSupported indicates whether erasureObjects implements tagging support.
-func (er erasureObjects) IsTaggingSupported() bool {
-	return true
-}

cmd/erasure-encode.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"sync"
 
+	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -83,9 +84,16 @@ func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []io.Writer
 	for {
 		var blocks [][]byte
 		n, err := io.ReadFull(src, buf)
-		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
-			logger.LogIf(ctx, err)
-			return 0, err
+		if err != nil {
+			if !IsErrIgnored(err, []error{
+				io.EOF,
+				io.ErrUnexpectedEOF,
+			}...) {
+				if !hash.IsChecksumMismatch(err) {
+					logger.LogIf(ctx, err)
+				}
+				return 0, err
+			}
 		}
 		eof := err == io.EOF || err == io.ErrUnexpectedEOF
 		if n == 0 && total != 0 {

cmd/erasure-encode_test.go

@@ -24,7 +24,7 @@ import (
 	"io"
 	"testing"
 
-	humanize "github.com/dustin/go-humanize"
+	"github.com/dustin/go-humanize"
 )
 
 type badDisk struct{ StorageAPI }
@@ -41,10 +41,6 @@ func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset
 	return nil, errFaultyDisk
 }
 
-func (a badDisk) UpdateBloomFilter(ctx context.Context, oldest, current uint64) (*bloomFilterResponse, error) {
-	return nil, errFaultyDisk
-}
-
 func (a badDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
 	return errFaultyDisk
 }

cmd/erasure-healing-common_test.go

@@ -223,7 +223,7 @@ func TestListOnlineDisks(t *testing.T) {
 	}
 
 	bucket := "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -398,7 +398,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 	}
 
 	bucket := "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -519,7 +519,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	z := obj.(*erasureServerPools)
 	s := z.serverPools[0].sets[0]
 	erasureDisks := s.getDisks()
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}

cmd/erasure-healing.go

@@ -66,16 +66,12 @@ func (fi FileInfo) DataShardFixed() bool {
 	return fi.Metadata[reservedMetadataPrefixLowerDataShardFix] == "true"
 }
 
-// Heals a bucket if it doesn't exist on one of the disks, additionally
+// HealBucket heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
 func (er erasureObjects) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (
 	result madmin.HealResultItem, err error,
 ) {
-	if !opts.DryRun {
-		defer NSUpdated(bucket, slashSeparator)
-	}
-
 	storageDisks := er.getDisks()
 	storageEndpoints := er.getEndpoints()
 
@@ -94,7 +90,7 @@ func (er erasureObjects) healBucket(ctx context.Context, storageDisks []StorageA
 	if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 {
 		startTime := time.Now()
 		defer func() {
-			healTrace(healingMetricBucket, startTime, bucket, "", "", &opts, err, &res)
+			healTrace(healingMetricBucket, startTime, bucket, "", &opts, err, &res)
 		}()
 	}
 
@@ -235,6 +231,10 @@ func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets
 				// we ignore disk not found errors
 				return nil
 			}
+			if storageDisks[index].Healing() != nil {
+				// we ignore disks under healing
+				return nil
+			}
 			volsInfo, err := storageDisks[index].ListVols(ctx)
 			if err != nil {
 				return err
@@ -305,7 +305,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 {
 		startTime := time.Now()
 		defer func() {
-			healTrace(healingMetricObject, startTime, bucket, object, versionID, &opts, err, &result)
+			healTrace(healingMetricObject, startTime, bucket, object, &opts, err, &result)
 		}()
 	}
 	// Initialize heal result object
@@ -313,6 +313,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		Type:      madmin.HealItemObject,
 		Bucket:    bucket,
 		Object:    object,
+		VersionID: versionID,
 		DiskCount: len(storageDisks),
 	}
 
@@ -323,7 +324,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			return result, err
 		}
 		ctx = lkctx.Context()
-		defer lk.Unlock(lkctx.Cancel)
+		defer lk.Unlock(lkctx)
 	}
 
 	// Re-read when we have lock...
@@ -406,6 +407,11 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			erasure.ShardFileSize(latestMeta.Parts[0].ActualSize) < smallFileThreshold)
 	}
 
+	result.ObjectSize, err = latestMeta.GetActualSize()
+	if err != nil {
+		return result, err
+	}
+
 	// Loop to find number of disks with valid data, per-drive
 	// data state and a list of outdated disks on which data needs
 	// to be healed.
@@ -416,9 +422,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		switch {
 		case v != nil:
 			driveState = madmin.DriveStateOk
-			// If data is sane on any one disk, we can
-			// extract the correct object size.
-			result.ObjectSize = partsMetadata[i].Size
 		case errs[i] == errDiskNotFound, dataErrs[i] == errDiskNotFound:
 			driveState = madmin.DriveStateOffline
 		case errs[i] == errFileNotFound, errs[i] == errFileVersionNotFound, errs[i] == errVolumeNotFound:
@@ -668,9 +671,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		})
 	}
 
-	// Set the size of the object in the heal result
-	result.ObjectSize = latestMeta.Size
-
 	return result, nil
 }
 
@@ -684,7 +684,7 @@ func (er *erasureObjects) checkAbandonedParts(ctx context.Context, bucket string
 	if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 {
 		startTime := time.Now()
 		defer func() {
-			healTrace(healingMetricCheckAbandonedParts, startTime, bucket, object, "", nil, err, nil)
+			healTrace(healingMetricCheckAbandonedParts, startTime, bucket, object, nil, err, nil)
 		}()
 	}
 	if !opts.NoLock {
@@ -694,7 +694,7 @@ func (er *erasureObjects) checkAbandonedParts(ctx context.Context, bucket string
 			return err
 		}
 		ctx = lkctx.Context()
-		defer lk.Unlock(lkctx.Cancel)
+		defer lk.Unlock(lkctx)
 	}
 	var wg sync.WaitGroup
 	for _, disk := range er.getDisks() {
@@ -750,7 +750,6 @@ func (er *erasureObjects) healObjectDir(ctx context.Context, bucket, object stri
 				}(index, disk)
 			}
 			wg.Wait()
-			NSUpdated(bucket, object)
 		}
 	}
 
@@ -932,19 +931,22 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	// We can consider an object data not reliable
 	// when xl.meta is not found in read quorum disks.
 	// or when xl.meta is not readable in read quorum disks.
-	danglingErrsCount := func(cerrs []error) (int, int) {
+	danglingErrsCount := func(cerrs []error) (int, int, int) {
 		var (
-			notFoundCount  int
-			corruptedCount int
+			notFoundCount     int
+			corruptedCount    int
+			diskNotFoundCount int
 		)
 		for _, readErr := range cerrs {
 			if errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound) {
 				notFoundCount++
 			} else if errors.Is(readErr, errFileCorrupt) {
 				corruptedCount++
+			} else if errors.Is(readErr, errDiskNotFound) {
+				diskNotFoundCount++
 			}
 		}
-		return notFoundCount, corruptedCount
+		return notFoundCount, corruptedCount, diskNotFoundCount
 	}
 
 	ndataErrs := make([]error, len(dataErrs))
@@ -958,9 +960,12 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 		}
 	}
 
-	notFoundMetaErrs, corruptedMetaErrs := danglingErrsCount(errs)
-	notFoundPartsErrs, corruptedPartsErrs := danglingErrsCount(ndataErrs)
+	notFoundMetaErrs, corruptedMetaErrs, driveNotFoundMetaErrs := danglingErrsCount(errs)
+	notFoundPartsErrs, corruptedPartsErrs, driveNotFoundPartsErrs := danglingErrsCount(ndataErrs)
 
+	if driveNotFoundMetaErrs > 0 || driveNotFoundPartsErrs > 0 {
+		return validMeta, false
+	}
 	for _, m := range metaArr {
 		if m.IsValid() {
 			validMeta = m
@@ -1042,7 +1047,7 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 }
 
 // healTrace sends healing results to trace output.
-func healTrace(funcName healingMetric, startTime time.Time, bucket, object, versionID string, opts *madmin.HealOpts, err error, result *madmin.HealResultItem) {
+func healTrace(funcName healingMetric, startTime time.Time, bucket, object string, opts *madmin.HealOpts, err error, result *madmin.HealResultItem) {
 	tr := madmin.TraceInfo{
 		TraceType: madmin.TraceHealing,
 		Time:      startTime,
@@ -1052,10 +1057,12 @@ func healTrace(funcName healingMetric, startTime time.Time, bucket, object, vers
 		Path:      pathJoin(bucket, decodeDirObject(object)),
 	}
 	if opts != nil {
-		tr.Message = fmt.Sprintf("dry:%v, rm:%v, recreate:%v mode:%v", opts.DryRun, opts.Remove, opts.Recreate, opts.ScanMode)
-	}
-	if versionID != "" && versionID != "null" {
-		tr.Path += " v=" + versionID
+		tr.Custom = map[string]string{
+			"dry":      fmt.Sprint(opts.DryRun),
+			"remove":   fmt.Sprint(opts.Remove),
+			"recreate": fmt.Sprint(opts.Recreate),
+			"mode":     fmt.Sprint(opts.ScanMode),
+		}
 	}
 	if err != nil {
 		tr.Error = err.Error()

cmd/erasure-healing_test.go

@@ -244,7 +244,7 @@ func TestHealing(t *testing.T) {
 	er := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -398,7 +398,7 @@ func TestHealingVersioned(t *testing.T) {
 	er := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{VersioningEnabled: true})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{VersioningEnabled: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -581,7 +581,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	object := getRandomObjectName()
 	data := bytes.Repeat([]byte("a"), 128*1024)
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+	err = objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -776,7 +776,7 @@ func TestHealCorrectQuorum(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+	err = objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -907,7 +907,7 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+	err = objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1084,7 +1084,7 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+	err = objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1227,7 +1227,7 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = objLayer.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+	err = objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1384,7 +1384,7 @@ func TestHealObjectErasure(t *testing.T) {
 	data := bytes.Repeat([]byte("a"), 5*1024*1024)
 	var opts ObjectOptions
 
-	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1481,7 +1481,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	object := "empty-dir/"
 	var opts ObjectOptions
 
-	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
@@ -1582,7 +1582,7 @@ func TestHealLastDataShard(t *testing.T) {
 			}
 			var opts ObjectOptions
 
-			err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+			err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{})
 			if err != nil {
 				t.Fatalf("Failed to make a bucket - %v", err)
 			}

cmd/erasure-metadata.go

@@ -23,15 +23,18 @@ import (
 	"fmt"
 	"net/http"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
 	"github.com/minio/minio/internal/amztime"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/sio"
 )
 
 // Object was stored with additional erasure codes due to degraded system at upload time
@@ -104,6 +107,52 @@ func (fi FileInfo) IsValid() bool {
 		correctIndexes)
 }
 
+func (fi FileInfo) checkMultipart() (int64, bool) {
+	if len(fi.Parts) == 0 {
+		return 0, false
+	}
+	if !crypto.IsMultiPart(fi.Metadata) {
+		return 0, false
+	}
+	var size int64
+	for _, part := range fi.Parts {
+		psize, err := sio.DecryptedSize(uint64(part.Size))
+		if err != nil {
+			return 0, false
+		}
+		size += int64(psize)
+	}
+
+	return size, len(extractETag(fi.Metadata)) != 32
+}
+
+// GetActualSize - returns the actual size of the stored object
+func (fi FileInfo) GetActualSize() (int64, error) {
+	if _, ok := fi.Metadata[ReservedMetadataPrefix+"compression"]; ok {
+		sizeStr, ok := fi.Metadata[ReservedMetadataPrefix+"actual-size"]
+		if !ok {
+			return -1, errInvalidDecompressedSize
+		}
+		size, err := strconv.ParseInt(sizeStr, 10, 64)
+		if err != nil {
+			return -1, errInvalidDecompressedSize
+		}
+		return size, nil
+	}
+	if _, ok := crypto.IsEncrypted(fi.Metadata); ok {
+		size, ok := fi.checkMultipart()
+		if !ok {
+			size, err := sio.DecryptedSize(uint64(fi.Size))
+			if err != nil {
+				err = errObjectTampered // assign correct error type
+			}
+			return int64(size), err
+		}
+		return size, nil
+	}
+	return fi.Size, nil
+}
+
 // ToObjectInfo - Converts metadata to object info.
 func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInfo {
 	object = decodeDirObject(object)
@@ -147,9 +196,9 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 	// Add replication status to the object info
 	objInfo.ReplicationStatusInternal = fi.ReplicationState.ReplicationStatusInternal
 	objInfo.VersionPurgeStatusInternal = fi.ReplicationState.VersionPurgeStatusInternal
-	objInfo.ReplicationStatus = fi.ReplicationState.CompositeReplicationStatus()
+	objInfo.ReplicationStatus = fi.ReplicationStatus()
+	objInfo.VersionPurgeStatus = fi.VersionPurgeStatus()
 
-	objInfo.VersionPurgeStatus = fi.ReplicationState.CompositeVersionPurgeStatus()
 	objInfo.TransitionedObject = TransitionedObject{
 		Name:        fi.TransitionedObjName,
 		VersionID:   fi.TransitionVersionID,
@@ -176,7 +225,6 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 		objInfo.StorageClass = globalMinioDefaultStorageClass
 	}
 
-	objInfo.VersionPurgeStatus = fi.VersionPurgeStatus()
 	// set restore status for transitioned object
 	restoreHdr, ok := fi.Metadata[xhttp.AmzRestore]
 	if ok {
@@ -429,7 +477,13 @@ func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int)
 			parities[index] = -1
 			continue
 		}
-		parities[index] = metadata.Erasure.ParityBlocks
+		if !metadata.IsValid() {
+			parities[index] = -1
+			continue
+		}
+		if !metadata.Deleted {
+			parities[index] = metadata.Erasure.ParityBlocks
+		}
 	}
 	return
 }
@@ -534,6 +588,11 @@ func (fi *FileInfo) VersionPurgeStatus() VersionPurgeStatusType {
 	return fi.ReplicationState.CompositeVersionPurgeStatus()
 }
 
+// ReplicationStatus returns overall version replication status for this object version across targets
+func (fi *FileInfo) ReplicationStatus() replication.StatusType {
+	return fi.ReplicationState.CompositeReplicationStatus()
+}
+
 // DeleteMarkerReplicationStatus returns overall replication status for this delete marker version across targets
 func (fi *FileInfo) DeleteMarkerReplicationStatus() replication.StatusType {
 	if fi.Deleted {

cmd/erasure-multipart.go

@@ -349,7 +349,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		}
 		rctx := lkctx.Context()
 		obj, err := er.getObjectInfo(rctx, bucket, object, opts)
-		lk.RUnlock(lkctx.Cancel)
+		lk.RUnlock(lkctx)
 		if err != nil && !isErrVersionNotFound(err) {
 			return nil, err
 		}
@@ -473,24 +473,6 @@ func (er erasureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObjec
 	return partInfo, nil
 }
 
-func undoRenamePart(disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, errs []error) {
-	// Undo rename object on disks where RenameFile succeeded.
-	g := errgroup.WithNErrs(len(disks))
-	for index, disk := range disks {
-		if disk == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			if errs[index] == nil {
-				_ = disks[index].RenameFile(context.TODO(), dstBucket, dstEntry, srcBucket, srcEntry)
-			}
-			return nil
-		}, index)
-	}
-	g.Wait()
-}
-
 // renamePart - renames multipart part to its relevant location under uploadID.
 func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
 	g := errgroup.WithNErrs(len(disks))
@@ -509,15 +491,13 @@ func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, ds
 	// Wait for all renames to finish.
 	errs := g.Wait()
 
+	// Do not need to undo partial successful operation since those will be cleaned up
+	// in 24hrs via multipart cleaner, never rename() back to `.minio.sys/tmp` as there
+	// is no way to clean them.
+
 	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
 	// otherwise return failure. Cleanup successful renames.
-	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-	if err == errErasureWriteQuorum {
-		// Undo all the partial rename operations.
-		undoRenamePart(disks, srcBucket, srcEntry, dstBucket, dstEntry, errs)
-	}
-
-	return evalDisks(disks, errs), err
+	return evalDisks(disks, errs), reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 }
 
 // writeAllDisks - writes 'b' to all provided disks.
@@ -577,7 +557,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		return PartInfo{}, err
 	}
 	rctx := rlkctx.Context()
-	defer uploadIDRLock.RUnlock(rlkctx.Cancel)
+	defer uploadIDRLock.RUnlock(rlkctx)
 
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 	// Validates if upload ID exists.
@@ -603,7 +583,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		return PartInfo{}, err
 	}
 	pctx := plkctx.Context()
-	defer partIDLock.Unlock(plkctx.Cancel)
+	defer partIDLock.Unlock(plkctx)
 
 	onlineDisks := er.getDisks()
 	writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
@@ -775,7 +755,7 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 		return MultipartInfo{}, err
 	}
 	ctx = lkctx.Context()
-	defer uploadIDLock.RUnlock(lkctx.Cancel)
+	defer uploadIDLock.RUnlock(lkctx)
 
 	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
 	if err != nil {
@@ -802,7 +782,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 		return ListPartsInfo{}, err
 	}
 	ctx = lkctx.Context()
-	defer uploadIDLock.RUnlock(lkctx.Cancel)
+	defer uploadIDLock.RUnlock(lkctx)
 
 	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
 	if err != nil {
@@ -935,7 +915,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		return oi, err
 	}
 	wctx := wlkctx.Context()
-	defer uploadIDLock.Unlock(wlkctx.Cancel)
+	defer uploadIDLock.Unlock(wlkctx)
 
 	fi, partsMetadata, err := er.checkUploadIDExists(wctx, bucket, object, uploadID, true)
 	if err != nil {
@@ -993,15 +973,35 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	//
 	// Therefore, we adjust all ETags sent by the client to match what is stored
 	// on the backend.
-	kind, isEncrypted := crypto.IsEncrypted(fi.Metadata)
+	kind, _ := crypto.IsEncrypted(fi.Metadata)
 
 	var objectEncryptionKey []byte
-	if isEncrypted && kind == crypto.S3 {
+	switch kind {
+	case crypto.SSEC:
+		if checksumType.IsSet() {
+			if opts.EncryptFn == nil {
+				return oi, crypto.ErrMissingCustomerKey
+			}
+			baseKey := opts.EncryptFn("", nil)
+			if len(baseKey) != 32 {
+				return oi, crypto.ErrInvalidCustomerKey
+			}
+			objectEncryptionKey, err = decryptObjectMeta(baseKey, bucket, object, fi.Metadata)
+			if err != nil {
+				return oi, err
+			}
+		}
+	case crypto.S3, crypto.S3KMS:
 		objectEncryptionKey, err = decryptObjectMeta(nil, bucket, object, fi.Metadata)
 		if err != nil {
 			return oi, err
 		}
 	}
+	if len(objectEncryptionKey) == 32 {
+		var key crypto.ObjectKey
+		copy(key[:], objectEncryptionKey)
+		opts.EncryptFn = metadataEncrypter(key)
+	}
 
 	for i, part := range partInfoFiles {
 		partID := parts[i].PartNumber
@@ -1184,7 +1184,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		return oi, err
 	}
 	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
+	defer lk.Unlock(lkctx)
 
 	// Write final `xl.meta` at uploadID location
 	onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum)
@@ -1211,12 +1211,15 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	// Rename the multipart object to final location.
-	if onlineDisks, err = renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,
-		partsMetadata, bucket, object, writeQuorum); err != nil {
+	onlineDisks, versionsDisparity, err := renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,
+		partsMetadata, bucket, object, writeQuorum)
+	if err != nil {
 		return oi, toObjectErr(err, bucket, object)
 	}
 
-	defer NSUpdated(bucket, object)
+	if !opts.Speedtest && versionsDisparity {
+		listAndHeal(ctx, bucket, object, &er, healObjectVersionsDisparity)
+	}
 
 	// Check if there is any offline disk and add it to the MRF list
 	for _, disk := range onlineDisks {
@@ -1259,7 +1262,7 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 		return err
 	}
 	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
+	defer lk.Unlock(lkctx)
 
 	// Validates if upload ID exists.
 	if _, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false); err != nil {

cmd/erasure-object.go

@@ -72,8 +72,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 		return oi, NotImplemented{}
 	}
 
-	defer NSUpdated(dstBucket, dstObject)
-
 	if !dstOpts.NoLock {
 		lk := er.NewNSLock(dstBucket, dstObject)
 		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
@@ -81,7 +79,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 			return oi, err
 		}
 		ctx = lkctx.Context()
-		defer lk.Unlock(lkctx.Cancel)
+		defer lk.Unlock(lkctx)
 	}
 	// Read metadata associated with the object from all disks.
 	storageDisks := er.getDisks()
@@ -93,7 +91,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	if srcOpts.VersionID != "" {
 		metaArr, errs = readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID, true)
 	} else {
-		metaArr, errs = readAllXL(ctx, storageDisks, srcBucket, srcObject, true)
+		metaArr, errs = readAllXL(ctx, storageDisks, srcBucket, srcObject, true, false)
 	}
 
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
@@ -188,6 +186,14 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	auditObjectErasureSet(ctx, object, &er)
 
+	// This is a special call attempted first to check for SOS-API calls.
+	gr, err = veeamSOSAPIGetObject(ctx, bucket, object, rs, opts)
+	if err == nil {
+		return gr, nil
+	}
+	// reset any error to 'nil'
+	err = nil
+
 	var unlockOnDefer bool
 	nsUnlocker := func() {}
 	defer func() {
@@ -206,14 +212,14 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 				return nil, err
 			}
 			ctx = lkctx.Context()
-			nsUnlocker = func() { lock.Unlock(lkctx.Cancel) }
+			nsUnlocker = func() { lock.Unlock(lkctx) }
 		case readLock:
 			lkctx, err := lock.GetRLock(ctx, globalOperationTimeout)
 			if err != nil {
 				return nil, err
 			}
 			ctx = lkctx.Context()
-			nsUnlocker = func() { lock.RUnlock(lkctx.Cancel) }
+			nsUnlocker = func() { lock.RUnlock(lkctx) }
 		}
 		unlockOnDefer = true
 	}
@@ -434,7 +440,7 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
 			return ObjectInfo{}, err
 		}
 		ctx = lkctx.Context()
-		defer lk.RUnlock(lkctx.Cancel)
+		defer lk.RUnlock(lkctx)
 	}
 
 	return er.getObjectInfo(ctx, bucket, object, opts)
@@ -475,7 +481,6 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		if opts.VersionID != "" {
 			err = errFileVersionNotFound
 		}
-		defer NSUpdated(bucket, object)
 
 		fi := FileInfo{
 			VersionID: m.VersionID,
@@ -483,7 +488,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		if opts.VersionID != "" {
 			fi.VersionID = opts.VersionID
 		}
-
+		fi.SetTierFreeVersionID(mustGetUUID())
 		disks := er.getDisks()
 		g := errgroup.WithNErrs(len(disks))
 		for index := range disks {
@@ -501,7 +506,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 	return m, err
 }
 
-func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, readData bool) ([]FileInfo, []error) {
+func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, readData, inclFreeVers bool) ([]FileInfo, []error) {
 	metadataArray := make([]*xlMetaV2, len(disks))
 	metaFileInfos := make([]FileInfo, len(metadataArray))
 	metadataShallowVersions := make([][]xlMetaV2ShallowVersion, len(disks))
@@ -566,7 +571,7 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 
 	readQuorum := (len(disks) + 1) / 2
 	meta := &xlMetaV2{versions: mergeXLV2Versions(readQuorum, false, 1, metadataShallowVersions...)}
-	lfi, err := meta.ToFileInfo(bucket, object, "")
+	lfi, err := meta.ToFileInfo(bucket, object, "", inclFreeVers)
 	if err != nil {
 		for i := range errs {
 			if errs[i] == nil {
@@ -596,7 +601,7 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 
 		// make sure to preserve this for diskmtime based healing bugfix.
 		diskMTime := metaFileInfos[index].DiskMTime
-		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID)
+		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID, inclFreeVers)
 		if errs[index] != nil {
 			continue
 		}
@@ -620,7 +625,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	if opts.VersionID != "" {
 		metaArr, errs = readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, readData)
 	} else {
-		metaArr, errs = readAllXL(ctx, disks, bucket, object, readData)
+		metaArr, errs = readAllXL(ctx, disks, bucket, object, readData, opts.InclFreeVersions)
 	}
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
@@ -636,7 +641,10 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
 		if errors.Is(reducedErr, errErasureReadQuorum) && !strings.HasPrefix(bucket, minioMetaBucket) {
-			er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			_, derr := er.deleteIfDangling(ctx, bucket, object, metaArr, errs, nil, opts)
+			if derr != nil {
+				reducedErr = derr
+			}
 		}
 		return fi, nil, nil, toObjectErr(reducedErr, bucket, object)
 	}
@@ -729,7 +737,7 @@ func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, obj
 }
 
 // Similar to rename but renames data from srcEntry to dstEntry at dataDir
-func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
+func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, bool, error) {
 	g := errgroup.WithNErrs(len(disks))
 
 	fvID := mustGetUUID()
@@ -768,20 +776,25 @@ func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry str
 	// Wait for all renames to finish.
 	errs := g.Wait()
 
+	var versionsDisparity bool
+
 	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	if err == nil {
 		versions := reduceCommonVersions(diskVersions, writeQuorum)
 		for index, dversions := range diskVersions {
+			if errs[index] != nil {
+				continue
+			}
 			if versions != dversions {
-				errs[index] = errFileNeedsHealing
+				versionsDisparity = true
+				break
 			}
 		}
-		err = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	}
 
 	// We can safely allow RenameData errors up to len(er.getDisks()) - writeQuorum
 	// otherwise return failure.
-	return evalDisks(disks, errs), err
+	return evalDisks(disks, errs), versionsDisparity, err
 }
 
 func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
@@ -934,6 +947,47 @@ func (er erasureObjects) PutObject(ctx context.Context, bucket string, object st
 	return er.putObject(ctx, bucket, object, data, opts)
 }
 
+// Heal up to two versions of one object when there is disparity between disks
+func healObjectVersionsDisparity(bucket string, entry metaCacheEntry) error {
+	if entry.isDir() {
+		return nil
+	}
+	// We might land at .metacache, .trash, .multipart
+	// no need to heal them skip, only when bucket
+	// is '.minio.sys'
+	if bucket == minioMetaBucket {
+		if wildcard.Match("buckets/*/.metacache/*", entry.name) {
+			return nil
+		}
+		if wildcard.Match("tmp/*", entry.name) {
+			return nil
+		}
+		if wildcard.Match("multipart/*", entry.name) {
+			return nil
+		}
+		if wildcard.Match("tmp-old/*", entry.name) {
+			return nil
+		}
+	}
+
+	fivs, err := entry.fileInfoVersions(bucket)
+	if err != nil {
+		go healObject(bucket, entry.name, "", madmin.HealNormalScan)
+		return err
+	}
+
+	if len(fivs.Versions) > 2 {
+		return fmt.Errorf("bucket(%s)/object(%s) object with %d versions needs healing, allowing lazy healing via scanner instead here for versions greater than 2",
+			bucket, entry.name, len(fivs.Versions))
+	}
+
+	for _, version := range fivs.Versions {
+		go healObject(bucket, entry.name, version.VersionID, madmin.HealNormalScan)
+	}
+
+	return nil
+}
+
 // putObject wrapper for erasureObjects PutObject
 func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	auditObjectErasureSet(ctx, object, &er)
@@ -1152,7 +1206,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			return ObjectInfo{}, err
 		}
 		ctx = lkctx.Context()
-		defer lk.Unlock(lkctx.Cancel)
+		defer lk.Unlock(lkctx)
 	}
 
 	modTime := opts.MTime
@@ -1205,7 +1259,15 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	// Rename the successfully written temporary object to final location.
-	onlineDisks, err = renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
+	onlineDisks, versionsDisparity, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum)
+	if err != nil {
+		if errors.Is(err, errFileNotFound) {
+			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
+		}
+		logger.LogIf(ctx, err)
+		return ObjectInfo{}, toObjectErr(err, bucket, object)
+	}
+
 	for i := 0; i < len(onlineDisks); i++ {
 		if onlineDisks[i] != nil && onlineDisks[i].IsOnline() {
 			// Object info is the same in all disks, so we can pick
@@ -1227,63 +1289,12 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			er.addPartial(bucket, object, fi.VersionID, fi.Size)
 			break
 		}
-	}
 
-	healEntry := func(entry metaCacheEntry) error {
-		if entry.isDir() {
-			return nil
-		}
-		// We might land at .metacache, .trash, .multipart
-		// no need to heal them skip, only when bucket
-		// is '.minio.sys'
-		if bucket == minioMetaBucket {
-			if wildcard.Match("buckets/*/.metacache/*", entry.name) {
-				return nil
-			}
-			if wildcard.Match("tmp/*", entry.name) {
-				return nil
-			}
-			if wildcard.Match("multipart/*", entry.name) {
-				return nil
-			}
-			if wildcard.Match("tmp-old/*", entry.name) {
-				return nil
-			}
-		}
-
-		fivs, err := entry.fileInfoVersions(bucket)
-		if err != nil {
-			healObject(bucket, entry.name, "", madmin.HealNormalScan)
-			return err
-		}
-
-		if len(fivs.Versions) > 2 {
-			return fmt.Errorf("bucket(%s)/object(%s) object with %d versions needs healing, allowing lazy healing via scanner instead here for versions greater than 2",
-				bucket, entry.name,
-				len(fivs.Versions))
-		}
-
-		for _, version := range fivs.Versions {
-			healObject(bucket, entry.name, version.VersionID, madmin.HealNormalScan)
-		}
-
-		return nil
-	}
-
-	if err != nil {
-		if errors.Is(err, errFileNotFound) {
-			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
-		}
-		if errors.Is(err, errFileNeedsHealing) && !opts.Speedtest {
-			listAndHeal(ctx, bucket, object, &er, healEntry)
-		} else {
-			logger.LogIf(ctx, err)
-			return ObjectInfo{}, toObjectErr(err, bucket, object)
+		if versionsDisparity {
+			listAndHeal(ctx, bucket, object, &er, healObjectVersionsDisparity)
 		}
 	}
 
-	defer NSUpdated(bucket, object)
-
 	fi.ReplicationState = opts.PutReplicationState()
 	online = countOnlineDisks(onlineDisks)
 
@@ -1459,8 +1470,6 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		} else {
 			errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName)
 		}
-
-		defer NSUpdated(bucket, objects[objIndex].ObjectName)
 	}
 
 	// Check failed deletes across multiple objects
@@ -1552,17 +1561,25 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		}
 	}
 
-	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
-	defer NSUpdated(bucket, object)
-
 	storageDisks := er.getDisks()
-
-	//  Determine whether to mark object deleted for replication
-	var markDelete bool
+	versionFound := true
+	objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response.
+	goi, _, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
+	if gerr != nil && goi.Name == "" {
+		switch gerr.(type) {
+		case InsufficientReadQuorum:
+			return objInfo, InsufficientWriteQuorum{}
+		}
+		// For delete marker replication, versionID being replicated will not exist on disk
+		if opts.DeleteMarker {
+			versionFound = false
+		} else {
+			return objInfo, gerr
+		}
+	}
 
 	if opts.Expiration.Expire {
-		goi, _, err := er.getObjectInfoAndQuorum(ctx, bucket, object, opts)
-		if err == nil {
+		if gerr == nil {
 			evt := evalActionFromLifecycle(ctx, *lc, rcfg, goi)
 			var isErr bool
 			switch evt.Action {
@@ -1584,21 +1601,16 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 					Object: object,
 				}
 			}
-			markDelete = goi.VersionID != ""
 		}
 	}
 
-	versionFound := !(opts.DeleteMarker && opts.VersionID != "")
+	//  Determine whether to mark object deleted for replication
+	markDelete := goi.VersionID != ""
 
-	if !markDelete {
-		markDelete = !opts.DeleteMarker && opts.VersionID != ""
-	}
 	// Default deleteMarker to true if object is under versioning
-	// versioning suspended means we add `null` version as
-	// delete marker, if its not decided already.
-	deleteMarker := (opts.Versioned || opts.VersionSuspended) && opts.VersionID == "" || (opts.DeleteMarker && opts.VersionID != "")
+	deleteMarker := opts.Versioned
 
-	if markDelete && opts.VersionID != "" {
+	if opts.VersionID != "" {
 		// case where replica version needs to be deleted on target cluster
 		if versionFound && opts.DeleteMarkerReplicationStatus() == replication.Replica {
 			markDelete = false
@@ -1609,6 +1621,22 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		if opts.VersionPurgeStatus() == Complete {
 			markDelete = false
 		}
+		// now, since VersionPurgeStatus() is already set, we can let the
+		// lower layers decide this. This fixes a regression that was introduced
+		// in PR #14555 where !VersionPurgeStatus.Empty() is automatically
+		// considered as Delete marker true to avoid listing such objects by
+		// regular ListObjects() calls. However for delete replication this
+		// ends up being a problem because "upon" a successful delete this
+		// ends up creating a new delete marker that is spurious and unnecessary.
+		//
+		// Regression introduced by #14555 was reintroduced in #15564
+		if versionFound {
+			if !goi.VersionPurgeStatus.Empty() {
+				deleteMarker = false
+			} else if !goi.DeleteMarker { // implies a versioned delete of object
+				deleteMarker = false
+			}
+		}
 	}
 
 	modTime := opts.MTime
@@ -1618,9 +1646,14 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	fvID := mustGetUUID()
 
 	if markDelete && (opts.Versioned || opts.VersionSuspended) {
+		if !deleteMarker {
+			// versioning suspended means we add `null` version as
+			// delete marker, if its not decided already.
+			deleteMarker = opts.VersionSuspended && opts.VersionID == ""
+		}
 		fi := FileInfo{
 			Name:             object,
-			Deleted:          true,
+			Deleted:          deleteMarker,
 			MarkDeleted:      markDelete,
 			ModTime:          modTime,
 			ReplicationState: opts.DeleteReplication,
@@ -1628,11 +1661,10 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 			ExpireRestored:   opts.Transition.ExpireRestored,
 		}
 		fi.SetTierFreeVersionID(fvID)
-		if opts.Versioned {
+		if opts.VersionID != "" {
+			fi.VersionID = opts.VersionID
+		} else if opts.Versioned {
 			fi.VersionID = mustGetUUID()
-			if opts.VersionID != "" {
-				fi.VersionID = opts.VersionID
-			}
 		}
 		// versioning suspended means we add `null` version as
 		// delete marker. Add delete marker, since we don't have
@@ -1693,7 +1725,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 			return ObjectInfo{}, err
 		}
 		ctx = lkctx.Context()
-		defer lk.Unlock(lkctx.Cancel)
+		defer lk.Unlock(lkctx)
 	}
 
 	disks := er.getDisks()
@@ -1705,7 +1737,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 	if opts.VersionID != "" {
 		metaArr, errs = readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
 	} else {
-		metaArr, errs = readAllXL(ctx, disks, bucket, object, false)
+		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false)
 	}
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
@@ -1767,7 +1799,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 		return ObjectInfo{}, err
 	}
 	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
+	defer lk.Unlock(lkctx)
 
 	disks := er.getDisks()
 
@@ -1778,7 +1810,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	if opts.VersionID != "" {
 		metaArr, errs = readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false)
 	} else {
-		metaArr, errs = readAllXL(ctx, disks, bucket, object, false)
+		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false)
 	}
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
@@ -1877,7 +1909,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		return err
 	}
 	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
+	defer lk.Unlock(lkctx)
 
 	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
 	if err != nil {
@@ -1898,7 +1930,6 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 	if fi.TransitionStatus == lifecycle.TransitionComplete {
 		return nil
 	}
-	defer NSUpdated(bucket, object)
 
 	if fi.XLV1 {
 		if _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{NoLock: true}); err != nil {
@@ -1972,15 +2003,13 @@ func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket,
 }
 
 // update restore status header in the metadata
-func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions, rerr error) error {
+func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions) error {
 	oi := objInfo.Clone()
 	oi.metadataOnly = true // Perform only metadata updates.
 
-	if rerr == nil {
-		oi.UserDefined[xhttp.AmzRestore] = completedRestoreObj(opts.Transition.RestoreExpiry).String()
-	} else { // allow retry in the case of failure to restore
-		delete(oi.UserDefined, xhttp.AmzRestore)
-	}
+	// allow retry in the case of failure to restore
+	delete(oi.UserDefined, xhttp.AmzRestore)
+
 	if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{
 		VersionID: oi.VersionID,
 	}, ObjectOptions{
@@ -1996,7 +2025,10 @@ func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, obje
 // as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta
 func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error {
 	setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error {
-		er.updateRestoreMetadata(ctx, bucket, object, oi, opts, rerr)
+		if rerr == nil {
+			return nil // nothing to do; restore object was successful
+		}
+		er.updateRestoreMetadata(ctx, bucket, object, oi, opts)
 		return rerr
 	}
 	var oi ObjectInfo
@@ -2020,7 +2052,6 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
 		}
 		pReader := NewPutObjReader(hashReader)
-		ropts.UserDefined[xhttp.AmzRestore] = completedRestoreObj(opts.Transition.RestoreExpiry).String()
 		_, err = er.PutObject(ctx, bucket, object, pReader, ropts)
 		return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
 	}

cmd/erasure-object_test.go

@@ -53,7 +53,7 @@ func TestRepeatPutObjectPart(t *testing.T) {
 	defer objLayer.Shutdown(context.Background())
 	defer removeRoots(disks)
 
-	err = objLayer.MakeBucketWithLocation(ctx, "bucket1", MakeBucketOptions{})
+	err = objLayer.MakeBucket(ctx, "bucket1", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -101,7 +101,7 @@ func TestErasureDeleteObjectBasic(t *testing.T) {
 	}
 	defer xl.Shutdown(context.Background())
 
-	err = xl.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = xl.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -155,7 +155,7 @@ func TestDeleteObjectsVersioned(t *testing.T) {
 		{bucketName, "dir/obj1"},
 	}
 
-	err = obj.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{
+	err = obj.MakeBucket(ctx, bucketName, MakeBucketOptions{
 		VersioningEnabled: true,
 	})
 	if err != nil {
@@ -214,22 +214,19 @@ func TestDeleteObjectsVersioned(t *testing.T) {
 func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	var objs []*erasureObjects
-	for i := 0; i < 32; i++ {
-		obj, fsDirs, err := prepareErasure(ctx, 16)
-		if err != nil {
-			t.Fatal("Unable to initialize 'Erasure' object layer.", err)
-		}
-		// Remove all dirs.
-		for _, dir := range fsDirs {
-			defer os.RemoveAll(dir)
-		}
-		z := obj.(*erasureServerPools)
-		xl := z.serverPools[0].sets[0]
-		objs = append(objs, xl)
+
+	obj, fsDirs, err := prepareErasureSets32(ctx)
+	if err != nil {
+		t.Fatal("Unable to initialize 'Erasure' object layer.", err)
 	}
 
-	erasureSets := &erasureSets{sets: objs, distributionAlgo: "CRCMOD"}
+	setObjectLayer(obj)
+	initConfigSubsystem(ctx, obj)
+
+	// Remove all dirs.
+	for _, dir := range fsDirs {
+		defer os.RemoveAll(dir)
+	}
 
 	type testCaseType struct {
 		bucket string
@@ -244,13 +241,12 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 		{bucketName, "obj_4"},
 	}
 
-	err := erasureSets.MakeBucketWithLocation(ctx, bucketName, MakeBucketOptions{})
-	if err != nil {
+	if err = obj.MakeBucket(ctx, bucketName, MakeBucketOptions{}); err != nil {
 		t.Fatal(err)
 	}
 
 	for _, testCase := range testCases {
-		_, err = erasureSets.PutObject(ctx, testCase.bucket, testCase.object,
+		_, err = obj.PutObject(ctx, testCase.bucket, testCase.object,
 			mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{})
 		if err != nil {
 			t.Fatalf("Erasure Object upload failed: <ERROR> %s", err)
@@ -270,7 +266,7 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 	}
 
 	objectNames := toObjectNames(testCases)
-	_, delErrs := erasureSets.DeleteObjects(ctx, bucketName, objectNames, ObjectOptions{})
+	_, delErrs := obj.DeleteObjects(ctx, bucketName, objectNames, ObjectOptions{})
 
 	for i := range delErrs {
 		if delErrs[i] != nil {
@@ -279,7 +275,7 @@ func TestErasureDeleteObjectsErasureSet(t *testing.T) {
 	}
 
 	for _, test := range testCases {
-		_, statErr := erasureSets.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{})
+		_, statErr := obj.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{})
 		switch statErr.(type) {
 		case ObjectNotFound:
 		default:
@@ -305,7 +301,7 @@ func TestErasureDeleteObjectDiskNotFound(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -374,7 +370,7 @@ func TestErasureDeleteObjectDiskNotFoundErasure4(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -434,7 +430,7 @@ func TestErasureDeleteObjectDiskNotFoundErr(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -505,7 +501,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -614,7 +610,7 @@ func TestHeadObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -692,7 +688,7 @@ func TestPutObjectNoQuorum(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -755,7 +751,7 @@ func TestPutObjectNoQuorumSmall(t *testing.T) {
 	xl := z.serverPools[0].sets[0]
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, "bucket", MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -824,7 +820,7 @@ func TestPutObjectSmallInlineData(t *testing.T) {
 	object := "object"
 
 	// Create "bucket"
-	err = obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+	err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -892,7 +888,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	ctx, cancel := context.WithCancel(GlobalContext)
 	defer cancel()
 
-	err := obj.MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
+	err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{})
 	if err != nil {
 		t.Fatalf("Failed to make a bucket %v", err)
 	}
@@ -1091,7 +1087,7 @@ func TestGetObjectInlineNotInline(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Create a testbucket
-	err = objLayer.MakeBucketWithLocation(ctx, "testbucket", MakeBucketOptions{})
+	err = objLayer.MakeBucket(ctx, "testbucket", MakeBucketOptions{})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1161,7 +1157,7 @@ func TestGetObjectWithOutdatedDisks(t *testing.T) {
 
 	for i, testCase := range testCases {
 		// Step 1: create a bucket
-		err = z.MakeBucketWithLocation(ctx, testCase.bucket, MakeBucketOptions{VersioningEnabled: testCase.versioned})
+		err = z.MakeBucket(ctx, testCase.bucket, MakeBucketOptions{VersioningEnabled: testCase.versioned})
 		if err != nil {
 			t.Fatalf("Test %d: Failed to create a bucket: %v", i+1, err)
 		}

cmd/erasure-server-pool-decom.go

@@ -126,7 +126,7 @@ type poolMeta struct {
 
 // A decommission resumable tells us if decommission is worth
 // resuming upon restart of a cluster.
-func (p *poolMeta) returnResumablePools(n int) []PoolStatus {
+func (p *poolMeta) returnResumablePools() []PoolStatus {
 	var newPools []PoolStatus
 	for _, pool := range p.Pools {
 		if pool.Decommission == nil {
@@ -139,11 +139,8 @@ func (p *poolMeta) returnResumablePools(n int) []PoolStatus {
 			continue
 		} // In all other situations we need to resume
 		newPools = append(newPools, pool)
-		if n > 0 && len(newPools) == n {
-			return newPools
-		}
 	}
-	return nil
+	return newPools
 }
 
 func (p *poolMeta) DecommissionComplete(idx int) bool {
@@ -251,18 +248,6 @@ var (
 )
 
 func (p *poolMeta) Decommission(idx int, pi poolSpaceInfo) error {
-	for i, pool := range p.Pools {
-		if idx == i {
-			continue
-		}
-		if pool.Decommission != nil {
-			// Do not allow multiple decommissions at the same time.
-			// We shall for now only allow one pool decommission at
-			// a time.
-			return fmt.Errorf("%w at index: %d", errDecommissionAlreadyRunning, i)
-		}
-	}
-
 	// Return an error when there is decommission on going - the user needs
 	// to explicitly cancel it first in order to restart decommissioning again.
 	if p.Pools[idx].Decommission != nil &&
@@ -510,7 +495,6 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 	z.StartRebalance()
 
 	meta := poolMeta{}
-
 	if err := meta.load(ctx, z.serverPools[0], z.serverPools); err != nil {
 		return err
 	}
@@ -524,38 +508,38 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 	if !update {
 		z.poolMeta = meta
 
-		// We are only supporting single pool decommission at this time
-		// so it makes sense to only resume single pools at any given
-		// time, in future meta.returnResumablePools() might take
-		// '-1' as argument to decommission multiple pools at a time
-		// but this is not a priority at the moment.
-		for _, pool := range meta.returnResumablePools(1) {
+		pools := meta.returnResumablePools()
+		poolIndices := make([]int, 0, len(pools))
+		for _, pool := range pools {
 			idx := globalEndpoints.GetPoolIdx(pool.CmdLine)
 			if idx == -1 {
 				return fmt.Errorf("unexpected state present for decommission status pool(%s) not found", pool.CmdLine)
 			}
-			if globalEndpoints[idx].Endpoints[0].IsLocal {
-				go func(pool PoolStatus) {
-					r := rand.New(rand.NewSource(time.Now().UnixNano()))
-					for {
-						if err := z.Decommission(ctx, pool.ID); err != nil {
-							if errors.Is(err, errDecommissionAlreadyRunning) {
-								// A previous decommission running found restart it.
+			poolIndices = append(poolIndices, idx)
+		}
+
+		if len(poolIndices) > 0 && globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
+			go func() {
+				r := rand.New(rand.NewSource(time.Now().UnixNano()))
+				for {
+					if err := z.Decommission(ctx, poolIndices...); err != nil {
+						if errors.Is(err, errDecommissionAlreadyRunning) {
+							// A previous decommission running found restart it.
+							for _, idx := range poolIndices {
 								z.doDecommissionInRoutine(ctx, idx)
-								return
 							}
-							if configRetriableErrors(err) {
-								logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w: retrying..", pool, err))
-								time.Sleep(time.Second + time.Duration(r.Float64()*float64(5*time.Second)))
-								continue
-							}
-							logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pool, err))
 							return
 						}
-						break
+						if configRetriableErrors(err) {
+							logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pools %v: %w: retrying..", pools, err))
+							time.Sleep(time.Second + time.Duration(r.Float64()*float64(5*time.Second)))
+							continue
+						}
+						logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pools, err))
+						return
 					}
-				}(pool)
-			}
+				}
+			}()
 		}
 
 		return nil
@@ -585,7 +569,10 @@ func (z *erasureServerPools) IsDecommissionRunning() bool {
 	defer z.poolMetaMutex.RUnlock()
 	meta := z.poolMeta
 	for _, pool := range meta.Pools {
-		if pool.Decommission != nil {
+		if pool.Decommission != nil &&
+			!pool.Decommission.Complete &&
+			!pool.Decommission.Failed &&
+			!pool.Decommission.Canceled {
 			return true
 		}
 	}
@@ -686,6 +673,39 @@ func (v versionsSorter) reverse() {
 	})
 }
 
+func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi decomBucketInfo, fn func(entry metaCacheEntry)) error {
+	disks := set.getOnlineDisks()
+	if len(disks) == 0 {
+		return fmt.Errorf("no online drives found for set with endpoints %s", set.getEndpoints())
+	}
+
+	// How to resolve partial results.
+	resolver := metadataResolutionParams{
+		dirQuorum: len(disks) / 2, // make sure to capture all quorum ratios
+		objQuorum: len(disks) / 2, // make sure to capture all quorum ratios
+		bucket:    bi.Name,
+	}
+
+	err := listPathRaw(ctx, listPathRawOptions{
+		disks:          disks,
+		bucket:         bi.Name,
+		path:           bi.Prefix,
+		recursive:      true,
+		forwardTo:      "",
+		minDisks:       len(disks) / 2, // to capture all quorum ratios
+		reportNotFound: false,
+		agreed:         fn,
+		partial: func(entries metaCacheEntries, _ []error) {
+			entry, ok := entries.resolve(&resolver)
+			if ok {
+				fn(*entry)
+			}
+		},
+		finished: nil,
+	})
+	return err
+}
+
 func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bi decomBucketInfo) error {
 	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})
 
@@ -700,12 +720,6 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 	for _, set := range pool.sets {
 		set := set
-		disks := set.getOnlineDisks()
-		if len(disks) == 0 {
-			logger.LogIf(GlobalContext, fmt.Errorf("no online drives found for set with endpoints %s",
-				set.getEndpoints()))
-			continue
-		}
 
 		vc, _ := globalBucketVersioningSys.Get(bi.Name)
 
@@ -879,39 +893,17 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			z.poolMetaMutex.Unlock()
 		}
 
-		// How to resolve partial results.
-		resolver := metadataResolutionParams{
-			dirQuorum: len(disks) / 2, // make sure to capture all quorum ratios
-			objQuorum: len(disks) / 2, // make sure to capture all quorum ratios
-			bucket:    bi.Name,
-		}
-
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			err := listPathRaw(ctx, listPathRawOptions{
-				disks:          disks,
-				bucket:         bi.Name,
-				path:           bi.Prefix,
-				recursive:      true,
-				forwardTo:      "",
-				minDisks:       len(disks) / 2, // to capture all quorum ratios
-				reportNotFound: false,
-				agreed: func(entry metaCacheEntry) {
-					parallelWorkers <- struct{}{}
+			err := set.listObjectsToDecommission(ctx, bi,
+				func(entry metaCacheEntry) {
+					// Wait must be synchronized here.
 					wg.Add(1)
+					parallelWorkers <- struct{}{}
 					go decommissionEntry(entry)
 				},
-				partial: func(entries metaCacheEntries, _ []error) {
-					entry, ok := entries.resolve(&resolver)
-					if ok {
-						parallelWorkers <- struct{}{}
-						wg.Add(1)
-						go decommissionEntry(*entry)
-					}
-				},
-				finished: nil,
-			})
+			)
 			logger.LogIf(ctx, err)
 		}()
 	}
@@ -992,6 +984,33 @@ func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx i
 	return nil
 }
 
+func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error {
+	buckets, err := z.getBucketsToDecommission(ctx)
+	if err != nil {
+		return err
+	}
+
+	pool := z.serverPools[idx]
+	for _, set := range pool.sets {
+		for _, bi := range buckets {
+			var objectsFound int
+			err := set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) {
+				if entry.isObject() {
+					objectsFound++
+				}
+			})
+			if err != nil {
+				return err
+			}
+			if objectsFound > 0 {
+				return fmt.Errorf("at least %d objects were found in bucket `%s` after decommissioning", objectsFound, bi.Name)
+			}
+		}
+	}
+
+	return nil
+}
+
 func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx int) {
 	z.poolMetaMutex.Lock()
 	var dctx context.Context
@@ -1008,9 +1027,19 @@ func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx in
 	}
 
 	z.poolMetaMutex.Lock()
-	failed := z.poolMeta.Pools[idx].Decommission.ItemsDecommissionFailed > 0
+	failed := z.poolMeta.Pools[idx].Decommission.ItemsDecommissionFailed > 0 || contextCanceled(dctx)
+	poolCmdLine := z.poolMeta.Pools[idx].CmdLine
 	z.poolMetaMutex.Unlock()
 
+	if !failed {
+		logger.Info("Decommissioning complete for pool '%s', verifying for any pending objects", poolCmdLine)
+		err := z.checkAfterDecom(dctx, idx)
+		if err != nil {
+			logger.LogIf(ctx, err)
+			failed = true
+		}
+	}
+
 	if failed {
 		// Decommission failed indicate as such.
 		logger.LogIf(GlobalContext, z.DecommissionFailed(dctx, idx))
@@ -1027,8 +1056,8 @@ func (z *erasureServerPools) IsSuspended(idx int) bool {
 }
 
 // Decommission - start decommission session.
-func (z *erasureServerPools) Decommission(ctx context.Context, idx int) error {
-	if idx < 0 {
+func (z *erasureServerPools) Decommission(ctx context.Context, indices ...int) error {
+	if len(indices) == 0 {
 		return errInvalidArgument
 	}
 
@@ -1037,11 +1066,15 @@ func (z *erasureServerPools) Decommission(ctx context.Context, idx int) error {
 	}
 
 	// Make pool unwritable before decommissioning.
-	if err := z.StartDecommission(ctx, idx); err != nil {
+	if err := z.StartDecommission(ctx, indices...); err != nil {
 		return err
 	}
 
-	go z.doDecommissionInRoutine(ctx, idx)
+	go func() {
+		for _, idx := range indices {
+			z.doDecommissionInRoutine(ctx, idx)
+		}
+	}()
 
 	// Successfully started decommissioning.
 	return nil
@@ -1193,25 +1226,10 @@ func (z *erasureServerPools) CompleteDecommission(ctx context.Context, idx int)
 	return nil
 }
 
-func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (err error) {
-	if idx < 0 {
-		return errInvalidArgument
-	}
-
-	if z.SinglePool() {
-		return errInvalidArgument
-	}
-
+func (z *erasureServerPools) getBucketsToDecommission(ctx context.Context) ([]decomBucketInfo, error) {
 	buckets, err := z.ListBuckets(ctx, BucketOptions{})
 	if err != nil {
-		return err
-	}
-
-	// Make sure to heal the buckets to ensure the new
-	// pool has the new buckets, this is to avoid
-	// failures later.
-	for _, bucket := range buckets {
-		z.HealBucket(ctx, bucket.Name, madmin.HealOpts{})
+		return nil, err
 	}
 
 	decomBuckets := make([]decomBucketInfo, len(buckets))
@@ -1221,6 +1239,38 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		}
 	}
 
+	// Buckets data are dispersed in multiple zones/sets, make
+	// sure to decommission the necessary metadata.
+	decomBuckets = append(decomBuckets, decomBucketInfo{
+		Name:   minioMetaBucket,
+		Prefix: minioConfigPrefix,
+	})
+	decomBuckets = append(decomBuckets, decomBucketInfo{
+		Name:   minioMetaBucket,
+		Prefix: bucketMetaPrefix,
+	})
+
+	return decomBuckets, nil
+}
+
+func (z *erasureServerPools) StartDecommission(ctx context.Context, indices ...int) (err error) {
+	if len(indices) == 0 {
+		return errInvalidArgument
+	}
+
+	if z.SinglePool() {
+		return errInvalidArgument
+	}
+
+	decomBuckets, err := z.getBucketsToDecommission(ctx)
+	if err != nil {
+		return err
+	}
+
+	for _, bucket := range decomBuckets {
+		z.HealBucket(ctx, bucket.Name, madmin.HealOpts{})
+	}
+
 	// TODO: Support decommissioning transition tiers.
 	for _, bucket := range decomBuckets {
 		if lc, err := globalLifecycleSys.Get(bucket.Name); err == nil {
@@ -1240,52 +1290,35 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, idx int) (er
 		pathJoin(minioMetaBucket, bucketMetaPrefix),
 	} {
 		var bucketExists BucketExists
-		if err = z.MakeBucketWithLocation(ctx, metaBucket, MakeBucketOptions{}); err != nil {
+		if err = z.MakeBucket(ctx, metaBucket, MakeBucketOptions{}); err != nil {
 			if !errors.As(err, &bucketExists) {
 				return err
 			}
 		}
 	}
 
-	// Buckets data are dispersed in multiple zones/sets, make
-	// sure to decommission the necessary metadata.
-	decomBuckets = append(decomBuckets, decomBucketInfo{
-		Name:   minioMetaBucket,
-		Prefix: minioConfigPrefix,
-	})
-	decomBuckets = append(decomBuckets, decomBucketInfo{
-		Name:   minioMetaBucket,
-		Prefix: bucketMetaPrefix,
-	})
-
-	var pool *erasureSets
-	for pidx := range z.serverPools {
-		if pidx == idx {
-			pool = z.serverPools[idx]
-			break
-		}
-	}
-
-	if pool == nil {
-		return errInvalidArgument
-	}
-
-	pi, err := z.getDecommissionPoolSpaceInfo(idx)
-	if err != nil {
-		return err
-	}
-
 	z.poolMetaMutex.Lock()
 	defer z.poolMetaMutex.Unlock()
 
-	if err = z.poolMeta.Decommission(idx, pi); err != nil {
-		return err
+	for _, idx := range indices {
+		pi, err := z.getDecommissionPoolSpaceInfo(idx)
+		if err != nil {
+			return err
+		}
+
+		if err = z.poolMeta.Decommission(idx, pi); err != nil {
+			return err
+		}
+
+		z.poolMeta.QueueBuckets(idx, decomBuckets)
 	}
-	z.poolMeta.QueueBuckets(idx, decomBuckets)
+
 	if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 		return err
 	}
+
 	globalNotificationSys.ReloadPoolMeta(ctx)
+
 	return nil
 }
 

cmd/erasure-server-pool-rebalance.go

@@ -57,15 +57,16 @@ type rebalanceStats struct {
 	Info              rebalanceInfo `json:"info" msg:"inf"`
 }
 
-func (rs *rebalanceStats) update(bucket string, oi ObjectInfo) {
-	if oi.IsLatest {
+func (rs *rebalanceStats) update(bucket string, fi FileInfo) {
+	if fi.IsLatest {
 		rs.NumObjects++
 	}
 
 	rs.NumVersions++
-	rs.Bytes += uint64(oi.Size)
+	onDiskSz := fi.Size * int64(fi.Erasure.DataBlocks+fi.Erasure.ParityBlocks) / int64(fi.Erasure.DataBlocks)
+	rs.Bytes += uint64(onDiskSz)
 	rs.Bucket = bucket
-	rs.Object = oi.Name
+	rs.Object = fi.Name
 }
 
 type rstats []*rebalanceStats
@@ -132,6 +133,11 @@ func (z *erasureServerPools) initRebalanceMeta(ctx context.Context, buckets []st
 	}, len(z.serverPools))
 	var totalCap, totalFree uint64
 	for _, disk := range si.Disks {
+		// Ignore invalid.
+		if disk.PoolIndex < 0 || len(diskStats) <= disk.PoolIndex {
+			// https://github.com/minio/minio/issues/16500
+			continue
+		}
 		totalCap += disk.TotalSpace
 		totalFree += disk.AvailableSpace
 
@@ -168,7 +174,7 @@ func (z *erasureServerPools) initRebalanceMeta(ctx context.Context, buckets []st
 	return r.ID, nil
 }
 
-func (z *erasureServerPools) updatePoolStats(poolIdx int, bucket string, oi ObjectInfo) {
+func (z *erasureServerPools) updatePoolStats(poolIdx int, bucket string, fi FileInfo) {
 	z.rebalMu.Lock()
 	defer z.rebalMu.Unlock()
 
@@ -177,7 +183,7 @@ func (z *erasureServerPools) updatePoolStats(poolIdx int, bucket string, oi Obje
 		return
 	}
 
-	r.PoolStats[poolIdx].update(bucket, oi)
+	r.PoolStats[poolIdx].update(bucket, fi)
 }
 
 const (
@@ -352,6 +358,7 @@ func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int)
 				z.rebalMu.Lock()
 				z.rebalMeta.PoolStats[poolIdx].Info.Status = rebalStopped
 				z.rebalMeta.PoolStats[poolIdx].Info.EndTime = now
+				z.rebalMeta.cancel = nil // remove the already used context.CancelFunc
 				z.rebalMu.Unlock()
 
 				rebalDone = true
@@ -534,14 +541,13 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					}
 
 					if !failure {
-						z.updatePoolStats(poolIdx, bucket, version.ToObjectInfo(bucket, version.Name, vc.PrefixEnabled(version.Name)))
+						z.updatePoolStats(poolIdx, bucket, version)
 						rebalanced++
 					}
 					continue
 				}
 
 				var failure bool
-				var oi ObjectInfo
 				for try := 0; try < 3; try++ {
 					// GetObjectReader.Close is called by rebalanceObject
 					gr, err := set.GetObjectNInfo(ctx,
@@ -574,14 +580,13 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 
 					stopFn(nil)
 					failure = false
-					oi = gr.ObjInfo
 					break
 				}
 
 				if failure {
 					break // break out on first error
 				}
-				z.updatePoolStats(poolIdx, bucket, oi)
+				z.updatePoolStats(poolIdx, bucket, version)
 				rebalanced++
 			}
 
@@ -656,7 +661,7 @@ func (z *erasureServerPools) saveRebalanceStats(ctx context.Context, poolIdx int
 		logger.LogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
 		return err
 	}
-	defer lock.Unlock(lkCtx.Cancel)
+	defer lock.Unlock(lkCtx)
 
 	ctx = lkCtx.Context()
 	noLockOpts := ObjectOptions{NoLock: true}

cmd/erasure-server-pool.go

@@ -32,7 +32,9 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
+	"github.com/google/uuid"
 	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -49,13 +51,15 @@ type erasureServerPools struct {
 	rebalMu   sync.RWMutex
 	rebalMeta *rebalanceMeta
 
-	serverPools []*erasureSets
+	deploymentID     [16]byte
+	distributionAlgo string
 
-	// Shut down async operations
-	shutdown context.CancelFunc
+	serverPools []*erasureSets
 
 	// Active decommission canceler
 	decommissionCancelers []context.CancelFunc
+
+	s3Peer *S3PeerSys
 }
 
 func (z *erasureServerPools) SinglePool() bool {
@@ -74,6 +78,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		storageDisks = make([][]StorageAPI, len(endpointServerPools))
 		z            = &erasureServerPools{
 			serverPools: make([]*erasureSets, len(endpointServerPools)),
+			s3Peer:      NewS3PeerSys(endpointServerPools),
 		}
 	)
 
@@ -86,11 +91,14 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		// -- Default for Standard Storage class is, parity = 3 - disks 6, 7
 		// -- Default for Standard Storage class is, parity = 4 - disks 8 to 16
 		if commonParityDrives == 0 {
-			commonParityDrives = ecDrivesNoConfig(ep.DrivesPerSet)
+			commonParityDrives, err = ecDrivesNoConfig(ep.DrivesPerSet)
+			if err != nil {
+				return nil, err
+			}
 		}
 
 		if err = storageclass.ValidateParity(commonParityDrives, ep.DrivesPerSet); err != nil {
-			return nil, fmt.Errorf("parity validation returned an error %w <- (%d, %d), for pool(%s)", err, commonParityDrives, ep.DrivesPerSet, humanize.Ordinal(i+1))
+			return nil, fmt.Errorf("parity validation returned an error: %w <- (%d, %d), for pool(%s)", err, commonParityDrives, ep.DrivesPerSet, humanize.Ordinal(i+1))
 		}
 
 		storageDisks[i], formats[i], err = waitForFormatErasure(local, ep.Endpoints, i+1,
@@ -123,6 +131,14 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		if err != nil {
 			return nil, err
 		}
+
+		if deploymentID != "" && bytes.Equal(z.deploymentID[:], []byte{}) {
+			z.deploymentID = uuid.MustParse(deploymentID)
+		}
+
+		if distributionAlgo != "" && z.distributionAlgo == "" {
+			z.distributionAlgo = distributionAlgo
+		}
 	}
 
 	z.decommissionCancelers = make([]context.CancelFunc, len(z.serverPools))
@@ -141,19 +157,16 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		break
 	}
 
-	drives := make([]string, 0, len(localDrives))
-	for _, localDrive := range localDrives {
-		drives = append(drives, localDrive.Endpoint().Path)
-	}
-
 	globalLocalDrives = localDrives
-	ctx, z.shutdown = context.WithCancel(ctx)
-	go intDataUpdateTracker.start(ctx, drives...)
 	return z, nil
 }
 
 func (z *erasureServerPools) NewNSLock(bucket string, objects ...string) RWLocker {
-	return z.serverPools[0].NewNSLock(bucket, objects...)
+	poolID := hashKey(z.distributionAlgo, "", len(z.serverPools), z.deploymentID)
+	if len(objects) >= 1 {
+		poolID = hashKey(z.distributionAlgo, objects[0], len(z.serverPools), z.deploymentID)
+	}
+	return z.serverPools[poolID].NewNSLock(bucket, objects...)
 }
 
 // GetDisksID will return disks by their ID.
@@ -164,10 +177,8 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
 	}
 	res := make([]StorageAPI, 0, len(idMap))
 	for _, s := range z.serverPools {
-		s.erasureDisksMu.RLock()
-		defer s.erasureDisksMu.RUnlock()
-		for _, disks := range s.erasureDisks {
-			for _, disk := range disks {
+		for _, set := range s.sets {
+			for _, disk := range set.getDisks() {
 				if disk == OfflineDisk {
 					continue
 				}
@@ -188,8 +199,8 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
 func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error {
 	found := 0
 	for _, s := range z.serverPools {
-		for _, disks := range s.erasureDisks {
-			for _, disk := range disks {
+		for _, set := range s.sets {
+			for _, disk := range set.getDisks() {
 				if disk == OfflineDisk {
 					continue
 				}
@@ -503,8 +514,6 @@ func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object stri
 }
 
 func (z *erasureServerPools) Shutdown(ctx context.Context) error {
-	defer z.shutdown()
-
 	g := errgroup.WithNErrs(len(z.serverPools))
 
 	for index := range z.serverPools {
@@ -573,7 +582,7 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context) StorageInfo {
 	return globalNotificationSys.StorageInfo(z)
 }
 
-func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
+func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
 	// Updates must be closed before we return.
 	defer close(updates)
 
@@ -618,7 +627,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 					}
 				}()
 				// Start scanner. Blocks until done.
-				err := erObj.nsScanner(ctx, allBuckets, bf, wantCycle, updates, healScanMode)
+				err := erObj.nsScanner(ctx, allBuckets, wantCycle, updates, healScanMode)
 				if err != nil {
 					logger.LogIf(ctx, err)
 					mu.Lock()
@@ -687,45 +696,38 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, bf *bloomFilter, upd
 	return firstErr
 }
 
-// MakeBucketWithLocation - creates a new bucket across all serverPools simultaneously
+// MakeBucket - creates a new bucket across all serverPools simultaneously
 // even if one of the sets fail to create buckets, we proceed all the successful
 // operations.
-func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
-	g := errgroup.WithNErrs(len(z.serverPools))
-
-	// Lock the bucket name before creating.
-	lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck")
-	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return err
-	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
-
-	// Create buckets in parallel across all sets.
-	for index := range z.serverPools {
-		index := index
-		g.Go(func() error {
-			if z.IsSuspended(index) {
-				return nil
-			}
-			return z.serverPools[index].MakeBucketWithLocation(ctx, bucket, opts)
-		}, index)
-	}
-
-	errs := g.Wait()
-	// Return the first encountered error
-	for _, err := range errs {
-		if err != nil {
-			if _, ok := err.(BucketExists); !ok {
-				// Delete created buckets, ignoring errors.
-				z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
-					Force:      false,
-					NoRecreate: true,
-				})
-			}
-			return err
+func (z *erasureServerPools) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error {
+	// Verify if bucket is valid.
+	if !isMinioMetaBucketName(bucket) {
+		if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil {
+			return BucketNameInvalid{Bucket: bucket}
 		}
+
+		if !opts.NoLock {
+			// Lock the bucket name before creating.
+			lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck")
+			lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+			if err != nil {
+				return err
+			}
+
+			ctx = lkctx.Context()
+			defer lk.Unlock(lkctx)
+		}
+	}
+
+	if err := z.s3Peer.MakeBucket(ctx, bucket, opts); err != nil {
+		if _, ok := err.(BucketExists); !ok {
+			// Delete created buckets, ignoring errors.
+			z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{
+				NoLock:     true,
+				NoRecreate: true,
+			})
+		}
+		return err
 	}
 
 	// If it doesn't exist we get a new, so ignore errors
@@ -779,14 +781,14 @@ func (z *erasureServerPools) GetObjectNInfo(ctx context.Context, bucket, object
 				return nil, err
 			}
 			ctx = lkctx.Context()
-			nsUnlocker = func() { lock.Unlock(lkctx.Cancel) }
+			nsUnlocker = func() { lock.Unlock(lkctx) }
 		case readLock:
 			lkctx, err := lock.GetRLock(ctx, globalOperationTimeout)
 			if err != nil {
 				return nil, err
 			}
 			ctx = lkctx.Context()
-			nsUnlocker = func() { lock.RUnlock(lkctx.Cancel) }
+			nsUnlocker = func() { lock.RUnlock(lkctx) }
 		}
 		unlockOnDefer = true
 	}
@@ -907,7 +909,7 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
 			return ObjectInfo{}, err
 		}
 		ctx = lkctx.Context()
-		defer lk.RUnlock(lkctx.Cancel)
+		defer lk.RUnlock(lkctx)
 	}
 
 	objInfo, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts)
@@ -936,7 +938,7 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 			return ObjectInfo{}, err
 		}
 		ctx = lkctx.Context()
-		defer ns.Unlock(lkctx.Cancel)
+		defer ns.Unlock(lkctx)
 		opts.NoLock = true
 	}
 
@@ -977,7 +979,7 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return ObjectInfo{}, err
 	}
 	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
+	defer lk.Unlock(lkctx)
 
 	gopts := opts
 	gopts.NoLock = true
@@ -1022,7 +1024,7 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 		return dobjects, derrs
 	}
 	ctx = lkctx.Context()
-	defer multiDeleteLock.Unlock(lkctx.Cancel)
+	defer multiDeleteLock.Unlock(lkctx)
 
 	// Fetch location of up to 10 objects concurrently.
 	poolObjIdxMap := map[int][]ObjectToDelete{}
@@ -1050,7 +1052,7 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 					derrs[j] = err
 				}
 				dobjects[j] = DeletedObject{
-					ObjectName: obj.ObjectName,
+					ObjectName: decodeDirObject(obj.ObjectName),
 					VersionID:  obj.VersionID,
 				}
 				return nil
@@ -1062,7 +1064,7 @@ func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, o
 					DeleteMarker:          pinfo.ObjInfo.DeleteMarker,
 					DeleteMarkerVersionID: pinfo.ObjInfo.VersionID,
 					DeleteMarkerMTime:     DeleteMarkerMTime{pinfo.ObjInfo.ModTime},
-					ObjectName:            pinfo.ObjInfo.Name,
+					ObjectName:            decodeDirObject(pinfo.ObjInfo.Name),
 				}
 				return nil
 			}
@@ -1122,7 +1124,7 @@ func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObjec
 			return ObjectInfo{}, err
 		}
 		ctx = lkctx.Context()
-		defer ns.Unlock(lkctx.Cancel)
+		defer ns.Unlock(lkctx)
 		dstOpts.NoLock = true
 	}
 
@@ -1191,6 +1193,7 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 	if marker == "" && versionMarker != "" {
 		return loi, NotImplemented{}
 	}
+
 	opts := listPathOptions{
 		Bucket:      bucket,
 		Prefix:      prefix,
@@ -1201,6 +1204,17 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 		AskDisks:    globalAPIConfig.getListQuorum(),
 		Versioned:   true,
 	}
+
+	// Shortcut for APN/1.0 Veeam/1.0 Backup/10.0
+	// It requests unique blocks with a specific prefix.
+	// We skip scanning the parent directory for
+	// more objects matching the prefix.
+	ri := logger.GetReqInfo(ctx)
+	if ri != nil && strings.Contains(ri.UserAgent, `1.0 Veeam/1.0 Backup`) && strings.HasSuffix(prefix, ".blk") {
+		opts.BaseDir = prefix
+		opts.Transient = true
+	}
+
 	// set bucket metadata in opts
 	opts.setBucketMeta(ctx)
 
@@ -1582,103 +1596,65 @@ func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket
 
 // GetBucketInfo - returns bucket info from one of the erasure coded serverPools.
 func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
-	if z.SinglePool() {
-		bucketInfo, err = z.serverPools[0].GetBucketInfo(ctx, bucket, opts)
-		if err != nil {
-			return bucketInfo, err
-		}
-		meta, err := globalBucketMetadataSys.Get(bucket)
-		if err == nil {
-			bucketInfo.Created = meta.Created
-			bucketInfo.Versioning = meta.LockEnabled || globalBucketVersioningSys.Enabled(bucket)
-			bucketInfo.ObjectLocking = meta.LockEnabled
-		}
-		return bucketInfo, nil
+	bucketInfo, err = z.s3Peer.GetBucketInfo(ctx, bucket, opts)
+	if err != nil {
+		return bucketInfo, toObjectErr(err, bucket)
 	}
-	for _, pool := range z.serverPools {
-		bucketInfo, err = pool.GetBucketInfo(ctx, bucket, opts)
-		if err != nil {
-			if isErrBucketNotFound(err) {
-				continue
-			}
-			return bucketInfo, err
-		}
-		meta, err := globalBucketMetadataSys.Get(bucket)
-		if err == nil {
-			bucketInfo.Created = meta.Created
-			bucketInfo.Versioning = meta.LockEnabled || globalBucketVersioningSys.Enabled(bucket)
-			bucketInfo.ObjectLocking = meta.LockEnabled
-		}
-		return bucketInfo, nil
+
+	meta, err := globalBucketMetadataSys.Get(bucket)
+	if err == nil {
+		bucketInfo.Created = meta.Created
+		bucketInfo.Versioning = meta.LockEnabled || globalBucketVersioningSys.Enabled(bucket)
+		bucketInfo.ObjectLocking = meta.LockEnabled
 	}
-	return bucketInfo, BucketNotFound{
-		Bucket: bucket,
-	}
-}
-
-// IsNotificationSupported returns whether bucket notification is applicable for this layer.
-func (z *erasureServerPools) IsNotificationSupported() bool {
-	return true
-}
-
-// IsListenSupported returns whether listen bucket notification is applicable for this layer.
-func (z *erasureServerPools) IsListenSupported() bool {
-	return true
-}
-
-// IsEncryptionSupported returns whether server side encryption is implemented for this layer.
-func (z *erasureServerPools) IsEncryptionSupported() bool {
-	return true
-}
-
-// IsCompressionSupported returns whether compression is applicable for this layer.
-func (z *erasureServerPools) IsCompressionSupported() bool {
-	return true
-}
-
-func (z *erasureServerPools) IsTaggingSupported() bool {
-	return true
+	return bucketInfo, nil
 }
 
 // DeleteBucket - deletes a bucket on all serverPools simultaneously,
 // even if one of the serverPools fail to delete buckets, we proceed to
 // undo a successful operation.
 func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
-	g := errgroup.WithNErrs(len(z.serverPools))
-
-	// Delete buckets in parallel across all serverPools.
-	for index := range z.serverPools {
-		index := index
-		g.Go(func() error {
-			if z.IsSuspended(index) {
-				return nil
-			}
-			return z.serverPools[index].DeleteBucket(ctx, bucket, opts)
-		}, index)
+	if isMinioMetaBucketName(bucket) {
+		return BucketNameInvalid{Bucket: bucket}
 	}
 
-	errs := g.Wait()
+	// Verify if bucket is valid.
+	if err := s3utils.CheckValidBucketName(bucket); err != nil {
+		return BucketNameInvalid{Bucket: bucket}
+	}
 
-	// For any write quorum failure, we undo all the delete
-	// buckets operation by creating all the buckets again.
-	for _, err := range errs {
+	if !opts.NoLock {
+		// Lock the bucket name before creating.
+		lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck")
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 		if err != nil {
-			if !z.SinglePool() && !opts.NoRecreate {
-				undoDeleteBucketServerPools(context.Background(), bucket, z.serverPools, errs)
-			}
 			return err
 		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx)
+	}
+
+	err := z.s3Peer.DeleteBucket(ctx, bucket, opts)
+	if err == nil || errors.Is(err, errVolumeNotFound) {
+		// If site replication is configured, hold on to deleted bucket state until sites sync
+		switch opts.SRDeleteOp {
+		case MarkDelete:
+			z.s3Peer.MakeBucket(context.Background(), pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket), MakeBucketOptions{})
+		}
 	}
 
-	// Purge the entire bucket metadata entirely.
-	z.deleteAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
-	// If site replication is configured, hold on to deleted bucket state until sites sync
-	switch opts.SRDeleteOp {
-	case MarkDelete:
-		z.markDelete(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix, bucket))
+	if err != nil && !errors.Is(err, errVolumeNotFound) {
+		if !opts.NoRecreate {
+			z.s3Peer.MakeBucket(ctx, bucket, MakeBucketOptions{})
+		}
 	}
-	// Success.
-	return nil
+
+	if err == nil {
+		// Purge the entire bucket metadata entirely.
+		z.deleteAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
+	}
+
+	return toObjectErr(err, bucket)
 }
 
 // deleteAll will rename bucket+prefix unconditionally across all disks to
@@ -1694,64 +1670,11 @@ func (z *erasureServerPools) deleteAll(ctx context.Context, bucket, prefix strin
 	}
 }
 
-// markDelete will create a directory of deleted bucket in .minio.sys/buckets/.deleted across all disks
-// in situations where the deleted bucket needs to be held on to until all sites are in sync for
-// site replication
-func (z *erasureServerPools) markDelete(ctx context.Context, bucket, prefix string) {
-	for _, servers := range z.serverPools {
-		for _, set := range servers.sets {
-			set.markDelete(ctx, bucket, prefix)
-		}
-	}
-}
-
-// purgeDelete deletes vol entry in .minio.sys/buckets/.deleted after site replication
-// syncs the delete to peers.
-func (z *erasureServerPools) purgeDelete(ctx context.Context, bucket, prefix string) {
-	for _, servers := range z.serverPools {
-		for _, set := range servers.sets {
-			set.purgeDelete(ctx, bucket, prefix)
-		}
-	}
-}
-
-// This function is used to undo a successful DeleteBucket operation.
-func undoDeleteBucketServerPools(ctx context.Context, bucket string, serverPools []*erasureSets, errs []error) {
-	g := errgroup.WithNErrs(len(serverPools))
-
-	// Undo previous delete bucket on all underlying serverPools.
-	for index := range serverPools {
-		index := index
-		g.Go(func() error {
-			if errs[index] == nil {
-				return serverPools[index].MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
-			}
-			return nil
-		}, index)
-	}
-
-	g.Wait()
-}
-
 // List all buckets from one of the serverPools, we are not doing merge
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all serverPools.
 func (z *erasureServerPools) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
-	if z.SinglePool() {
-		buckets, err = z.serverPools[0].ListBuckets(ctx, opts)
-	} else {
-		for idx, pool := range z.serverPools {
-			if z.IsSuspended(idx) {
-				continue
-			}
-			buckets, err = pool.ListBuckets(ctx, opts)
-			if err != nil {
-				logger.LogIf(ctx, err)
-				continue
-			}
-			break
-		}
-	}
+	buckets, err = z.s3Peer.ListBuckets(ctx, opts)
 	if err != nil {
 		return nil, err
 	}
@@ -1772,7 +1695,7 @@ func (z *erasureServerPools) HealFormat(ctx context.Context, dryRun bool) (madmi
 		return madmin.HealResultItem{}, err
 	}
 	ctx = lkctx.Context()
-	defer formatLock.Unlock(lkctx.Cancel)
+	defer formatLock.Unlock(lkctx)
 
 	r := madmin.HealResultItem{
 		Type:   madmin.HealItemMetadata,
@@ -1954,7 +1877,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 // HealObjectFn closure function heals the object.
 type HealObjectFn func(bucket, object, versionID string) error
 
-func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects, healEntry func(metaCacheEntry) error) error {
+func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects, healEntry func(string, metaCacheEntry) error) error {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
@@ -1987,7 +1910,7 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 		minDisks:       1,
 		reportNotFound: false,
 		agreed: func(entry metaCacheEntry) {
-			if err := healEntry(entry); err != nil {
+			if err := healEntry(bucket, entry); err != nil {
 				logger.LogIf(ctx, err)
 				cancel()
 			}
@@ -2000,7 +1923,7 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 				entry, _ = entries.firstFound()
 			}
 
-			if err := healEntry(*entry); err != nil {
+			if err := healEntry(bucket, *entry); err != nil {
 				logger.LogIf(ctx, err)
 				cancel()
 				return
@@ -2017,7 +1940,7 @@ func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects
 }
 
 func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error {
-	healEntry := func(entry metaCacheEntry) error {
+	healEntry := func(bucket string, entry metaCacheEntry) error {
 		if entry.isDir() {
 			return nil
 		}

cmd/erasure-sets.go

@@ -26,7 +26,6 @@ import (
 	"math/rand"
 	"net/http"
 	"reflect"
-	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -355,10 +354,6 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 		endpointStrings[i] = endpoint.String()
 	}
 
-	if defaultParityCount == 0 {
-		logger.Error("Warning: Default parity set to 0. This can lead to data loss.")
-	}
-
 	// Initialize the erasure sets instance.
 	s := &erasureSets{
 		sets:               make([]*erasureObjects, setCount),
@@ -677,32 +672,6 @@ func (s *erasureSets) Shutdown(ctx context.Context) error {
 	return nil
 }
 
-// MakeBucketLocation - creates a new bucket across all sets simultaneously,
-// then return the first encountered error
-func (s *erasureSets) MakeBucketWithLocation(ctx context.Context, bucket string, opts MakeBucketOptions) error {
-	g := errgroup.WithNErrs(len(s.sets))
-
-	// Create buckets in parallel across all sets.
-	for index := range s.sets {
-		index := index
-		g.Go(func() error {
-			return s.sets[index].MakeBucketWithLocation(ctx, bucket, opts)
-		}, index)
-	}
-
-	errs := g.Wait()
-
-	// Return the first encountered error
-	for _, err := range errs {
-		if err != nil {
-			return err
-		}
-	}
-
-	// Success.
-	return nil
-}
-
 // hashes the key returning an integer based on the input algorithm.
 // This function currently supports
 // - CRCMOD
@@ -749,131 +718,6 @@ func (s *erasureSets) getHashedSet(input string) (set *erasureObjects) {
 	return s.sets[s.getHashedSetIndex(input)]
 }
 
-// GetBucketInfo - returns bucket info from one of the erasure coded set.
-func (s *erasureSets) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error) {
-	return s.getHashedSet("").GetBucketInfo(ctx, bucket, opts)
-}
-
-// IsNotificationSupported returns whether bucket notification is applicable for this layer.
-func (s *erasureSets) IsNotificationSupported() bool {
-	return s.getHashedSet("").IsNotificationSupported()
-}
-
-// IsListenSupported returns whether listen bucket notification is applicable for this layer.
-func (s *erasureSets) IsListenSupported() bool {
-	return true
-}
-
-// IsEncryptionSupported returns whether server side encryption is implemented for this layer.
-func (s *erasureSets) IsEncryptionSupported() bool {
-	return s.getHashedSet("").IsEncryptionSupported()
-}
-
-// IsCompressionSupported returns whether compression is applicable for this layer.
-func (s *erasureSets) IsCompressionSupported() bool {
-	return s.getHashedSet("").IsCompressionSupported()
-}
-
-func (s *erasureSets) IsTaggingSupported() bool {
-	return true
-}
-
-// DeleteBucket - deletes a bucket on all sets simultaneously,
-// even if one of the sets fail to delete buckets, we proceed to
-// undo a successful operation.
-func (s *erasureSets) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error {
-	g := errgroup.WithNErrs(len(s.sets))
-
-	// Delete buckets in parallel across all sets.
-	for index := range s.sets {
-		index := index
-		g.Go(func() error {
-			return s.sets[index].DeleteBucket(ctx, bucket, opts)
-		}, index)
-	}
-
-	errs := g.Wait()
-	// For any failure, we attempt undo all the delete buckets operation
-	// by creating buckets again on all sets which were successfully deleted.
-	for _, err := range errs {
-		if err != nil && !opts.NoRecreate {
-			undoDeleteBucketSets(ctx, bucket, s.sets, errs)
-			return err
-		}
-	}
-
-	// Success.
-	return nil
-}
-
-// This function is used to undo a successful DeleteBucket operation.
-func undoDeleteBucketSets(ctx context.Context, bucket string, sets []*erasureObjects, errs []error) {
-	g := errgroup.WithNErrs(len(sets))
-
-	// Undo previous delete bucket on all underlying sets.
-	for index := range sets {
-		index := index
-		g.Go(func() error {
-			if errs[index] == nil {
-				return sets[index].MakeBucketWithLocation(ctx, bucket, MakeBucketOptions{})
-			}
-			return nil
-		}, index)
-	}
-
-	g.Wait()
-}
-
-// List all buckets from one of the set, we are not doing merge
-// sort here just for simplification. As per design it is assumed
-// that all buckets are present on all sets.
-func (s *erasureSets) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
-	var listBuckets []BucketInfo
-	healBuckets := map[string]VolInfo{}
-	for _, set := range s.sets {
-		// lists all unique buckets across drives.
-		if err := listAllBuckets(ctx, set.getDisks(), healBuckets, s.defaultParityCount); err != nil {
-			return nil, err
-		}
-	}
-
-	// include deleted buckets in listBuckets output
-	deletedBuckets := map[string]VolInfo{}
-
-	if opts.Deleted {
-		for _, set := range s.sets {
-			// lists all unique buckets across drives.
-			if err := listDeletedBuckets(ctx, set.getDisks(), deletedBuckets, s.defaultParityCount); err != nil {
-				return nil, err
-			}
-		}
-	}
-	for _, v := range healBuckets {
-		bi := BucketInfo{
-			Name:    v.Name,
-			Created: v.Created,
-		}
-		if vi, ok := deletedBuckets[v.Name]; ok {
-			bi.Deleted = vi.Created
-		}
-		listBuckets = append(listBuckets, bi)
-	}
-	for _, v := range deletedBuckets {
-		if _, ok := healBuckets[v.Name]; !ok {
-			listBuckets = append(listBuckets, BucketInfo{
-				Name:    v.Name,
-				Deleted: v.Created,
-			})
-		}
-	}
-
-	sort.Slice(listBuckets, func(i, j int) bool {
-		return listBuckets[i].Name < listBuckets[j].Name
-	})
-
-	return listBuckets, nil
-}
-
 // listDeletedBuckets lists deleted buckets from all disks.
 func listDeletedBuckets(ctx context.Context, storageDisks []StorageAPI, delBuckets map[string]VolInfo, readQuorum int) error {
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -887,25 +731,21 @@ func listDeletedBuckets(ctx context.Context, storageDisks []StorageAPI, delBucke
 			}
 			volsInfo, err := storageDisks[index].ListDir(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix), -1)
 			if err != nil {
-				if err == errFileNotFound {
+				if errors.Is(err, errFileNotFound) {
 					return nil
 				}
 				return err
 			}
 			for _, volName := range volsInfo {
-				mu.Lock()
-				if _, ok := delBuckets[volName]; !ok {
-					vi, err := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, volName))
-					if err != nil {
-						return err
+				vi, err := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, volName))
+				if err == nil {
+					vi.Name = strings.TrimSuffix(volName, SlashSeparator)
+					mu.Lock()
+					if _, ok := delBuckets[volName]; !ok {
+						delBuckets[volName] = vi
 					}
-					bkt := strings.TrimPrefix(vi.Name, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix))
-					bkt = strings.TrimPrefix(bkt, slashSeparator)
-					bkt = strings.TrimSuffix(bkt, slashSeparator)
-					vi.Name = bkt
-					delBuckets[bkt] = vi
+					mu.Unlock()
 				}
-				mu.Unlock()
 			}
 			return nil
 		}, index)
@@ -1281,6 +1121,8 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 		return res, errNoHealRequired
 	}
 
+	formatOpID := mustGetUUID()
+
 	// Initialize a new set of set formats which will be written to disk.
 	newFormatSets := newHealFormatSets(refFormat, s.setCount, s.setDriveCount, formats, sErrs)
 
@@ -1302,7 +1144,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			if storageDisks[index] == nil || format == nil {
 				continue
 			}
-			if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
+			if err := saveFormatErasure(storageDisks[index], format, formatOpID); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Drive %s failed to write updated 'format.json': %v", storageDisks[index], err))
 				tmpNewFormats[index] = nil // this disk failed to write new format
 			}

cmd/erasure-sets_test.go

@@ -191,7 +191,12 @@ func TestNewErasureSets(t *testing.T) {
 	}
 
 	ep := PoolEndpoints{Endpoints: endpoints}
-	if _, err := newErasureSets(ctx, ep, storageDisks, format, ecDrivesNoConfig(16), 0); err != nil {
+
+	parity, err := ecDrivesNoConfig(16)
+	if err != nil {
+		t.Fatalf("Unexpected error during EC drive config: %v", err)
+	}
+	if _, err := newErasureSets(ctx, ep, storageDisks, format, parity, 0); err != nil {
 		t.Fatalf("Unable to initialize erasure")
 	}
 }

cmd/erasure.go

@@ -33,9 +33,14 @@ import (
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
-	"github.com/minio/pkg/console"
 )
 
+// list all errors that can be ignore in a bucket operation.
+var bucketOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errUnformattedDisk)
+
+// list all errors that can be ignored in a bucket metadata operation.
+var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound)
+
 // OfflineDisk represents an unavailable disk.
 var OfflineDisk StorageAPI // zero value is nil
 
@@ -89,10 +94,6 @@ func (er erasureObjects) defaultWQuorum() int {
 	return dataCount
 }
 
-func (er erasureObjects) defaultRQuorum() int {
-	return er.setDriveCount - er.defaultParityCount
-}
-
 // byDiskTotal is a collection satisfying sort.Interface.
 type byDiskTotal []madmin.Disk
 
@@ -349,7 +350,7 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
 
 // nsScanner will start scanning buckets and send updated totals as they are traversed.
 // Updates are sent on a regular basis and the caller *must* consume them.
-func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf *bloomFilter, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
+func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error {
 	if len(buckets) == 0 {
 		return nil
 	}
@@ -375,7 +376,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 		},
 		Cache: make(map[string]dataUsageEntry, len(oldCache.Cache)),
 	}
-	bloom := bf.bytes()
 
 	// Put all buckets into channel.
 	bucketCh := make(chan BucketInfo, len(buckets))
@@ -471,7 +471,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 				if cache.Info.Name == "" {
 					cache.Info.Name = bucket.Name
 				}
-				cache.Info.BloomFilter = bloom
 				cache.Info.SkipHealing = healing
 				cache.Info.NextCycle = wantCycle
 				if cache.Info.Name != bucket.Name {
@@ -494,16 +493,12 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, bf
 							Parent: dataUsageRoot,
 							Entry:  update,
 						}
-						if intDataUpdateTracker.debug {
-							console.Debugln("z:", er.poolIndex, "s:", er.setIndex, "bucket", name, "got update", update)
-						}
 					}
 				}(cache.Info.Name)
 				// Calc usage
 				before := cache.Info.LastUpdate
 				var err error
 				cache, err = disk.NSScanner(ctx, cache, updates, healScanMode)
-				cache.Info.BloomFilter = nil
 				if err != nil {
 					if !cache.Info.LastUpdate.IsZero() && cache.Info.LastUpdate.After(before) {
 						logger.LogIf(ctx, cache.save(ctx, er, cacheName))

cmd/event-notification.go

@@ -248,6 +248,7 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 
 	respElements := map[string]string{
 		"x-amz-request-id": args.RespElements["requestId"],
+		"x-amz-id-2":       args.RespElements["nodeId"],
 		"x-minio-origin-endpoint": func() string {
 			if globalMinioEndpoint != "" {
 				return globalMinioEndpoint
@@ -255,17 +256,18 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 			return getAPIEndpoints()[0]
 		}(), // MinIO specific custom elements.
 	}
-	// Add deployment as part of
-	if globalDeploymentID != "" {
-		respElements["x-minio-deployment-id"] = globalDeploymentID
-	}
+
+	// Add deployment as part of response elements.
+	respElements["x-minio-deployment-id"] = globalDeploymentID
 	if args.RespElements["content-length"] != "" {
 		respElements["content-length"] = args.RespElements["content-length"]
 	}
+
 	keyName := args.Object.Name
 	if escape {
 		keyName = url.QueryEscape(args.Object.Name)
 	}
+
 	newEvent := event.Event{
 		EventVersion:      "2.0",
 		EventSource:       "minio:s3",
@@ -312,12 +314,13 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 }
 
 func sendEvent(args eventArgs) {
-	args.Object.Size, _ = args.Object.GetActualSize()
-
 	// avoid generating a notification for REPLICA creation event.
 	if _, ok := args.ReqParams[xhttp.MinIOSourceReplicationRequest]; ok {
 		return
 	}
+
+	args.Object.Size, _ = args.Object.GetActualSize()
+
 	// remove sensitive encryption entries in metadata.
 	crypto.RemoveSensitiveEntries(args.Object.UserDefined)
 	crypto.RemoveInternalEntries(args.Object.UserDefined)

cmd/format-erasure.go

@@ -304,11 +304,6 @@ func countErrs(errs []error, err error) int {
 	return i
 }
 
-// Does all errors indicate we need to initialize all disks?.
-func shouldInitErasureDisks(errs []error) bool {
-	return countErrs(errs, errUnformattedDisk) == len(errs)
-}
-
 // Check if unformatted disks are equal to write quorum.
 func quorumUnformattedDisks(errs []error) bool {
 	return countErrs(errs, errUnformattedDisk) >= (len(errs)/2)+1
@@ -347,7 +342,7 @@ func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasur
 	return formats, g.Wait()
 }
 
-func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) error {
+func saveFormatErasure(disk StorageAPI, format *formatErasureV3, healID string) error {
 	if disk == nil || format == nil {
 		return errDiskNotFound
 	}
@@ -383,9 +378,9 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, heal bool) erro
 	}
 
 	disk.SetDiskID(diskID)
-	if heal {
+	if healID != "" {
 		ctx := context.Background()
-		ht := newHealingTracker(disk)
+		ht := newHealingTracker(disk, healID)
 		return ht.save(ctx)
 	}
 	return nil
@@ -541,7 +536,7 @@ func formatErasureFixLocalDeploymentID(endpoints Endpoints, storageDisks []Stora
 				}
 				format.ID = refFormat.ID
 				// Heal the drive if we fixed its deployment ID.
-				if err := saveFormatErasure(storageDisks[index], format, true); err != nil {
+				if err := saveFormatErasure(storageDisks[index], format, mustGetUUID()); err != nil {
 					logger.LogIf(GlobalContext, err)
 					return fmt.Errorf("Unable to save format.json, %w", err)
 				}
@@ -642,7 +637,7 @@ func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, format
 			if formats[index] == nil {
 				return errDiskNotFound
 			}
-			return saveFormatErasure(storageDisks[index], formats[index], false)
+			return saveFormatErasure(storageDisks[index], formats[index], "")
 		}, index)
 	}
 
@@ -722,7 +717,7 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats
 			if formats[i].Erasure.This == "" {
 				formats[i].Erasure.This = formats[i].Erasure.Sets[0][i]
 				// Heal the drive if drive has .This empty.
-				if err := saveFormatErasure(storageDisks[i], formats[i], true); err != nil {
+				if err := saveFormatErasure(storageDisks[i], formats[i], mustGetUUID()); err != nil {
 					return err
 				}
 			}
@@ -741,12 +736,18 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats
 func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, setDriveCount int, deploymentID, distributionAlgo string, sErrs []error) (*formatErasureV3, error) {
 	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, len(storageDisks))
-	wantAtMost := ecDrivesNoConfig(setDriveCount)
+	wantAtMost, err := ecDrivesNoConfig(setDriveCount)
+	if err != nil {
+		return nil, err
+	}
 
 	for i := 0; i < setCount; i++ {
 		hostCount := make(map[string]int, setDriveCount)
 		for j := 0; j < setDriveCount; j++ {
 			disk := storageDisks[i*setDriveCount+j]
+			if disk == nil {
+				continue
+			}
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
 			if distributionAlgo != "" {
@@ -758,26 +759,24 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 			hostCount[disk.Hostname()]++
 			formats[i*setDriveCount+j] = newFormat
 		}
-		if len(hostCount) > 0 {
-			var once sync.Once
-			for host, count := range hostCount {
-				if count > wantAtMost {
-					if host == "" {
-						host = "local"
-					}
-					once.Do(func() {
-						if len(hostCount) == 1 {
-							return
-						}
-						logger.Info(" * Set %v:", i+1)
-						for j := 0; j < setDriveCount; j++ {
-							disk := storageDisks[i*setDriveCount+j]
-							logger.Info("   - Drive: %s", disk.String())
-						}
-					})
-					logger.Info(color.Yellow("WARNING:")+" Host %v has more than %v drives of set. "+
-						"A host failure will result in data becoming unavailable.", host, wantAtMost)
+		var once sync.Once
+		for host, count := range hostCount {
+			if count > wantAtMost {
+				if host == "" {
+					host = "local"
 				}
+				once.Do(func() {
+					if len(hostCount) == 1 {
+						return
+					}
+					logger.Info(" * Set %v:", i+1)
+					for j := 0; j < setDriveCount; j++ {
+						disk := storageDisks[i*setDriveCount+j]
+						logger.Info("   - Drive: %s", disk.String())
+					}
+				})
+				logger.Info(color.Yellow("WARNING:")+" Host %v has more than %v drives of set. "+
+					"A host failure will result in data becoming unavailable.", host, wantAtMost)
 			}
 		}
 	}
@@ -795,13 +794,12 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 
 // ecDrivesNoConfig returns the erasure coded drives in a set if no config has been set.
 // It will attempt to read it from env variable and fall back to drives/2.
-func ecDrivesNoConfig(setDriveCount int) int {
-	sc, _ := storageclass.LookupConfig(config.KVS{}, setDriveCount)
-	ecDrives := sc.GetParityForSC(storageclass.STANDARD)
-	if ecDrives < 0 {
-		ecDrives = storageclass.DefaultParityBlocks(setDriveCount)
+func ecDrivesNoConfig(setDriveCount int) (int, error) {
+	sc, err := storageclass.LookupConfig(config.KVS{}, setDriveCount)
+	if err != nil {
+		return 0, err
 	}
-	return ecDrives
+	return sc.GetParityForSC(storageclass.STANDARD), nil
 }
 
 // Make Erasure backend meta volumes.