protect bpool from buffer pollution by invalid buffers (#20342)

Currently, the bucket events and replication targets are only reloaded
with buckets that failed to load during the first cluster startup,
which is wrong because if one bucket change was done in one node but
that node was not able to notify other nodes; the other nodes will
reload the bucket metadata config but fails to set the events and bucket
targets in the memory.

CONTRIBUTING.md

@@ -12,8 +12,9 @@ Fork [MinIO upstream](https://github.com/minio/minio/fork) source repository to
 
 ```sh
 git clone https://github.com/minio/minio
+cd minio
 go install -v
-ls /go/bin/minio
+ls $(go env GOPATH)/bin/minio
 ```
 
 ### Set up git remote as ``upstream``

Dockerfile.hotfix

@@ -1,24 +1,26 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.22-alpine as build
 
 ARG TARGETARCH
 ARG RELEASE
 
-ENV GOPATH /go
-ENV CGO_ENABLED 0
+ENV GOPATH=/go
+ENV CGO_ENABLED=0
 
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
     go install aead.dev/minisign/cmd/minisign@v0.2.1
 
-# Download minio binary and signature file
+# Download minio binary and signature files
 RUN curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \
     chmod +x /go/bin/minio
 
-# Download mc binary and signature file
+# Download mc binary and signature files
 RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \
     chmod +x /go/bin/mc
 
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
@@ -52,8 +54,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MC_CONFIG_DIR=/tmp/.mc
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
-COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/minio* /usr/bin/
+COPY --from=build /go/bin/mc* /usr/bin/
 COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS

Dockerfile.release

@@ -1,24 +1,26 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.22-alpine AS build
 
 ARG TARGETARCH
 ARG RELEASE
 
-ENV GOPATH /go
-ENV CGO_ENABLED 0
+ENV GOPATH=/go
+ENV CGO_ENABLED=0
 
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
     go install aead.dev/minisign/cmd/minisign@v0.2.1
 
-# Download minio binary and signature file
+# Download minio binary and signature files
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \
     chmod +x /go/bin/minio
 
-# Download mc binary and signature file
+# Download mc binary and signature files
 RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \
     chmod +x /go/bin/mc
 
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
@@ -52,8 +54,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MC_CONFIG_DIR=/tmp/.mc
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
-COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/minio* /usr/bin/
+COPY --from=build /go/bin/mc* /usr/bin/
 COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS

Dockerfile.release.fips

@@ -1,21 +1,28 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.22-alpine AS build
 
 ARG TARGETARCH
 ARG RELEASE
 
-ENV GOPATH /go
-ENV CGO_ENABLED 0
+ENV GOPATH=/go
+ENV CGO_ENABLED=0
 
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
     go install aead.dev/minisign/cmd/minisign@v0.2.1
 
-# Download minio binary and signature file
+# Download minio binary and signature files
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /go/bin/minio.minisig && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /go/bin/minio.sha256sum && \
     chmod +x /go/bin/minio
 
+# Download mc binary and signature files
+RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips -o /go/bin/mc && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips.minisig -o /go/bin/mc.minisig && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.fips.sha256sum -o /go/bin/mc.sha256sum && \
+    chmod +x /go/bin/mc
+
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
        curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \
        chmod +x /go/bin/curl; \
@@ -45,7 +52,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_CONFIG_ENV_FILE=config.env
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/minio* /usr/bin/
+COPY --from=build /go/bin/mc* /usr/bin/
 COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS

Dockerfile.release.old_cpu

@@ -1,24 +1,26 @@
-FROM golang:1.21-alpine as build
+FROM golang:1.22-alpine AS build
 
 ARG TARGETARCH
 ARG RELEASE
 
-ENV GOPATH /go
-ENV CGO_ENABLED 0
+ENV GOPATH=/go
+ENV CGO_ENABLED=0
 
 # Install curl and minisign
 RUN apk add -U --no-cache ca-certificates && \
     apk add -U --no-cache curl && \
     go install aead.dev/minisign/cmd/minisign@v0.2.1
 
-# Download minio binary and signature file
+# Download minio binary and signature files
 RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \
     chmod +x /go/bin/minio
 
-# Download mc binary and signature file
+# Download mc binary and signature files
 RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \
     chmod +x /go/bin/mc
 
 RUN if [ "$TARGETARCH" = "amd64" ]; then \
@@ -52,8 +54,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MC_CONFIG_DIR=/tmp/.mc
 
 COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=build /go/bin/minio /usr/bin/minio
-COPY --from=build /go/bin/mc /usr/bin/mc
+COPY --from=build /go/bin/minio* /usr/bin/
+COPY --from=build /go/bin/mc* /usr/bin/
 COPY --from=build /go/bin/cur* /usr/bin/
 
 COPY CREDITS /licenses/CREDITS

cmd/admin-bucket-handlers.go

@@ -427,6 +427,21 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 			cfgPath := pathJoin(bi.Name, cfgFile)
 			bucket := bi.Name
 			switch cfgFile {
+			case bucketPolicyConfig:
+				config, _, err := globalBucketMetadataSys.GetBucketPolicy(bucket)
+				if err != nil {
+					if errors.Is(err, BucketPolicyNotFound{Bucket: bucket}) {
+						continue
+					}
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				configData, err := json.Marshal(config)
+				if err != nil {
+					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
+					return
+				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketNotificationConfig:
 				config, err := globalBucketMetadataSys.GetNotificationConfig(bucket)
 				if err != nil {
@@ -796,11 +811,12 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			}
 
 			bucketMap[bucket].NotificationConfigXML = configData
+			bucketMap[bucket].NotificationConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
 		case bucketPolicyConfig:
 			// Error out if Content-Length is beyond allowed size.
 			if sz > maxBucketPolicySize {
-				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyTooLarge.String()))
+				rpt.SetStatus(bucket, fileName, errors.New(ErrPolicyTooLarge.String()))
 				continue
 			}
 
@@ -818,7 +834,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 
 			// Version in policy must not be empty
 			if bucketPolicy.Version == "" {
-				rpt.SetStatus(bucket, fileName, fmt.Errorf(ErrPolicyInvalidVersion.String()))
+				rpt.SetStatus(bucket, fileName, errors.New(ErrPolicyInvalidVersion.String()))
 				continue
 			}
 

cmd/admin-handlers-users.go

@@ -34,7 +34,6 @@ import (
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
-	"github.com/minio/minio/internal/cachevalue"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/mux"
 	xldap "github.com/minio/pkg/v3/ldap"
@@ -1235,18 +1234,6 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	bucketStorageCache.InitOnce(10*time.Second,
-		cachevalue.Opts{ReturnLastGood: true},
-		func(ctx context.Context) (DataUsageInfo, error) {
-			ctx, done := context.WithTimeout(ctx, 2*time.Second)
-			defer done()
-
-			return loadDataUsageFromBackend(ctx, objectAPI)
-		},
-	)
-
-	dataUsageInfo, _ := bucketStorageCache.Get()
-
 	// If etcd, dns federation configured list buckets from etcd.
 	var err error
 	var buckets []BucketInfo
@@ -1333,15 +1320,12 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		rd, wr := isAllowedAccess(bucket.Name)
 		if rd || wr {
 			// Fetch the data usage of the current bucket
-			var size uint64
-			var objectsCount uint64
-			var objectsHist, versionsHist map[string]uint64
-			if !dataUsageInfo.LastUpdate.IsZero() {
-				size = dataUsageInfo.BucketsUsage[bucket.Name].Size
-				objectsCount = dataUsageInfo.BucketsUsage[bucket.Name].ObjectsCount
-				objectsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectSizesHistogram
-				versionsHist = dataUsageInfo.BucketsUsage[bucket.Name].ObjectVersionsHistogram
-			}
+			bui := globalBucketQuotaSys.GetBucketUsageInfo(ctx, bucket.Name)
+			size := bui.Size
+			objectsCount := bui.ObjectsCount
+			objectsHist := bui.ObjectSizesHistogram
+			versionsHist := bui.ObjectVersionsHistogram
+
 			// Fetch the prefix usage of the current bucket
 			var prefixUsage map[string]uint64
 			if enablePrefixUsage {

cmd/admin-handlers.go

@@ -848,9 +848,10 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 }
 
 func lriToLockEntry(l lockRequesterInfo, now time.Time, resource, server string) *madmin.LockEntry {
+	t := time.Unix(0, l.Timestamp)
 	entry := &madmin.LockEntry{
-		Timestamp:  l.Timestamp,
-		Elapsed:    now.Sub(l.Timestamp),
+		Timestamp:  t,
+		Elapsed:    now.Sub(t),
 		Resource:   resource,
 		ServerList: []string{server},
 		Source:     l.Source,
@@ -1035,7 +1036,7 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 	// Start profiling on remote servers.
 	var hostErrs []NotificationPeerErr
 	for _, profiler := range profiles {
-		hostErrs = append(hostErrs, globalNotificationSys.StartProfiling(profiler)...)
+		hostErrs = append(hostErrs, globalNotificationSys.StartProfiling(ctx, profiler)...)
 
 		// Start profiling locally as well.
 		prof, err := startProfiler(profiler)
@@ -1116,7 +1117,11 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 
 	// Start profiling on remote servers.
 	for _, profiler := range profiles {
-		globalNotificationSys.StartProfiling(profiler)
+		// Limit start time to max 10s.
+		ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+		globalNotificationSys.StartProfiling(ctx, profiler)
+		// StartProfiling blocks, so we can cancel now.
+		cancel()
 
 		// Start profiling locally as well.
 		prof, err := startProfiler(profiler)
@@ -1131,6 +1136,10 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 	for {
 		select {
 		case <-ctx.Done():
+			// Stop remote profiles
+			go globalNotificationSys.DownloadProfilingData(GlobalContext, io.Discard)
+
+			// Stop local
 			globalProfilerMu.Lock()
 			defer globalProfilerMu.Unlock()
 			for k, v := range globalProfiler {
@@ -2186,7 +2195,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// KMSKeyStatusHandler - GET /minio/admin/v3/kms/status
+// KMSStatusHandler - GET /minio/admin/v3/kms/status
 func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
@@ -2330,6 +2339,7 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]
 }
 
 func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) madmin.InfoMessage {
+	const operationTimeout = 10 * time.Second
 	ldap := madmin.LDAP{}
 	if globalIAMSys.LDAPConfig.Enabled() {
 		ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
@@ -2370,7 +2380,9 @@ func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) ma
 		mode = madmin.ItemOnline
 
 		// Load data usage
-		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
+		ctx2, cancel := context.WithTimeout(ctx, operationTimeout)
+		dataUsageInfo, err := loadDataUsageFromBackend(ctx2, objectAPI)
+		cancel()
 		if err == nil {
 			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
 			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
@@ -2404,18 +2416,24 @@ func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) ma
 		}
 
 		if pools {
-			poolsInfo, _ = getPoolsInfo(ctx, allDisks)
+			ctx2, cancel := context.WithTimeout(ctx, operationTimeout)
+			poolsInfo, _ = getPoolsInfo(ctx2, allDisks)
+			cancel()
 		}
 	}
 
 	domain := globalDomainNames
 	services := madmin.Services{
-		KMSStatus:     fetchKMSStatus(ctx),
 		LDAP:          ldap,
 		Logger:        log,
 		Audit:         audit,
 		Notifications: notifyTarget,
 	}
+	{
+		ctx2, cancel := context.WithTimeout(ctx, operationTimeout)
+		services.KMSStatus = fetchKMSStatus(ctx2)
+		cancel()
+	}
 
 	return madmin.InfoMessage{
 		Mode:          string(mode),
@@ -3049,7 +3067,7 @@ func targetStatus(ctx context.Context, h logger.Target) madmin.Status {
 	return madmin.Status{Status: string(madmin.ItemOffline)}
 }
 
-// fetchLoggerDetails return log info
+// fetchLoggerInfo return log info
 func fetchLoggerInfo(ctx context.Context) ([]madmin.Logger, []madmin.Audit) {
 	var loggerInfo []madmin.Logger
 	var auditloggerInfo []madmin.Audit

cmd/admin-handlers_test.go

@@ -463,6 +463,7 @@ func TestTopLockEntries(t *testing.T) {
 			Owner:      lri.Owner,
 			ID:         lri.UID,
 			Quorum:     lri.Quorum,
+			Timestamp:  time.Unix(0, lri.Timestamp),
 		})
 	}
 

cmd/admin-heal-ops.go

@@ -761,6 +761,15 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		return nil
 	}
 
+	countOKDrives := func(drives []madmin.HealDriveInfo) (count int) {
+		for _, drive := range drives {
+			if drive.State == madmin.DriveStateOk {
+				count++
+			}
+		}
+		return count
+	}
+
 	// task queued, now wait for the response.
 	select {
 	case res := <-task.respCh:
@@ -781,6 +790,11 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 		if res.err != nil {
 			res.result.Detail = res.err.Error()
 		}
+		if res.result.ParityBlocks > 0 && res.result.DataBlocks > 0 && res.result.DataBlocks > res.result.ParityBlocks {
+			if got := countOKDrives(res.result.After.Drives); got < res.result.ParityBlocks {
+				res.result.Detail = fmt.Sprintf("quorum loss - expected %d minimum, got drive states in OK %d", res.result.ParityBlocks, got)
+			}
+		}
 		return h.pushHealResultItem(res.result)
 	case <-h.ctx.Done():
 		return nil

cmd/api-errors.go

@@ -28,7 +28,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore"
 	"github.com/minio/minio/internal/ioutil"
 	"google.golang.org/api/googleapi"
 
@@ -1486,7 +1486,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrTooManyRequests: {
 		Code:           "TooManyRequests",
-		Description:    "Deadline exceeded while waiting in incoming queue, please reduce your request rate",
+		Description:    "Please reduce your request rate",
 		HTTPStatusCode: http.StatusTooManyRequests,
 	},
 	ErrUnsupportedMetadata: {
@@ -2549,11 +2549,11 @@ func toAPIError(ctx context.Context, err error) APIError {
 			if len(e.Errors) >= 1 {
 				apiErr.Code = e.Errors[0].Reason
 			}
-		case azblob.StorageError:
+		case *azcore.ResponseError:
 			apiErr = APIError{
-				Code:           string(e.ServiceCode()),
+				Code:           e.ErrorCode,
 				Description:    e.Error(),
-				HTTPStatusCode: e.Response().StatusCode,
+				HTTPStatusCode: e.StatusCode,
 			}
 			// Add more other SDK related errors here if any in future.
 		default:
@@ -2592,7 +2592,7 @@ func getAPIError(code APIErrorCode) APIError {
 	return errorCodes.ToAPIErr(ErrInternalError)
 }
 
-// getErrorResponse gets in standard error and resource value and
+// getAPIErrorResponse gets in standard error and resource value and
 // provides a encodable populated response values
 func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID, hostID string) APIErrorResponse {
 	reqInfo := logger.GetReqInfo(ctx)

cmd/api-response.go

@@ -947,19 +947,10 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 // writeErrorResponse writes error headers
 func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
 	switch err.HTTPStatusCode {
-	case http.StatusServiceUnavailable:
+	case http.StatusServiceUnavailable, http.StatusTooManyRequests:
 		// Set retry-after header to indicate user-agents to retry request after 60 seconds.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set(xhttp.RetryAfter, "60")
-	case http.StatusTooManyRequests:
-		_, deadline := globalAPIConfig.getRequestsPool()
-		if deadline <= 0 {
-			// Set retry-after header to indicate user-agents to retry request after 10 seconds.
-			// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
-			w.Header().Set(xhttp.RetryAfter, "10")
-		} else {
-			w.Header().Set(xhttp.RetryAfter, strconv.Itoa(int(deadline.Seconds())))
-		}
 	}
 
 	switch err.Code {

cmd/api-router.go

@@ -456,6 +456,14 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodGet).
 			HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)).
 			Queries("cors", "")
+		// PutBucketCors - this is a dummy call.
+		router.Methods(http.MethodPut).
+			HandlerFunc(s3APIMiddleware(api.PutBucketCorsHandler)).
+			Queries("cors", "")
+		// DeleteBucketCors - this is a dummy call.
+		router.Methods(http.MethodDelete).
+			HandlerFunc(s3APIMiddleware(api.DeleteBucketCorsHandler)).
+			Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
 		router.Methods(http.MethodGet).
 			HandlerFunc(s3APIMiddleware(api.GetBucketWebsiteHandler)).
@@ -472,6 +480,7 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodGet).
 			HandlerFunc(s3APIMiddleware(api.GetBucketLoggingHandler)).
 			Queries("logging", "")
+
 		// GetBucketTaggingHandler
 		router.Methods(http.MethodGet).
 			HandlerFunc(s3APIMiddleware(api.GetBucketTaggingHandler)).

cmd/background-newdisks-heal-ops.go

@@ -72,10 +72,12 @@ type healingTracker struct {
 
 	// Numbers when current bucket started healing,
 	// for resuming with correct numbers.
-	ResumeItemsHealed uint64 `json:"-"`
-	ResumeItemsFailed uint64 `json:"-"`
-	ResumeBytesDone   uint64 `json:"-"`
-	ResumeBytesFailed uint64 `json:"-"`
+	ResumeItemsHealed  uint64 `json:"-"`
+	ResumeItemsFailed  uint64 `json:"-"`
+	ResumeItemsSkipped uint64 `json:"-"`
+	ResumeBytesDone    uint64 `json:"-"`
+	ResumeBytesFailed  uint64 `json:"-"`
+	ResumeBytesSkipped uint64 `json:"-"`
 
 	// Filled on startup/restarts.
 	QueuedBuckets []string
@@ -90,6 +92,9 @@ type healingTracker struct {
 	BytesSkipped uint64
 
 	RetryAttempts uint64
+
+	Finished bool // finished healing, whether with errors or not
+
 	// Add future tracking capabilities
 	// Be sure that they are included in toHealingDisk
 }
@@ -143,6 +148,26 @@ func initHealingTracker(disk StorageAPI, healID string) *healingTracker {
 	return h
 }
 
+func (h *healingTracker) resetHealing() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	h.ItemsHealed = 0
+	h.ItemsFailed = 0
+	h.BytesDone = 0
+	h.BytesFailed = 0
+	h.ResumeItemsHealed = 0
+	h.ResumeItemsFailed = 0
+	h.ResumeBytesDone = 0
+	h.ResumeBytesFailed = 0
+	h.ItemsSkipped = 0
+	h.BytesSkipped = 0
+
+	h.HealedBuckets = nil
+	h.Object = ""
+	h.Bucket = ""
+}
+
 func (h *healingTracker) getLastUpdate() time.Time {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
@@ -262,8 +287,10 @@ func (h *healingTracker) resume() {
 
 	h.ItemsHealed = h.ResumeItemsHealed
 	h.ItemsFailed = h.ResumeItemsFailed
+	h.ItemsSkipped = h.ResumeItemsSkipped
 	h.BytesDone = h.ResumeBytesDone
 	h.BytesFailed = h.ResumeBytesFailed
+	h.BytesSkipped = h.ResumeBytesSkipped
 }
 
 // bucketDone should be called when a bucket is done healing.
@@ -274,8 +301,10 @@ func (h *healingTracker) bucketDone(bucket string) {
 
 	h.ResumeItemsHealed = h.ItemsHealed
 	h.ResumeItemsFailed = h.ItemsFailed
+	h.ResumeItemsSkipped = h.ItemsSkipped
 	h.ResumeBytesDone = h.BytesDone
 	h.ResumeBytesFailed = h.BytesFailed
+	h.ResumeBytesSkipped = h.BytesSkipped
 	h.HealedBuckets = append(h.HealedBuckets, bucket)
 	for i, b := range h.QueuedBuckets {
 		if b == bucket {
@@ -324,6 +353,7 @@ func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
 		PoolIndex:         h.PoolIndex,
 		SetIndex:          h.SetIndex,
 		DiskIndex:         h.DiskIndex,
+		Finished:          h.Finished,
 		Path:              h.Path,
 		Started:           h.Started.UTC(),
 		LastUpdate:        h.LastUpdate.UTC(),
@@ -339,6 +369,7 @@ func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
 		Object:            h.Object,
 		QueuedBuckets:     h.QueuedBuckets,
 		HealedBuckets:     h.HealedBuckets,
+		RetryAttempts:     h.RetryAttempts,
 
 		ObjectsHealed: h.ItemsHealed, // Deprecated July 2021
 		ObjectsFailed: h.ItemsFailed, // Deprecated July 2021
@@ -353,16 +384,18 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	}
 
 	initBackgroundHealing(ctx, objAPI) // start quick background healing
-
-	if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn || env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
+	if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn {
 		globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 		go monitorLocalDisksAndHeal(ctx, z)
 	}
+
+	go globalMRFState.startMRFPersistence()
+	go globalMRFState.healRoutine(z)
 }
 
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 	for _, disk := range localDrives {
 		_, err := disk.DiskInfo(context.Background(), DiskInfoOptions{})
@@ -370,7 +403,7 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 			disksToHeal = append(disksToHeal, disk.Endpoint())
 			continue
 		}
-		if disk.Healing() != nil {
+		if h := disk.Healing(); h != nil && !h.Finished {
 			disksToHeal = append(disksToHeal, disk.Endpoint())
 		}
 	}
@@ -469,16 +502,19 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	// if objects have failed healing, we attempt a retry to heal the drive upto 3 times before giving up.
 	if tracker.ItemsFailed > 0 && tracker.RetryAttempts < 4 {
 		tracker.RetryAttempts++
-		bugLogIf(ctx, tracker.update(ctx))
 
 		healingLogEvent(ctx, "Healing of drive '%s' is incomplete, retrying %s time (healed: %d, skipped: %d, failed: %d).", disk,
 			humanize.Ordinal(int(tracker.RetryAttempts)), tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
+
+		tracker.resetHealing()
+		bugLogIf(ctx, tracker.update(ctx))
+
 		return errRetryHealing
 	}
 
 	if tracker.ItemsFailed > 0 {
 		healingLogEvent(ctx, "Healing of drive '%s' is incomplete, retried %d times (healed: %d, skipped: %d, failed: %d).", disk,
-			tracker.RetryAttempts-1, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
+			tracker.RetryAttempts, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed)
 	} else {
 		if tracker.RetryAttempts > 0 {
 			healingLogEvent(ctx, "Healing of drive '%s' is complete, retried %d times (healed: %d, skipped: %d).", disk,
@@ -516,7 +552,8 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 			continue
 		}
 		if t.HealID == tracker.HealID {
-			t.delete(ctx)
+			t.Finished = true
+			t.update(ctx)
 		}
 	}
 

cmd/background-newdisks-heal-ops_gen.go

@@ -132,6 +132,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ResumeItemsFailed")
 				return
 			}
+		case "ResumeItemsSkipped":
+			z.ResumeItemsSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeItemsSkipped")
+				return
+			}
 		case "ResumeBytesDone":
 			z.ResumeBytesDone, err = dc.ReadUint64()
 			if err != nil {
@@ -144,6 +150,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ResumeBytesFailed")
 				return
 			}
+		case "ResumeBytesSkipped":
+			z.ResumeBytesSkipped, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeBytesSkipped")
+				return
+			}
 		case "QueuedBuckets":
 			var zb0002 uint32
 			zb0002, err = dc.ReadArrayHeader()
@@ -206,6 +218,12 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "RetryAttempts")
 				return
 			}
+		case "Finished":
+			z.Finished, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "Finished")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -219,9 +237,9 @@ func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 26
+	// map header, size 29
 	// write "ID"
-	err = en.Append(0xde, 0x0, 0x1a, 0xa2, 0x49, 0x44)
+	err = en.Append(0xde, 0x0, 0x1d, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -400,6 +418,16 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ResumeItemsFailed")
 		return
 	}
+	// write "ResumeItemsSkipped"
+	err = en.Append(0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ResumeItemsSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "ResumeItemsSkipped")
+		return
+	}
 	// write "ResumeBytesDone"
 	err = en.Append(0xaf, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65)
 	if err != nil {
@@ -420,6 +448,16 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ResumeBytesFailed")
 		return
 	}
+	// write "ResumeBytesSkipped"
+	err = en.Append(0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.ResumeBytesSkipped)
+	if err != nil {
+		err = msgp.WrapError(err, "ResumeBytesSkipped")
+		return
+	}
 	// write "QueuedBuckets"
 	err = en.Append(0xad, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73)
 	if err != nil {
@@ -494,15 +532,25 @@ func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "RetryAttempts")
 		return
 	}
+	// write "Finished"
+	err = en.Append(0xa8, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.Finished)
+	if err != nil {
+		err = msgp.WrapError(err, "Finished")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 26
+	// map header, size 29
 	// string "ID"
-	o = append(o, 0xde, 0x0, 0x1a, 0xa2, 0x49, 0x44)
+	o = append(o, 0xde, 0x0, 0x1d, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "PoolIndex"
 	o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
@@ -555,12 +603,18 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "ResumeItemsFailed"
 	o = append(o, 0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
 	o = msgp.AppendUint64(o, z.ResumeItemsFailed)
+	// string "ResumeItemsSkipped"
+	o = append(o, 0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ResumeItemsSkipped)
 	// string "ResumeBytesDone"
 	o = append(o, 0xaf, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65)
 	o = msgp.AppendUint64(o, z.ResumeBytesDone)
 	// string "ResumeBytesFailed"
 	o = append(o, 0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64)
 	o = msgp.AppendUint64(o, z.ResumeBytesFailed)
+	// string "ResumeBytesSkipped"
+	o = append(o, 0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64)
+	o = msgp.AppendUint64(o, z.ResumeBytesSkipped)
 	// string "QueuedBuckets"
 	o = append(o, 0xad, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(len(z.QueuedBuckets)))
@@ -585,6 +639,9 @@ func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "RetryAttempts"
 	o = append(o, 0xad, 0x52, 0x65, 0x74, 0x72, 0x79, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
 	o = msgp.AppendUint64(o, z.RetryAttempts)
+	// string "Finished"
+	o = append(o, 0xa8, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64)
+	o = msgp.AppendBool(o, z.Finished)
 	return
 }
 
@@ -714,6 +771,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ResumeItemsFailed")
 				return
 			}
+		case "ResumeItemsSkipped":
+			z.ResumeItemsSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeItemsSkipped")
+				return
+			}
 		case "ResumeBytesDone":
 			z.ResumeBytesDone, bts, err = msgp.ReadUint64Bytes(bts)
 			if err != nil {
@@ -726,6 +789,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ResumeBytesFailed")
 				return
 			}
+		case "ResumeBytesSkipped":
+			z.ResumeBytesSkipped, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ResumeBytesSkipped")
+				return
+			}
 		case "QueuedBuckets":
 			var zb0002 uint32
 			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
@@ -788,6 +857,12 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "RetryAttempts")
 				return
 			}
+		case "Finished":
+			z.Finished, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Finished")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -802,7 +877,7 @@ func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *healingTracker) Msgsize() (s int) {
-	s = 3 + 3 + msgp.StringPrefixSize + len(z.ID) + 10 + msgp.IntSize + 9 + msgp.IntSize + 10 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Path) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 8 + msgp.TimeSize + 11 + msgp.TimeSize + 18 + msgp.Uint64Size + 17 + msgp.Uint64Size + 12 + msgp.Uint64Size + 12 + msgp.Uint64Size + 10 + msgp.Uint64Size + 12 + msgp.Uint64Size + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 18 + msgp.Uint64Size + 18 + msgp.Uint64Size + 16 + msgp.Uint64Size + 18 + msgp.Uint64Size + 14 + msgp.ArrayHeaderSize
+	s = 3 + 3 + msgp.StringPrefixSize + len(z.ID) + 10 + msgp.IntSize + 9 + msgp.IntSize + 10 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Path) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 8 + msgp.TimeSize + 11 + msgp.TimeSize + 18 + msgp.Uint64Size + 17 + msgp.Uint64Size + 12 + msgp.Uint64Size + 12 + msgp.Uint64Size + 10 + msgp.Uint64Size + 12 + msgp.Uint64Size + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 18 + msgp.Uint64Size + 18 + msgp.Uint64Size + 19 + msgp.Uint64Size + 16 + msgp.Uint64Size + 18 + msgp.Uint64Size + 19 + msgp.Uint64Size + 14 + msgp.ArrayHeaderSize
 	for za0001 := range z.QueuedBuckets {
 		s += msgp.StringPrefixSize + len(z.QueuedBuckets[za0001])
 	}
@@ -810,6 +885,6 @@ func (z *healingTracker) Msgsize() (s int) {
 	for za0002 := range z.HealedBuckets {
 		s += msgp.StringPrefixSize + len(z.HealedBuckets[za0002])
 	}
-	s += 7 + msgp.StringPrefixSize + len(z.HealID) + 13 + msgp.Uint64Size + 13 + msgp.Uint64Size + 14 + msgp.Uint64Size
+	s += 7 + msgp.StringPrefixSize + len(z.HealID) + 13 + msgp.Uint64Size + 13 + msgp.Uint64Size + 14 + msgp.Uint64Size + 9 + msgp.BoolSize
 	return
 }

cmd/batch-expire.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"runtime"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -36,6 +37,7 @@ import (
 	"github.com/minio/pkg/v3/env"
 	"github.com/minio/pkg/v3/wildcard"
 	"github.com/minio/pkg/v3/workers"
+	"github.com/minio/pkg/v3/xtime"
 	"gopkg.in/yaml.v3"
 )
 
@@ -116,7 +118,7 @@ func (p BatchJobExpirePurge) Validate() error {
 // BatchJobExpireFilter holds all the filters currently supported for batch replication
 type BatchJobExpireFilter struct {
 	line, col     int
-	OlderThan     time.Duration       `yaml:"olderThan,omitempty" json:"olderThan"`
+	OlderThan     xtime.Duration      `yaml:"olderThan,omitempty" json:"olderThan"`
 	CreatedBefore *time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
 	Tags          []BatchJobKV        `yaml:"tags,omitempty" json:"tags"`
 	Metadata      []BatchJobKV        `yaml:"metadata,omitempty" json:"metadata"`
@@ -162,7 +164,7 @@ func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool {
 	if len(ef.Name) > 0 && !wildcard.Match(ef.Name, obj.Name) {
 		return false
 	}
-	if ef.OlderThan > 0 && now.Sub(obj.ModTime) <= ef.OlderThan {
+	if ef.OlderThan > 0 && now.Sub(obj.ModTime) <= ef.OlderThan.D() {
 		return false
 	}
 
@@ -280,7 +282,7 @@ type BatchJobExpire struct {
 	line, col       int
 	APIVersion      string                 `yaml:"apiVersion" json:"apiVersion"`
 	Bucket          string                 `yaml:"bucket" json:"bucket"`
-	Prefix          string                 `yaml:"prefix" json:"prefix"`
+	Prefix          BatchJobPrefix         `yaml:"prefix" json:"prefix"`
 	NotificationCfg BatchJobNotification   `yaml:"notify" json:"notify"`
 	Retry           BatchJobRetry          `yaml:"retry" json:"retry"`
 	Rules           []BatchJobExpireFilter `yaml:"rules" json:"rules"`
@@ -340,8 +342,24 @@ func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versio
 		PrefixEnabledFn:  vc.PrefixEnabled,
 		VersionSuspended: vc.Suspended(),
 	}
-	_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel, opts)
-	return errs
+
+	allErrs := make([]error, 0, len(objsToDel))
+
+	for {
+		count := len(objsToDel)
+		if count == 0 {
+			break
+		}
+		if count > maxDeleteList {
+			count = maxDeleteList
+		}
+		_, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel[:count], opts)
+		allErrs = append(allErrs, errs...)
+		// Next batch of deletion
+		objsToDel = objsToDel[count:]
+	}
+
+	return allErrs
 }
 
 const (
@@ -534,40 +552,54 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		return err
 	}
 
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
+	ctx, cancelCause := context.WithCancelCause(ctx)
+	defer cancelCause(nil)
 
 	results := make(chan itemOrErr[ObjectInfo], workerSize)
-	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{
-		Marker:       lastObject,
-		LatestOnly:   false, // we need to visit all versions of the object to implement purge: retainVersions
-		VersionsSort: WalkVersionsSortDesc,
-	}); err != nil {
-		// Do not need to retry if we can't list objects on source.
-		return err
-	}
+	go func() {
+		for _, prefix := range r.Prefix.F() {
+			prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize)
+			err := api.Walk(ctx, r.Bucket, strings.TrimSpace(prefix), prefixResultCh, WalkOptions{
+				Marker:       lastObject,
+				LatestOnly:   false, // we need to visit all versions of the object to implement purge: retainVersions
+				VersionsSort: WalkVersionsSortDesc,
+			})
+			if err != nil {
+				cancelCause(err)
+				xioutil.SafeClose(results)
+				return
+			}
+			for result := range prefixResultCh {
+				results <- result
+			}
+		}
+		xioutil.SafeClose(results)
+	}()
 
 	// Goroutine to periodically save batch-expire job's in-memory state
 	saverQuitCh := make(chan struct{})
 	go func() {
 		saveTicker := time.NewTicker(10 * time.Second)
 		defer saveTicker.Stop()
-		for {
+		quit := false
+		after := time.Minute
+		for !quit {
 			select {
 			case <-saveTicker.C:
-				// persist in-memory state to disk after every 10secs.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
-
 			case <-ctx.Done():
-				// persist in-memory state immediately before exiting due to context cancellation.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
-				return
-
+				quit = true
 			case <-saverQuitCh:
-				// persist in-memory state immediately to disk.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
-				return
+				quit = true
 			}
+
+			if quit {
+				// save immediately if we are quitting
+				after = 0
+			}
+
+			ctx, cancel := context.WithTimeout(GlobalContext, 30*time.Second) // independent context
+			batchLogIf(ctx, ri.updateAfter(ctx, api, after, job))
+			cancel()
 		}
 	}()
 
@@ -584,7 +616,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		versionsCount int
 		toDel         []expireObjInfo
 	)
-	failed := true
+	failed := false
 	for result := range results {
 		if result.Err != nil {
 			failed = true
@@ -653,6 +685,10 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 			ObjectInfo: result.Item,
 		})
 	}
+	if context.Cause(ctx) != nil {
+		xioutil.SafeClose(expireCh)
+		return context.Cause(ctx)
+	}
 	// Send any remaining objects downstream
 	if len(toDel) > 0 {
 		select {

cmd/batch-expire_gen.go

@@ -39,7 +39,7 @@ func (z *BatchJobExpire) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Prefix":
-			z.Prefix, err = dc.ReadString()
+			err = z.Prefix.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "Prefix")
 				return
@@ -114,7 +114,7 @@ func (z *BatchJobExpire) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteString(z.Prefix)
+	err = z.Prefix.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "Prefix")
 		return
@@ -171,7 +171,11 @@ func (z *BatchJobExpire) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendString(o, z.Bucket)
 	// string "Prefix"
 	o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
-	o = msgp.AppendString(o, z.Prefix)
+	o, err = z.Prefix.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Prefix")
+		return
+	}
 	// string "NotificationCfg"
 	o = append(o, 0xaf, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x66, 0x67)
 	o, err = z.NotificationCfg.MarshalMsg(o)
@@ -230,7 +234,7 @@ func (z *BatchJobExpire) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "Prefix":
-			z.Prefix, bts, err = msgp.ReadStringBytes(bts)
+			bts, err = z.Prefix.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Prefix")
 				return
@@ -280,7 +284,7 @@ func (z *BatchJobExpire) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobExpire) Msgsize() (s int) {
-	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 16 + z.NotificationCfg.Msgsize() + 6 + z.Retry.Msgsize() + 6 + msgp.ArrayHeaderSize
+	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + z.Prefix.Msgsize() + 16 + z.NotificationCfg.Msgsize() + 6 + z.Retry.Msgsize() + 6 + msgp.ArrayHeaderSize
 	for za0001 := range z.Rules {
 		s += z.Rules[za0001].Msgsize()
 	}
@@ -306,7 +310,7 @@ func (z *BatchJobExpireFilter) DecodeMsg(dc *msgp.Reader) (err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "OlderThan":
-			z.OlderThan, err = dc.ReadDuration()
+			err = z.OlderThan.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -433,7 +437,7 @@ func (z *BatchJobExpireFilter) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteDuration(z.OlderThan)
+	err = z.OlderThan.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "OlderThan")
 		return
@@ -544,7 +548,11 @@ func (z *BatchJobExpireFilter) MarshalMsg(b []byte) (o []byte, err error) {
 	// map header, size 8
 	// string "OlderThan"
 	o = append(o, 0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
-	o = msgp.AppendDuration(o, z.OlderThan)
+	o, err = z.OlderThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "OlderThan")
+		return
+	}
 	// string "CreatedBefore"
 	o = append(o, 0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65)
 	if z.CreatedBefore == nil {
@@ -613,7 +621,7 @@ func (z *BatchJobExpireFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "OlderThan":
-			z.OlderThan, bts, err = msgp.ReadDurationBytes(bts)
+			bts, err = z.OlderThan.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -734,7 +742,7 @@ func (z *BatchJobExpireFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobExpireFilter) Msgsize() (s int) {
-	s = 1 + 10 + msgp.DurationSize + 14
+	s = 1 + 10 + z.OlderThan.Msgsize() + 14
 	if z.CreatedBefore == nil {
 		s += msgp.NilSize
 	} else {

cmd/batch-expire_test.go

@@ -18,9 +18,10 @@
 package cmd
 
 import (
+	"slices"
 	"testing"
 
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 func TestParseBatchJobExpire(t *testing.T) {
@@ -32,7 +33,7 @@ expire: # Expire objects that match a condition
   rules:
     - type: object  # regular objects with zero or more older versions
       name: NAME # match object names that satisfy the wildcard expression.
-      olderThan: 70h # match objects older than this value
+      olderThan: 7d10h # match objects older than this value
       createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
       tags:
         - key: name
@@ -64,8 +65,61 @@ expire: # Expire objects that match a condition
     delay: 500ms # least amount of delay between each retry
 `
 	var job BatchJobRequest
-	err := yaml.UnmarshalStrict([]byte(expireYaml), &job)
+	err := yaml.Unmarshal([]byte(expireYaml), &job)
 	if err != nil {
 		t.Fatal("Failed to parse batch-job-expire yaml", err)
 	}
+	if !slices.Equal(job.Expire.Prefix.F(), []string{"myprefix"}) {
+		t.Fatal("Failed to parse batch-job-expire yaml")
+	}
+
+	multiPrefixExpireYaml := `
+expire: # Expire objects that match a condition
+  apiVersion: v1
+  bucket: mybucket # Bucket where this batch job will expire matching objects from
+  prefix: # (Optional) Prefix under which this job will expire objects matching the rules below.
+    - myprefix
+    - myprefix1
+  rules:
+    - type: object  # regular objects with zero or more older versions
+      name: NAME # match object names that satisfy the wildcard expression.
+      olderThan: 7d10h # match objects older than this value
+      createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+      tags:
+        - key: name
+          value: pick* # match objects with tag 'name', all values starting with 'pick'
+      metadata:
+        - key: content-type
+          value: image/* # match objects with 'content-type', all values starting with 'image/'
+      size:
+        lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB)
+        greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB)
+      purge:
+          # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+          # retainVersions: 5 # keep the latest 5 versions of the object.
+  
+    - type: deleted # objects with delete marker as their latest version
+      name: NAME # match object names that satisfy the wildcard expression.
+      olderThan: 10h # match objects older than this value (e.g. 7d10h31s)
+      createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date"
+      purge:
+          # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest.
+          # retainVersions: 5 # keep the latest 5 versions of the object including delete markers.
+
+  notify:
+    endpoint: https://notify.endpoint # notification endpoint to receive job completion status
+    token: Bearer xxxxx # optional authentication token for the notification endpoint
+  
+  retry:
+    attempts: 10 # number of retries for the job before giving up
+    delay: 500ms # least amount of delay between each retry
+`
+	var multiPrefixJob BatchJobRequest
+	err = yaml.Unmarshal([]byte(multiPrefixExpireYaml), &multiPrefixJob)
+	if err != nil {
+		t.Fatal("Failed to parse batch-job-expire yaml", err)
+	}
+	if !slices.Equal(multiPrefixJob.Expire.Prefix.F(), []string{"myprefix", "myprefix1"}) {
+		t.Fatal("Failed to parse batch-job-expire yaml")
+	}
 }

cmd/batch-handlers.go

@@ -287,12 +287,12 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	isStorageClassOnly := len(r.Flags.Filter.Metadata) == 1 && strings.EqualFold(r.Flags.Filter.Metadata[0].Key, xhttp.AmzStorageClass)
 
 	skip := func(oi ObjectInfo) (ok bool) {
-		if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan {
+		if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan.D() {
 			// skip all objects that are newer than specified older duration
 			return true
 		}
 
-		if r.Flags.Filter.NewerThan > 0 && time.Since(oi.ModTime) >= r.Flags.Filter.NewerThan {
+		if r.Flags.Filter.NewerThan > 0 && time.Since(oi.ModTime) >= r.Flags.Filter.NewerThan.D() {
 			// skip all objects that are older than specified newer duration
 			return true
 		}
@@ -385,12 +385,23 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
 		minioSrc := r.Source.Type == BatchJobReplicateResourceMinIO
 		ctx, cancel := context.WithCancel(ctx)
-		objInfoCh := c.ListObjects(ctx, r.Source.Bucket, miniogo.ListObjectsOptions{
-			Prefix:       r.Source.Prefix,
-			WithVersions: minioSrc,
-			Recursive:    true,
-			WithMetadata: true,
-		})
+
+		objInfoCh := make(chan miniogo.ObjectInfo, 1)
+		go func() {
+			for _, prefix := range r.Source.Prefix.F() {
+				prefixObjInfoCh := c.ListObjects(ctx, r.Source.Bucket, miniogo.ListObjectsOptions{
+					Prefix:       strings.TrimSpace(prefix),
+					WithVersions: minioSrc,
+					Recursive:    true,
+					WithMetadata: true,
+				})
+				for obj := range prefixObjInfoCh {
+					objInfoCh <- obj
+				}
+			}
+			xioutil.SafeClose(objInfoCh)
+		}()
+
 		prevObj := ""
 		skipReplicate := false
 
@@ -540,7 +551,7 @@ func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo
 	return oi
 }
 
-func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo) error {
+func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string) error {
 	input := make(chan minio.SnowballObject, 1)
 	opts := minio.SnowballOptions{
 		Opts:     minio.PutObjectOptions{},
@@ -562,6 +573,10 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
 				continue
 			}
 
+			if prefix != "" {
+				entry.Name = pathJoin(prefix, entry.Name)
+			}
+
 			snowballObj := minio.SnowballObject{
 				// Create path to store objects within the bucket.
 				Key:       entry.Name,
@@ -725,7 +740,7 @@ const (
 
 	batchReplJobAPIVersion        = "v1"
 	batchReplJobDefaultRetries    = 3
-	batchReplJobDefaultRetryDelay = 250 * time.Millisecond
+	batchReplJobDefaultRetryDelay = time.Second
 )
 
 func getJobPath(job BatchJobRequest) string {
@@ -1016,18 +1031,18 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	lastObject := ri.Object
 
 	delay := job.Replicate.Flags.Retry.Delay
-	if delay == 0 {
+	if delay < time.Second {
 		delay = batchReplJobDefaultRetryDelay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	selectObj := func(info FileInfo) (ok bool) {
-		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
+		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan.D() {
 			// skip all objects that are newer than specified older duration
 			return false
 		}
 
-		if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan {
+		if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan.D() {
 			// skip all objects that are older than specified newer duration
 			return false
 		}
@@ -1115,7 +1130,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			slowCh = make(chan itemOrErr[ObjectInfo], 100)
 		)
 
-		if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
+		if r.Source.Snowball.Disable != nil && !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
 			go func() {
 				// Snowball currently needs the high level minio-go Client, not the Core one
 				cl, err := miniogo.New(u.Host, &miniogo.Options{
@@ -1125,7 +1140,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 					BucketLookup: lookupStyle(r.Target.Path),
 				})
 				if err != nil {
-					batchLogIf(ctx, err)
+					batchLogOnceIf(ctx, err, job.ID+"miniogo.New")
 					return
 				}
 
@@ -1135,8 +1150,8 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch)
 				writeFn := func(batch []ObjectInfo) {
 					if len(batch) > 0 {
-						if err := r.writeAsArchive(ctx, api, cl, batch); err != nil {
-							batchLogIf(ctx, err)
+						if err := r.writeAsArchive(ctx, api, cl, batch, r.Target.Prefix); err != nil {
+							batchLogOnceIf(ctx, err, job.ID+"writeAsArchive")
 							for _, b := range batch {
 								slowCh <- itemOrErr[ObjectInfo]{Item: b}
 							}
@@ -1144,7 +1159,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 							ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
 							globalBatchJobsMetrics.save(job.ID, ri)
 							// persist in-memory state to disk after every 10secs.
-							batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+							batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job), job.ID+"updateAfter")
 						}
 					}
 				}
@@ -1184,19 +1199,28 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		if walkQuorum == "" {
 			walkQuorum = "strict"
 		}
-		ctx, cancel := context.WithCancel(ctx)
+		ctx, cancelCause := context.WithCancelCause(ctx)
 		// one of source/target is s3, skip delete marker and all versions under the same object name.
 		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
 
-		if err := api.Walk(ctx, r.Source.Bucket, r.Source.Prefix, walkCh, WalkOptions{
-			Marker:   lastObject,
-			Filter:   selectObj,
-			AskDisks: walkQuorum,
-		}); err != nil {
-			cancel()
-			// Do not need to retry if we can't list objects on source.
-			return err
-		}
+		go func() {
+			for _, prefix := range r.Source.Prefix.F() {
+				prefixWalkCh := make(chan itemOrErr[ObjectInfo], 100)
+				if err := api.Walk(ctx, r.Source.Bucket, strings.TrimSpace(prefix), prefixWalkCh, WalkOptions{
+					Marker:   lastObject,
+					Filter:   selectObj,
+					AskDisks: walkQuorum,
+				}); err != nil {
+					cancelCause(err)
+					xioutil.SafeClose(walkCh)
+					return
+				}
+				for obj := range prefixWalkCh {
+					walkCh <- obj
+				}
+			}
+			xioutil.SafeClose(walkCh)
+		}()
 
 		prevObj := ""
 
@@ -1204,7 +1228,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		for res := range slowCh {
 			if res.Err != nil {
 				ri.Failed = true
-				batchLogIf(ctx, res.Err)
+				batchLogOnceIf(ctx, res.Err, job.ID+"res.Err")
 				continue
 			}
 			result := res.Item
@@ -1231,7 +1255,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						return
 					}
 					stopFn(result, err)
-					batchLogIf(ctx, err)
+					batchLogOnceIf(ctx, err, job.ID+"ReplicateToTarget")
 					success = false
 				} else {
 					stopFn(result, nil)
@@ -1239,7 +1263,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				ri.trackCurrentBucketObject(r.Source.Bucket, result, success, attempts)
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
-				batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
+				batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job), job.ID+"updateAfter2")
 
 				if wait := globalBatchConfig.ReplicationWait(); wait > 0 {
 					time.Sleep(wait)
@@ -1247,20 +1271,23 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			}()
 		}
 		wk.Wait()
-
+		// Do not need to retry if we can't list objects on source.
+		if context.Cause(ctx) != nil {
+			return context.Cause(ctx)
+		}
 		ri.RetryAttempts = attempts
 		ri.Complete = ri.ObjectsFailed == 0
 		ri.Failed = ri.ObjectsFailed > 0
 
 		globalBatchJobsMetrics.save(job.ID, ri)
 		// persist in-memory state to disk.
-		batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job))
+		batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 0, job), job.ID+"updateAfter3")
 
 		if err := r.Notify(ctx, ri); err != nil {
-			batchLogIf(ctx, fmt.Errorf("unable to notify %v", err))
+			batchLogOnceIf(ctx, fmt.Errorf("unable to notify %v", err), job.ID+"notify")
 		}
 
-		cancel()
+		cancelCause(nil)
 		if ri.Failed {
 			ri.ObjectsFailed = 0
 			ri.Bucket = ""
@@ -1567,6 +1594,9 @@ func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request)
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, result.Err), r.URL)
 			return
 		}
+		if strings.HasPrefix(result.Item.Name, batchJobReportsPrefix+slashSeparator) {
+			continue
+		}
 		req := &BatchJobRequest{}
 		if err := req.load(ctx, objectAPI, result.Item.Name); err != nil {
 			if !errors.Is(err, errNoSuchJob) {
@@ -1675,7 +1705,7 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 	w.Write(buf)
 }
 
-// StarBatchJob queue a new job for execution
+// StartBatchJob queue a new job for execution
 func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
@@ -1883,6 +1913,9 @@ func (j *BatchJobPool) resume(randomWait func() time.Duration) {
 			batchLogIf(j.ctx, result.Err)
 			continue
 		}
+		if strings.HasPrefix(result.Item.Name, batchJobReportsPrefix+slashSeparator) {
+			continue
+		}
 		// ignore batch-replicate.bin and batch-rotate.bin entries
 		if strings.HasSuffix(result.Item.Name, slashSeparator) {
 			continue
@@ -2222,3 +2255,30 @@ func lookupStyle(s string) miniogo.BucketLookupType {
 	}
 	return lookup
 }
+
+// BatchJobPrefix - to support prefix field yaml unmarshalling with string or slice of strings
+type BatchJobPrefix []string
+
+var _ yaml.Unmarshaler = &BatchJobPrefix{}
+
+// UnmarshalYAML - to support prefix field yaml unmarshalling with string or slice of strings
+func (b *BatchJobPrefix) UnmarshalYAML(value *yaml.Node) error {
+	// try slice first
+	tmpSlice := []string{}
+	if err := value.Decode(&tmpSlice); err == nil {
+		*b = tmpSlice
+		return nil
+	}
+	// try string
+	tmpStr := ""
+	if err := value.Decode(&tmpStr); err == nil {
+		*b = []string{tmpStr}
+		return nil
+	}
+	return fmt.Errorf("unable to decode %s", value.Value)
+}
+
+// F - return prefix(es) as slice
+func (b *BatchJobPrefix) F() []string {
+	return *b
+}

cmd/batch-handlers_gen.go

@@ -6,6 +6,89 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobPrefix) DecodeMsg(dc *msgp.Reader) (err error) {
+	var zb0002 uint32
+	zb0002, err = dc.ReadArrayHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if cap((*z)) >= int(zb0002) {
+		(*z) = (*z)[:zb0002]
+	} else {
+		(*z) = make(BatchJobPrefix, zb0002)
+	}
+	for zb0001 := range *z {
+		(*z)[zb0001], err = dc.ReadString()
+		if err != nil {
+			err = msgp.WrapError(err, zb0001)
+			return
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobPrefix) EncodeMsg(en *msgp.Writer) (err error) {
+	err = en.WriteArrayHeader(uint32(len(z)))
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0003 := range z {
+		err = en.WriteString(z[zb0003])
+		if err != nil {
+			err = msgp.WrapError(err, zb0003)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobPrefix) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	o = msgp.AppendArrayHeader(o, uint32(len(z)))
+	for zb0003 := range z {
+		o = msgp.AppendString(o, z[zb0003])
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobPrefix) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var zb0002 uint32
+	zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	if cap((*z)) >= int(zb0002) {
+		(*z) = (*z)[:zb0002]
+	} else {
+		(*z) = make(BatchJobPrefix, zb0002)
+	}
+	for zb0001 := range *z {
+		(*z)[zb0001], bts, err = msgp.ReadStringBytes(bts)
+		if err != nil {
+			err = msgp.WrapError(err, zb0001)
+			return
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobPrefix) Msgsize() (s int) {
+	s = msgp.ArrayHeaderSize
+	for zb0003 := range z {
+		s += msgp.StringPrefixSize + len(z[zb0003])
+	}
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/batch-handlers_gen_test.go

@@ -9,6 +9,119 @@ import (
 	"github.com/tinylib/msgp/msgp"
 )
 
+func TestMarshalUnmarshalBatchJobPrefix(t *testing.T) {
+	v := BatchJobPrefix{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobPrefix(b *testing.B) {
+	v := BatchJobPrefix{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobPrefix(b *testing.B) {
+	v := BatchJobPrefix{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobPrefix(b *testing.B) {
+	v := BatchJobPrefix{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobPrefix(t *testing.T) {
+	v := BatchJobPrefix{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobPrefix Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobPrefix{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobPrefix(b *testing.B) {
+	v := BatchJobPrefix{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobPrefix(b *testing.B) {
+	v := BatchJobPrefix{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalBatchJobRequest(t *testing.T) {
 	v := BatchJobRequest{}
 	bts, err := v.MarshalMsg(nil)

cmd/batch-handlers_test.go

@@ -0,0 +1,75 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"slices"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestBatchJobPrefix_UnmarshalYAML(t *testing.T) {
+	type args struct {
+		yamlStr string
+	}
+	type PrefixTemp struct {
+		Prefix BatchJobPrefix `yaml:"prefix"`
+	}
+	tests := []struct {
+		name    string
+		b       PrefixTemp
+		args    args
+		want    []string
+		wantErr bool
+	}{
+		{
+			name: "test1",
+			b:    PrefixTemp{},
+			args: args{
+				yamlStr: `
+prefix: "foo"
+`,
+			},
+			want:    []string{"foo"},
+			wantErr: false,
+		},
+		{
+			name: "test2",
+			b:    PrefixTemp{},
+			args: args{
+				yamlStr: `
+prefix:
+  - "foo"
+  - "bar"
+`,
+			},
+			want: []string{"foo", "bar"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := yaml.Unmarshal([]byte(tt.args.yamlStr), &tt.b); (err != nil) != tt.wantErr {
+				t.Errorf("UnmarshalYAML() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if !slices.Equal(tt.b.Prefix.F(), tt.want) {
+				t.Errorf("UnmarshalYAML() = %v, want %v", tt.b.Prefix.F(), tt.want)
+			}
+		})
+	}
+}

cmd/batch-replicate.go

@@ -21,8 +21,8 @@ import (
 	"time"
 
 	miniogo "github.com/minio/minio-go/v7"
-
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/pkg/v3/xtime"
 )
 
 //go:generate msgp -file $GOFILE
@@ -65,12 +65,12 @@ import (
 
 // BatchReplicateFilter holds all the filters currently supported for batch replication
 type BatchReplicateFilter struct {
-	NewerThan     time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
-	OlderThan     time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
-	CreatedAfter  time.Time     `yaml:"createdAfter,omitempty" json:"createdAfter"`
-	CreatedBefore time.Time     `yaml:"createdBefore,omitempty" json:"createdBefore"`
-	Tags          []BatchJobKV  `yaml:"tags,omitempty" json:"tags"`
-	Metadata      []BatchJobKV  `yaml:"metadata,omitempty" json:"metadata"`
+	NewerThan     xtime.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
+	OlderThan     xtime.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedAfter  time.Time      `yaml:"createdAfter,omitempty" json:"createdAfter"`
+	CreatedBefore time.Time      `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchJobKV   `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchJobKV   `yaml:"metadata,omitempty" json:"metadata"`
 }
 
 // BatchJobReplicateFlags various configurations for replication job definition currently includes
@@ -151,7 +151,7 @@ func (t BatchJobReplicateTarget) ValidPath() bool {
 type BatchJobReplicateSource struct {
 	Type     BatchJobReplicateResourceType `yaml:"type" json:"type"`
 	Bucket   string                        `yaml:"bucket" json:"bucket"`
-	Prefix   string                        `yaml:"prefix" json:"prefix"`
+	Prefix   BatchJobPrefix                `yaml:"prefix" json:"prefix"`
 	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
 	Path     string                        `yaml:"path" json:"path"`
 	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`

cmd/batch-replicate_gen.go

@@ -411,7 +411,7 @@ func (z *BatchJobReplicateSource) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Prefix":
-			z.Prefix, err = dc.ReadString()
+			err = z.Prefix.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "Prefix")
 				return
@@ -514,7 +514,7 @@ func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteString(z.Prefix)
+	err = z.Prefix.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "Prefix")
 		return
@@ -600,7 +600,11 @@ func (z *BatchJobReplicateSource) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendString(o, z.Bucket)
 	// string "Prefix"
 	o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78)
-	o = msgp.AppendString(o, z.Prefix)
+	o, err = z.Prefix.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Prefix")
+		return
+	}
 	// string "Endpoint"
 	o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
 	o = msgp.AppendString(o, z.Endpoint)
@@ -664,7 +668,7 @@ func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error)
 				return
 			}
 		case "Prefix":
-			z.Prefix, bts, err = msgp.ReadStringBytes(bts)
+			bts, err = z.Prefix.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Prefix")
 				return
@@ -742,7 +746,7 @@ func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error)
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobReplicateSource) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken) + 9 + z.Snowball.Msgsize()
+	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + z.Prefix.Msgsize() + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken) + 9 + z.Snowball.Msgsize()
 	return
 }
 
@@ -1409,13 +1413,13 @@ func (z *BatchReplicateFilter) DecodeMsg(dc *msgp.Reader) (err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "NewerThan":
-			z.NewerThan, err = dc.ReadDuration()
+			err = z.NewerThan.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "NewerThan")
 				return
 			}
 		case "OlderThan":
-			z.OlderThan, err = dc.ReadDuration()
+			err = z.OlderThan.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -1489,7 +1493,7 @@ func (z *BatchReplicateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteDuration(z.NewerThan)
+	err = z.NewerThan.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "NewerThan")
 		return
@@ -1499,7 +1503,7 @@ func (z *BatchReplicateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteDuration(z.OlderThan)
+	err = z.OlderThan.EncodeMsg(en)
 	if err != nil {
 		err = msgp.WrapError(err, "OlderThan")
 		return
@@ -1567,10 +1571,18 @@ func (z *BatchReplicateFilter) MarshalMsg(b []byte) (o []byte, err error) {
 	// map header, size 6
 	// string "NewerThan"
 	o = append(o, 0x86, 0xa9, 0x4e, 0x65, 0x77, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
-	o = msgp.AppendDuration(o, z.NewerThan)
+	o, err = z.NewerThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "NewerThan")
+		return
+	}
 	// string "OlderThan"
 	o = append(o, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e)
-	o = msgp.AppendDuration(o, z.OlderThan)
+	o, err = z.OlderThan.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "OlderThan")
+		return
+	}
 	// string "CreatedAfter"
 	o = append(o, 0xac, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x66, 0x74, 0x65, 0x72)
 	o = msgp.AppendTime(o, z.CreatedAfter)
@@ -1619,13 +1631,13 @@ func (z *BatchReplicateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		}
 		switch msgp.UnsafeString(field) {
 		case "NewerThan":
-			z.NewerThan, bts, err = msgp.ReadDurationBytes(bts)
+			bts, err = z.NewerThan.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "NewerThan")
 				return
 			}
 		case "OlderThan":
-			z.OlderThan, bts, err = msgp.ReadDurationBytes(bts)
+			bts, err = z.OlderThan.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "OlderThan")
 				return
@@ -1694,7 +1706,7 @@ func (z *BatchReplicateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchReplicateFilter) Msgsize() (s int) {
-	s = 1 + 10 + msgp.DurationSize + 10 + msgp.DurationSize + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize
+	s = 1 + 10 + z.NewerThan.Msgsize() + 10 + z.OlderThan.Msgsize() + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize
 	for za0001 := range z.Tags {
 		s += z.Tags[za0001].Msgsize()
 	}

cmd/batch-replicate_test.go

@@ -0,0 +1,182 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"slices"
+	"testing"
+
+	"gopkg.in/yaml.v3"
+)
+
+func TestParseBatchJobReplicate(t *testing.T) {
+	replicateYaml := `
+replicate:
+  apiVersion: v1
+  # source of the objects to be replicated
+  source:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: object-prefix1 # 'PREFIX' is optional
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+#    endpoint: "http://127.0.0.1:9000"
+#    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+#    credentials:
+#      accessKey: minioadmin # Required
+#      secretKey: minioadmin # Required
+#    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+    snowball: # automatically activated if the source is local
+      disable: true # optionally turn-off snowball archive transfer
+#      batch: 100 # upto this many objects per archive
+#      inmemory: true # indicates if the archive must be staged locally or in-memory
+#      compress: false # S2/Snappy compressed archive
+#      smallerThan: 5MiB # create archive for all objects smaller than 5MiB
+#      skipErrs: false # skips any source side read() errors
+
+  # target where the objects must be replicated
+  target:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: stage # 'PREFIX' is optional
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+    endpoint: "http://127.0.0.1:9001"
+    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+    credentials:
+      accessKey: minioadmin
+      secretKey: minioadmin
+    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+
+  # NOTE: All flags are optional
+  # - filtering criteria only applies for all source objects match the criteria
+  # - configurable notification endpoints
+  # - configurable retries for the job (each retry skips successfully previously replaced objects)
+  flags:
+    filter:
+      newerThan: "7d10h31s" # match objects newer than this value (e.g. 7d10h31s)
+      olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
+#      createdAfter: "date" # match objects created after "date"
+#      createdBefore: "date" # match objects created before "date"
+
+      ## NOTE: tags are not supported when "source" is remote.
+      tags:
+         - key: "name"
+           value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
+
+      metadata:
+         - key: "content-type"
+           value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
+
+#    notify:
+#      endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
+#      token: "Bearer xxxxx" # optional authentication token for the notification endpoint
+#
+#    retry:
+#      attempts: 10 # number of retries for the job before giving up
+#      delay: "500ms" # least amount of delay between each retry
+
+`
+	var job BatchJobRequest
+	err := yaml.Unmarshal([]byte(replicateYaml), &job)
+	if err != nil {
+		t.Fatal("Failed to parse batch-job-replicate yaml", err)
+	}
+	if !slices.Equal(job.Replicate.Source.Prefix.F(), []string{"object-prefix1"}) {
+		t.Fatal("Failed to parse batch-job-replicate yaml", err)
+	}
+	multiPrefixReplicateYaml := `
+replicate:
+  apiVersion: v1
+  # source of the objects to be replicated
+  source:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: # 'PREFIX' is optional
+      - object-prefix1 
+      - object-prefix2
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+#    endpoint: "http://127.0.0.1:9000"
+#    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+#    credentials:
+#      accessKey: minioadmin # Required
+#      secretKey: minioadmin # Required
+#    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+    snowball: # automatically activated if the source is local
+      disable: true # optionally turn-off snowball archive transfer
+#      batch: 100 # upto this many objects per archive
+#      inmemory: true # indicates if the archive must be staged locally or in-memory
+#      compress: false # S2/Snappy compressed archive
+#      smallerThan: 5MiB # create archive for all objects smaller than 5MiB
+#      skipErrs: false # skips any source side read() errors
+
+  # target where the objects must be replicated
+  target:
+    type: minio # valid values are "s3" or "minio"
+    bucket: mytest
+    prefix: stage # 'PREFIX' is optional
+    # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted
+
+    # Either the 'source' or 'remote' *must* be the "local" deployment
+    endpoint: "http://127.0.0.1:9001"
+    # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto"
+    credentials:
+      accessKey: minioadmin
+      secretKey: minioadmin
+    # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used
+
+  # NOTE: All flags are optional
+  # - filtering criteria only applies for all source objects match the criteria
+  # - configurable notification endpoints
+  # - configurable retries for the job (each retry skips successfully previously replaced objects)
+  flags:
+    filter:
+      newerThan: "7d10h31s" # match objects newer than this value (e.g. 7d10h31s)
+      olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
+#      createdAfter: "date" # match objects created after "date"
+#      createdBefore: "date" # match objects created before "date"
+
+      ## NOTE: tags are not supported when "source" is remote.
+      tags:
+         - key: "name"
+           value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
+
+      metadata:
+         - key: "content-type"
+           value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
+
+#    notify:
+#      endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
+#      token: "Bearer xxxxx" # optional authentication token for the notification endpoint
+#
+#    retry:
+#      attempts: 10 # number of retries for the job before giving up
+#      delay: "500ms" # least amount of delay between each retry
+
+`
+	var multiPrefixJob BatchJobRequest
+	err = yaml.Unmarshal([]byte(multiPrefixReplicateYaml), &multiPrefixJob)
+	if err != nil {
+		t.Fatal("Failed to parse batch-job-replicate yaml", err)
+	}
+	if !slices.Equal(multiPrefixJob.Replicate.Source.Prefix.F(), []string{"object-prefix1", "object-prefix2"}) {
+		t.Fatal("Failed to parse batch-job-replicate yaml")
+	}
+}

cmd/bitrot-streaming.go

@@ -141,13 +141,7 @@ func (b *streamingBitrotReader) Close() error {
 	}
 	if closer, ok := b.rc.(io.Closer); ok {
 		// drain the body for connection reuse at network layer.
-		xhttp.DrainBody(struct {
-			io.Reader
-			io.Closer
-		}{
-			Reader: b.rc,
-			Closer: closeWrapper(func() error { return nil }),
-		})
+		xhttp.DrainBody(io.NopCloser(b.rc))
 		return closer.Close()
 	}
 	return nil

cmd/bucket-handlers.go

@@ -1682,7 +1682,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 	}
 
 	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
-	globalReplicationPool.deleteResyncMetadata(ctx, bucket)
+	globalReplicationPool.Get().deleteResyncMetadata(ctx, bucket)
 
 	// Call site replication hook.
 	replLogIf(ctx, globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete))

cmd/bucket-lifecycle-audit.go

@@ -17,7 +17,11 @@
 
 package cmd
 
-import "github.com/minio/minio/internal/bucket/lifecycle"
+import (
+	"strconv"
+
+	"github.com/minio/minio/internal/bucket/lifecycle"
+)
 
 //go:generate stringer -type lcEventSrc -trimprefix lcEventSrc_ $GOFILE
 type lcEventSrc uint8
@@ -43,7 +47,7 @@ type lcAuditEvent struct {
 	source lcEventSrc
 }
 
-func (lae lcAuditEvent) Tags() map[string]interface{} {
+func (lae lcAuditEvent) Tags() map[string]string {
 	event := lae.Event
 	src := lae.source
 	const (
@@ -55,7 +59,7 @@ func (lae lcAuditEvent) Tags() map[string]interface{} {
 		ilmNewerNoncurrentVersions = "ilm-newer-noncurrent-versions"
 		ilmNoncurrentDays          = "ilm-noncurrent-days"
 	)
-	tags := make(map[string]interface{}, 5)
+	tags := make(map[string]string, 5)
 	if src > lcEventSrc_None {
 		tags[ilmSrc] = src.String()
 	}
@@ -63,7 +67,7 @@ func (lae lcAuditEvent) Tags() map[string]interface{} {
 	tags[ilmRuleID] = event.RuleID
 
 	if !event.Due.IsZero() {
-		tags[ilmDue] = event.Due
+		tags[ilmDue] = event.Due.Format(iso8601Format)
 	}
 
 	// rule with Transition/NoncurrentVersionTransition in effect
@@ -73,10 +77,10 @@ func (lae lcAuditEvent) Tags() map[string]interface{} {
 
 	// rule with NewernoncurrentVersions in effect
 	if event.NewerNoncurrentVersions > 0 {
-		tags[ilmNewerNoncurrentVersions] = event.NewerNoncurrentVersions
+		tags[ilmNewerNoncurrentVersions] = strconv.Itoa(event.NewerNoncurrentVersions)
 	}
 	if event.NoncurrentDays > 0 {
-		tags[ilmNoncurrentDays] = event.NoncurrentDays
+		tags[ilmNoncurrentDays] = strconv.Itoa(event.NoncurrentDays)
 	}
 	return tags
 }

cmd/bucket-lifecycle.go

@@ -71,7 +71,7 @@ func NewLifecycleSys() *LifecycleSys {
 	return &LifecycleSys{}
 }
 
-func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string) madmin.TraceInfo {
+func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string, metadata map[string]string) madmin.TraceInfo {
 	sz, _ := oi.GetActualSize()
 	return madmin.TraceInfo{
 		TraceType: madmin.TraceILM,
@@ -83,16 +83,16 @@ func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event
 		Bytes:     sz,
 		Error:     "",
 		Message:   getSource(4),
-		Custom:    map[string]string{"version-id": oi.VersionID},
+		Custom:    metadata,
 	}
 }
 
-func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string) {
+func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string, metadata map[string]string) {
 	startTime := time.Now()
-	return func(event string) {
+	return func(event string, metadata map[string]string) {
 		duration := time.Since(startTime)
 		if globalTrace.NumSubscribers(madmin.TraceILM) > 0 {
-			globalTrace.Publish(ilmTrace(startTime, duration, oi, event))
+			globalTrace.Publish(ilmTrace(startTime, duration, oi, event, metadata))
 		}
 	}
 }
@@ -707,6 +707,11 @@ type auditTierOp struct {
 	Error            string `json:"error,omitempty"`
 }
 
+func (op auditTierOp) String() string {
+	// flattening the auditTierOp{} for audit
+	return fmt.Sprintf("tier:%s,respNS:%d,tx:%d,err:%s", op.Tier, op.TimeToResponseNS, op.OutputBytes, op.Error)
+}
+
 func auditTierActions(ctx context.Context, tier string, bytes int64) func(err error) {
 	startTime := time.Now()
 	return func(err error) {
@@ -730,7 +735,7 @@ func auditTierActions(ctx context.Context, tier string, bytes int64) func(err er
 			globalTierMetrics.logFailure(tier)
 		}
 
-		logger.GetReqInfo(ctx).AppendTags("tierStats", op)
+		logger.GetReqInfo(ctx).AppendTags("tierStats", op.String())
 	}
 }
 
@@ -998,7 +1003,7 @@ func ongoingRestoreObj() restoreObjStatus {
 	}
 }
 
-// completeRestoreObj constructs restoreObjStatus for a completed restore-object with given expiry.
+// completedRestoreObj constructs restoreObjStatus for a completed restore-object with given expiry.
 func completedRestoreObj(expiry time.Time) restoreObjStatus {
 	return restoreObjStatus{
 		ongoing: false,

cmd/bucket-metadata-sys.go

@@ -39,6 +39,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/policy"
 	"github.com/minio/pkg/v3/sync/errgroup"
+	"golang.org/x/sync/singleflight"
 )
 
 // BucketMetadataSys captures all bucket metadata for a given cluster.
@@ -47,6 +48,7 @@ type BucketMetadataSys struct {
 
 	sync.RWMutex
 	initialized bool
+	group       *singleflight.Group
 	metadataMap map[string]BucketMetadata
 }
 
@@ -62,6 +64,7 @@ func (sys *BucketMetadataSys) Count() int {
 func (sys *BucketMetadataSys) Remove(buckets ...string) {
 	sys.Lock()
 	for _, bucket := range buckets {
+		sys.group.Forget(bucket)
 		delete(sys.metadataMap, bucket)
 		globalBucketMonitor.DeleteBucket(bucket)
 	}
@@ -121,6 +124,7 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 		meta.PolicyConfigUpdatedAt = updatedAt
 	case bucketNotificationConfig:
 		meta.NotificationConfigXML = configData
+		meta.NotificationConfigUpdatedAt = updatedAt
 	case bucketLifecycleConfig:
 		meta.LifecycleConfigXML = configData
 		meta.LifecycleConfigUpdatedAt = updatedAt
@@ -150,12 +154,13 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 		if err != nil {
 			return updatedAt, fmt.Errorf("Error encrypting bucket target metadata %w", err)
 		}
+		meta.BucketTargetsConfigUpdatedAt = updatedAt
+		meta.BucketTargetsConfigMetaUpdatedAt = updatedAt
 	default:
 		return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
 
-	err = sys.save(ctx, meta)
-	return updatedAt, err
+	return updatedAt, sys.save(ctx, meta)
 }
 
 func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
@@ -262,6 +267,21 @@ func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Ve
 	return meta.versioningConfig, meta.VersioningConfigUpdatedAt, nil
 }
 
+// GetBucketPolicy returns configured bucket policy
+func (sys *BucketMetadataSys) GetBucketPolicy(bucket string) (*policy.BucketPolicy, time.Time, error) {
+	meta, _, err := sys.GetConfig(GlobalContext, bucket)
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket}
+		}
+		return nil, time.Time{}, err
+	}
+	if meta.policyConfig == nil {
+		return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket}
+	}
+	return meta.policyConfig, meta.PolicyConfigUpdatedAt, nil
+}
+
 // GetTaggingConfig returns configured tagging config
 // The returned object may not be modified.
 func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, time.Time, error) {
@@ -451,13 +471,20 @@ func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (met
 	if ok {
 		return meta, reloaded, nil
 	}
-	meta, err = loadBucketMetadata(ctx, objAPI, bucket)
-	if err != nil {
-		if !sys.Initialized() {
-			// bucket metadata not yet initialized
-			return newBucketMetadata(bucket), reloaded, errBucketMetadataNotInitialized
+
+	val, err, _ := sys.group.Do(bucket, func() (val interface{}, err error) {
+		meta, err = loadBucketMetadata(ctx, objAPI, bucket)
+		if err != nil {
+			if !sys.Initialized() {
+				// bucket metadata not yet initialized
+				return newBucketMetadata(bucket), errBucketMetadataNotInitialized
+			}
 		}
-		return meta, reloaded, err
+		return meta, err
+	})
+	meta, _ = val.(BucketMetadata)
+	if err != nil {
+		return meta, false, err
 	}
 	sys.Lock()
 	sys.metadataMap[bucket] = meta
@@ -480,7 +507,7 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []string, objAPI
 }
 
 // concurrently load bucket metadata to speed up loading bucket metadata.
-func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []string, failedBuckets map[string]struct{}) {
+func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []string) {
 	g := errgroup.WithNErrs(len(buckets))
 	bucketMetas := make([]BucketMetadata, len(buckets))
 	for index := range buckets {
@@ -521,10 +548,6 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []stri
 
 	for i, meta := range bucketMetas {
 		if errs[i] != nil {
-			if failedBuckets == nil {
-				failedBuckets = make(map[string]struct{})
-			}
-			failedBuckets[buckets[i]] = struct{}{}
 			continue
 		}
 		globalEventNotifier.set(buckets[i], meta)   // set notification targets
@@ -532,7 +555,7 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []stri
 	}
 }
 
-func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, failedBuckets map[string]struct{}) {
+func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
 	const bucketMetadataRefresh = 15 * time.Minute
 
 	sleeper := newDynamicSleeper(2, 150*time.Millisecond, false)
@@ -562,7 +585,10 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa
 			for i := range buckets {
 				wait := sleeper.Timer(ctx)
 
-				meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[i].Name)
+				bucket := buckets[i].Name
+				updated := false
+
+				meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket)
 				if err != nil {
 					internalLogIf(ctx, err, logger.WarningKind)
 					wait() // wait to proceed to next entry.
@@ -570,14 +596,16 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context, fa
 				}
 
 				sys.Lock()
-				sys.metadataMap[buckets[i].Name] = meta
+				// Update if the bucket metadata in the memory is older than on-disk one
+				if lu := sys.metadataMap[bucket].lastUpdate(); lu.Before(meta.lastUpdate()) {
+					updated = true
+					sys.metadataMap[bucket] = meta
+				}
 				sys.Unlock()
 
-				// Initialize the failed buckets
-				if _, ok := failedBuckets[buckets[i].Name]; ok {
-					globalEventNotifier.set(buckets[i].Name, meta)
-					globalBucketTargetSys.set(buckets[i].Name, meta)
-					delete(failedBuckets, buckets[i].Name)
+				if updated {
+					globalEventNotifier.set(bucket, meta)
+					globalBucketTargetSys.set(bucket, meta)
 				}
 
 				wait() // wait to proceed to next entry.
@@ -598,13 +626,12 @@ func (sys *BucketMetadataSys) Initialized() bool {
 // Loads bucket metadata for all buckets into BucketMetadataSys.
 func (sys *BucketMetadataSys) init(ctx context.Context, buckets []string) {
 	count := globalEndpoints.ESCount() * 10
-	failedBuckets := make(map[string]struct{})
 	for {
 		if len(buckets) < count {
-			sys.concurrentLoad(ctx, buckets, failedBuckets)
+			sys.concurrentLoad(ctx, buckets)
 			break
 		}
-		sys.concurrentLoad(ctx, buckets[:count], failedBuckets)
+		sys.concurrentLoad(ctx, buckets[:count])
 		buckets = buckets[count:]
 	}
 
@@ -613,7 +640,7 @@ func (sys *BucketMetadataSys) init(ctx context.Context, buckets []string) {
 	sys.Unlock()
 
 	if globalIsDistErasure {
-		go sys.refreshBucketsMetadataLoop(ctx, failedBuckets)
+		go sys.refreshBucketsMetadataLoop(ctx)
 	}
 }
 
@@ -630,5 +657,6 @@ func (sys *BucketMetadataSys) Reset() {
 func NewBucketMetadataSys() *BucketMetadataSys {
 	return &BucketMetadataSys{
 		metadataMap: make(map[string]BucketMetadata),
+		group:       &singleflight.Group{},
 	}
 }

cmd/bucket-metadata.go

@@ -81,14 +81,19 @@ type BucketMetadata struct {
 	ReplicationConfigXML        []byte
 	BucketTargetsConfigJSON     []byte
 	BucketTargetsConfigMetaJSON []byte
-	PolicyConfigUpdatedAt       time.Time
-	ObjectLockConfigUpdatedAt   time.Time
-	EncryptionConfigUpdatedAt   time.Time
-	TaggingConfigUpdatedAt      time.Time
-	QuotaConfigUpdatedAt        time.Time
-	ReplicationConfigUpdatedAt  time.Time
-	VersioningConfigUpdatedAt   time.Time
-	LifecycleConfigUpdatedAt    time.Time
+
+	PolicyConfigUpdatedAt            time.Time
+	ObjectLockConfigUpdatedAt        time.Time
+	EncryptionConfigUpdatedAt        time.Time
+	TaggingConfigUpdatedAt           time.Time
+	QuotaConfigUpdatedAt             time.Time
+	ReplicationConfigUpdatedAt       time.Time
+	VersioningConfigUpdatedAt        time.Time
+	LifecycleConfigUpdatedAt         time.Time
+	NotificationConfigUpdatedAt      time.Time
+	BucketTargetsConfigUpdatedAt     time.Time
+	BucketTargetsConfigMetaUpdatedAt time.Time
+	// Add a new UpdatedAt field and update lastUpdate function
 
 	// Unexported fields. Must be updated atomically.
 	policyConfig           *policy.BucketPolicy
@@ -120,6 +125,46 @@ func newBucketMetadata(name string) BucketMetadata {
 	}
 }
 
+// Return the last update of this bucket metadata, which
+// means, the last update of any policy document.
+func (b BucketMetadata) lastUpdate() (t time.Time) {
+	if b.PolicyConfigUpdatedAt.After(t) {
+		t = b.PolicyConfigUpdatedAt
+	}
+	if b.ObjectLockConfigUpdatedAt.After(t) {
+		t = b.ObjectLockConfigUpdatedAt
+	}
+	if b.EncryptionConfigUpdatedAt.After(t) {
+		t = b.EncryptionConfigUpdatedAt
+	}
+	if b.TaggingConfigUpdatedAt.After(t) {
+		t = b.TaggingConfigUpdatedAt
+	}
+	if b.QuotaConfigUpdatedAt.After(t) {
+		t = b.QuotaConfigUpdatedAt
+	}
+	if b.ReplicationConfigUpdatedAt.After(t) {
+		t = b.ReplicationConfigUpdatedAt
+	}
+	if b.VersioningConfigUpdatedAt.After(t) {
+		t = b.VersioningConfigUpdatedAt
+	}
+	if b.LifecycleConfigUpdatedAt.After(t) {
+		t = b.LifecycleConfigUpdatedAt
+	}
+	if b.NotificationConfigUpdatedAt.After(t) {
+		t = b.NotificationConfigUpdatedAt
+	}
+	if b.BucketTargetsConfigUpdatedAt.After(t) {
+		t = b.BucketTargetsConfigUpdatedAt
+	}
+	if b.BucketTargetsConfigMetaUpdatedAt.After(t) {
+		t = b.BucketTargetsConfigMetaUpdatedAt
+	}
+
+	return
+}
+
 // Versioning returns true if versioning is enabled
 func (b BucketMetadata) Versioning() bool {
 	return b.LockEnabled || (b.versioningConfig != nil && b.versioningConfig.Enabled()) || (b.objectLockConfig != nil && b.objectLockConfig.Enabled())
@@ -440,6 +485,18 @@ func (b *BucketMetadata) defaultTimestamps() {
 	if b.LifecycleConfigUpdatedAt.IsZero() {
 		b.LifecycleConfigUpdatedAt = b.Created
 	}
+
+	if b.NotificationConfigUpdatedAt.IsZero() {
+		b.NotificationConfigUpdatedAt = b.Created
+	}
+
+	if b.BucketTargetsConfigUpdatedAt.IsZero() {
+		b.BucketTargetsConfigUpdatedAt = b.Created
+	}
+
+	if b.BucketTargetsConfigMetaUpdatedAt.IsZero() {
+		b.BucketTargetsConfigMetaUpdatedAt = b.Created
+	}
 }
 
 // Save config to supplied ObjectLayer api.

cmd/bucket-metadata_gen.go

@@ -156,6 +156,24 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
 				return
 			}
+		case "NotificationConfigUpdatedAt":
+			z.NotificationConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationConfigUpdatedAt")
+				return
+			}
+		case "BucketTargetsConfigUpdatedAt":
+			z.BucketTargetsConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigUpdatedAt")
+				return
+			}
+		case "BucketTargetsConfigMetaUpdatedAt":
+			z.BucketTargetsConfigMetaUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -169,9 +187,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 22
+	// map header, size 25
 	// write "Name"
-	err = en.Append(0xde, 0x0, 0x16, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -390,15 +408,45 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
 		return
 	}
+	// write "NotificationConfigUpdatedAt"
+	err = en.Append(0xbb, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.NotificationConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "NotificationConfigUpdatedAt")
+		return
+	}
+	// write "BucketTargetsConfigUpdatedAt"
+	err = en.Append(0xbc, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.BucketTargetsConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "BucketTargetsConfigUpdatedAt")
+		return
+	}
+	// write "BucketTargetsConfigMetaUpdatedAt"
+	err = en.Append(0xd9, 0x20, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.BucketTargetsConfigMetaUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 22
+	// map header, size 25
 	// string "Name"
-	o = append(o, 0xde, 0x0, 0x16, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -463,6 +511,15 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "LifecycleConfigUpdatedAt"
 	o = append(o, 0xb8, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
 	o = msgp.AppendTime(o, z.LifecycleConfigUpdatedAt)
+	// string "NotificationConfigUpdatedAt"
+	o = append(o, 0xbb, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.NotificationConfigUpdatedAt)
+	// string "BucketTargetsConfigUpdatedAt"
+	o = append(o, 0xbc, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.BucketTargetsConfigUpdatedAt)
+	// string "BucketTargetsConfigMetaUpdatedAt"
+	o = append(o, 0xd9, 0x20, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.BucketTargetsConfigMetaUpdatedAt)
 	return
 }
 
@@ -616,6 +673,24 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
 				return
 			}
+		case "NotificationConfigUpdatedAt":
+			z.NotificationConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "NotificationConfigUpdatedAt")
+				return
+			}
+		case "BucketTargetsConfigUpdatedAt":
+			z.BucketTargetsConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigUpdatedAt")
+				return
+			}
+		case "BucketTargetsConfigMetaUpdatedAt":
+			z.BucketTargetsConfigMetaUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -630,6 +705,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize
+	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize + 28 + msgp.TimeSize + 29 + msgp.TimeSize + 34 + msgp.TimeSize
 	return
 }

cmd/bucket-quota.go

@@ -50,6 +50,9 @@ func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 	bucketStorageCache.InitOnce(10*time.Second,
 		cachevalue.Opts{ReturnLastGood: true, NoWait: true},
 		func(ctx context.Context) (DataUsageInfo, error) {
+			if objAPI == nil {
+				return DataUsageInfo{}, errServerNotInitialized
+			}
 			ctx, done := context.WithTimeout(ctx, 2*time.Second)
 			defer done()
 
@@ -59,7 +62,9 @@ func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 }
 
 // GetBucketUsageInfo return bucket usage info for a given bucket
-func (sys *BucketQuotaSys) GetBucketUsageInfo(ctx context.Context, bucket string) (BucketUsageInfo, error) {
+func (sys *BucketQuotaSys) GetBucketUsageInfo(ctx context.Context, bucket string) BucketUsageInfo {
+	sys.Init(newObjectLayerFn())
+
 	dui, err := bucketStorageCache.GetWithCtx(ctx)
 	timedout := OperationTimedOut{}
 	if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &timedout) {
@@ -73,10 +78,10 @@ func (sys *BucketQuotaSys) GetBucketUsageInfo(ctx context.Context, bucket string
 	if len(dui.BucketsUsage) > 0 {
 		bui, ok := dui.BucketsUsage[bucket]
 		if ok {
-			return bui, nil
+			return bui
 		}
 	}
-	return BucketUsageInfo{}, nil
+	return BucketUsageInfo{}
 }
 
 // parseBucketQuota parses BucketQuota from json
@@ -118,11 +123,7 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
 			return BucketQuotaExceeded{Bucket: bucket}
 		}
 
-		bui, err := sys.GetBucketUsageInfo(ctx, bucket)
-		if err != nil {
-			return err
-		}
-
+		bui := sys.GetBucketUsageInfo(ctx, bucket)
 		if bui.Size > 0 && ((bui.Size + uint64(size)) >= quotaSize) {
 			return BucketQuotaExceeded{Bucket: bucket}
 		}

cmd/bucket-replication-handlers.go

@@ -75,7 +75,7 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
-	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, &validateReplicationDestinationOptions{CheckRemoteBucket: true})
 	if apiErr != noError {
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
@@ -230,7 +230,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	w.Header().Set(xhttp.ContentType, string(mimeJSON))
 
 	enc := json.NewEncoder(w)
-	stats := globalReplicationStats.getLatestReplicationStats(bucket)
+	stats := globalReplicationStats.Load().getLatestReplicationStats(bucket)
 	bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
 	bwMap := bwRpt.BucketStats
 	for arn, st := range stats.ReplicationStats.Stats {
@@ -286,7 +286,7 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsV2Handler(w http.Respons
 	w.Header().Set(xhttp.ContentType, string(mimeJSON))
 
 	enc := json.NewEncoder(w)
-	stats := globalReplicationStats.getLatestReplicationStats(bucket)
+	stats := globalReplicationStats.Load().getLatestReplicationStats(bucket)
 	bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
 	bwMap := bwRpt.BucketStats
 	for arn, st := range stats.ReplicationStats.Stats {
@@ -422,7 +422,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 		return
 	}
 
-	if err := globalReplicationPool.resyncer.start(ctx, objectAPI, resyncOpts{
+	if err := globalReplicationPool.Get().resyncer.start(ctx, objectAPI, resyncOpts{
 		bucket:       bucket,
 		arn:          arn,
 		resyncID:     resetID,
@@ -559,7 +559,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
 		lockEnabled = lcfg.Enabled()
 	}
 
-	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, &validateReplicationDestinationOptions{CheckRemoteBucket: true})
 	if apiErr != noError {
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return

cmd/bucket-replication-metrics.go

@@ -119,7 +119,7 @@ func (a *ActiveWorkerStat) update() {
 	if a == nil {
 		return
 	}
-	a.Curr = globalReplicationPool.ActiveWorkers()
+	a.Curr = globalReplicationPool.Get().ActiveWorkers()
 	a.hist.Update(int64(a.Curr))
 	a.Avg = float32(a.hist.Mean())
 	a.Max = int(a.hist.Max())

cmd/bucket-replication-stats.go

@@ -87,6 +87,9 @@ func (r *ReplicationStats) updateMovingAvg() {
 
 // ActiveWorkers returns worker stats
 func (r *ReplicationStats) ActiveWorkers() ActiveWorkerStat {
+	if r == nil {
+		return ActiveWorkerStat{}
+	}
 	r.wlock.RLock()
 	defer r.wlock.RUnlock()
 	w := r.workers.get()
@@ -351,6 +354,9 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
 }
 
 func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketStats) {
+	if r == nil {
+		return nil
+	}
 	peerBucketStatsList := globalNotificationSys.GetClusterAllBucketStats(GlobalContext)
 	bucketsReplicationStats = make(map[string]BucketStats, len(bucketsUsage))
 
@@ -460,6 +466,9 @@ func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, bucket
 
 // get the most current of in-memory replication stats  and data usage info from crawler.
 func (r *ReplicationStats) getLatestReplicationStats(bucket string) (s BucketStats) {
+	if r == nil {
+		return s
+	}
 	bucketStats := globalNotificationSys.GetClusterBucketStats(GlobalContext, bucket)
 	return r.calculateBucketReplicationStats(bucket, bucketStats)
 }
@@ -495,9 +504,14 @@ func (r *ReplicationStats) decQ(bucket string, sz int64, isDelMarker bool, opTyp
 
 // incProxy increments proxy metrics for proxied calls
 func (r *ReplicationStats) incProxy(bucket string, api replProxyAPI, isErr bool) {
-	r.pCache.inc(bucket, api, isErr)
+	if r != nil {
+		r.pCache.inc(bucket, api, isErr)
+	}
 }
 
 func (r *ReplicationStats) getProxyStats(bucket string) ProxyMetric {
+	if r == nil {
+		return ProxyMetric{}
+	}
 	return r.pCache.getBucketStats(bucket)
 }

cmd/bucket-replication.go

@@ -50,6 +50,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/once"
 	"github.com/tinylib/msgp/msgp"
 	"github.com/zeebo/xxh3"
 	"golang.org/x/exp/maps"
@@ -91,7 +92,10 @@ func getReplicationConfig(ctx context.Context, bucketName string) (rc *replicati
 
 // validateReplicationDestination returns error if replication destination bucket missing or not configured
 // It also returns true if replication destination is same as this server.
-func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config, checkRemote bool) (bool, APIError) {
+func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config, opts *validateReplicationDestinationOptions) (bool, APIError) {
+	if opts == nil {
+		opts = &validateReplicationDestinationOptions{}
+	}
 	var arns []string
 	if rCfg.RoleArn != "" {
 		arns = append(arns, rCfg.RoleArn)
@@ -113,7 +117,7 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 		if clnt == nil {
 			return sameTarget, toAPIError(ctx, BucketRemoteTargetNotFound{Bucket: bucket})
 		}
-		if checkRemote { // validate remote bucket
+		if opts.CheckRemoteBucket { // validate remote bucket
 			found, err := clnt.BucketExists(ctx, arn.Bucket)
 			if err != nil {
 				return sameTarget, errorCodes.ToAPIErrWithErr(ErrRemoteDestinationNotFoundError, err)
@@ -133,23 +137,29 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 				}
 			}
 		}
+		// if checked bucket, then check the ready is unnecessary
+		if !opts.CheckRemoteBucket && opts.CheckReady {
+			endpoint := clnt.EndpointURL().String()
+			if errInt, ok := opts.checkReadyErr.Load(endpoint); !ok {
+				err = checkRemoteEndpoint(ctx, clnt.EndpointURL())
+				opts.checkReadyErr.Store(endpoint, err)
+			} else {
+				if errInt == nil {
+					err = nil
+				} else {
+					err = errInt.(error)
+				}
+			}
+			switch err.(type) {
+			case BucketRemoteIdenticalToSource:
+				return true, errorCodes.ToAPIErrWithErr(ErrBucketRemoteIdenticalToSource, fmt.Errorf("remote target endpoint %s is self referential", clnt.EndpointURL().String()))
+			default:
+			}
+		}
 		// validate replication ARN against target endpoint
-		c := globalBucketTargetSys.GetRemoteTargetClient(bucket, arnStr)
-		if c != nil {
-			if err := checkRemoteEndpoint(ctx, c.EndpointURL()); err != nil {
-				switch err.(type) {
-				case BucketRemoteIdenticalToSource:
-					return true, errorCodes.ToAPIErrWithErr(ErrBucketRemoteIdenticalToSource, fmt.Errorf("remote target endpoint %s is self referential", c.EndpointURL().String()))
-				default:
-				}
-			}
-			if c.EndpointURL().String() == clnt.EndpointURL().String() {
-				selfTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
-				if !sameTarget {
-					sameTarget = selfTarget
-				}
-				continue
-			}
+		selfTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
+		if !sameTarget {
+			sameTarget = selfTarget
 		}
 	}
 
@@ -469,7 +479,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	lk := objectAPI.NewNSLock(bucket, "/[replicate]/"+dobj.ObjectName)
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
-		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
+		globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -539,7 +549,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	// to decrement pending count later.
 	for _, rinfo := range rinfos.Targets {
 		if rinfo.ReplicationStatus != rinfo.PrevReplicationStatus {
-			globalReplicationStats.Update(dobj.Bucket, rinfo, replicationStatus,
+			globalReplicationStats.Load().Update(dobj.Bucket, rinfo, replicationStatus,
 				prevStatus)
 		}
 	}
@@ -547,7 +557,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	eventName := event.ObjectReplicationComplete
 	if replicationStatus == replication.Failed {
 		eventName = event.ObjectReplicationFailed
-		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
+		globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry())
 	}
 	drs := getReplicationState(rinfos, dobj.ReplicationState, dobj.VersionID)
 	if replicationStatus != prevStatus {
@@ -765,21 +775,6 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
 	return "", false
 }
 
-func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) map[string]string {
-	meta := make(map[string]string)
-	cs := oi.decryptChecksums(partNum, h)
-	for k, v := range cs {
-		cksum := hash.NewChecksumString(k, v)
-		if cksum == nil {
-			continue
-		}
-		if cksum.Valid() {
-			meta[cksum.Type.Key()] = v
-		}
-	}
-	return meta
-}
-
 func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, partNum int) (putOpts minio.PutObjectOptions, err error) {
 	meta := make(map[string]string)
 	isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined)
@@ -788,11 +783,6 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
 		// In case of SSE-C objects copy the allowed internal headers as well
 		if !isSSEC || !slices.Contains(maps.Keys(validSSEReplicationHeaders), k) {
 			if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
-				if strings.EqualFold(k, ReservedMetadataPrefixLower+"crc") {
-					for k, v := range getCRCMeta(objInfo, partNum, nil) {
-						meta[k] = v
-					}
-				}
 				continue
 			}
 			if isStandardHeader(k) {
@@ -811,8 +801,12 @@ func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo, part
 		if isSSEC {
 			meta[ReplicationSsecChecksumHeader] = base64.StdEncoding.EncodeToString(objInfo.Checksum)
 		} else {
-			for k, v := range getCRCMeta(objInfo, 0, nil) {
-				meta[k] = v
+			for _, pi := range objInfo.Parts {
+				if pi.Number == partNum {
+					for k, v := range pi.Checksums {
+						meta[k] = v
+					}
+				}
 			}
 		}
 	}
@@ -1061,7 +1055,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
-		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+		globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
 		return
 	}
 	ctx = lkctx.Context()
@@ -1146,7 +1140,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		for _, rinfo := range rinfos.Targets {
 			if rinfo.ReplicationStatus != rinfo.PrevReplicationStatus {
 				rinfo.OpType = opType // update optype to reflect correct operation.
-				globalReplicationStats.Update(bucket, rinfo, rinfo.ReplicationStatus, rinfo.PrevReplicationStatus)
+				globalReplicationStats.Load().Update(bucket, rinfo, rinfo.ReplicationStatus, rinfo.PrevReplicationStatus)
 			}
 		}
 	}
@@ -1166,7 +1160,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		ri.EventType = ReplicateMRF
 		ri.ReplicationStatusInternal = rinfos.ReplicationStatusInternal()
 		ri.RetryCount++
-		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+		globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
 	}
 }
 
@@ -1306,7 +1300,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 		HeaderSize: headerSize,
 	}
 	newCtx := ctx
-	if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) {
+	if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) && objInfo.Size < minLargeObjSize {
 		var cancel context.CancelFunc
 		newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
 		defer cancel()
@@ -1581,7 +1575,7 @@ applyAction:
 			HeaderSize: headerSize,
 		}
 		newCtx := ctx
-		if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) {
+		if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) && objInfo.Size < minLargeObjSize {
 			var cancel context.CancelFunc
 			newCtx, cancel = context.WithTimeout(ctx, throttleDeadline)
 			defer cancel()
@@ -1648,7 +1642,6 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 
 	var (
 		hr     *hash.Reader
-		pInfo  minio.ObjectPart
 		isSSEC = crypto.SSEC.IsEncrypted(objInfo.UserDefined)
 	)
 
@@ -1666,8 +1659,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		cHeader := http.Header{}
 		cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true")
 		if !isSSEC {
-			crc := getCRCMeta(objInfo, partInfo.Number, nil) // No SSE-C keys here.
-			for k, v := range crc {
+			for k, v := range partInfo.Checksums {
 				cHeader.Add(k, v)
 			}
 		}
@@ -1676,18 +1668,19 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 			CustomHeader: cHeader,
 		}
 
+		var size int64
 		if isSSEC {
-			objectSize += partInfo.Size
-			pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.Size, popts)
+			size = partInfo.Size
 		} else {
-			objectSize += partInfo.ActualSize
-			pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
+			size = partInfo.ActualSize
 		}
+		objectSize += size
+		pInfo, err := c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, size, popts)
 		if err != nil {
 			return err
 		}
-		if !isSSEC && pInfo.Size != partInfo.ActualSize {
-			return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, partInfo.ActualSize)
+		if pInfo.Size != size {
+			return fmt.Errorf("ssec(%t): Part size mismatch: got %d, want %d", isSSEC, pInfo.Size, size)
 		}
 		uploadedParts = append(uploadedParts, minio.CompletePart{
 			PartNumber:     pInfo.PartNumber,
@@ -1699,7 +1692,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 		})
 	}
 	userMeta := map[string]string{
-		validSSEReplicationHeaders[ReservedMetadataPrefix+"Actual-Object-Size"]: objInfo.UserDefined[ReservedMetadataPrefix+"actual-size"],
+		xhttp.MinIOReplicationActualObjectSize: objInfo.UserDefined[ReservedMetadataPrefix+"actual-size"],
 	}
 	if isSSEC && objInfo.UserDefined[ReplicationSsecChecksumHeader] != "" {
 		userMeta[ReplicationSsecChecksumHeader] = objInfo.UserDefined[ReplicationSsecChecksumHeader]
@@ -1795,8 +1788,8 @@ const (
 )
 
 var (
-	globalReplicationPool  *ReplicationPool
-	globalReplicationStats *ReplicationStats
+	globalReplicationPool  = once.NewSingleton[ReplicationPool]()
+	globalReplicationStats atomic.Pointer[ReplicationStats]
 )
 
 // ReplicationPool describes replication pool
@@ -1811,6 +1804,7 @@ type ReplicationPool struct {
 	priority    string
 	maxWorkers  int
 	maxLWorkers int
+	stats       *ReplicationStats
 
 	mu       sync.RWMutex
 	mrfMU    sync.Mutex
@@ -1857,7 +1851,7 @@ const (
 )
 
 // NewReplicationPool creates a pool of replication workers of specified size
-func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPoolOpts) *ReplicationPool {
+func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPoolOpts, stats *ReplicationStats) *ReplicationPool {
 	var workers, failedWorkers int
 	priority := "auto"
 	maxWorkers := WorkerMaxLimit
@@ -1899,6 +1893,7 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 		mrfStopCh:       make(chan struct{}, 1),
 		ctx:             ctx,
 		objLayer:        o,
+		stats:           stats,
 		priority:        priority,
 		maxWorkers:      maxWorkers,
 		maxLWorkers:     maxLWorkers,
@@ -1926,11 +1921,11 @@ func (p *ReplicationPool) AddMRFWorker() {
 			}
 			switch v := oi.(type) {
 			case ReplicateObjectInfo:
-				globalReplicationStats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				atomic.AddInt32(&p.activeMRFWorkers, 1)
 				replicateObject(p.ctx, v, p.objLayer)
 				atomic.AddInt32(&p.activeMRFWorkers, -1)
-				globalReplicationStats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 
 			default:
 				bugLogIf(p.ctx, fmt.Errorf("unknown mrf replication type: %T", oi), "unknown-mrf-replicate-type")
@@ -1958,9 +1953,9 @@ func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opT
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, 1)
 				}
-				globalReplicationStats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				replicateObject(p.ctx, v, p.objLayer)
-				globalReplicationStats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, -1)
 				}
@@ -1968,10 +1963,10 @@ func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opT
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, 1)
 				}
-				globalReplicationStats.incQ(v.Bucket, 0, true, v.OpType)
+				p.stats.incQ(v.Bucket, 0, true, v.OpType)
 
 				replicateDelete(p.ctx, v, p.objLayer)
-				globalReplicationStats.decQ(v.Bucket, 0, true, v.OpType)
+				p.stats.decQ(v.Bucket, 0, true, v.OpType)
 
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, -1)
@@ -1998,9 +1993,9 @@ func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, 1)
 				}
-				globalReplicationStats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				replicateObject(p.ctx, v, p.objLayer)
-				globalReplicationStats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
+				p.stats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType)
 				if opTracker != nil {
 					atomic.AddInt32(opTracker, -1)
 				}
@@ -2162,9 +2157,9 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		h := xxh3.HashString(ri.Bucket + ri.Name)
 		select {
 		case <-p.ctx.Done():
-		case p.lrgworkers[h%LargeWorkerCount] <- ri:
+		case p.lrgworkers[h%uint64(len(p.lrgworkers))] <- ri:
 		default:
-			globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+			p.queueMRFSave(ri.ToMRFEntry())
 			p.mu.RLock()
 			maxLWorkers := p.maxLWorkers
 			existing := len(p.lrgworkers)
@@ -2195,7 +2190,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 	case healCh <- ri:
 	case ch <- ri:
 	default:
-		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+		globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry())
 		p.mu.RLock()
 		prio := p.priority
 		maxWorkers := p.maxWorkers
@@ -2231,7 +2226,7 @@ func queueReplicateDeletesWrapper(doi DeletedObjectReplicationInfo, existingObje
 			doi.ResetID = v.ResetID
 			doi.TargetArn = k
 
-			globalReplicationPool.queueReplicaDeleteTask(doi)
+			globalReplicationPool.Get().queueReplicaDeleteTask(doi)
 		}
 	}
 }
@@ -2252,7 +2247,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 	case <-p.ctx.Done():
 	case ch <- doi:
 	default:
-		globalReplicationPool.queueMRFSave(doi.ToMRFEntry())
+		p.queueMRFSave(doi.ToMRFEntry())
 		p.mu.RLock()
 		prio := p.priority
 		maxWorkers := p.maxWorkers
@@ -2282,9 +2277,10 @@ type replicationPoolOpts struct {
 }
 
 func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) {
-	globalReplicationPool = NewReplicationPool(ctx, objectAPI, globalAPIConfig.getReplicationOpts())
-	globalReplicationStats = NewReplicationStats(ctx, objectAPI)
-	go globalReplicationStats.trackEWMA()
+	stats := NewReplicationStats(ctx, objectAPI)
+	globalReplicationPool.Set(NewReplicationPool(ctx, objectAPI, globalAPIConfig.getReplicationOpts(), stats))
+	globalReplicationStats.Store(stats)
+	go stats.trackEWMA()
 }
 
 type proxyResult struct {
@@ -2490,7 +2486,7 @@ func scheduleReplication(ctx context.Context, oi ObjectInfo, o ObjectLayer, dsc
 	if dsc.Synchronous() {
 		replicateObject(ctx, ri, o)
 	} else {
-		globalReplicationPool.queueReplicaTask(ri)
+		globalReplicationPool.Get().queueReplicaTask(ri)
 	}
 }
 
@@ -2618,9 +2614,9 @@ func proxyGetTaggingToRepTarget(ctx context.Context, bucket, object string, opts
 }
 
 func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectReplicationInfo, o ObjectLayer) {
-	globalReplicationPool.queueReplicaDeleteTask(dv)
+	globalReplicationPool.Get().queueReplicaDeleteTask(dv)
 	for arn := range dv.ReplicationState.Targets {
-		globalReplicationStats.Update(dv.Bucket, replicatedTargetInfo{Arn: arn, Size: 0, Duration: 0, OpType: replication.DeleteReplicationType}, replication.Pending, replication.StatusType(""))
+		globalReplicationStats.Load().Update(dv.Bucket, replicatedTargetInfo{Arn: arn, Size: 0, Duration: 0, OpType: replication.DeleteReplicationType}, replication.Pending, replication.StatusType(""))
 	}
 }
 
@@ -3050,9 +3046,9 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
 	if len(tgtArns) == 0 {
 		return fmt.Errorf("arn %s specified for resync not found in replication config", opts.arn)
 	}
-	globalReplicationPool.resyncer.RLock()
-	data, ok := globalReplicationPool.resyncer.statusMap[opts.bucket]
-	globalReplicationPool.resyncer.RUnlock()
+	globalReplicationPool.Get().resyncer.RLock()
+	data, ok := globalReplicationPool.Get().resyncer.statusMap[opts.bucket]
+	globalReplicationPool.Get().resyncer.RUnlock()
 	if !ok {
 		data, err = loadBucketResyncMetadata(ctx, opts.bucket, objAPI)
 		if err != nil {
@@ -3078,9 +3074,9 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
 		return err
 	}
 
-	globalReplicationPool.resyncer.Lock()
-	defer globalReplicationPool.resyncer.Unlock()
-	brs, ok := globalReplicationPool.resyncer.statusMap[opts.bucket]
+	globalReplicationPool.Get().resyncer.Lock()
+	defer globalReplicationPool.Get().resyncer.Unlock()
+	brs, ok := globalReplicationPool.Get().resyncer.statusMap[opts.bucket]
 	if !ok {
 		brs = BucketReplicationResyncStatus{
 			Version:    resyncMetaVersion,
@@ -3088,8 +3084,8 @@ func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opt
 		}
 	}
 	brs.TargetsMap[opts.arn] = status
-	globalReplicationPool.resyncer.statusMap[opts.bucket] = brs
-	go globalReplicationPool.resyncer.resyncBucket(GlobalContext, objAPI, false, opts)
+	globalReplicationPool.Get().resyncer.statusMap[opts.bucket] = brs
+	go globalReplicationPool.Get().resyncer.resyncBucket(GlobalContext, objAPI, false, opts)
 	return nil
 }
 
@@ -3421,7 +3417,7 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 		if roi.ReplicationStatus == replication.Pending ||
 			roi.ReplicationStatus == replication.Failed ||
 			roi.VersionPurgeStatus == Failed || roi.VersionPurgeStatus == Pending {
-			globalReplicationPool.queueReplicaDeleteTask(dv)
+			globalReplicationPool.Get().queueReplicaDeleteTask(dv)
 			return
 		}
 		// if replication status is Complete on DeleteMarker and existing object resync required
@@ -3437,12 +3433,12 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 	switch roi.ReplicationStatus {
 	case replication.Pending, replication.Failed:
 		roi.EventType = ReplicateHeal
-		globalReplicationPool.queueReplicaTask(roi)
+		globalReplicationPool.Get().queueReplicaTask(roi)
 		return
 	}
 	if roi.ExistingObjResync.mustResync() {
 		roi.EventType = ReplicateExisting
-		globalReplicationPool.queueReplicaTask(roi)
+		globalReplicationPool.Get().queueReplicaTask(roi)
 	}
 	return
 }
@@ -3507,8 +3503,8 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
 		return
 	}
 	if entry.RetryCount > mrfRetryLimit { // let scanner catch up if retry count exceeded
-		atomic.AddUint64(&globalReplicationStats.mrfStats.TotalDroppedCount, 1)
-		atomic.AddUint64(&globalReplicationStats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
+		atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1)
+		atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
 		return
 	}
 
@@ -3521,8 +3517,8 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
 		select {
 		case p.mrfSaveCh <- entry:
 		default:
-			atomic.AddUint64(&globalReplicationStats.mrfStats.TotalDroppedCount, 1)
-			atomic.AddUint64(&globalReplicationStats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
+			atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1)
+			atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz))
 		}
 	}
 }
@@ -3553,7 +3549,7 @@ func (p *ReplicationPool) persistToDrive(ctx context.Context, v MRFReplicateEntr
 	}
 
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	for _, localDrive := range localDrives {
@@ -3571,7 +3567,7 @@ func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string
 	if !p.initialized() {
 		return
 	}
-	atomic.StoreUint64(&globalReplicationStats.mrfStats.LastFailedCount, uint64(len(entries)))
+	atomic.StoreUint64(&p.stats.mrfStats.LastFailedCount, uint64(len(entries)))
 	if len(entries) == 0 {
 		return
 	}
@@ -3620,7 +3616,7 @@ func (p *ReplicationPool) loadMRF() (mrfRec MRFReplicateEntries, err error) {
 	}
 
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	for _, localDrive := range localDrives {
@@ -3741,3 +3737,12 @@ func (p *ReplicationPool) getMRF(ctx context.Context, bucket string) (ch <-chan
 
 	return mrfCh, nil
 }
+
+// validateReplicationDestinationOptions is used to configure the validation of the replication destination.
+// validateReplicationDestination uses this to configure the validation.
+type validateReplicationDestinationOptions struct {
+	CheckRemoteBucket bool
+	CheckReady        bool
+
+	checkReadyErr sync.Map
+}

cmd/bucket-stats.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"fmt"
 	"math"
+	"sync/atomic"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
@@ -127,8 +128,7 @@ func (l *ReplicationLastHour) getTotal() AccElem {
 
 // forwardTo time t, clearing any entries in between.
 func (l *ReplicationLastHour) forwardTo(t int64) {
-	tMin := t / 60
-	if l.LastMin >= tMin {
+	if l.LastMin >= t {
 		return
 	}
 	if t-l.LastMin >= 60 {
@@ -310,10 +310,13 @@ type ReplQNodeStats struct {
 func (r *ReplicationStats) getNodeQueueStats(bucket string) (qs ReplQNodeStats) {
 	qs.NodeName = globalLocalNodeName
 	qs.Uptime = UTCNow().Unix() - globalBootTime.Unix()
-	qs.ActiveWorkers = globalReplicationStats.ActiveWorkers()
+	qs.ActiveWorkers = globalReplicationStats.Load().ActiveWorkers()
 	qs.XferStats = make(map[RMetricName]XferStats)
 	qs.QStats = r.qCache.getBucketStats(bucket)
 	qs.TgtXferStats = make(map[string]map[RMetricName]XferStats)
+	qs.MRFStats = ReplicationMRFStats{
+		LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount),
+	}
 
 	r.RLock()
 	defer r.RUnlock()
@@ -399,10 +402,12 @@ func (r *ReplicationStats) getNodeQueueStats(bucket string) (qs ReplQNodeStats)
 func (r *ReplicationStats) getNodeQueueStatsSummary() (qs ReplQNodeStats) {
 	qs.NodeName = globalLocalNodeName
 	qs.Uptime = UTCNow().Unix() - globalBootTime.Unix()
-	qs.ActiveWorkers = globalReplicationStats.ActiveWorkers()
+	qs.ActiveWorkers = globalReplicationStats.Load().ActiveWorkers()
 	qs.XferStats = make(map[RMetricName]XferStats)
 	qs.QStats = r.qCache.getSiteStats()
-
+	qs.MRFStats = ReplicationMRFStats{
+		LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount),
+	}
 	r.RLock()
 	defer r.RUnlock()
 	tx := newXferStats()

cmd/common-main.go

@@ -685,16 +685,6 @@ func loadEnvVarsFromFiles() {
 		}
 	}
 
-	if env.IsSet(kms.EnvKMSSecretKeyFile) {
-		kmsSecret, err := readFromSecret(env.Get(kms.EnvKMSSecretKeyFile, ""))
-		if err != nil {
-			logger.Fatal(err, "Unable to read the KMS secret key inherited from secret file")
-		}
-		if kmsSecret != "" {
-			os.Setenv(kms.EnvKMSSecretKey, kmsSecret)
-		}
-	}
-
 	if env.IsSet(config.EnvConfigEnvFile) {
 		ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
 		if err != nil && !os.IsNotExist(err) {
@@ -768,7 +758,7 @@ func serverHandleEnvVars() {
 	if len(domains) != 0 {
 		for _, domainName := range strings.Split(domains, config.ValueSeparator) {
 			if _, ok := dns2.IsDomainName(domainName); !ok {
-				logger.Fatal(config.ErrInvalidDomainValue(nil).Msg("Unknown value `%s`", domainName),
+				logger.Fatal(config.ErrInvalidDomainValue(nil).Msgf("Unknown value `%s`", domainName),
 					"Invalid MINIO_DOMAIN value in environment variable")
 			}
 			globalDomainNames = append(globalDomainNames, domainName)
@@ -777,7 +767,7 @@ func serverHandleEnvVars() {
 		lcpSuf := lcpSuffix(globalDomainNames)
 		for _, domainName := range globalDomainNames {
 			if domainName == lcpSuf && len(globalDomainNames) > 1 {
-				logger.Fatal(config.ErrOverlappingDomainValue(nil).Msg("Overlapping domains `%s` not allowed", globalDomainNames),
+				logger.Fatal(config.ErrOverlappingDomainValue(nil).Msgf("Overlapping domains `%s` not allowed", globalDomainNames),
 					"Invalid MINIO_DOMAIN value in environment variable")
 			}
 		}
@@ -881,6 +871,12 @@ func loadRootCredentials() {
 	} else {
 		globalActiveCred = auth.DefaultCredentials
 	}
+
+	var err error
+	globalNodeAuthToken, err = authenticateNode(globalActiveCred.AccessKey, globalActiveCred.SecretKey)
+	if err != nil {
+		logger.Fatal(err, "Unable to generate internode credentials")
+	}
 }
 
 // Initialize KMS global variable after valiadating and loading the configuration.

cmd/data-scanner.go

@@ -569,7 +569,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				APIName: "Scanner",
 				Bucket:  f.root,
 				Object:  prefixName,
-				Tags: map[string]interface{}{
+				Tags: map[string]string{
 					"x-minio-prefixes-total": strconv.Itoa(totalFolders),
 				},
 			})
@@ -1138,7 +1138,7 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			APIName: "Scanner",
 			Bucket:  i.bucket,
 			Object:  i.objectPath(),
-			Tags: map[string]interface{}{
+			Tags: map[string]string{
 				"x-minio-versions": strconv.Itoa(len(objInfos)),
 			},
 		})
@@ -1170,7 +1170,7 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			APIName: "Scanner",
 			Bucket:  i.bucket,
 			Object:  i.objectPath(),
-			Tags: map[string]interface{}{
+			Tags: map[string]string{
 				"x-minio-versions-count": strconv.Itoa(len(objInfos)),
 				"x-minio-versions-size":  strconv.FormatInt(cumulativeSize, 10),
 			},
@@ -1336,6 +1336,8 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	}
 
 	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
+	tags["version-id"] = dobj.VersionID
+
 	// Send audit for the lifecycle delete operation
 	auditLogLifecycle(ctx, dobj, ILMExpiry, tags, traceFn)
 
@@ -1547,7 +1549,7 @@ const (
 	ILMTransition = " ilm:transition"
 )
 
-func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]interface{}, traceFn func(event string)) {
+func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string)) {
 	var apiName string
 	switch event {
 	case ILMExpiry:
@@ -1565,5 +1567,5 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags ma
 		VersionID: oi.VersionID,
 		Tags:      tags,
 	})
-	traceFn(event)
+	traceFn(event, tags)
 }

cmd/dummy-handlers.go

@@ -44,7 +44,7 @@ func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	// Allow getBucketCors if policy action is set, since this is a dummy call
+	// Allow GetBucketWebsite if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -76,7 +76,7 @@ func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Allow getBucketCors if policy action is set, since this is a dummy call
+	// Allow GetBucketAccelerate if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -109,7 +109,7 @@ func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWrite
 		return
 	}
 
-	// Allow getBucketCors if policy action is set, since this is a dummy call
+	// Allow GetBucketRequestPaymentHandler if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -143,7 +143,7 @@ func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	// Allow getBucketCors if policy action is set, since this is a dummy call
+	// Allow GetBucketLoggingHandler if policy action is set, since this is a dummy call
 	// we are simply re-purposing the bucketPolicyAction.
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
@@ -181,9 +181,7 @@ func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Allow getBucketCors if policy action is set, since this is a dummy call
-	// we are simply re-purposing the bucketPolicyAction.
-	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone {
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketCorsAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
@@ -197,3 +195,63 @@ func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http
 
 	writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchCORSConfiguration), r.URL)
 }
+
+// PutBucketCorsHandler - PUT bucket cors, a dummy api
+func (api objectAPIHandlers) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "PutBucketCors")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketCorsAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	// Validate if bucket exists, before proceeding further...
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+}
+
+// DeleteBucketCorsHandler - DELETE bucket cors, a dummy api
+func (api objectAPIHandlers) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "DeleteBucketCors")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objAPI := api.ObjectAPI()
+	if objAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketCorsAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	// Validate if bucket exists, before proceeding further...
+	_, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+}

cmd/encryption-v1.go

@@ -601,28 +601,44 @@ func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint3
 	partEncRelOffset := int64(seqNumber) * (SSEDAREPackageBlockSize + SSEDAREPackageMetaSize)
 
 	w := &DecryptBlocksReader{
-		reader:            inputReader,
-		startSeqNum:       seqNumber,
-		partDecRelOffset:  partDecRelOffset,
-		partEncRelOffset:  partEncRelOffset,
-		parts:             oi.Parts,
-		partIndex:         partStart,
-		header:            h,
-		bucket:            bucket,
-		object:            object,
-		customerKeyHeader: h.Get(xhttp.AmzServerSideEncryptionCustomerKey),
-		copySource:        copySource,
-		metadata:          cloneMSS(oi.UserDefined),
+		reader:           inputReader,
+		startSeqNum:      seqNumber,
+		partDecRelOffset: partDecRelOffset,
+		partEncRelOffset: partEncRelOffset,
+		parts:            oi.Parts,
+		partIndex:        partStart,
 	}
 
-	if w.copySource {
-		w.customerKeyHeader = h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey)
+	// In case of SSE-C, we have to decrypt the OEK using the client-provided key.
+	// In case of a SSE-C server-side copy, the client might provide two keys,
+	// one for the source and one for the target. This reader is the source.
+	var ssecClientKey []byte
+	if crypto.SSEC.IsEncrypted(oi.UserDefined) {
+		if copySource && crypto.SSECopy.IsRequested(h) {
+			key, err := crypto.SSECopy.ParseHTTP(h)
+			if err != nil {
+				return nil, err
+			}
+			ssecClientKey = key[:]
+		} else {
+			key, err := crypto.SSEC.ParseHTTP(h)
+			if err != nil {
+				return nil, err
+			}
+			ssecClientKey = key[:]
+		}
 	}
 
+	// Decrypt the OEK once and reuse it for all subsequent parts.
+	objectEncryptionKey, err := decryptObjectMeta(ssecClientKey, bucket, object, oi.UserDefined)
+	if err != nil {
+		return nil, err
+	}
+	w.objectEncryptionKey = objectEncryptionKey
+
 	if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil {
 		return nil, err
 	}
-
 	return w, nil
 }
 
@@ -638,48 +654,17 @@ type DecryptBlocksReader struct {
 	// Current part index
 	partIndex int
 	// Parts information
-	parts          []ObjectPartInfo
-	header         http.Header
-	bucket, object string
-	metadata       map[string]string
+	parts []ObjectPartInfo
 
+	objectEncryptionKey                []byte
 	partDecRelOffset, partEncRelOffset int64
-
-	copySource bool
-	// Customer Key
-	customerKeyHeader string
 }
 
 func (d *DecryptBlocksReader) buildDecrypter(partID int) error {
-	m := cloneMSS(d.metadata)
-	// Initialize the first decrypter; new decrypters will be
-	// initialized in Read() operation as needed.
-	var key []byte
-	var err error
-	if d.copySource {
-		if crypto.SSEC.IsEncrypted(d.metadata) {
-			d.header.Set(xhttp.AmzServerSideEncryptionCopyCustomerKey, d.customerKeyHeader)
-			key, err = ParseSSECopyCustomerRequest(d.header, d.metadata)
-		}
-	} else {
-		if crypto.SSEC.IsEncrypted(d.metadata) {
-			d.header.Set(xhttp.AmzServerSideEncryptionCustomerKey, d.customerKeyHeader)
-			key, err = ParseSSECustomerHeader(d.header)
-		}
-	}
-	if err != nil {
-		return err
-	}
-
-	objectEncryptionKey, err := decryptObjectMeta(key, d.bucket, d.object, m)
-	if err != nil {
-		return err
-	}
-
 	var partIDbin [4]byte
 	binary.LittleEndian.PutUint32(partIDbin[:], uint32(partID)) // marshal part ID
 
-	mac := hmac.New(sha256.New, objectEncryptionKey) // derive part encryption key from part ID and object key
+	mac := hmac.New(sha256.New, d.objectEncryptionKey) // derive part encryption key from part ID and object key
 	mac.Write(partIDbin[:])
 	partEncryptionKey := mac.Sum(nil)
 
@@ -743,8 +728,9 @@ func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
 // but has an invalid size.
 func (o ObjectInfo) DecryptedSize() (int64, error) {
 	if _, ok := crypto.IsEncrypted(o.UserDefined); !ok {
-		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
+		return -1, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
+
 	if !o.isMultipart() {
 		size, err := sio.DecryptedSize(uint64(o.Size))
 		if err != nil {
@@ -757,7 +743,7 @@ func (o ObjectInfo) DecryptedSize() (int64, error) {
 	for _, part := range o.Parts {
 		partSize, err := sio.DecryptedSize(uint64(part.Size))
 		if err != nil {
-			return 0, errObjectTampered
+			return -1, errObjectTampered
 		}
 		size += int64(partSize)
 	}
@@ -1103,7 +1089,7 @@ func (o *ObjectInfo) metadataDecrypter(h http.Header) objectMetaDecryptFn {
 	}
 }
 
-// decryptChecksums will attempt to decode checksums and return it/them if set.
+// decryptPartsChecksums will attempt to decode checksums and return it/them if set.
 // if part > 0, and we have the checksum for the part that will be returned.
 func (o *ObjectInfo) decryptPartsChecksums(h http.Header) {
 	data := o.Checksum
@@ -1172,6 +1158,14 @@ func (o *ObjectInfo) decryptChecksums(part int, h http.Header) map[string]string
 	if len(data) == 0 {
 		return nil
 	}
+	if part > 0 && !crypto.SSEC.IsEncrypted(o.UserDefined) {
+		// already decrypted in ToObjectInfo for multipart objects
+		for _, pi := range o.Parts {
+			if pi.Number == part {
+				return pi.Checksums
+			}
+		}
+	}
 	if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted {
 		decrypted, err := o.metadataDecrypter(h)("object-checksum", data)
 		if err != nil {

cmd/endpoint-ellipses.go

@@ -311,7 +311,7 @@ func GetAllSets(setDriveCount uint64, args ...string) ([][]string, error) {
 	for _, sargs := range setArgs {
 		for _, arg := range sargs {
 			if uniqueArgs.Contains(arg) {
-				return nil, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("Input args (%s) has duplicate ellipses", args))
+				return nil, config.ErrInvalidErasureEndpoints(nil).Msgf("Input args (%s) has duplicate ellipses", args)
 			}
 			uniqueArgs.Add(arg)
 		}

cmd/erasure-common.go

@@ -19,13 +19,9 @@ package cmd
 
 import (
 	"context"
-	"fmt"
-	"io"
 	"math/rand"
 	"sync"
 	"time"
-
-	"github.com/minio/pkg/v3/sync/errgroup"
 )
 
 func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
@@ -87,89 +83,3 @@ func (er erasureObjects) getLocalDisks() (newDisks []StorageAPI) {
 	}
 	return newDisks
 }
-
-// readMultipleFiles Reads raw data from all specified files from all disks.
-func readMultipleFiles(ctx context.Context, disks []StorageAPI, req ReadMultipleReq, readQuorum int) ([]ReadMultipleResp, error) {
-	resps := make([]chan ReadMultipleResp, len(disks))
-	for i := range resps {
-		resps[i] = make(chan ReadMultipleResp, len(req.Files))
-	}
-	g := errgroup.WithNErrs(len(disks))
-	// Read files in parallel across disks.
-	for index := range disks {
-		index := index
-		g.Go(func() (err error) {
-			if disks[index] == nil {
-				return errDiskNotFound
-			}
-			return disks[index].ReadMultiple(ctx, req, resps[index])
-		}, index)
-	}
-
-	dataArray := make([]ReadMultipleResp, 0, len(req.Files))
-	// Merge results. They should come in order from each.
-	for _, wantFile := range req.Files {
-		quorum := 0
-		toAdd := ReadMultipleResp{
-			Bucket: req.Bucket,
-			Prefix: req.Prefix,
-			File:   wantFile,
-		}
-		for i := range resps {
-			if disks[i] == nil {
-				continue
-			}
-			select {
-			case <-ctx.Done():
-			case gotFile, ok := <-resps[i]:
-				if !ok {
-					continue
-				}
-				if gotFile.Error != "" || !gotFile.Exists {
-					continue
-				}
-				if gotFile.File != wantFile || gotFile.Bucket != req.Bucket || gotFile.Prefix != req.Prefix {
-					continue
-				}
-				quorum++
-				if toAdd.Modtime.After(gotFile.Modtime) || len(gotFile.Data) < len(toAdd.Data) {
-					// Pick latest, or largest to avoid possible truncated entries.
-					continue
-				}
-				toAdd = gotFile
-			}
-		}
-		if quorum < readQuorum {
-			toAdd.Exists = false
-			toAdd.Error = errErasureReadQuorum.Error()
-			toAdd.Data = nil
-		}
-		dataArray = append(dataArray, toAdd)
-	}
-
-	ignoredErrs := []error{
-		errFileNotFound,
-		errVolumeNotFound,
-		errFileVersionNotFound,
-		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
-		io.EOF,              // some times we would read without locks, ignore these errors
-		context.DeadlineExceeded,
-		context.Canceled,
-	}
-	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)
-
-	errs := g.Wait()
-	for index, err := range errs {
-		if err == nil {
-			continue
-		}
-		if !IsErr(err, ignoredErrs...) {
-			storageLogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
-				disks[index], req.Bucket, req.Prefix, err),
-				disks[index].String())
-		}
-	}
-
-	// Return all the metadata.
-	return dataArray, nil
-}

cmd/erasure-decode.go

@@ -48,14 +48,14 @@ func newParallelReader(readers []io.ReaderAt, e Erasure, offset, totalLength int
 		r2b[i] = i
 	}
 	bufs := make([][]byte, len(readers))
-	// Fill buffers
-	b := globalBytePoolCap.Load().Get()
 	shardSize := int(e.ShardSize())
-	if cap(b) < len(readers)*shardSize {
-		// We should always have enough capacity, but older objects may be bigger.
-		globalBytePoolCap.Load().Put(b)
-		b = nil
-	} else {
+	var b []byte
+
+	// We should always have enough capacity, but older objects may be bigger
+	// we do not need stashbuffer for them.
+	if globalBytePoolCap.Load().WidthCap() >= len(readers)*shardSize {
+		// Fill buffers
+		b = globalBytePoolCap.Load().Get()
 		// Seed the buffers.
 		for i := range bufs {
 			bufs[i] = b[i*shardSize : (i+1)*shardSize]

cmd/erasure-healing-common_test.go

@@ -23,6 +23,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"testing"
 	"time"
 
@@ -147,6 +148,10 @@ func TestCommonTime(t *testing.T) {
 // TestListOnlineDisks - checks if listOnlineDisks and outDatedDisks
 // are consistent with each other.
 func TestListOnlineDisks(t *testing.T) {
+	if runtime.GOOS == globalWindowsOSName {
+		t.Skip()
+	}
+
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -230,7 +235,7 @@ func TestListOnlineDisks(t *testing.T) {
 	}
 
 	object := "object"
-	data := bytes.Repeat([]byte("a"), smallFileThreshold*16)
+	data := bytes.Repeat([]byte("a"), smallFileThreshold*32)
 	z := obj.(*erasureServerPools)
 
 	erasureDisks, err := z.GetDisks(0, 0)

cmd/erasure-healing.go

@@ -226,21 +226,28 @@ func (er *erasureObjects) auditHealObject(ctx context.Context, bucket, object, v
 	opts := AuditLogOptions{
 		Event:     "HealObject",
 		Bucket:    bucket,
-		Object:    object,
+		Object:    decodeDirObject(object),
 		VersionID: versionID,
 	}
 	if err != nil {
 		opts.Error = err.Error()
 	}
 
-	opts.Tags = map[string]interface{}{
-		"healResult": result,
-		"objectLocation": auditObjectOp{
-			Name:   decodeDirObject(object),
-			Pool:   er.poolIndex + 1,
-			Set:    er.setIndex + 1,
-			Drives: er.getEndpointStrings(),
-		},
+	b, a := result.GetCorruptedCounts()
+	if b == a {
+		opts.Error = fmt.Sprintf("unable to heal %d corrupted blocks on drives", b)
+	}
+	b, a = result.GetMissingCounts()
+	if b == a {
+		opts.Error = fmt.Sprintf("unable to heal %d missing blocks on drives", b)
+	}
+
+	opts.Tags = map[string]string{
+		"healObject": auditObjectOp{
+			Name: opts.Object,
+			Pool: er.poolIndex + 1,
+			Set:  er.setIndex + 1,
+		}.String(),
 	}
 
 	auditLogInternal(ctx, opts)

cmd/erasure-metadata.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/amztime"
+	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/replication"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash/sha256"
@@ -173,6 +174,7 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 		}
 	}
 	objInfo.Checksum = fi.Checksum
+	objInfo.decryptPartsChecksums(nil)
 	objInfo.Inlined = fi.InlineData()
 	// Success.
 	return objInfo
@@ -287,13 +289,14 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 		mtimeValid := meta.ModTime.Equal(modTime)
 		if mtimeValid || etagOnly {
 			fmt.Fprintf(h, "%v", meta.XLV1)
-			if !etagOnly {
-				// Verify dataDir is same only when mtime is valid and etag is not considered.
-				fmt.Fprintf(h, "%v", meta.GetDataDir())
-			}
 			for _, part := range meta.Parts {
 				fmt.Fprintf(h, "part.%d", part.Number)
+				fmt.Fprintf(h, "part.%d", part.Size)
 			}
+			// Previously we checked if we had quorum on DataDir value.
+			// We have removed this check to allow reading objects with different DataDir
+			// values in a few drives (due to a rebalance-stop race bug)
+			// provided their their etags or ModTimes match.
 
 			if !meta.Deleted && meta.Size != 0 {
 				fmt.Fprintf(h, "%v+%v", meta.Erasure.DataBlocks, meta.Erasure.ParityBlocks)
@@ -388,8 +391,7 @@ func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Tim
 	return findFileInfoInQuorum(ctx, metaArr, modTime, etag, quorum)
 }
 
-// writeUniqueFileInfo - writes unique `xl.meta` content for each disk concurrently.
-func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
+func writeAllMetadataWithRevert(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int, revert bool) ([]StorageAPI, error) {
 	g := errgroup.WithNErrs(len(disks))
 
 	// Start writing `xl.meta` to all disks in parallel.
@@ -413,9 +415,37 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, origbucket, bu
 	mErrs := g.Wait()
 
 	err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, quorum)
+	if err != nil && revert {
+		ng := errgroup.WithNErrs(len(disks))
+		for index := range disks {
+			if mErrs[index] != nil {
+				continue
+			}
+			index := index
+			ng.Go(func() error {
+				if disks[index] == nil {
+					return errDiskNotFound
+				}
+				return disks[index].Delete(ctx, bucket, pathJoin(prefix, xlStorageFormatFile), DeleteOptions{
+					Recursive: true,
+				})
+			}, index)
+		}
+		ng.Wait()
+	}
+
 	return evalDisks(disks, mErrs), err
 }
 
+func writeAllMetadata(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
+	return writeAllMetadataWithRevert(ctx, disks, origbucket, bucket, prefix, files, quorum, true)
+}
+
+// writeUniqueFileInfo - writes unique `xl.meta` content for each disk concurrently.
+func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
+	return writeAllMetadataWithRevert(ctx, disks, origbucket, bucket, prefix, files, quorum, false)
+}
+
 func commonParity(parities []int, defaultParityCount int) int {
 	N := len(parities)
 
@@ -456,6 +486,7 @@ func commonParity(parities []int, defaultParityCount int) int {
 }
 
 func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int) {
+	totalShards := len(partsMetadata)
 	parities = make([]int, len(partsMetadata))
 	for index, metadata := range partsMetadata {
 		if errs[index] != nil {
@@ -466,9 +497,13 @@ func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int)
 			parities[index] = -1
 			continue
 		}
+		//nolint:gocritic
 		// Delete marker or zero byte objects take highest parity.
 		if metadata.Deleted || metadata.Size == 0 {
-			parities[index] = len(partsMetadata) / 2
+			parities[index] = totalShards / 2
+		} else if metadata.TransitionStatus == lifecycle.TransitionComplete {
+			// For tiered objects, read quorum is N/2+1 to ensure simple majority on xl.meta. It is not equal to EcM because the data integrity is entrusted with the warm tier.
+			parities[index] = totalShards - (totalShards/2 + 1)
 		} else {
 			parities[index] = metadata.Erasure.ParityBlocks
 		}

cmd/erasure-metadata_test.go

@@ -19,6 +19,8 @@ package cmd
 
 import (
 	"context"
+	"fmt"
+	"slices"
 	"strconv"
 	"testing"
 	"time"
@@ -359,3 +361,124 @@ func TestSkipTierFreeVersion(t *testing.T) {
 		t.Fatal("Expected SkipTierFreeVersion to be set on FileInfo but wasn't")
 	}
 }
+
+func TestListObjectParities(t *testing.T) {
+	mkMetaArr := func(N, parity, agree int) []FileInfo {
+		fi := newFileInfo("obj-1", N-parity, parity)
+		fi.TransitionTier = "WARM-TIER"
+		fi.TransitionedObjName = mustGetUUID()
+		fi.TransitionStatus = "complete"
+		fi.Size = 1 << 20
+
+		metaArr := make([]FileInfo, N)
+		for i := range N {
+			fi.Erasure.Index = i + 1
+			metaArr[i] = fi
+			if i < agree {
+				continue
+			}
+			metaArr[i].TransitionTier, metaArr[i].TransitionedObjName = "", ""
+			metaArr[i].TransitionStatus = ""
+		}
+		return metaArr
+	}
+	mkParities := func(N, agreedParity, disagreedParity, agree int) []int {
+		ps := make([]int, N)
+		for i := range N {
+			if i < agree {
+				ps[i] = agreedParity
+				continue
+			}
+			ps[i] = disagreedParity // disagree
+		}
+		return ps
+	}
+
+	mkTest := func(N, parity, agree int) (res struct {
+		metaArr  []FileInfo
+		errs     []error
+		parities []int
+		parity   int
+	},
+	) {
+		res.metaArr = mkMetaArr(N, parity, agree)
+		res.parities = mkParities(N, N-(N/2+1), parity, agree)
+		res.errs = make([]error, N)
+		if agree >= N/2+1 { // simple majority consensus
+			res.parity = N - (N/2 + 1)
+		} else {
+			res.parity = -1
+		}
+		return res
+	}
+
+	nonTieredTest := func(N, parity, agree int) (res struct {
+		metaArr  []FileInfo
+		errs     []error
+		parities []int
+		parity   int
+	},
+	) {
+		fi := newFileInfo("obj-1", N-parity, parity)
+		fi.Size = 1 << 20
+		metaArr := make([]FileInfo, N)
+		parities := make([]int, N)
+		for i := range N {
+			fi.Erasure.Index = i + 1
+			metaArr[i] = fi
+			parities[i] = parity
+			if i < agree {
+				continue
+			}
+			metaArr[i].Erasure.Index = 0 // creates invalid fi on remaining drives
+			parities[i] = -1             // invalid fi are assigned parity -1
+		}
+		res.metaArr = metaArr
+		res.parities = parities
+		res.errs = make([]error, N)
+		if agree >= N-parity {
+			res.parity = parity
+		} else {
+			res.parity = -1
+		}
+
+		return res
+	}
+	tests := []struct {
+		metaArr  []FileInfo
+		errs     []error
+		parities []int
+		parity   int
+	}{
+		// More than simple majority consensus
+		mkTest(15, 3, 11),
+		// No simple majority consensus
+		mkTest(15, 3, 7),
+		// Exact simple majority consensus
+		mkTest(15, 3, 8),
+		// More than simple majority consensus
+		mkTest(16, 4, 11),
+		// No simple majority consensus
+		mkTest(16, 4, 8),
+		// Exact simple majority consensus
+		mkTest(16, 4, 9),
+		// non-tiered object require read quorum of EcM
+		nonTieredTest(15, 3, 12),
+		// non-tiered object with fewer than EcM in consensus
+		nonTieredTest(15, 3, 11),
+		// non-tiered object require read quorum of EcM
+		nonTieredTest(16, 4, 12),
+		// non-tiered object with fewer than EcM in consensus
+		nonTieredTest(16, 4, 11),
+	}
+	for i, test := range tests {
+		t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) {
+			if got := listObjectParities(test.metaArr, test.errs); !slices.Equal(got, test.parities) {
+				t.Fatalf("Expected parities %v but got %v", test.parities, got)
+			}
+			if got := commonParity(test.parities, len(test.metaArr)/2); got != test.parity {
+				t.Fatalf("Expected common parity %v but got %v", test.parity, got)
+			}
+		})
+	}
+}

cmd/erasure-multipart.go

@@ -80,6 +80,14 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 		return fi, nil, err
 	}
 
+	if readQuorum < 0 {
+		return fi, nil, errErasureReadQuorum
+	}
+
+	if writeQuorum < 0 {
+		return fi, nil, errErasureWriteQuorum
+	}
+
 	quorum := readQuorum
 	if write {
 		quorum = writeQuorum
@@ -88,14 +96,13 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	// List all online disks.
 	_, modTime, etag := listOnlineDisks(storageDisks, partsMetadata, errs, quorum)
 
-	var reducedErr error
 	if write {
-		reducedErr = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+		err = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	} else {
-		reducedErr = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum)
+		err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum)
 	}
-	if reducedErr != nil {
-		return fi, nil, reducedErr
+	if err != nil {
+		return fi, nil, err
 	}
 
 	// Pick one from the first valid metadata.
@@ -103,10 +110,9 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	return fi, partsMetadata, err
 }
 
-// Removes part.meta given by partName belonging to a multipart upload from minioMetaBucket
-func (er erasureObjects) removePartMeta(bucket, object, uploadID, dataDir string, partNumber int) {
-	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
-	curpartPath := pathJoin(uploadIDPath, dataDir, fmt.Sprintf("part.%d", partNumber))
+// cleanupMultipartPath removes all extraneous files and parts from the multipart folder, this is used per CompleteMultipart.
+// do not use this function outside of completeMultipartUpload()
+func (er erasureObjects) cleanupMultipartPath(ctx context.Context, paths ...string) {
 	storageDisks := er.getDisks()
 
 	g := errgroup.WithNErrs(len(storageDisks))
@@ -116,42 +122,7 @@ func (er erasureObjects) removePartMeta(bucket, object, uploadID, dataDir string
 		}
 		index := index
 		g.Go(func() error {
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath+".meta", DeleteOptions{
-				Recursive: false,
-				Immediate: false,
-			})
-
-			return nil
-		}, index)
-	}
-	g.Wait()
-}
-
-// Removes part given by partName belonging to a multipart upload from minioMetaBucket
-func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir string, partNumber int) {
-	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
-	curpartPath := pathJoin(uploadIDPath, dataDir, fmt.Sprintf("part.%d", partNumber))
-	storageDisks := er.getDisks()
-
-	g := errgroup.WithNErrs(len(storageDisks))
-	for index, disk := range storageDisks {
-		if disk == nil {
-			continue
-		}
-		index := index
-		g.Go(func() error {
-			// Ignoring failure to remove parts that weren't present in CompleteMultipartUpload
-			// requests. xl.meta is the authoritative source of truth on which parts constitute
-			// the object. The presence of parts that don't belong in the object doesn't affect correctness.
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath, DeleteOptions{
-				Recursive: false,
-				Immediate: false,
-			})
-			_ = storageDisks[index].Delete(context.TODO(), minioMetaMultipartBucket, curpartPath+".meta", DeleteOptions{
-				Recursive: false,
-				Immediate: false,
-			})
-
+			_ = storageDisks[index].DeleteBulk(ctx, minioMetaMultipartBucket, paths...)
 			return nil
 		}, index)
 	}
@@ -159,7 +130,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 }
 
 // Clean-up the old multipart uploads. Should be run in a Go routine.
-func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
+func (er erasureObjects) cleanupStaleUploads(ctx context.Context) {
 	// run multiple cleanup's local to this server.
 	var wg sync.WaitGroup
 	for _, disk := range er.getLocalDisks() {
@@ -167,7 +138,7 @@ func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Du
 			wg.Add(1)
 			go func(disk StorageAPI) {
 				defer wg.Done()
-				er.cleanupStaleUploadsOnDisk(ctx, disk, expiry)
+				er.cleanupStaleUploadsOnDisk(ctx, disk)
 			}(disk)
 		}
 	}
@@ -193,7 +164,7 @@ func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) {
 }
 
 // Remove the old multipart uploads on the given disk.
-func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI, expiry time.Duration) {
+func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI) {
 	drivePath := disk.Endpoint().Path
 
 	readDirFn(pathJoin(drivePath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
@@ -219,7 +190,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 				modTime = fi.ModTime
 				wait()
 			}
-			if time.Since(modTime) < expiry {
+			if time.Since(modTime) < globalAPIConfig.getStaleUploadsExpiry() {
 				return nil
 			}
 			w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
@@ -238,7 +209,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 			return nil
 		}
 		// Modtime is returned in the Created field. See (*xlStorage).StatVol
-		if time.Since(vi.Created) < expiry {
+		if time.Since(vi.Created) < globalAPIConfig.getStaleUploadsExpiry() {
 			return nil
 		}
 		w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
@@ -267,7 +238,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout())
 		return w.Run(func() error {
 			wait := deleteMultipartCleanupSleeper.Timer(ctx)
-			if time.Since(vi.Created) > expiry {
+			if time.Since(vi.Created) > globalAPIConfig.getStaleUploadsExpiry() {
 				pathUUID := mustGetUUID()
 				targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID)
 
@@ -287,7 +258,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 // towards simplification of multipart APIs.
 // The resulting ListMultipartsInfo structure is unmarshalled directly as XML.
 func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) {
-	auditObjectErasureSet(ctx, object, &er)
+	auditObjectErasureSet(ctx, "ListMultipartUploads", object, &er)
 
 	result.MaxUploads = maxUploads
 	result.KeyMarker = keyMarker
@@ -526,9 +497,10 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadUUID)
 
 	// Write updated `xl.meta` to all disks.
-	if _, err := writeUniqueFileInfo(ctx, onlineDisks, bucket, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
+	if _, err := writeAllMetadata(ctx, onlineDisks, bucket, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
 		return nil, toObjectErr(err, bucket, object)
 	}
+
 	return &NewMultipartUploadResult{
 		UploadID:     uploadID,
 		ChecksumAlgo: userDefined[hash.MinIOMultipartChecksum],
@@ -542,14 +514,14 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 // Implements S3 compatible initiate multipart API.
 func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (*NewMultipartUploadResult, error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "NewMultipartUpload", object, &er)
 	}
 
 	return er.newMultipartUpload(ctx, bucket, object, opts)
 }
 
 // renamePart - renames multipart part to its relevant location under uploadID.
-func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
+func (er erasureObjects) renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, optsMeta []byte, writeQuorum int) ([]StorageAPI, error) {
 	g := errgroup.WithNErrs(len(disks))
 
 	// Rename file on all underlying storage disks.
@@ -559,60 +531,25 @@ func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, ds
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			return disks[index].RenameFile(ctx, srcBucket, srcEntry, dstBucket, dstEntry)
+			return disks[index].RenamePart(ctx, srcBucket, srcEntry, dstBucket, dstEntry, optsMeta)
 		}, index)
 	}
 
 	// Wait for all renames to finish.
 	errs := g.Wait()
 
-	// Do not need to undo partial successful operation since those will be cleaned up
-	// in 24hrs via multipart cleaner, never rename() back to `.minio.sys/tmp` as there
-	// is no way to clean them.
-
-	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
-	// otherwise return failure. Cleanup successful renames.
-	return evalDisks(disks, errs), reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-}
-
-// writeAllDisks - writes 'b' to all provided disks.
-// If write cannot reach quorum, the files will be deleted from all disks.
-func writeAllDisks(ctx context.Context, disks []StorageAPI, dstBucket, dstEntry string, b []byte, writeQuorum int) ([]StorageAPI, error) {
-	g := errgroup.WithNErrs(len(disks))
-
-	// Write file to all underlying storage disks.
-	for index := range disks {
-		index := index
-		g.Go(func() error {
-			if disks[index] == nil {
-				return errDiskNotFound
-			}
-			return disks[index].WriteAll(ctx, dstBucket, dstEntry, b)
-		}, index)
+	paths := []string{
+		dstEntry,
+		dstEntry + ".meta",
 	}
 
-	// Wait for all renames to finish.
-	errs := g.Wait()
-
-	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
-	// otherwise return failure. Cleanup successful renames.
 	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-	if errors.Is(err, errErasureWriteQuorum) {
-		// Remove all written
-		g := errgroup.WithNErrs(len(disks))
-		for index := range disks {
-			if disks[index] == nil || errs[index] != nil {
-				continue
-			}
-			index := index
-			g.Go(func() error {
-				return disks[index].Delete(ctx, dstBucket, dstEntry, DeleteOptions{Immediate: true})
-			}, index)
-		}
-		// Ignore these errors.
-		g.WaitErr()
+	if err != nil {
+		er.cleanupMultipartPath(ctx, paths...)
 	}
 
+	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
+	// otherwise return failure. Cleanup successful renames.
 	return evalDisks(disks, errs), err
 }
 
@@ -623,7 +560,7 @@ func writeAllDisks(ctx context.Context, disks []StorageAPI, dstBucket, dstEntry
 // Implements S3 compatible Upload Part API.
 func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "PutObjectPart", object, &er)
 	}
 
 	data := r.Reader
@@ -768,19 +705,6 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 
 	// Rename temporary part file to its final location.
 	partPath := pathJoin(uploadIDPath, fi.DataDir, partSuffix)
-	onlineDisks, err = renamePart(ctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, writeQuorum)
-	if err != nil {
-		if errors.Is(err, errFileNotFound) {
-			// An in-quorum errFileNotFound means that client stream
-			// prematurely closed and we do not find any xl.meta or
-			// part.1's - in such a scenario we must return as if client
-			// disconnected. This means that erasure.Encode() CreateFile()
-			// did not do anything.
-			return pi, IncompleteBody{Bucket: bucket, Object: object}
-		}
-
-		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
-	}
 
 	md5hex := r.MD5CurrentHexString()
 	if opts.PreserveETag != "" {
@@ -802,15 +726,22 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		Checksums:  r.ContentCRC(),
 	}
 
-	fi.Parts = []ObjectPartInfo{partInfo}
-	partFI, err := fi.MarshalMsg(nil)
+	partFI, err := partInfo.MarshalMsg(nil)
 	if err != nil {
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
-	// Write part metadata to all disks.
-	onlineDisks, err = writeAllDisks(ctx, onlineDisks, minioMetaMultipartBucket, partPath+".meta", partFI, writeQuorum)
+	onlineDisks, err = er.renamePart(ctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, partFI, writeQuorum)
 	if err != nil {
+		if errors.Is(err, errFileNotFound) {
+			// An in-quorum errFileNotFound means that client stream
+			// prematurely closed and we do not find any xl.meta or
+			// part.1's - in such a scenario we must return as if client
+			// disconnected. This means that erasure.Encode() CreateFile()
+			// did not do anything.
+			return pi, IncompleteBody{Bucket: bucket, Object: object}
+		}
+
 		return pi, toObjectErr(err, minioMetaMultipartBucket, partPath)
 	}
 
@@ -835,7 +766,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 // Does not contain currently uploaded parts by design.
 func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (MultipartInfo, error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "GetMultipartInfo", object, &er)
 	}
 
 	result := MultipartInfo{
@@ -873,7 +804,7 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 // replied back to the client.
 func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "ListObjectParts", object, &er)
 	}
 
 	uploadIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
@@ -953,7 +884,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 
 	g := errgroup.WithNErrs(len(req.Files)).WithConcurrency(32)
 
-	partsInfo := make([]ObjectPartInfo, len(req.Files))
+	partsInfo := make([]*ObjectPartInfo, len(req.Files))
 	for i, file := range req.Files {
 		file := file
 		partN := i + start
@@ -965,21 +896,17 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 				return err
 			}
 
-			var pfi FileInfo
-			_, err = pfi.UnmarshalMsg(buf)
+			pinfo := &ObjectPartInfo{}
+			_, err = pinfo.UnmarshalMsg(buf)
 			if err != nil {
 				return err
 			}
 
-			if len(pfi.Parts) != 1 {
-				return errors.New("invalid number of parts expected 1, got 0")
+			if partN != pinfo.Number {
+				return fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partN, partN, pinfo.Number)
 			}
 
-			if partN != pfi.Parts[0].Number {
-				return fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partN, partN, pfi.Parts[0].Number)
-			}
-
-			partsInfo[i] = pfi.Parts[0]
+			partsInfo[i] = pinfo
 			return nil
 		}, i)
 	}
@@ -987,7 +914,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	g.Wait()
 
 	for _, part := range partsInfo {
-		if part.Number != 0 && !part.ModTime.IsZero() {
+		if part != nil && part.Number != 0 && !part.ModTime.IsZero() {
 			fi.AddObjectPart(part.Number, part.ETag, part.Size, part.ActualSize, part.ModTime, part.Index, part.Checksums)
 		}
 	}
@@ -1023,6 +950,106 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	return result, nil
 }
 
+func readParts(ctx context.Context, disks []StorageAPI, bucket string, partMetaPaths []string, partNumbers []int, readQuorum int) ([]*ObjectPartInfo, error) {
+	g := errgroup.WithNErrs(len(disks))
+
+	objectPartInfos := make([][]*ObjectPartInfo, len(disks))
+	// Rename file on all underlying storage disks.
+	for index := range disks {
+		index := index
+		g.Go(func() (err error) {
+			if disks[index] == nil {
+				return errDiskNotFound
+			}
+			objectPartInfos[index], err = disks[index].ReadParts(ctx, bucket, partMetaPaths...)
+			return err
+		}, index)
+	}
+
+	if err := reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum); err != nil {
+		return nil, err
+	}
+
+	partInfosInQuorum := make([]*ObjectPartInfo, len(partMetaPaths))
+	partMetaQuorumMap := make(map[string]int, len(partNumbers))
+	for pidx := range partMetaPaths {
+		var pinfos []*ObjectPartInfo
+		for idx := range disks {
+			if len(objectPartInfos[idx]) == 0 {
+				partMetaQuorumMap[partMetaPaths[pidx]]++
+				continue
+			}
+
+			pinfo := objectPartInfos[idx][pidx]
+			if pinfo == nil {
+				partMetaQuorumMap[partMetaPaths[pidx]]++
+				continue
+			}
+
+			if pinfo.ETag == "" {
+				partMetaQuorumMap[partMetaPaths[pidx]]++
+			} else {
+				pinfos = append(pinfos, pinfo)
+				partMetaQuorumMap[pinfo.ETag]++
+			}
+		}
+
+		var maxQuorum int
+		var maxETag string
+		var maxPartMeta string
+		for etag, quorum := range partMetaQuorumMap {
+			if maxQuorum < quorum {
+				maxQuorum = quorum
+				maxETag = etag
+				maxPartMeta = etag
+			}
+		}
+
+		var pinfo *ObjectPartInfo
+		for _, pinfo = range pinfos {
+			if pinfo != nil && maxETag != "" && pinfo.ETag == maxETag {
+				break
+			}
+			if maxPartMeta != "" && path.Base(maxPartMeta) == fmt.Sprintf("part.%d.meta", pinfo.Number) {
+				break
+			}
+		}
+
+		if pinfo != nil && pinfo.ETag != "" && partMetaQuorumMap[maxETag] >= readQuorum {
+			partInfosInQuorum[pidx] = pinfo
+			continue
+		}
+
+		if partMetaQuorumMap[maxPartMeta] == len(disks) {
+			if pinfo != nil && pinfo.Error != "" {
+				partInfosInQuorum[pidx] = &ObjectPartInfo{Error: pinfo.Error}
+			} else {
+				partInfosInQuorum[pidx] = &ObjectPartInfo{
+					Error: InvalidPart{
+						PartNumber: partNumbers[pidx],
+					}.Error(),
+				}
+			}
+		} else {
+			partInfosInQuorum[pidx] = &ObjectPartInfo{Error: errErasureReadQuorum.Error()}
+		}
+	}
+	return partInfosInQuorum, nil
+}
+
+func errStrToPartErr(errStr string) error {
+	if strings.Contains(errStr, "file not found") {
+		return InvalidPart{}
+	}
+	if strings.Contains(errStr, "Specified part could not be found") {
+		return InvalidPart{}
+	}
+	if strings.Contains(errStr, errErasureReadQuorum.Error()) {
+		return errErasureReadQuorum
+	}
+	return errors.New(errStr)
+}
+
 // CompleteMultipartUpload - completes an ongoing multipart
 // transaction after receiving all the parts indicated by the client.
 // Returns an md5sum calculated by concatenating all the individual
@@ -1031,7 +1058,7 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 // Implements S3 compatible Complete multipart API.
 func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "CompleteMultipartUpload", object, &er)
 	}
 
 	if opts.CheckPrecondFn != nil {
@@ -1076,24 +1103,22 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)
 	onlineDisks := er.getDisks()
 	writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
+	readQuorum := fi.ReadQuorum(er.defaultRQuorum())
 
 	// Read Part info for all parts
 	partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
-	req := ReadMultipleReq{
-		Bucket:       minioMetaMultipartBucket,
-		Prefix:       partPath,
-		MaxSize:      1 << 20, // Each part should realistically not be > 1MiB.
-		Files:        make([]string, 0, len(parts)),
-		AbortOn404:   true,
-		MetadataOnly: true,
+	partMetaPaths := make([]string, len(parts))
+	partNumbers := make([]int, len(parts))
+	for idx, part := range parts {
+		partMetaPaths[idx] = pathJoin(partPath, fmt.Sprintf("part.%d.meta", part.PartNumber))
+		partNumbers[idx] = part.PartNumber
 	}
-	for _, part := range parts {
-		req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", part.PartNumber))
-	}
-	partInfoFiles, err := readMultipleFiles(ctx, onlineDisks, req, writeQuorum)
+
+	partInfoFiles, err := readParts(ctx, onlineDisks, minioMetaMultipartBucket, partMetaPaths, partNumbers, readQuorum)
 	if err != nil {
 		return oi, err
 	}
+
 	if len(partInfoFiles) != len(parts) {
 		// Should only happen through internal error
 		err := fmt.Errorf("unexpected part result count: %d, want %d", len(partInfoFiles), len(parts))
@@ -1155,35 +1180,22 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		opts.EncryptFn = metadataEncrypter(key)
 	}
 
-	for i, part := range partInfoFiles {
-		partID := parts[i].PartNumber
-		if part.Error != "" || !part.Exists {
-			return oi, InvalidPart{
-				PartNumber: partID,
-			}
-		}
-
-		var pfi FileInfo
-		_, err := pfi.UnmarshalMsg(part.Data)
-		if err != nil {
-			// Maybe crash or similar.
+	for idx, part := range partInfoFiles {
+		if part.Error != "" {
+			err = errStrToPartErr(part.Error)
 			bugLogIf(ctx, err)
-			return oi, InvalidPart{
-				PartNumber: partID,
-			}
+			return oi, err
 		}
 
-		partI := pfi.Parts[0]
-		partNumber := partI.Number
-		if partID != partNumber {
-			internalLogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partID, partID, partI.Number))
+		if parts[idx].PartNumber != part.Number {
+			internalLogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", parts[idx].PartNumber, parts[idx].PartNumber, part.Number))
 			return oi, InvalidPart{
-				PartNumber: partID,
+				PartNumber: part.Number,
 			}
 		}
 
 		// Add the current part.
-		fi.AddObjectPart(partI.Number, partI.ETag, partI.Size, partI.ActualSize, partI.ModTime, partI.Index, partI.Checksums)
+		fi.AddObjectPart(part.Number, part.ETag, part.Size, part.ActualSize, part.ModTime, part.Index, part.Checksums)
 	}
 
 	// Calculate full object size.
@@ -1337,7 +1349,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 
 	// Save the consolidated actual size.
 	if opts.ReplicationRequest {
-		fi.Metadata[ReservedMetadataPrefix+"actual-size"] = opts.UserDefined["X-Minio-Internal-Actual-Object-Size"]
+		if v := opts.UserDefined[ReservedMetadataPrefix+"Actual-Object-Size"]; v != "" {
+			fi.Metadata[ReservedMetadataPrefix+"actual-size"] = v
+		}
 	} else {
 		fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
 	}
@@ -1359,10 +1373,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
+	paths := make([]string, 0, len(currentFI.Parts))
 	// Remove parts that weren't present in CompleteMultipartUpload request.
 	for _, curpart := range currentFI.Parts {
-		// Remove part.meta which is not needed anymore.
-		er.removePartMeta(bucket, object, uploadID, currentFI.DataDir, curpart.Number)
+		paths = append(paths, pathJoin(uploadIDPath, currentFI.DataDir, fmt.Sprintf("part.%d.meta", curpart.Number)))
 
 		if objectPartIndex(fi.Parts, curpart.Number) == -1 {
 			// Delete the missing part files. e.g,
@@ -1371,10 +1385,12 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			// Request 3: PutObjectPart 2
 			// Request 4: CompleteMultipartUpload --part 2
 			// N.B. 1st part is not present. This part should be removed from the storage.
-			er.removeObjectPart(bucket, object, uploadID, currentFI.DataDir, curpart.Number)
+			paths = append(paths, pathJoin(uploadIDPath, currentFI.DataDir, fmt.Sprintf("part.%d", curpart.Number)))
 		}
 	}
 
+	er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename().
+
 	defer func() {
 		if err == nil {
 			er.deleteAll(context.Background(), minioMetaMultipartBucket, uploadIDPath)
@@ -1393,13 +1409,13 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	}
 
 	if !opts.Speedtest && len(versions) > 0 {
-		globalMRFState.addPartialOp(partialOperation{
-			bucket:    bucket,
-			object:    object,
-			queued:    time.Now(),
-			versions:  versions,
-			setIndex:  er.setIndex,
-			poolIndex: er.poolIndex,
+		globalMRFState.addPartialOp(PartialOperation{
+			Bucket:    bucket,
+			Object:    object,
+			Queued:    time.Now(),
+			Versions:  versions,
+			SetIndex:  er.setIndex,
+			PoolIndex: er.poolIndex,
 		})
 	}
 
@@ -1439,7 +1455,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 // operation.
 func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er)
 	}
 
 	lk := er.NewNSLock(bucket, pathJoin(object, uploadID))

cmd/erasure-object.go

@@ -70,7 +70,7 @@ func countOnlineDisks(onlineDisks []StorageAPI) (online int) {
 // update metadata.
 func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, err error) {
 	if !dstOpts.NoAuditLog {
-		auditObjectErasureSet(ctx, dstObject, &er)
+		auditObjectErasureSet(ctx, "CopyObject", dstObject, &er)
 	}
 
 	// This call shouldn't be used for anything other than metadata updates or adding self referential versions.
@@ -97,7 +97,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	if srcOpts.VersionID != "" {
 		metaArr, errs = readAllFileInfo(ctx, storageDisks, "", srcBucket, srcObject, srcOpts.VersionID, true, false)
 	} else {
-		metaArr, errs = readAllXL(ctx, storageDisks, srcBucket, srcObject, true, false, true)
+		metaArr, errs = readAllXL(ctx, storageDisks, srcBucket, srcObject, true, false)
 	}
 
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
@@ -199,7 +199,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 // Read(Closer). When err != nil, the returned reader is always nil.
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "GetObject", object, &er)
 	}
 
 	var unlockOnDefer bool
@@ -283,7 +283,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 	}
 
 	if unlockOnDefer {
-		unlockOnDefer = fi.InlineData()
+		unlockOnDefer = fi.InlineData() || len(fi.Data) > 0
 	}
 
 	pr, pw := xioutil.WaitPipe()
@@ -395,24 +395,16 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 			// that we have some parts or data blocks missing or corrupted
 			// - attempt a heal to successfully heal them for future calls.
 			if written == partLength {
-				var scan madmin.HealScanMode
-				switch {
-				case errors.Is(err, errFileNotFound):
-					scan = madmin.HealNormalScan
-				case errors.Is(err, errFileCorrupt):
-					scan = madmin.HealDeepScan
-				}
-				switch scan {
-				case madmin.HealNormalScan, madmin.HealDeepScan:
+				if errors.Is(err, errFileNotFound) || errors.Is(err, errFileCorrupt) {
 					healOnce.Do(func() {
-						globalMRFState.addPartialOp(partialOperation{
-							bucket:    bucket,
-							object:    object,
-							versionID: fi.VersionID,
-							queued:    time.Now(),
-							setIndex:  er.setIndex,
-							poolIndex: er.poolIndex,
-							scanMode:  scan,
+						globalMRFState.addPartialOp(PartialOperation{
+							Bucket:     bucket,
+							Object:     object,
+							VersionID:  fi.VersionID,
+							Queued:     time.Now(),
+							SetIndex:   er.setIndex,
+							PoolIndex:  er.poolIndex,
+							BitrotScan: errors.Is(err, errFileCorrupt),
 						})
 					})
 					// Healing is triggered and we have written
@@ -439,7 +431,7 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 // GetObjectInfo - reads object metadata and replies back ObjectInfo.
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "GetObjectInfo", object, &er)
 	}
 
 	if !opts.NoLock {
@@ -456,7 +448,7 @@ func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object strin
 	return er.getObjectInfo(ctx, bucket, object, opts)
 }
 
-func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID string, tags map[string]interface{}) {
+func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID string, tags map[string]string) {
 	if len(logger.AuditTargets()) == 0 {
 		return
 	}
@@ -472,13 +464,16 @@ func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID
 	auditLogInternal(ctx, opts)
 }
 
-func joinErrs(errs []error) []string {
-	s := make([]string, len(errs))
+func joinErrs(errs []error) string {
+	var s string
 	for i := range s {
+		if s != "" {
+			s += ","
+		}
 		if errs[i] == nil {
-			s[i] = "<nil>"
+			s += "<nil>"
 		} else {
-			s[i] = errs[i].Error()
+			s += errs[i].Error()
 		}
 	}
 	return s
@@ -491,20 +486,18 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		// can be deleted safely, in such a scenario return ReadQuorum error.
 		return FileInfo{}, errErasureReadQuorum
 	}
-	tags := make(map[string]interface{}, 4)
-	tags["set"] = er.setIndex
-	tags["pool"] = er.poolIndex
+	tags := make(map[string]string, 16)
+	tags["set"] = strconv.Itoa(er.setIndex)
+	tags["pool"] = strconv.Itoa(er.poolIndex)
 	tags["merrs"] = joinErrs(errs)
-	tags["derrs"] = dataErrsByPart
+	tags["derrs"] = fmt.Sprintf("%v", dataErrsByPart)
 	if m.IsValid() {
-		tags["size"] = m.Size
-		tags["mtime"] = m.ModTime.Format(http.TimeFormat)
-		tags["data"] = m.Erasure.DataBlocks
-		tags["parity"] = m.Erasure.ParityBlocks
+		tags["sz"] = strconv.FormatInt(m.Size, 10)
+		tags["mt"] = m.ModTime.Format(iso8601Format)
+		tags["d:p"] = fmt.Sprintf("%d:%d", m.Erasure.DataBlocks, m.Erasure.ParityBlocks)
 	} else {
-		tags["invalid-meta"] = true
-		tags["data"] = er.setDriveCount - er.defaultParityCount
-		tags["parity"] = er.defaultParityCount
+		tags["invalid"] = "1"
+		tags["d:p"] = fmt.Sprintf("%d:%d", er.setDriveCount-er.defaultParityCount, er.defaultParityCount)
 	}
 
 	// count the number of offline disks
@@ -527,7 +520,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		}
 	}
 	if offline > 0 {
-		tags["offline"] = offline
+		tags["offline"] = strconv.Itoa(offline)
 	}
 
 	_, file, line, cok := runtime.Caller(1)
@@ -556,30 +549,23 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		}, index)
 	}
 
-	rmDisks := make(map[string]string, len(disks))
 	for index, err := range g.Wait() {
-		var errStr, diskName string
+		var errStr string
 		if err != nil {
 			errStr = err.Error()
 		} else {
 			errStr = "<nil>"
 		}
-		if disks[index] != nil {
-			diskName = disks[index].String()
-		} else {
-			diskName = fmt.Sprintf("disk-%d", index)
-		}
-		rmDisks[diskName] = errStr
+		tags[fmt.Sprintf("ddisk-%d", index)] = errStr
 	}
-	tags["cleanupResult"] = rmDisks
+
 	return m, nil
 }
 
-func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVers, allParts bool) (FileInfo, error) {
+func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVers bool) (FileInfo, error) {
 	return getFileInfo(ri.Buf, bucket, object, "", fileInfoOpts{
 		Data:             readData,
 		InclFreeVersions: inclFreeVers,
-		AllParts:         allParts,
 	})
 }
 
@@ -604,7 +590,7 @@ func readAllRawFileInfo(ctx context.Context, disks []StorageAPI, bucket, object
 	return rawFileInfos, g.Wait()
 }
 
-func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo, errs []error, bucket, object string, readData, inclFreeVers, allParts bool) ([]FileInfo, []error) {
+func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo, errs []error, bucket, object string, readData, inclFreeVers bool) ([]FileInfo, []error) {
 	metadataArray := make([]*xlMetaV2, len(rawFileInfos))
 	metaFileInfos := make([]FileInfo, len(rawFileInfos))
 	metadataShallowVersions := make([][]xlMetaV2ShallowVersion, len(rawFileInfos))
@@ -641,7 +627,7 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 
 	readQuorum := (len(rawFileInfos) + 1) / 2
 	meta := &xlMetaV2{versions: mergeXLV2Versions(readQuorum, false, 1, metadataShallowVersions...)}
-	lfi, err := meta.ToFileInfo(bucket, object, "", inclFreeVers, allParts)
+	lfi, err := meta.ToFileInfo(bucket, object, "", inclFreeVers, true)
 	if err != nil {
 		for i := range errs {
 			if errs[i] == nil {
@@ -670,7 +656,7 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 		}
 
 		// make sure to preserve this for diskmtime based healing bugfix.
-		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID, inclFreeVers, allParts)
+		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID, inclFreeVers, true)
 		if errs[index] != nil {
 			continue
 		}
@@ -715,9 +701,9 @@ func shouldCheckForDangling(err error, errs []error, bucket string) bool {
 	return false
 }
 
-func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, readData, inclFreeVers, allParts bool) ([]FileInfo, []error) {
+func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, readData, inclFreeVers bool) ([]FileInfo, []error) {
 	rawFileInfos, errs := readAllRawFileInfo(ctx, disks, bucket, object, readData)
-	return pickLatestQuorumFilesInfo(ctx, rawFileInfos, errs, bucket, object, readData, inclFreeVers, allParts)
+	return pickLatestQuorumFilesInfo(ctx, rawFileInfos, errs, bucket, object, readData, inclFreeVers)
 }
 
 func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (FileInfo, []FileInfo, []StorageAPI, error) {
@@ -774,7 +760,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 					// Read the latest version
 					rfi, err = disk.ReadXL(ctx, bucket, object, readData)
 					if err == nil {
-						fi, err = fileInfoFromRaw(rfi, bucket, object, readData, opts.InclFreeVersions, true)
+						fi, err = fileInfoFromRaw(rfi, bucket, object, readData, opts.InclFreeVersions)
 					}
 				}
 
@@ -820,13 +806,13 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		// additionally do not heal delete markers inline, let them be
 		// healed upon regular heal process.
 		if missingBlocks > 0 && missingBlocks < fi.Erasure.DataBlocks {
-			globalMRFState.addPartialOp(partialOperation{
-				bucket:    fi.Volume,
-				object:    fi.Name,
-				versionID: fi.VersionID,
-				queued:    time.Now(),
-				setIndex:  er.setIndex,
-				poolIndex: er.poolIndex,
+			globalMRFState.addPartialOp(PartialOperation{
+				Bucket:    fi.Volume,
+				Object:    fi.Name,
+				VersionID: fi.VersionID,
+				Queued:    time.Now(),
+				SetIndex:  er.setIndex,
+				PoolIndex: er.poolIndex,
 			})
 		}
 
@@ -908,14 +894,16 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		}
 
 		rw.Lock()
+		// when its a versioned bucket and empty versionID - at totalResp == setDriveCount
+		// we must use rawFileInfo to resolve versions to figure out the latest version.
 		if opts.VersionID == "" && totalResp == er.setDriveCount {
 			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(pickLatestQuorumFilesInfo(ctx,
-				rawArr, errs, bucket, object, readData, opts.InclFreeVersions, true))
+				rawArr, errs, bucket, object, readData, opts.InclFreeVersions))
 		} else {
 			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(metaArr, errs)
 		}
 		rw.Unlock()
-		if err == nil && fi.InlineData() {
+		if err == nil && (fi.InlineData() || len(fi.Data) > 0) {
 			break
 		}
 	}
@@ -1250,7 +1238,7 @@ func (er erasureObjects) PutObject(ctx context.Context, bucket string, object st
 // putObject wrapper for erasureObjects PutObject
 func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "PutObject", object, &er)
 	}
 
 	data := r.Reader
@@ -1399,7 +1387,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	writers := make([]io.Writer, len(onlineDisks))
 	var inlineBuffers []*bytes.Buffer
 	if shardFileSize >= 0 {
-		if !opts.Versioned && shardFileSize < inlineBlock {
+		if !opts.Versioned && shardFileSize <= inlineBlock {
 			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
 		} else if shardFileSize < inlineBlock/8 {
 			inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
@@ -1407,7 +1395,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	} else {
 		// If compressed, use actual size to determine.
 		if sz := erasure.ShardFileSize(data.ActualSize()); sz > 0 {
-			if !opts.Versioned && sz < inlineBlock {
+			if !opts.Versioned && sz <= inlineBlock {
 				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
 			} else if sz < inlineBlock/8 {
 				inlineBuffers = make([]*bytes.Buffer, len(onlineDisks))
@@ -1576,13 +1564,13 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 				break
 			}
 		} else {
-			globalMRFState.addPartialOp(partialOperation{
-				bucket:    bucket,
-				object:    object,
-				queued:    time.Now(),
-				versions:  versions,
-				setIndex:  er.setIndex,
-				poolIndex: er.poolIndex,
+			globalMRFState.addPartialOp(PartialOperation{
+				Bucket:    bucket,
+				Object:    object,
+				Queued:    time.Now(),
+				Versions:  versions,
+				SetIndex:  er.setIndex,
+				PoolIndex: er.poolIndex,
 			})
 		}
 	}
@@ -1625,7 +1613,7 @@ func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object
 func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) {
 	if !opts.NoAuditLog {
 		for _, obj := range objects {
-			auditObjectErasureSet(ctx, obj.ObjectV.ObjectName, &er)
+			auditObjectErasureSet(ctx, "DeleteObjects", obj.ObjectV.ObjectName, &er)
 		}
 	}
 
@@ -1845,7 +1833,7 @@ func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string
 // response to the client request.
 func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	if !opts.NoAuditLog {
-		auditObjectErasureSet(ctx, object, &er)
+		auditObjectErasureSet(ctx, "DeleteObject", object, &er)
 	}
 
 	var lc *lifecycle.Lifecycle
@@ -2111,11 +2099,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 // Send the successful but partial upload/delete, however ignore
 // if the channel is blocked by other items.
 func (er erasureObjects) addPartial(bucket, object, versionID string) {
-	globalMRFState.addPartialOp(partialOperation{
-		bucket:    bucket,
-		object:    object,
-		versionID: versionID,
-		queued:    time.Now(),
+	globalMRFState.addPartialOp(PartialOperation{
+		Bucket:    bucket,
+		Object:    object,
+		VersionID: versionID,
+		Queued:    time.Now(),
 	})
 }
 
@@ -2140,7 +2128,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 	if opts.VersionID != "" {
 		metaArr, errs = readAllFileInfo(ctx, disks, "", bucket, object, opts.VersionID, false, false)
 	} else {
-		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false, true)
+		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false)
 	}
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
@@ -2217,7 +2205,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	if opts.VersionID != "" {
 		metaArr, errs = readAllFileInfo(ctx, disks, "", bucket, object, opts.VersionID, false, false)
 	} else {
-		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false, true)
+		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false)
 	}
 
 	readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)

cmd/erasure-server-pool-decom.go

@@ -608,7 +608,7 @@ func (z *erasureServerPools) IsDecommissionRunning() bool {
 	return false
 }
 
-func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket string, gr *GetObjectReader) (err error) {
+func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bucket string, gr *GetObjectReader) (err error) {
 	objInfo := gr.ObjInfo
 
 	defer func() {
@@ -623,9 +623,11 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 
 	if objInfo.isMultipart() {
 		res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
-			VersionID:   objInfo.VersionID,
-			UserDefined: objInfo.UserDefined,
-			NoAuditLog:  true,
+			VersionID:    objInfo.VersionID,
+			UserDefined:  objInfo.UserDefined,
+			NoAuditLog:   true,
+			SrcPoolIdx:   idx,
+			DataMovement: true,
 		})
 		if err != nil {
 			return fmt.Errorf("decommissionObject: NewMultipartUpload() %w", err)
@@ -660,6 +662,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 			}
 		}
 		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, parts, ObjectOptions{
+			SrcPoolIdx:   idx,
 			DataMovement: true,
 			MTime:        objInfo.ModTime,
 			NoAuditLog:   true,
@@ -681,6 +684,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		NewPutObjReader(hr),
 		ObjectOptions{
 			DataMovement: true,
+			SrcPoolIdx:   idx,
 			VersionID:    objInfo.VersionID,
 			MTime:        objInfo.ModTime,
 			UserDefined:  objInfo.UserDefined,
@@ -855,6 +859,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					versionID = nullVersionID
 				}
 
+				var failure, ignore bool
 				if version.Deleted {
 					_, err := z.DeleteObject(ctx,
 						bi.Name,
@@ -867,14 +872,19 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 							VersionID:          versionID,
 							MTime:              version.ModTime,
 							DeleteReplication:  version.ReplicationState,
+							SrcPoolIdx:         idx,
+							DataMovement:       true,
 							DeleteMarker:       true, // make sure we create a delete marker
 							SkipDecommissioned: true, // make sure we skip the decommissioned pool
 							NoAuditLog:         true,
 						})
-					var failure bool
 					if err != nil {
-						if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-							err = nil
+						// This can happen when rebalance stop races with ongoing rebalance workers.
+						// These rebalance failures can be ignored.
+						if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
+							ignore = true
+							stopFn(0, nil)
+							continue
 						}
 					}
 					stopFn(version.Size, err)
@@ -893,22 +903,26 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					continue
 				}
 
-				var failure, ignore bool
 				// gr.Close() is ensured by decommissionObject().
 				for try := 0; try < 3; try++ {
 					if version.IsRemote() {
 						if err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{
-							VersionID:   versionID,
-							MTime:       version.ModTime,
-							UserDefined: version.Metadata,
+							VersionID:    versionID,
+							MTime:        version.ModTime,
+							UserDefined:  version.Metadata,
+							SrcPoolIdx:   idx,
+							DataMovement: true,
 						}); err != nil {
-							stopFn(version.Size, err)
-							failure = true
-							decomLogIf(ctx, err)
-							continue
+							if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
+								ignore = true
+								stopFn(0, nil)
+							}
+						}
+						if !ignore {
+							stopFn(version.Size, err)
+							failure = err != nil
+							decomLogIf(ctx, err)
 						}
-						stopFn(version.Size, nil)
-						failure = false
 						break
 					}
 					gr, err := set.GetObjectNInfo(ctx,
@@ -925,7 +939,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
 						ignore = true
-						stopFn(version.Size, nil)
+						stopFn(0, nil)
 						break
 					}
 					if err != nil && !ignore {
@@ -943,7 +957,12 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						stopFn(version.Size, err)
 						continue
 					}
-					if err = z.decommissionObject(ctx, bi.Name, gr); err != nil {
+					if err = z.decommissionObject(ctx, idx, bi.Name, gr); err != nil {
+						if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
+							ignore = true
+							stopFn(0, nil)
+							break
+						}
 						stopFn(version.Size, err)
 						failure = true
 						decomLogIf(ctx, err)

cmd/erasure-server-pool-rebalance.go

@@ -626,14 +626,18 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 
 			var rebalanced, expired int
 			for _, version := range fivs.Versions {
+				stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceObject, poolIdx, bucket, version.Name, version.VersionID)
+
 				// Skip transitioned objects for now. TBD
 				if version.IsRemote() {
+					stopFn(version.Size, errors.New("ILM Tiered version will be skipped for now"))
 					continue
 				}
 
 				// Apply lifecycle rules on the objects that are expired.
 				if filterLifecycle(bucket, version.Name, version) {
 					expired++
+					stopFn(version.Size, errors.New("ILM expired object/version will be skipped"))
 					continue
 				}
 
@@ -643,6 +647,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 				remainingVersions := len(fivs.Versions) - expired
 				if version.Deleted && remainingVersions == 1 {
 					rebalanced++
+					stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped"))
 					continue
 				}
 
@@ -651,6 +656,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					versionID = nullVersionID
 				}
 
+				var failure, ignore bool
 				if version.Deleted {
 					_, err := z.DeleteObject(ctx,
 						bucket,
@@ -660,16 +666,26 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 							VersionID:         versionID,
 							MTime:             version.ModTime,
 							DeleteReplication: version.ReplicationState,
+							SrcPoolIdx:        poolIdx,
+							DataMovement:      true,
 							DeleteMarker:      true, // make sure we create a delete marker
 							SkipRebalancing:   true, // make sure we skip the decommissioned pool
 							NoAuditLog:        true,
 						})
-					var failure bool
-					if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
-						rebalanceLogIf(ctx, err)
-						failure = true
+					// This can happen when rebalance stop races with ongoing rebalance workers.
+					// These rebalance failures can be ignored.
+					if err != nil {
+						// This can happen when rebalance stop races with ongoing rebalance workers.
+						// These rebalance failures can be ignored.
+						if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
+							ignore = true
+							stopFn(0, nil)
+							continue
+						}
 					}
-
+					stopFn(version.Size, err)
+					rebalanceLogIf(ctx, err)
+					failure = err != nil
 					if !failure {
 						z.updatePoolStats(poolIdx, bucket, version)
 						rebalanced++
@@ -678,10 +694,8 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					continue
 				}
 
-				var failure, ignore bool
 				for try := 0; try < 3; try++ {
 					// GetObjectReader.Close is called by rebalanceObject
-					stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceObject, poolIdx, bucket, version.Name, version.VersionID)
 					gr, err := set.GetObjectNInfo(ctx,
 						bucket,
 						encodeDirObject(version.Name),
@@ -706,7 +720,14 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 						continue
 					}
 
-					if err = z.rebalanceObject(ctx, bucket, gr); err != nil {
+					if err = z.rebalanceObject(ctx, poolIdx, bucket, gr); err != nil {
+						// This can happen when rebalance stop races with ongoing rebalance workers.
+						// These rebalance failures can be ignored.
+						if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) {
+							ignore = true
+							stopFn(0, nil)
+							break
+						}
 						failure = true
 						rebalanceLogIf(ctx, err)
 						stopFn(version.Size, err)
@@ -822,7 +843,7 @@ func auditLogRebalance(ctx context.Context, apiName, bucket, object, versionID s
 	})
 }
 
-func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string, gr *GetObjectReader) (err error) {
+func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, bucket string, gr *GetObjectReader) (err error) {
 	oi := gr.ObjInfo
 
 	defer func() {
@@ -837,9 +858,11 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 
 	if oi.isMultipart() {
 		res, err := z.NewMultipartUpload(ctx, bucket, oi.Name, ObjectOptions{
-			VersionID:   oi.VersionID,
-			UserDefined: oi.UserDefined,
-			NoAuditLog:  true,
+			VersionID:    oi.VersionID,
+			UserDefined:  oi.UserDefined,
+			NoAuditLog:   true,
+			DataMovement: true,
+			SrcPoolIdx:   poolIdx,
 		})
 		if err != nil {
 			return fmt.Errorf("rebalanceObject: NewMultipartUpload() %w", err)
@@ -891,6 +914,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 		oi.Name,
 		NewPutObjReader(hr),
 		ObjectOptions{
+			SrcPoolIdx:   poolIdx,
 			DataMovement: true,
 			VersionID:    oi.VersionID,
 			MTime:        oi.ModTime,
@@ -981,6 +1005,8 @@ const (
 	rebalanceMetricSaveMetadata
 )
 
+var errDataMovementSrcDstPoolSame = errors.New("source and destination pool are the same")
+
 func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duration time.Duration, err error, path string, sz int64) madmin.TraceInfo {
 	var errStr string
 	if err != nil {

cmd/erasure-server-pool.go

@@ -168,7 +168,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 
 	if !globalIsDistErasure {
 		globalLocalDrivesMu.Lock()
-		globalLocalDrives = localDrives
+		globalLocalDrivesMap = make(map[string]StorageAPI, len(localDrives))
 		for _, drive := range localDrives {
 			globalLocalDrivesMap[drive.Endpoint().String()] = drive
 		}
@@ -187,19 +187,12 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 		globalLeaderLock = newSharedLock(GlobalContext, z, "leader.lock")
 	})
 
-	// Enable background operations on
-	//
-	// - Disk auto healing
-	// - MRF (most recently failed) healing
-	// - Background expiration routine for lifecycle policies
+	// Start self healing after the object initialization
+	// so various tasks will be useful
 	bootstrapTrace("initAutoHeal", func() {
 		initAutoHeal(GlobalContext, z)
 	})
 
-	bootstrapTrace("initHealMRF", func() {
-		go globalMRFState.healRoutine(z)
-	})
-
 	// initialize the object layer.
 	defer setObjectLayer(z)
 
@@ -1087,6 +1080,14 @@ func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, objec
 		return ObjectInfo{}, err
 	}
 
+	if opts.DataMovement && idx == opts.SrcPoolIdx {
+		return ObjectInfo{}, DataMovementOverwriteErr{
+			Bucket:    bucket,
+			Object:    object,
+			VersionID: opts.VersionID,
+			Err:       errDataMovementSrcDstPoolSame,
+		}
+	}
 	// Overwrite the object at the right pool
 	return z.serverPools[idx].PutObject(ctx, bucket, object, data, opts)
 }
@@ -1139,6 +1140,16 @@ func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, ob
 		return pinfo.ObjInfo, nil
 	}
 
+	// Datamovement must never be allowed on the same pool.
+	if opts.DataMovement && opts.SrcPoolIdx == pinfo.Index {
+		return pinfo.ObjInfo, DataMovementOverwriteErr{
+			Bucket:    bucket,
+			Object:    object,
+			VersionID: opts.VersionID,
+			Err:       errDataMovementSrcDstPoolSame,
+		}
+	}
+
 	// Delete concurrently in all server pools with read quorum error for unversioned objects.
 	if len(noReadQuorumPools) > 0 && !opts.Versioned && !opts.VersionSuspended {
 		return z.deleteObjectFromAllPools(ctx, bucket, object, opts, noReadQuorumPools)
@@ -1526,14 +1537,12 @@ func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, pre
 			loi.NextMarker = last.Name
 		}
 
-		if merged.lastSkippedEntry != "" {
-			if merged.lastSkippedEntry > loi.NextMarker {
-				// An object hidden by ILM was found during listing. Since the number of entries
-				// fetched from drives is limited, set IsTruncated to true to ask the s3 client
-				// to continue listing if it wishes in order to find if there is more objects.
-				loi.IsTruncated = true
-				loi.NextMarker = merged.lastSkippedEntry
-			}
+		if loi.IsTruncated && merged.lastSkippedEntry > loi.NextMarker {
+			// An object hidden by ILM was found during a truncated listing. Since the number of entries
+			// fetched from drives is limited by max-keys, we should use the last ILM filtered entry
+			// as a continuation token if it is lexially higher than the last visible object so that the
+			// next call of WalkDir() with the max-keys can reach new objects not seen previously.
+			loi.NextMarker = merged.lastSkippedEntry
 		}
 
 		if loi.NextMarker != "" {
@@ -1754,6 +1763,15 @@ func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, obj
 		return nil, err
 	}
 
+	if opts.DataMovement && idx == opts.SrcPoolIdx {
+		return nil, DataMovementOverwriteErr{
+			Bucket:    bucket,
+			Object:    object,
+			VersionID: opts.VersionID,
+			Err:       errDataMovementSrcDstPoolSame,
+		}
+	}
+
 	return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts)
 }
 
@@ -1962,6 +1980,34 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 		defer lk.Unlock(lkctx)
 	}
 
+	if !opts.Force {
+		results := make(chan itemOrErr[ObjectInfo])
+
+		ctx, cancel := context.WithTimeout(ctx, time.Minute)
+		defer cancel()
+		err := z.Walk(ctx, bucket, "", results, WalkOptions{Limit: 1})
+		if err != nil {
+			s3LogIf(ctx, fmt.Errorf("unable to verify if the bucket %s is empty: %w", bucket, err))
+			return toObjectErr(err, bucket)
+		}
+
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case r, found := <-results:
+			if found {
+				if r.Err != nil {
+					s3LogIf(ctx, fmt.Errorf("unable to verify if the bucket %s is empty: %w", bucket, r.Err))
+					return toObjectErr(r.Err, bucket)
+				}
+				return toObjectErr(errVolumeNotEmpty, bucket)
+			}
+		}
+
+		// Always pass force to the lower level
+		opts.Force = true
+	}
+
 	err := z.s3Peer.DeleteBucket(ctx, bucket, opts)
 	if err == nil || isErrBucketNotFound(err) {
 		// If site replication is configured, hold on to deleted bucket state until sites sync
@@ -2104,7 +2150,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 			disks, infos, _ := set.getOnlineDisksWithHealingAndInfo(true)
 			if len(disks) == 0 {
 				xioutil.SafeClose(results)
-				err := fmt.Errorf("Walk: no online disks found in pool %d, set %d", setIdx, poolIdx)
+				err := fmt.Errorf("Walk: no online disks found in (set:%d pool:%d) %w", setIdx, poolIdx, errErasureReadQuorum)
 				cancelCause(err)
 				return err
 			}
@@ -2180,6 +2226,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 					filterPrefix:   filterPrefix,
 					recursive:      true,
 					forwardTo:      opts.Marker,
+					perDiskLimit:   opts.Limit,
 					minDisks:       listingQuorum,
 					reportNotFound: false,
 					agreed:         send,
@@ -2782,5 +2829,14 @@ func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, obje
 		return err
 	}
 
+	if opts.DataMovement && idx == opts.SrcPoolIdx {
+		return DataMovementOverwriteErr{
+			Bucket:    bucket,
+			Object:    object,
+			VersionID: opts.VersionID,
+			Err:       errDataMovementSrcDstPoolSame,
+		}
+	}
+
 	return z.serverPools[idx].DecomTieredObject(ctx, bucket, object, fi, opts)
 }

cmd/erasure-sets.go

@@ -230,8 +230,11 @@ func (s *erasureSets) connectDisks(log bool) {
 				}
 				return
 			}
-			if disk.IsLocal() && disk.Healing() != nil {
-				globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint())
+			if disk.IsLocal() {
+				h := disk.Healing()
+				if h != nil && !h.Finished {
+					globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint())
+				}
 			}
 			s.erasureDisksMu.Lock()
 			setIndex, diskIndex, err := findDiskIndex(s.format, format)
@@ -262,13 +265,7 @@ func (s *erasureSets) connectDisks(log bool) {
 				if globalIsDistErasure {
 					globalLocalSetDrives[s.poolIndex][setIndex][diskIndex] = disk
 				}
-				for i, ldisk := range globalLocalDrives {
-					_, k, l := ldisk.GetDiskLoc()
-					if k == setIndex && l == diskIndex {
-						globalLocalDrives[i] = disk
-						break
-					}
-				}
+				globalLocalDrivesMap[disk.Endpoint().String()] = disk
 				globalLocalDrivesMu.Unlock()
 			}
 			s.erasureDisksMu.Unlock()
@@ -531,7 +528,7 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
 					if set == nil {
 						return
 					}
-					set.cleanupStaleUploads(ctx, globalAPIConfig.getStaleUploadsExpiry())
+					set.cleanupStaleUploads(ctx)
 				}(set)
 			}
 			wg.Wait()
@@ -543,26 +540,29 @@ func (s *erasureSets) cleanupStaleUploads(ctx context.Context) {
 }
 
 type auditObjectOp struct {
-	Name   string   `json:"name"`
-	Pool   int      `json:"poolId"`
-	Set    int      `json:"setId"`
-	Drives []string `json:"drives"`
+	Name string `json:"name"`
+	Pool int    `json:"poolId"`
+	Set  int    `json:"setId"`
+}
+
+func (op auditObjectOp) String() string {
+	// Flatten the auditObjectOp
+	return fmt.Sprintf("name=%s,pool=%d,set=%d", op.Name, op.Pool, op.Set)
 }
 
 // Add erasure set information to the current context
-func auditObjectErasureSet(ctx context.Context, object string, set *erasureObjects) {
+func auditObjectErasureSet(ctx context.Context, api, object string, set *erasureObjects) {
 	if len(logger.AuditTargets()) == 0 {
 		return
 	}
 
 	op := auditObjectOp{
-		Name:   decodeDirObject(object),
-		Pool:   set.poolIndex + 1,
-		Set:    set.setIndex + 1,
-		Drives: set.getEndpointStrings(),
+		Name: decodeDirObject(object),
+		Pool: set.poolIndex + 1,
+		Set:  set.setIndex + 1,
 	}
 
-	logger.GetReqInfo(ctx).AppendTags("objectLocation", op)
+	logger.GetReqInfo(ctx).AppendTags(api, op.String())
 }
 
 // NewNSLock - initialize a new namespace RWLocker instance.
@@ -1135,13 +1135,7 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 					if globalIsDistErasure {
 						globalLocalSetDrives[s.poolIndex][m][n] = disk
 					}
-					for i, ldisk := range globalLocalDrives {
-						_, k, l := ldisk.GetDiskLoc()
-						if k == m && l == n {
-							globalLocalDrives[i] = disk
-							break
-						}
-					}
+					globalLocalDrivesMap[disk.Endpoint().String()] = disk
 					globalLocalDrivesMu.Unlock()
 				}
 			}

cmd/erasure.go

@@ -90,6 +90,11 @@ func (er erasureObjects) defaultWQuorum() int {
 	return dataCount
 }
 
+// defaultRQuorum read quorum based on setDriveCount and defaultParityCount
+func (er erasureObjects) defaultRQuorum() int {
+	return er.setDriveCount - er.defaultParityCount
+}
+
 func diskErrToDriveState(err error) (state string) {
 	switch {
 	case errors.Is(err, errDiskNotFound) || errors.Is(err, context.DeadlineExceeded):
@@ -198,11 +203,9 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint, metrics bool) (disks
 			di.State = diskErrToDriveState(err)
 			di.FreeInodes = info.FreeInodes
 			di.UsedInodes = info.UsedInodes
-			if info.Healing {
-				if hi := disks[index].Healing(); hi != nil {
-					hd := hi.toHealingDisk()
-					di.HealInfo = &hd
-				}
+			if hi := disks[index].Healing(); hi != nil {
+				hd := hi.toHealingDisk()
+				di.HealInfo = &hd
 			}
 			di.Metrics = &madmin.DiskMetrics{
 				LastMinute:              make(map[string]madmin.TimedAction, len(info.Metrics.LastMinute)),

cmd/fmt-gen.go

@@ -0,0 +1,119 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"encoding/json"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/klauspost/compress/zip"
+	"github.com/minio/cli"
+)
+
+var fmtGenFlags = []cli.Flag{
+	cli.IntFlag{
+		Name:  "parity",
+		Usage: "specify erasure code parity",
+	},
+	cli.StringFlag{
+		Name:  "deployment-id",
+		Usage: "deployment-id of the MinIO cluster for which format.json is needed",
+	},
+	cli.StringFlag{
+		Name:   "address",
+		Value:  ":" + GlobalMinioDefaultPort,
+		Usage:  "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname",
+		EnvVar: "MINIO_ADDRESS",
+	},
+}
+
+var fmtGenCmd = cli.Command{
+	Name:   "fmt-gen",
+	Usage:  "Generate format.json files for an erasure server pool",
+	Flags:  append(fmtGenFlags, GlobalFlags...),
+	Action: fmtGenMain,
+	Hidden: true,
+	CustomHelpTemplate: `NAME:
+  {{.HelpName}} - {{.Usage}}
+
+USAGE:
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR1 [DIR2..]
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64}
+  {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64} DIR{65...128}
+
+DIR:
+  DIR points to a directory on a filesystem. When you want to combine
+  multiple drives into a single large system, pass one directory per
+  filesystem separated by space. You may also use a '...' convention
+  to abbreviate the directory arguments. Remote directories in a
+  distributed setup are encoded as HTTP(s) URIs.
+{{if .VisibleFlags}}
+FLAGS:
+  {{range .VisibleFlags}}{{.}}
+  {{end}}{{end}}
+EXAMPLES:
+  1. Generate format.json.zip containing format.json files for all drives in a distributed MinIO server pool of 32 nodes with 32 drives each.
+     {{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export{1...32}
+
+`,
+}
+
+func fmtGenMain(ctxt *cli.Context) {
+	deploymentID := ctxt.String("deployment-id")
+	err := buildServerCtxt(ctxt, &globalServerCtxt)
+	if err != nil {
+		log.Fatalln(err)
+	}
+	handleCommonArgs(globalServerCtxt)
+	pools, _, err := createServerEndpoints(globalMinioAddr, globalServerCtxt.Layout.pools, globalServerCtxt.Layout.legacy)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	zipFile, err := os.Create("format.json.zip")
+	if err != nil {
+		log.Fatalf("failed to create format.json.zip: %v", err)
+	}
+	defer zipFile.Close()
+	fmtZipW := zip.NewWriter(zipFile)
+	defer fmtZipW.Close()
+	for _, pool := range pools { // for each pool
+		setCount, setDriveCount := pool.SetCount, pool.DrivesPerSet
+		format := newFormatErasureV3(setCount, setDriveCount)
+		format.ID = deploymentID
+		for i := 0; i < setCount; i++ { // for each erasure set
+			for j := 0; j < setDriveCount; j++ {
+				newFormat := format.Clone()
+				newFormat.Erasure.This = format.Erasure.Sets[i][j]
+				if deploymentID != "" {
+					newFormat.ID = deploymentID
+				}
+				drive := pool.Endpoints[i*setDriveCount+j]
+				fmtBytes, err := json.Marshal(newFormat)
+				if err != nil {
+					//nolint:gocritic
+					log.Fatalf("failed to marshal format.json for %s: %v", drive.String(), err)
+				}
+				fmtJSON := filepath.Join(drive.Host, drive.Path, minioMetaBucket, "format.json")
+				embedFileInZip(fmtZipW, fmtJSON, fmtBytes, 0o600)
+			}
+		}
+	}
+}

cmd/ftp-server-driver.go

@@ -114,7 +114,7 @@ func ftpTrace(s *ftp.Context, startTime time.Time, source, objPath string, err e
 		TraceType: madmin.TraceFTP,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
-		FuncName:  fmt.Sprintf(s.Cmd),
+		FuncName:  s.Cmd,
 		Duration:  time.Since(startTime),
 		Path:      objPath,
 		Error:     errStr,

cmd/generic-handlers.go

@@ -247,11 +247,14 @@ func guessIsRPCReq(req *http.Request) bool {
 	if req == nil {
 		return false
 	}
-	if req.Method == http.MethodGet && req.URL != nil && req.URL.Path == grid.RoutePath {
-		return true
+	if req.Method == http.MethodGet && req.URL != nil {
+		switch req.URL.Path {
+		case grid.RoutePath, grid.RouteLockPath:
+			return true
+		}
 	}
 
-	return req.Method == http.MethodPost &&
+	return (req.Method == http.MethodPost || req.Method == http.MethodGet) &&
 		strings.HasPrefix(req.URL.Path, minioReservedBucketPath+SlashSeparator)
 }
 
@@ -469,7 +472,7 @@ func setBucketForwardingMiddleware(h http.Handler) http.Handler {
 		}
 		if globalDNSConfig == nil || !globalBucketFederation ||
 			guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
-			guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) {
+			guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) || isKMSReq(r) {
 			h.ServeHTTP(w, r)
 			return
 		}

cmd/generic-handlers_test.go

@@ -51,9 +51,10 @@ func TestGuessIsRPC(t *testing.T) {
 	r = &http.Request{
 		Proto:  "HTTP/1.1",
 		Method: http.MethodGet,
+		URL:    u,
 	}
-	if guessIsRPCReq(r) {
-		t.Fatal("Test shouldn't report as net/rpc for a non net/rpc request.")
+	if !guessIsRPCReq(r) {
+		t.Fatal("Test shouldn't fail for a possible net/rpc request.")
 	}
 	r = &http.Request{
 		Proto:  "HTTP/1.1",
@@ -63,6 +64,14 @@ func TestGuessIsRPC(t *testing.T) {
 	if !guessIsRPCReq(r) {
 		t.Fatal("Grid RPC path not detected")
 	}
+	r = &http.Request{
+		Proto:  "HTTP/1.1",
+		Method: http.MethodGet,
+		URL:    &url.URL{Path: grid.RouteLockPath},
+	}
+	if !guessIsRPCReq(r) {
+		t.Fatal("Grid RPC path not detected")
+	}
 }
 
 var isHTTPHeaderSizeTooLargeTests = []struct {

cmd/global-heal.go

@@ -150,6 +150,11 @@ type healEntryResult struct {
 
 // healErasureSet lists and heals all objects in a specific erasure set
 func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, tracker *healingTracker) error {
+	bgSeq, found := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID)
+	if !found {
+		return errors.New("no local healing sequence initialized, unable to heal the drive")
+	}
+
 	scanMode := madmin.HealNormalScan
 
 	// Make sure to copy since `buckets slice`
@@ -162,12 +167,21 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		return errServerNotInitialized
 	}
 
+	started := tracker.Started
+	if started.IsZero() || started.Equal(timeSentinel) {
+		healingLogIf(ctx, fmt.Errorf("unexpected tracker healing start time found: %v", started))
+		started = time.Time{}
+	}
+
+	// Final tracer update before quitting
+	defer func() {
+		tracker.setObject("")
+		tracker.setBucket("")
+		healingLogIf(ctx, tracker.update(ctx))
+	}()
+
 	for _, bucket := range healBuckets {
-		_, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{
-			Recreate: true,
-			ScanMode: scanMode,
-		})
-		if err != nil {
+		if err := bgSeq.healBucket(objAPI, bucket, true); err != nil {
 			// Log bucket healing error if any, we shall retry again.
 			healingLogIf(ctx, err)
 		}
@@ -193,7 +207,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		numHealers = uint64(v)
 	}
 
-	healingLogEvent(ctx, fmt.Sprintf("Healing drive '%s' - use %d parallel workers.", tracker.disk.String(), numHealers))
+	healingLogEvent(ctx, "Healing drive '%s' - use %d parallel workers.", tracker.disk.String(), numHealers)
 
 	jt, _ := workers.New(int(numHealers))
 
@@ -226,7 +240,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 
 	// Collect updates to tracker from concurrent healEntry calls
 	results := make(chan healEntryResult, 1000)
-	defer close(results)
+	quitting := make(chan struct{})
+	defer func() {
+		close(results)
+		<-quitting
+	}()
 
 	go func() {
 		for res := range results {
@@ -240,6 +258,9 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 
 			tracker.updateProgress(res.success, res.skipped, res.bytes)
 		}
+
+		healingLogIf(ctx, tracker.update(ctx))
+		close(quitting)
 	}()
 
 	var retErr error
@@ -264,10 +285,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		tracker.setBucket(bucket)
 		// Heal current bucket again in case if it is failed
 		// in the beginning of erasure set healing
-		if _, err := objAPI.HealBucket(ctx, bucket, madmin.HealOpts{
-			Recreate: true,
-			ScanMode: scanMode,
-		}); err != nil {
+		if err := bgSeq.healBucket(objAPI, bucket, true); err != nil {
 			// Set this such that when we return this function
 			// we let the caller retry this disk again for the
 			// buckets that failed healing.
@@ -366,6 +384,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				}
 				return false
 			case results <- result:
+				bgSeq.countScanned(madmin.HealItemObject)
 				return true
 			}
 		}
@@ -416,8 +435,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 						return
 					}
 					result = healEntryFailure(0)
+					bgSeq.countFailed(madmin.HealItemObject)
 					healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err))
 				} else {
+					bgSeq.countHealed(madmin.HealItemObject)
 					result = healEntrySuccess(uint64(res.ObjectSize))
 				}
 
@@ -427,13 +448,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 
 			var versionNotFound int
 			for _, version := range fivs.Versions {
-				// Ignore a version with a modtime newer than healing start time.
-				if version.ModTime.After(tracker.Started) {
-					continue
-				}
-
-				// Apply lifecycle rules on the objects that are expired.
-				if filterLifecycle(bucket, version.Name, version) {
+				// Ignore healing a version if:
+				// - It is uploaded after the drive healing is started
+				// - An object that is already expired by ILM rule.
+				if !started.IsZero() && version.ModTime.After(started) || filterLifecycle(bucket, version.Name, version) {
 					versionNotFound++
 					if !send(healEntrySkipped(uint64(version.Size))) {
 						return
@@ -463,8 +481,10 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				}
 
 				if versionHealed {
+					bgSeq.countHealed(madmin.HealItemObject)
 					result = healEntrySuccess(uint64(version.Size))
 				} else {
+					bgSeq.countFailed(madmin.HealItemObject)
 					result = healEntryFailure(uint64(version.Size))
 					if version.VersionID != "" {
 						healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s-v(%s): %w", bucket, version.Name, version.VersionID, err))
@@ -546,10 +566,6 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			healingLogIf(ctx, tracker.update(ctx))
 		}
 	}
-
-	tracker.setObject("")
-	tracker.setBucket("")
-
 	if retErr != nil {
 		return retErr
 	}

cmd/globals.go

@@ -310,6 +310,7 @@ var (
 	globalBootTime = UTCNow()
 
 	globalActiveCred         auth.Credentials
+	globalNodeAuthToken      string
 	globalSiteReplicatorCred siteReplicatorCred
 
 	// Captures if root credentials are set via ENV.
@@ -383,9 +384,7 @@ var (
 	globalBackgroundHealRoutine = newHealRoutine()
 	globalBackgroundHealState   = newHealState(GlobalContext, false)
 
-	globalMRFState = mrfState{
-		opCh: make(chan partialOperation, mrfOpsQueueSize),
-	}
+	globalMRFState = newMRFState()
 
 	// If writes to FS backend should be O_SYNC.
 	globalFSOSync bool
@@ -413,10 +412,9 @@ var (
 	globalServiceFreezeCnt int32
 	globalServiceFreezeMu  sync.Mutex // Updates.
 
-	// List of local drives to this node, this is only set during server startup,
-	// and is only mutated by HealFormat. Hold globalLocalDrivesMu to access.
-	globalLocalDrives    []StorageAPI
-	globalLocalDrivesMap = make(map[string]StorageAPI)
+	// Map of local drives to this node, this is set during server startup,
+	// disk reconnect and mutated by HealFormat. Hold globalLocalDrivesMu to access.
+	globalLocalDrivesMap map[string]StorageAPI
 	globalLocalDrivesMu  sync.RWMutex
 
 	globalDriveMonitoring = env.Get("_MINIO_DRIVE_ACTIVE_MONITORING", config.EnableOn) == config.EnableOn

cmd/grid.go

@@ -31,9 +31,15 @@ import (
 // globalGrid is the global grid manager.
 var globalGrid atomic.Pointer[grid.Manager]
 
+// globalLockGrid is the global lock grid manager.
+var globalLockGrid atomic.Pointer[grid.Manager]
+
 // globalGridStart is a channel that will block startup of grid connections until closed.
 var globalGridStart = make(chan struct{})
 
+// globalLockGridStart is a channel that will block startup of lock grid connections until closed.
+var globalLockGridStart = make(chan struct{})
+
 func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
 	hosts, local := eps.GridHosts()
 	lookupHost := globalDNSCache.LookupHost
@@ -55,9 +61,10 @@ func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
 		AuthFn:       newCachedAuthToken(),
 		BlockConnect: globalGridStart,
 		// Record incoming and outgoing bytes.
-		Incoming: globalConnStats.incInternodeInputBytes,
-		Outgoing: globalConnStats.incInternodeOutputBytes,
-		TraceTo:  globalTrace,
+		Incoming:  globalConnStats.incInternodeInputBytes,
+		Outgoing:  globalConnStats.incInternodeOutputBytes,
+		TraceTo:   globalTrace,
+		RoutePath: grid.RoutePath,
 	})
 	if err != nil {
 		return err
@@ -65,3 +72,36 @@ func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
 	globalGrid.Store(g)
 	return nil
 }
+
+func initGlobalLockGrid(ctx context.Context, eps EndpointServerPools) error {
+	hosts, local := eps.GridHosts()
+	lookupHost := globalDNSCache.LookupHost
+	g, err := grid.NewManager(ctx, grid.ManagerOptions{
+		// Pass Dialer for websocket grid, make sure we do not
+		// provide any DriveOPTimeout() function, as that is not
+		// useful over persistent connections.
+		Dialer: grid.ConnectWSWithRoutePath(
+			grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))),
+			newCachedAuthToken(),
+			&tls.Config{
+				RootCAs:          globalRootCAs,
+				CipherSuites:     fips.TLSCiphers(),
+				CurvePreferences: fips.TLSCurveIDs(),
+			}, grid.RouteLockPath),
+		Local:        local,
+		Hosts:        hosts,
+		AuthToken:    validateStorageRequestToken,
+		AuthFn:       newCachedAuthToken(),
+		BlockConnect: globalGridStart,
+		// Record incoming and outgoing bytes.
+		Incoming:  globalConnStats.incInternodeInputBytes,
+		Outgoing:  globalConnStats.incInternodeOutputBytes,
+		TraceTo:   globalTrace,
+		RoutePath: grid.RouteLockPath,
+	})
+	if err != nil {
+		return err
+	}
+	globalLockGrid.Store(g)
+	return nil
+}

cmd/handler-api.go

@@ -39,7 +39,6 @@ import (
 type apiConfig struct {
 	mu sync.RWMutex
 
-	requestsDeadline       time.Duration
 	requestsPool           chan struct{}
 	clusterDeadline        time.Duration
 	listQuorum             string
@@ -164,15 +163,14 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) {
 		// but this shouldn't last long.
 		t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode)
 	}
-	t.requestsDeadline = cfg.RequestsDeadline
 	listQuorum := cfg.ListQuorum
 	if listQuorum == "" {
 		listQuorum = "strict"
 	}
 	t.listQuorum = listQuorum
-	if globalReplicationPool != nil &&
+	if r := globalReplicationPool.GetNonBlocking(); r != nil &&
 		(cfg.ReplicationPriority != t.replicationPriority || cfg.ReplicationMaxWorkers != t.replicationMaxWorkers || cfg.ReplicationMaxLWorkers != t.replicationMaxLWorkers) {
-		globalReplicationPool.ResizeWorkerPriority(cfg.ReplicationPriority, cfg.ReplicationMaxWorkers, cfg.ReplicationMaxLWorkers)
+		r.ResizeWorkerPriority(cfg.ReplicationPriority, cfg.ReplicationMaxWorkers, cfg.ReplicationMaxLWorkers)
 	}
 	t.replicationPriority = cfg.ReplicationPriority
 	t.replicationMaxWorkers = cfg.ReplicationMaxWorkers
@@ -290,19 +288,15 @@ func (t *apiConfig) getRequestsPoolCapacity() int {
 	return cap(t.requestsPool)
 }
 
-func (t *apiConfig) getRequestsPool() (chan struct{}, time.Duration) {
+func (t *apiConfig) getRequestsPool() chan struct{} {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
 	if t.requestsPool == nil {
-		return nil, 10 * time.Second
+		return nil
 	}
 
-	if t.requestsDeadline <= 0 {
-		return t.requestsPool, 10 * time.Second
-	}
-
-	return t.requestsPool, t.requestsDeadline
+	return t.requestsPool
 }
 
 // maxClients throttles the S3 API calls
@@ -325,27 +319,19 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 		}
 
 		globalHTTPStats.addRequestsInQueue(1)
-		pool, deadline := globalAPIConfig.getRequestsPool()
+		pool := globalAPIConfig.getRequestsPool()
 		if pool == nil {
 			globalHTTPStats.addRequestsInQueue(-1)
 			f.ServeHTTP(w, r)
 			return
 		}
 
-		// No deadline to wait, there is nothing to queue
-		// perform the API call immediately.
-		if deadline <= 0 {
-			globalHTTPStats.addRequestsInQueue(-1)
-			f.ServeHTTP(w, r)
-			return
-		}
-
 		if tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt); ok {
 			tc.FuncName = "s3.MaxClients"
 		}
 
-		deadlineTimer := time.NewTimer(deadline)
-		defer deadlineTimer.Stop()
+		w.Header().Set("X-RateLimit-Limit", strconv.Itoa(cap(pool)))
+		w.Header().Set("X-RateLimit-Remaining", strconv.Itoa(cap(pool)-len(pool)))
 
 		ctx := r.Context()
 		select {
@@ -357,7 +343,13 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 				return
 			}
 			f.ServeHTTP(w, r)
-		case <-deadlineTimer.C:
+		case <-r.Context().Done():
+			globalHTTPStats.addRequestsInQueue(-1)
+			// When the client disconnects before getting the S3 handler
+			// status code response, set the status code to 499 so this request
+			// will be properly audited and traced.
+			w.WriteHeader(499)
+		default:
 			globalHTTPStats.addRequestsInQueue(-1)
 			if contextCanceled(ctx) {
 				w.WriteHeader(499)
@@ -367,12 +359,7 @@ func maxClients(f http.HandlerFunc) http.HandlerFunc {
 			writeErrorResponse(ctx, w,
 				errorCodes.ToAPIErr(ErrTooManyRequests),
 				r.URL)
-		case <-r.Context().Done():
-			globalHTTPStats.addRequestsInQueue(-1)
-			// When the client disconnects before getting the S3 handler
-			// status code response, set the status code to 499 so this request
-			// will be properly audited and traced.
-			w.WriteHeader(499)
+
 		}
 	}
 }

cmd/http-stats.go

@@ -27,10 +27,6 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-const (
-	apiGetObject = "GetObject"
-)
-
 // connStats - Network statistics
 // Count total input/output transferred bytes during
 // the server's life.
@@ -132,7 +128,7 @@ func (bh *bucketHTTPStats) updateHTTPStats(bucket, api string, w *xhttp.Response
 		return
 	}
 
-	if w != nil && api == apiGetObject {
+	if w != nil {
 		// Increment the prometheus http request response histogram with API, Bucket
 		bucketHTTPRequestsDuration.With(prometheus.Labels{
 			"api":    api,
@@ -437,9 +433,7 @@ func (st *HTTPStats) updateStats(api string, w *xhttp.ResponseRecorder) {
 	st.totalS3Requests.Inc(api)
 
 	// Increment the prometheus http request response histogram with appropriate label
-	if api == apiGetObject {
-		httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TimeToFirstByte.Seconds())
-	}
+	httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TimeToFirstByte.Seconds())
 
 	code := w.StatusCode
 

cmd/iam-etcd-store.go

@@ -249,6 +249,18 @@ func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMU
 	return nil
 }
 
+func (ies *IAMEtcdStore) loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error) {
+	var u UserIdentity
+	err := ies.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return "", errNoSuchUser
+		}
+		return "", err
+	}
+	return u.Credentials.SecretKey, nil
+}
+
 func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
 	var u UserIdentity
 	err := ies.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))

cmd/iam-object-store.go

@@ -34,6 +34,7 @@ import (
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/v3/sync/errgroup"
 	"github.com/puzpuzpuz/xsync/v3"
 )
 
@@ -182,19 +183,20 @@ func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy
 	}
 }
 
-func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
+func (iamOS *IAMObjectStore) loadPolicy(ctx context.Context, policy string) (PolicyDoc, error) {
+	var p PolicyDoc
+
 	data, objInfo, err := iamOS.loadIAMConfigBytesWithMetadata(ctx, getPolicyDocPath(policy))
 	if err != nil {
 		if err == errConfigNotFound {
-			return errNoSuchPolicy
+			return p, errNoSuchPolicy
 		}
-		return err
+		return p, err
 	}
 
-	var p PolicyDoc
 	err = p.parseJSON(data)
 	if err != nil {
-		return err
+		return p, err
 	}
 
 	if p.Version == 0 {
@@ -205,6 +207,14 @@ func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m
 		p.UpdateDate = objInfo.ModTime
 	}
 
+	return p, nil
+}
+
+func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
+	p, err := iamOS.loadPolicy(ctx, policy)
+	if err != nil {
+		return err
+	}
 	m[policy] = p
 	return nil
 }
@@ -225,21 +235,33 @@ func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]Po
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
+func (iamOS *IAMObjectStore) loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error) {
+	var u UserIdentity
+	err := iamOS.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))
+	if err != nil {
+		if errors.Is(err, errConfigNotFound) {
+			return "", errNoSuchUser
+		}
+		return "", err
+	}
+	return u.Credentials.SecretKey, nil
+}
+
+func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string, userType IAMUserType) (UserIdentity, error) {
 	var u UserIdentity
 	err := iamOS.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType))
 	if err != nil {
 		if err == errConfigNotFound {
-			return errNoSuchUser
+			return u, errNoSuchUser
 		}
-		return err
+		return u, err
 	}
 
 	if u.Credentials.IsExpired() {
 		// Delete expired identity - ignoring errors here.
 		iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
 		iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
-		return nil
+		return u, errNoSuchUser
 	}
 
 	if u.Credentials.AccessKey == "" {
@@ -255,7 +277,7 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 				iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType))
 				iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false))
 			}
-			return nil
+			return u, errNoSuchUser
 
 		}
 		u.Credentials.Claims = jwtClaims.Map()
@@ -265,6 +287,35 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 		u.Credentials.Description = u.Credentials.Comment
 	}
 
+	return u, nil
+}
+
+func (iamOS *IAMObjectStore) loadUserConcurrent(ctx context.Context, userType IAMUserType, users ...string) ([]UserIdentity, error) {
+	userIdentities := make([]UserIdentity, len(users))
+	g := errgroup.WithNErrs(len(users))
+
+	for index := range users {
+		index := index
+		g.Go(func() error {
+			userName := path.Dir(users[index])
+			user, err := iamOS.loadUserIdentity(ctx, userName, userType)
+			if err != nil && !errors.Is(err, errNoSuchUser) {
+				return fmt.Errorf("unable to load the user `%s`: %w", userName, err)
+			}
+			userIdentities[index] = user
+			return nil
+		}, index)
+	}
+
+	err := errors.Join(g.Wait()...)
+	return userIdentities, err
+}
+
+func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error {
+	u, err := iamOS.loadUserIdentity(ctx, user, userType)
+	if err != nil {
+		return err
+	}
 	m[user] = u
 	return nil
 }
@@ -346,16 +397,44 @@ func (iamOS *IAMObjectStore) loadMappedPolicyWithRetry(ctx context.Context, name
 	}
 }
 
-func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
+func (iamOS *IAMObjectStore) loadMappedPolicyInternal(ctx context.Context, name string, userType IAMUserType, isGroup bool) (MappedPolicy, error) {
 	var p MappedPolicy
 	err := iamOS.loadIAMConfig(ctx, &p, getMappedPolicyPath(name, userType, isGroup))
 	if err != nil {
 		if err == errConfigNotFound {
-			return errNoSuchPolicy
+			return p, errNoSuchPolicy
 		}
-		return err
+		return p, err
+	}
+	return p, nil
+}
+
+func (iamOS *IAMObjectStore) loadMappedPolicyConcurrent(ctx context.Context, userType IAMUserType, isGroup bool, users ...string) ([]MappedPolicy, error) {
+	mappedPolicies := make([]MappedPolicy, len(users))
+	g := errgroup.WithNErrs(len(users))
+
+	for index := range users {
+		index := index
+		g.Go(func() error {
+			userName := strings.TrimSuffix(users[index], ".json")
+			userMP, err := iamOS.loadMappedPolicyInternal(ctx, userName, userType, isGroup)
+			if err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return fmt.Errorf("unable to load the user policy map `%s`: %w", userName, err)
+			}
+			mappedPolicies[index] = userMP
+			return nil
+		}, index)
 	}
 
+	err := errors.Join(g.Wait()...)
+	return mappedPolicies, err
+}
+
+func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error {
+	p, err := iamOS.loadMappedPolicyInternal(ctx, name, userType, isGroup)
+	if err != nil {
+		return err
+	}
 	m.Store(name, p)
 	return nil
 }
@@ -443,6 +522,27 @@ const (
 	maxIAMLoadOpTime = 5 * time.Second
 )
 
+func (iamOS *IAMObjectStore) loadPolicyDocConcurrent(ctx context.Context, policies ...string) ([]PolicyDoc, error) {
+	policyDocs := make([]PolicyDoc, len(policies))
+	g := errgroup.WithNErrs(len(policies))
+
+	for index := range policies {
+		index := index
+		g.Go(func() error {
+			policyName := path.Dir(policies[index])
+			policyDoc, err := iamOS.loadPolicy(ctx, policyName)
+			if err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return fmt.Errorf("unable to load the policy doc `%s`: %w", policyName, err)
+			}
+			policyDocs[index] = policyDoc
+			return nil
+		}, index)
+	}
+
+	err := errors.Join(g.Wait()...)
+	return policyDocs, err
+}
+
 // Assumes cache is locked by caller.
 func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache, firstTime bool) error {
 	bootstrapTraceMsgFirstTime := func(s string) {
@@ -478,12 +578,37 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 
 	policyLoadStartTime := UTCNow()
 	policiesList := listedConfigItems[policiesListKey]
-	for _, item := range policiesList {
-		policyName := path.Dir(item)
-		if err := iamOS.loadPolicyDoc(ctx, policyName, cache.iamPolicyDocsMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return fmt.Errorf("unable to load the policy doc `%s`: %w", policyName, err)
+	count := 32 // number of parallel IAM loaders
+	for {
+		if len(policiesList) < count {
+			policyDocs, err := iamOS.loadPolicyDocConcurrent(ctx, policiesList...)
+			if err != nil {
+				return err
+			}
+			for index := range policiesList {
+				if policyDocs[index].Policy.Version != "" {
+					policyName := path.Dir(policiesList[index])
+					cache.iamPolicyDocsMap[policyName] = policyDocs[index]
+				}
+			}
+			break
 		}
+
+		policyDocs, err := iamOS.loadPolicyDocConcurrent(ctx, policiesList[:count]...)
+		if err != nil {
+			return err
+		}
+
+		for index := range policiesList[:count] {
+			if policyDocs[index].Policy.Version != "" {
+				policyName := path.Dir(policiesList[index])
+				cache.iamPolicyDocsMap[policyName] = policyDocs[index]
+			}
+		}
+
+		policiesList = policiesList[count:]
 	}
+
 	if took := time.Since(policyLoadStartTime); took > maxIAMLoadOpTime {
 		logger.Info("Policy docs load took %.2fs (for %d items)", took.Seconds(), len(policiesList))
 	}
@@ -492,12 +617,37 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 		bootstrapTraceMsgFirstTime("loading regular IAM users")
 		regUsersLoadStartTime := UTCNow()
 		regUsersList := listedConfigItems[usersListKey]
-		for _, item := range regUsersList {
-			userName := path.Dir(item)
-			if err := iamOS.loadUser(ctx, userName, regUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
-				return fmt.Errorf("unable to load the user: %w", err)
+
+		for {
+			if len(regUsersList) < count {
+				users, err := iamOS.loadUserConcurrent(ctx, regUser, regUsersList...)
+				if err != nil {
+					return err
+				}
+				for index := range regUsersList {
+					if users[index].Credentials.AccessKey != "" {
+						userName := path.Dir(regUsersList[index])
+						cache.iamUsersMap[userName] = users[index]
+					}
+				}
+				break
 			}
+
+			users, err := iamOS.loadUserConcurrent(ctx, regUser, regUsersList[:count]...)
+			if err != nil {
+				return err
+			}
+
+			for index := range regUsersList[:count] {
+				if users[index].Credentials.AccessKey != "" {
+					userName := path.Dir(regUsersList[index])
+					cache.iamUsersMap[userName] = users[index]
+				}
+			}
+
+			regUsersList = regUsersList[count:]
 		}
+
 		if took := time.Since(regUsersLoadStartTime); took > maxIAMLoadOpTime {
 			actualLoaded := len(cache.iamUsersMap)
 			logger.Info("Reg. users load took %.2fs (for %d items with %d expired items)", took.Seconds(),
@@ -521,12 +671,38 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	bootstrapTraceMsgFirstTime("loading user policy mapping")
 	userPolicyMappingLoadStartTime := UTCNow()
 	userPolicyMappingsList := listedConfigItems[policyDBUsersListKey]
-	for _, item := range userPolicyMappingsList {
-		userName := strings.TrimSuffix(item, ".json")
-		if err := iamOS.loadMappedPolicy(ctx, userName, regUser, false, cache.iamUserPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return fmt.Errorf("unable to load the policy mapping for the user: %w", err)
+	for {
+		if len(userPolicyMappingsList) < count {
+			mappedPolicies, err := iamOS.loadMappedPolicyConcurrent(ctx, regUser, false, userPolicyMappingsList...)
+			if err != nil {
+				return err
+			}
+
+			for index := range userPolicyMappingsList {
+				if mappedPolicies[index].Policies != "" {
+					userName := strings.TrimSuffix(userPolicyMappingsList[index], ".json")
+					cache.iamUserPolicyMap.Store(userName, mappedPolicies[index])
+				}
+			}
+
+			break
 		}
+
+		mappedPolicies, err := iamOS.loadMappedPolicyConcurrent(ctx, regUser, false, userPolicyMappingsList[:count]...)
+		if err != nil {
+			return err
+		}
+
+		for index := range userPolicyMappingsList[:count] {
+			if mappedPolicies[index].Policies != "" {
+				userName := strings.TrimSuffix(userPolicyMappingsList[index], ".json")
+				cache.iamUserPolicyMap.Store(userName, mappedPolicies[index])
+			}
+		}
+
+		userPolicyMappingsList = userPolicyMappingsList[count:]
 	}
+
 	if took := time.Since(userPolicyMappingLoadStartTime); took > maxIAMLoadOpTime {
 		logger.Info("User policy mappings load took %.2fs (for %d items)", took.Seconds(), len(userPolicyMappingsList))
 	}
@@ -605,21 +781,16 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range listedConfigItems[stsListKey] {
 		userName := path.Dir(item)
 		// loadUser() will delete expired user during the load.
-		err := iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore)
-		if err != nil && !errors.Is(err, errNoSuchUser) {
-			return fmt.Errorf("unable to load user during STS purge: %w (%s)", err, item)
-		}
-
+		iamLogIf(ctx, iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore))
+		// No need to return errors for failed expiration of STS users
 	}
+
 	// Loading the STS policy mappings from disk ensures that stale entries
 	// (removed during loadUser() in the loop above) are removed from memory.
 	for _, item := range listedConfigItems[policyDBSTSUsersListKey] {
 		stsName := strings.TrimSuffix(item, ".json")
-		err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore)
-		if err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return fmt.Errorf("unable to load policies during STS purge: %w (%s)", err, item)
-		}
-
+		iamLogIf(ctx, iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore))
+		// No need to return errors for failed expiration of STS users
 	}
 
 	took := time.Since(purgeStart).Seconds()
@@ -631,7 +802,11 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	// Store the newly populated map in the iam cache. This takes care of
 	// removing stale entries from the existing map.
 	cache.iamSTSAccountsMap = stsAccountsFromStore
-	cache.iamSTSPolicyMap = stsAccPoliciesFromStore
+
+	stsAccPoliciesFromStore.Range(func(k string, v MappedPolicy) bool {
+		cache.iamSTSPolicyMap.Store(k, v)
+		return true
+	})
 
 	return nil
 }

cmd/iam-store.go

@@ -23,6 +23,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"path"
 	"sort"
 	"strings"
 	"time"
@@ -31,9 +32,10 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/jwt"
-	"github.com/minio/pkg/v3/console"
+	"github.com/minio/pkg/v3/env"
 	"github.com/minio/pkg/v3/policy"
 	"github.com/puzpuzpuz/xsync/v3"
 	"golang.org/x/sync/singleflight"
@@ -366,7 +368,7 @@ func (c *iamCache) removeGroupFromMembershipsMap(group string) {
 // information in IAM (i.e sys.iam*Map) - this info is stored only in the STS
 // generated credentials. Thus we skip looking up group memberships, user map,
 // and group map and check the appropriate policy maps directly.
-func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([]string, time.Time, error) {
+func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool, policyPresent bool) ([]string, time.Time, error) {
 	if isGroup {
 		if store.getUsersSysType() == MinIOUsersSysType {
 			g, ok := c.iamGroupsMap[name]
@@ -391,105 +393,107 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 		if ok {
 			return policy.toSlice(), policy.UpdatedAt, nil
 		}
-		if err := store.loadMappedPolicyWithRetry(context.TODO(), name, regUser, true, c.iamGroupPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return nil, time.Time{}, err
-		}
-		policy, _ = c.iamGroupPolicyMap.Load(name)
-		return policy.toSlice(), policy.UpdatedAt, nil
-	}
-
-	// When looking for a user's policies, we also check if the user
-	// and the groups they are member of are enabled.
-	u, ok := c.iamUsersMap[name]
-	if ok {
-		if !u.Credentials.IsValid() {
-			return nil, time.Time{}, nil
-		}
-	}
-
-	// For internal IDP regular/service account user accounts, the policy
-	// mapping is iamUserPolicyMap. For STS accounts, the parent user would be
-	// passed here and we lookup the mapping in iamSTSPolicyMap.
-	mp, ok := c.iamUserPolicyMap.Load(name)
-	if !ok {
-		if err := store.loadMappedPolicyWithRetry(context.TODO(), name, regUser, false, c.iamUserPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return nil, time.Time{}, err
-		}
-
-		mp, ok = c.iamUserPolicyMap.Load(name)
-		if !ok {
-			// Since user "name" could be a parent user of an STS account, we look up
-			// mappings for those too.
-			mp, ok = c.iamSTSPolicyMap.Load(name)
-			if !ok {
-				// Attempt to load parent user mapping for STS accounts
-				if err := store.loadMappedPolicyWithRetry(context.TODO(), name, stsUser, false, c.iamSTSPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
-					return nil, time.Time{}, err
-				}
-				mp, _ = c.iamSTSPolicyMap.Load(name)
+		if !policyPresent {
+			if err := store.loadMappedPolicy(context.TODO(), name, regUser, true, c.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return nil, time.Time{}, err
 			}
+			policy, _ = c.iamGroupPolicyMap.Load(name)
+			return policy.toSlice(), policy.UpdatedAt, nil
 		}
+		return nil, time.Time{}, nil
 	}
 
 	// returned policy could be empty, we use set to de-duplicate.
-	policies := set.CreateStringSet(mp.toSlice()...)
+	var policies set.StringSet
+	var updatedAt time.Time
 
-	for _, group := range u.Credentials.Groups {
-		if store.getUsersSysType() == MinIOUsersSysType {
-			g, ok := c.iamGroupsMap[group]
-			if !ok {
-				if err := store.loadGroup(context.Background(), group, c.iamGroupsMap); err != nil {
-					return nil, time.Time{}, err
-				}
-				g, ok = c.iamGroupsMap[group]
-				if !ok {
-					return nil, time.Time{}, errNoSuchGroup
-				}
+	if store.getUsersSysType() == LDAPUsersSysType {
+		// For LDAP policy mapping is part of STS users, we only need to lookup
+		// those mappings.
+		mp, ok := c.iamSTSPolicyMap.Load(name)
+		if !ok {
+			// Attempt to load parent user mapping for STS accounts
+			if err := store.loadMappedPolicy(context.TODO(), name, stsUser, false, c.iamSTSPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return nil, time.Time{}, err
 			}
-
-			// Group is disabled, so we return no policy - this
-			// ensures the request is denied.
-			if g.Status == statusDisabled {
+			mp, _ = c.iamSTSPolicyMap.Load(name)
+		}
+		policies = set.CreateStringSet(mp.toSlice()...)
+		updatedAt = mp.UpdatedAt
+	} else {
+		// When looking for a user's policies, we also check if the user
+		// and the groups they are member of are enabled.
+		u, ok := c.iamUsersMap[name]
+		if ok {
+			if !u.Credentials.IsValid() {
 				return nil, time.Time{}, nil
 			}
 		}
 
-		policy, ok := c.iamGroupPolicyMap.Load(group)
+		// For internal IDP regular/service account user accounts, the policy
+		// mapping is iamUserPolicyMap. For STS accounts, the parent user would be
+		// passed here and we lookup the mapping in iamSTSPolicyMap.
+		mp, ok := c.iamUserPolicyMap.Load(name)
 		if !ok {
-			if err := store.loadMappedPolicyWithRetry(context.TODO(), group, regUser, true, c.iamGroupPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+			if err := store.loadMappedPolicy(context.TODO(), name, regUser, false, c.iamUserPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
 				return nil, time.Time{}, err
 			}
-			policy, _ = c.iamGroupPolicyMap.Load(group)
+			mp, ok = c.iamUserPolicyMap.Load(name)
+			if !ok {
+				// Since user "name" could be a parent user of an STS account, we look up
+				// mappings for those too.
+				mp, ok = c.iamSTSPolicyMap.Load(name)
+				if !ok {
+					// Attempt to load parent user mapping for STS accounts
+					if err := store.loadMappedPolicy(context.TODO(), name, stsUser, false, c.iamSTSPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
+						return nil, time.Time{}, err
+					}
+					mp, _ = c.iamSTSPolicyMap.Load(name)
+				}
+			}
 		}
+		policies = set.CreateStringSet(mp.toSlice()...)
 
-		for _, p := range policy.toSlice() {
-			policies.Add(p)
+		for _, group := range u.Credentials.Groups {
+			g, ok := c.iamGroupsMap[group]
+			if ok {
+				// Group is disabled, so we return no policy - this
+				// ensures the request is denied.
+				if g.Status == statusDisabled {
+					return nil, time.Time{}, nil
+				}
+			}
+
+			policy, ok := c.iamGroupPolicyMap.Load(group)
+			if !ok {
+				if err := store.loadMappedPolicy(context.TODO(), group, regUser, true, c.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
+					return nil, time.Time{}, err
+				}
+				policy, _ = c.iamGroupPolicyMap.Load(group)
+			}
+
+			for _, p := range policy.toSlice() {
+				policies.Add(p)
+			}
 		}
+		updatedAt = mp.UpdatedAt
 	}
 
 	for _, group := range c.iamUserGroupMemberships[name].ToSlice() {
 		if store.getUsersSysType() == MinIOUsersSysType {
 			g, ok := c.iamGroupsMap[group]
-			if !ok {
-				if err := store.loadGroup(context.Background(), group, c.iamGroupsMap); err != nil {
-					return nil, time.Time{}, err
+			if ok {
+				// Group is disabled, so we return no policy - this
+				// ensures the request is denied.
+				if g.Status == statusDisabled {
+					return nil, time.Time{}, nil
 				}
-				g, ok = c.iamGroupsMap[group]
-				if !ok {
-					return nil, time.Time{}, errNoSuchGroup
-				}
-			}
-
-			// Group is disabled, so we return no policy - this
-			// ensures the request is denied.
-			if g.Status == statusDisabled {
-				return nil, time.Time{}, nil
 			}
 		}
 
 		policy, ok := c.iamGroupPolicyMap.Load(group)
 		if !ok {
-			if err := store.loadMappedPolicyWithRetry(context.TODO(), group, regUser, true, c.iamGroupPolicyMap, 3); err != nil && !errors.Is(err, errNoSuchPolicy) {
+			if err := store.loadMappedPolicy(context.TODO(), group, regUser, true, c.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
 				return nil, time.Time{}, err
 			}
 			policy, _ = c.iamGroupPolicyMap.Load(group)
@@ -500,7 +504,7 @@ func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([
 		}
 	}
 
-	return policies.ToSlice(), mp.UpdatedAt, nil
+	return policies.ToSlice(), updatedAt, nil
 }
 
 func (c *iamCache) updateUserWithClaims(key string, u UserIdentity) error {
@@ -535,6 +539,7 @@ type IAMStorageAPI interface {
 	loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, retries int) error
 	loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error
 	loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error
+	loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error)
 	loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error
 	loadGroup(ctx context.Context, group string, m map[string]GroupInfo) error
 	loadGroups(ctx context.Context, m map[string]GroupInfo) error
@@ -641,7 +646,7 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) erro
 	// An in-memory cache must be replaced only if we know for sure that the
 	// values loaded from disk are not stale. They might be stale if the
 	// cached.updatedAt is more recent than the refresh cycle began.
-	if cache.updatedAt.Before(loadedAt) {
+	if cache.updatedAt.Before(loadedAt) || firstTime {
 		// No one has updated anything since the config was loaded,
 		// so we just replace whatever is on the disk into memory.
 		cache.iamGroupPolicyMap = newCache.iamGroupPolicyMap
@@ -752,13 +757,14 @@ func (store *IAMStoreSys) PolicyDBGet(name string, groups ...string) ([]string,
 	cache := store.rlock()
 	defer store.runlock()
 
-	policies, _, err := cache.policyDBGet(store, name, false)
+	policies, _, err := cache.policyDBGet(store, name, false, false)
 	if err != nil {
 		return nil, err
 	}
 
+	userPolicyPresent := len(policies) > 0
 	for _, group := range groups {
-		ps, _, err := cache.policyDBGet(store, group, true)
+		ps, _, err := cache.policyDBGet(store, group, true, userPolicyPresent)
 		if err != nil {
 			return nil, err
 		}
@@ -943,7 +949,7 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
 	cache := store.rlock()
 	defer store.runlock()
 
-	ps, updatedAt, err := cache.policyDBGet(store, group, true)
+	ps, updatedAt, err := cache.policyDBGet(store, group, true, false)
 	if err != nil {
 		return gd, err
 	}
@@ -972,34 +978,46 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
 	}, nil
 }
 
+// updateGroups updates the group from the persistent store, and also related policy mapping if any.
 func (store *IAMStoreSys) updateGroups(ctx context.Context, cache *iamCache) (res []string, err error) {
-	if store.getUsersSysType() == MinIOUsersSysType {
-		m := map[string]GroupInfo{}
-		err = store.loadGroups(ctx, m)
+	if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok {
+		listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
 		if err != nil {
-			return
+			return nil, err
 		}
-		cache.iamGroupsMap = m
-		cache.updatedAt = time.Now()
-		for k := range cache.iamGroupsMap {
-			res = append(res, k)
+		if store.getUsersSysType() == MinIOUsersSysType {
+			groupsList := listedConfigItems[groupsListKey]
+			for _, item := range groupsList {
+				group := path.Dir(item)
+				if err = iamOS.loadGroup(ctx, group, cache.iamGroupsMap); err != nil && !errors.Is(err, errNoSuchGroup) {
+					return nil, fmt.Errorf("unable to load the group: %w", err)
+				}
+				res = append(res, group)
+			}
 		}
+
+		groupPolicyMappingsList := listedConfigItems[policyDBGroupsListKey]
+		for _, item := range groupPolicyMappingsList {
+			group := strings.TrimSuffix(item, ".json")
+			if err = iamOS.loadMappedPolicy(ctx, group, regUser, true, cache.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return nil, fmt.Errorf("unable to load the policy mapping for the group: %w", err)
+			}
+			res = append(res, group)
+		}
+
+		return res, nil
 	}
 
-	if store.getUsersSysType() == LDAPUsersSysType {
-		m := xsync.NewMapOf[string, MappedPolicy]()
-		err = store.loadMappedPolicies(ctx, stsUser, true, m)
-		if err != nil {
-			return
-		}
-		cache.iamGroupPolicyMap = m
-		cache.updatedAt = time.Now()
-		cache.iamGroupPolicyMap.Range(func(k string, v MappedPolicy) bool {
-			res = append(res, k)
-			return true
-		})
+	// For etcd just return from cache.
+	for k := range cache.iamGroupsMap {
+		res = append(res, k)
 	}
 
+	cache.iamGroupPolicyMap.Range(func(k string, v MappedPolicy) bool {
+		res = append(res, k)
+		return true
+	})
+
 	return res, nil
 }
 
@@ -1862,8 +1880,12 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 					delete(cache.iamUsersMap, u.AccessKey)
 				case u.IsTemp():
 					_ = store.deleteUserIdentity(ctx, u.AccessKey, stsUser)
+					delete(cache.iamSTSAccountsMap, u.AccessKey)
 					delete(cache.iamUsersMap, u.AccessKey)
 				}
+				if store.group != nil {
+					store.group.Forget(u.AccessKey)
+				}
 			}
 		}
 	}
@@ -1877,8 +1899,13 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 		// ignore if user is already deleted.
 		err = nil
 	}
+	if userType == stsUser {
+		delete(cache.iamSTSAccountsMap, accessKey)
+	}
 	delete(cache.iamUsersMap, accessKey)
-	store.group.Forget(accessKey)
+	if store.group != nil {
+		store.group.Forget(accessKey)
+	}
 
 	cache.updatedAt = time.Now()
 
@@ -1953,8 +1980,13 @@ func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error
 			// we are only logging errors, not handling them.
 			err := store.deleteUserIdentity(ctx, user, userType)
 			iamLogIf(GlobalContext, err)
+			if userType == stsUser {
+				delete(cache.iamSTSAccountsMap, user)
+			}
 			delete(cache.iamUsersMap, user)
-			store.group.Forget(user)
+			if store.group != nil {
+				store.group.Forget(user)
+			}
 
 			deleted = true
 		}
@@ -2725,115 +2757,161 @@ func (store *IAMStoreSys) UpdateUserIdentity(ctx context.Context, cred auth.Cred
 
 // LoadUser - attempts to load user info from storage and updates cache.
 func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) error {
-	// We use singleflight to de-duplicate requests when server
-	// is coming up and loading accessKey and its associated assets
-	val, err, shared := store.group.Do(accessKey, func() (val interface{}, err error) {
-		cache := store.lock()
-		defer func() {
-			cache.updatedAt = time.Now()
-			store.unlock()
-		}()
+	groupLoad := env.Get("_MINIO_IAM_GROUP_REFRESH", config.EnableOff) == config.EnableOn
 
-		_, found := cache.iamUsersMap[accessKey]
+	newCachePopulate := func() (val interface{}, err error) {
+		newCache := newIamCache()
 
-		// Check for regular user access key
-		if !found {
-			err = store.loadUser(ctx, accessKey, regUser, cache.iamUsersMap)
-			if _, found = cache.iamUsersMap[accessKey]; found {
-				// load mapped policies
-				err = store.loadMappedPolicyWithRetry(ctx, accessKey, regUser, false, cache.iamUserPolicyMap, 3)
+		// Check for service account first
+		store.loadUser(ctx, accessKey, svcUser, newCache.iamUsersMap)
+
+		svc, found := newCache.iamUsersMap[accessKey]
+		if found {
+			// Load parent user and mapped policies.
+			if store.getUsersSysType() == MinIOUsersSysType {
+				err = store.loadUser(ctx, svc.Credentials.ParentUser, regUser, newCache.iamUsersMap)
+				// NOTE: we are not worried about loading errors from policies.
+				store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, regUser, false, newCache.iamUserPolicyMap, 3)
+			} else {
+				// In case of LDAP the parent user's policy mapping needs to be loaded into sts map
+				// NOTE: we are not worried about loading errors from policies.
+				store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, stsUser, false, newCache.iamSTSPolicyMap, 3)
 			}
 		}
 
-		// Check for service account
 		if !found {
-			err = store.loadUser(ctx, accessKey, svcUser, cache.iamUsersMap)
-			var svc UserIdentity
-			svc, found = cache.iamUsersMap[accessKey]
-			if found {
-				// Load parent user and mapped policies.
-				if store.getUsersSysType() == MinIOUsersSysType {
-					err = store.loadUser(ctx, svc.Credentials.ParentUser, regUser, cache.iamUsersMap)
-					if err == nil {
-						err = store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, regUser, false, cache.iamUserPolicyMap, 3)
-					}
-				} else {
-					// In case of LDAP the parent user's policy mapping needs to be
-					// loaded into sts map
-					err = store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap, 3)
-				}
+			err = store.loadUser(ctx, accessKey, regUser, newCache.iamUsersMap)
+			if _, found = newCache.iamUsersMap[accessKey]; found {
+				// NOTE: we are not worried about loading errors from policies.
+				store.loadMappedPolicyWithRetry(ctx, accessKey, regUser, false, newCache.iamUserPolicyMap, 3)
 			}
 		}
 
 		// Check for STS account
-		stsAccountFound := false
 		var stsUserCred UserIdentity
 		if !found {
-			err = store.loadUser(ctx, accessKey, stsUser, cache.iamSTSAccountsMap)
-			if stsUserCred, found = cache.iamSTSAccountsMap[accessKey]; found {
+			err = store.loadUser(ctx, accessKey, stsUser, newCache.iamSTSAccountsMap)
+			if stsUserCred, found = newCache.iamSTSAccountsMap[accessKey]; found {
 				// Load mapped policy
-				err = store.loadMappedPolicyWithRetry(ctx, stsUserCred.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap, 3)
-				stsAccountFound = true
+				// NOTE: we are not worried about loading errors from policies.
+				store.loadMappedPolicyWithRetry(ctx, stsUserCred.Credentials.ParentUser, stsUser, false, newCache.iamSTSPolicyMap, 3)
 			}
 		}
 
 		// Load any associated policy definitions
-		if !stsAccountFound {
-			pols, _ := cache.iamUserPolicyMap.Load(accessKey)
-			for _, policy := range pols.toSlice() {
-				if _, found = cache.iamPolicyDocsMap[policy]; !found {
-					err = store.loadPolicyDocWithRetry(ctx, policy, cache.iamPolicyDocsMap, 3)
-				}
-			}
-		} else {
-			pols, _ := cache.iamSTSPolicyMap.Load(stsUserCred.Credentials.AccessKey)
-			for _, policy := range pols.toSlice() {
-				if _, found = cache.iamPolicyDocsMap[policy]; !found {
-					err = store.loadPolicyDocWithRetry(ctx, policy, cache.iamPolicyDocsMap, 3)
-				}
+		pols, _ := newCache.iamUserPolicyMap.Load(accessKey)
+		for _, policy := range pols.toSlice() {
+			if _, found = newCache.iamPolicyDocsMap[policy]; !found {
+				// NOTE: we are not worried about loading errors from policies.
+				store.loadPolicyDocWithRetry(ctx, policy, newCache.iamPolicyDocsMap, 3)
 			}
 		}
 
-		load := len(cache.iamGroupsMap) == 0
-		if store.getUsersSysType() == LDAPUsersSysType && cache.iamGroupPolicyMap.Size() == 0 {
-			load = true
-		}
-		if load {
-			if _, err = store.updateGroups(ctx, cache); err != nil {
-				return "done", err
+		pols, _ = newCache.iamSTSPolicyMap.Load(stsUserCred.Credentials.AccessKey)
+		for _, policy := range pols.toSlice() {
+			if _, found = newCache.iamPolicyDocsMap[policy]; !found {
+				// NOTE: we are not worried about loading errors from policies.
+				store.loadPolicyDocWithRetry(ctx, policy, newCache.iamPolicyDocsMap, 3)
 			}
 		}
 
-		cache.buildUserGroupMemberships()
+		if groupLoad {
+			// NOTE: we are not worried about loading errors from groups.
+			store.updateGroups(ctx, newCache)
+			newCache.buildUserGroupMemberships()
+		}
 
-		return "done", err
+		return newCache, err
+	}
+
+	var (
+		val interface{}
+		err error
+	)
+	if store.group != nil {
+		val, err, _ = store.group.Do(accessKey, newCachePopulate)
+	} else {
+		val, err = newCachePopulate()
+	}
+
+	// Return error right away if any.
+	if err != nil {
+		if errors.Is(err, errNoSuchUser) || errors.Is(err, errConfigNotFound) {
+			return nil
+		}
+		return err
+	}
+
+	newCache := val.(*iamCache)
+
+	cache := store.lock()
+	defer store.unlock()
+
+	// We need to merge the new cache with the existing cache because the
+	// periodic IAM reload is partial. The periodic load here is to account.
+	newCache.iamGroupPolicyMap.Range(func(k string, v MappedPolicy) bool {
+		cache.iamGroupPolicyMap.Store(k, v)
+		return true
 	})
 
-	if serverDebugLog {
-		console.Debugln("loadUser: loading shared", val, err, shared)
+	for k, v := range newCache.iamGroupsMap {
+		cache.iamGroupsMap[k] = v
 	}
 
-	if IsErr(err, errNoSuchUser, errNoSuchPolicy, errNoSuchGroup) {
-		return nil
+	for k, v := range newCache.iamPolicyDocsMap {
+		cache.iamPolicyDocsMap[k] = v
 	}
 
-	return err
+	for k, v := range newCache.iamUserGroupMemberships {
+		cache.iamUserGroupMemberships[k] = v
+	}
+
+	newCache.iamUserPolicyMap.Range(func(k string, v MappedPolicy) bool {
+		cache.iamUserPolicyMap.Store(k, v)
+		return true
+	})
+
+	for k, v := range newCache.iamUsersMap {
+		cache.iamUsersMap[k] = v
+	}
+
+	for k, v := range newCache.iamSTSAccountsMap {
+		cache.iamSTSAccountsMap[k] = v
+	}
+
+	newCache.iamSTSPolicyMap.Range(func(k string, v MappedPolicy) bool {
+		cache.iamSTSPolicyMap.Store(k, v)
+		return true
+	})
+
+	cache.updatedAt = time.Now()
+
+	return nil
 }
 
-func extractJWTClaims(u UserIdentity) (*jwt.MapClaims, error) {
-	jwtClaims, err := auth.ExtractClaims(u.Credentials.SessionToken, u.Credentials.SecretKey)
-	if err != nil {
-		secretKey, err := getTokenSigningKey()
-		if err != nil {
-			return nil, err
+func extractJWTClaims(u UserIdentity) (jwtClaims *jwt.MapClaims, err error) {
+	keys := make([]string, 0, 3)
+	// Append credentials secret key itself
+	keys = append(keys, u.Credentials.SecretKey)
+	// Use site-replication credentials if found
+	if globalSiteReplicationSys.isEnabled() {
+		siteReplSecretKey, e := globalSiteReplicatorCred.Get(GlobalContext)
+		if e != nil {
+			return nil, e
 		}
-		// Session tokens for STS creds will be generated with root secret or site-replicator-0 secret
-		jwtClaims, err = auth.ExtractClaims(u.Credentials.SessionToken, secretKey)
-		if err != nil {
-			return nil, err
+		keys = append(keys, siteReplSecretKey)
+	}
+	// Use root credentials for credentials created with older deployments
+	keys = append(keys, globalActiveCred.SecretKey)
+
+	// Iterate over all keys and return with the first successful claim extraction
+	for _, key := range keys {
+		jwtClaims, err = auth.ExtractClaims(u.Credentials.SessionToken, key)
+		if err == nil {
+			break
 		}
 	}
-	return jwtClaims, nil
+	return
 }
 
 func validateSvcExpirationInUTC(expirationInUTC time.Time) error {

cmd/iam.go

@@ -46,9 +46,9 @@ import (
 	"github.com/minio/minio/internal/config/policy/opa"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	xhttp "github.com/minio/minio/internal/http"
-	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/v3/env"
 	"github.com/minio/pkg/v3/ldap"
 	"github.com/minio/pkg/v3/policy"
 	etcd "go.etcd.io/etcd/client/v3"
@@ -178,9 +178,16 @@ func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) {
 	}
 
 	if etcdClient == nil {
-		sys.store = &IAMStoreSys{newIAMObjectStore(objAPI, sys.usersSysType), &singleflight.Group{}}
+		var group *singleflight.Group
+		if env.Get("_MINIO_IAM_SINGLE_FLIGHT", config.EnableOn) == config.EnableOn {
+			group = &singleflight.Group{}
+		}
+		sys.store = &IAMStoreSys{
+			IAMStorageAPI: newIAMObjectStore(objAPI, sys.usersSysType),
+			group:         group,
+		}
 	} else {
-		sys.store = &IAMStoreSys{newIAMEtcdStore(etcdClient, sys.usersSysType), &singleflight.Group{}}
+		sys.store = &IAMStoreSys{IAMStorageAPI: newIAMEtcdStore(etcdClient, sys.usersSysType)}
 	}
 }
 
@@ -211,7 +218,7 @@ func (sys *IAMSys) Load(ctx context.Context, firstTime bool) error {
 	if !globalSiteReplicatorCred.IsValid() {
 		sa, _, err := sys.getServiceAccount(ctx, siteReplicatorSvcAcc)
 		if err == nil {
-			globalSiteReplicatorCred.Set(sa.Credentials)
+			globalSiteReplicatorCred.Set(sa.Credentials.SecretKey)
 		}
 	}
 
@@ -225,7 +232,7 @@ func (sys *IAMSys) Load(ctx context.Context, firstTime bool) error {
 	select {
 	case <-sys.configLoaded:
 	default:
-		xioutil.SafeClose(sys.configLoaded)
+		close(sys.configLoaded)
 	}
 	return nil
 }
@@ -307,8 +314,9 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		// Migrate IAM configuration, if necessary.
 		if err := saveIAMFormat(retryCtx, sys.store); err != nil {
 			if configRetriableErrors(err) {
-				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v)", err)
-				time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
+				retryInterval := time.Duration(r.Float64() * float64(time.Second))
+				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v) (retrying in %s)", err, retryInterval)
+				time.Sleep(retryInterval)
 				continue
 			}
 			iamLogIf(ctx, fmt.Errorf("IAM sub-system is partially initialized, unable to write the IAM format: %w", err), logger.WarningKind)
@@ -340,8 +348,9 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	for {
 		if err := sys.Load(retryCtx, true); err != nil {
 			if configRetriableErrors(err) {
-				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v)", err)
-				time.Sleep(time.Duration(r.Float64() * float64(time.Second)))
+				retryInterval := time.Duration(r.Float64() * float64(time.Second))
+				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v) (retrying in %s)", err, retryInterval)
+				time.Sleep(retryInterval)
 				continue
 			}
 			if err != nil {
@@ -408,7 +417,7 @@ func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Durat
 				took := time.Since(refreshStart).Seconds()
 				if took > maxDurationSecondsForLog {
 					// Log if we took a lot of time to load.
-					logger.Info("IAM refresh took (duration: %s)", took)
+					logger.Info("IAM refresh took (duration: %.2fs)", took)
 				}
 			}
 
@@ -1212,9 +1221,18 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (Use
 
 // GetTemporaryAccount - wrapper method to get information about a temporary account
 func (sys *IAMSys) GetTemporaryAccount(ctx context.Context, accessKey string) (auth.Credentials, *policy.Policy, error) {
+	if !sys.Initialized() {
+		return auth.Credentials{}, nil, errServerNotInitialized
+	}
 	tmpAcc, embeddedPolicy, err := sys.getTempAccount(ctx, accessKey)
 	if err != nil {
-		return auth.Credentials{}, nil, err
+		if err == errNoSuchTempAccount {
+			sys.store.LoadUser(ctx, accessKey)
+			tmpAcc, embeddedPolicy, err = sys.getTempAccount(ctx, accessKey)
+		}
+		if err != nil {
+			return auth.Credentials{}, nil, err
+		}
 	}
 	// Hide secret & session keys
 	tmpAcc.Credentials.SecretKey = ""
@@ -1442,11 +1460,13 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	// 1. Collect all LDAP users with active creds.
 	allCreds := sys.store.GetSTSAndServiceAccounts()
 	// List of unique LDAP (parent) user DNs that have active creds
-	var parentUsers []string
-	// Map of LDAP user to list of active credential objects
+	var parentUserActualDNList []string
+	// Map of LDAP user (internal representation) to list of active credential objects
 	parentUserToCredsMap := make(map[string][]auth.Credentials)
 	// DN to ldap username mapping for each LDAP user
-	parentUserToLDAPUsernameMap := make(map[string]string)
+	actualDNToLDAPUsernameMap := make(map[string]string)
+	// External (actual) LDAP DN to internal normalized representation
+	actualDNToParentUserMap := make(map[string]string)
 	for _, cred := range allCreds {
 		// Expired credentials don't need parent user updates.
 		if cred.IsExpired() {
@@ -1489,25 +1509,28 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 				continue
 			}
 
-			ldapUsername, ok := jwtClaims.Lookup(ldapUserN)
-			if !ok {
+			ldapUsername, okUserN := jwtClaims.Lookup(ldapUserN)
+			ldapActualDN, okDN := jwtClaims.Lookup(ldapActualUser)
+			if !okUserN || !okDN {
 				// skip this cred - we dont have the
 				// username info needed
 				continue
 			}
 
 			// Collect each new cred.ParentUser into parentUsers
-			parentUsers = append(parentUsers, cred.ParentUser)
+			parentUserActualDNList = append(parentUserActualDNList, ldapActualDN)
 
 			// Update the ldapUsernameMap
-			parentUserToLDAPUsernameMap[cred.ParentUser] = ldapUsername
+			actualDNToLDAPUsernameMap[ldapActualDN] = ldapUsername
+
+			// Update the actualDNToParentUserMap
+			actualDNToParentUserMap[ldapActualDN] = cred.ParentUser
 		}
 		parentUserToCredsMap[cred.ParentUser] = append(parentUserToCredsMap[cred.ParentUser], cred)
-
 	}
 
 	// 2. Query LDAP server for groups of the LDAP users collected.
-	updatedGroups, err := sys.LDAPConfig.LookupGroupMemberships(parentUsers, parentUserToLDAPUsernameMap)
+	updatedGroups, err := sys.LDAPConfig.LookupGroupMemberships(parentUserActualDNList, actualDNToLDAPUsernameMap)
 	if err != nil {
 		// Log and return on error - perhaps it'll work the next time.
 		iamLogIf(GlobalContext, err)
@@ -1515,8 +1538,9 @@ func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) {
 	}
 
 	// 3. Update creds for those users whose groups are changed
-	for _, parentUser := range parentUsers {
-		currGroupsSet := updatedGroups[parentUser]
+	for _, parentActualDN := range parentUserActualDNList {
+		currGroupsSet := updatedGroups[parentActualDN]
+		parentUser := actualDNToParentUserMap[parentActualDN]
 		currGroups := currGroupsSet.ToSlice()
 		for _, cred := range parentUserToCredsMap[parentUser] {
 			gSet := set.CreateStringSet(cred.Groups...)

cmd/jwt.go

@@ -24,10 +24,8 @@ import (
 
 	jwtgo "github.com/golang-jwt/jwt/v4"
 	jwtreq "github.com/golang-jwt/jwt/v4/request"
-	"github.com/hashicorp/golang-lru/v2/expirable"
 	"github.com/minio/minio/internal/auth"
 	xjwt "github.com/minio/minio/internal/jwt"
-	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/policy"
 )
 
@@ -37,8 +35,8 @@ const (
 	// Default JWT token for web handlers is one day.
 	defaultJWTExpiry = 24 * time.Hour
 
-	// Inter-node JWT token expiry is 15 minutes.
-	defaultInterNodeJWTExpiry = 15 * time.Minute
+	// Inter-node JWT token expiry is 100 years approx.
+	defaultInterNodeJWTExpiry = 100 * 365 * 24 * time.Hour
 )
 
 var (
@@ -50,17 +48,10 @@ var (
 	errMalformedAuth      = errors.New("Malformed authentication input")
 )
 
-type cacheKey struct {
-	accessKey, secretKey, audience string
-}
-
-var cacheLRU = expirable.NewLRU[cacheKey, string](1000, nil, 15*time.Second)
-
-func authenticateNode(accessKey, secretKey, audience string) (string, error) {
+func authenticateNode(accessKey, secretKey string) (string, error) {
 	claims := xjwt.NewStandardClaims()
 	claims.SetExpiry(UTCNow().Add(defaultInterNodeJWTExpiry))
 	claims.SetAccessKey(accessKey)
-	claims.SetAudience(audience)
 
 	jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, claims)
 	return jwt.SignedString([]byte(secretKey))
@@ -141,27 +132,9 @@ func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, b
 	return claims, groups, owner, nil
 }
 
-// newCachedAuthToken returns a token that is cached up to 15 seconds.
-// If globalActiveCred is updated it is reflected at once.
-func newCachedAuthToken() func(audience string) string {
-	fn := func(accessKey, secretKey, audience string) (s string, err error) {
-		k := cacheKey{accessKey: accessKey, secretKey: secretKey, audience: audience}
-
-		var ok bool
-		s, ok = cacheLRU.Get(k)
-		if !ok {
-			s, err = authenticateNode(accessKey, secretKey, audience)
-			if err != nil {
-				return "", err
-			}
-			cacheLRU.Add(k, s)
-		}
-		return s, nil
-	}
-	return func(audience string) string {
-		cred := globalActiveCred
-		token, err := fn(cred.AccessKey, cred.SecretKey, audience)
-		logger.CriticalIf(GlobalContext, err)
-		return token
+// newCachedAuthToken returns the cached token.
+func newCachedAuthToken() func() string {
+	return func() string {
+		return globalNodeAuthToken
 	}
 }

cmd/jwt_test.go

@@ -107,7 +107,7 @@ func BenchmarkParseJWTStandardClaims(b *testing.B) {
 	}
 
 	creds := globalActiveCred
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -138,7 +138,7 @@ func BenchmarkParseJWTMapClaims(b *testing.B) {
 	}
 
 	creds := globalActiveCred
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		b.Fatal(err)
 	}
@@ -176,7 +176,7 @@ func BenchmarkAuthenticateNode(b *testing.B) {
 		b.ResetTimer()
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
-			fn(creds.AccessKey, creds.SecretKey, "aud")
+			fn(creds.AccessKey, creds.SecretKey)
 		}
 	})
 	b.Run("cached", func(b *testing.B) {
@@ -184,7 +184,7 @@ func BenchmarkAuthenticateNode(b *testing.B) {
 		b.ResetTimer()
 		b.ReportAllocs()
 		for i := 0; i < b.N; i++ {
-			fn("aud")
+			fn()
 		}
 	})
 }

cmd/kms-handlers.go

@@ -22,8 +22,8 @@ import (
 	"encoding/json"
 	"net/http"
 
-	"github.com/minio/kms-go/kes"
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v3/policy"
@@ -56,7 +56,7 @@ func (a kmsAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request)
 	writeSuccessResponseJSON(w, resp)
 }
 
-// KMSMetricsHandler - POST /minio/kms/v1/metrics
+// KMSMetricsHandler - GET /minio/kms/v1/metrics
 func (a kmsAPIHandlers) KMSMetricsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSMetrics")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -83,7 +83,7 @@ func (a kmsAPIHandlers) KMSMetricsHandler(w http.ResponseWriter, r *http.Request
 	}
 }
 
-// KMSAPIsHandler - POST /minio/kms/v1/apis
+// KMSAPIsHandler - GET /minio/kms/v1/apis
 func (a kmsAPIHandlers) KMSAPIsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSAPIs")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -114,7 +114,7 @@ type versionResponse struct {
 	Version string `json:"version"`
 }
 
-// KMSVersionHandler - POST /minio/kms/v1/version
+// KMSVersionHandler - GET /minio/kms/v1/version
 func (a kmsAPIHandlers) KMSVersionHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSVersion")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -159,7 +159,20 @@ func (a kmsAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	if err := GlobalKMS.CreateKey(ctx, &kms.CreateKeyRequest{Name: r.Form.Get("key-id")}); err != nil {
+	keyID := r.Form.Get("key-id")
+
+	// Ensure policy allows the user to create this key name
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+	if !checkKMSActionAllowed(r, owner, cred, policy.KMSCreateKeyAction, keyID) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
+	if err := GlobalKMS.CreateKey(ctx, &kms.CreateKeyRequest{Name: keyID}); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -171,6 +184,9 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 	ctx := newContext(r, w, "KMSListKeys")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
+	// This only checks if the action (kms:ListKeys) is allowed, it does not check
+	// each key name against the policy's Resources. We check that below, once
+	// we have the list of key names from the KMS.
 	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSListKeysAction)
 	if objectAPI == nil {
 		return
@@ -180,7 +196,7 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
 		return
 	}
-	names, _, err := GlobalKMS.ListKeyNames(ctx, &kms.ListRequest{
+	allKeys, _, err := GlobalKMS.ListKeys(ctx, &kms.ListRequest{
 		Prefix: r.Form.Get("pattern"),
 	})
 	if err != nil {
@@ -188,13 +204,25 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	values := make([]kes.KeyInfo, 0, len(names))
-	for _, name := range names {
-		values = append(values, kes.KeyInfo{
-			Name: name,
-		})
+	// Get the cred and owner for checking authz below.
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
 	}
-	if res, err := json.Marshal(values); err != nil {
+
+	// Now we have all the key names, for each of them, check whether the policy grants permission for
+	// the user to list it. Filter in place to leave only allowed keys.
+	n := 0
+	for _, k := range allKeys {
+		if checkKMSActionAllowed(r, owner, cred, policy.KMSListKeysAction, k.Name) {
+			allKeys[n] = k
+			n++
+		}
+	}
+	allKeys = allKeys[:n]
+
+	if res, err := json.Marshal(allKeys); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
 	} else {
 		writeSuccessResponseJSON(w, res)
@@ -224,6 +252,17 @@ func (a kmsAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Reque
 		KeyID: keyID,
 	}
 
+	// Ensure policy allows the user to get this key's status
+	cred, owner, s3Err := validateAdminSignature(ctx, r, "")
+	if s3Err != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		return
+	}
+	if !checkKMSActionAllowed(r, owner, cred, policy.KMSKeyStatusAction, keyID) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
 	kmsContext := kms.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation
 	// 1. Generate a new key using the KMS.
 	key, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{Name: keyID, AssociatedData: kmsContext})
@@ -274,3 +313,16 @@ func (a kmsAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Reque
 	}
 	writeSuccessResponseJSON(w, resp)
 }
+
+// checkKMSActionAllowed checks for authorization for a specific action on a resource.
+func checkKMSActionAllowed(r *http.Request, owner bool, cred auth.Credentials, action policy.KMSAction, resource string) bool {
+	return globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.Action(action),
+		ConditionValues: getConditionValues(r, "", cred),
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+		BucketName:      resource, // overloading BucketName as that's what the policy engine uses to assemble a Resource.
+	})
+}

cmd/kms-handlers_test.go

@@ -0,0 +1,848 @@
+// Copyright (c) 2015-2024 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"strings"
+	"testing"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio/internal/kms"
+	"github.com/minio/pkg/v3/policy"
+)
+
+const (
+	// KMS API paths
+	// For example: /minio/kms/v1/key/list?pattern=*
+	kmsURL           = kmsPathPrefix + kmsAPIVersionPrefix
+	kmsStatusPath    = kmsURL + "/status"
+	kmsMetricsPath   = kmsURL + "/metrics"
+	kmsAPIsPath      = kmsURL + "/apis"
+	kmsVersionPath   = kmsURL + "/version"
+	kmsKeyCreatePath = kmsURL + "/key/create"
+	kmsKeyListPath   = kmsURL + "/key/list"
+	kmsKeyStatusPath = kmsURL + "/key/status"
+
+	// Admin API paths
+	// For example: /minio/admin/v3/kms/status
+	adminURL              = adminPathPrefix + adminAPIVersionPrefix
+	kmsAdminStatusPath    = adminURL + "/kms/status"
+	kmsAdminKeyStatusPath = adminURL + "/kms/key/status"
+	kmsAdminKeyCreate     = adminURL + "/kms/key/create"
+)
+
+const (
+	userAccessKey = "miniofakeuseraccesskey"
+	userSecretKey = "miniofakeusersecret"
+)
+
+type kmsTestCase struct {
+	name   string
+	method string
+	path   string
+	query  map[string]string
+
+	// User credentials and policy for request
+	policy string
+	asRoot bool
+
+	// Wanted in response.
+	wantStatusCode int
+	wantKeyNames   []string
+	wantResp       []string
+}
+
+func TestKMSHandlersCreateKey(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		// Create key test
+		{
+			name:   "create key as user with no policy want forbidden",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "create key as user with no resources specified want success",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:CreateKey"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user set policy to allow want success",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "second-new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:CreateKey"],
+				"Resource": ["arn:minio:kms:::second-new-test-*"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user set policy to non matching resource want forbidden",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "third-new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:CreateKey"],
+				"Resource": ["arn:minio:kms:::non-matching-key-name"] }`,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+	}
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlersKeyStatus(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		{
+			name:   "create a first key root user",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "key status as root want success",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"abc-test-key"},
+		},
+		{
+			name:   "key status as user no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "key status as user legacy no resources specified want success",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:KeyStatus"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"abc-test-key"},
+		},
+		{
+			name:   "key status as user set policy to allow only one key",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:KeyStatus"],
+					"Resource": ["arn:minio:kms:::abc-test-*"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"abc-test-key"},
+		},
+		{
+			name:   "key status as user set policy to allow non-matching key",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:KeyStatus"],
+					"Resource": ["arn:minio:kms:::xyz-test-key"] }`,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+	}
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlersAPIs(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		// Version test
+		{
+			name:   "version as root want success",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"version"},
+		},
+		{
+			name:   "version as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "version as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:Version"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"version"},
+		},
+
+		// APIs test
+		{
+			name:   "apis as root want success",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"stub/path"},
+		},
+		{
+			name:   "apis as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "apis as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:API"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"stub/path"},
+		},
+
+		// Metrics test
+		{
+			name:   "metrics as root want success",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"kms"},
+		},
+		{
+			name:   "metrics as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "metrics as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:Metrics"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"kms"},
+		},
+
+		// Status tests
+		{
+			name:   "status as root want success",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+		{
+			name:   "status as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "status as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:Status"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+	}
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlersListKeys(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		{
+			name:   "create a first key root user",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create a second key root user",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "xyz-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+
+		// List keys tests
+		{
+			name:   "list keys as root want all to be returned",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"default-test-key", "abc-test-key", "xyz-test-key"},
+		},
+		{
+			name:   "list keys as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "list keys as user with no resources specified want success",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["kms:ListKeys"]
+			}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"default-test-key", "abc-test-key", "xyz-test-key"},
+		},
+		{
+			name:   "list keys as user set policy resource to allow only one key",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::abc*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"abc-test-key"},
+		},
+		{
+			name:   "list keys as user set policy to allow only one key, use pattern that includes correct key",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "abc*"},
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::abc*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{"abc-test-key"},
+		},
+		{
+			name:   "list keys as user set policy to allow only one key, use pattern that excludes correct key",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "xyz*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::abc*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{},
+		},
+		{
+			name:   "list keys as user set policy that has no matching key resources",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+					"Action": ["kms:ListKeys"],
+					"Resource": ["arn:minio:kms:::nonematch*"]}`,
+
+			wantStatusCode: http.StatusOK,
+			wantKeyNames:   []string{},
+		},
+		{
+			name:   "list keys as user set policy that allows listing but denies specific keys",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+			asRoot: false,
+
+			// It looks like this should allow listing any key that isn't "default-test-key", however
+			// the policy engine matches all Deny statements first, without regard to Resources (for KMS).
+			// This is for backwards compatibility where historically KMS statements ignored Resources.
+			policy: `{
+						"Effect": "Allow",
+						"Action": ["kms:ListKeys"]
+					},{
+						"Effect": "Deny",
+						"Action": ["kms:ListKeys"],
+						"Resource": ["arn:minio:kms:::default-test-key"]
+					}`,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+	}
+
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+func TestKMSHandlerAdminAPI(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, true)
+	defer tearDown()
+
+	tests := []kmsTestCase{
+		// Create key tests
+		{
+			name:   "create a key root user",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "abc-test-key"},
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user with no policy want forbidden",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "create key as user with no resources specified want success",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "new-test-key"},
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSCreateKey"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+		{
+			name:   "create key as user set policy to non matching resource want success",
+			method: http.MethodPost,
+			path:   kmsAdminKeyCreate,
+			query:  map[string]string{"key-id": "third-new-test-key"},
+			asRoot: false,
+
+			// Admin actions ignore Resources
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSCreateKey"],
+				"Resource": ["arn:minio:kms:::this-is-disregarded"] }`,
+
+			wantStatusCode: http.StatusOK,
+		},
+
+		// Status tests
+		{
+			name:   "status as root want success",
+			method: http.MethodPost,
+			path:   kmsAdminStatusPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+		{
+			name:   "status as user with no policy want forbidden",
+			method: http.MethodPost,
+			path:   kmsAdminStatusPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "status as user with policy ignores resource want success",
+			method: http.MethodPost,
+			path:   kmsAdminStatusPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSKeyStatus"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"MinIO builtin"},
+		},
+
+		// Key status tests
+		{
+			name:   "key status as root want success",
+			method: http.MethodGet,
+			path:   kmsAdminKeyStatusPath,
+			asRoot: true,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"key-id"},
+		},
+		{
+			name:   "key status as user with no policy want forbidden",
+			method: http.MethodGet,
+			path:   kmsAdminKeyStatusPath,
+			asRoot: false,
+
+			wantStatusCode: http.StatusForbidden,
+			wantResp:       []string{"AccessDenied"},
+		},
+		{
+			name:   "key status as user with policy ignores resource want success",
+			method: http.MethodGet,
+			path:   kmsAdminKeyStatusPath,
+			asRoot: false,
+
+			policy: `{"Effect": "Allow",
+				"Action": ["admin:KMSKeyStatus"],
+				"Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`,
+
+			wantStatusCode: http.StatusOK,
+			wantResp:       []string{"key-id"},
+		},
+	}
+
+	for testNum, test := range tests {
+		t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) {
+			execKMSTest(t, test, adminTestBed)
+		})
+	}
+}
+
+// execKMSTest runs a single test case for KMS handlers
+func execKMSTest(t *testing.T, test kmsTestCase, adminTestBed *adminErasureTestBed) {
+	var accessKey, secretKey string
+	if test.asRoot {
+		accessKey, secretKey = globalActiveCred.AccessKey, globalActiveCred.SecretKey
+	} else {
+		setupKMSUser(t, userAccessKey, userSecretKey, test.policy)
+		accessKey = userAccessKey
+		secretKey = userSecretKey
+	}
+
+	req := buildKMSRequest(t, test.method, test.path, accessKey, secretKey, test.query)
+	rec := httptest.NewRecorder()
+	adminTestBed.router.ServeHTTP(rec, req)
+
+	t.Logf("HTTP req: %s, resp code: %d, resp body: %s", req.URL.String(), rec.Code, rec.Body.String())
+
+	// Check status code
+	if rec.Code != test.wantStatusCode {
+		t.Errorf("want status code %d, got %d", test.wantStatusCode, rec.Code)
+	}
+
+	// Check returned key list is correct
+	if test.wantKeyNames != nil {
+		keys := []madmin.KMSKeyInfo{}
+		err := json.Unmarshal(rec.Body.Bytes(), &keys)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if len(keys) != len(test.wantKeyNames) {
+			t.Fatalf("want %d keys, got %d", len(test.wantKeyNames), len(keys))
+		}
+
+		for i, want := range keys {
+			if want.CreatedBy != kms.StubCreatedBy {
+				t.Fatalf("want key created by %s, got %s", kms.StubCreatedBy, want.CreatedBy)
+			}
+			if want.CreatedAt != kms.StubCreatedAt {
+				t.Fatalf("want key created at %s, got %s", kms.StubCreatedAt, want.CreatedAt)
+			}
+			if test.wantKeyNames[i] != want.Name {
+				t.Fatalf("want key name %s, got %s", test.wantKeyNames[i], want.Name)
+			}
+		}
+	}
+
+	// Check generic text in the response
+	if test.wantResp != nil {
+		for _, want := range test.wantResp {
+			if !strings.Contains(rec.Body.String(), want) {
+				t.Fatalf("want response to contain %s, got %s", want, rec.Body.String())
+			}
+		}
+	}
+}
+
+// TestKMSHandlerNotConfiguredOrInvalidCreds tests KMS handlers for situations where KMS is not configured
+// or invalid credentials are provided.
+func TestKMSHandlerNotConfiguredOrInvalidCreds(t *testing.T) {
+	adminTestBed, tearDown := setupKMSTest(t, false)
+	defer tearDown()
+
+	tests := []struct {
+		name   string
+		method string
+		path   string
+		query  map[string]string
+	}{
+		{
+			name:   "GET status",
+			method: http.MethodGet,
+			path:   kmsStatusPath,
+		},
+		{
+			name:   "GET metrics",
+			method: http.MethodGet,
+			path:   kmsMetricsPath,
+		},
+		{
+			name:   "GET apis",
+			method: http.MethodGet,
+			path:   kmsAPIsPath,
+		},
+		{
+			name:   "GET version",
+			method: http.MethodGet,
+			path:   kmsVersionPath,
+		},
+		{
+			name:   "POST key create",
+			method: http.MethodPost,
+			path:   kmsKeyCreatePath,
+			query:  map[string]string{"key-id": "master-key-id"},
+		},
+		{
+			name:   "GET key list",
+			method: http.MethodGet,
+			path:   kmsKeyListPath,
+			query:  map[string]string{"pattern": "*"},
+		},
+		{
+			name:   "GET key status",
+			method: http.MethodGet,
+			path:   kmsKeyStatusPath,
+			query:  map[string]string{"key-id": "master-key-id"},
+		},
+	}
+
+	// Test when the GlobalKMS is not configured
+	for _, test := range tests {
+		t.Run(test.name+" not configured", func(t *testing.T) {
+			req := buildKMSRequest(t, test.method, test.path, "", "", test.query)
+			rec := httptest.NewRecorder()
+			adminTestBed.router.ServeHTTP(rec, req)
+			if rec.Code != http.StatusNotImplemented {
+				t.Errorf("want status code %d, got %d", http.StatusNotImplemented, rec.Code)
+			}
+		})
+	}
+
+	// Test when the GlobalKMS is configured but the credentials are invalid
+	GlobalKMS = kms.NewStub("default-test-key")
+	for _, test := range tests {
+		t.Run(test.name+" invalid credentials", func(t *testing.T) {
+			req := buildKMSRequest(t, test.method, test.path, userAccessKey, userSecretKey, test.query)
+			rec := httptest.NewRecorder()
+			adminTestBed.router.ServeHTTP(rec, req)
+			if rec.Code != http.StatusForbidden {
+				t.Errorf("want status code %d, got %d", http.StatusForbidden, rec.Code)
+			}
+		})
+	}
+}
+
+func setupKMSTest(t *testing.T, enableKMS bool) (*adminErasureTestBed, func()) {
+	adminTestBed, err := prepareAdminErasureTestBed(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	registerKMSRouter(adminTestBed.router)
+
+	if enableKMS {
+		GlobalKMS = kms.NewStub("default-test-key")
+	}
+
+	tearDown := func() {
+		adminTestBed.TearDown()
+		GlobalKMS = nil
+	}
+	return adminTestBed, tearDown
+}
+
+func buildKMSRequest(t *testing.T, method, path, accessKey, secretKey string, query map[string]string) *http.Request {
+	if len(query) > 0 {
+		queryVal := url.Values{}
+		for k, v := range query {
+			queryVal.Add(k, v)
+		}
+		path = path + "?" + queryVal.Encode()
+	}
+
+	if accessKey == "" && secretKey == "" {
+		accessKey = globalActiveCred.AccessKey
+		secretKey = globalActiveCred.SecretKey
+	}
+
+	req, err := newTestSignedRequestV4(method, path, 0, nil, accessKey, secretKey, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return req
+}
+
+// setupKMSUser is a test helper that creates a new user with the provided access key and secret key
+// and applies the given policy to the user.
+func setupKMSUser(t *testing.T, accessKey, secretKey, p string) {
+	ctx := context.Background()
+	createUserParams := madmin.AddOrUpdateUserReq{
+		SecretKey: secretKey,
+		Status:    madmin.AccountEnabled,
+	}
+	_, err := globalIAMSys.CreateUser(ctx, accessKey, createUserParams)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testKMSPolicyName := "testKMSPolicy"
+	if p != "" {
+		p = `{"Version":"2012-10-17","Statement":[` + p + `]}`
+		policyData, err := policy.ParseConfig(strings.NewReader(p))
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = globalIAMSys.SetPolicy(ctx, testKMSPolicyName, *policyData)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = globalIAMSys.PolicyDBSet(ctx, accessKey, testKMSPolicyName, regUser, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+	} else {
+		err = globalIAMSys.DeletePolicy(ctx, testKMSPolicyName, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+		_, err = globalIAMSys.PolicyDBSet(ctx, accessKey, "", regUser, false)
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+}

cmd/local-locker.go

@@ -31,16 +31,16 @@ import (
 
 // lockRequesterInfo stores various info from the client for each lock that is requested.
 type lockRequesterInfo struct {
-	Name            string    // name of the resource lock was requested for
-	Writer          bool      // Bool whether write or read lock.
-	UID             string    // UID to uniquely identify request of client.
-	Timestamp       time.Time // Timestamp set at the time of initialization.
-	TimeLastRefresh time.Time // Timestamp for last lock refresh.
-	Source          string    // Contains line, function and filename requesting the lock.
-	Group           bool      // indicates if it was a group lock.
-	Owner           string    // Owner represents the UUID of the owner who originally requested the lock.
-	Quorum          int       // Quorum represents the quorum required for this lock to be active.
-	idx             int       `msg:"-"` // index of the lock in the lockMap.
+	Name            string // name of the resource lock was requested for
+	Writer          bool   // Bool whether write or read lock.
+	UID             string // UID to uniquely identify request of client.
+	Timestamp       int64  // Timestamp set at the time of initialization.
+	TimeLastRefresh int64  // Timestamp for last lock refresh.
+	Source          string // Contains line, function and filename requesting the lock.
+	Group           bool   // indicates if it was a group lock.
+	Owner           string // Owner represents the UUID of the owner who originally requested the lock.
+	Quorum          int    // Quorum represents the quorum required for this lock to be active.
+	idx             int    `msg:"-"` // index of the lock in the lockMap.
 }
 
 // isWriteLock returns whether the lock is a write or read lock.
@@ -87,6 +87,7 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 
 	// No locks held on the all resources, so claim write
 	// lock on all resources at once.
+	now := UTCNow()
 	for i, resource := range args.Resources {
 		l.lockMap[resource] = []lockRequesterInfo{
 			{
@@ -95,10 +96,10 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 				Source:          args.Source,
 				Owner:           args.Owner,
 				UID:             args.UID,
-				Timestamp:       UTCNow(),
-				TimeLastRefresh: UTCNow(),
+				Timestamp:       now.UnixNano(),
+				TimeLastRefresh: now.UnixNano(),
 				Group:           len(args.Resources) > 1,
-				Quorum:          args.Quorum,
+				Quorum:          *args.Quorum,
 				idx:             i,
 			},
 		}
@@ -108,7 +109,7 @@ func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool
 }
 
 func formatUUID(s string, idx int) string {
-	return s + strconv.Itoa(idx)
+	return concat(s, strconv.Itoa(idx))
 }
 
 func (l *localLocker) Unlock(_ context.Context, args dsync.LockArgs) (reply bool, err error) {
@@ -167,17 +168,19 @@ func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply boo
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 	resource := args.Resources[0]
+	now := UTCNow()
 	lrInfo := lockRequesterInfo{
 		Name:            resource,
 		Writer:          false,
 		Source:          args.Source,
 		Owner:           args.Owner,
 		UID:             args.UID,
-		Timestamp:       UTCNow(),
-		TimeLastRefresh: UTCNow(),
-		Quorum:          args.Quorum,
+		Timestamp:       now.UnixNano(),
+		TimeLastRefresh: now.UnixNano(),
+		Quorum:          *args.Quorum,
 	}
-	if lri, ok := l.lockMap[resource]; ok {
+	lri, ok := l.lockMap[resource]
+	if ok {
 		if reply = !isWriteLock(lri); reply {
 			// Unless there is a write lock
 			l.lockMap[resource] = append(l.lockMap[resource], lrInfo)
@@ -351,9 +354,10 @@ func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refresh
 				delete(l.lockUID, formatUUID(args.UID, idx))
 				return idx > 0, nil
 			}
+			now := UTCNow()
 			for i := range lris {
 				if lris[i].UID == args.UID {
-					lris[i].TimeLastRefresh = UTCNow()
+					lris[i].TimeLastRefresh = now.UnixNano()
 				}
 			}
 			idx++
@@ -376,7 +380,7 @@ func (l *localLocker) expireOldLocks(interval time.Duration) {
 		modified := false
 		for i := 0; i < len(lris); {
 			lri := &lris[i]
-			if time.Since(lri.TimeLastRefresh) > interval {
+			if time.Since(time.Unix(0, lri.TimeLastRefresh)) > interval {
 				delete(l.lockUID, formatUUID(lri.UID, lri.idx))
 				if len(lris) == 1 {
 					// Remove the write lock.

cmd/local-locker_gen.go

@@ -204,13 +204,13 @@ func (z *lockRequesterInfo) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Timestamp":
-			z.Timestamp, err = dc.ReadTime()
+			z.Timestamp, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "Timestamp")
 				return
 			}
 		case "TimeLastRefresh":
-			z.TimeLastRefresh, err = dc.ReadTime()
+			z.TimeLastRefresh, err = dc.ReadInt64()
 			if err != nil {
 				err = msgp.WrapError(err, "TimeLastRefresh")
 				return
@@ -288,7 +288,7 @@ func (z *lockRequesterInfo) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteTime(z.Timestamp)
+	err = en.WriteInt64(z.Timestamp)
 	if err != nil {
 		err = msgp.WrapError(err, "Timestamp")
 		return
@@ -298,7 +298,7 @@ func (z *lockRequesterInfo) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = en.WriteTime(z.TimeLastRefresh)
+	err = en.WriteInt64(z.TimeLastRefresh)
 	if err != nil {
 		err = msgp.WrapError(err, "TimeLastRefresh")
 		return
@@ -361,10 +361,10 @@ func (z *lockRequesterInfo) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendString(o, z.UID)
 	// string "Timestamp"
 	o = append(o, 0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70)
-	o = msgp.AppendTime(o, z.Timestamp)
+	o = msgp.AppendInt64(o, z.Timestamp)
 	// string "TimeLastRefresh"
 	o = append(o, 0xaf, 0x54, 0x69, 0x6d, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68)
-	o = msgp.AppendTime(o, z.TimeLastRefresh)
+	o = msgp.AppendInt64(o, z.TimeLastRefresh)
 	// string "Source"
 	o = append(o, 0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65)
 	o = msgp.AppendString(o, z.Source)
@@ -417,13 +417,13 @@ func (z *lockRequesterInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "Timestamp":
-			z.Timestamp, bts, err = msgp.ReadTimeBytes(bts)
+			z.Timestamp, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Timestamp")
 				return
 			}
 		case "TimeLastRefresh":
-			z.TimeLastRefresh, bts, err = msgp.ReadTimeBytes(bts)
+			z.TimeLastRefresh, bts, err = msgp.ReadInt64Bytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "TimeLastRefresh")
 				return
@@ -466,7 +466,7 @@ func (z *lockRequesterInfo) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *lockRequesterInfo) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 7 + msgp.BoolSize + 4 + msgp.StringPrefixSize + len(z.UID) + 10 + msgp.TimeSize + 16 + msgp.TimeSize + 7 + msgp.StringPrefixSize + len(z.Source) + 6 + msgp.BoolSize + 6 + msgp.StringPrefixSize + len(z.Owner) + 7 + msgp.IntSize
+	s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 7 + msgp.BoolSize + 4 + msgp.StringPrefixSize + len(z.UID) + 10 + msgp.Int64Size + 16 + msgp.Int64Size + 7 + msgp.StringPrefixSize + len(z.Source) + 6 + msgp.BoolSize + 6 + msgp.StringPrefixSize + len(z.Owner) + 7 + msgp.IntSize
 	return
 }
 

cmd/local-locker_test.go

@@ -32,6 +32,7 @@ import (
 func TestLocalLockerExpire(t *testing.T) {
 	wResources := make([]string, 1000)
 	rResources := make([]string, 1000)
+	quorum := 0
 	l := newLocker()
 	ctx := context.Background()
 	for i := range wResources {
@@ -40,7 +41,7 @@ func TestLocalLockerExpire(t *testing.T) {
 			Resources: []string{mustGetUUID()},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.Lock(ctx, arg)
 		if err != nil {
@@ -58,14 +59,14 @@ func TestLocalLockerExpire(t *testing.T) {
 			Resources: []string{name},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RLock(ctx, arg)
 		if err != nil {
 			t.Fatal(err)
 		}
 		if !ok {
-			t.Fatal("did not get write lock")
+			t.Fatal("did not get read lock")
 		}
 		// RLock twice
 		ok, err = l.RLock(ctx, arg)
@@ -112,6 +113,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 	rUIDs := make([]string, 0, n*2)
 	l := newLocker()
 	ctx := context.Background()
+	quorum := 0
 	for i := range wResources {
 		names := [m]string{}
 		for j := range names {
@@ -123,7 +125,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: names[:],
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.Lock(ctx, arg)
 		if err != nil {
@@ -144,7 +146,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: []string{name},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RLock(ctx, arg)
 		if err != nil {
@@ -183,7 +185,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: []string{name},
 			Source:    t.Name(),
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RUnlock(ctx, arg)
 		if err != nil {
@@ -203,6 +205,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 	if len(l.lockUID) != len(rResources)+len(wResources)*m {
 		t.Fatalf("lockUID len, got %d, want %d + %d", len(l.lockUID), len(rResources), len(wResources)*m)
 	}
+
 	// RUnlock again, different uids
 	for i, name := range rResources {
 		arg := dsync.LockArgs{
@@ -210,7 +213,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: []string{name},
 			Source:    "minio",
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.RUnlock(ctx, arg)
 		if err != nil {
@@ -238,7 +241,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 			Resources: names[:],
 			Source:    "minio",
 			Owner:     "owner",
-			Quorum:    0,
+			Quorum:    &quorum,
 		}
 		ok, err := l.Unlock(ctx, arg)
 		if err != nil {
@@ -261,6 +264,7 @@ func TestLocalLockerUnlock(t *testing.T) {
 
 func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 	rng := rand.New(rand.NewSource(0))
+	quorum := 0
 	// Numbers of unique locks
 	for _, locks := range []int{100, 1000, 1e6} {
 		if testing.Short() && locks > 100 {
@@ -289,7 +293,7 @@ func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 								Resources: res,
 								Source:    hex.EncodeToString(tmp[:8]),
 								Owner:     hex.EncodeToString(tmp[8:]),
-								Quorum:    0,
+								Quorum:    &quorum,
 							})
 							if !ok || err != nil {
 								t.Fatal("failed:", err, ok)
@@ -311,7 +315,7 @@ func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 					for _, v := range l.lockMap {
 						for i := range v {
 							if rng.Intn(2) == 0 {
-								v[i].TimeLastRefresh = expired
+								v[i].TimeLastRefresh = expired.UnixNano()
 							}
 						}
 					}
@@ -347,6 +351,7 @@ func Test_localLocker_expireOldLocksExpire(t *testing.T) {
 
 func Test_localLocker_RUnlock(t *testing.T) {
 	rng := rand.New(rand.NewSource(0))
+	quorum := 0
 	// Numbers of unique locks
 	for _, locks := range []int{1, 100, 1000, 1e6} {
 		if testing.Short() && locks > 100 {
@@ -375,7 +380,7 @@ func Test_localLocker_RUnlock(t *testing.T) {
 								Resources: res,
 								Source:    hex.EncodeToString(tmp[:8]),
 								Owner:     hex.EncodeToString(tmp[8:]),
-								Quorum:    0,
+								Quorum:    &quorum,
 							})
 							if !ok || err != nil {
 								t.Fatal("failed:", err, ok)

cmd/lock-rest-client.go

@@ -107,5 +107,5 @@ func newLockAPI(endpoint Endpoint) dsync.NetLocker {
 
 // Returns a lock rest client.
 func newlockRESTClient(ep Endpoint) *lockRESTClient {
-	return &lockRESTClient{globalGrid.Load().Connection(ep.GridHost())}
+	return &lockRESTClient{globalLockGrid.Load().Connection(ep.GridHost())}
 }

cmd/lock-rest-client_test.go

@@ -39,7 +39,7 @@ func TestLockRESTlient(t *testing.T) {
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	err = initGlobalGrid(ctx, []PoolEndpoints{{Endpoints: Endpoints{endpoint, endpointLocal}}})
+	err = initGlobalLockGrid(ctx, []PoolEndpoints{{Endpoints: Endpoints{endpoint, endpointLocal}}})
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/lock-rest-server-common_test.go

@@ -44,7 +44,7 @@ func createLockTestServer(ctx context.Context, t *testing.T) (string, *lockRESTS
 		},
 	}
 	creds := globalActiveCred
-	token, err := authenticateNode(creds.AccessKey, creds.SecretKey, "")
+	token, err := authenticateNode(creds.AccessKey, creds.SecretKey)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -63,15 +63,15 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 		Owner:           "owner",
 		Writer:          true,
 		UID:             "0123-4567",
-		Timestamp:       UTCNow(),
-		TimeLastRefresh: UTCNow(),
+		Timestamp:       UTCNow().UnixNano(),
+		TimeLastRefresh: UTCNow().UnixNano(),
 	}
 	lockRequesterInfo2 := lockRequesterInfo{
 		Owner:           "owner",
 		Writer:          true,
 		UID:             "89ab-cdef",
-		Timestamp:       UTCNow(),
-		TimeLastRefresh: UTCNow(),
+		Timestamp:       UTCNow().UnixNano(),
+		TimeLastRefresh: UTCNow().UnixNano(),
 	}
 
 	locker.ll.lockMap["name"] = []lockRequesterInfo{

cmd/lock-rest-server.go

@@ -111,17 +111,17 @@ func newLockHandler(h grid.HandlerID) *grid.SingleHandler[*dsync.LockArgs, *dsyn
 }
 
 // registerLockRESTHandlers - register lock rest router.
-func registerLockRESTHandlers() {
+func registerLockRESTHandlers(gm *grid.Manager) {
 	lockServer := &lockRESTServer{
 		ll: newLocker(),
 	}
 
-	logger.FatalIf(lockRPCForceUnlock.Register(globalGrid.Load(), lockServer.ForceUnlockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCRefresh.Register(globalGrid.Load(), lockServer.RefreshHandler), "unable to register handler")
-	logger.FatalIf(lockRPCLock.Register(globalGrid.Load(), lockServer.LockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCUnlock.Register(globalGrid.Load(), lockServer.UnlockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCRLock.Register(globalGrid.Load(), lockServer.RLockHandler), "unable to register handler")
-	logger.FatalIf(lockRPCRUnlock.Register(globalGrid.Load(), lockServer.RUnlockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCForceUnlock.Register(gm, lockServer.ForceUnlockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCRefresh.Register(gm, lockServer.RefreshHandler), "unable to register handler")
+	logger.FatalIf(lockRPCLock.Register(gm, lockServer.LockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCUnlock.Register(gm, lockServer.UnlockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCRLock.Register(gm, lockServer.RLockHandler), "unable to register handler")
+	logger.FatalIf(lockRPCRUnlock.Register(gm, lockServer.RUnlockHandler), "unable to register handler")
 
 	globalLockServer = lockServer.ll
 

cmd/main.go

@@ -107,6 +107,11 @@ func newApp(name string) *cli.App {
 
 	// registerCommand registers a cli command.
 	registerCommand := func(command cli.Command) {
+		// avoid registering commands which are not being built (via
+		// go:build tags)
+		if command.Name == "" {
+			return
+		}
 		commands = append(commands, command)
 		commandsTree.Insert(command.Name)
 	}
@@ -134,6 +139,7 @@ func newApp(name string) *cli.App {
 
 	// Register all commands.
 	registerCommand(serverCmd)
+	registerCommand(fmtGenCmd)
 
 	// Set up app.
 	cli.HelpFlag = cli.BoolFlag{

cmd/metacache-entries.go

@@ -257,7 +257,7 @@ func (e *metaCacheEntry) fileInfo(bucket string) (FileInfo, error) {
 				ModTime:  timeSentinel1970,
 			}, nil
 		}
-		return e.cached.ToFileInfo(bucket, e.name, "", false, false)
+		return e.cached.ToFileInfo(bucket, e.name, "", false, true)
 	}
 	return getFileInfo(e.metadata, bucket, e.name, "", fileInfoOpts{})
 }
@@ -300,7 +300,7 @@ func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, erro
 		}, nil
 	}
 	// Too small gains to reuse cache here.
-	return getFileInfoVersions(e.metadata, bucket, e.name, false, true)
+	return getFileInfoVersions(e.metadata, bucket, e.name, true)
 }
 
 // metaCacheEntries is a slice of metacache entries.

cmd/metrics-resource.go

@@ -262,7 +262,7 @@ func collectDriveMetrics(m madmin.RealtimeMetrics) {
 	latestDriveStatsMu.Unlock()
 
 	globalLocalDrivesMu.RLock()
-	localDrives := cloneDrives(globalLocalDrives)
+	localDrives := cloneDrives(globalLocalDrivesMap)
 	globalLocalDrivesMu.RUnlock()
 
 	for _, d := range localDrives {

cmd/metrics-v2.go

@@ -2313,8 +2313,8 @@ func getReplicationNodeMetrics(opts MetricsGroupOpts) *MetricsGroupV2 {
 		var ml []MetricV2
 		// common operational metrics for bucket replication and site replication - published
 		// at cluster level
-		if globalReplicationStats != nil {
-			qs := globalReplicationStats.getNodeQueueStatsSummary()
+		if rStats := globalReplicationStats.Load(); rStats != nil {
+			qs := rStats.getNodeQueueStatsSummary()
 			activeWorkersCount := MetricV2{
 				Description: getClusterReplActiveWorkersCountMD(),
 			}
@@ -3245,7 +3245,7 @@ func getBucketUsageMetrics(opts MetricsGroupOpts) *MetricsGroupV2 {
 
 		var bucketReplStats map[string]BucketStats
 		if !globalSiteReplicationSys.isEnabled() {
-			bucketReplStats = globalReplicationStats.getAllLatest(dataUsageInfo.BucketsUsage)
+			bucketReplStats = globalReplicationStats.Load().getAllLatest(dataUsageInfo.BucketsUsage)
 		}
 		for bucket, usage := range dataUsageInfo.BucketsUsage {
 			quota, _ := globalBucketQuotaSys.Get(ctx, bucket)

cmd/metrics-v3-api.go

@@ -19,6 +19,8 @@ package cmd
 
 import (
 	"context"
+
+	"github.com/minio/minio-go/v7/pkg/set"
 )
 
 const (
@@ -127,7 +129,8 @@ func loadAPIRequestsHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsC
 // This is a `MetricsLoaderFn`.
 func loadAPIRequestsTTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error {
 	renameLabels := map[string]string{"api": "name"}
-	m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, renameLabels, nil,
+	labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
+	m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, labelsFilter, renameLabels, nil,
 		"type", "s3")
 	return nil
 }
@@ -214,7 +217,8 @@ func loadBucketAPIHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCac
 // This is a `MetricsLoaderFn`.
 func loadBucketAPITTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error {
 	renameLabels := map[string]string{"api": "name"}
-	m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, renameLabels,
+	labelsFilter := map[string]set.StringSet{"api": set.CreateStringSet("GetObject")}
+	m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, labelsFilter, renameLabels,
 		buckets, "type", "s3")
 	return nil
 }

cmd/metrics-v3-bucket-replication.go

@@ -119,7 +119,7 @@ func loadBucketReplicationMetrics(ctx context.Context, m MetricValues, c *metric
 		return nil
 	}
 
-	bucketReplStats := globalReplicationStats.getAllLatest(dataUsageInfo.BucketsUsage)
+	bucketReplStats := globalReplicationStats.Load().getAllLatest(dataUsageInfo.BucketsUsage)
 	for _, bucket := range buckets {
 		labels := []string{bucketL, bucket}
 		if s, ok := bucketReplStats[bucket]; ok {

cmd/metrics-v3-replication.go

@@ -34,6 +34,7 @@ const (
 	replicationMaxQueuedBytes          = "max_queued_bytes"
 	replicationMaxQueuedCount          = "max_queued_count"
 	replicationMaxDataTransferRate     = "max_data_transfer_rate"
+	replicationRecentBacklogCount      = "recent_backlog_count"
 )
 
 var (
@@ -61,16 +62,19 @@ var (
 		"Maximum number of objects queued for replication since server start")
 	replicationMaxDataTransferRateMD = NewGaugeMD(replicationMaxDataTransferRate,
 		"Maximum replication data transfer rate in bytes/sec seen since server start")
+	replicationRecentBacklogCountMD = NewGaugeMD(replicationRecentBacklogCount,
+		"Total number of objects seen in replication backlog in the last 5 minutes")
 )
 
 // loadClusterReplicationMetrics - `MetricsLoaderFn` for cluster replication metrics
 // such as transfer rate and objects queued.
 func loadClusterReplicationMetrics(ctx context.Context, m MetricValues, c *metricsCache) error {
-	if globalReplicationStats == nil {
+	st := globalReplicationStats.Load()
+	if st == nil {
 		return nil
 	}
 
-	qs := globalReplicationStats.getNodeQueueStatsSummary()
+	qs := st.getNodeQueueStatsSummary()
 
 	qt := qs.QStats
 	m.Set(replicationAverageQueuedBytes, float64(qt.Avg.Bytes))
@@ -91,6 +95,7 @@ func loadClusterReplicationMetrics(ctx context.Context, m MetricValues, c *metri
 		m.Set(replicationCurrentDataTransferRate, tots.Curr)
 		m.Set(replicationMaxDataTransferRate, tots.Peak)
 	}
+	m.Set(replicationRecentBacklogCount, float64(qs.MRFStats.LastFailedCount))
 
 	return nil
 }

cmd/metrics-v3-types.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"sync"
 
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/logger"
 	"github.com/pkg/errors"
 	"github.com/prometheus/client_golang/prometheus"
@@ -247,28 +248,38 @@ func (m *MetricValues) Set(name MetricName, value float64, labels ...string) {
 // SetHistogram - sets values for the given MetricName using the provided
 // histogram.
 //
+// `filterByLabels` is a map of label names to list of allowed label values to
+// filter by. Note that this filtering happens before any renaming of labels.
+//
 // `renameLabels` is a map of label names to rename. The keys are the original
 // label names and the values are the new label names.
 //
-// TODO: bucketFilter doc
+// `bucketFilter` is a list of bucket values to filter. If this is non-empty,
+// only metrics for the given buckets are added.
 //
 // `extraLabels` are additional labels to add to each metric. They are ordered
 // label name and value pairs.
 func (m *MetricValues) SetHistogram(name MetricName, hist *prometheus.HistogramVec,
-	renameLabels map[string]string, bucketFilter []string, extraLabels ...string,
+	filterByLabels map[string]set.StringSet, renameLabels map[string]string, bucketFilter []string,
+	extraLabels ...string,
 ) {
 	if _, ok := m.descriptors[name]; !ok {
 		panic(fmt.Sprintf("metric has no description: %s", name))
 	}
 	dummyDesc := MetricDescription{}
 	metricsV2 := getHistogramMetrics(hist, dummyDesc, false)
+mainLoop:
 	for _, metric := range metricsV2 {
+		for label, allowedValues := range filterByLabels {
+			if !allowedValues.Contains(metric.VariableLabels[label]) {
+				continue mainLoop
+			}
+		}
+
 		// If a bucket filter is provided, only add metrics for the given
 		// buckets.
-		if len(bucketFilter) > 0 {
-			if !slices.Contains(bucketFilter, metric.VariableLabels["bucket"]) {
-				continue
-			}
+		if len(bucketFilter) > 0 && !slices.Contains(bucketFilter, metric.VariableLabels["bucket"]) {
+			continue
 		}
 
 		labels := make([]string, 0, len(metric.VariableLabels)*2)

cmd/metrics-v3.go

@@ -341,6 +341,7 @@ func newMetricGroups(r *prometheus.Registry) *metricsV3Collection {
 			replicationMaxQueuedBytesMD,
 			replicationMaxQueuedCountMD,
 			replicationMaxDataTransferRateMD,
+			replicationRecentBacklogCountMD,
 		},
 		loadClusterReplicationMetrics,
 	)

cmd/metrics.go

@@ -300,7 +300,7 @@ func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) {
 	}
 
 	for bucket, usageInfo := range dataUsageInfo.BucketsUsage {
-		stat := globalReplicationStats.getLatestReplicationStats(bucket)
+		stat := globalReplicationStats.Load().getLatestReplicationStats(bucket)
 		// Total space used by bucket
 		ch <- prometheus.MustNewConstMetric(
 			prometheus.NewDesc(

cmd/mrf.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -15,51 +15,203 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
+//go:generate msgp -file=$GOFILE
+
 package cmd
 
 import (
 	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"io"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/pkg/v3/wildcard"
+	"github.com/tinylib/msgp/msgp"
 )
 
 const (
 	mrfOpsQueueSize = 100000
 )
 
-// partialOperation is a successful upload/delete of an object
+const (
+	healDir              = ".heal"
+	healMRFDir           = bucketMetaPrefix + SlashSeparator + healDir + SlashSeparator + "mrf"
+	healMRFMetaFormat    = 1
+	healMRFMetaVersionV1 = 1
+)
+
+// PartialOperation is a successful upload/delete of an object
 // but not written in all disks (having quorum)
-type partialOperation struct {
-	bucket              string
-	object              string
-	versionID           string
-	versions            []byte
-	setIndex, poolIndex int
-	queued              time.Time
-	scanMode            madmin.HealScanMode
+type PartialOperation struct {
+	Bucket              string
+	Object              string
+	VersionID           string
+	Versions            []byte
+	SetIndex, PoolIndex int
+	Queued              time.Time
+	BitrotScan          bool
 }
 
 // mrfState sncapsulates all the information
 // related to the global background MRF.
 type mrfState struct {
-	opCh chan partialOperation
+	opCh chan PartialOperation
+
+	closed  int32
+	closing int32
+	wg      sync.WaitGroup
+}
+
+func newMRFState() mrfState {
+	return mrfState{
+		opCh: make(chan PartialOperation, mrfOpsQueueSize),
+	}
 }
 
 // Add a partial S3 operation (put/delete) when one or more disks are offline.
-func (m *mrfState) addPartialOp(op partialOperation) {
+func (m *mrfState) addPartialOp(op PartialOperation) {
 	if m == nil {
 		return
 	}
 
+	if atomic.LoadInt32(&m.closed) == 1 {
+		return
+	}
+
+	m.wg.Add(1)
+	defer m.wg.Done()
+
+	if atomic.LoadInt32(&m.closing) == 1 {
+		return
+	}
+
 	select {
 	case m.opCh <- op:
 	default:
 	}
 }
 
+// Do not accept new MRF operations anymore and start to save
+// the current heal status in one available disk
+func (m *mrfState) shutdown() {
+	atomic.StoreInt32(&m.closing, 1)
+	m.wg.Wait()
+	close(m.opCh)
+	atomic.StoreInt32(&m.closed, 1)
+
+	if len(m.opCh) > 0 {
+		healingLogEvent(context.Background(), "Saving MRF healing data (%d entries)", len(m.opCh))
+	}
+
+	newReader := func() io.ReadCloser {
+		r, w := io.Pipe()
+		go func() {
+			// Initialize MRF meta header.
+			var data [4]byte
+			binary.LittleEndian.PutUint16(data[0:2], healMRFMetaFormat)
+			binary.LittleEndian.PutUint16(data[2:4], healMRFMetaVersionV1)
+			mw := msgp.NewWriter(w)
+			n, err := mw.Write(data[:])
+			if err != nil {
+				w.CloseWithError(err)
+				return
+			}
+			if n != len(data) {
+				w.CloseWithError(io.ErrShortWrite)
+				return
+			}
+			for item := range m.opCh {
+				err = item.EncodeMsg(mw)
+				if err != nil {
+					break
+				}
+			}
+			mw.Flush()
+			w.CloseWithError(err)
+		}()
+		return r
+	}
+
+	globalLocalDrivesMu.RLock()
+	localDrives := cloneDrives(globalLocalDrivesMap)
+	globalLocalDrivesMu.RUnlock()
+
+	for _, localDrive := range localDrives {
+		r := newReader()
+		err := localDrive.CreateFile(context.Background(), "", minioMetaBucket, pathJoin(healMRFDir, "list.bin"), -1, r)
+		r.Close()
+		if err == nil {
+			break
+		}
+	}
+}
+
+func (m *mrfState) startMRFPersistence() {
+	loadMRF := func(rc io.ReadCloser, opCh chan PartialOperation) error {
+		defer rc.Close()
+		var data [4]byte
+		n, err := rc.Read(data[:])
+		if err != nil {
+			return err
+		}
+		if n != len(data) {
+			return errors.New("heal mrf: no data")
+		}
+		// Read resync meta header
+		switch binary.LittleEndian.Uint16(data[0:2]) {
+		case healMRFMetaFormat:
+		default:
+			return fmt.Errorf("heal mrf: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+		}
+		switch binary.LittleEndian.Uint16(data[2:4]) {
+		case healMRFMetaVersionV1:
+		default:
+			return fmt.Errorf("heal mrf: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+		}
+
+		mr := msgp.NewReader(rc)
+		for {
+			op := PartialOperation{}
+			err = op.DecodeMsg(mr)
+			if err != nil {
+				break
+			}
+			opCh <- op
+		}
+
+		return nil
+	}
+
+	globalLocalDrivesMu.RLock()
+	localDrives := cloneDrives(globalLocalDrivesMap)
+	globalLocalDrivesMu.RUnlock()
+
+	for _, localDrive := range localDrives {
+		if localDrive == nil {
+			continue
+		}
+		rc, err := localDrive.ReadFileStream(context.Background(), minioMetaBucket, pathJoin(healMRFDir, "list.bin"), 0, -1)
+		if err != nil {
+			continue
+		}
+		err = loadMRF(rc, m.opCh)
+		if err != nil {
+			continue
+		}
+		// finally delete the file after processing mrf entries
+		localDrive.Delete(GlobalContext, minioMetaBucket, pathJoin(healMRFDir, "list.bin"), DeleteOptions{})
+		break
+	}
+
+	return
+}
+
 var healSleeper = newDynamicSleeper(5, time.Second, false)
 
 // healRoutine listens to new disks reconnection events and
@@ -78,24 +230,24 @@ func (m *mrfState) healRoutine(z *erasureServerPools) {
 			// We might land at .metacache, .trash, .multipart
 			// no need to heal them skip, only when bucket
 			// is '.minio.sys'
-			if u.bucket == minioMetaBucket {
+			if u.Bucket == minioMetaBucket {
 				// No MRF needed for temporary objects
-				if wildcard.Match("buckets/*/.metacache/*", u.object) {
+				if wildcard.Match("buckets/*/.metacache/*", u.Object) {
 					continue
 				}
-				if wildcard.Match("tmp/*", u.object) {
+				if wildcard.Match("tmp/*", u.Object) {
 					continue
 				}
-				if wildcard.Match("multipart/*", u.object) {
+				if wildcard.Match("multipart/*", u.Object) {
 					continue
 				}
-				if wildcard.Match("tmp-old/*", u.object) {
+				if wildcard.Match("tmp-old/*", u.Object) {
 					continue
 				}
 			}
 
 			now := time.Now()
-			if now.Sub(u.queued) < time.Second {
+			if now.Sub(u.Queued) < time.Second {
 				// let recently failed networks to reconnect
 				// making MRF wait for 1s before retrying,
 				// i.e 4 reconnect attempts.
@@ -106,21 +258,22 @@ func (m *mrfState) healRoutine(z *erasureServerPools) {
 			wait := healSleeper.Timer(context.Background())
 
 			scan := madmin.HealNormalScan
-			if u.scanMode != 0 {
-				scan = u.scanMode
+			if u.BitrotScan {
+				scan = madmin.HealDeepScan
 			}
-			if u.object == "" {
-				healBucket(u.bucket, scan)
+
+			if u.Object == "" {
+				healBucket(u.Bucket, scan)
 			} else {
-				if len(u.versions) > 0 {
-					vers := len(u.versions) / 16
+				if len(u.Versions) > 0 {
+					vers := len(u.Versions) / 16
 					if vers > 0 {
 						for i := 0; i < vers; i++ {
-							healObject(u.bucket, u.object, uuid.UUID(u.versions[16*i:]).String(), scan)
+							healObject(u.Bucket, u.Object, uuid.UUID(u.Versions[16*i:]).String(), scan)
 						}
 					}
 				} else {
-					healObject(u.bucket, u.object, u.versionID, scan)
+					healObject(u.Bucket, u.Object, u.VersionID, scan)
 				}
 			}
 

cmd/mrf_gen.go

@@ -0,0 +1,285 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *PartialOperation) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Bucket":
+			z.Bucket, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Object":
+			z.Object, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		case "VersionID":
+			z.VersionID, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "VersionID")
+				return
+			}
+		case "Versions":
+			z.Versions, err = dc.ReadBytes(z.Versions)
+			if err != nil {
+				err = msgp.WrapError(err, "Versions")
+				return
+			}
+		case "SetIndex":
+			z.SetIndex, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "SetIndex")
+				return
+			}
+		case "PoolIndex":
+			z.PoolIndex, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "PoolIndex")
+				return
+			}
+		case "Queued":
+			z.Queued, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "Queued")
+				return
+			}
+		case "BitrotScan":
+			z.BitrotScan, err = dc.ReadBool()
+			if err != nil {
+				err = msgp.WrapError(err, "BitrotScan")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *PartialOperation) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 8
+	// write "Bucket"
+	err = en.Append(0x88, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Bucket)
+	if err != nil {
+		err = msgp.WrapError(err, "Bucket")
+		return
+	}
+	// write "Object"
+	err = en.Append(0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Object)
+	if err != nil {
+		err = msgp.WrapError(err, "Object")
+		return
+	}
+	// write "VersionID"
+	err = en.Append(0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.VersionID)
+	if err != nil {
+		err = msgp.WrapError(err, "VersionID")
+		return
+	}
+	// write "Versions"
+	err = en.Append(0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteBytes(z.Versions)
+	if err != nil {
+		err = msgp.WrapError(err, "Versions")
+		return
+	}
+	// write "SetIndex"
+	err = en.Append(0xa8, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.SetIndex)
+	if err != nil {
+		err = msgp.WrapError(err, "SetIndex")
+		return
+	}
+	// write "PoolIndex"
+	err = en.Append(0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.PoolIndex)
+	if err != nil {
+		err = msgp.WrapError(err, "PoolIndex")
+		return
+	}
+	// write "Queued"
+	err = en.Append(0xa6, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.Queued)
+	if err != nil {
+		err = msgp.WrapError(err, "Queued")
+		return
+	}
+	// write "BitrotScan"
+	err = en.Append(0xaa, 0x42, 0x69, 0x74, 0x72, 0x6f, 0x74, 0x53, 0x63, 0x61, 0x6e)
+	if err != nil {
+		return
+	}
+	err = en.WriteBool(z.BitrotScan)
+	if err != nil {
+		err = msgp.WrapError(err, "BitrotScan")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *PartialOperation) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 8
+	// string "Bucket"
+	o = append(o, 0x88, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
+	o = msgp.AppendString(o, z.Bucket)
+	// string "Object"
+	o = append(o, 0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74)
+	o = msgp.AppendString(o, z.Object)
+	// string "VersionID"
+	o = append(o, 0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44)
+	o = msgp.AppendString(o, z.VersionID)
+	// string "Versions"
+	o = append(o, 0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73)
+	o = msgp.AppendBytes(o, z.Versions)
+	// string "SetIndex"
+	o = append(o, 0xa8, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	o = msgp.AppendInt(o, z.SetIndex)
+	// string "PoolIndex"
+	o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78)
+	o = msgp.AppendInt(o, z.PoolIndex)
+	// string "Queued"
+	o = append(o, 0xa6, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64)
+	o = msgp.AppendTime(o, z.Queued)
+	// string "BitrotScan"
+	o = append(o, 0xaa, 0x42, 0x69, 0x74, 0x72, 0x6f, 0x74, 0x53, 0x63, 0x61, 0x6e)
+	o = msgp.AppendBool(o, z.BitrotScan)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *PartialOperation) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Bucket":
+			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Bucket")
+				return
+			}
+		case "Object":
+			z.Object, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Object")
+				return
+			}
+		case "VersionID":
+			z.VersionID, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "VersionID")
+				return
+			}
+		case "Versions":
+			z.Versions, bts, err = msgp.ReadBytesBytes(bts, z.Versions)
+			if err != nil {
+				err = msgp.WrapError(err, "Versions")
+				return
+			}
+		case "SetIndex":
+			z.SetIndex, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "SetIndex")
+				return
+			}
+		case "PoolIndex":
+			z.PoolIndex, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PoolIndex")
+				return
+			}
+		case "Queued":
+			z.Queued, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Queued")
+				return
+			}
+		case "BitrotScan":
+			z.BitrotScan, bts, err = msgp.ReadBoolBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "BitrotScan")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *PartialOperation) Msgsize() (s int) {
+	s = 1 + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 10 + msgp.StringPrefixSize + len(z.VersionID) + 9 + msgp.BytesPrefixSize + len(z.Versions) + 9 + msgp.IntSize + 10 + msgp.IntSize + 7 + msgp.TimeSize + 11 + msgp.BoolSize
+	return
+}

cmd/mrf_gen_test.go

@@ -0,0 +1,123 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalPartialOperation(t *testing.T) {
+	v := PartialOperation{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgPartialOperation(b *testing.B) {
+	v := PartialOperation{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgPartialOperation(b *testing.B) {
+	v := PartialOperation{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalPartialOperation(b *testing.B) {
+	v := PartialOperation{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodePartialOperation(t *testing.T) {
+	v := PartialOperation{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodePartialOperation Msgsize() is inaccurate")
+	}
+
+	vn := PartialOperation{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodePartialOperation(b *testing.B) {
+	v := PartialOperation{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodePartialOperation(b *testing.B) {
+	v := PartialOperation{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/naughty-disk_test.go

@@ -208,6 +208,20 @@ func (d *naughtyDisk) RenameData(ctx context.Context, srcVolume, srcPath string,
 	return d.disk.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts)
 }
 
+func (d *naughtyDisk) RenamePart(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string, meta []byte) error {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.RenamePart(ctx, srcVolume, srcPath, dstVolume, dstPath, meta)
+}
+
+func (d *naughtyDisk) ReadParts(ctx context.Context, bucket string, partMetaPaths ...string) ([]*ObjectPartInfo, error) {
+	if err := d.calcError(); err != nil {
+		return nil, err
+	}
+	return d.disk.ReadParts(ctx, bucket, partMetaPaths...)
+}
+
 func (d *naughtyDisk) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error {
 	if err := d.calcError(); err != nil {
 		return err
@@ -222,6 +236,13 @@ func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string
 	return d.disk.CheckParts(ctx, volume, path, fi)
 }
 
+func (d *naughtyDisk) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) {
+	if err := d.calcError(); err != nil {
+		return err
+	}
+	return d.disk.DeleteBulk(ctx, volume, paths...)
+}
+
 func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) {
 	if err := d.calcError(); err != nil {
 		return err

cmd/notification.go

@@ -84,6 +84,9 @@ func WithNPeersThrottled(nerrs, wks int) *NotificationGroup {
 	if nerrs <= 0 {
 		nerrs = 1
 	}
+	if wks > nerrs {
+		wks = nerrs
+	}
 	wk, _ := workers.New(wks)
 	return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs), workers: wk, retryCount: 3}
 }
@@ -292,15 +295,15 @@ func (sys *NotificationSys) BackgroundHealStatus(ctx context.Context) ([]madmin.
 }
 
 // StartProfiling - start profiling on remote peers, by initiating a remote RPC.
-func (sys *NotificationSys) StartProfiling(profiler string) []NotificationPeerErr {
+func (sys *NotificationSys) StartProfiling(ctx context.Context, profiler string) []NotificationPeerErr {
 	ng := WithNPeers(len(sys.peerClients))
 	for idx, client := range sys.peerClients {
 		if client == nil {
 			continue
 		}
 		client := client
-		ng.Go(GlobalContext, func() error {
-			return client.StartProfiling(profiler)
+		ng.Go(ctx, func() error {
+			return client.StartProfiling(ctx, profiler)
 		}, idx, *client.host)
 	}
 	return ng.Wait()
@@ -313,28 +316,49 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 	zipWriter := zip.NewWriter(writer)
 	defer zipWriter.Close()
 
-	for _, client := range sys.peerClients {
+	// Start by embedding cluster info.
+	if b := getClusterMetaInfo(ctx); len(b) > 0 {
+		internalLogIf(ctx, embedFileInZip(zipWriter, "cluster.info", b, 0o600))
+	}
+
+	// Profiles can be quite big, so we limit to max 16 concurrent downloads.
+	ng := WithNPeersThrottled(len(sys.peerClients), 16)
+	var writeMu sync.Mutex
+	for i, client := range sys.peerClients {
 		if client == nil {
 			continue
 		}
-		data, err := client.DownloadProfileData()
-		if err != nil {
-			reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
-			ctx := logger.SetReqInfo(ctx, reqInfo)
-			peersLogOnceIf(ctx, err, client.host.String())
-			continue
-		}
-
-		profilingDataFound = true
-
-		for typ, data := range data {
-			err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", client.host.String(), typ), data, 0o600)
+		ng.Go(ctx, func() error {
+			// Give 15 seconds to each remote call.
+			// Errors are logged but not returned.
+			ctx, cancel := context.WithTimeout(ctx, 15*time.Second)
+			defer cancel()
+			data, err := client.DownloadProfileData(ctx)
 			if err != nil {
 				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 				ctx := logger.SetReqInfo(ctx, reqInfo)
 				peersLogOnceIf(ctx, err, client.host.String())
+				return nil
 			}
-		}
+
+			for typ, data := range data {
+				// zip writer only handles one concurrent write
+				writeMu.Lock()
+				profilingDataFound = true
+				err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", client.host.String(), typ), data, 0o600)
+				writeMu.Unlock()
+				if err != nil {
+					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
+					ctx := logger.SetReqInfo(ctx, reqInfo)
+					peersLogOnceIf(ctx, err, client.host.String())
+				}
+			}
+			return nil
+		}, i, *client.host)
+	}
+	ng.Wait()
+	if ctx.Err() != nil {
+		return false
 	}
 
 	// Local host
@@ -359,9 +383,6 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 		err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", thisAddr, typ), data, 0o600)
 		internalLogIf(ctx, err)
 	}
-	if b := getClusterMetaInfo(ctx); len(b) > 0 {
-		internalLogIf(ctx, embedFileInZip(zipWriter, "cluster.info", b, 0o600))
-	}
 
 	return
 }
@@ -383,10 +404,6 @@ func (sys *NotificationSys) VerifyBinary(ctx context.Context, u *url.URL, sha256
 	// further discussion advised. Remove this comment and remove the worker model
 	// for this function in future.
 	maxWorkers := runtime.GOMAXPROCS(0) / 2
-	if maxWorkers > len(sys.peerClients) {
-		maxWorkers = len(sys.peerClients)
-	}
-
 	ng := WithNPeersThrottled(len(sys.peerClients), maxWorkers)
 	for idx, client := range sys.peerClients {
 		if client == nil {
@@ -520,7 +537,7 @@ func (sys *NotificationSys) LoadBucketMetadata(ctx context.Context, bucketName s
 
 // DeleteBucketMetadata - calls DeleteBucketMetadata call on all peers
 func (sys *NotificationSys) DeleteBucketMetadata(ctx context.Context, bucketName string) {
-	globalReplicationStats.Delete(bucketName)
+	globalReplicationStats.Load().Delete(bucketName)
 	globalBucketMetadataSys.Remove(bucketName)
 	globalBucketTargetSys.Delete(bucketName)
 	globalEventNotifier.RemoveNotification(bucketName)
@@ -574,7 +591,7 @@ func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []Buck
 		}
 	}
 
-	replicationStatsList := globalReplicationStats.GetAll()
+	replicationStatsList := globalReplicationStats.Load().GetAll()
 	bucketStatsMap := BucketStatsMap{
 		Stats:     make(map[string]BucketStats, len(replicationStatsList)),
 		Timestamp: UTCNow(),
@@ -582,7 +599,7 @@ func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []Buck
 	for k, replicationStats := range replicationStatsList {
 		bucketStatsMap.Stats[k] = BucketStats{
 			ReplicationStats: replicationStats,
-			ProxyStats:       globalReplicationStats.getProxyStats(k),
+			ProxyStats:       globalReplicationStats.Load().getProxyStats(k),
 		}
 	}
 
@@ -615,11 +632,13 @@ func (sys *NotificationSys) GetClusterBucketStats(ctx context.Context, bucketNam
 			peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
-	bucketStats = append(bucketStats, BucketStats{
-		ReplicationStats: globalReplicationStats.Get(bucketName),
-		QueueStats:       ReplicationQueueStats{Nodes: []ReplQNodeStats{globalReplicationStats.getNodeQueueStats(bucketName)}},
-		ProxyStats:       globalReplicationStats.getProxyStats(bucketName),
-	})
+	if st := globalReplicationStats.Load(); st != nil {
+		bucketStats = append(bucketStats, BucketStats{
+			ReplicationStats: st.Get(bucketName),
+			QueueStats:       ReplicationQueueStats{Nodes: []ReplQNodeStats{st.getNodeQueueStats(bucketName)}},
+			ProxyStats:       st.getProxyStats(bucketName),
+		})
+	}
 	return bucketStats
 }
 
@@ -648,7 +667,7 @@ func (sys *NotificationSys) GetClusterSiteMetrics(ctx context.Context) []SRMetri
 			peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
-	siteStats = append(siteStats, globalReplicationStats.getSRMetricsForNode())
+	siteStats = append(siteStats, globalReplicationStats.Load().getSRMetricsForNode())
 	return siteStats
 }
 
@@ -1099,6 +1118,8 @@ func (sys *NotificationSys) ServerInfo(ctx context.Context, metrics bool) []madm
 		wg.Add(1)
 		go func(client *peerRESTClient, idx int) {
 			defer wg.Done()
+			ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+			defer cancel()
 			info, err := client.ServerInfo(ctx, metrics)
 			if err != nil {
 				info.Endpoint = client.host.String()
@@ -1586,7 +1607,7 @@ func (sys *NotificationSys) GetReplicationMRF(ctx context.Context, bucket, node
 		if node != "all" && node != globalLocalNodeName {
 			return nil
 		}
-		mCh, err := globalReplicationPool.getMRF(ctx, bucket)
+		mCh, err := globalReplicationPool.Get().getMRF(ctx, bucket)
 		if err != nil {
 			return err
 		}

cmd/object-api-deleteobject_test.go

@@ -93,14 +93,18 @@ func testDeleteObject(obj ObjectLayer, instanceType string, t TestErrHandler) {
 			md5Bytes := md5.Sum([]byte(object.content))
 			oi, err := obj.PutObject(context.Background(), testCase.bucketName, object.name, mustGetPutObjReader(t, strings.NewReader(object.content),
 				int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{})
-			t.Log(oi)
 			if err != nil {
+				t.Log(oi)
 				t.Fatalf("%s : %s", instanceType, err.Error())
 			}
 		}
 
 		oi, err := obj.DeleteObject(context.Background(), testCase.bucketName, testCase.pathToDelete, ObjectOptions{})
-		t.Log(oi, err)
+		if err != nil && !isErrObjectNotFound(err) {
+			t.Log(oi)
+			t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err)
+			continue
+		}
 
 		result, err := obj.ListObjects(context.Background(), testCase.bucketName, "", "", "", 1000)
 		if err != nil {

cmd/object-api-errors.go

@@ -751,12 +751,6 @@ func isErrSignatureDoesNotMatch(err error) bool {
 	return errors.As(err, &signatureDoesNotMatch)
 }
 
-// isErrObjectNameInvalid - Check if error type is ObjectNameInvalid.
-func isErrObjectNameInvalid(err error) bool {
-	var invalidObject ObjectNameInvalid
-	return errors.As(err, &invalidObject)
-}
-
 // PreConditionFailed - Check if copy precondition failed
 type PreConditionFailed struct{}
 
@@ -794,3 +788,20 @@ func isReplicationPermissionCheck(err error) bool {
 	_, ok := err.(ReplicationPermissionCheck)
 	return ok
 }
+
+// DataMovementOverwriteErr - captures the error when a data movement activity
+// like rebalance incorrectly tries to overwrite an object.
+type DataMovementOverwriteErr GenericError
+
+func (de DataMovementOverwriteErr) Error() string {
+	objInfoStr := fmt.Sprintf("bucket=%s object=%s", de.Bucket, de.Object)
+	if de.VersionID != "" {
+		objInfoStr = fmt.Sprintf("%s version-id=%s", objInfoStr, de.VersionID)
+	}
+	return fmt.Sprintf("invalid data movement operation, source and destination pool are the same for %s", objInfoStr)
+}
+
+func isDataMovementOverWriteErr(err error) bool {
+	var de DataMovementOverwriteErr
+	return errors.As(err, &de)
+}

cmd/object-api-interface.go

@@ -113,6 +113,8 @@ type ObjectOptions struct {
 	// participating in a rebalance operation. Typically set for 'write' operations.
 	SkipRebalancing bool
 
+	SrcPoolIdx int // set by PutObject/CompleteMultipart operations due to rebalance; used to prevent rebalance src, dst pools to be the same
+
 	DataMovement bool // indicates an going decommisionning or rebalacing
 
 	PrefixEnabledFn func(prefix string) bool // function which returns true if versioning is enabled on prefix
@@ -143,6 +145,7 @@ type WalkOptions struct {
 	LatestOnly   bool                     // returns only latest versions for all matching objects
 	AskDisks     string                   // dictates how many disks are being listed
 	VersionsSort WalkVersionsSortOrder    // sort order for versions of the same object; default: Ascending order in ModTime
+	Limit        int                      // maximum number of items, 0 means no limit
 }
 
 // ExpirationOptions represents object options for object expiration at objectLayer.

cmd/object-api-interface_gen.go

@@ -219,9 +219,9 @@ func (z *MakeBucketOptions) Msgsize() (s int) {
 // MarshalMsg implements msgp.Marshaler
 func (z *WalkOptions) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 4
+	// map header, size 5
 	// string "Marker"
-	o = append(o, 0x84, 0xa6, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72)
+	o = append(o, 0x85, 0xa6, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72)
 	o = msgp.AppendString(o, z.Marker)
 	// string "LatestOnly"
 	o = append(o, 0xaa, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4f, 0x6e, 0x6c, 0x79)
@@ -232,6 +232,9 @@ func (z *WalkOptions) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "VersionsSort"
 	o = append(o, 0xac, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x53, 0x6f, 0x72, 0x74)
 	o = msgp.AppendUint8(o, uint8(z.VersionsSort))
+	// string "Limit"
+	o = append(o, 0xa5, 0x4c, 0x69, 0x6d, 0x69, 0x74)
+	o = msgp.AppendInt(o, z.Limit)
 	return
 }
 
@@ -281,6 +284,12 @@ func (z *WalkOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				}
 				z.VersionsSort = WalkVersionsSortOrder(zb0002)
 			}
+		case "Limit":
+			z.Limit, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Limit")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -295,7 +304,7 @@ func (z *WalkOptions) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *WalkOptions) Msgsize() (s int) {
-	s = 1 + 7 + msgp.StringPrefixSize + len(z.Marker) + 11 + msgp.BoolSize + 9 + msgp.StringPrefixSize + len(z.AskDisks) + 13 + msgp.Uint8Size
+	s = 1 + 7 + msgp.StringPrefixSize + len(z.Marker) + 11 + msgp.BoolSize + 9 + msgp.StringPrefixSize + len(z.AskDisks) + 13 + msgp.Uint8Size + 6 + msgp.IntSize
 	return
 }