Added replication graphs for site replication metrics (#17951)

This dashboard graphs the metrics when site replication is enabled
across MinIO instances.

Signed-off-by: Shubhendu Ram Tripathi <shubhendu@minio.io>

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,9 @@
+## Community Contribution License
+All community contributions in this pull request are licensed to the project maintainers
+under the terms of the [Apache 2 license] (https://www.apache.org/licenses/LICENSE-2.0). 
+By creating this pull request I represent that I have the right to license the 
+contributions to the project maintainers under the Apache 2 license.
+
 ## Description
 
 

.github/workflows/go-cross.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3

.github/workflows/go-fips.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v3

.github/workflows/go-healing.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v3

.github/workflows/go-lint.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v3

.github/workflows/go.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v3

.github/workflows/iam-integrations.yaml

@@ -61,7 +61,7 @@ jobs:
       # are turned off - i.e. if ldap="", then ldap server is not enabled for
       # the tests.
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         ldap: ["", "localhost:389"]
         etcd: ["", "http://localhost:2379"]
         openid: ["", "http://127.0.0.1:5556/dex"]
@@ -82,9 +82,9 @@ jobs:
           check-latest: true
       - name: Test LDAP/OpenID/Etcd combo
         env:
-          LDAP_TEST_SERVER: ${{ matrix.ldap }}
-          ETCD_SERVER: ${{ matrix.etcd }}
-          OPENID_TEST_SERVER: ${{ matrix.openid }}
+          _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
+          _MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
+          _MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
@@ -92,20 +92,20 @@ jobs:
       - name: Test with multiple OpenID providers
         if: matrix.openid == 'http://127.0.0.1:5556/dex'
         env:
-          LDAP_TEST_SERVER: ${{ matrix.ldap }}
-          ETCD_SERVER: ${{ matrix.etcd }}
-          OPENID_TEST_SERVER: ${{ matrix.openid }}
-          OPENID_TEST_SERVER_2: "http://127.0.0.1:5557/dex"
+          _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
+          _MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
+          _MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
+          _MINIO_OPENID_TEST_SERVER_2: "http://127.0.0.1:5557/dex"
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-iam
       - name: Test with Access Management Plugin enabled
         env:
-          LDAP_TEST_SERVER: ${{ matrix.ldap }}
-          ETCD_SERVER: ${{ matrix.etcd }}
-          OPENID_TEST_SERVER: ${{ matrix.openid }}
-          POLICY_PLUGIN_ENDPOINT: "http://127.0.0.1:8080"
+          _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
+          _MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
+          _MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
+          _MINIO_POLICY_PLUGIN_TEST_ENDPOINT: "http://127.0.0.1:8080"
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0

.github/workflows/mint.yml

@@ -29,7 +29,7 @@ jobs:
       - name: setup-go-step
         uses: actions/setup-go@v2
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: github sha short
         id: vars

.github/workflows/replication.yaml

@@ -21,7 +21,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/root-disable.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
 
     steps:

.github/workflows/upgrade-ci-cd.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
 
     steps:

.github/workflows/vulncheck.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.11
+          go-version: 1.21.0
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

CREDITS

@@ -2386,10 +2386,13 @@ SOFTWARE.
 
 ================================================================
 
-github.com/Shopify/sarama
-https://github.com/Shopify/sarama
+github.com/IBM/sarama
+https://github.com/IBM/sarama
 ----------------------------------------------------------------
+# MIT License
+
 Copyright (c) 2013 Shopify
+Copyright (c) 2023 IBM Corporation
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
@@ -2410,34 +2413,6 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-================================================================
-
-github.com/Shopify/toxiproxy/v2
-https://github.com/Shopify/toxiproxy/v2
-----------------------------------------------------------------
-The MIT License (MIT)
-
-Copyright (c) 2014 Shopify
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-
 ================================================================
 
 github.com/alecthomas/participle

Dockerfile.release

@@ -38,6 +38,7 @@ RUN \
      curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /opt/bin/mc && \
      microdnf clean all && \
      chmod +x /opt/bin/minio && \
+     chmod +x /opt/bin/mc && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
      /usr/bin/verify-minio.sh && \

README.md

@@ -200,7 +200,7 @@ service iptables restart
 
 MinIO Server comes with an embedded web based object browser. Point your web browser to <http://127.0.0.1:9000> to ensure your server has started successfully.
 
-> NOTE: MinIO runs console on random port by default if you wish choose a specific port use `--console-address` to pick a specific interface and port.
+> NOTE: MinIO runs console on random port by default, if you wish to choose a specific port use `--console-address` to pick a specific interface and port.
 
 ### Things to consider
 
@@ -241,7 +241,7 @@ mc admin update <minio alias, e.g., myminio>
 ### Upgrade Checklist
 
 - Test all upgrades in a lower environment (DEV, QA, UAT) before applying to production. Performing blind upgrades in production environments carries significant risk.
-- Read the release notes for MinIO *before* performing any upgrade, there is no forced requirement to upgrade to latest releases upon every releases. Some releases may not be relevant to your setup, avoid upgrading production environments unnecessarily.
+- Read the release notes for MinIO *before* performing any upgrade, there is no forced requirement to upgrade to latest release upon every release. Some release may not be relevant to your setup, avoid upgrading production environments unnecessarily.
 - If you plan to use `mc admin update`, MinIO process must have write access to the parent directory where the binary is present on the host system.
 - `mc admin update` is not supported and should be avoided in kubernetes/container environments, please upgrade containers by upgrading relevant container images.
 - **We do not recommend upgrading one MinIO server at a time, the product is designed to support parallel upgrades please follow our recommended guidelines.**
@@ -259,6 +259,6 @@ Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/ma
 
 ## License
 
-- MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
-- MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+- MinIO source is licensed under the [GNU AGPLv3](https://github.com/minio/minio/blob/master/LICENSE).
+- MinIO [documentation](https://github.com/minio/minio/tree/master/docs) is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
 - [License Compliance](https://github.com/minio/minio/blob/master/COMPLIANCE.md)

buildscripts/resolve-right-versions.sh

@@ -3,6 +3,7 @@
 set -E
 set -o pipefail
 set -x
+set -e
 
 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"

cmd/admin-bucket-handlers.go

@@ -662,12 +662,31 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
-	bucketMap := make(map[string]struct{}, 1)
 	rpt := importMetaReport{
 		madmin.BucketMetaImportErrs{
 			Buckets: make(map[string]madmin.BucketStatus, len(zr.File)),
 		},
 	}
+
+	bucketMap := make(map[string]*BucketMetadata, len(zr.File))
+
+	updatedAt := UTCNow()
+
+	for _, file := range zr.File {
+		slc := strings.Split(file.Name, slashSeparator)
+		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
+			rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
+			continue
+		}
+		bucket := slc[0]
+		meta, err := readBucketMetadata(ctx, objectAPI, bucket)
+		if err == nil {
+			bucketMap[bucket] = &meta
+		} else if err != errConfigNotFound {
+			rpt.SetStatus(bucket, "", err)
+		}
+	}
+
 	// import object lock config if any - order of import matters here.
 	for _, file := range zr.File {
 		slc := strings.Split(file.Name, slashSeparator)
@@ -704,7 +723,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 						continue
 					}
 				}
-				bucketMap[bucket] = struct{}{}
+				v := newBucketMetadata(bucket)
+				bucketMap[bucket] = &v
 			}
 
 			// Deny object locking configuration settings on existing buckets without object lock enabled.
@@ -713,27 +733,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
-			if err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].ObjectLockConfigXML = configData
+			bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
-
-			// Call site replication hook.
-			//
-			// We encode the xml bytes as base64 to ensure there are no encoding
-			// errors.
-			cfgStr := base64.StdEncoding.EncodeToString(configData)
-			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:             madmin.SRBucketMetaTypeObjectLockConfig,
-				Bucket:           bucket,
-				ObjectLockConfig: &cfgStr,
-				UpdatedAt:        updatedAt,
-			}); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
 		}
 	}
 
@@ -763,7 +765,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 						continue
 					}
 				}
-				bucketMap[bucket] = struct{}{}
+				v := newBucketMetadata(bucket)
+				bucketMap[bucket] = &v
 			}
 
 			if globalSiteReplicationSys.isEnabled() && v.Suspended() {
@@ -786,10 +789,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].VersioningConfigXML = configData
+			bucketMap[bucket].VersioningConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
 		}
 	}
@@ -807,6 +808,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			continue
 		}
 		bucket, fileName := slc[0], slc[1]
+
 		// create bucket if it does not exist yet.
 		if _, ok := bucketMap[bucket]; !ok {
 			err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{})
@@ -816,7 +818,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 					continue
 				}
 			}
-			bucketMap[bucket] = struct{}{}
+			v := newBucketMetadata(bucket)
+			bucketMap[bucket] = &v
 		}
 		if _, ok := bucketMap[bucket]; !ok {
 			continue
@@ -835,12 +838,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
-			rulesMap := config.ToRulesMap()
-			globalEventNotifier.AddRulesMap(bucket, rulesMap)
+			bucketMap[bucket].NotificationConfigXML = configData
 			rpt.SetStatus(bucket, fileName, nil)
 		case bucketPolicyConfig:
 			// Error out if Content-Length is beyond allowed size.
@@ -873,22 +871,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
-			if err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].PolicyConfigJSON = configData
+			bucketMap[bucket].PolicyConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
-			// Call site replication hook.
-			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:      madmin.SRBucketMetaTypePolicy,
-				Bucket:    bucket,
-				Policy:    bucketPolicyBytes,
-				UpdatedAt: updatedAt,
-			}); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
 		case bucketLifecycleConfig:
 			bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
 			if err != nil {
@@ -914,10 +899,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].LifecycleConfigXML = configData
+			bucketMap[bucket].LifecycleConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
 		case bucketSSEConfig:
 			// Parse bucket encryption xml
@@ -952,29 +935,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			// Store the bucket encryption configuration in the object layer
-			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
-			if err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].EncryptionConfigXML = configData
+			bucketMap[bucket].EncryptionConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
-
-			// Call site replication hook.
-			//
-			// We encode the xml bytes as base64 to ensure there are no encoding
-			// errors.
-			cfgStr := base64.StdEncoding.EncodeToString(configData)
-			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:      madmin.SRBucketMetaTypeSSEConfig,
-				Bucket:    bucket,
-				SSEConfig: &cfgStr,
-				UpdatedAt: updatedAt,
-			}); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
-
 		case bucketTaggingConfig:
 			tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
 			if err != nil {
@@ -988,27 +951,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
-			if err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].TaggingConfigXML = configData
+			bucketMap[bucket].TaggingConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
-
-			// Call site replication hook.
-			//
-			// We encode the xml bytes as base64 to ensure there are no encoding
-			// errors.
-			cfgStr := base64.StdEncoding.EncodeToString(configData)
-			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:      madmin.SRBucketMetaTypeTags,
-				Bucket:    bucket,
-				Tags:      &cfgStr,
-				UpdatedAt: updatedAt,
-			}); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
 		case bucketQuotaConfigFile:
 			data, err := io.ReadAll(reader)
 			if err != nil {
@@ -1016,37 +961,49 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			quotaConfig, err := parseBucketQuota(bucket, data)
+			_, err = parseBucketQuota(bucket, data)
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, err)
 				continue
 			}
 
-			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
-			if err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].QuotaConfigJSON = data
+			bucketMap[bucket].QuotaConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
-
-			bucketMeta := madmin.SRBucketMeta{
-				Type:      madmin.SRBucketMetaTypeQuotaConfig,
-				Bucket:    bucket,
-				Quota:     data,
-				UpdatedAt: updatedAt,
-			}
-			if quotaConfig.Quota == 0 {
-				bucketMeta.Quota = nil
-			}
-
-			// Call site replication hook.
-			if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
 		}
 	}
 
+	enc := func(b []byte) *string {
+		if b == nil {
+			return nil
+		}
+		v := base64.StdEncoding.EncodeToString(b)
+		return &v
+	}
+
+	for bucket, meta := range bucketMap {
+		err := globalBucketMetadataSys.save(ctx, *meta)
+		if err != nil {
+			rpt.SetStatus(bucket, "", err)
+			continue
+		}
+		// Call site replication hook.
+		if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+			Bucket:           bucket,
+			Quota:            meta.QuotaConfigJSON,
+			Policy:           meta.PolicyConfigJSON,
+			Versioning:       enc(meta.VersioningConfigXML),
+			Tags:             enc(meta.TaggingConfigXML),
+			ObjectLockConfig: enc(meta.ObjectLockConfigXML),
+			SSEConfig:        enc(meta.EncryptionConfigXML),
+			UpdatedAt:        updatedAt,
+		}); err != nil {
+			rpt.SetStatus(bucket, "", err)
+			continue
+		}
+
+	}
+
 	rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)

cmd/admin-handlers-pools.go

@@ -30,7 +30,7 @@ import (
 )
 
 var (
-	errRebalanceDecommissionAlreadyRunning = errors.New("Rebalance cannot be started, decommission is aleady in progress")
+	errRebalanceDecommissionAlreadyRunning = errors.New("Rebalance cannot be started, decommission is already in progress")
 	errDecommissionRebalanceAlreadyRunning = errors.New("Decommission cannot be started, rebalance is already in progress")
 )
 

cmd/admin-handlers-site-replication.go

@@ -207,10 +207,15 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 		return
 	}
 
+	if item.Bucket == "" {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errSRInvalidRequest(errInvalidArgument)), r.URL)
+		return
+	}
+
 	var err error
 	switch item.Type {
 	default:
-		err = errSRInvalidRequest(errInvalidArgument)
+		err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, item)
 	case madmin.SRBucketMetaTypePolicy:
 		if item.Policy == nil {
 			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
@@ -236,7 +241,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 				return
 			}
 			if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig, item.UpdatedAt); err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
 			}
 		}
@@ -426,6 +431,7 @@ func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
 	opts.Entity = madmin.GetSREntityType(q.Get("entity"))
 	opts.EntityValue = q.Get("entityvalue")
 	opts.ShowDeleted = q.Get("showDeleted") == "true"
+	opts.Metrics = q.Get("metrics") == "true"
 	return
 }
 

cmd/admin-handlers-users_test.go

@@ -27,7 +27,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"os"
 	"runtime"
 	"strings"
 	"testing"
@@ -40,6 +39,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/signer"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/pkg/env"
 )
 
 const (
@@ -121,7 +121,7 @@ var iamTestSuites = func() []*TestSuiteIAM {
 }()
 
 const (
-	EnvTestEtcdBackend = "ETCD_SERVER"
+	EnvTestEtcdBackend = "_MINIO_ETCD_TEST_SERVER"
 )
 
 func (s *TestSuiteIAM) setUpEtcd(c *check, etcdServer string) {
@@ -144,7 +144,7 @@ func (s *TestSuiteIAM) setUpEtcd(c *check, etcdServer string) {
 func (s *TestSuiteIAM) SetUpSuite(c *check) {
 	// If etcd backend is specified and etcd server is not present, the test
 	// is skipped.
-	etcdServer := os.Getenv(EnvTestEtcdBackend)
+	etcdServer := env.Get(EnvTestEtcdBackend, "")
 	if s.withEtcdBackend && etcdServer == "" {
 		c.Skip("Skipping etcd backend IAM test as no etcd server is configured.")
 	}
@@ -983,9 +983,9 @@ func (s *TestSuiteIAM) SetUpAccMgmtPlugin(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()
 
-	pluginEndpoint := os.Getenv("POLICY_PLUGIN_ENDPOINT")
+	pluginEndpoint := env.Get("_MINIO_POLICY_PLUGIN_ENDPOINT", "")
 	if pluginEndpoint == "" {
-		c.Skip("POLICY_PLUGIN_ENDPOINT not given - skipping.")
+		c.Skip("_MINIO_POLICY_PLUGIN_ENDPOINT not given - skipping.")
 	}
 
 	configCmds := []string{

cmd/admin-handlers.go

@@ -41,6 +41,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -1024,7 +1025,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 {
 			clientToken := nh.clientToken
 			if globalIsDistErasure {
-				clientToken = fmt.Sprintf("%s@%d", nh.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
+				clientToken = fmt.Sprintf("%s:%d", nh.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
 			}
 			b, err := json.Marshal(madmin.HealStartSuccess{
 				ClientToken:   clientToken,
@@ -1146,6 +1147,7 @@ func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
 		return
 	}
+	ctx = lkctx.Context()
 	defer nsLock.Unlock(lkctx)
 
 	durationStr := r.Form.Get(peerRESTDuration)
@@ -1172,6 +1174,66 @@ func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request
 	}
 }
 
+// ClientDevNullExtraTime - return extratime for last devnull
+// [POST] /minio/admin/v3/speedtest/client/devnull/extratime
+func (a adminAPIHandlers) ClientDevNullExtraTime(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.BandwidthMonitorAction)
+	if objectAPI == nil {
+		return
+	}
+
+	enc := json.NewEncoder(w)
+	if err := enc.Encode(madmin.ClientPerfExtraTime{TimeSpent: atomic.LoadInt64(&globalLastClientPerfExtraTime)}); err != nil {
+		return
+	}
+}
+
+// ClientDevNull - everything goes to io.Discard
+// [POST] /minio/admin/v3/speedtest/client/devnull
+func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	timeStart := time.Now()
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.BandwidthMonitorAction)
+	if objectAPI == nil {
+		return
+	}
+
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "client-perf")
+	lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
+		return
+	}
+	ctx = lkctx.Context()
+	defer nsLock.Unlock(lkctx)
+	timeEnd := time.Now()
+
+	atomic.SwapInt64(&globalLastClientPerfExtraTime, timeEnd.Sub(timeStart).Nanoseconds())
+
+	ctx, cancel := context.WithTimeout(ctx, madmin.MaxClientPerfTimeout)
+	defer cancel()
+	totalRx := int64(0)
+	connectTime := time.Now()
+	for {
+		n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			// would mean the network is not stable. Logging here will help in debugging network issues.
+			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
+				logger.LogIf(ctx, err)
+			}
+		}
+		totalRx += n
+		if err != nil || ctx.Err() != nil || totalRx > 100*humanize.GiByte {
+			break
+		}
+
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
 // NetperfHandler - perform mesh style network throughput test
 func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
@@ -1192,6 +1254,7 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
 		return
 	}
+	ctx = lkctx.Context()
 	defer nsLock.Unlock(lkctx)
 
 	durationStr := r.Form.Get(peerRESTDuration)

cmd/admin-heal-ops.go

@@ -253,7 +253,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 	} else {
 		clientToken := he.clientToken
 		if globalIsDistErasure {
-			clientToken = fmt.Sprintf("%s@%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
+			clientToken = fmt.Sprintf("%s:%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
 		}
 
 		hsp = madmin.HealStopSuccess{
@@ -327,7 +327,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLay
 
 	clientToken := h.clientToken
 	if globalIsDistErasure {
-		clientToken = fmt.Sprintf("%s@%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
+		clientToken = fmt.Sprintf("%s:%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
 	}
 
 	b, err := json.Marshal(madmin.HealStartSuccess{

cmd/admin-router.go

@@ -36,6 +36,8 @@ const (
 	adminAPIVersionPrefix          = SlashSeparator + adminAPIVersion
 	adminAPISiteReplicationDevNull = "/site-replication/devnull"
 	adminAPISiteReplicationNetPerf = "/site-replication/netperf"
+	adminAPIClientDevNull          = "/speedtest/client/devnull"
+	adminAPIClientDevExtraTime     = "/speedtest/client/devnull/extratime"
 )
 
 var gzipHandler = func() func(http.Handler) http.HandlerFunc {
@@ -388,6 +390,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(adminMiddleware(adminAPI.DriveSpeedtestHandler, noGZFlag))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(adminMiddleware(adminAPI.NetperfHandler, noGZFlag))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/site").HandlerFunc(adminMiddleware(adminAPI.SitePerfHandler, noGZFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPIClientDevNull).HandlerFunc(adminMiddleware(adminAPI.ClientDevNull, noGZFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPIClientDevExtraTime).HandlerFunc(adminMiddleware(adminAPI.ClientDevNullExtraTime, noGZFlag))
 
 		// HTTP Trace
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(adminMiddleware(adminAPI.TraceHandler, noObjLayerFlag))

cmd/api-errors.go

@@ -190,6 +190,7 @@ const (
 	ErrMaximumExpires
 	ErrSlowDownRead
 	ErrSlowDownWrite
+	ErrMaxVersionsExceeded
 	ErrInvalidPrefixMarker
 	ErrBadRequest
 	ErrKeyTooLongError
@@ -857,6 +858,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Resource requested is unwritable, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrMaxVersionsExceeded: {
+		Code:           "MaxVersionsExceeded",
+		Description:    "You've exceeded the limit on the number of versions you can create on this object",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidPrefixMarker: {
 		Code:           "InvalidPrefixMarker",
 		Description:    "Invalid marker prefix combination",
@@ -1397,7 +1403,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrAdminConfigIDPCfgNameAlreadyExists: {
 		Code:           "XMinioAdminConfigIDPCfgNameAlreadyExists",
-		Description:    "An IDP configuration with the given name aleady exists",
+		Description:    "An IDP configuration with the given name already exists",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrAdminConfigIDPCfgNameDoesNotExist: {
@@ -2110,6 +2116,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrSlowDownRead
 	case errErasureWriteQuorum:
 		apiErr = ErrSlowDownWrite
+	case errMaxVersionsExceeded:
+		apiErr = ErrMaxVersionsExceeded
 	// SSE errors
 	case errInvalidEncryptionParameters:
 		apiErr = ErrInvalidEncryptionParameters

cmd/api-headers.go

@@ -23,11 +23,11 @@ import (
 	"encoding/xml"
 	"fmt"
 	"net/http"
-	"net/url"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -140,9 +140,13 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 
 	// Set tag count if object has tags
 	if len(objInfo.UserTags) > 0 {
-		tags, _ := url.ParseQuery(objInfo.UserTags)
-		if len(tags) > 0 {
-			w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(len(tags))}
+		tags, _ := tags.ParseObjectTags(objInfo.UserTags)
+		if tags.Count() > 0 {
+			w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tags.Count())}
+			if opts.Tagging {
+				// This is MinIO only extension to return back tags along with the count.
+				w.Header()[xhttp.AmzObjectTagging] = []string{objInfo.UserTags}
+			}
 		}
 	}
 

cmd/api-router.go

@@ -461,9 +461,12 @@ func registerAPIRouter(router *mux.Router) {
 
 		// MinIO extension API for replication.
 		//
-		// GetBucketReplicationMetrics
+		router.Methods(http.MethodGet).HandlerFunc(
+			collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsV2Handler))))).Queries("replication-metrics", "2")
+		// deprecated handler
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "")
+
 		// ValidateBucketReplicationCreds
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("checkbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.ValidateBucketReplicationCredsHandler))))).Queries("replication-check", "")
@@ -545,12 +548,5 @@ func corsHandler(handler http.Handler) http.Handler {
 		ExposedHeaders:   commonS3Headers,
 		AllowCredentials: true,
 	}
-	for _, origin := range globalAPIConfig.getCorsAllowOrigins() {
-		if origin == "*" {
-			opts.AllowOriginFunc = nil
-			opts.AllowedOrigins = globalAPIConfig.getCorsAllowOrigins()
-			break
-		}
-	}
 	return cors.New(opts).Handler(handler)
 }

cmd/background-heal-ops.go

@@ -75,9 +75,9 @@ func waitForLowIO(maxIO int, maxWait time.Duration, currentIO func() int) {
 		if tmpMaxWait > 0 {
 			if tmpMaxWait < waitTick {
 				time.Sleep(tmpMaxWait)
-			} else {
-				time.Sleep(waitTick)
+				return
 			}
+			time.Sleep(waitTick)
 			tmpMaxWait -= waitTick
 		}
 		if tmpMaxWait <= 0 {

cmd/batch-handlers.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -42,230 +42,18 @@ import (
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
 	"github.com/minio/pkg/env"
 	iampolicy "github.com/minio/pkg/iam/policy"
-	"github.com/minio/pkg/wildcard"
 	"github.com/minio/pkg/workers"
 	"gopkg.in/yaml.v2"
 )
 
-// replicate:
-//   # source of the objects to be replicated
-//   source:
-//     type: "minio"
-//     bucket: "testbucket"
-//     prefix: "spark/"
-//
-//   # optional flags based filtering criteria
-//   # for source objects
-//   flags:
-//     filter:
-//       newerThan: "7d"
-//       olderThan: "7d"
-//       createdAfter: "date"
-//       createdBefore: "date"
-//       tags:
-//         - key: "name"
-//           value: "value*"
-//       metadata:
-//         - key: "content-type"
-//           value: "image/*"
-//     notify:
-//       endpoint: "https://splunk-hec.dev.com"
-//       token: "Splunk ..." # e.g. "Bearer token"
-//
-//   # target where the objects must be replicated
-//   target:
-//     type: "minio"
-//     bucket: "testbucket1"
-//     endpoint: "https://play.min.io"
-//     path: "on"
-//     credentials:
-//       accessKey: "minioadmin"
-//       secretKey: "minioadmin"
-//       sessionToken: ""
-
-// BatchJobReplicateKV is a datatype that holds key and values for filtering of objects
-// used by metadata filter as well as tags based filtering.
-type BatchJobReplicateKV struct {
-	Key   string `yaml:"key" json:"key"`
-	Value string `yaml:"value" json:"value"`
-}
-
-// Validate returns an error if key is empty
-func (kv BatchJobReplicateKV) Validate() error {
-	if kv.Key == "" {
-		return errInvalidArgument
-	}
-	return nil
-}
-
-// Empty indicates if kv is not set
-func (kv BatchJobReplicateKV) Empty() bool {
-	return kv.Key == "" && kv.Value == ""
-}
-
-// Match matches input kv with kv, value will be wildcard matched depending on the user input
-func (kv BatchJobReplicateKV) Match(ikv BatchJobReplicateKV) bool {
-	if kv.Empty() {
-		return true
-	}
-	if strings.EqualFold(kv.Key, ikv.Key) {
-		return wildcard.Match(kv.Value, ikv.Value)
-	}
-	return false
-}
-
-// BatchReplicateRetry datatype represents total retry attempts and delay between each retries.
-type BatchReplicateRetry struct {
-	Attempts int           `yaml:"attempts" json:"attempts"` // number of retry attempts
-	Delay    time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
-}
-
-// Validate validates input replicate retries.
-func (r BatchReplicateRetry) Validate() error {
-	if r.Attempts < 0 {
-		return errInvalidArgument
-	}
-
-	if r.Delay < 0 {
-		return errInvalidArgument
-	}
-
-	return nil
-}
-
-// BatchReplicateFilter holds all the filters currently supported for batch replication
-type BatchReplicateFilter struct {
-	NewerThan     time.Duration         `yaml:"newerThan,omitempty" json:"newerThan"`
-	OlderThan     time.Duration         `yaml:"olderThan,omitempty" json:"olderThan"`
-	CreatedAfter  time.Time             `yaml:"createdAfter,omitempty" json:"createdAfter"`
-	CreatedBefore time.Time             `yaml:"createdBefore,omitempty" json:"createdBefore"`
-	Tags          []BatchJobReplicateKV `yaml:"tags,omitempty" json:"tags"`
-	Metadata      []BatchJobReplicateKV `yaml:"metadata,omitempty" json:"metadata"`
-}
-
-// BatchReplicateNotification success or failure notification endpoint for each job attempts
-type BatchReplicateNotification struct {
-	Endpoint string `yaml:"endpoint" json:"endpoint"`
-	Token    string `yaml:"token" json:"token"`
-}
-
-// BatchJobReplicateFlags various configurations for replication job definition currently includes
-// - filter
-// - notify
-// - retry
-type BatchJobReplicateFlags struct {
-	Filter BatchReplicateFilter       `yaml:"filter" json:"filter"`
-	Notify BatchReplicateNotification `yaml:"notify" json:"notify"`
-	Retry  BatchReplicateRetry        `yaml:"retry" json:"retry"`
-}
-
-// BatchJobReplicateResourceType defines the type of batch jobs
-type BatchJobReplicateResourceType string
-
-// Validate validates if the replicate resource type is recognized and supported
-func (t BatchJobReplicateResourceType) Validate() error {
-	switch t {
-	case BatchJobReplicateResourceMinIO:
-	case BatchJobReplicateResourceS3:
-	default:
-		return errInvalidArgument
-	}
-	return nil
-}
-
-func (t BatchJobReplicateResourceType) isMinio() bool {
-	return t == BatchJobReplicateResourceMinIO
-}
-
-// Different types of batch jobs..
-const (
-	BatchJobReplicateResourceMinIO BatchJobReplicateResourceType = "minio"
-	BatchJobReplicateResourceS3    BatchJobReplicateResourceType = "s3"
-
-	// add future targets
-)
-
-// BatchJobReplicateCredentials access credentials for batch replication it may
-// be either for target or source.
-type BatchJobReplicateCredentials struct {
-	AccessKey    string `xml:"AccessKeyId" json:"accessKey,omitempty" yaml:"accessKey"`
-	SecretKey    string `xml:"SecretAccessKey" json:"secretKey,omitempty" yaml:"secretKey"`
-	SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"`
-}
-
-// Empty indicates if credentials are not set
-func (c BatchJobReplicateCredentials) Empty() bool {
-	return c.AccessKey == "" && c.SecretKey == "" && c.SessionToken == ""
-}
-
-// Validate validates if credentials are valid
-func (c BatchJobReplicateCredentials) Validate() error {
-	if !auth.IsAccessKeyValid(c.AccessKey) || !auth.IsSecretKeyValid(c.SecretKey) {
-		return errInvalidArgument
-	}
-	return nil
-}
-
-// BatchJobReplicateTarget describes target element of the replication job that receives
-// the filtered data from source
-type BatchJobReplicateTarget struct {
-	Type     BatchJobReplicateResourceType `yaml:"type" json:"type"`
-	Bucket   string                        `yaml:"bucket" json:"bucket"`
-	Prefix   string                        `yaml:"prefix" json:"prefix"`
-	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
-	Path     string                        `yaml:"path" json:"path"`
-	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
-}
-
-// ValidPath returns true if path is valid
-func (t BatchJobReplicateTarget) ValidPath() bool {
-	return t.Path == "on" || t.Path == "off" || t.Path == "auto" || t.Path == ""
-}
-
-// BatchJobReplicateSource describes source element of the replication job that is
-// the source of the data for the target
-type BatchJobReplicateSource struct {
-	Type     BatchJobReplicateResourceType `yaml:"type" json:"type"`
-	Bucket   string                        `yaml:"bucket" json:"bucket"`
-	Prefix   string                        `yaml:"prefix" json:"prefix"`
-	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
-	Path     string                        `yaml:"path" json:"path"`
-	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
-}
-
-// ValidPath returns true if path is valid
-func (s BatchJobReplicateSource) ValidPath() bool {
-	switch s.Path {
-	case "on", "off", "auto", "":
-		return true
-	default:
-		return false
-	}
-}
-
-// BatchJobReplicateV1 v1 of batch job replication
-type BatchJobReplicateV1 struct {
-	APIVersion string                  `yaml:"apiVersion" json:"apiVersion"`
-	Flags      BatchJobReplicateFlags  `yaml:"flags" json:"flags"`
-	Target     BatchJobReplicateTarget `yaml:"target" json:"target"`
-	Source     BatchJobReplicateSource `yaml:"source" json:"source"`
-
-	clnt *miniogo.Core `msg:"-"`
-}
-
-// RemoteToLocal returns true if source is remote and target is local
-func (r BatchJobReplicateV1) RemoteToLocal() bool {
-	return !r.Source.Creds.Empty()
-}
-
 // BatchJobRequest this is an internal data structure not for external consumption.
 type BatchJobRequest struct {
 	ID        string               `yaml:"-" json:"name"`
@@ -277,23 +65,28 @@ type BatchJobRequest struct {
 	ctx       context.Context      `msg:"-"`
 }
 
-// Notify notifies notification endpoint if configured regarding job failure or success.
-func (r BatchJobReplicateV1) Notify(ctx context.Context, body io.Reader) error {
-	if r.Flags.Notify.Endpoint == "" {
+func notifyEndpoint(ctx context.Context, ri *batchJobInfo, endpoint, token string) error {
+	if endpoint == "" {
 		return nil
 	}
 
+	buf, err := json.Marshal(ri)
+	if err != nil {
+		return err
+	}
+
 	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.Flags.Notify.Endpoint, body)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(buf))
 	if err != nil {
 		return err
 	}
 
-	if r.Flags.Notify.Token != "" {
-		req.Header.Set("Authorization", r.Flags.Notify.Token)
+	if token != "" {
+		req.Header.Set("Authorization", token)
 	}
+	req.Header.Set("Content-Type", "application/json")
 
 	clnt := http.Client{Transport: getRemoteInstanceTransport}
 	resp, err := clnt.Do(req)
@@ -309,6 +102,11 @@ func (r BatchJobReplicateV1) Notify(ctx context.Context, body io.Reader) error {
 	return nil
 }
 
+// Notify notifies notification endpoint if configured regarding job failure or success.
+func (r BatchJobReplicateV1) Notify(ctx context.Context, ri *batchJobInfo) error {
+	return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token)
+}
+
 // ReplicateFromSource - this is not implemented yet where source is 'remote' and target is local.
 func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *miniogo.Core, srcObjInfo ObjectInfo, retry bool) error {
 	srcBucket := r.Source.Bucket
@@ -319,17 +117,17 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
 	}
 
-	versioned := globalBucketVersioningSys.PrefixEnabled(tgtBucket, tgtObject)
-	versionSuspended := globalBucketVersioningSys.PrefixSuspended(tgtBucket, tgtObject)
 	versionID := srcObjInfo.VersionID
 	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
 		versionID = ""
 	}
 	if srcObjInfo.DeleteMarker {
 		_, err := api.DeleteObject(ctx, tgtBucket, tgtObject, ObjectOptions{
-			VersionID:          versionID,
-			VersionSuspended:   versionSuspended,
-			Versioned:          versioned,
+			VersionID: versionID,
+			// Since we are preserving a delete marker, we have to make sure this is always true.
+			// regardless of the current configuration of the bucket we must preserve all versions
+			// on the pool being batch replicated from source.
+			Versioned:          true,
 			MTime:              srcObjInfo.ModTime,
 			DeleteMarker:       srcObjInfo.DeleteMarker,
 			ReplicationRequest: true,
@@ -338,12 +136,10 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 	}
 
 	opts := ObjectOptions{
-		VersionID:        srcObjInfo.VersionID,
-		Versioned:        versioned,
-		VersionSuspended: versionSuspended,
-		MTime:            srcObjInfo.ModTime,
-		PreserveETag:     srcObjInfo.ETag,
-		UserDefined:      srcObjInfo.UserDefined,
+		VersionID:    srcObjInfo.VersionID,
+		MTime:        srcObjInfo.ModTime,
+		PreserveETag: srcObjInfo.ETag,
+		UserDefined:  srcObjInfo.UserDefined,
 	}
 	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
 		opts.VersionID = ""
@@ -507,7 +303,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			}
 			for _, kv := range r.Flags.Filter.Tags {
 				for t, v := range tagMap {
-					if kv.Match(BatchJobReplicateKV{Key: t, Value: v}) {
+					if kv.Match(BatchJobKV{Key: t, Value: v}) {
 						return true
 					}
 				}
@@ -524,7 +320,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 						continue
 					}
 					// We only need to match x-amz-meta or standardHeaders
-					if kv.Match(BatchJobReplicateKV{Key: k, Value: v}) {
+					if kv.Match(BatchJobKV{Key: k, Value: v}) {
 						return true
 					}
 				}
@@ -645,8 +441,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		// persist in-memory state to disk.
 		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
-		buf, _ := json.Marshal(ri)
-		if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
+		if err := r.Notify(ctx, ri); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
 		}
 
@@ -1062,7 +857,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 			for _, kv := range r.Flags.Filter.Tags {
 				for t, v := range tagMap {
-					if kv.Match(BatchJobReplicateKV{Key: t, Value: v}) {
+					if kv.Match(BatchJobKV{Key: t, Value: v}) {
 						return true
 					}
 				}
@@ -1079,7 +874,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						continue
 					}
 					// We only need to match x-amz-meta or standardHeaders
-					if kv.Match(BatchJobReplicateKV{Key: k, Value: v}) {
+					if kv.Match(BatchJobKV{Key: k, Value: v}) {
 						return true
 					}
 				}
@@ -1193,8 +988,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		// persist in-memory state to disk.
 		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
-		buf, _ := json.Marshal(ri)
-		if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
+		if err := r.Notify(ctx, ri); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
 		}
 
@@ -1406,7 +1200,6 @@ func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
 	case j.KeyRotate != nil:
 		deleteConfig(ctx, api, pathJoin(j.Location, batchKeyRotationName))
 	}
-	globalBatchJobsMetrics.delete(j.ID)
 	deleteConfig(ctx, api, j.Location)
 }
 
@@ -1520,14 +1313,14 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	id := r.Form.Get("jobId")
-	if id == "" {
+	jobID := r.Form.Get("jobId")
+	if jobID == "" {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL)
 		return
 	}
 
 	req := &BatchJobRequest{}
-	if err := req.load(ctx, objectAPI, pathJoin(batchJobPrefix, id)); err != nil {
+	if err := req.load(ctx, objectAPI, pathJoin(batchJobPrefix, jobID)); err != nil {
 		if !errors.Is(err, errNoSuchJob) {
 			logger.LogIf(ctx, err)
 		}
@@ -1555,7 +1348,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	buf, err := io.ReadAll(r.Body)
+	buf, err := io.ReadAll(ioutil.HardLimitReader(r.Body, humanize.MiByte*4))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1567,12 +1360,12 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 	}
 
 	job := &BatchJobRequest{}
-	if err = yaml.Unmarshal(buf, job); err != nil {
+	if err = yaml.UnmarshalStrict(buf, job); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
-	job.ID = shortuuid.New()
+	job.ID = fmt.Sprintf("%s:%d", shortuuid.New(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
 	job.User = user
 	job.Started = time.Now()
 
@@ -1608,19 +1401,27 @@ func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request)
 	if objectAPI == nil {
 		return
 	}
+
 	jobID := r.Form.Get("id")
 	if jobID == "" {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL)
 		return
 	}
+
+	if _, success := proxyRequestByToken(ctx, w, r, jobID); success {
+		return
+	}
+
 	if err := globalBatchJobPool.canceler(jobID, true); err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL)
 		return
 	}
+
 	j := BatchJobRequest{
 		ID:       jobID,
 		Location: pathJoin(batchJobPrefix, jobID),
 	}
+
 	j.delete(ctx, objectAPI)
 
 	writeSuccessNoContent(w)
@@ -1675,6 +1476,11 @@ func (j *BatchJobPool) resume() {
 			logger.LogIf(ctx, err)
 			continue
 		}
+		_, nodeIdx := parseRequestToken(req.ID)
+		if nodeIdx > -1 && GetProxyEndpointLocalIndex(globalProxyEndpoints) != nodeIdx {
+			// This job doesn't belong on this node.
+			continue
+		}
 		if err := j.queueJob(req); err != nil {
 			logger.LogIf(ctx, err)
 			continue
@@ -1795,10 +1601,6 @@ type batchJobMetrics struct {
 	metrics map[string]*batchJobInfo
 }
 
-var globalBatchJobsMetrics = batchJobMetrics{
-	metrics: make(map[string]*batchJobInfo),
-}
-
 //msgp:ignore batchJobMetric
 //go:generate stringer -type=batchJobMetric -trimprefix=batchJobMetric $GOFILE
 type batchJobMetric uint8
@@ -1838,9 +1640,17 @@ func (m *batchJobMetrics) report(jobID string) (metrics *madmin.BatchJobMetrics)
 	metrics = &madmin.BatchJobMetrics{CollectedAt: time.Now(), Jobs: make(map[string]madmin.JobMetric)}
 	m.RLock()
 	defer m.RUnlock()
+
+	match := true
 	for id, job := range m.metrics {
-		match := jobID != "" && id == jobID
-		metrics.Jobs[id] = madmin.JobMetric{
+		if jobID != "" {
+			match = id == jobID
+		}
+		if !match {
+			continue
+		}
+
+		m := madmin.JobMetric{
 			JobID:         job.JobID,
 			JobType:       job.JobType,
 			StartTime:     job.StartTime,
@@ -1848,28 +1658,58 @@ func (m *batchJobMetrics) report(jobID string) (metrics *madmin.BatchJobMetrics)
 			RetryAttempts: job.RetryAttempts,
 			Complete:      job.Complete,
 			Failed:        job.Failed,
-			Replicate: &madmin.ReplicateInfo{
+		}
+
+		switch job.JobType {
+		case string(madmin.BatchJobReplicate):
+			m.Replicate = &madmin.ReplicateInfo{
 				Bucket:           job.Bucket,
 				Object:           job.Object,
 				Objects:          job.Objects,
 				ObjectsFailed:    job.ObjectsFailed,
 				BytesTransferred: job.BytesTransferred,
 				BytesFailed:      job.BytesFailed,
-			},
-			KeyRotate: &madmin.KeyRotationInfo{
+			}
+		case string(madmin.BatchJobKeyRotate):
+			m.KeyRotate = &madmin.KeyRotationInfo{
 				Bucket:        job.Bucket,
 				Object:        job.Object,
 				Objects:       job.Objects,
 				ObjectsFailed: job.ObjectsFailed,
-			},
-		}
-		if match {
-			break
+			}
 		}
+
+		metrics.Jobs[id] = m
 	}
 	return metrics
 }
 
+// keep job metrics for some time after the job is completed
+// in-case some one wants to look at the older results.
+func (m *batchJobMetrics) purgeJobMetrics() {
+	t := time.NewTicker(6 * time.Hour)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-GlobalContext.Done():
+			return
+		case <-t.C:
+			var toDeleteJobMetrics []string
+			m.RLock()
+			for id, metrics := range m.metrics {
+				if time.Since(metrics.LastUpdate) > 24*time.Hour && (metrics.Complete || metrics.Failed) {
+					toDeleteJobMetrics = append(toDeleteJobMetrics, id)
+				}
+			}
+			m.RUnlock()
+			for _, jobID := range toDeleteJobMetrics {
+				m.delete(jobID)
+			}
+		}
+	}
+}
+
 func (m *batchJobMetrics) delete(jobID string) {
 	m.Lock()
 	defer m.Unlock()

cmd/batch-job-common-types.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"strings"
+	"time"
+
+	"github.com/minio/pkg/wildcard"
+)
+
+//go:generate msgp -file $GOFILE
+
+// BatchJobKV is a key-value data type which supports wildcard matching
+type BatchJobKV struct {
+	Key   string `yaml:"key" json:"key"`
+	Value string `yaml:"value" json:"value"`
+}
+
+// Validate returns an error if key is empty
+func (kv BatchJobKV) Validate() error {
+	if kv.Key == "" {
+		return errInvalidArgument
+	}
+	return nil
+}
+
+// Empty indicates if kv is not set
+func (kv BatchJobKV) Empty() bool {
+	return kv.Key == "" && kv.Value == ""
+}
+
+// Match matches input kv with kv, value will be wildcard matched depending on the user input
+func (kv BatchJobKV) Match(ikv BatchJobKV) bool {
+	if kv.Empty() {
+		return true
+	}
+	if strings.EqualFold(kv.Key, ikv.Key) {
+		return wildcard.Match(kv.Value, ikv.Value)
+	}
+	return false
+}
+
+// BatchJobNotification stores notification endpoint and token information.
+// Used by batch jobs to notify of their status.
+type BatchJobNotification struct {
+	Endpoint string `yaml:"endpoint" json:"endpoint"`
+	Token    string `yaml:"token" json:"token"`
+}
+
+// BatchJobRetry stores retry configuration used in the event of failures.
+type BatchJobRetry struct {
+	Attempts int           `yaml:"attempts" json:"attempts"` // number of retry attempts
+	Delay    time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
+}
+
+// Validate validates input replicate retries.
+func (r BatchJobRetry) Validate() error {
+	if r.Attempts < 0 {
+		return errInvalidArgument
+	}
+
+	if r.Delay < 0 {
+		return errInvalidArgument
+	}
+
+	return nil
+}

cmd/batch-job-common-types_gen.go

@@ -0,0 +1,391 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobKV) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Key":
+			z.Key, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Key")
+				return
+			}
+		case "Value":
+			z.Value, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Value")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobKV) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Key"
+	err = en.Append(0x82, 0xa3, 0x4b, 0x65, 0x79)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Key)
+	if err != nil {
+		err = msgp.WrapError(err, "Key")
+		return
+	}
+	// write "Value"
+	err = en.Append(0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Value)
+	if err != nil {
+		err = msgp.WrapError(err, "Value")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobKV) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Key"
+	o = append(o, 0x82, 0xa3, 0x4b, 0x65, 0x79)
+	o = msgp.AppendString(o, z.Key)
+	// string "Value"
+	o = append(o, 0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
+	o = msgp.AppendString(o, z.Value)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobKV) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Key":
+			z.Key, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Key")
+				return
+			}
+		case "Value":
+			z.Value, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Value")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobKV) Msgsize() (s int) {
+	s = 1 + 4 + msgp.StringPrefixSize + len(z.Key) + 6 + msgp.StringPrefixSize + len(z.Value)
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobNotification) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Endpoint":
+			z.Endpoint, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "Token":
+			z.Token, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Token")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobNotification) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Endpoint"
+	err = en.Append(0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Endpoint)
+	if err != nil {
+		err = msgp.WrapError(err, "Endpoint")
+		return
+	}
+	// write "Token"
+	err = en.Append(0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Token)
+	if err != nil {
+		err = msgp.WrapError(err, "Token")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobNotification) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Endpoint"
+	o = append(o, 0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	o = msgp.AppendString(o, z.Endpoint)
+	// string "Token"
+	o = append(o, 0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e)
+	o = msgp.AppendString(o, z.Token)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobNotification) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Endpoint":
+			z.Endpoint, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "Token":
+			z.Token, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Token")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobNotification) Msgsize() (s int) {
+	s = 1 + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 6 + msgp.StringPrefixSize + len(z.Token)
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobRetry) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Attempts":
+			z.Attempts, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "Attempts")
+				return
+			}
+		case "Delay":
+			z.Delay, err = dc.ReadDuration()
+			if err != nil {
+				err = msgp.WrapError(err, "Delay")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobRetry) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Attempts"
+	err = en.Append(0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Attempts)
+	if err != nil {
+		err = msgp.WrapError(err, "Attempts")
+		return
+	}
+	// write "Delay"
+	err = en.Append(0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
+	if err != nil {
+		return
+	}
+	err = en.WriteDuration(z.Delay)
+	if err != nil {
+		err = msgp.WrapError(err, "Delay")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobRetry) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Attempts"
+	o = append(o, 0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
+	o = msgp.AppendInt(o, z.Attempts)
+	// string "Delay"
+	o = append(o, 0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
+	o = msgp.AppendDuration(o, z.Delay)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobRetry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Attempts":
+			z.Attempts, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Attempts")
+				return
+			}
+		case "Delay":
+			z.Delay, bts, err = msgp.ReadDurationBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Delay")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobRetry) Msgsize() (s int) {
+	s = 1 + 9 + msgp.IntSize + 6 + msgp.DurationSize
+	return
+}

cmd/batch-job-common-types_gen_test.go

@@ -0,0 +1,349 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalBatchJobKV(t *testing.T) {
+	v := BatchJobKV{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobKV(b *testing.B) {
+	v := BatchJobKV{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobKV(b *testing.B) {
+	v := BatchJobKV{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobKV(b *testing.B) {
+	v := BatchJobKV{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobKV(t *testing.T) {
+	v := BatchJobKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobKV Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobKV{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobKV(b *testing.B) {
+	v := BatchJobKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobKV(b *testing.B) {
+	v := BatchJobKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobNotification(t *testing.T) {
+	v := BatchJobNotification{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobNotification(b *testing.B) {
+	v := BatchJobNotification{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobNotification(b *testing.B) {
+	v := BatchJobNotification{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobNotification(b *testing.B) {
+	v := BatchJobNotification{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobNotification(t *testing.T) {
+	v := BatchJobNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobNotification Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobNotification{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobNotification(b *testing.B) {
+	v := BatchJobNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobNotification(b *testing.B) {
+	v := BatchJobNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobRetry(t *testing.T) {
+	v := BatchJobRetry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobRetry(b *testing.B) {
+	v := BatchJobRetry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobRetry(b *testing.B) {
+	v := BatchJobRetry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobRetry(b *testing.B) {
+	v := BatchJobRetry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobRetry(t *testing.T) {
+	v := BatchJobRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobRetry Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobRetry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobRetry(b *testing.B) {
+	v := BatchJobRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobRetry(b *testing.B) {
+	v := BatchJobRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batch-replicate.go

@@ -0,0 +1,183 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"time"
+
+	miniogo "github.com/minio/minio-go/v7"
+
+	"github.com/minio/minio/internal/auth"
+)
+
+//go:generate msgp -file $GOFILE
+
+// replicate:
+//   # source of the objects to be replicated
+//   source:
+//     type: "minio"
+//     bucket: "testbucket"
+//     prefix: "spark/"
+//
+//   # optional flags based filtering criteria
+//   # for source objects
+//   flags:
+//     filter:
+//       newerThan: "7d"
+//       olderThan: "7d"
+//       createdAfter: "date"
+//       createdBefore: "date"
+//       tags:
+//         - key: "name"
+//           value: "value*"
+//       metadata:
+//         - key: "content-type"
+//           value: "image/*"
+//     notify:
+//       endpoint: "https://splunk-hec.dev.com"
+//       token: "Splunk ..." # e.g. "Bearer token"
+//
+//   # target where the objects must be replicated
+//   target:
+//     type: "minio"
+//     bucket: "testbucket1"
+//     endpoint: "https://play.min.io"
+//     path: "on"
+//     credentials:
+//       accessKey: "minioadmin"
+//       secretKey: "minioadmin"
+//       sessionToken: ""
+
+// BatchReplicateFilter holds all the filters currently supported for batch replication
+type BatchReplicateFilter struct {
+	NewerThan     time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
+	OlderThan     time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedAfter  time.Time     `yaml:"createdAfter,omitempty" json:"createdAfter"`
+	CreatedBefore time.Time     `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchJobKV  `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchJobKV  `yaml:"metadata,omitempty" json:"metadata"`
+}
+
+// BatchJobReplicateFlags various configurations for replication job definition currently includes
+// - filter
+// - notify
+// - retry
+type BatchJobReplicateFlags struct {
+	Filter BatchReplicateFilter `yaml:"filter" json:"filter"`
+	Notify BatchJobNotification `yaml:"notify" json:"notify"`
+	Retry  BatchJobRetry        `yaml:"retry" json:"retry"`
+}
+
+// BatchJobReplicateResourceType defines the type of batch jobs
+type BatchJobReplicateResourceType string
+
+// Validate validates if the replicate resource type is recognized and supported
+func (t BatchJobReplicateResourceType) Validate() error {
+	switch t {
+	case BatchJobReplicateResourceMinIO:
+	case BatchJobReplicateResourceS3:
+	default:
+		return errInvalidArgument
+	}
+	return nil
+}
+
+func (t BatchJobReplicateResourceType) isMinio() bool {
+	return t == BatchJobReplicateResourceMinIO
+}
+
+// Different types of batch jobs..
+const (
+	BatchJobReplicateResourceMinIO BatchJobReplicateResourceType = "minio"
+	BatchJobReplicateResourceS3    BatchJobReplicateResourceType = "s3"
+
+	// add future targets
+)
+
+// BatchJobReplicateCredentials access credentials for batch replication it may
+// be either for target or source.
+type BatchJobReplicateCredentials struct {
+	AccessKey    string `xml:"AccessKeyId" json:"accessKey,omitempty" yaml:"accessKey"`
+	SecretKey    string `xml:"SecretAccessKey" json:"secretKey,omitempty" yaml:"secretKey"`
+	SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"`
+}
+
+// Empty indicates if credentials are not set
+func (c BatchJobReplicateCredentials) Empty() bool {
+	return c.AccessKey == "" && c.SecretKey == "" && c.SessionToken == ""
+}
+
+// Validate validates if credentials are valid
+func (c BatchJobReplicateCredentials) Validate() error {
+	if !auth.IsAccessKeyValid(c.AccessKey) || !auth.IsSecretKeyValid(c.SecretKey) {
+		return errInvalidArgument
+	}
+	return nil
+}
+
+// BatchJobReplicateTarget describes target element of the replication job that receives
+// the filtered data from source
+type BatchJobReplicateTarget struct {
+	Type     BatchJobReplicateResourceType `yaml:"type" json:"type"`
+	Bucket   string                        `yaml:"bucket" json:"bucket"`
+	Prefix   string                        `yaml:"prefix" json:"prefix"`
+	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
+	Path     string                        `yaml:"path" json:"path"`
+	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
+}
+
+// ValidPath returns true if path is valid
+func (t BatchJobReplicateTarget) ValidPath() bool {
+	return t.Path == "on" || t.Path == "off" || t.Path == "auto" || t.Path == ""
+}
+
+// BatchJobReplicateSource describes source element of the replication job that is
+// the source of the data for the target
+type BatchJobReplicateSource struct {
+	Type     BatchJobReplicateResourceType `yaml:"type" json:"type"`
+	Bucket   string                        `yaml:"bucket" json:"bucket"`
+	Prefix   string                        `yaml:"prefix" json:"prefix"`
+	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
+	Path     string                        `yaml:"path" json:"path"`
+	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
+}
+
+// ValidPath returns true if path is valid
+func (s BatchJobReplicateSource) ValidPath() bool {
+	switch s.Path {
+	case "on", "off", "auto", "":
+		return true
+	default:
+		return false
+	}
+}
+
+// BatchJobReplicateV1 v1 of batch job replication
+type BatchJobReplicateV1 struct {
+	APIVersion string                  `yaml:"apiVersion" json:"apiVersion"`
+	Flags      BatchJobReplicateFlags  `yaml:"flags" json:"flags"`
+	Target     BatchJobReplicateTarget `yaml:"target" json:"target"`
+	Source     BatchJobReplicateSource `yaml:"source" json:"source"`
+
+	clnt *miniogo.Core `msg:"-"`
+}
+
+// RemoteToLocal returns true if source is remote and target is local
+func (r BatchJobReplicateV1) RemoteToLocal() bool {
+	return !r.Source.Creds.Empty()
+}

cmd/batch-replicate_gen_test.go

@@ -0,0 +1,688 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalBatchJobReplicateCredentials(t *testing.T) {
+	v := BatchJobReplicateCredentials{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobReplicateCredentials(b *testing.B) {
+	v := BatchJobReplicateCredentials{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobReplicateCredentials(b *testing.B) {
+	v := BatchJobReplicateCredentials{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobReplicateCredentials(b *testing.B) {
+	v := BatchJobReplicateCredentials{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobReplicateCredentials(t *testing.T) {
+	v := BatchJobReplicateCredentials{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobReplicateCredentials Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobReplicateCredentials{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobReplicateCredentials(b *testing.B) {
+	v := BatchJobReplicateCredentials{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobReplicateCredentials(b *testing.B) {
+	v := BatchJobReplicateCredentials{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobReplicateFlags(t *testing.T) {
+	v := BatchJobReplicateFlags{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobReplicateFlags(b *testing.B) {
+	v := BatchJobReplicateFlags{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobReplicateFlags(b *testing.B) {
+	v := BatchJobReplicateFlags{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobReplicateFlags(b *testing.B) {
+	v := BatchJobReplicateFlags{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobReplicateFlags(t *testing.T) {
+	v := BatchJobReplicateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobReplicateFlags Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobReplicateFlags{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobReplicateFlags(b *testing.B) {
+	v := BatchJobReplicateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobReplicateFlags(b *testing.B) {
+	v := BatchJobReplicateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobReplicateSource(t *testing.T) {
+	v := BatchJobReplicateSource{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobReplicateSource(b *testing.B) {
+	v := BatchJobReplicateSource{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobReplicateSource(b *testing.B) {
+	v := BatchJobReplicateSource{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobReplicateSource(b *testing.B) {
+	v := BatchJobReplicateSource{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobReplicateSource(t *testing.T) {
+	v := BatchJobReplicateSource{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobReplicateSource Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobReplicateSource{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobReplicateSource(b *testing.B) {
+	v := BatchJobReplicateSource{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobReplicateSource(b *testing.B) {
+	v := BatchJobReplicateSource{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobReplicateTarget(t *testing.T) {
+	v := BatchJobReplicateTarget{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobReplicateTarget(b *testing.B) {
+	v := BatchJobReplicateTarget{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobReplicateTarget(b *testing.B) {
+	v := BatchJobReplicateTarget{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobReplicateTarget(b *testing.B) {
+	v := BatchJobReplicateTarget{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobReplicateTarget(t *testing.T) {
+	v := BatchJobReplicateTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobReplicateTarget Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobReplicateTarget{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobReplicateTarget(b *testing.B) {
+	v := BatchJobReplicateTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobReplicateTarget(b *testing.B) {
+	v := BatchJobReplicateTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobReplicateV1(t *testing.T) {
+	v := BatchJobReplicateV1{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobReplicateV1(b *testing.B) {
+	v := BatchJobReplicateV1{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobReplicateV1(b *testing.B) {
+	v := BatchJobReplicateV1{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobReplicateV1(b *testing.B) {
+	v := BatchJobReplicateV1{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobReplicateV1(t *testing.T) {
+	v := BatchJobReplicateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobReplicateV1 Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobReplicateV1{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobReplicateV1(b *testing.B) {
+	v := BatchJobReplicateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobReplicateV1(b *testing.B) {
+	v := BatchJobReplicateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchReplicateFilter(t *testing.T) {
+	v := BatchReplicateFilter{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchReplicateFilter(b *testing.B) {
+	v := BatchReplicateFilter{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchReplicateFilter(b *testing.B) {
+	v := BatchReplicateFilter{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchReplicateFilter(b *testing.B) {
+	v := BatchReplicateFilter{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchReplicateFilter(t *testing.T) {
+	v := BatchReplicateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchReplicateFilter Msgsize() is inaccurate")
+	}
+
+	vn := BatchReplicateFilter{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchReplicateFilter(b *testing.B) {
+	v := BatchReplicateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchReplicateFilter(b *testing.B) {
+	v := BatchReplicateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batch-rotate.go

@@ -18,13 +18,9 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"encoding/base64"
-	"encoding/json"
-	"errors"
 	"fmt"
-	"io"
 	"math/rand"
 	"net/http"
 	"runtime"
@@ -39,7 +35,6 @@ import (
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/env"
-	"github.com/minio/pkg/wildcard"
 	"github.com/minio/pkg/workers"
 )
 
@@ -76,56 +71,6 @@ import (
 
 //go:generate msgp -file $GOFILE -unexported
 
-// BatchKeyRotateKV is a datatype that holds key and values for filtering of objects
-// used by metadata filter as well as tags based filtering.
-type BatchKeyRotateKV struct {
-	Key   string `yaml:"key" json:"key"`
-	Value string `yaml:"value" json:"value"`
-}
-
-// Validate returns an error if key is empty
-func (kv BatchKeyRotateKV) Validate() error {
-	if kv.Key == "" {
-		return errInvalidArgument
-	}
-	return nil
-}
-
-// Empty indicates if kv is not set
-func (kv BatchKeyRotateKV) Empty() bool {
-	return kv.Key == "" && kv.Value == ""
-}
-
-// Match matches input kv with kv, value will be wildcard matched depending on the user input
-func (kv BatchKeyRotateKV) Match(ikv BatchKeyRotateKV) bool {
-	if kv.Empty() {
-		return true
-	}
-	if strings.EqualFold(kv.Key, ikv.Key) {
-		return wildcard.Match(kv.Value, ikv.Value)
-	}
-	return false
-}
-
-// BatchKeyRotateRetry datatype represents total retry attempts and delay between each retries.
-type BatchKeyRotateRetry struct {
-	Attempts int           `yaml:"attempts" json:"attempts"` // number of retry attempts
-	Delay    time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
-}
-
-// Validate validates input replicate retries.
-func (r BatchKeyRotateRetry) Validate() error {
-	if r.Attempts < 0 {
-		return errInvalidArgument
-	}
-
-	if r.Delay < 0 {
-		return errInvalidArgument
-	}
-
-	return nil
-}
-
 // BatchKeyRotationType defines key rotation type
 type BatchKeyRotationType string
 
@@ -178,13 +123,13 @@ func (e BatchJobKeyRotateEncryption) Validate() error {
 
 // BatchKeyRotateFilter holds all the filters currently supported for batch replication
 type BatchKeyRotateFilter struct {
-	NewerThan     time.Duration      `yaml:"newerThan,omitempty" json:"newerThan"`
-	OlderThan     time.Duration      `yaml:"olderThan,omitempty" json:"olderThan"`
-	CreatedAfter  time.Time          `yaml:"createdAfter,omitempty" json:"createdAfter"`
-	CreatedBefore time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
-	Tags          []BatchKeyRotateKV `yaml:"tags,omitempty" json:"tags"`
-	Metadata      []BatchKeyRotateKV `yaml:"metadata,omitempty" json:"metadata"`
-	KMSKeyID      string             `yaml:"kmskeyid" json:"kmskey"`
+	NewerThan     time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
+	OlderThan     time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedAfter  time.Time     `yaml:"createdAfter,omitempty" json:"createdAfter"`
+	CreatedBefore time.Time     `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchJobKV  `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchJobKV  `yaml:"metadata,omitempty" json:"metadata"`
+	KMSKeyID      string        `yaml:"kmskeyid" json:"kmskey"`
 }
 
 // BatchKeyRotateNotification success or failure notification endpoint for each job attempts
@@ -198,9 +143,9 @@ type BatchKeyRotateNotification struct {
 // - notify
 // - retry
 type BatchJobKeyRotateFlags struct {
-	Filter BatchKeyRotateFilter       `yaml:"filter" json:"filter"`
-	Notify BatchKeyRotateNotification `yaml:"notify" json:"notify"`
-	Retry  BatchKeyRotateRetry        `yaml:"retry" json:"retry"`
+	Filter BatchKeyRotateFilter `yaml:"filter" json:"filter"`
+	Notify BatchJobNotification `yaml:"notify" json:"notify"`
+	Retry  BatchJobRetry        `yaml:"retry" json:"retry"`
 }
 
 // BatchJobKeyRotateV1 v1 of batch key rotation job
@@ -214,35 +159,8 @@ type BatchJobKeyRotateV1 struct {
 }
 
 // Notify notifies notification endpoint if configured regarding job failure or success.
-func (r BatchJobKeyRotateV1) Notify(ctx context.Context, body io.Reader) error {
-	if r.Flags.Notify.Endpoint == "" {
-		return nil
-	}
-
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.Flags.Notify.Endpoint, body)
-	if err != nil {
-		return err
-	}
-
-	if r.Flags.Notify.Token != "" {
-		req.Header.Set("Authorization", r.Flags.Notify.Token)
-	}
-
-	clnt := http.Client{Transport: getRemoteInstanceTransport}
-	resp, err := clnt.Do(req)
-	if err != nil {
-		return err
-	}
-
-	xhttp.DrainBody(resp.Body)
-	if resp.StatusCode != http.StatusOK {
-		return errors.New(resp.Status)
-	}
-
-	return nil
+func (r BatchJobKeyRotateV1) Notify(ctx context.Context, ri *batchJobInfo) error {
+	return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token)
 }
 
 // KeyRotate rotates encryption key of an object
@@ -330,7 +248,7 @@ const (
 	batchKeyRotateVersion              = batchKeyRotateVersionV1
 	batchKeyRotateAPIVersion           = "v1"
 	batchKeyRotateJobDefaultRetries    = 3
-	batchKeyRotateJobDefaultRetryDelay = 250 * time.Millisecond
+	batchKeyRotateJobDefaultRetryDelay = 25 * time.Millisecond
 )
 
 // Start the batch key rottion job, resumes if there was a pending job via "job.ID"
@@ -351,6 +269,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	if delay == 0 {
 		delay = batchKeyRotateJobDefaultRetryDelay
 	}
+
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	skip := func(info FileInfo) (ok bool) {
@@ -388,7 +307,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 			for _, kv := range r.Flags.Filter.Tags {
 				for t, v := range tagMap {
-					if kv.Match(BatchKeyRotateKV{Key: t, Value: v}) {
+					if kv.Match(BatchJobKV{Key: t, Value: v}) {
 						return true
 					}
 				}
@@ -405,7 +324,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						continue
 					}
 					// We only need to match x-amz-meta or standardHeaders
-					if kv.Match(BatchKeyRotateKV{Key: k, Value: v}) {
+					if kv.Match(BatchJobKV{Key: k, Value: v}) {
 						return true
 					}
 				}
@@ -475,6 +394,9 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				if success {
 					break
 				}
+				if delay > 0 {
+					time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
+				}
 			}
 		}()
 	}
@@ -486,20 +408,11 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	// persist in-memory state to disk.
 	logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
-	buf, _ := json.Marshal(ri)
-	if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
+	if err := r.Notify(ctx, ri); err != nil {
 		logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
 	}
 
 	cancel()
-	if ri.Failed {
-		ri.ObjectsFailed = 0
-		ri.Bucket = ""
-		ri.Object = ""
-		ri.Objects = 0
-		time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
-	}
-
 	return nil
 }
 

cmd/batch-rotate_gen.go

@@ -192,75 +192,17 @@ func (z *BatchJobKeyRotateFlags) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Notify":
-			var zb0002 uint32
-			zb0002, err = dc.ReadMapHeader()
+			err = z.Notify.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "Notify")
 				return
 			}
-			for zb0002 > 0 {
-				zb0002--
-				field, err = dc.ReadMapKeyPtr()
-				if err != nil {
-					err = msgp.WrapError(err, "Notify")
-					return
-				}
-				switch msgp.UnsafeString(field) {
-				case "Endpoint":
-					z.Notify.Endpoint, err = dc.ReadString()
-					if err != nil {
-						err = msgp.WrapError(err, "Notify", "Endpoint")
-						return
-					}
-				case "Token":
-					z.Notify.Token, err = dc.ReadString()
-					if err != nil {
-						err = msgp.WrapError(err, "Notify", "Token")
-						return
-					}
-				default:
-					err = dc.Skip()
-					if err != nil {
-						err = msgp.WrapError(err, "Notify")
-						return
-					}
-				}
-			}
 		case "Retry":
-			var zb0003 uint32
-			zb0003, err = dc.ReadMapHeader()
+			err = z.Retry.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "Retry")
 				return
 			}
-			for zb0003 > 0 {
-				zb0003--
-				field, err = dc.ReadMapKeyPtr()
-				if err != nil {
-					err = msgp.WrapError(err, "Retry")
-					return
-				}
-				switch msgp.UnsafeString(field) {
-				case "Attempts":
-					z.Retry.Attempts, err = dc.ReadInt()
-					if err != nil {
-						err = msgp.WrapError(err, "Retry", "Attempts")
-						return
-					}
-				case "Delay":
-					z.Retry.Delay, err = dc.ReadDuration()
-					if err != nil {
-						err = msgp.WrapError(err, "Retry", "Delay")
-						return
-					}
-				default:
-					err = dc.Skip()
-					if err != nil {
-						err = msgp.WrapError(err, "Retry")
-						return
-					}
-				}
-			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -290,25 +232,9 @@ func (z *BatchJobKeyRotateFlags) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	// map header, size 2
-	// write "Endpoint"
-	err = en.Append(0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	err = z.Notify.EncodeMsg(en)
 	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Notify.Endpoint)
-	if err != nil {
-		err = msgp.WrapError(err, "Notify", "Endpoint")
-		return
-	}
-	// write "Token"
-	err = en.Append(0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Notify.Token)
-	if err != nil {
-		err = msgp.WrapError(err, "Notify", "Token")
+		err = msgp.WrapError(err, "Notify")
 		return
 	}
 	// write "Retry"
@@ -316,25 +242,9 @@ func (z *BatchJobKeyRotateFlags) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	// map header, size 2
-	// write "Attempts"
-	err = en.Append(0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
+	err = z.Retry.EncodeMsg(en)
 	if err != nil {
-		return
-	}
-	err = en.WriteInt(z.Retry.Attempts)
-	if err != nil {
-		err = msgp.WrapError(err, "Retry", "Attempts")
-		return
-	}
-	// write "Delay"
-	err = en.Append(0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
-	if err != nil {
-		return
-	}
-	err = en.WriteDuration(z.Retry.Delay)
-	if err != nil {
-		err = msgp.WrapError(err, "Retry", "Delay")
+		err = msgp.WrapError(err, "Retry")
 		return
 	}
 	return
@@ -353,22 +263,18 @@ func (z *BatchJobKeyRotateFlags) MarshalMsg(b []byte) (o []byte, err error) {
 	}
 	// string "Notify"
 	o = append(o, 0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79)
-	// map header, size 2
-	// string "Endpoint"
-	o = append(o, 0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
-	o = msgp.AppendString(o, z.Notify.Endpoint)
-	// string "Token"
-	o = append(o, 0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e)
-	o = msgp.AppendString(o, z.Notify.Token)
+	o, err = z.Notify.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Notify")
+		return
+	}
 	// string "Retry"
 	o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
-	// map header, size 2
-	// string "Attempts"
-	o = append(o, 0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
-	o = msgp.AppendInt(o, z.Retry.Attempts)
-	// string "Delay"
-	o = append(o, 0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
-	o = msgp.AppendDuration(o, z.Retry.Delay)
+	o, err = z.Retry.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Retry")
+		return
+	}
 	return
 }
 
@@ -397,75 +303,17 @@ func (z *BatchJobKeyRotateFlags) UnmarshalMsg(bts []byte) (o []byte, err error)
 				return
 			}
 		case "Notify":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			bts, err = z.Notify.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Notify")
 				return
 			}
-			for zb0002 > 0 {
-				zb0002--
-				field, bts, err = msgp.ReadMapKeyZC(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Notify")
-					return
-				}
-				switch msgp.UnsafeString(field) {
-				case "Endpoint":
-					z.Notify.Endpoint, bts, err = msgp.ReadStringBytes(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Notify", "Endpoint")
-						return
-					}
-				case "Token":
-					z.Notify.Token, bts, err = msgp.ReadStringBytes(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Notify", "Token")
-						return
-					}
-				default:
-					bts, err = msgp.Skip(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Notify")
-						return
-					}
-				}
-			}
 		case "Retry":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			bts, err = z.Retry.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Retry")
 				return
 			}
-			for zb0003 > 0 {
-				zb0003--
-				field, bts, err = msgp.ReadMapKeyZC(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Retry")
-					return
-				}
-				switch msgp.UnsafeString(field) {
-				case "Attempts":
-					z.Retry.Attempts, bts, err = msgp.ReadIntBytes(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Retry", "Attempts")
-						return
-					}
-				case "Delay":
-					z.Retry.Delay, bts, err = msgp.ReadDurationBytes(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Retry", "Delay")
-						return
-					}
-				default:
-					bts, err = msgp.Skip(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Retry")
-						return
-					}
-				}
-			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -480,7 +328,7 @@ func (z *BatchJobKeyRotateFlags) UnmarshalMsg(bts []byte) (o []byte, err error)
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobKeyRotateFlags) Msgsize() (s int) {
-	s = 1 + 7 + z.Filter.Msgsize() + 7 + 1 + 9 + msgp.StringPrefixSize + len(z.Notify.Endpoint) + 6 + msgp.StringPrefixSize + len(z.Notify.Token) + 6 + 1 + 9 + msgp.IntSize + 6 + msgp.DurationSize
+	s = 1 + 7 + z.Filter.Msgsize() + 7 + z.Notify.Msgsize() + 6 + z.Retry.Msgsize()
 	return
 }
 
@@ -509,11 +357,46 @@ func (z *BatchJobKeyRotateV1) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Flags":
-			err = z.Flags.DecodeMsg(dc)
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Flags")
 				return
 			}
+			for zb0002 > 0 {
+				zb0002--
+				field, err = dc.ReadMapKeyPtr()
+				if err != nil {
+					err = msgp.WrapError(err, "Flags")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "Filter":
+					err = z.Flags.Filter.DecodeMsg(dc)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Filter")
+						return
+					}
+				case "Notify":
+					err = z.Flags.Notify.DecodeMsg(dc)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Notify")
+						return
+					}
+				case "Retry":
+					err = z.Flags.Retry.DecodeMsg(dc)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Retry")
+						return
+					}
+				default:
+					err = dc.Skip()
+					if err != nil {
+						err = msgp.WrapError(err, "Flags")
+						return
+					}
+				}
+			}
 		case "Bucket":
 			z.Bucket, err = dc.ReadString()
 			if err != nil {
@@ -567,9 +450,35 @@ func (z *BatchJobKeyRotateV1) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = z.Flags.EncodeMsg(en)
+	// map header, size 3
+	// write "Filter"
+	err = en.Append(0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
 	if err != nil {
-		err = msgp.WrapError(err, "Flags")
+		return
+	}
+	err = z.Flags.Filter.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Flags", "Filter")
+		return
+	}
+	// write "Notify"
+	err = en.Append(0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79)
+	if err != nil {
+		return
+	}
+	err = z.Flags.Notify.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Flags", "Notify")
+		return
+	}
+	// write "Retry"
+	err = en.Append(0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
+	if err != nil {
+		return
+	}
+	err = z.Flags.Retry.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Flags", "Retry")
 		return
 	}
 	// write "Bucket"
@@ -624,9 +533,26 @@ func (z *BatchJobKeyRotateV1) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendString(o, z.APIVersion)
 	// string "Flags"
 	o = append(o, 0xa5, 0x46, 0x6c, 0x61, 0x67, 0x73)
-	o, err = z.Flags.MarshalMsg(o)
+	// map header, size 3
+	// string "Filter"
+	o = append(o, 0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
+	o, err = z.Flags.Filter.MarshalMsg(o)
 	if err != nil {
-		err = msgp.WrapError(err, "Flags")
+		err = msgp.WrapError(err, "Flags", "Filter")
+		return
+	}
+	// string "Notify"
+	o = append(o, 0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79)
+	o, err = z.Flags.Notify.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Flags", "Notify")
+		return
+	}
+	// string "Retry"
+	o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
+	o, err = z.Flags.Retry.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Flags", "Retry")
 		return
 	}
 	// string "Bucket"
@@ -673,11 +599,46 @@ func (z *BatchJobKeyRotateV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "Flags":
-			bts, err = z.Flags.UnmarshalMsg(bts)
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Flags")
 				return
 			}
+			for zb0002 > 0 {
+				zb0002--
+				field, bts, err = msgp.ReadMapKeyZC(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Flags")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "Filter":
+					bts, err = z.Flags.Filter.UnmarshalMsg(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Filter")
+						return
+					}
+				case "Notify":
+					bts, err = z.Flags.Notify.UnmarshalMsg(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Notify")
+						return
+					}
+				case "Retry":
+					bts, err = z.Flags.Retry.UnmarshalMsg(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Retry")
+						return
+					}
+				default:
+					bts, err = msgp.Skip(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags")
+						return
+					}
+				}
+			}
 		case "Bucket":
 			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
 			if err != nil {
@@ -716,7 +677,7 @@ func (z *BatchJobKeyRotateV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobKeyRotateV1) Msgsize() (s int) {
-	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 6 + z.Flags.Msgsize() + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 11 + z.Encryption.Msgsize()
+	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 6 + 1 + 7 + z.Flags.Filter.Msgsize() + 7 + z.Flags.Notify.Msgsize() + 6 + z.Flags.Retry.Msgsize() + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 11 + z.Encryption.Msgsize()
 	return
 }
 
@@ -772,91 +733,33 @@ func (z *BatchKeyRotateFilter) DecodeMsg(dc *msgp.Reader) (err error) {
 			if cap(z.Tags) >= int(zb0002) {
 				z.Tags = (z.Tags)[:zb0002]
 			} else {
-				z.Tags = make([]BatchKeyRotateKV, zb0002)
+				z.Tags = make([]BatchJobKV, zb0002)
 			}
 			for za0001 := range z.Tags {
-				var zb0003 uint32
-				zb0003, err = dc.ReadMapHeader()
+				err = z.Tags[za0001].DecodeMsg(dc)
 				if err != nil {
 					err = msgp.WrapError(err, "Tags", za0001)
 					return
 				}
-				for zb0003 > 0 {
-					zb0003--
-					field, err = dc.ReadMapKeyPtr()
-					if err != nil {
-						err = msgp.WrapError(err, "Tags", za0001)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "Key":
-						z.Tags[za0001].Key, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001, "Key")
-							return
-						}
-					case "Value":
-						z.Tags[za0001].Value, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001, "Value")
-							return
-						}
-					default:
-						err = dc.Skip()
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001)
-							return
-						}
-					}
-				}
 			}
 		case "Metadata":
-			var zb0004 uint32
-			zb0004, err = dc.ReadArrayHeader()
+			var zb0003 uint32
+			zb0003, err = dc.ReadArrayHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Metadata")
 				return
 			}
-			if cap(z.Metadata) >= int(zb0004) {
-				z.Metadata = (z.Metadata)[:zb0004]
+			if cap(z.Metadata) >= int(zb0003) {
+				z.Metadata = (z.Metadata)[:zb0003]
 			} else {
-				z.Metadata = make([]BatchKeyRotateKV, zb0004)
+				z.Metadata = make([]BatchJobKV, zb0003)
 			}
 			for za0002 := range z.Metadata {
-				var zb0005 uint32
-				zb0005, err = dc.ReadMapHeader()
+				err = z.Metadata[za0002].DecodeMsg(dc)
 				if err != nil {
 					err = msgp.WrapError(err, "Metadata", za0002)
 					return
 				}
-				for zb0005 > 0 {
-					zb0005--
-					field, err = dc.ReadMapKeyPtr()
-					if err != nil {
-						err = msgp.WrapError(err, "Metadata", za0002)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "Key":
-						z.Metadata[za0002].Key, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002, "Key")
-							return
-						}
-					case "Value":
-						z.Metadata[za0002].Value, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002, "Value")
-							return
-						}
-					default:
-						err = dc.Skip()
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002)
-							return
-						}
-					}
-				}
 			}
 		case "KMSKeyID":
 			z.KMSKeyID, err = dc.ReadString()
@@ -929,25 +832,9 @@ func (z *BatchKeyRotateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 		return
 	}
 	for za0001 := range z.Tags {
-		// map header, size 2
-		// write "Key"
-		err = en.Append(0x82, 0xa3, 0x4b, 0x65, 0x79)
+		err = z.Tags[za0001].EncodeMsg(en)
 		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Tags[za0001].Key)
-		if err != nil {
-			err = msgp.WrapError(err, "Tags", za0001, "Key")
-			return
-		}
-		// write "Value"
-		err = en.Append(0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Tags[za0001].Value)
-		if err != nil {
-			err = msgp.WrapError(err, "Tags", za0001, "Value")
+			err = msgp.WrapError(err, "Tags", za0001)
 			return
 		}
 	}
@@ -962,25 +849,9 @@ func (z *BatchKeyRotateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 		return
 	}
 	for za0002 := range z.Metadata {
-		// map header, size 2
-		// write "Key"
-		err = en.Append(0x82, 0xa3, 0x4b, 0x65, 0x79)
+		err = z.Metadata[za0002].EncodeMsg(en)
 		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Metadata[za0002].Key)
-		if err != nil {
-			err = msgp.WrapError(err, "Metadata", za0002, "Key")
-			return
-		}
-		// write "Value"
-		err = en.Append(0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Metadata[za0002].Value)
-		if err != nil {
-			err = msgp.WrapError(err, "Metadata", za0002, "Value")
+			err = msgp.WrapError(err, "Metadata", za0002)
 			return
 		}
 	}
@@ -1017,25 +888,21 @@ func (z *BatchKeyRotateFilter) MarshalMsg(b []byte) (o []byte, err error) {
 	o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(len(z.Tags)))
 	for za0001 := range z.Tags {
-		// map header, size 2
-		// string "Key"
-		o = append(o, 0x82, 0xa3, 0x4b, 0x65, 0x79)
-		o = msgp.AppendString(o, z.Tags[za0001].Key)
-		// string "Value"
-		o = append(o, 0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-		o = msgp.AppendString(o, z.Tags[za0001].Value)
+		o, err = z.Tags[za0001].MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Tags", za0001)
+			return
+		}
 	}
 	// string "Metadata"
 	o = append(o, 0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61)
 	o = msgp.AppendArrayHeader(o, uint32(len(z.Metadata)))
 	for za0002 := range z.Metadata {
-		// map header, size 2
-		// string "Key"
-		o = append(o, 0x82, 0xa3, 0x4b, 0x65, 0x79)
-		o = msgp.AppendString(o, z.Metadata[za0002].Key)
-		// string "Value"
-		o = append(o, 0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-		o = msgp.AppendString(o, z.Metadata[za0002].Value)
+		o, err = z.Metadata[za0002].MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Metadata", za0002)
+			return
+		}
 	}
 	// string "KMSKeyID"
 	o = append(o, 0xa8, 0x4b, 0x4d, 0x53, 0x4b, 0x65, 0x79, 0x49, 0x44)
@@ -1095,91 +962,33 @@ func (z *BatchKeyRotateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			if cap(z.Tags) >= int(zb0002) {
 				z.Tags = (z.Tags)[:zb0002]
 			} else {
-				z.Tags = make([]BatchKeyRotateKV, zb0002)
+				z.Tags = make([]BatchJobKV, zb0002)
 			}
 			for za0001 := range z.Tags {
-				var zb0003 uint32
-				zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+				bts, err = z.Tags[za0001].UnmarshalMsg(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Tags", za0001)
 					return
 				}
-				for zb0003 > 0 {
-					zb0003--
-					field, bts, err = msgp.ReadMapKeyZC(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Tags", za0001)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "Key":
-						z.Tags[za0001].Key, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001, "Key")
-							return
-						}
-					case "Value":
-						z.Tags[za0001].Value, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001, "Value")
-							return
-						}
-					default:
-						bts, err = msgp.Skip(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001)
-							return
-						}
-					}
-				}
 			}
 		case "Metadata":
-			var zb0004 uint32
-			zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Metadata")
 				return
 			}
-			if cap(z.Metadata) >= int(zb0004) {
-				z.Metadata = (z.Metadata)[:zb0004]
+			if cap(z.Metadata) >= int(zb0003) {
+				z.Metadata = (z.Metadata)[:zb0003]
 			} else {
-				z.Metadata = make([]BatchKeyRotateKV, zb0004)
+				z.Metadata = make([]BatchJobKV, zb0003)
 			}
 			for za0002 := range z.Metadata {
-				var zb0005 uint32
-				zb0005, bts, err = msgp.ReadMapHeaderBytes(bts)
+				bts, err = z.Metadata[za0002].UnmarshalMsg(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Metadata", za0002)
 					return
 				}
-				for zb0005 > 0 {
-					zb0005--
-					field, bts, err = msgp.ReadMapKeyZC(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Metadata", za0002)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "Key":
-						z.Metadata[za0002].Key, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002, "Key")
-							return
-						}
-					case "Value":
-						z.Metadata[za0002].Value, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002, "Value")
-							return
-						}
-					default:
-						bts, err = msgp.Skip(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002)
-							return
-						}
-					}
-				}
 			}
 		case "KMSKeyID":
 			z.KMSKeyID, bts, err = msgp.ReadStringBytes(bts)
@@ -1203,144 +1012,16 @@ func (z *BatchKeyRotateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 func (z *BatchKeyRotateFilter) Msgsize() (s int) {
 	s = 1 + 10 + msgp.DurationSize + 10 + msgp.DurationSize + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize
 	for za0001 := range z.Tags {
-		s += 1 + 4 + msgp.StringPrefixSize + len(z.Tags[za0001].Key) + 6 + msgp.StringPrefixSize + len(z.Tags[za0001].Value)
+		s += z.Tags[za0001].Msgsize()
 	}
 	s += 9 + msgp.ArrayHeaderSize
 	for za0002 := range z.Metadata {
-		s += 1 + 4 + msgp.StringPrefixSize + len(z.Metadata[za0002].Key) + 6 + msgp.StringPrefixSize + len(z.Metadata[za0002].Value)
+		s += z.Metadata[za0002].Msgsize()
 	}
 	s += 9 + msgp.StringPrefixSize + len(z.KMSKeyID)
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *BatchKeyRotateKV) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Key":
-			z.Key, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Key")
-				return
-			}
-		case "Value":
-			z.Value, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Value")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z BatchKeyRotateKV) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
-	// write "Key"
-	err = en.Append(0x82, 0xa3, 0x4b, 0x65, 0x79)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Key)
-	if err != nil {
-		err = msgp.WrapError(err, "Key")
-		return
-	}
-	// write "Value"
-	err = en.Append(0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Value)
-	if err != nil {
-		err = msgp.WrapError(err, "Value")
-		return
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z BatchKeyRotateKV) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
-	// string "Key"
-	o = append(o, 0x82, 0xa3, 0x4b, 0x65, 0x79)
-	o = msgp.AppendString(o, z.Key)
-	// string "Value"
-	o = append(o, 0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-	o = msgp.AppendString(o, z.Value)
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *BatchKeyRotateKV) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Key":
-			z.Key, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Key")
-				return
-			}
-		case "Value":
-			z.Value, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Value")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z BatchKeyRotateKV) Msgsize() (s int) {
-	s = 1 + 4 + msgp.StringPrefixSize + len(z.Key) + 6 + msgp.StringPrefixSize + len(z.Value)
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *BatchKeyRotateNotification) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -1469,134 +1150,6 @@ func (z BatchKeyRotateNotification) Msgsize() (s int) {
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *BatchKeyRotateRetry) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Attempts":
-			z.Attempts, err = dc.ReadInt()
-			if err != nil {
-				err = msgp.WrapError(err, "Attempts")
-				return
-			}
-		case "Delay":
-			z.Delay, err = dc.ReadDuration()
-			if err != nil {
-				err = msgp.WrapError(err, "Delay")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z BatchKeyRotateRetry) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
-	// write "Attempts"
-	err = en.Append(0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt(z.Attempts)
-	if err != nil {
-		err = msgp.WrapError(err, "Attempts")
-		return
-	}
-	// write "Delay"
-	err = en.Append(0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
-	if err != nil {
-		return
-	}
-	err = en.WriteDuration(z.Delay)
-	if err != nil {
-		err = msgp.WrapError(err, "Delay")
-		return
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z BatchKeyRotateRetry) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
-	// string "Attempts"
-	o = append(o, 0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
-	o = msgp.AppendInt(o, z.Attempts)
-	// string "Delay"
-	o = append(o, 0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
-	o = msgp.AppendDuration(o, z.Delay)
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *BatchKeyRotateRetry) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Attempts":
-			z.Attempts, bts, err = msgp.ReadIntBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Attempts")
-				return
-			}
-		case "Delay":
-			z.Delay, bts, err = msgp.ReadDurationBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Delay")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z BatchKeyRotateRetry) Msgsize() (s int) {
-	s = 1 + 9 + msgp.IntSize + 6 + msgp.DurationSize
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *BatchKeyRotationType) DecodeMsg(dc *msgp.Reader) (err error) {
 	{

cmd/batch-rotate_gen_test.go

@@ -461,119 +461,6 @@ func BenchmarkDecodeBatchKeyRotateFilter(b *testing.B) {
 	}
 }
 
-func TestMarshalUnmarshalBatchKeyRotateKV(t *testing.T) {
-	v := BatchKeyRotateKV{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgBatchKeyRotateKV(b *testing.B) {
-	v := BatchKeyRotateKV{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgBatchKeyRotateKV(b *testing.B) {
-	v := BatchKeyRotateKV{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalBatchKeyRotateKV(b *testing.B) {
-	v := BatchKeyRotateKV{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodeBatchKeyRotateKV(t *testing.T) {
-	v := BatchKeyRotateKV{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateKV Msgsize() is inaccurate")
-	}
-
-	vn := BatchKeyRotateKV{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodeBatchKeyRotateKV(b *testing.B) {
-	v := BatchKeyRotateKV{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodeBatchKeyRotateKV(b *testing.B) {
-	v := BatchKeyRotateKV{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
 func TestMarshalUnmarshalBatchKeyRotateNotification(t *testing.T) {
 	v := BatchKeyRotateNotification{}
 	bts, err := v.MarshalMsg(nil)
@@ -686,116 +573,3 @@ func BenchmarkDecodeBatchKeyRotateNotification(b *testing.B) {
 		}
 	}
 }
-
-func TestMarshalUnmarshalBatchKeyRotateRetry(t *testing.T) {
-	v := BatchKeyRotateRetry{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgBatchKeyRotateRetry(b *testing.B) {
-	v := BatchKeyRotateRetry{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgBatchKeyRotateRetry(b *testing.B) {
-	v := BatchKeyRotateRetry{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalBatchKeyRotateRetry(b *testing.B) {
-	v := BatchKeyRotateRetry{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodeBatchKeyRotateRetry(t *testing.T) {
-	v := BatchKeyRotateRetry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateRetry Msgsize() is inaccurate")
-	}
-
-	vn := BatchKeyRotateRetry{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodeBatchKeyRotateRetry(b *testing.B) {
-	v := BatchKeyRotateRetry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodeBatchKeyRotateRetry(b *testing.B) {
-	v := BatchKeyRotateRetry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}

cmd/bitrot-streaming.go

@@ -94,16 +94,23 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
 	r, w := io.Pipe()
 	h := algo.New()
 
-	bw := &streamingBitrotWriter{iow: w, closeWithErr: w.CloseWithError, h: h, shardSize: shardSize, canClose: &sync.WaitGroup{}}
+	bw := &streamingBitrotWriter{
+		iow:          ioutil.NewDeadlineWriter(w, diskMaxTimeout),
+		closeWithErr: w.CloseWithError,
+		h:            h,
+		shardSize:    shardSize,
+		canClose:     &sync.WaitGroup{},
+	}
 	bw.canClose.Add(1)
 	go func() {
+		defer bw.canClose.Done()
+
 		totalFileSize := int64(-1) // For compressed objects length will be unknown (represented by length=-1)
 		if length != -1 {
 			bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
 			totalFileSize = bitrotSumsTotalSize + length
 		}
 		r.CloseWithError(disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r))
-		bw.canClose.Done()
 	}()
 	return bw
 }

cmd/bootstrap-messages.go

@@ -19,98 +19,51 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"sync"
-	"time"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/pubsub"
 )
 
-const bootstrapMsgsLimit = 4 << 10
+const bootstrapTraceLimit = 4 << 10
 
-type bootstrapInfo struct {
-	msg    string
-	ts     time.Time
-	source string
-}
 type bootstrapTracer struct {
-	mu         sync.RWMutex
-	idx        int
-	info       [bootstrapMsgsLimit]bootstrapInfo
-	lastUpdate time.Time
+	mu   sync.RWMutex
+	info []madmin.TraceInfo
 }
 
 var globalBootstrapTracer = &bootstrapTracer{}
 
-func (bs *bootstrapTracer) DropEvents() {
+func (bs *bootstrapTracer) Record(info madmin.TraceInfo) {
 	bs.mu.Lock()
 	defer bs.mu.Unlock()
 
-	if time.Now().UTC().Sub(bs.lastUpdate) > 24*time.Hour {
-		bs.info = [4096]bootstrapInfo{}
-		bs.idx = 0
+	if len(bs.info) > bootstrapTraceLimit {
+		return
 	}
-}
-
-func (bs *bootstrapTracer) Empty() bool {
-	var empty bool
-	bs.mu.RLock()
-	empty = bs.info[0].msg == ""
-	bs.mu.RUnlock()
-
-	return empty
-}
-
-func (bs *bootstrapTracer) Record(msg string, skip int) {
-	source := getSource(skip + 1)
-	bs.mu.Lock()
-	now := time.Now().UTC()
-	bs.info[bs.idx] = bootstrapInfo{
-		msg:    msg,
-		ts:     now,
-		source: source,
-	}
-	bs.lastUpdate = now
-	bs.idx = (bs.idx + 1) % bootstrapMsgsLimit
-	bs.mu.Unlock()
+	bs.info = append(bs.info, info)
 }
 
 func (bs *bootstrapTracer) Events() []madmin.TraceInfo {
-	traceInfo := make([]madmin.TraceInfo, 0, bootstrapMsgsLimit)
-
-	// Add all messages in order
-	addAll := func(info []bootstrapInfo) {
-		for _, msg := range info {
-			if msg.ts.IsZero() {
-				continue // skip empty events
-			}
-			traceInfo = append(traceInfo, madmin.TraceInfo{
-				TraceType: madmin.TraceBootstrap,
-				Time:      msg.ts,
-				NodeName:  globalLocalNodeName,
-				FuncName:  "BOOTSTRAP",
-				Message:   fmt.Sprintf("%s %s", msg.source, msg.msg),
-			})
-		}
-	}
+	traceInfo := make([]madmin.TraceInfo, 0, bootstrapTraceLimit)
 
 	bs.mu.RLock()
-	addAll(bs.info[bs.idx:])
-	addAll(bs.info[:bs.idx])
+	for _, i := range bs.info {
+		traceInfo = append(traceInfo, i)
+	}
 	bs.mu.RUnlock()
+
 	return traceInfo
 }
 
 func (bs *bootstrapTracer) Publish(ctx context.Context, trace *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) {
-	if bs.Empty() {
-		return
-	}
 	for _, bsEvent := range bs.Events() {
-		select {
-		case <-ctx.Done():
-		default:
-			trace.Publish(bsEvent)
+		if bsEvent.Message != "" {
+			select {
+			case <-ctx.Done():
+			default:
+				trace.Publish(bsEvent)
+			}
 		}
 	}
 }

cmd/bootstrap-messages_test.go

@@ -1,60 +0,0 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestBootstrap(t *testing.T) {
-	// Bootstrap events exceed bootstrap messages limit
-	bsTracer := &bootstrapTracer{}
-	for i := 0; i < bootstrapMsgsLimit+10; i++ {
-		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
-	}
-
-	traceInfos := bsTracer.Events()
-	if len(traceInfos) != bootstrapMsgsLimit {
-		t.Fatalf("Expected length of events %d but got %d", bootstrapMsgsLimit, len(traceInfos))
-	}
-
-	// Simulate the case where bootstrap events were updated a day ago
-	bsTracer.lastUpdate = time.Now().UTC().Add(-25 * time.Hour)
-	bsTracer.DropEvents()
-	if !bsTracer.Empty() {
-		t.Fatalf("Expected all bootstrap events to have been dropped, but found %d events", len(bsTracer.Events()))
-	}
-
-	// Fewer than 4K bootstrap events
-	for i := 0; i < 10; i++ {
-		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
-	}
-	events := bsTracer.Events()
-	if len(events) != 10 {
-		t.Fatalf("Expected length of events %d but got %d", 10, len(events))
-	}
-	for i, traceInfo := range bsTracer.Events() {
-		msg := fmt.Sprintf("msg-%d", i)
-		if !strings.HasSuffix(traceInfo.Message, msg) {
-			t.Fatalf("Expected %s but got %s", msg, traceInfo.Message)
-		}
-	}
-}

cmd/bootstrap-peer-server.go

@@ -221,7 +221,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 	for onlineServers < len(clnts)/2 {
 		for _, clnt := range clnts {
 			if err := clnt.Verify(ctx, srcCfg); err != nil {
-				bootstrapTrace(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
+				bootstrapTraceMsg(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
 				if !isNetworkError(err) {
 					logger.LogOnceIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err), clnt.String())
 					incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))

cmd/bucket-handlers_test.go

@@ -355,7 +355,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 			maxUploads:         "0",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusNotFound,
+			expectedRespStatus: http.StatusBadRequest,
 			shouldPass:         false,
 		},
 		// Test case - 2.

cmd/bucket-lifecycle.go

@@ -280,8 +280,10 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
 			if err := transitionObject(t.ctx, objectAPI, task.objInfo, newLifecycleAuditEvent(task.src, task.event)); err != nil {
-				logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
-					task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
+				if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+					logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
+						task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
+				}
 			} else {
 				ts := tierStats{
 					TotalSize:   uint64(task.objInfo.Size),
@@ -403,13 +405,11 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		TierName:  oi.TransitionedObject.Tier,
 	}
 	if err := globalTierJournal.AddEntry(entry); err != nil {
-		logger.LogIf(ctx, err)
 		return err
 	}
 	// Delete metadata on source, now that data in remote tier has been
 	// marked for deletion.
 	if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
-		logger.LogIf(ctx, err)
 		return err
 	}
 

cmd/bucket-listobjects-handlers.go

@@ -231,7 +231,7 @@ func parseRequestToken(token string) (subToken string, nodeIndex int) {
 	if token == "" {
 		return token, -1
 	}
-	i := strings.Index(token, "@")
+	i := strings.Index(token, ":")
 	if i < 0 {
 		return token, -1
 	}

cmd/bucket-metadata-sys.go

@@ -151,14 +151,27 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 		return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
 
-	if err := meta.Save(ctx, objAPI); err != nil {
-		return updatedAt, err
+	err = sys.save(ctx, meta)
+	return updatedAt, err
+}
+
+func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
 	}
 
-	sys.Set(bucket, meta)
-	globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket) // Do not use caller context here
+	if isMinioMetaBucketName(meta.Name) {
+		return errInvalidArgument
+	}
 
-	return updatedAt, nil
+	if err := meta.Save(ctx, objAPI); err != nil {
+		return err
+	}
+
+	sys.Set(meta.Name, meta)
+	globalNotificationSys.LoadBucketMetadata(bgContext(ctx), meta.Name) // Do not use caller context here
+	return nil
 }
 
 // Delete delete the bucket metadata for the specified bucket.

cmd/bucket-metadata.go

@@ -143,39 +143,38 @@ func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
 
 // Load - loads the metadata of bucket by name from ObjectLayer api.
 // If an error is returned the returned metadata will be default initialized.
-func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
+func readBucketMetadata(ctx context.Context, api ObjectLayer, name string) (BucketMetadata, error) {
 	if name == "" {
 		logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
-		return errInvalidArgument
+		return BucketMetadata{}, errInvalidArgument
 	}
+	b := newBucketMetadata(name)
 	configFile := path.Join(bucketMetaPrefix, name, bucketMetadataFile)
 	data, err := readConfig(ctx, api, configFile)
 	if err != nil {
-		return err
+		return b, err
 	}
 	if len(data) <= 4 {
-		return fmt.Errorf("loadBucketMetadata: no data")
+		return b, fmt.Errorf("loadBucketMetadata: no data")
 	}
 	// Read header
 	switch binary.LittleEndian.Uint16(data[0:2]) {
 	case bucketMetadataFormat:
 	default:
-		return fmt.Errorf("loadBucketMetadata: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+		return b, fmt.Errorf("loadBucketMetadata: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
 	}
 	switch binary.LittleEndian.Uint16(data[2:4]) {
 	case bucketMetadataVersion:
 	default:
-		return fmt.Errorf("loadBucketMetadata: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+		return b, fmt.Errorf("loadBucketMetadata: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
 	}
-	// OK, parse data.
 	_, err = b.UnmarshalMsg(data[4:])
-	b.Name = name // in-case parsing failed for some reason, make sure bucket name is not empty.
-	return err
+	return b, err
 }
 
 func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
-	b := newBucketMetadata(bucket)
-	err := b.Load(ctx, objectAPI, b.Name)
+	b, err := readBucketMetadata(ctx, objectAPI, bucket)
+	b.Name = bucket // in-case parsing failed for some reason, make sure bucket name is not empty.
 	if err != nil && !errors.Is(err, errConfigNotFound) {
 		return b, err
 	}

cmd/bucket-policy-handlers_test.go

@@ -289,7 +289,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			policyLen:          len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusNotFound,
+			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 10.
 		// Existent bucket with policy with Version field empty.
@@ -463,7 +463,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			accessKey:            credentials.AccessKey,
 			secretKey:            credentials.SecretKey,
 			expectedBucketPolicy: "",
-			expectedRespStatus:   http.StatusNotFound,
+			expectedRespStatus:   http.StatusBadRequest,
 		},
 	}
 	// Iterating over the cases, fetching the policy and validating the response.
@@ -686,7 +686,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 			bucketName:         ".invalid-bucket-name",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusNotFound,
+			expectedRespStatus: http.StatusBadRequest,
 		},
 	}
 	// Iterating over the cases and deleting the bucket policy and then asserting response.

cmd/bucket-replication-handlers.go

@@ -193,7 +193,7 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// GetBucketReplicationMetricsHandler - GET Bucket replication metrics.
+// GetBucketReplicationMetricsHandler - GET Bucket replication metrics.		// Deprecated Aug 2023
 // ----------
 // Gets the replication metrics for a bucket.
 func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseWriter, r *http.Request) {
@@ -227,29 +227,81 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 		return
 	}
 
-	var usageInfo BucketUsageInfo
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	if err == nil && !dataUsageInfo.LastUpdate.IsZero() {
-		usageInfo = dataUsageInfo.BucketsUsage[bucket]
+	w.Header().Set(xhttp.ContentType, string(mimeJSON))
+
+	enc := json.NewEncoder(w)
+	stats := globalReplicationStats.getLatestReplicationStats(bucket)
+	bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
+	bwMap := bwRpt.BucketStats[bucket]
+	for arn, st := range stats.ReplicationStats.Stats {
+		if bwMap != nil {
+			if bw, ok := bwMap[arn]; ok {
+				st.BandWidthLimitInBytesPerSecond = bw.LimitInBytesPerSecond
+				st.CurrentBandwidthInBytesPerSecond = bw.CurrentBandwidthInBytesPerSecond
+				stats.ReplicationStats.Stats[arn] = st
+			}
+		}
+	}
+
+	if err := enc.Encode(stats.ReplicationStats); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+}
+
+// GetBucketReplicationMetricsV2Handler - GET Bucket replication metrics.
+// ----------
+// Gets the replication metrics for a bucket.
+func (api objectAPIHandlers) GetBucketReplicationMetricsV2Handler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "GetBucketReplicationMetricsV2")
+
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+
+	// check if user has permissions to perform this operation
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+
+	if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
 	}
 
 	w.Header().Set(xhttp.ContentType, string(mimeJSON))
 
 	enc := json.NewEncoder(w)
-	stats := globalReplicationStats.getLatestReplicationStats(bucket, usageInfo)
+	stats := globalReplicationStats.getLatestReplicationStats(bucket)
 	bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
 	bwMap := bwRpt.BucketStats[bucket]
-	for arn, st := range stats.Stats {
+	for arn, st := range stats.ReplicationStats.Stats {
 		if bwMap != nil {
 			if bw, ok := bwMap[arn]; ok {
 				st.BandWidthLimitInBytesPerSecond = bw.LimitInBytesPerSecond
 				st.CurrentBandwidthInBytesPerSecond = bw.CurrentBandwidthInBytesPerSecond
-				stats.Stats[arn] = st
+				stats.ReplicationStats.Stats[arn] = st
 			}
 		}
 	}
-	if err = enc.Encode(stats); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+	stats.Uptime = UTCNow().Unix() - globalBootTime.Unix()
+
+	if err := enc.Encode(stats); err != nil {
+		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 }
@@ -533,7 +585,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
 		if rule.Status == replication.Disabled {
 			continue
 		}
-		clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rule.Destination.Bucket)
+		clnt := globalBucketTargetSys.GetRemoteTargetClient(rule.Destination.Bucket)
 		if clnt == nil {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetNotFoundError, fmt.Errorf("replication config with rule ID %s has a stale target", rule.ID)), r.URL)
 			return

cmd/bucket-replication-metrics.go

@@ -0,0 +1,381 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/rcrowley/go-metrics"
+)
+
+//go:generate msgp -file $GOFILE
+
+const (
+	// beta is the weight used to calculate exponential moving average
+	beta = 0.1 // Number of averages considered = 1/(1-beta)
+)
+
+// rateMeasurement captures the transfer details for one bucket/target
+//msgp:ignore rateMeasurement
+
+type rateMeasurement struct {
+	lock                 sync.Mutex
+	bytesSinceLastWindow uint64    // Total bytes since last window was processed
+	startTime            time.Time // Start time for window
+	expMovingAvg         float64   // Previously calculated exponential moving average
+}
+
+// newRateMeasurement creates a new instance of the measurement with the initial start time.
+func newRateMeasurement(initTime time.Time) *rateMeasurement {
+	return &rateMeasurement{
+		startTime: initTime,
+	}
+}
+
+// incrementBytes add bytes reported for a bucket/target.
+func (m *rateMeasurement) incrementBytes(bytes uint64) {
+	atomic.AddUint64(&m.bytesSinceLastWindow, bytes)
+}
+
+// updateExponentialMovingAverage processes the measurements captured so far.
+func (m *rateMeasurement) updateExponentialMovingAverage(endTime time.Time) {
+	// Calculate aggregate avg bandwidth and exp window avg
+	m.lock.Lock()
+	defer func() {
+		m.startTime = endTime
+		m.lock.Unlock()
+	}()
+
+	if m.startTime.IsZero() {
+		return
+	}
+
+	if endTime.Before(m.startTime) {
+		return
+	}
+
+	duration := endTime.Sub(m.startTime)
+
+	bytesSinceLastWindow := atomic.SwapUint64(&m.bytesSinceLastWindow, 0)
+
+	if m.expMovingAvg == 0 {
+		// Should address initial calculation and should be fine for resuming from 0
+		m.expMovingAvg = float64(bytesSinceLastWindow) / duration.Seconds()
+		return
+	}
+
+	increment := float64(bytesSinceLastWindow) / duration.Seconds()
+	m.expMovingAvg = exponentialMovingAverage(beta, m.expMovingAvg, increment)
+}
+
+// exponentialMovingAverage calculates the exponential moving average
+func exponentialMovingAverage(beta, previousAvg, incrementAvg float64) float64 {
+	return (1-beta)*incrementAvg + beta*previousAvg
+}
+
+// getExpMovingAvgBytesPerSecond returns the exponential moving average for the bucket/target in bytes
+func (m *rateMeasurement) getExpMovingAvgBytesPerSecond() float64 {
+	m.lock.Lock()
+	defer m.lock.Unlock()
+	return m.expMovingAvg
+}
+
+// ActiveWorkerStat is stat for active replication workers
+type ActiveWorkerStat struct {
+	Curr int     `json:"curr"`
+	Avg  float32 `json:"avg"`
+	Max  int     `json:"max"`
+	hist metrics.Histogram
+}
+
+func newActiveWorkerStat(r metrics.Registry) *ActiveWorkerStat {
+	h := metrics.NewHistogram(metrics.NewUniformSample(100))
+	r.Register("replication.active_workers", h)
+	return &ActiveWorkerStat{
+		hist: h,
+	}
+}
+
+// update curr and max workers;
+func (a *ActiveWorkerStat) update() {
+	if a == nil {
+		return
+	}
+	a.Curr = globalReplicationPool.ActiveWorkers()
+	a.hist.Update(int64(a.Curr))
+	a.Avg = float32(a.hist.Mean())
+	a.Max = int(a.hist.Max())
+}
+
+func (a *ActiveWorkerStat) get() ActiveWorkerStat {
+	w := ActiveWorkerStat{
+		Curr: a.Curr,
+		Avg:  a.Avg,
+		Max:  a.Max,
+	}
+	return w
+}
+
+// QStat holds queue stats for replication
+type QStat struct {
+	Count float64 `json:"count"`
+	Bytes float64 `json:"bytes"`
+}
+
+func (q *QStat) add(o QStat) QStat {
+	return QStat{Bytes: q.Bytes + o.Bytes, Count: q.Count + o.Count}
+}
+
+// InQueueMetric holds queue stats for replication
+type InQueueMetric struct {
+	Curr QStat `json:"curr" msg:"cq"`
+	Avg  QStat `json:"avg" msg:"aq"`
+	Max  QStat `json:"max" msg:"pq"`
+}
+
+func (qm InQueueMetric) merge(o InQueueMetric) InQueueMetric {
+	return InQueueMetric{
+		Curr: qm.Curr.add(o.Curr),
+		Avg:  qm.Avg.add(o.Avg),
+		Max:  qm.Max.add(o.Max),
+	}
+}
+
+type queueCache struct {
+	srQueueStats InQueueStats
+	bucketStats  map[string]InQueueStats
+	sync.RWMutex // mutex for queue stats
+}
+
+func newQueueCache(r metrics.Registry) queueCache {
+	return queueCache{
+		bucketStats:  make(map[string]InQueueStats),
+		srQueueStats: newInQueueStats(r, "site"),
+	}
+}
+
+func (q *queueCache) update() {
+	q.Lock()
+	defer q.Unlock()
+	q.srQueueStats.update()
+	for _, s := range q.bucketStats {
+		s.update()
+	}
+}
+
+func (q *queueCache) getBucketStats(bucket string) InQueueMetric {
+	q.RLock()
+	defer q.RUnlock()
+	v, ok := q.bucketStats[bucket]
+	if !ok {
+		return InQueueMetric{}
+	}
+	return InQueueMetric{
+		Curr: QStat{Bytes: float64(v.nowBytes), Count: float64(v.nowCount)},
+		Max:  QStat{Bytes: float64(v.histBytes.Max()), Count: float64(v.histCounts.Max())},
+		Avg:  QStat{Bytes: v.histBytes.Mean(), Count: v.histCounts.Mean()},
+	}
+}
+
+func (q *queueCache) getSiteStats() InQueueMetric {
+	q.RLock()
+	defer q.RUnlock()
+	v := q.srQueueStats
+	return InQueueMetric{
+		Curr: QStat{Bytes: float64(v.nowBytes), Count: float64(v.nowCount)},
+		Max:  QStat{Bytes: float64(v.histBytes.Max()), Count: float64(v.histCounts.Max())},
+		Avg:  QStat{Bytes: v.histBytes.Mean(), Count: v.histCounts.Mean()},
+	}
+}
+
+// InQueueStats holds queue stats for replication
+type InQueueStats struct {
+	nowBytes   int64 `json:"-"`
+	nowCount   int64 `json:"-"`
+	histCounts metrics.Histogram
+	histBytes  metrics.Histogram
+}
+
+func newInQueueStats(r metrics.Registry, lbl string) InQueueStats {
+	histCounts := metrics.NewHistogram(metrics.NewUniformSample(100))
+	histBytes := metrics.NewHistogram(metrics.NewUniformSample(100))
+
+	r.Register("replication.queue.counts."+lbl, histCounts)
+	r.Register("replication.queue.bytes."+lbl, histBytes)
+
+	return InQueueStats{
+		histCounts: histCounts,
+		histBytes:  histBytes,
+	}
+}
+
+func (q *InQueueStats) update() {
+	q.histBytes.Update(atomic.LoadInt64(&q.nowBytes))
+	q.histCounts.Update(atomic.LoadInt64(&q.nowCount))
+}
+
+// XferStats has transfer stats for replication
+type XferStats struct {
+	Curr    float64          `json:"currRate" msg:"cr"`
+	Avg     float64          `json:"avgRate" msg:"av"`
+	Peak    float64          `json:"peakRate" msg:"p"`
+	N       int64            `json:"n" msg:"n"`
+	measure *rateMeasurement `json:"-"`
+	sma     *SMA             `json:"-"`
+}
+
+// Clone returns a copy of XferStats
+func (rx *XferStats) Clone() *XferStats {
+	curr := rx.curr()
+	peak := rx.Peak
+	if curr > peak {
+		peak = curr
+	}
+	return &XferStats{
+		Curr:    curr,
+		Avg:     rx.Avg,
+		Peak:    peak,
+		N:       rx.N,
+		measure: rx.measure,
+	}
+}
+
+func newXferStats() *XferStats {
+	return &XferStats{
+		measure: newRateMeasurement(time.Now()),
+		sma:     newSMA(50),
+	}
+}
+
+func (rx *XferStats) String() string {
+	return fmt.Sprintf("curr=%f avg=%f, peak=%f", rx.curr(), rx.Avg, rx.Peak)
+}
+
+func (rx *XferStats) curr() float64 {
+	if rx.measure == nil {
+		return 0.0
+	}
+	return rx.measure.getExpMovingAvgBytesPerSecond()
+}
+
+func (rx *XferStats) merge(o XferStats) XferStats {
+	curr := calcAvg(rx.curr(), o.curr(), rx.N, o.N)
+	peak := rx.Peak
+	if o.Peak > peak {
+		peak = o.Peak
+	}
+	if curr > peak {
+		peak = curr
+	}
+	return XferStats{
+		Avg:     calcAvg(rx.Avg, o.Avg, rx.N, o.N),
+		Peak:    peak,
+		Curr:    curr,
+		measure: rx.measure,
+		N:       rx.N + o.N,
+	}
+}
+
+func calcAvg(x, y float64, n1, n2 int64) float64 {
+	if n1+n2 == 0 {
+		return 0
+	}
+	avg := (x*float64(n1) + y*float64(n2)) / float64(n1+n2)
+	return avg
+}
+
+// Add a new transfer
+func (rx *XferStats) addSize(sz int64, t time.Duration) {
+	if rx.measure == nil {
+		rx.measure = newRateMeasurement(time.Now())
+	}
+	rx.measure.incrementBytes(uint64(sz))
+	rx.Curr = rx.measure.getExpMovingAvgBytesPerSecond()
+	rx.sma.addSample(rx.Curr)
+	rx.Avg = rx.sma.simpleMovingAvg()
+	if rx.Curr > rx.Peak {
+		rx.Peak = rx.Curr
+	}
+	rx.N++
+}
+
+// ReplicationMRFStats holds stats of MRF backlog saved to disk in the last 5 minutes
+// and number of entries that failed replication after 3 retries
+type ReplicationMRFStats struct {
+	LastFailedCount uint64 `json:"failedCount_last5min"`
+	// Count of unreplicated entries that were dropped after MRF retry limit reached since cluster start.
+	TotalDroppedCount uint64 `json:"droppedCount_since_uptime"`
+	// Bytes of unreplicated entries that were dropped after MRF retry limit reached since cluster start.
+	TotalDroppedBytes uint64 `json:"droppedBytes_since_uptime"`
+}
+
+// SMA struct for calculating simple moving average
+type SMA struct {
+	buf       []float64
+	window    int     // len of buf
+	idx       int     // current index in buf
+	CAvg      float64 // cumulative average
+	prevSMA   float64
+	filledBuf bool
+}
+
+func newSMA(len int) *SMA {
+	if len <= 0 {
+		len = defaultWindowSize
+	}
+	return &SMA{
+		buf:    make([]float64, len),
+		window: len,
+		idx:    0,
+	}
+}
+
+func (s *SMA) addSample(next float64) {
+	prev := s.buf[s.idx]
+	s.buf[s.idx] = next
+
+	if s.filledBuf {
+		s.prevSMA += (next - prev) / float64(s.window)
+		s.CAvg += (next - s.CAvg) / float64(s.window)
+	} else {
+		s.CAvg = s.simpleMovingAvg()
+		s.prevSMA = s.CAvg
+	}
+	if s.idx == s.window-1 {
+		s.filledBuf = true
+	}
+	s.idx = (s.idx + 1) % s.window
+}
+
+func (s *SMA) simpleMovingAvg() float64 {
+	if s.filledBuf {
+		return s.prevSMA
+	}
+	var tot float64
+	for _, r := range s.buf {
+		tot += r
+	}
+	return tot / float64(s.idx+1)
+}
+
+const (
+	defaultWindowSize = 10
+)

cmd/bucket-replication-metrics_gen_test.go

@@ -0,0 +1,801 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalActiveWorkerStat(t *testing.T) {
+	v := ActiveWorkerStat{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgActiveWorkerStat(b *testing.B) {
+	v := ActiveWorkerStat{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgActiveWorkerStat(b *testing.B) {
+	v := ActiveWorkerStat{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalActiveWorkerStat(b *testing.B) {
+	v := ActiveWorkerStat{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeActiveWorkerStat(t *testing.T) {
+	v := ActiveWorkerStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeActiveWorkerStat Msgsize() is inaccurate")
+	}
+
+	vn := ActiveWorkerStat{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeActiveWorkerStat(b *testing.B) {
+	v := ActiveWorkerStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeActiveWorkerStat(b *testing.B) {
+	v := ActiveWorkerStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalInQueueMetric(t *testing.T) {
+	v := InQueueMetric{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgInQueueMetric(b *testing.B) {
+	v := InQueueMetric{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgInQueueMetric(b *testing.B) {
+	v := InQueueMetric{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalInQueueMetric(b *testing.B) {
+	v := InQueueMetric{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeInQueueMetric(t *testing.T) {
+	v := InQueueMetric{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeInQueueMetric Msgsize() is inaccurate")
+	}
+
+	vn := InQueueMetric{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeInQueueMetric(b *testing.B) {
+	v := InQueueMetric{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeInQueueMetric(b *testing.B) {
+	v := InQueueMetric{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalInQueueStats(t *testing.T) {
+	v := InQueueStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgInQueueStats(b *testing.B) {
+	v := InQueueStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgInQueueStats(b *testing.B) {
+	v := InQueueStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalInQueueStats(b *testing.B) {
+	v := InQueueStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeInQueueStats(t *testing.T) {
+	v := InQueueStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeInQueueStats Msgsize() is inaccurate")
+	}
+
+	vn := InQueueStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeInQueueStats(b *testing.B) {
+	v := InQueueStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeInQueueStats(b *testing.B) {
+	v := InQueueStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalQStat(t *testing.T) {
+	v := QStat{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgQStat(b *testing.B) {
+	v := QStat{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgQStat(b *testing.B) {
+	v := QStat{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalQStat(b *testing.B) {
+	v := QStat{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeQStat(t *testing.T) {
+	v := QStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeQStat Msgsize() is inaccurate")
+	}
+
+	vn := QStat{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeQStat(b *testing.B) {
+	v := QStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeQStat(b *testing.B) {
+	v := QStat{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalReplicationMRFStats(t *testing.T) {
+	v := ReplicationMRFStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgReplicationMRFStats(b *testing.B) {
+	v := ReplicationMRFStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgReplicationMRFStats(b *testing.B) {
+	v := ReplicationMRFStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalReplicationMRFStats(b *testing.B) {
+	v := ReplicationMRFStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeReplicationMRFStats(t *testing.T) {
+	v := ReplicationMRFStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeReplicationMRFStats Msgsize() is inaccurate")
+	}
+
+	vn := ReplicationMRFStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeReplicationMRFStats(b *testing.B) {
+	v := ReplicationMRFStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeReplicationMRFStats(b *testing.B) {
+	v := ReplicationMRFStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalSMA(t *testing.T) {
+	v := SMA{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgSMA(b *testing.B) {
+	v := SMA{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgSMA(b *testing.B) {
+	v := SMA{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalSMA(b *testing.B) {
+	v := SMA{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeSMA(t *testing.T) {
+	v := SMA{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeSMA Msgsize() is inaccurate")
+	}
+
+	vn := SMA{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeSMA(b *testing.B) {
+	v := SMA{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeSMA(b *testing.B) {
+	v := SMA{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalXferStats(t *testing.T) {
+	v := XferStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgXferStats(b *testing.B) {
+	v := XferStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgXferStats(b *testing.B) {
+	v := XferStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalXferStats(b *testing.B) {
+	v := XferStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeXferStats(t *testing.T) {
+	v := XferStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeXferStats Msgsize() is inaccurate")
+	}
+
+	vn := XferStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeXferStats(b *testing.B) {
+	v := XferStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeXferStats(b *testing.B) {
+	v := XferStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/bucket-replication-stats.go

@@ -19,12 +19,12 @@ package cmd
 
 import (
 	"context"
-	"encoding/binary"
-	"math"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/rcrowley/go-metrics"
 )
 
 func (b *BucketReplicationStats) hasReplicationUsage() bool {
@@ -38,12 +38,93 @@ func (b *BucketReplicationStats) hasReplicationUsage() bool {
 
 // ReplicationStats holds the global in-memory replication stats
 type ReplicationStats struct {
+	// map of site deployment ID to site replication status
+	// for site replication - maintain stats at global level
+	srStats *SRStats
+	// active worker stats
+	workers *ActiveWorkerStat
+	// queue stats cache
+	qCache queueCache
+	// mrf backlog stats
+	mrfStats ReplicationMRFStats
+	// for bucket replication, continue to use existing cache
 	Cache             map[string]*BucketReplicationStats
-	UsageCache        map[string]*BucketReplicationStats
 	mostRecentStats   BucketStatsMap
-	sync.RWMutex                   // mutex for Cache
-	ulock             sync.RWMutex // mutex for UsageCache
-	mostRecentStatsMu sync.Mutex   // mutex for mostRecentStats
+	registry          metrics.Registry
+	sync.RWMutex                 // mutex for Cache
+	mostRecentStatsMu sync.Mutex // mutex for mostRecentStats
+
+	wlock sync.RWMutex // mutex for active workers
+
+	movingAvgTicker *time.Ticker // Ticker for calculating moving averages
+	wTimer          *time.Ticker // ticker for calculating active workers
+	qTimer          *time.Ticker // ticker for calculating queue stats
+}
+
+func (r *ReplicationStats) trackEWMA() {
+	for {
+		select {
+		case <-r.movingAvgTicker.C:
+			r.updateMovingAvg()
+		case <-GlobalContext.Done():
+			return
+		}
+	}
+}
+
+func (r *ReplicationStats) updateMovingAvg() {
+	r.RLock()
+	for _, s := range r.Cache {
+		for _, st := range s.Stats {
+			st.XferRateLrg.measure.updateExponentialMovingAverage(time.Now())
+			st.XferRateSml.measure.updateExponentialMovingAverage(time.Now())
+		}
+	}
+	r.RUnlock()
+}
+
+// ActiveWorkers returns worker stats
+func (r *ReplicationStats) ActiveWorkers() ActiveWorkerStat {
+	r.wlock.RLock()
+	defer r.wlock.RUnlock()
+	w := r.workers.get()
+	return ActiveWorkerStat{
+		Curr: w.Curr,
+		Max:  w.Max,
+		Avg:  w.Avg,
+	}
+}
+
+func (r *ReplicationStats) collectWorkerMetrics(ctx context.Context) {
+	if r == nil {
+		return
+	}
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-r.wTimer.C:
+			r.wlock.Lock()
+			r.workers.update()
+			r.wlock.Unlock()
+
+		}
+	}
+}
+
+func (r *ReplicationStats) collectQueueMetrics(ctx context.Context) {
+	if r == nil {
+		return
+	}
+
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-r.qTimer.C:
+			r.qCache.update()
+		}
+	}
 }
 
 // Delete deletes in-memory replication statistics for a bucket.
@@ -55,10 +136,6 @@ func (r *ReplicationStats) Delete(bucket string) {
 	r.Lock()
 	defer r.Unlock()
 	delete(r.Cache, bucket)
-
-	r.ulock.Lock()
-	defer r.ulock.Unlock()
-	delete(r.UsageCache, bucket)
 }
 
 // UpdateReplicaStat updates in-memory replica statistics with new values.
@@ -71,83 +148,130 @@ func (r *ReplicationStats) UpdateReplicaStat(bucket string, n int64) {
 	defer r.Unlock()
 	bs, ok := r.Cache[bucket]
 	if !ok {
-		bs = &BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+		bs = newBucketReplicationStats()
 	}
 	bs.ReplicaSize += n
+	bs.ReplicaCount++
 	r.Cache[bucket] = bs
+	r.srUpdateReplicaStat(n)
 }
 
-// Update updates in-memory replication statistics with new values.
-func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration time.Duration, status, prevStatus replication.StatusType, opType replication.Type) {
+func (r *ReplicationStats) srUpdateReplicaStat(sz int64) {
 	if r == nil {
 		return
 	}
-	r.Lock()
-	defer r.Unlock()
+	atomic.AddInt64(&r.srStats.ReplicaSize, sz)
+	atomic.AddInt64(&r.srStats.ReplicaCount, 1)
+}
 
-	bs, ok := r.Cache[bucket]
-	if !ok {
-		bs = &BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
-		r.Cache[bucket] = bs
-	}
-	b, ok := bs.Stats[arn]
-	if !ok {
-		b = &BucketReplicationStat{}
-		bs.Stats[arn] = b
-	}
-	switch status {
-	case replication.Pending:
-		if opType.IsDataReplication() && prevStatus != status {
-			b.PendingSize += n
-			b.PendingCount++
-		}
-	case replication.Completed:
-		switch prevStatus { // adjust counters based on previous state
-		case replication.Pending:
-			b.PendingCount--
-		case replication.Failed:
-			b.FailedCount--
-		}
-		if opType.IsDataReplication() {
-			b.ReplicatedSize += n
-			switch prevStatus {
-			case replication.Pending:
-				b.PendingSize -= n
-			case replication.Failed:
-				b.FailedSize -= n
-			}
-			if duration > 0 {
-				b.Latency.update(n, duration)
-			}
-		}
-	case replication.Failed:
-		if opType.IsDataReplication() {
-			if prevStatus == replication.Pending {
-				b.FailedSize += n
-				b.FailedCount++
-				b.PendingSize -= n
-				b.PendingCount--
-			}
-		}
-	case replication.Replica:
-		if opType == replication.ObjectReplicationType {
-			b.ReplicaSize += n
-		}
+func (r *ReplicationStats) srUpdate(sr replStat) {
+	dID, err := globalSiteReplicationSys.getDeplIDForEndpoint(sr.endpoint())
+	if err == nil {
+		r.srStats.update(sr, dID)
 	}
 }
 
-// GetInitialUsage get replication metrics available at the time of cluster initialization
-func (r *ReplicationStats) GetInitialUsage(bucket string) BucketReplicationStats {
+// Update updates in-memory replication statistics with new values.
+func (r *ReplicationStats) Update(bucket string, ri replicatedTargetInfo, status, prevStatus replication.StatusType) {
 	if r == nil {
-		return BucketReplicationStats{}
+		return
 	}
-	r.ulock.RLock()
-	defer r.ulock.RUnlock()
-	st, ok := r.UsageCache[bucket]
+	var rs replStat
+	switch status {
+	case replication.Pending:
+		if ri.OpType.IsDataReplication() && prevStatus != status {
+			rs.set(ri.Arn, ri.Size, 0, status, ri.OpType, ri.endpoint, ri.secure, ri.Err)
+		}
+	case replication.Completed:
+		if ri.OpType.IsDataReplication() {
+			rs.set(ri.Arn, ri.Size, ri.Duration, status, ri.OpType, ri.endpoint, ri.secure, ri.Err)
+		}
+	case replication.Failed:
+		if ri.OpType.IsDataReplication() && prevStatus == replication.Pending {
+			rs.set(ri.Arn, ri.Size, ri.Duration, status, ri.OpType, ri.endpoint, ri.secure, ri.Err)
+		}
+	case replication.Replica:
+		if ri.OpType == replication.ObjectReplicationType {
+			rs.set(ri.Arn, ri.Size, 0, status, ri.OpType, "", false, ri.Err)
+		}
+	}
+
+	// update site-replication in-memory stats
+	if rs.Completed || rs.Failed {
+		r.srUpdate(rs)
+	}
+
+	r.Lock()
+	defer r.Unlock()
+
+	// update bucket replication in-memory stats
+	bs, ok := r.Cache[bucket]
 	if !ok {
-		return BucketReplicationStats{}
+		bs = newBucketReplicationStats()
+		r.Cache[bucket] = bs
+	}
+	b, ok := bs.Stats[ri.Arn]
+	if !ok {
+		b = &BucketReplicationStat{
+			XferRateLrg: newXferStats(),
+			XferRateSml: newXferStats(),
+		}
+		bs.Stats[ri.Arn] = b
+	}
+
+	switch {
+	case rs.Completed:
+		b.ReplicatedSize += rs.TransferSize
+		b.ReplicatedCount++
+		if rs.TransferDuration > 0 {
+			b.Latency.update(rs.TransferSize, rs.TransferDuration)
+			b.updateXferRate(rs.TransferSize, rs.TransferDuration)
+		}
+	case rs.Failed:
+		b.FailStats.addsize(rs.TransferSize, rs.Err)
+	case rs.Pending:
+	}
+}
+
+type replStat struct {
+	Arn       string
+	Completed bool
+	Pending   bool
+	Failed    bool
+	opType    replication.Type
+	// transfer size
+	TransferSize int64
+	// transfer duration
+	TransferDuration time.Duration
+	Endpoint         string
+	Secure           bool
+	Err              error
+}
+
+func (rs *replStat) endpoint() string {
+	scheme := "http"
+	if rs.Secure {
+		scheme = "https"
+	}
+	return scheme + "://" + rs.Endpoint
+}
+
+func (rs *replStat) set(arn string, n int64, duration time.Duration, status replication.StatusType, opType replication.Type, endpoint string, secure bool, err error) {
+	rs.Endpoint = endpoint
+	rs.Secure = secure
+	rs.TransferSize = n
+	rs.Arn = arn
+	rs.TransferDuration = duration
+	rs.opType = opType
+	switch status {
+	case replication.Completed:
+		rs.Completed = true
+	case replication.Pending:
+		rs.Pending = true
+	case replication.Failed:
+		rs.Failed = true
+		rs.Err = err
 	}
-	return st.Clone()
 }
 
 // GetAll returns replication metrics for all buckets at once.
@@ -157,16 +281,36 @@ func (r *ReplicationStats) GetAll() map[string]BucketReplicationStats {
 	}
 
 	r.RLock()
-	defer r.RUnlock()
 
 	bucketReplicationStats := make(map[string]BucketReplicationStats, len(r.Cache))
 	for k, v := range r.Cache {
 		bucketReplicationStats[k] = v.Clone()
 	}
+	r.RUnlock()
+	for k, v := range bucketReplicationStats {
+		v.QStat = r.qCache.getBucketStats(k)
+		bucketReplicationStats[k] = v
+	}
 
 	return bucketReplicationStats
 }
 
+func (r *ReplicationStats) getSRMetricsForNode() SRMetricsSummary {
+	if r == nil {
+		return SRMetricsSummary{}
+	}
+
+	m := SRMetricsSummary{
+		Uptime:        UTCNow().Unix() - globalBootTime.Unix(),
+		Queued:        r.qCache.getSiteStats(),
+		ActiveWorkers: r.ActiveWorkers(),
+		Metrics:       r.srStats.get(),
+		ReplicaSize:   atomic.LoadInt64(&r.srStats.ReplicaSize),
+		ReplicaCount:  atomic.LoadInt64(&r.srStats.ReplicaCount),
+	}
+	return m
+}
+
 // Get replication metrics for a bucket from this node since this node came up.
 func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 	if r == nil {
@@ -178,99 +322,35 @@ func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 
 	st, ok := r.Cache[bucket]
 	if !ok {
-		return BucketReplicationStats{}
+		return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
 	}
 	return st.Clone()
 }
 
 // NewReplicationStats initialize in-memory replication statistics
 func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *ReplicationStats {
-	return &ReplicationStats{
-		Cache:      make(map[string]*BucketReplicationStats),
-		UsageCache: make(map[string]*BucketReplicationStats),
+	r := metrics.NewRegistry()
+	rs := ReplicationStats{
+		Cache:           make(map[string]*BucketReplicationStats),
+		qCache:          newQueueCache(r),
+		srStats:         newSRStats(),
+		movingAvgTicker: time.NewTicker(2 * time.Second),
+		wTimer:          time.NewTicker(2 * time.Second),
+		qTimer:          time.NewTicker(2 * time.Second),
+
+		workers:  newActiveWorkerStat(r),
+		registry: r,
 	}
+	go rs.collectWorkerMetrics(ctx)
+	go rs.collectQueueMetrics(ctx)
+	return &rs
 }
 
-// load replication metrics at cluster start from latest replication stats saved in .minio.sys/buckets/replication/node-name.stats
-// fallback to replication stats in data usage to be backward compatible
-func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
-	m := make(map[string]*BucketReplicationStats)
-	if stats, err := globalReplicationPool.loadStatsFromDisk(); err == nil {
-		for b, st := range stats {
-			c := st.Clone()
-			m[b] = &c
-		}
-		r.ulock.Lock()
-		r.UsageCache = m
-		r.ulock.Unlock()
-		return
-	}
-	rTimer := time.NewTimer(time.Second * 5)
-	defer rTimer.Stop()
-	var (
-		dui DataUsageInfo
-		err error
-	)
-outer:
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-rTimer.C:
-			dui, err = loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
-			// If LastUpdate is set, data usage is available.
-			if err == nil {
-				break outer
-			}
-			rTimer.Reset(time.Second * 5)
-		}
-	}
-	for bucket, usage := range dui.BucketsUsage {
-		b := &BucketReplicationStats{
-			Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
-		}
-		for arn, uinfo := range usage.ReplicationInfo {
-			b.Stats[arn] = &BucketReplicationStat{
-				FailedSize:     int64(uinfo.ReplicationFailedSize),
-				ReplicatedSize: int64(uinfo.ReplicatedSize),
-				ReplicaSize:    int64(uinfo.ReplicaSize),
-				FailedCount:    int64(uinfo.ReplicationFailedCount),
-			}
-		}
-		b.ReplicaSize += int64(usage.ReplicaSize)
-		if b.hasReplicationUsage() {
-			m[bucket] = b
-		}
-	}
-	r.ulock.Lock()
-	r.UsageCache = m
-	r.ulock.Unlock()
-}
-
-// serializeStats will serialize the current stats.
-// Will return (nil, nil) if no data.
-func (r *ReplicationStats) serializeStats() ([]byte, error) {
-	if r == nil {
-		return nil, nil
-	}
-	r.mostRecentStatsMu.Lock()
-	defer r.mostRecentStatsMu.Unlock()
-	if len(r.mostRecentStats.Stats) == 0 {
-		return nil, nil
-	}
-	data := make([]byte, 4, 4+r.mostRecentStats.Msgsize())
-	// Add the replication stats meta header.
-	binary.LittleEndian.PutUint16(data[0:2], replStatsMetaFormat)
-	binary.LittleEndian.PutUint16(data[2:4], replStatsVersion)
-	// Add data
-	return r.mostRecentStats.MarshalMsg(data)
-}
-
-func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketReplicationStats) {
+func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketStats) {
 	peerBucketStatsList := globalNotificationSys.GetClusterAllBucketStats(GlobalContext)
-	bucketsReplicationStats = make(map[string]BucketReplicationStats, len(bucketsUsage))
+	bucketsReplicationStats = make(map[string]BucketStats, len(bucketsUsage))
 
-	for bucket, u := range bucketsUsage {
+	for bucket := range bucketsUsage {
 		bucketStats := make([]BucketStats, len(peerBucketStatsList))
 		for i, peerBucketStats := range peerBucketStatsList {
 			bucketStat, ok := peerBucketStats.Stats[bucket]
@@ -279,110 +359,126 @@ func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo)
 			}
 			bucketStats[i] = bucketStat
 		}
-		bucketsReplicationStats[bucket] = r.calculateBucketReplicationStats(bucket, u, bucketStats)
+		bucketsReplicationStats[bucket] = r.calculateBucketReplicationStats(bucket, bucketStats)
 	}
 	return bucketsReplicationStats
 }
 
-func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, u BucketUsageInfo, bucketStats []BucketStats) (s BucketReplicationStats) {
+func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, bucketStats []BucketStats) (bs BucketStats) {
 	if r == nil {
-		s = BucketReplicationStats{
-			Stats: make(map[string]*BucketReplicationStat),
+		bs = BucketStats{
+			ReplicationStats: BucketReplicationStats{
+				Stats: make(map[string]*BucketReplicationStat),
+			},
+			QueueStats: ReplicationQueueStats{},
 		}
-		return s
+		return bs
 	}
-
+	var s BucketReplicationStats
 	// accumulate cluster bucket stats
 	stats := make(map[string]*BucketReplicationStat)
-	var totReplicaSize int64
+	var (
+		totReplicaSize, totReplicatedSize   int64
+		totReplicaCount, totReplicatedCount int64
+		totFailed                           RTimedMetrics
+		tq                                  InQueueMetric
+	)
 	for _, bucketStat := range bucketStats {
 		totReplicaSize += bucketStat.ReplicationStats.ReplicaSize
+		totReplicaCount += bucketStat.ReplicationStats.ReplicaCount
+		for _, q := range bucketStat.QueueStats.Nodes {
+			tq = tq.merge(q.QStats)
+		}
+
 		for arn, stat := range bucketStat.ReplicationStats.Stats {
 			oldst := stats[arn]
 			if oldst == nil {
-				oldst = &BucketReplicationStat{}
+				oldst = &BucketReplicationStat{
+					XferRateLrg: newXferStats(),
+					XferRateSml: newXferStats(),
+				}
 			}
+			fstats := stat.FailStats.merge(oldst.FailStats)
+			lrg := oldst.XferRateLrg.merge(*stat.XferRateLrg)
+			sml := oldst.XferRateSml.merge(*stat.XferRateSml)
 			stats[arn] = &BucketReplicationStat{
-				FailedCount:    stat.FailedCount + oldst.FailedCount,
-				FailedSize:     stat.FailedSize + oldst.FailedSize,
-				ReplicatedSize: stat.ReplicatedSize + oldst.ReplicatedSize,
-				Latency:        stat.Latency.merge(oldst.Latency),
-				PendingCount:   stat.PendingCount + oldst.PendingCount,
-				PendingSize:    stat.PendingSize + oldst.PendingSize,
+				Failed:          fstats.toMetric(),
+				FailStats:       fstats,
+				ReplicatedSize:  stat.ReplicatedSize + oldst.ReplicatedSize,
+				ReplicatedCount: stat.ReplicatedCount + oldst.ReplicatedCount,
+				Latency:         stat.Latency.merge(oldst.Latency),
+				XferRateLrg:     &lrg,
+				XferRateSml:     &sml,
 			}
+			totReplicatedSize += stat.ReplicatedSize
+			totReplicatedCount += stat.ReplicatedCount
+			totFailed = totFailed.merge(stat.FailStats)
 		}
 	}
 
-	// add initial usage stat to cluster stats
-	usageStat := globalReplicationStats.GetInitialUsage(bucket)
-
-	totReplicaSize += usageStat.ReplicaSize
-	for arn, stat := range usageStat.Stats {
-		st, ok := stats[arn]
-		if !ok {
-			st = &BucketReplicationStat{}
-			stats[arn] = st
-		}
-		st.ReplicatedSize += stat.ReplicatedSize
-		st.FailedSize += stat.FailedSize
-		st.FailedCount += stat.FailedCount
-		st.PendingSize += stat.PendingSize
-		st.PendingCount += stat.PendingCount
-	}
-
 	s = BucketReplicationStats{
-		Stats: make(map[string]*BucketReplicationStat, len(stats)),
+		Stats:           stats,
+		QStat:           tq,
+		ReplicaSize:     totReplicaSize,
+		ReplicaCount:    totReplicaCount,
+		ReplicatedSize:  totReplicatedSize,
+		ReplicatedCount: totReplicatedCount,
+		Failed:          totFailed.toMetric(),
 	}
 
-	var latestTotReplicatedSize int64
-	for _, st := range u.ReplicationInfo {
-		latestTotReplicatedSize += int64(st.ReplicatedSize)
+	var qs ReplicationQueueStats
+	for _, bs := range bucketStats {
+		qs.Nodes = append(qs.Nodes, bs.QueueStats.Nodes...)
 	}
 
-	// normalize computed real time stats with latest usage stat
-	for arn, tgtstat := range stats {
-		st := BucketReplicationStat{}
-		bu, ok := u.ReplicationInfo[arn]
-		if !ok {
-			bu = BucketTargetUsageInfo{}
-		}
-		// use in memory replication stats if it is ahead of usage info.
-		st.ReplicatedSize = int64(bu.ReplicatedSize)
-		if tgtstat.ReplicatedSize >= int64(bu.ReplicatedSize) {
-			st.ReplicatedSize = tgtstat.ReplicatedSize
-		}
-		s.ReplicatedSize += st.ReplicatedSize
-		// Reset FailedSize and FailedCount to 0 for negative overflows which can
-		// happen since data usage picture can lag behind actual usage state at the time of cluster start
-		st.FailedSize = int64(math.Max(float64(tgtstat.FailedSize), 0))
-		st.FailedCount = int64(math.Max(float64(tgtstat.FailedCount), 0))
-		st.PendingSize = int64(math.Max(float64(tgtstat.PendingSize), 0))
-		st.PendingCount = int64(math.Max(float64(tgtstat.PendingCount), 0))
-		st.Latency = tgtstat.Latency
-
-		s.Stats[arn] = &st
-		s.FailedSize += st.FailedSize
-		s.FailedCount += st.FailedCount
-		s.PendingCount += st.PendingCount
-		s.PendingSize += st.PendingSize
+	qs.Uptime = UTCNow().Unix() - globalBootTime.Unix()
+	bs = BucketStats{
+		ReplicationStats: s,
+		QueueStats:       qs,
 	}
-	// normalize overall stats
-	s.ReplicaSize = int64(math.Max(float64(totReplicaSize), float64(u.ReplicaSize)))
-	s.ReplicatedSize = int64(math.Max(float64(s.ReplicatedSize), float64(latestTotReplicatedSize)))
 	r.mostRecentStatsMu.Lock()
 	if len(r.mostRecentStats.Stats) == 0 {
 		r.mostRecentStats = BucketStatsMap{Stats: make(map[string]BucketStats, 1), Timestamp: UTCNow()}
 	}
-	if len(s.Stats) > 0 {
-		r.mostRecentStats.Stats[bucket] = BucketStats{ReplicationStats: s}
+	if len(bs.ReplicationStats.Stats) > 0 {
+		r.mostRecentStats.Stats[bucket] = bs
 	}
 	r.mostRecentStats.Timestamp = UTCNow()
 	r.mostRecentStatsMu.Unlock()
-	return s
+	return bs
 }
 
 // get the most current of in-memory replication stats  and data usage info from crawler.
-func (r *ReplicationStats) getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplicationStats) {
+func (r *ReplicationStats) getLatestReplicationStats(bucket string) (s BucketStats) {
 	bucketStats := globalNotificationSys.GetClusterBucketStats(GlobalContext, bucket)
-	return r.calculateBucketReplicationStats(bucket, u, bucketStats)
+	return r.calculateBucketReplicationStats(bucket, bucketStats)
+}
+
+func (r *ReplicationStats) incQ(bucket string, sz int64, isDeleleRepl bool, opType replication.Type) {
+	r.qCache.Lock()
+	defer r.qCache.Unlock()
+	v, ok := r.qCache.bucketStats[bucket]
+	if !ok {
+		v = newInQueueStats(r.registry, bucket)
+	}
+	atomic.AddInt64(&v.nowBytes, sz)
+	atomic.AddInt64(&v.nowCount, 1)
+	r.qCache.bucketStats[bucket] = v
+	atomic.AddInt64(&r.qCache.srQueueStats.nowBytes, sz)
+	atomic.AddInt64(&r.qCache.srQueueStats.nowCount, 1)
+}
+
+func (r *ReplicationStats) decQ(bucket string, sz int64, isDelMarker bool, opType replication.Type) {
+	r.qCache.Lock()
+	defer r.qCache.Unlock()
+	v, ok := r.qCache.bucketStats[bucket]
+	if !ok {
+		v = newInQueueStats(r.registry, bucket)
+	}
+	atomic.AddInt64(&v.nowBytes, -1*sz)
+	atomic.AddInt64(&v.nowCount, -1)
+	r.qCache.bucketStats[bucket] = v
+
+	atomic.AddInt64(&r.qCache.srQueueStats.nowBytes, -1*sz)
+	atomic.AddInt64(&r.qCache.srQueueStats.nowCount, -1)
 }

cmd/bucket-replication-utils.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -31,6 +32,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
 )
 
 //go:generate msgp -file=$GOFILE
@@ -47,6 +49,9 @@ type replicatedTargetInfo struct {
 	VersionPurgeStatus    VersionPurgeStatusType
 	ResyncTimestamp       string
 	ReplicationResynced   bool // true only if resync attempted for this target
+	endpoint              string
+	secure                bool
+	Err                   error // replication error if any
 }
 
 // Empty returns true for a target if arn is empty
@@ -180,6 +185,7 @@ type replicateTargetDecision struct {
 	Synchronous bool   // Synchronous replication configured.
 	Arn         string // ARN of replication target
 	ID          string
+	Tgt         *TargetClient
 }
 
 func (t *replicateTargetDecision) String() string {
@@ -288,7 +294,7 @@ var errInvalidReplicateDecisionFormat = fmt.Errorf("ReplicateDecision has invali
 
 // parse k-v pairs of target ARN to stringified ReplicateTargetDecision delimited by ',' into a
 // ReplicateDecision struct
-func parseReplicateDecision(s string) (r ReplicateDecision, err error) {
+func parseReplicateDecision(ctx context.Context, bucket, s string) (r ReplicateDecision, err error) {
 	r = ReplicateDecision{
 		targetsMap: make(map[string]replicateTargetDecision),
 	}
@@ -317,7 +323,13 @@ func parseReplicateDecision(s string) (r ReplicateDecision, err error) {
 		if err != nil {
 			return r, err
 		}
-		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: replicate, Synchronous: sync, Arn: tgt[2], ID: tgt[3]}
+		tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(slc[0])
+		if tgtClnt == nil {
+			// Skip stale targets if any and log them to be missing atleast once.
+			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, slc[0]), slc[0])
+			// We save the targetDecision even when its not configured or stale.
+		}
+		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: replicate, Synchronous: sync, Arn: tgt[2], ID: tgt[3], Tgt: tgtClnt}
 	}
 	return
 }
@@ -340,8 +352,6 @@ type ReplicationState struct {
 // Equal returns true if replication state is identical for version purge statuses and (replica)tion statuses.
 func (rs *ReplicationState) Equal(o ReplicationState) bool {
 	return rs.ReplicaStatus == o.ReplicaStatus &&
-		rs.ReplicaTimeStamp.Equal(o.ReplicaTimeStamp) &&
-		rs.ReplicationTimeStamp.Equal(o.ReplicationTimeStamp) &&
 		rs.ReplicationStatusInternal == o.ReplicationStatusInternal &&
 		rs.VersionPurgeStatusInternal == o.VersionPurgeStatusInternal
 }
@@ -357,7 +367,10 @@ func (rs *ReplicationState) CompositeReplicationStatus() (st replication.StatusT
 			replStatus := getCompositeReplicationStatus(rs.Targets)
 			// return REPLICA status if replica received timestamp is later than replication timestamp
 			// provided object replication completed for all targets.
-			if !rs.ReplicaTimeStamp.Equal(timeSentinel) && replStatus == replication.Completed && rs.ReplicaTimeStamp.After(rs.ReplicationTimeStamp) {
+			if rs.ReplicaTimeStamp.Equal(timeSentinel) || rs.ReplicaTimeStamp.IsZero() {
+				return replStatus
+			}
+			if replStatus == replication.Completed && rs.ReplicaTimeStamp.After(rs.ReplicationTimeStamp) {
 				return rs.ReplicaStatus
 			}
 			return replStatus
@@ -720,6 +733,7 @@ type TargetReplicationResyncStatus struct {
 	// Last bucket/object replicated.
 	Bucket string `json:"-" msg:"bkt"`
 	Object string `json:"-" msg:"obj"`
+	Error  error  `json:"-" msg:"-"`
 }
 
 // BucketReplicationResyncStatus captures current replication resync status
@@ -789,6 +803,7 @@ type MRFReplicateEntry struct {
 	Object     string `json:"object" msg:"o"`
 	versionID  string `json:"-"`
 	RetryCount int    `json:"retryCount" msg:"rc"`
+	sz         int64  `json:"-"`
 }
 
 // MRFReplicateEntries has the map of MRF entries to save to disk
@@ -803,17 +818,7 @@ func (ri ReplicateObjectInfo) ToMRFEntry() MRFReplicateEntry {
 		Bucket:     ri.Bucket,
 		Object:     ri.Name,
 		versionID:  ri.VersionID,
+		sz:         ri.Size,
 		RetryCount: int(ri.RetryCount),
 	}
 }
-
-func getReplicationStatsPath() string {
-	return bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + "replication.stats"
-}
-
-const (
-	replStatsMetaFormat   = 1
-	replStatsVersionV1    = 1
-	replStatsVersion      = replStatsVersionV1
-	replStatsSaveInterval = time.Minute * 5
-)

cmd/bucket-replication-utils_test.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"context"
 	"testing"
 
 	"github.com/minio/minio/internal/bucket/replication"
@@ -183,7 +184,7 @@ var parseReplicationDecisionTest = []struct {
 
 func TestParseReplicateDecision(t *testing.T) {
 	for i, test := range parseReplicationDecisionTest {
-		dsc, err := parseReplicateDecision(test.expDsc.String())
+		dsc, err := parseReplicateDecision(context.Background(), "bucket", test.expDsc.String())
 		if err != nil {
 			if test.expErr != err {
 				t.Errorf("Test%d (%s): Expected parse error got %t , want %t", i+1, test.name, err, test.expErr)

cmd/bucket-stats.go

@@ -19,7 +19,10 @@ package cmd
 
 import (
 	"fmt"
+	"math"
 	"time"
+
+	"github.com/minio/madmin-go/v3"
 )
 
 //go:generate msgp -file $GOFILE
@@ -52,6 +55,94 @@ func (rl *ReplicationLatency) update(size int64, duration time.Duration) {
 	rl.UploadHistogram.Add(size, duration)
 }
 
+// ReplicationLastMinute has last minute replication counters
+type ReplicationLastMinute struct {
+	LastMinute lastMinuteLatency
+}
+
+func (rl ReplicationLastMinute) merge(other ReplicationLastMinute) (nl ReplicationLastMinute) {
+	nl = ReplicationLastMinute{rl.LastMinute.merge(other.LastMinute)}
+	return
+}
+
+func (rl *ReplicationLastMinute) addsize(n int64) {
+	t := time.Now().Unix()
+	rl.LastMinute.addAll(t-1, AccElem{Total: t - 1, Size: n, N: 1})
+}
+
+func (rl *ReplicationLastMinute) String() string {
+	t := rl.LastMinute.getTotal()
+	return fmt.Sprintf("ReplicationLastMinute sz= %d, n=%d , dur=%d", t.Size, t.N, t.Total)
+}
+
+func (rl *ReplicationLastMinute) getTotal() AccElem {
+	return rl.LastMinute.getTotal()
+}
+
+// ReplicationLastHour keeps track of replication counts over the last hour
+type ReplicationLastHour struct {
+	Totals  [60]AccElem
+	LastMin int64
+}
+
+// Merge data of two ReplicationLastHour structure
+func (l ReplicationLastHour) merge(o ReplicationLastHour) (merged ReplicationLastHour) {
+	if l.LastMin > o.LastMin {
+		o.forwardTo(l.LastMin)
+		merged.LastMin = l.LastMin
+	} else {
+		l.forwardTo(o.LastMin)
+		merged.LastMin = o.LastMin
+	}
+
+	for i := range merged.Totals {
+		merged.Totals[i] = AccElem{
+			Total: l.Totals[i].Total + o.Totals[i].Total,
+			N:     l.Totals[i].N + o.Totals[i].N,
+			Size:  l.Totals[i].Size + o.Totals[i].Size,
+		}
+	}
+	return merged
+}
+
+// Add  a new duration data
+func (l *ReplicationLastHour) addsize(sz int64) {
+	min := time.Now().Unix() / 60
+	l.forwardTo(min)
+	winIdx := min % 60
+	l.Totals[winIdx].merge(AccElem{Total: min, Size: sz, N: 1})
+	l.LastMin = min
+}
+
+// Merge all recorded counts of last hour into one
+func (l *ReplicationLastHour) getTotal() AccElem {
+	var res AccElem
+	min := time.Now().Unix() / 60
+	l.forwardTo(min)
+	for _, elem := range l.Totals[:] {
+		res.merge(elem)
+	}
+	return res
+}
+
+// forwardTo time t, clearing any entries in between.
+func (l *ReplicationLastHour) forwardTo(t int64) {
+	tMin := t / 60
+	if l.LastMin >= tMin {
+		return
+	}
+	if t-l.LastMin >= 60 {
+		l.Totals = [60]AccElem{}
+		return
+	}
+	for l.LastMin != t {
+		// Clear next element.
+		idx := (l.LastMin + 1) % 60
+		l.Totals[idx] = AccElem{}
+		l.LastMin++
+	}
+}
+
 // BucketStatsMap captures bucket statistics for all buckets
 type BucketStatsMap struct {
 	Stats     map[string]BucketStats
@@ -60,19 +151,32 @@ type BucketStatsMap struct {
 
 // BucketStats bucket statistics
 type BucketStats struct {
-	ReplicationStats BucketReplicationStats
+	Uptime           int64                  `json:"uptime"`
+	ReplicationStats BucketReplicationStats `json:"currStats"`  // current replication stats since cluster startup
+	QueueStats       ReplicationQueueStats  `json:"queueStats"` // replication queue stats
 }
 
 // BucketReplicationStats represents inline replication statistics
 // such as pending, failed and completed bytes in total for a bucket
 type BucketReplicationStats struct {
 	Stats map[string]*BucketReplicationStat `json:",omitempty"`
-	// Pending size in bytes
-	PendingSize int64 `json:"pendingReplicationSize"`
 	// Completed size in bytes
 	ReplicatedSize int64 `json:"completedReplicationSize"`
 	// Total Replica size in bytes
 	ReplicaSize int64 `json:"replicaSize"`
+	// Total failed operations including metadata updates for various time frames
+	Failed madmin.TimedErrStats `json:"failed"`
+
+	// Total number of completed operations
+	ReplicatedCount int64 `json:"replicationCount"`
+	// Total number of replica received
+	ReplicaCount int64 `json:"replicaCount"`
+
+	// in Queue stats for bucket - from qCache
+	QStat InQueueMetric `json:"queued"`
+	// Deprecated fields
+	// Pending size in bytes
+	PendingSize int64 `json:"pendingReplicationSize"`
 	// Failed size in bytes
 	FailedSize int64 `json:"failedReplicationSize"`
 	// Total number of pending operations including metadata updates
@@ -81,6 +185,12 @@ type BucketReplicationStats struct {
 	FailedCount int64 `json:"failedReplicationCount"`
 }
 
+func newBucketReplicationStats() *BucketReplicationStats {
+	return &BucketReplicationStats{
+		Stats: make(map[string]*BucketReplicationStat),
+	}
+}
+
 // Empty returns true if there are no target stats
 func (brs *BucketReplicationStats) Empty() bool {
 	return len(brs.Stats) == 0 && brs.ReplicaSize == 0
@@ -96,7 +206,24 @@ func (brs BucketReplicationStats) Clone() (c BucketReplicationStats) {
 	c.Stats = make(map[string]*BucketReplicationStat, len(brs.Stats))
 	for arn, st := range brs.Stats {
 		// make a copy of `*st`
-		s := *st
+		s := BucketReplicationStat{
+			ReplicatedSize:                   st.ReplicatedSize,
+			ReplicaSize:                      st.ReplicaSize,
+			Latency:                          st.Latency,
+			BandWidthLimitInBytesPerSecond:   st.BandWidthLimitInBytesPerSecond,
+			CurrentBandwidthInBytesPerSecond: st.CurrentBandwidthInBytesPerSecond,
+			XferRateLrg:                      st.XferRateLrg.Clone(),
+			XferRateSml:                      st.XferRateSml.Clone(),
+			ReplicatedCount:                  st.ReplicatedCount,
+			Failed:                           st.Failed,
+			FailStats:                        st.FailStats,
+		}
+		if s.Failed.ErrCounts == nil {
+			s.Failed.ErrCounts = make(map[string]int)
+			for k, v := range st.Failed.ErrCounts {
+				s.Failed.ErrCounts[k] = v
+			}
+		}
 		c.Stats[arn] = &s
 	}
 	return c
@@ -107,38 +234,189 @@ func (brs BucketReplicationStats) Clone() (c BucketReplicationStats) {
 // remote target
 type BucketReplicationStat struct {
 	// Pending size in bytes
-	PendingSize int64 `json:"pendingReplicationSize"`
+	//	PendingSize int64 `json:"pendingReplicationSize"`
 	// Completed size in bytes
 	ReplicatedSize int64 `json:"completedReplicationSize"`
 	// Total Replica size in bytes
 	ReplicaSize int64 `json:"replicaSize"`
-	// Failed size in bytes
-	FailedSize int64 `json:"failedReplicationSize"`
-	// Total number of pending operations including metadata updates
-	PendingCount int64 `json:"pendingReplicationCount"`
-	// Total number of failed operations including metadata updates
-	FailedCount int64 `json:"failedReplicationCount"`
+	// Collect stats for failures
+	FailStats RTimedMetrics `json:"-"`
+
+	// Total number of failed operations including metadata updates in the last minute
+	Failed madmin.TimedErrStats `json:"failed"`
+	// Total number of completed operations
+	ReplicatedCount int64 `json:"replicationCount"`
 	// Replication latency information
 	Latency ReplicationLatency `json:"replicationLatency"`
 	// bandwidth limit for target
 	BandWidthLimitInBytesPerSecond int64 `json:"limitInBits"`
 	// current bandwidth reported
 	CurrentBandwidthInBytesPerSecond float64 `json:"currentBandwidth"`
+	// transfer rate for large uploads
+	XferRateLrg *XferStats `json:"-" msg:"lt"`
+	// transfer rate for small uploads
+	XferRateSml *XferStats `json:"-" msg:"st"`
+
+	// Deprecated fields
+	// Pending size in bytes
+	PendingSize int64 `json:"pendingReplicationSize"`
+	// Failed size in bytes
+	FailedSize int64 `json:"failedReplicationSize"`
+	// Total number of pending operations including metadata updates
+	PendingCount int64 `json:"pendingReplicationCount"`
+	// Total number of failed operations including metadata updates
+	FailedCount int64 `json:"failedReplicationCount"`
 }
 
 func (bs *BucketReplicationStat) hasReplicationUsage() bool {
-	return bs.FailedSize > 0 ||
+	return bs.FailStats.SinceUptime.Count > 0 ||
 		bs.ReplicatedSize > 0 ||
-		bs.ReplicaSize > 0 ||
-		bs.FailedCount > 0 ||
-		bs.PendingCount > 0 ||
-		bs.PendingSize > 0
+		bs.ReplicaSize > 0
 }
 
-func (brs BucketReplicationStats) String() string {
-	s := "ReplicatedSize=" + fmt.Sprintf("%d", brs.ReplicatedSize) + "+\n ReplicaSize=" + fmt.Sprintf("%d", brs.ReplicaSize)
-	for arn, st := range brs.Stats {
-		s += "\n arn: " + arn + " ReplicatedSize=" + fmt.Sprintf("%d", st.ReplicatedSize) + "  +::ReplicaSize=" + fmt.Sprintf("%d", st.ReplicaSize)
+func (bs *BucketReplicationStat) updateXferRate(sz int64, duration time.Duration) {
+	if sz > minLargeObjSize {
+		bs.XferRateLrg.addSize(sz, duration)
+	} else {
+		bs.XferRateSml.addSize(sz, duration)
 	}
-	return s
+}
+
+// RMetricName - name of replication metric
+type RMetricName string
+
+const (
+	// Large - objects larger than 128MiB
+	Large RMetricName = "Large"
+	// Small - objects smaller than 128MiB
+	Small RMetricName = "Small"
+	// Total - metric pertaining to totals
+	Total RMetricName = "Total"
+)
+
+// ReplQNodeStats holds queue stats for replication per node
+type ReplQNodeStats struct {
+	NodeName      string                               `json:"nodeName"`
+	Uptime        int64                                `json:"uptime"`
+	ActiveWorkers ActiveWorkerStat                     `json:"activeWorkers"`
+	XferStats     map[RMetricName]XferStats            `json:"transferSummary"`
+	TgtXferStats  map[string]map[RMetricName]XferStats `json:"tgtTransferStats"`
+	QStats        InQueueMetric                        `json:"queueStats"`
+	MRFStats      ReplicationMRFStats                  `json:"mrfStats"`
+}
+
+// getNodeQueueStats returns replication operational stats at the node level
+func (r *ReplicationStats) getNodeQueueStats(bucket string) (qs ReplQNodeStats) {
+	qs.NodeName = globalLocalNodeName
+	qs.Uptime = UTCNow().Unix() - globalBootTime.Unix()
+	qs.ActiveWorkers = globalReplicationStats.ActiveWorkers()
+	qs.XferStats = make(map[RMetricName]XferStats)
+	qs.QStats = r.qCache.getBucketStats(bucket)
+	qs.TgtXferStats = make(map[string]map[RMetricName]XferStats)
+
+	r.RLock()
+	defer r.RUnlock()
+
+	brs, ok := r.Cache[bucket]
+	if !ok {
+		return qs
+	}
+	for arn := range brs.Stats {
+		qs.TgtXferStats[arn] = make(map[RMetricName]XferStats)
+	}
+	count := 0
+	var totPeak float64
+	// calculate large, small transfers and total transfer rates per replication target at bucket level
+	for arn, v := range brs.Stats {
+		lcurrTgt := v.XferRateLrg.curr()
+		scurrTgt := v.XferRateSml.curr()
+		totPeak = math.Max(math.Max(v.XferRateLrg.Peak, v.XferRateSml.Peak), totPeak)
+		totPeak = math.Max(math.Max(lcurrTgt, scurrTgt), totPeak)
+		tcount := 0
+		if v.XferRateLrg.Peak > 0 {
+			tcount++
+		}
+		if v.XferRateSml.Peak > 0 {
+			tcount++
+		}
+		qs.TgtXferStats[arn][Large] = XferStats{
+			Avg:  v.XferRateLrg.Avg,
+			Curr: lcurrTgt,
+			Peak: math.Max(v.XferRateLrg.Peak, lcurrTgt),
+		}
+		qs.TgtXferStats[arn][Small] = XferStats{
+			Avg:  v.XferRateSml.Avg,
+			Curr: scurrTgt,
+			Peak: math.Max(v.XferRateSml.Peak, scurrTgt),
+		}
+		if tcount > 0 {
+			qs.TgtXferStats[arn][Total] = XferStats{
+				Avg:  (v.XferRateLrg.Avg + v.XferRateSml.Avg) / float64(tcount),
+				Curr: (scurrTgt + lcurrTgt) / float64(tcount),
+				Peak: totPeak,
+			}
+		}
+	}
+	// calculate large, small and total transfer rates for a minio node
+	var lavg, lcurr, lpeak, savg, scurr, speak, totpeak float64
+	for _, v := range qs.TgtXferStats {
+		tot := v[Total]
+		lavg += v[Large].Avg
+		lcurr += v[Large].Curr
+		savg += v[Small].Avg
+		scurr += v[Small].Curr
+		totpeak = math.Max(math.Max(tot.Peak, totpeak), tot.Curr)
+		lpeak = math.Max(math.Max(v[Large].Peak, lpeak), v[Large].Curr)
+		speak = math.Max(math.Max(v[Small].Peak, speak), v[Small].Curr)
+		if lpeak > 0 || speak > 0 {
+			count++
+		}
+	}
+	if count > 0 {
+		lrg := XferStats{
+			Avg:  lavg / float64(count),
+			Curr: lcurr / float64(count),
+			Peak: lpeak,
+		}
+		sml := XferStats{
+			Avg:  savg / float64(count),
+			Curr: scurr / float64(count),
+			Peak: speak,
+		}
+		qs.XferStats[Large] = lrg
+		qs.XferStats[Small] = sml
+		qs.XferStats[Total] = XferStats{
+			Avg:  (savg + lavg) / float64(count),
+			Curr: (lcurr + scurr) / float64(count),
+			Peak: totpeak,
+		}
+	}
+	return qs
+}
+
+// populate queue totals for node and active workers in use for metrics
+func (r *ReplicationStats) getNodeQueueStatsSummary() (qs ReplQNodeStats) {
+	qs.NodeName = globalLocalNodeName
+	qs.Uptime = UTCNow().Unix() - globalBootTime.Unix()
+	qs.ActiveWorkers = globalReplicationStats.ActiveWorkers()
+	qs.XferStats = make(map[RMetricName]XferStats)
+	qs.QStats = r.qCache.getSiteStats()
+
+	r.RLock()
+	defer r.RUnlock()
+	tx := newXferStats()
+	for _, brs := range r.Cache {
+		for _, v := range brs.Stats {
+			tx := tx.merge(*v.XferRateLrg)
+			tx = tx.merge(*v.XferRateSml)
+		}
+	}
+	qs.XferStats[Total] = *tx
+	return qs
+}
+
+// ReplicationQueueStats holds overall queue stats for replication
+type ReplicationQueueStats struct {
+	Nodes  []ReplQNodeStats `json:"nodes"`
+	Uptime int64            `json:"uptime"`
 }

cmd/bucket-stats_gen_test.go

@@ -461,6 +461,345 @@ func BenchmarkDecodeBucketStatsMap(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalReplQNodeStats(t *testing.T) {
+	v := ReplQNodeStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgReplQNodeStats(b *testing.B) {
+	v := ReplQNodeStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgReplQNodeStats(b *testing.B) {
+	v := ReplQNodeStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalReplQNodeStats(b *testing.B) {
+	v := ReplQNodeStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeReplQNodeStats(t *testing.T) {
+	v := ReplQNodeStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeReplQNodeStats Msgsize() is inaccurate")
+	}
+
+	vn := ReplQNodeStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeReplQNodeStats(b *testing.B) {
+	v := ReplQNodeStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeReplQNodeStats(b *testing.B) {
+	v := ReplQNodeStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalReplicationLastHour(t *testing.T) {
+	v := ReplicationLastHour{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgReplicationLastHour(b *testing.B) {
+	v := ReplicationLastHour{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgReplicationLastHour(b *testing.B) {
+	v := ReplicationLastHour{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalReplicationLastHour(b *testing.B) {
+	v := ReplicationLastHour{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeReplicationLastHour(t *testing.T) {
+	v := ReplicationLastHour{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeReplicationLastHour Msgsize() is inaccurate")
+	}
+
+	vn := ReplicationLastHour{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeReplicationLastHour(b *testing.B) {
+	v := ReplicationLastHour{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeReplicationLastHour(b *testing.B) {
+	v := ReplicationLastHour{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalReplicationLastMinute(t *testing.T) {
+	v := ReplicationLastMinute{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgReplicationLastMinute(b *testing.B) {
+	v := ReplicationLastMinute{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgReplicationLastMinute(b *testing.B) {
+	v := ReplicationLastMinute{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalReplicationLastMinute(b *testing.B) {
+	v := ReplicationLastMinute{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeReplicationLastMinute(t *testing.T) {
+	v := ReplicationLastMinute{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeReplicationLastMinute Msgsize() is inaccurate")
+	}
+
+	vn := ReplicationLastMinute{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeReplicationLastMinute(b *testing.B) {
+	v := ReplicationLastMinute{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeReplicationLastMinute(b *testing.B) {
+	v := ReplicationLastMinute{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalReplicationLatency(t *testing.T) {
 	v := ReplicationLatency{}
 	bts, err := v.MarshalMsg(nil)
@@ -573,3 +912,116 @@ func BenchmarkDecodeReplicationLatency(b *testing.B) {
 		}
 	}
 }
+
+func TestMarshalUnmarshalReplicationQueueStats(t *testing.T) {
+	v := ReplicationQueueStats{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgReplicationQueueStats(b *testing.B) {
+	v := ReplicationQueueStats{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgReplicationQueueStats(b *testing.B) {
+	v := ReplicationQueueStats{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalReplicationQueueStats(b *testing.B) {
+	v := ReplicationQueueStats{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeReplicationQueueStats(t *testing.T) {
+	v := ReplicationQueueStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeReplicationQueueStats Msgsize() is inaccurate")
+	}
+
+	vn := ReplicationQueueStats{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeReplicationQueueStats(b *testing.B) {
+	v := ReplicationQueueStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeReplicationQueueStats(b *testing.B) {
+	v := ReplicationQueueStats{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/bucket-targets.go

@@ -50,15 +50,37 @@ type BucketTargetSys struct {
 	hcClient      *madmin.AnonymousClient
 }
 
+type latencyStat struct {
+	lastmin lastMinuteLatency
+	curr    time.Duration
+	avg     time.Duration
+	peak    time.Duration
+	N       int64
+}
+
+func (l *latencyStat) update(d time.Duration) {
+	l.lastmin.add(d)
+	l.N++
+	if d > l.peak {
+		l.peak = d
+	}
+	l.curr = l.lastmin.getTotal().avg()
+	l.avg = time.Duration((int64(l.avg)*(l.N-1) + int64(l.curr)) / l.N)
+}
+
 // epHealth struct represents health of a replication target endpoint.
 type epHealth struct {
-	Endpoint string
-	Scheme   string
-	Online   bool
+	Endpoint        string
+	Scheme          string
+	Online          bool
+	lastOnline      time.Time
+	lastHCAt        time.Time
+	offlineDuration time.Duration
+	latency         latencyStat
 }
 
 // isOffline returns current liveness result of remote target. Add endpoint to
-// healthcheck map if missing and default to online status
+// healthCheck map if missing and default to online status
 func (sys *BucketTargetSys) isOffline(ep *url.URL) bool {
 	sys.hMutex.RLock()
 	defer sys.hMutex.RUnlock()
@@ -69,6 +91,16 @@ func (sys *BucketTargetSys) isOffline(ep *url.URL) bool {
 	return false
 }
 
+// markOffline sets endpoint to offline if network i/o timeout seen.
+func (sys *BucketTargetSys) markOffline(ep *url.URL) {
+	sys.hMutex.Lock()
+	defer sys.hMutex.Unlock()
+	if h, ok := sys.hc[ep.Host]; ok {
+		h.Online = false
+		sys.hc[ep.Host] = h
+	}
+}
+
 func (sys *BucketTargetSys) initHC(ep *url.URL) {
 	sys.hMutex.Lock()
 	sys.hc[ep.Host] = epHealth{
@@ -107,11 +139,40 @@ func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
 			if len(eps) > 0 {
 				cctx, cancel := context.WithTimeout(ctx, 30*time.Second)
 				m := make(map[string]epHealth, len(eps))
+				start := time.Now()
+
 				for result := range sys.hcClient.Alive(cctx, madmin.AliveOpts{}, eps...) {
+					var lastOnline time.Time
+					var offline time.Duration
+					//	var deploymentID string
+					sys.hMutex.RLock()
+					prev, ok := sys.hc[result.Endpoint.Host]
+					sys.hMutex.RUnlock()
+					if ok {
+						if prev.Online != result.Online || !result.Online {
+							if !prev.lastHCAt.IsZero() {
+								offline = time.Since(prev.lastHCAt) + prev.offlineDuration
+							} else {
+								offline = prev.offlineDuration
+							}
+						} else if result.Online {
+							offline = prev.offlineDuration
+						}
+					}
+					lastOnline = prev.lastOnline
+					if result.Online {
+						lastOnline = time.Now()
+					}
+					l := prev.latency
+					l.update(time.Since(start))
 					m[result.Endpoint.Host] = epHealth{
-						Endpoint: result.Endpoint.Host,
-						Scheme:   result.Endpoint.Scheme,
-						Online:   result.Online,
+						Endpoint:        result.Endpoint.Host,
+						Scheme:          result.Endpoint.Scheme,
+						Online:          result.Online,
+						lastOnline:      lastOnline,
+						offlineDuration: offline,
+						lastHCAt:        time.Now(),
+						latency:         l,
 					}
 				}
 				cancel()
@@ -126,37 +187,66 @@ func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
 	}
 }
 
-// periodically rebuild the healthcheck map from list of targets to clear
+// periodically rebuild the healthCheck map from list of targets to clear
 // out stale endpoints
 func (sys *BucketTargetSys) reloadHealthCheckers(ctx context.Context) {
 	m := make(map[string]epHealth)
 	tgts := sys.ListTargets(ctx, "", "")
+	sys.hMutex.Lock()
 	for _, t := range tgts {
 		if _, ok := m[t.Endpoint]; !ok {
 			scheme := "http"
 			if t.Secure {
 				scheme = "https"
 			}
-			m[t.Endpoint] = epHealth{
+			epHealth := epHealth{
 				Online:   true,
 				Endpoint: t.Endpoint,
 				Scheme:   scheme,
 			}
+			if prev, ok := sys.hc[t.Endpoint]; ok {
+				epHealth.lastOnline = prev.lastOnline
+				epHealth.offlineDuration = prev.offlineDuration
+				epHealth.lastHCAt = prev.lastHCAt
+				epHealth.latency = prev.latency
+			}
+			m[t.Endpoint] = epHealth
 		}
 	}
-	sys.hMutex.Lock()
 	// swap out the map
 	sys.hc = m
 	sys.hMutex.Unlock()
 }
 
+func (sys *BucketTargetSys) healthStats() map[string]epHealth {
+	sys.hMutex.RLock()
+	defer sys.hMutex.RUnlock()
+	m := make(map[string]epHealth, len(sys.hc))
+	for k, v := range sys.hc {
+		m[k] = v
+	}
+	return m
+}
+
 // ListTargets lists bucket targets across tenant or for individual bucket, and returns
 // results filtered by arnType
 func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType string) (targets []madmin.BucketTarget) {
+	h := sys.healthStats()
+
 	if bucket != "" {
 		if ts, err := sys.ListBucketTargets(ctx, bucket); err == nil {
 			for _, t := range ts.Targets {
 				if string(t.Type) == arnType || arnType == "" {
+					if hs, ok := h[t.URL().Host]; ok {
+						t.TotalDowntime = hs.offlineDuration
+						t.Online = hs.Online
+						t.LastOnline = hs.lastOnline
+						t.Latency = madmin.LatencyStat{
+							Curr: hs.latency.curr,
+							Avg:  hs.latency.avg,
+							Max:  hs.latency.peak,
+						}
+					}
 					targets = append(targets, t.Clone())
 				}
 			}
@@ -168,6 +258,16 @@ func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType str
 	for _, tgts := range sys.targetsMap {
 		for _, t := range tgts {
 			if string(t.Type) == arnType || arnType == "" {
+				if hs, ok := h[t.URL().Host]; ok {
+					t.TotalDowntime = hs.offlineDuration
+					t.Online = hs.Online
+					t.LastOnline = hs.lastOnline
+					t.Latency = madmin.LatencyStat{
+						Curr: hs.latency.curr,
+						Avg:  hs.latency.avg,
+						Max:  hs.latency.peak,
+					}
+				}
 				targets = append(targets, t.Clone())
 			}
 		}
@@ -333,7 +433,7 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 }
 
 // GetRemoteTargetClient returns minio-go client for replication target instance
-func (sys *BucketTargetSys) GetRemoteTargetClient(ctx context.Context, arn string) *TargetClient {
+func (sys *BucketTargetSys) GetRemoteTargetClient(arn string) *TargetClient {
 	sys.RLock()
 	defer sys.RUnlock()
 	return sys.arnRemotesMap[arn]
@@ -362,7 +462,7 @@ func NewBucketTargetSys(ctx context.Context) *BucketTargetSys {
 		hc:            make(map[string]epHealth),
 		hcClient:      newHCClient(),
 	}
-	// reload healthcheck endpoints map periodically to remove stale endpoints from the map.
+	// reload healthCheck endpoints map periodically to remove stale endpoints from the map.
 	go func() {
 		rTimer := time.NewTimer(defaultHealthCheckReloadDuration)
 		defer rTimer.Stop()
@@ -479,6 +579,8 @@ func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTar
 	if target == nil {
 		return
 	}
+	sys.RLock()
+	defer sys.RUnlock()
 	tgts := sys.targetsMap[bucket]
 	for _, tgt := range tgts {
 		if tgt.Type == target.Type &&
@@ -496,6 +598,8 @@ func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTar
 
 // getRemoteARNForPeer returns the remote target for a peer site in site replication
 func (sys *BucketTargetSys) getRemoteARNForPeer(bucket string, peer madmin.PeerInfo) string {
+	sys.RLock()
+	defer sys.RUnlock()
 	tgts := sys.targetsMap[bucket]
 	for _, target := range tgts {
 		ep, _ := url.Parse(peer.Endpoint)

cmd/common-main.go

@@ -95,6 +95,9 @@ func init() {
 
 	initGlobalContext()
 
+	globalBatchJobsMetrics = batchJobMetrics{metrics: make(map[string]*batchJobInfo)}
+	go globalBatchJobsMetrics.purgeJobMetrics()
+
 	t, _ := minioVersionToReleaseTime(Version)
 	if !t.IsZero() {
 		globalVersionUnix = uint64(t.Unix())
@@ -102,22 +105,6 @@ func init() {
 
 	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != ""
 
-	// Call to refresh will refresh names in cache.
-	go func() {
-		// Baremetal setups set DNS refresh window to 10 minutes.
-		t := time.NewTicker(10 * time.Minute)
-		defer t.Stop()
-		for {
-			select {
-			case <-t.C:
-				globalDNSCache.Refresh()
-
-			case <-GlobalContext.Done():
-				return
-			}
-		}
-	}()
-
 	console.SetColor("Debug", fcolor.New())
 
 	gob.Register(StorageErr(""))
@@ -188,6 +175,10 @@ func minioConfigToConsoleFeatures() {
 	if value := env.Get(config.EnvBrowserLoginAnimation, "on"); value != "" {
 		os.Setenv("CONSOLE_ANIMATED_LOGIN", value)
 	}
+	// Pass on the session duration environment variable, else we will default to 12 hours
+	if value := env.Get(config.EnvBrowserSessionDuration, ""); value != "" {
+		os.Setenv("CONSOLE_STS_DURATION", value)
+	}
 
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
@@ -443,6 +434,28 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)}
 
 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
+
+	// Check if we have configured a custom DNS cache TTL.
+	dnsTTL := ctx.Duration("dns-cache-ttl")
+	if dnsTTL <= 0 {
+		dnsTTL = 10 * time.Minute
+	}
+
+	// Call to refresh will refresh names in cache.
+	go func() {
+		// Baremetal setups set DNS refresh window up to dnsTTL duration.
+		t := time.NewTicker(dnsTTL)
+		defer t.Stop()
+		for {
+			select {
+			case <-t.C:
+				globalDNSCache.Refresh()
+
+			case <-GlobalContext.Done():
+				return
+			}
+		}
+	}()
 }
 
 type envKV struct {

cmd/config-common.go

@@ -61,7 +61,8 @@ type objectDeleter interface {
 
 func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string) error {
 	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{
-		DeletePrefix: true,
+		DeletePrefix:       true,
+		DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
 	})
 	if err != nil && isErrObjectNotFound(err) {
 		return errConfigNotFound

cmd/config-current.go

@@ -386,7 +386,7 @@ func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, o
 		}
 	default:
 		if config.LoggerSubSystems.Contains(subSys) {
-			if err := logger.ValidateSubSysConfig(s, subSys); err != nil {
+			if err := logger.ValidateSubSysConfig(ctx, s, subSys); err != nil {
 				return err
 			}
 		}
@@ -440,7 +440,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
 	}
 	if err == nil && dnsURL != "" {
-		bootstrapTrace("initialize remote bucket DNS store")
+		bootstrapTraceMsg("initialize remote bucket DNS store")
 		globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
 			dns.Authentication(dnsUser, dnsPass),
 			dns.RootCAs(globalRootCAs))
@@ -455,7 +455,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 
 	if etcdCfg.Enabled {
-		bootstrapTrace("initialize etcd store")
+		bootstrapTraceMsg("initialize etcd store")
 		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
@@ -514,19 +514,19 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	transport := NewHTTPTransport()
 
-	bootstrapTrace("initialize the event notification targets")
+	bootstrapTraceMsg("initialize the event notification targets")
 	globalNotifyTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
-	bootstrapTrace("initialize the lambda targets")
+	bootstrapTraceMsg("initialize the lambda targets")
 	globalLambdaTargetList, err = lambda.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err))
 	}
 
-	bootstrapTrace("applying the dynamic configuration")
+	bootstrapTraceMsg("applying the dynamic configuration")
 	// Apply dynamic config values
 	if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
 		logger.LogIf(ctx, err)
@@ -575,7 +575,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		scannerCycle.Store(scannerCfg.Cycle)
 		logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
 	case config.LoggerWebhookSubSys:
-		loggerCfg, err := logger.LookupConfigForSubSys(s, config.LoggerWebhookSubSys)
+		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.LoggerWebhookSubSys)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to load logger webhook config: %w", err))
 		}
@@ -592,7 +592,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
 		}
 	case config.AuditWebhookSubSys:
-		loggerCfg, err := logger.LookupConfigForSubSys(s, config.AuditWebhookSubSys)
+		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.AuditWebhookSubSys)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to load audit webhook config: %w", err))
 		}
@@ -610,7 +610,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
 		}
 	case config.AuditKafkaSubSys:
-		loggerCfg, err := logger.LookupConfigForSubSys(s, config.AuditKafkaSubSys)
+		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.AuditKafkaSubSys)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to load audit kafka config: %w", err))
 		}
@@ -786,13 +786,13 @@ func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
 // from env if found and valid
 // data is optional. If nil it will be loaded from backend.
 func loadConfig(objAPI ObjectLayer, data []byte) error {
-	bootstrapTrace("load the configuration")
+	bootstrapTraceMsg("load the configuration")
 	srvCfg, err := readServerConfig(GlobalContext, objAPI, data)
 	if err != nil {
 		return err
 	}
 
-	bootstrapTrace("lookup the configuration")
+	bootstrapTraceMsg("lookup the configuration")
 	// Override any values from ENVs.
 	lookupConfigs(srvCfg, objAPI)
 

cmd/config-migrate.go

@@ -2423,7 +2423,7 @@ func migrateV27ToV28() error {
 // Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
 // if etcd is configured then migrates /config/config.json to '<export_path>/.minio.sys/config/config.json'
 func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
-	bootstrapTrace("migrate config to .minio.sys/config/config.json")
+	bootstrapTraceMsg("migrate config to .minio.sys/config/config.json")
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 

cmd/config.go

@@ -97,7 +97,8 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
 func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) error {
 	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
 	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, historyFile, ObjectOptions{
-		DeletePrefix: true,
+		DeletePrefix:       true,
+		DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
 	})
 	return err
 }
@@ -196,10 +197,10 @@ func NewConfigSys() *ConfigSys {
 
 // Initialize and load config from remote etcd or local config directory
 func initConfig(objAPI ObjectLayer) (err error) {
-	bootstrapTrace("load the configuration")
+	bootstrapTraceMsg("load the configuration")
 	defer func() {
 		if err != nil {
-			bootstrapTrace(fmt.Sprintf("loading configuration failed: %v", err))
+			bootstrapTraceMsg(fmt.Sprintf("loading configuration failed: %v", err))
 		}
 	}()
 
@@ -212,7 +213,7 @@ func initConfig(objAPI ObjectLayer) (err error) {
 		return err
 	}
 
-	bootstrapTrace("lookup the configuration")
+	bootstrapTraceMsg("lookup the configuration")
 
 	// Override any values from ENVs.
 	lookupConfigs(srvCfg, objAPI)

cmd/data-scanner-metric.go

@@ -142,14 +142,14 @@ func (p *scannerMetrics) incTime(s scannerMetric, d time.Duration) {
 // timeILM times an ILM action.
 // lifecycle.NoneAction is ignored.
 // Use for s < scannerMetricLastRealtime
-func (p *scannerMetrics) timeILM(a lifecycle.Action) func() {
+func (p *scannerMetrics) timeILM(a lifecycle.Action) func(versions uint64) {
 	if a == lifecycle.NoneAction || a >= lifecycle.ActionCount {
-		return func() {}
+		return func(_ uint64) {}
 	}
 	startTime := time.Now()
-	return func() {
+	return func(versions uint64) {
 		duration := time.Since(startTime)
-		atomic.AddUint64(&p.actions[a], 1)
+		atomic.AddUint64(&p.actions[a], versions)
 		p.actionsLatency[a].add(duration)
 	}
 }

cmd/data-scanner.go

@@ -550,7 +550,6 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				dst = &dataUsageEntry{Compacted: false}
 			}
 			if err := f.scanFolder(ctx, folder, dst); err != nil {
-				logger.LogIf(ctx, err)
 				return
 			}
 			if !into.Compacted {
@@ -847,9 +846,11 @@ type sizeSummary struct {
 	versions        uint64
 	deleteMarkers   uint64
 	replicatedSize  int64
+	replicatedCount int64
 	pendingSize     int64
 	failedSize      int64
 	replicaSize     int64
+	replicaCount    int64
 	pendingCount    uint64
 	failedCount     uint64
 	replTargetStats map[string]replTargetSizeSummary
@@ -858,11 +859,12 @@ type sizeSummary struct {
 
 // replTargetSizeSummary holds summary of replication stats by target
 type replTargetSizeSummary struct {
-	replicatedSize int64
-	pendingSize    int64
-	failedSize     int64
-	pendingCount   uint64
-	failedCount    uint64
+	replicatedSize  int64
+	replicatedCount int64
+	pendingSize     int64
+	failedSize      int64
+	pendingCount    uint64
+	failedCount     uint64
 }
 
 type getSizeFn func(item scannerItem) (sizeSummary, error)
@@ -900,15 +902,11 @@ func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi Object
 		Remove:   healDeleteDangling,
 		ScanMode: scanMode,
 	}
-	res, err := o.HealObject(ctx, i.bucket, i.objectPath(), oi.VersionID, healOpts)
-	if err != nil {
-		if errors.Is(err, NotImplemented{}) || isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-			err = nil
-		}
-		logger.LogIf(ctx, err)
-		return 0
+	res, _ := o.HealObject(ctx, i.bucket, i.objectPath(), oi.VersionID, healOpts)
+	if res.ObjectSize > 0 {
+		return res.ObjectSize
 	}
-	return res.ObjectSize
+	return 0
 }
 
 func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi ObjectInfo) (applied bool, size int64) {
@@ -930,7 +928,15 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), lcEvt.Action)
 		}
 	}
-	defer globalScannerMetrics.timeILM(lcEvt.Action)()
+	defer func() {
+		if applied {
+			numVersions := uint64(1)
+			if lcEvt.Action == lifecycle.DeleteAllVersionsAction {
+				numVersions = uint64(oi.NumVersions)
+			}
+			globalScannerMetrics.timeILM(lcEvt.Action)(numVersions)
+		}
+	}()
 
 	switch lcEvt.Action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction, lifecycle.DeleteAllVersionsAction:
@@ -990,7 +996,6 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 
 	versioned := vcfg != nil && vcfg.Versioned(i.objectPath())
 
-	// current version + most recent lim noncurrent versions
 	objectInfos := make([]ObjectInfo, 0, len(fivs))
 
 	if i.lifeCycle == nil {
@@ -1010,6 +1015,10 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 	}
 
 	overflowVersions := fivs[lim+1:]
+	// Retain the current version + most recent lim noncurrent versions
+	for _, fi := range fivs[:lim+1] {
+		objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
+	}
 
 	toDel := make([]ObjectToDelete, 0, len(overflowVersions))
 	for _, fi := range overflowVersions {
@@ -1178,10 +1187,10 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	if lcEvent.Action.DeleteVersioned() {
 		opts.VersionID = obj.VersionID
 	}
-	if opts.VersionID == "" {
-		opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
-		opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
-	}
+
+	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
+	opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
+
 	if lcEvent.Action.DeleteAll() {
 		opts.DeletePrefix = true
 	}
@@ -1276,13 +1285,16 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
 			sizeS.failedCount++
 		case replication.Completed, replication.CompletedLegacy:
 			tgtSizeS.replicatedSize += oi.Size
+			tgtSizeS.replicatedCount++
 			sizeS.replicatedSize += oi.Size
+			sizeS.replicatedCount++
 		}
 		sizeS.replTargetStats[arn] = tgtSizeS
 	}
 
 	if oi.ReplicationStatus == replication.Replica {
 		sizeS.replicaSize += oi.Size
+		sizeS.replicaCount++
 	}
 }
 

cmd/data-scanner_test.go

@@ -0,0 +1,137 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/xml"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+)
+
+func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
+	objAPI, disks, err := prepareErasure(context.Background(), 8)
+	if err != nil {
+		t.Fatalf("Failed to initialize object layer: %v", err)
+	}
+	defer removeRoots(disks)
+	setObjectLayer(objAPI)
+	globalBucketMetadataSys = NewBucketMetadataSys()
+	globalBucketObjectLockSys = &BucketObjectLockSys{}
+	globalBucketVersioningSys = &BucketVersioningSys{}
+	globalExpiryState = newExpiryState()
+	var wg sync.WaitGroup
+	wg.Add(1)
+	expired := make([]ObjectToDelete, 0, 5)
+	go func() {
+		defer wg.Done()
+		for t := range globalExpiryState.byNewerNoncurrentCh {
+			expired = append(expired, t.versions...)
+		}
+	}()
+	lc := lifecycle.Lifecycle{
+		Rules: []lifecycle.Rule{
+			{
+				ID:     "max-versions",
+				Status: "Enabled",
+				NoncurrentVersionExpiration: lifecycle.NoncurrentVersionExpiration{
+					NewerNoncurrentVersions: 1,
+				},
+			},
+		},
+	}
+	lcXML, err := xml.Marshal(lc)
+	if err != nil {
+		t.Fatalf("Failed to marshal lifecycle config: %v", err)
+	}
+	vcfg := versioning.Versioning{
+		Status: "Enabled",
+	}
+	vcfgXML, err := xml.Marshal(vcfg)
+	if err != nil {
+		t.Fatalf("Failed to marshal versioning config: %v", err)
+	}
+
+	bucket := "bucket"
+	obj := "obj-1"
+	now := time.Now()
+	meta := BucketMetadata{
+		Name:                      bucket,
+		Created:                   now,
+		LifecycleConfigXML:        lcXML,
+		VersioningConfigXML:       vcfgXML,
+		VersioningConfigUpdatedAt: now,
+		LifecycleConfigUpdatedAt:  now,
+		lifecycleConfig:           &lc,
+		versioningConfig:          &vcfg,
+	}
+	globalBucketMetadataSys.Set(bucket, meta)
+	item := scannerItem{
+		Path:       obj,
+		bucket:     bucket,
+		prefix:     "",
+		objectName: obj,
+		lifeCycle:  &lc,
+	}
+
+	modTime := time.Now()
+	uuids := make([]uuid.UUID, 5)
+	for i := range uuids {
+		uuids[i] = uuid.UUID([16]byte{15: uint8(i + 1)})
+	}
+	fivs := make([]FileInfo, 5)
+	for i := 0; i < 5; i++ {
+		fivs[i] = FileInfo{
+			Volume:      bucket,
+			Name:        obj,
+			VersionID:   uuids[i].String(),
+			IsLatest:    i == 0,
+			ModTime:     modTime.Add(-1 * time.Duration(i) * time.Minute),
+			Size:        1 << 10,
+			NumVersions: 5,
+		}
+	}
+	versioned := vcfg.Status == "Enabled"
+	wants := make([]ObjectInfo, 2)
+	for i, fi := range fivs[:2] {
+		wants[i] = fi.ToObjectInfo(bucket, obj, versioned)
+	}
+	gots, err := item.applyNewerNoncurrentVersionLimit(context.TODO(), objAPI, fivs)
+	if err != nil {
+		t.Fatalf("Failed with err: %v", err)
+	}
+	if len(gots) != len(wants) {
+		t.Fatalf("Expected %d objects but got %d", len(wants), len(gots))
+	}
+
+	// Close expiry state's channel to inspect object versions enqueued for expiration
+	close(globalExpiryState.byNewerNoncurrentCh)
+	wg.Wait()
+	for _, obj := range expired {
+		switch obj.ObjectV.VersionID {
+		case uuids[2].String(), uuids[3].String(), uuids[4].String():
+		default:
+			t.Errorf("Unexpected versionID being expired: %#v\n", obj)
+		}
+	}
+}

cmd/data-usage-cache.go

@@ -164,6 +164,7 @@ type replicationStats struct {
 	AfterThresholdSize   uint64
 	MissedThresholdCount uint64
 	AfterThresholdCount  uint64
+	ReplicatedCount      uint64
 }
 
 func (rs replicationStats) Empty() bool {
@@ -173,14 +174,16 @@ func (rs replicationStats) Empty() bool {
 }
 
 type replicationAllStats struct {
-	Targets     map[string]replicationStats `msg:"t,omitempty"`
-	ReplicaSize uint64                      `msg:"r,omitempty"`
+	Targets      map[string]replicationStats `msg:"t,omitempty"`
+	ReplicaSize  uint64                      `msg:"r,omitempty"`
+	ReplicaCount uint64                      `msg:"rc,omitempty"`
 }
 
 //msgp:tuple replicationAllStatsV1
 type replicationAllStatsV1 struct {
-	Targets     map[string]replicationStats
-	ReplicaSize uint64 `msg:"ReplicaSize,omitempty"`
+	Targets      map[string]replicationStats
+	ReplicaSize  uint64 `msg:"ReplicaSize,omitempty"`
+	ReplicaCount uint64 `msg:"ReplicaCount,omitempty"`
 }
 
 // clone creates a deep-copy clone.
@@ -341,6 +344,7 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 		e.ReplicationStats.Targets = make(map[string]replicationStats)
 	}
 	e.ReplicationStats.ReplicaSize += uint64(summary.replicaSize)
+	e.ReplicationStats.ReplicaCount += uint64(summary.replicaCount)
 
 	if summary.replTargetStats != nil {
 		for arn, st := range summary.replTargetStats {
@@ -351,6 +355,7 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 			tgtStat.PendingSize += uint64(st.pendingSize)
 			tgtStat.FailedSize += uint64(st.failedSize)
 			tgtStat.ReplicatedSize += uint64(st.replicatedSize)
+			tgtStat.ReplicatedCount += uint64(st.replicatedCount)
 			tgtStat.FailedCount += st.failedCount
 			tgtStat.PendingCount += st.pendingCount
 			e.ReplicationStats.Targets[arn] = tgtStat
@@ -377,14 +382,16 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 			e.ReplicationStats.Targets = make(map[string]replicationStats)
 		}
 		e.ReplicationStats.ReplicaSize += other.ReplicationStats.ReplicaSize
+		e.ReplicationStats.ReplicaCount += other.ReplicationStats.ReplicaCount
 		for arn, stat := range other.ReplicationStats.Targets {
 			st := e.ReplicationStats.Targets[arn]
 			e.ReplicationStats.Targets[arn] = replicationStats{
-				PendingSize:    stat.PendingSize + st.PendingSize,
-				FailedSize:     stat.FailedSize + st.FailedSize,
-				ReplicatedSize: stat.ReplicatedSize + st.ReplicatedSize,
-				PendingCount:   stat.PendingCount + st.PendingCount,
-				FailedCount:    stat.FailedCount + st.FailedCount,
+				PendingSize:     stat.PendingSize + st.PendingSize,
+				FailedSize:      stat.FailedSize + st.FailedSize,
+				ReplicatedSize:  stat.ReplicatedSize + st.ReplicatedSize,
+				PendingCount:    stat.PendingCount + st.PendingCount,
+				FailedCount:     stat.FailedCount + st.FailedCount,
+				ReplicatedCount: stat.ReplicatedCount + st.ReplicatedCount,
 			}
 		}
 	}
@@ -815,6 +822,8 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 		}
 		if flat.ReplicationStats != nil {
 			bui.ReplicaSize = flat.ReplicationStats.ReplicaSize
+			bui.ReplicaCount = flat.ReplicationStats.ReplicaCount
+
 			bui.ReplicationInfo = make(map[string]BucketTargetUsageInfo, len(flat.ReplicationStats.Targets))
 			for arn, stat := range flat.ReplicationStats.Targets {
 				bui.ReplicationInfo[arn] = BucketTargetUsageInfo{
@@ -823,6 +832,7 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 					ReplicationFailedSize:   stat.FailedSize,
 					ReplicationPendingCount: stat.PendingCount,
 					ReplicationFailedCount:  stat.FailedCount,
+					ReplicatedCount:         stat.ReplicatedCount,
 				}
 			}
 		}
@@ -1236,8 +1246,9 @@ func (d *dataUsageCache) deserialize(r io.Reader) error {
 			var replicationStats *replicationAllStats
 			if v.ReplicationStats != nil {
 				replicationStats = &replicationAllStats{
-					Targets:     v.ReplicationStats.Targets,
-					ReplicaSize: v.ReplicationStats.ReplicaSize,
+					Targets:      v.ReplicationStats.Targets,
+					ReplicaSize:  v.ReplicationStats.ReplicaSize,
+					ReplicaCount: v.ReplicationStats.ReplicaCount,
 				}
 			}
 			due := dataUsageEntry{

cmd/data-usage-cache_gen.go

@@ -2790,6 +2790,12 @@ func (z *replicationAllStats) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "ReplicaSize")
 				return
 			}
+		case "rc":
+			z.ReplicaCount, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicaCount")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -2804,8 +2810,8 @@ func (z *replicationAllStats) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 	// omitempty: check for empty values
-	zb0001Len := uint32(2)
-	var zb0001Mask uint8 /* 2 bits */
+	zb0001Len := uint32(3)
+	var zb0001Mask uint8 /* 3 bits */
 	_ = zb0001Mask
 	if z.Targets == nil {
 		zb0001Len--
@@ -2815,6 +2821,10 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 		zb0001Len--
 		zb0001Mask |= 0x2
 	}
+	if z.ReplicaCount == 0 {
+		zb0001Len--
+		zb0001Mask |= 0x4
+	}
 	// variable map header, size zb0001Len
 	err = en.Append(0x80 | uint8(zb0001Len))
 	if err != nil {
@@ -2859,6 +2869,18 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
+	if (zb0001Mask & 0x4) == 0 { // if not empty
+		// write "rc"
+		err = en.Append(0xa2, 0x72, 0x63)
+		if err != nil {
+			return
+		}
+		err = en.WriteUint64(z.ReplicaCount)
+		if err != nil {
+			err = msgp.WrapError(err, "ReplicaCount")
+			return
+		}
+	}
 	return
 }
 
@@ -2866,8 +2888,8 @@ func (z *replicationAllStats) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// omitempty: check for empty values
-	zb0001Len := uint32(2)
-	var zb0001Mask uint8 /* 2 bits */
+	zb0001Len := uint32(3)
+	var zb0001Mask uint8 /* 3 bits */
 	_ = zb0001Mask
 	if z.Targets == nil {
 		zb0001Len--
@@ -2877,6 +2899,10 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 		zb0001Len--
 		zb0001Mask |= 0x2
 	}
+	if z.ReplicaCount == 0 {
+		zb0001Len--
+		zb0001Mask |= 0x4
+	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
 	if zb0001Len == 0 {
@@ -2900,6 +2926,11 @@ func (z *replicationAllStats) MarshalMsg(b []byte) (o []byte, err error) {
 		o = append(o, 0xa1, 0x72)
 		o = msgp.AppendUint64(o, z.ReplicaSize)
 	}
+	if (zb0001Mask & 0x4) == 0 { // if not empty
+		// string "rc"
+		o = append(o, 0xa2, 0x72, 0x63)
+		o = msgp.AppendUint64(o, z.ReplicaCount)
+	}
 	return
 }
 
@@ -2957,6 +2988,12 @@ func (z *replicationAllStats) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "ReplicaSize")
 				return
 			}
+		case "rc":
+			z.ReplicaCount, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ReplicaCount")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -2978,7 +3015,7 @@ func (z *replicationAllStats) Msgsize() (s int) {
 			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
 		}
 	}
-	s += 2 + msgp.Uint64Size
+	s += 2 + msgp.Uint64Size + 3 + msgp.Uint64Size
 	return
 }
 
@@ -2990,8 +3027,8 @@ func (z *replicationAllStatsV1) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 2 {
-		err = msgp.ArrayError{Wanted: 2, Got: zb0001}
+	if zb0001 != 3 {
+		err = msgp.ArrayError{Wanted: 3, Got: zb0001}
 		return
 	}
 	var zb0002 uint32
@@ -3028,13 +3065,18 @@ func (z *replicationAllStatsV1) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err, "ReplicaSize")
 		return
 	}
+	z.ReplicaCount, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaCount")
+		return
+	}
 	return
 }
 
 // EncodeMsg implements msgp.Encodable
 func (z *replicationAllStatsV1) EncodeMsg(en *msgp.Writer) (err error) {
-	// array header, size 2
-	err = en.Append(0x92)
+	// array header, size 3
+	err = en.Append(0x93)
 	if err != nil {
 		return
 	}
@@ -3060,14 +3102,19 @@ func (z *replicationAllStatsV1) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "ReplicaSize")
 		return
 	}
+	err = en.WriteUint64(z.ReplicaCount)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaCount")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *replicationAllStatsV1) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// array header, size 2
-	o = append(o, 0x92)
+	// array header, size 3
+	o = append(o, 0x93)
 	o = msgp.AppendMapHeader(o, uint32(len(z.Targets)))
 	for za0001, za0002 := range z.Targets {
 		o = msgp.AppendString(o, za0001)
@@ -3078,6 +3125,7 @@ func (z *replicationAllStatsV1) MarshalMsg(b []byte) (o []byte, err error) {
 		}
 	}
 	o = msgp.AppendUint64(o, z.ReplicaSize)
+	o = msgp.AppendUint64(o, z.ReplicaCount)
 	return
 }
 
@@ -3089,8 +3137,8 @@ func (z *replicationAllStatsV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 2 {
-		err = msgp.ArrayError{Wanted: 2, Got: zb0001}
+	if zb0001 != 3 {
+		err = msgp.ArrayError{Wanted: 3, Got: zb0001}
 		return
 	}
 	var zb0002 uint32
@@ -3127,6 +3175,11 @@ func (z *replicationAllStatsV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err, "ReplicaSize")
 		return
 	}
+	z.ReplicaCount, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicaCount")
+		return
+	}
 	o = bts
 	return
 }
@@ -3140,7 +3193,7 @@ func (z *replicationAllStatsV1) Msgsize() (s int) {
 			s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize()
 		}
 	}
-	s += msgp.Uint64Size
+	s += msgp.Uint64Size + msgp.Uint64Size
 	return
 }
 
@@ -3152,8 +3205,8 @@ func (z *replicationStats) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 9 {
-		err = msgp.ArrayError{Wanted: 9, Got: zb0001}
+	if zb0001 != 10 {
+		err = msgp.ArrayError{Wanted: 10, Got: zb0001}
 		return
 	}
 	z.PendingSize, err = dc.ReadUint64()
@@ -3201,13 +3254,18 @@ func (z *replicationStats) DecodeMsg(dc *msgp.Reader) (err error) {
 		err = msgp.WrapError(err, "AfterThresholdCount")
 		return
 	}
+	z.ReplicatedCount, err = dc.ReadUint64()
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedCount")
+		return
+	}
 	return
 }
 
 // EncodeMsg implements msgp.Encodable
 func (z *replicationStats) EncodeMsg(en *msgp.Writer) (err error) {
-	// array header, size 9
-	err = en.Append(0x99)
+	// array header, size 10
+	err = en.Append(0x9a)
 	if err != nil {
 		return
 	}
@@ -3256,14 +3314,19 @@ func (z *replicationStats) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "AfterThresholdCount")
 		return
 	}
+	err = en.WriteUint64(z.ReplicatedCount)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedCount")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *replicationStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// array header, size 9
-	o = append(o, 0x99)
+	// array header, size 10
+	o = append(o, 0x9a)
 	o = msgp.AppendUint64(o, z.PendingSize)
 	o = msgp.AppendUint64(o, z.ReplicatedSize)
 	o = msgp.AppendUint64(o, z.FailedSize)
@@ -3273,6 +3336,7 @@ func (z *replicationStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendUint64(o, z.AfterThresholdSize)
 	o = msgp.AppendUint64(o, z.MissedThresholdCount)
 	o = msgp.AppendUint64(o, z.AfterThresholdCount)
+	o = msgp.AppendUint64(o, z.ReplicatedCount)
 	return
 }
 
@@ -3284,8 +3348,8 @@ func (z *replicationStats) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err)
 		return
 	}
-	if zb0001 != 9 {
-		err = msgp.ArrayError{Wanted: 9, Got: zb0001}
+	if zb0001 != 10 {
+		err = msgp.ArrayError{Wanted: 10, Got: zb0001}
 		return
 	}
 	z.PendingSize, bts, err = msgp.ReadUint64Bytes(bts)
@@ -3333,13 +3397,18 @@ func (z *replicationStats) UnmarshalMsg(bts []byte) (o []byte, err error) {
 		err = msgp.WrapError(err, "AfterThresholdCount")
 		return
 	}
+	z.ReplicatedCount, bts, err = msgp.ReadUint64Bytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err, "ReplicatedCount")
+		return
+	}
 	o = bts
 	return
 }
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *replicationStats) Msgsize() (s int) {
-	s = 1 + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size
+	s = 1 + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size
 	return
 }
 

cmd/data-usage-utils.go

@@ -38,6 +38,7 @@ type BucketTargetUsageInfo struct {
 	ReplicaSize             uint64 `json:"objectReplicaTotalSize"`
 	ReplicationPendingCount uint64 `json:"objectsPendingReplicationCount"`
 	ReplicationFailedCount  uint64 `json:"objectsFailedReplicationCount"`
+	ReplicatedCount         uint64 `json:"objectsReplicatedCount"`
 }
 
 // BucketUsageInfo - bucket usage info provides
@@ -64,6 +65,7 @@ type BucketUsageInfo struct {
 	VersionsCount           uint64                           `json:"versionsCount"`
 	DeleteMarkersCount      uint64                           `json:"deleteMarkersCount"`
 	ReplicaSize             uint64                           `json:"objectReplicaTotalSize"`
+	ReplicaCount            uint64                           `json:"objectReplicaCount"`
 	ReplicationInfo         map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
 }
 

cmd/disk-cache-backend.go

@@ -231,7 +231,7 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 // stops when disk usage drops to 56%
 func (c *diskCache) diskUsageLow() bool {
 	gcStopPct := c.quotaPct * c.lowWatermark / 100
-	di, err := disk.GetInfo(c.dir)
+	di, err := disk.GetInfo(c.dir, false)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
 		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
@@ -254,7 +254,7 @@ func (c *diskCache) diskSpaceAvailable(size int64) bool {
 	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 
 	gcTriggerPct := c.quotaPct * c.highWatermark / 100
-	di, err := disk.GetInfo(c.dir)
+	di, err := disk.GetInfo(c.dir, false)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return false
@@ -288,7 +288,7 @@ func (c *diskCache) queueGC() {
 // toClear returns how many bytes should be cleared to reach the low watermark quota.
 // returns 0 if below quota.
 func (c *diskCache) toClear() uint64 {
-	di, err := disk.GetInfo(c.dir)
+	di, err := disk.GetInfo(c.dir, false)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
 		ctx := logger.SetReqInfo(GlobalContext, reqInfo)

cmd/endpoint-ellipses.go

@@ -360,7 +360,7 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			return nil, -1, err
 		}
 		for i := range endpointList {
-			endpointList[i].SetPool(0)
+			endpointList[i].SetPoolIndex(0)
 		}
 		endpointServerPools = append(endpointServerPools, PoolEndpoints{
 			Legacy:       true,

cmd/endpoint.go

@@ -69,8 +69,9 @@ type Node struct {
 // Endpoint - any type of endpoint.
 type Endpoint struct {
 	*url.URL
-	Pool    int
 	IsLocal bool
+
+	PoolIdx, SetIdx, DiskIdx int
 }
 
 func (endpoint Endpoint) String() string {
@@ -106,9 +107,19 @@ func (endpoint *Endpoint) UpdateIsLocal() (err error) {
 	return nil
 }
 
-// SetPool sets a specific pool number to this node
-func (endpoint *Endpoint) SetPool(i int) {
-	endpoint.Pool = i
+// SetPoolIndex sets a specific pool number to this node
+func (endpoint *Endpoint) SetPoolIndex(i int) {
+	endpoint.PoolIdx = i
+}
+
+// SetSetIndex sets a specific set number to this node
+func (endpoint *Endpoint) SetSetIndex(i int) {
+	endpoint.SetIdx = i
+}
+
+// SetDiskIndex sets a specific disk number to this node
+func (endpoint *Endpoint) SetDiskIndex(i int) {
+	endpoint.DiskIdx = i
 }
 
 // NewEndpoint - returns new endpoint based on given arguments.
@@ -205,6 +216,9 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 	return Endpoint{
 		URL:     u,
 		IsLocal: isLocal,
+		PoolIdx: -1,
+		SetIdx:  -1,
+		DiskIdx: -1,
 	}, nil
 }
 
@@ -236,8 +250,8 @@ func (l EndpointServerPools) GetNodes() (nodes []Node) {
 					Host:   ep.Host,
 				}
 			}
-			if !slices.Contains(node.Pools, ep.Pool) {
-				node.Pools = append(node.Pools, ep.Pool)
+			if !slices.Contains(node.Pools, ep.PoolIdx) {
+				node.Pools = append(node.Pools, ep.PoolIdx)
 			}
 			nodesMap[ep.Host] = node
 		}
@@ -811,7 +825,9 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 			return nil, setupType, config.ErrInvalidEndpoint(nil).Msg("use path style endpoint for single node setup")
 		}
 
-		endpoint.SetPool(0)
+		endpoint.SetPoolIndex(0)
+		endpoint.SetSetIndex(0)
+		endpoint.SetDiskIndex(0)
 
 		var endpoints Endpoints
 		endpoints = append(endpoints, endpoint)
@@ -828,7 +844,7 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 
 	for poolIdx, args := range poolArgs {
 		var endpoints Endpoints
-		for _, iargs := range args {
+		for setIdx, iargs := range args {
 			// Convert args to endpoints
 			eps, err := NewEndpoints(iargs...)
 			if err != nil {
@@ -840,8 +856,10 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 				return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 			}
 
-			for i := range eps {
-				eps[i].SetPool(poolIdx)
+			for diskIdx := range eps {
+				eps[diskIdx].SetPoolIndex(poolIdx)
+				eps[diskIdx].SetSetIndex(setIdx)
+				eps[diskIdx].SetDiskIndex(diskIdx)
 			}
 
 			endpoints = append(endpoints, eps...)
@@ -1022,6 +1040,11 @@ func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType,
 		if endpoint.Type() != PathEndpointType {
 			return endpoints, setupType, config.ErrInvalidEndpoint(nil).Msg("use path style endpoint for single node setup")
 		}
+
+		endpoint.SetPoolIndex(0)
+		endpoint.SetSetIndex(0)
+		endpoint.SetDiskIndex(0)
+
 		endpoints = append(endpoints, endpoint)
 		setupType = ErasureSDSetupType
 
@@ -1033,7 +1056,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType,
 		return endpoints, setupType, nil
 	}
 
-	for _, iargs := range args {
+	for setIdx, iargs := range args {
 		// Convert args to endpoints
 		eps, err := NewEndpoints(iargs...)
 		if err != nil {
@@ -1045,6 +1068,11 @@ func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType,
 			return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
+		for diskIdx := range eps {
+			eps[diskIdx].SetSetIndex(setIdx)
+			eps[diskIdx].SetDiskIndex(diskIdx)
+		}
+
 		endpoints = append(endpoints, eps...)
 	}
 

cmd/endpoint_test.go

@@ -37,9 +37,9 @@ func TestNewEndpoint(t *testing.T) {
 		expectedType     EndpointType
 		expectedErr      error
 	}{
-		{"/foo", Endpoint{URL: &url.URL{Path: rootSlashFoo}, IsLocal: true}, PathEndpointType, nil},
-		{"https://example.org/path", Endpoint{URL: u2, IsLocal: false}, URLEndpointType, nil},
-		{"http://192.168.253.200/path", Endpoint{URL: u4, IsLocal: false}, URLEndpointType, nil},
+		{"/foo", Endpoint{&url.URL{Path: rootSlashFoo}, true, -1, -1, -1}, PathEndpointType, nil},
+		{"https://example.org/path", Endpoint{u2, false, -1, -1, -1}, URLEndpointType, nil},
+		{"http://192.168.253.200/path", Endpoint{u4, false, -1, -1, -1}, URLEndpointType, nil},
 		{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{SlashSeparator, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},

cmd/erasure-common.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"io"
 	"math/rand"
 	"sync"
 	"time"
@@ -41,7 +42,7 @@ func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 			if disks[i] == nil {
 				return
 			}
-			di, err := disks[i].DiskInfo(context.Background())
+			di, err := disks[i].DiskInfo(context.Background(), false)
 			if err != nil || di.Healing {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.
@@ -99,7 +100,7 @@ func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 			if disks[i] == nil {
 				return
 			}
-			di, err := disks[i].DiskInfo(context.Background())
+			di, err := disks[i].DiskInfo(context.Background(), false)
 			if err != nil || di.Healing {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.
@@ -188,18 +189,21 @@ func readMultipleFiles(ctx context.Context, disks []StorageAPI, req ReadMultiple
 		dataArray = append(dataArray, toAdd)
 	}
 
+	ignoredErrs := []error{
+		errFileNotFound,
+		errVolumeNotFound,
+		errFileVersionNotFound,
+		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
+		io.EOF,              // some times we would read without locks, ignore these errors
+	}
+	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)
+
 	errs := g.Wait()
 	for index, err := range errs {
 		if err == nil {
 			continue
 		}
-		if !IsErr(err, []error{
-			errFileNotFound,
-			errVolumeNotFound,
-			errFileVersionNotFound,
-			errDiskNotFound,
-			errUnformattedDisk,
-		}...) {
+		if !IsErr(err, ignoredErrs...) {
 			logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
 				disks[index], req.Bucket, req.Prefix, err),
 				disks[index].String())

cmd/erasure-decode.go

@@ -289,12 +289,15 @@ func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.Read
 }
 
 // Heal reads from readers, reconstruct shards and writes the data to the writers.
-func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.ReaderAt, totalLength int64) (derr error) {
+func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.ReaderAt, totalLength int64, prefer []bool) (derr error) {
 	if len(writers) != e.parityBlocks+e.dataBlocks {
 		return errInvalidArgument
 	}
 
 	reader := newParallelReader(readers, e, 0, totalLength)
+	if len(readers) == len(prefer) {
+		reader.preferReaders(prefer)
+	}
 
 	startBlock := int64(0)
 	endBlock := totalLength / e.blockSize

cmd/erasure-heal_test.go

@@ -132,7 +132,7 @@ func TestErasureHeal(t *testing.T) {
 		}
 
 		// test case setup is complete - now call Heal()
-		err = erasure.Heal(context.Background(), staleWriters, readers, test.size)
+		err = erasure.Heal(context.Background(), staleWriters, readers, test.size, nil)
 		closeBitrotReaders(readers)
 		closeBitrotWriters(staleWriters)
 		if err != nil && !test.shouldFail {

cmd/erasure-healing-common.go

@@ -347,13 +347,6 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 					dataErrs[i] = errFileCorrupt
 					continue
 				}
-
-				// Since erasure.Distribution is trustable we can fix the mismatching erasure.Index
-				if meta.Erasure.Distribution[i] != meta.Erasure.Index {
-					partsMetadata[i] = FileInfo{}
-					dataErrs[i] = errFileCorrupt
-					continue
-				}
 			}
 		}
 

cmd/erasure-healing-common_test.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -426,7 +425,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 			}
 
 			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", true)
-			_, err = getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
+			fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
 			}
@@ -484,11 +483,6 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 				}
 
 			}
-			partsMetadata, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", true)
-			fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
-			if !errors.Is(err, errErasureReadQuorum) {
-				t.Fatalf("Failed to getLatestFileInfo, expected %v, got %v", errErasureReadQuorum, err)
-			}
 
 			rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
 			onlineDisks, modTime, _ := listOnlineDisks(erasureDisks, partsMetadata, test.errs, rQuorum)

cmd/erasure-healing.go

@@ -101,7 +101,6 @@ func (er erasureObjects) listAndHeal(bucket, prefix string, healEntry func(strin
 		reportNotFound: false,
 		agreed: func(entry metaCacheEntry) {
 			if err := healEntry(bucket, entry); err != nil {
-				logger.LogIf(ctx, err)
 				cancel()
 			}
 		},
@@ -114,7 +113,6 @@ func (er erasureObjects) listAndHeal(bucket, prefix string, healEntry func(strin
 			}
 
 			if err := healEntry(bucket, *entry); err != nil {
-				logger.LogIf(ctx, err)
 				cancel()
 				return
 			}
@@ -661,6 +659,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			partChecksums := latestMeta.Parts[partIndex].Checksums
 			tillOffset := erasure.ShardFileOffset(0, partSize, partSize)
 			readers := make([]io.ReaderAt, len(latestDisks))
+			prefer := make([]bool, len(latestDisks))
 			checksumAlgo := erasureInfo.GetChecksumInfo(partNumber).Algorithm
 			for i, disk := range latestDisks {
 				if disk == OfflineDisk {
@@ -670,6 +669,8 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 				partPath := pathJoin(object, srcDataDir, fmt.Sprintf("part.%d", partNumber))
 				readers[i] = newBitrotReader(disk, copyPartsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
 					checksumInfo.Hash, erasure.ShardSize())
+				prefer[i] = disk.Hostname() == ""
+
 			}
 			writers := make([]io.Writer, len(outDatedDisks))
 			for i, disk := range outDatedDisks {
@@ -689,7 +690,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			// Heal each part. erasure.Heal() will write the healed
 			// part to .minio/tmp/uuid/ which needs to be renamed
 			// later to the final location.
-			err = erasure.Heal(ctx, writers, readers, partSize)
+			err = erasure.Heal(ctx, writers, readers, partSize, prefer)
 			closeBitrotReaders(readers)
 			closeBitrotWriters(writers)
 			if err != nil {
@@ -749,7 +750,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		// Attempt a rename now from healed data to final location.
 		partsMetadata[i].SetHealing()
 		if _, err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object); err != nil {
-			logger.LogIf(ctx, err)
 			return result, err
 		}
 
@@ -902,7 +902,6 @@ func (er *erasureObjects) healObjectDir(ctx context.Context, bucket, object stri
 			case errDiskNotFound:
 				hr.After.Drives[i].State = madmin.DriveStateOffline
 			default:
-				logger.LogIf(ctx, merr)
 				hr.After.Drives[i].State = madmin.DriveStateCorrupt
 			}
 		}

cmd/erasure-healing_test.go

@@ -1065,8 +1065,8 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	}
 
 	for i := 0; i < (nfi.Erasure.DataBlocks + nfi.Erasure.ParityBlocks); i++ {
-		_, err = erasureDisks[i].StatInfoFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
-		if err == nil {
+		stats, _ := erasureDisks[i].StatInfoFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		if len(stats) != 0 {
 			t.Errorf("Expected xl.meta file to be not present, but succeeeded")
 		}
 	}

cmd/erasure-metadata-utils.go

@@ -170,11 +170,10 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 		errFileNotFound,
 		errVolumeNotFound,
 		errFileVersionNotFound,
-		errDiskNotFound,
-		errUnformattedDisk,
 		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
 		io.EOF,              // some times we would read without locks, ignore these errors
 	}
+	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)
 	errs := g.Wait()
 	for index, err := range errs {
 		if err == nil {

cmd/erasure-metadata.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
-	"net/http"
 	"sort"
 	"strconv"
 	"strings"
@@ -365,8 +364,6 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 			// Server-side replication fields
 			fmt.Fprintf(h, "%v", meta.MarkDeleted)
 			fmt.Fprint(h, meta.Metadata[string(meta.ReplicationState.ReplicaStatus)])
-			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.ReplicationTimeStamp.Format(http.TimeFormat)])
-			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.ReplicaTimeStamp.Format(http.TimeFormat)])
 			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.ReplicationStatusInternal])
 			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.VersionPurgeStatusInternal])
 

cmd/erasure-multipart.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -36,6 +36,7 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/mimedb"
 	"github.com/minio/pkg/sync/errgroup"
@@ -100,7 +101,6 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 
 	// Pick one from the first valid metadata.
 	fi, err = pickValidFileInfo(ctx, partsMetadata, modTime, etag, quorum)
-
 	return fi, partsMetadata, err
 }
 
@@ -205,25 +205,31 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 			if err != nil {
 				return nil
 			}
-			wait := deletedCleanupSleeper.Timer(ctx)
-			if now.Sub(fi.ModTime) > expiry {
-				removeAll(pathJoin(diskPath, minioMetaMultipartBucket, uploadIDPath))
-			}
-			wait()
-			return nil
+			w := xioutil.NewDeadlineWorker(diskMaxTimeout)
+			return w.Run(func() error {
+				wait := deletedCleanupSleeper.Timer(ctx)
+				if now.Sub(fi.ModTime) > expiry {
+					removeAll(pathJoin(diskPath, minioMetaMultipartBucket, uploadIDPath))
+				}
+				wait()
+				return nil
+			})
 		})
 		vi, err := disk.StatVol(ctx, pathJoin(minioMetaMultipartBucket, shaDir))
 		if err != nil {
 			return nil
 		}
-		wait := deletedCleanupSleeper.Timer(ctx)
-		if now.Sub(vi.Created) > expiry {
-			// We are not deleting shaDir recursively here, if shaDir is empty
-			// and its older then we can happily delete it.
-			Remove(pathJoin(diskPath, minioMetaMultipartBucket, shaDir))
-		}
-		wait()
-		return nil
+		w := xioutil.NewDeadlineWorker(diskMaxTimeout)
+		return w.Run(func() error {
+			wait := deletedCleanupSleeper.Timer(ctx)
+			if now.Sub(vi.Created) > expiry {
+				// We are not deleting shaDir recursively here, if shaDir is empty
+				// and its older then we can happily delete it.
+				Remove(pathJoin(diskPath, minioMetaMultipartBucket, shaDir))
+			}
+			wait()
+			return nil
+		})
 	})
 
 	readDirFn(pathJoin(diskPath, minioMetaTmpBucket), func(tmpDir string, typ os.FileMode) error {
@@ -234,12 +240,15 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		if err != nil {
 			return nil
 		}
-		wait := deletedCleanupSleeper.Timer(ctx)
-		if now.Sub(vi.Created) > expiry {
-			removeAll(pathJoin(diskPath, minioMetaTmpBucket, tmpDir))
-		}
-		wait()
-		return nil
+		w := xioutil.NewDeadlineWorker(diskMaxTimeout)
+		return w.Run(func() error {
+			wait := deletedCleanupSleeper.Timer(ctx)
+			if now.Sub(vi.Created) > expiry {
+				removeAll(pathJoin(diskPath, minioMetaTmpBucket, tmpDir))
+			}
+			wait()
+			return nil
+		})
 	})
 }
 
@@ -300,7 +309,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 		populatedUploadIds.Add(uploadID)
 		uploads = append(uploads, MultipartInfo{
 			Object:    object,
-			UploadID:  uploadID,
+			UploadID:  base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID, uploadID))),
 			Initiated: fi.ModTime,
 		})
 	}
@@ -405,7 +414,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		wg.Add(1)
 		go func(disk StorageAPI) {
 			defer wg.Done()
-			di, err := disk.DiskInfo(ctx)
+			di, err := disk.DiskInfo(ctx, false)
 			if err != nil || di.ID == "" {
 				atomicOfflineDrives.Inc()
 				atomicParityDrives.Inc()
@@ -635,9 +644,8 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	tmpPartPath := pathJoin(tmpPart, partSuffix)
 
 	// Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete.
-	var online int
 	defer func() {
-		if online != len(onlineDisks) {
+		if countOnlineDisks(onlineDisks) != len(onlineDisks) {
 			er.deleteAll(context.Background(), minioMetaTmpBucket, tmpPart)
 		}
 	}()
@@ -944,10 +952,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	if err != nil {
 		return oi, err
 	}
-	wctx := wlkctx.Context()
+	ctx = wlkctx.Context()
 	defer uploadIDLock.Unlock(wlkctx)
 
-	fi, partsMetadata, err := er.checkUploadIDExists(wctx, bucket, object, uploadID, true)
+	fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
 	if err != nil {
 		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
@@ -1205,6 +1213,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			partsMetadata[index].Metadata = fi.Metadata
 			partsMetadata[index].Parts = fi.Parts
 			partsMetadata[index].Checksum = fi.Checksum
+			partsMetadata[index].Versioned = opts.Versioned || opts.VersionSuspended
 		}
 	}
 
@@ -1217,14 +1226,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	ctx = lkctx.Context()
 	defer lk.Unlock(lkctx)
 
-	// Write final `xl.meta` at uploadID location
-	onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum)
-	if err != nil {
-		return oi, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
-	}
-
 	// Remove parts that weren't present in CompleteMultipartUpload request.
 	for _, curpart := range currentFI.Parts {
+		// Remove part.meta which is not needed anymore.
+		er.removePartMeta(bucket, object, uploadID, currentFI.DataDir, curpart.Number)
+
 		if objectPartIndex(fi.Parts, curpart.Number) == -1 {
 			// Delete the missing part files. e.g,
 			// Request 1: NewMultipart
@@ -1232,14 +1238,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			// Request 3: PutObjectPart 2
 			// Request 4: CompleteMultipartUpload --part 2
 			// N.B. 1st part is not present. This part should be removed from the storage.
-			er.removeObjectPart(bucket, object, uploadID, fi.DataDir, curpart.Number)
+			er.removeObjectPart(bucket, object, uploadID, currentFI.DataDir, curpart.Number)
 		}
 	}
 
-	// Remove part.meta which is not needed anymore.
-	for _, part := range fi.Parts {
-		er.removePartMeta(bucket, object, uploadID, fi.DataDir, part.Number)
-	}
+	defer er.deleteAll(context.Background(), minioMetaMultipartBucket, uploadIDPath)
 
 	// Rename the multipart object to final location.
 	onlineDisks, versionsDisparity, err := renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,

cmd/erasure-object.go

@@ -412,9 +412,14 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 				switch scan {
 				case madmin.HealNormalScan, madmin.HealDeepScan:
 					healOnce.Do(func() {
-						if _, healing := er.getOnlineDisksWithHealing(); !healing {
-							go healObject(bucket, object, fi.VersionID, scan)
-						}
+						globalMRFState.addPartialOp(partialOperation{
+							bucket:    bucket,
+							object:    object,
+							versionID: fi.VersionID,
+							queued:    time.Now(),
+							setIndex:  er.setIndex,
+							poolIndex: er.poolIndex,
+						})
 					})
 					// Healing is triggered and we have written
 					// successfully the content to client for
@@ -571,10 +576,10 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		errFileNameTooLong,
 		errVolumeNotFound,
 		errFileVersionNotFound,
-		errDiskNotFound,
 		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
 		io.EOF,              // some times we would read without locks, ignore these errors
 	}
+	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)
 
 	errs := g.Wait()
 	for index, err := range errs {
@@ -732,9 +737,14 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	// additionally do not heal delete markers inline, let them be
 	// healed upon regular heal process.
 	if !fi.Deleted && missingBlocks > 0 && missingBlocks < readQuorum {
-		if _, healing := er.getOnlineDisksWithHealing(); !healing {
-			go healObject(bucket, object, fi.VersionID, madmin.HealNormalScan)
-		}
+		globalMRFState.addPartialOp(partialOperation{
+			bucket:    bucket,
+			object:    object,
+			versionID: fi.VersionID,
+			queued:    time.Now(),
+			setIndex:  er.setIndex,
+			poolIndex: er.poolIndex,
+		})
 	}
 
 	return fi, metaArr, onlineDisks, nil
@@ -1102,7 +1112,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			wg.Add(1)
 			go func(disk StorageAPI) {
 				defer wg.Done()
-				di, err := disk.DiskInfo(ctx)
+				di, err := disk.DiskInfo(ctx, false)
 				if err != nil || di.ID == "" {
 					atomicOfflineDrives.Inc()
 					atomicParityDrives.Inc()
@@ -1186,15 +1196,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	partName := "part.1"
 	tempErasureObj := pathJoin(uniqueID, fi.DataDir, partName)
 
-	// Delete temporary object in the event of failure.
-	// If PutObject succeeded there would be no temporary
-	// object to delete.
-	var online int
-	defer func() {
-		if online != len(onlineDisks) {
-			er.deleteAll(context.Background(), minioMetaTmpBucket, tempObj)
-		}
-	}()
+	defer er.deleteAll(context.Background(), minioMetaTmpBucket, tempObj)
 
 	shardFileSize := erasure.ShardFileSize(data.Size())
 	writers := make([]io.Writer, len(onlineDisks))
@@ -1299,6 +1301,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			Algorithm:  DefaultBitrotAlgorithm,
 			Hash:       bitrotWriterSum(w),
 		})
+		partsMetadata[i].Versioned = opts.Versioned || opts.VersionSuspended
 	}
 
 	userDefined["etag"] = r.MD5CurrentHexString()
@@ -1379,7 +1382,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	fi.ReplicationState = opts.PutReplicationState()
-	online = countOnlineDisks(onlineDisks)
 
 	// we are adding a new version to this object under the namespace lock, so this is the latest version.
 	fi.IsLatest = true
@@ -1959,8 +1961,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
-// updateObjectMeta will update the metadata of a file.
-func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object string, fi FileInfo, onlineDisks []StorageAPI) error {
+func (er erasureObjects) updateObjectMetaWithOpts(ctx context.Context, bucket, object string, fi FileInfo, onlineDisks []StorageAPI, opts UpdateMetadataOpts) error {
 	if len(fi.Metadata) == 0 {
 		return nil
 	}
@@ -1974,7 +1975,7 @@ func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object st
 			if onlineDisks[index] == nil {
 				return errDiskNotFound
 			}
-			return onlineDisks[index].UpdateMetadata(ctx, bucket, object, fi)
+			return onlineDisks[index].UpdateMetadata(ctx, bucket, object, fi, opts)
 		}, index)
 	}
 
@@ -1984,6 +1985,11 @@ func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object st
 	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, er.defaultWQuorum())
 }
 
+// updateObjectMeta will update the metadata of a file.
+func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object string, fi FileInfo, onlineDisks []StorageAPI) error {
+	return er.updateObjectMetaWithOpts(ctx, bucket, object, fi, onlineDisks, UpdateMetadataOpts{})
+}
+
 // DeleteObjectTags - delete object tags from an existing object
 func (er erasureObjects) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 	return er.PutObjectTags(ctx, bucket, object, "", opts)

cmd/erasure-server-pool-decom.go

@@ -696,7 +696,9 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 	// Check if bucket is object locked.
 	lr, _ := globalBucketObjectLockSys.Get(bi.Name)
 
-	for _, set := range pool.sets {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	for setIdx, set := range pool.sets {
 		set := set
 
 		filterLifecycle := func(bucket, object string, fi FileInfo) bool {
@@ -895,16 +897,26 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 		}
 
 		wk.Take()
-		go func() {
+		go func(setIdx int) {
 			defer wk.Give()
-			err := set.listObjectsToDecommission(ctx, bi,
-				func(entry metaCacheEntry) {
-					wk.Take()
-					go decommissionEntry(entry)
-				},
-			)
-			logger.LogIf(ctx, err)
-		}()
+			// We will perpetually retry listing if it fails, since we cannot
+			// possibly give up in this matter
+			for {
+				err := set.listObjectsToDecommission(ctx, bi,
+					func(entry metaCacheEntry) {
+						wk.Take()
+						go decommissionEntry(entry)
+					},
+				)
+				if err == nil {
+					break
+				}
+				setN := humanize.Ordinal(setIdx + 1)
+				retryDur := time.Duration(r.Float64() * float64(5*time.Second))
+				logger.LogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v, retrying in %v", setN, err, retryDur), "decom-listing-failed"+setN)
+				time.Sleep(retryDur)
+			}
+		}(setIdx)
 	}
 	wk.Wait()
 	return nil

cmd/erasure-server-pool.go

@@ -263,8 +263,8 @@ type serverPoolsAvailableSpace []poolAvailableSpace
 
 type poolAvailableSpace struct {
 	Index      int
-	Available  uint64
-	MaxUsedPct int // Used disk percentage of most filled disk, rounded down.
+	Available  uint64 // in bytes
+	MaxUsedPct int    // Used disk percentage of most filled disk, rounded down.
 }
 
 // TotalAvailable - total available space
@@ -286,7 +286,7 @@ func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
 	}
 	var ok bool
 	for _, z := range p {
-		if z.MaxUsedPct < max {
+		if z.Available > 0 && z.MaxUsedPct < max {
 			ok = true
 			break
 		}
@@ -299,7 +299,7 @@ func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
 
 	// Remove entries that are above.
 	for i, z := range p {
-		if z.MaxUsedPct < max {
+		if z.Available > 0 && z.MaxUsedPct < max {
 			continue
 		}
 		p[i].Available = 0
@@ -365,7 +365,7 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 		}
 		var available uint64
 		if !isMinioMetaBucketName(bucket) {
-			if avail, err := hasSpaceFor(zinfo, size); err != nil && !avail {
+			if avail, err := hasSpaceFor(zinfo, size); err != nil || !avail {
 				serverPools[i] = poolAvailableSpace{Index: i}
 				continue
 			}
@@ -1254,6 +1254,11 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 		return loi, err
 	}
 	defer merged.truncate(0) // Release when returning
+
+	if contextCanceled(ctx) {
+		return ListObjectVersionsInfo{}, ctx.Err()
+	}
+
 	if versionMarker == "" {
 		o := listPathOptions{Marker: marker}
 		// If we are not looking for a specific version skip it.
@@ -1341,6 +1346,10 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 			loi.Objects = append(loi.Objects, objInfo)
 			return loi, nil
 		}
+
+		if contextCanceled(ctx) {
+			return ListObjectsInfo{}, ctx.Err()
+		}
 	}
 
 	merged, err := z.listPath(ctx, &opts)
@@ -1354,6 +1363,10 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	merged.forwardPast(opts.Marker)
 	defer merged.truncate(0) // Release when returning
 
+	if contextCanceled(ctx) {
+		return ListObjectsInfo{}, ctx.Err()
+	}
+
 	// Default is recursive, if delimiter is set then list non recursive.
 	objects := merged.fileInfos(bucket, prefix, delimiter)
 	loi.IsTruncated = err == nil && len(objects) > 0
@@ -1475,7 +1488,7 @@ func (z *erasureServerPools) CopyObjectPart(ctx context.Context, srcBucket, srcO
 
 // PutObjectPart - writes part of an object to hashedSet based on the object name.
 func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (PartInfo, error) {
-	if err := checkPutObjectPartArgs(ctx, bucket, object, z); err != nil {
+	if err := checkPutObjectPartArgs(ctx, bucket, object, uploadID, z); err != nil {
 		return PartInfo{}, err
 	}
 
@@ -1507,7 +1520,7 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 }
 
 func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (MultipartInfo, error) {
-	if err := checkListPartsArgs(ctx, bucket, object, z); err != nil {
+	if err := checkListPartsArgs(ctx, bucket, object, uploadID, z); err != nil {
 		return MultipartInfo{}, err
 	}
 
@@ -1538,7 +1551,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 
 // ListObjectParts - lists all uploaded parts to an object in hashedSet.
 func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (ListPartsInfo, error) {
-	if err := checkListPartsArgs(ctx, bucket, object, z); err != nil {
+	if err := checkListPartsArgs(ctx, bucket, object, uploadID, z); err != nil {
 		return ListPartsInfo{}, err
 	}
 
@@ -1567,7 +1580,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 
 // Aborts an in-progress multipart operation on hashedSet based on the object name.
 func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
-	if err := checkAbortMultipartArgs(ctx, bucket, object, z); err != nil {
+	if err := checkAbortMultipartArgs(ctx, bucket, object, uploadID, z); err != nil {
 		return err
 	}
 
@@ -1598,7 +1611,7 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 
 // CompleteMultipartUpload - completes a pending multipart transaction, on hashedSet based on object name.
 func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	if err = checkCompleteMultipartArgs(ctx, bucket, object, z); err != nil {
+	if err = checkCompleteMultipartArgs(ctx, bucket, object, uploadID, z); err != nil {
 		return objInfo, err
 	}
 
@@ -1822,45 +1835,86 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 						return
 					}
 
+					send := func(objInfo ObjectInfo) bool {
+						select {
+						case <-ctx.Done():
+							return false
+						case results <- objInfo:
+							return true
+						}
+					}
+
+					askDisks := getListQuorum(opts.WalkAskDisks, set.setDriveCount)
+					var fallbackDisks []StorageAPI
+
+					// Special case: ask all disks if the drive count is 4
+					if set.setDriveCount == 4 || askDisks > len(disks) {
+						askDisks = len(disks) // use all available drives
+					}
+
+					if askDisks > 0 && len(disks) > askDisks {
+						fallbackDisks = disks[askDisks:]
+						disks = disks[:askDisks]
+					}
+
+					requestedVersions := 0
+					if opts.WalkLatestOnly {
+						requestedVersions = 1
+					}
 					loadEntry := func(entry metaCacheEntry) {
 						if entry.isDir() {
 							return
 						}
 
-						fivs, err := entry.fileInfoVersions(bucket)
-						if err != nil {
-							cancel()
-							return
-						}
-
-						versionsSorter(fivs.Versions).reverse()
-
-						for _, version := range fivs.Versions {
-							send := true
-							if opts.WalkFilter != nil && !opts.WalkFilter(version) {
-								send = false
-							}
-
-							if !send {
-								continue
-							}
-
-							versioned := vcfg != nil && vcfg.Versioned(version.Name)
-							objInfo := version.ToObjectInfo(bucket, version.Name, versioned)
-
-							select {
-							case <-ctx.Done():
+						if opts.WalkLatestOnly {
+							fi, err := entry.fileInfo(bucket)
+							if err != nil {
+								cancel()
 								return
-							case results <- objInfo:
+							}
+							if opts.WalkFilter != nil {
+								if opts.WalkFilter(fi) {
+									if !send(fi.ToObjectInfo(bucket, fi.Name, vcfg != nil && vcfg.Versioned(fi.Name))) {
+										return
+									}
+								}
+							} else {
+								if !send(fi.ToObjectInfo(bucket, fi.Name, vcfg != nil && vcfg.Versioned(fi.Name))) {
+									return
+								}
+							}
+
+						} else {
+							fivs, err := entry.fileInfoVersions(bucket)
+							if err != nil {
+								cancel()
+								return
+							}
+
+							versionsSorter(fivs.Versions).reverse()
+
+							for _, version := range fivs.Versions {
+								if opts.WalkFilter != nil {
+									if opts.WalkFilter(version) {
+										if !send(version.ToObjectInfo(bucket, version.Name, vcfg != nil && vcfg.Versioned(version.Name))) {
+											return
+										}
+									}
+								} else {
+									if !send(version.ToObjectInfo(bucket, version.Name, vcfg != nil && vcfg.Versioned(version.Name))) {
+										return
+									}
+								}
 							}
 						}
 					}
 
 					// How to resolve partial results.
 					resolver := metadataResolutionParams{
-						dirQuorum: 1,
-						objQuorum: 1,
-						bucket:    bucket,
+						dirQuorum:         1,
+						objQuorum:         1,
+						bucket:            bucket,
+						requestedVersions: requestedVersions,
 					}
 
 					path := baseDirFromPrefix(prefix)
@@ -1871,6 +1925,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 
 					lopts := listPathRawOptions{
 						disks:          disks,
+						fallbackDisks:  fallbackDisks,
 						bucket:         bucket,
 						path:           path,
 						filterPrefix:   filterPrefix,
@@ -2060,9 +2115,13 @@ type HealthOptions struct {
 // additionally with any specific heuristic information which
 // was queried
 type HealthResult struct {
-	Healthy       bool
-	HealingDrives int
-	PoolID, SetID int
+	Healthy        bool
+	HealingDrives  int
+	UnhealthyPools []struct {
+		Maintenance   bool
+		PoolID, SetID int
+		WriteQuorum   int
+	}
 	WriteQuorum   int
 	UsingDefaults bool
 }
@@ -2164,24 +2223,6 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 		usingDefaults = true
 	}
 
-	for poolIdx := range erasureSetUpCount {
-		for setIdx := range erasureSetUpCount[poolIdx] {
-			if erasureSetUpCount[poolIdx][setIdx] < poolWriteQuorums[poolIdx] {
-				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
-					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
-						poolIdx, setIdx, poolWriteQuorums[poolIdx]))
-				return HealthResult{
-					Healthy:       false,
-					HealingDrives: len(aggHealStateResult.HealDisks),
-					PoolID:        poolIdx,
-					SetID:         setIdx,
-					WriteQuorum:   poolWriteQuorums[poolIdx],
-					UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
-				}
-			}
-		}
-	}
-
 	var maximumWriteQuorum int
 	for _, writeQuorum := range poolWriteQuorums {
 		if maximumWriteQuorum == 0 {
@@ -2192,6 +2233,35 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 		}
 	}
 
+	result := HealthResult{
+		HealingDrives: len(aggHealStateResult.HealDisks),
+		WriteQuorum:   maximumWriteQuorum,
+		UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
+	}
+
+	for poolIdx := range erasureSetUpCount {
+		for setIdx := range erasureSetUpCount[poolIdx] {
+			if erasureSetUpCount[poolIdx][setIdx] < poolWriteQuorums[poolIdx] {
+				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
+					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
+						poolIdx, setIdx, poolWriteQuorums[poolIdx]))
+				result.UnhealthyPools = append(result.UnhealthyPools, struct {
+					Maintenance                bool
+					PoolID, SetID, WriteQuorum int
+				}{
+					Maintenance: opts.Maintenance,
+					SetID:       setIdx,
+					PoolID:      poolIdx,
+					WriteQuorum: poolWriteQuorums[poolIdx],
+				})
+			}
+		}
+		if len(result.UnhealthyPools) > 0 {
+			// We have unhealthy pools return error.
+			return result
+		}
+	}
+
 	// when maintenance is not specified we don't have
 	// to look at the healing side of the code.
 	if !opts.Maintenance {

cmd/erasure-sets.go

@@ -116,7 +116,10 @@ func (s *erasureSets) getDiskMap() map[Endpoint]StorageAPI {
 // Initializes a new StorageAPI from the endpoint argument, returns
 // StorageAPI and also `format` which exists on the disk.
 func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
-	disk, err := newStorageAPI(endpoint, false)
+	disk, err := newStorageAPI(endpoint, storageOpts{
+		cleanUp:     false,
+		healthCheck: false,
+	})
 	if err != nil {
 		return nil, nil, err
 	}
@@ -124,7 +127,7 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 	format, err := loadFormatErasure(disk)
 	if err != nil {
 		if errors.Is(err, errUnformattedDisk) {
-			info, derr := disk.DiskInfo(context.TODO())
+			info, derr := disk.DiskInfo(context.TODO(), false)
 			if derr != nil && info.RootDisk {
 				return nil, nil, fmt.Errorf("Drive: %s is a root drive", disk)
 			}
@@ -132,6 +135,15 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 		return nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error
 	}
 
+	disk.Close()
+	disk, err = newStorageAPI(endpoint, storageOpts{
+		cleanUp:     true,
+		healthCheck: true,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
 	return disk, format, nil
 }
 
@@ -241,21 +253,10 @@ func (s *erasureSets) connectDisks() {
 				}
 				s.erasureDisks[setIndex][diskIndex].Close()
 			}
-			if disk.IsLocal() {
-				disk.SetDiskID(format.Erasure.This)
-				s.erasureDisks[setIndex][diskIndex] = disk
-			} else {
-				// Enable healthcheck disk for remote endpoint.
-				disk, err = newStorageAPI(endpoint, true)
-				if err != nil {
-					printEndpointError(endpoint, err, false)
-					s.erasureDisksMu.Unlock()
-					return
-				}
-				disk.SetDiskID(format.Erasure.This)
-				s.erasureDisks[setIndex][diskIndex] = disk
-			}
+
+			disk.SetDiskID(format.Erasure.This)
 			disk.SetDiskLoc(s.poolIndex, setIndex, diskIndex)
+			s.erasureDisks[setIndex][diskIndex] = disk
 			s.erasureDisksMu.Unlock()
 		}(endpoint)
 	}
@@ -775,7 +776,7 @@ func (s *erasureSets) deletePrefix(ctx context.Context, bucket string, prefix st
 
 // DeleteObject - deletes an object from the hashedSet based on the object name.
 func (s *erasureSets) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	if opts.DeletePrefix {
+	if opts.DeletePrefix && !opts.DeletePrefixObject {
 		err := s.deletePrefix(ctx, bucket, object)
 		return ObjectInfo{}, err
 	}
@@ -1025,7 +1026,7 @@ func getHealDiskInfos(storageDisks []StorageAPI, errs []error) ([]DiskInfo, []er
 				return errDiskNotFound
 			}
 			var err error
-			infos[index], err = storageDisks[index].DiskInfo(context.TODO())
+			infos[index], err = storageDisks[index].DiskInfo(context.TODO(), false)
 			return err
 		}, index)
 	}
@@ -1055,7 +1056,10 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 
 // HealFormat - heals missing `format.json` on fresh unformatted disks.
 func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
-	storageDisks, _ := initStorageDisksWithErrors(s.endpoints.Endpoints, false)
+	storageDisks, _ := initStorageDisksWithErrors(s.endpoints.Endpoints, storageOpts{
+		cleanUp:     true,
+		healthCheck: true,
+	})
 
 	defer func(storageDisks []StorageAPI) {
 		if err != nil {

cmd/erasure-sets_test.go

@@ -173,7 +173,7 @@ func TestNewErasureSets(t *testing.T) {
 		defer os.RemoveAll(disk)
 	}
 
-	endpoints := mustGetNewEndpoints(0, erasureDisks...)
+	endpoints := mustGetNewEndpoints(0, 16, erasureDisks...)
 	_, _, err := waitForFormatErasure(true, endpoints, 1, 0, 16, "", "")
 	if err != errInvalidArgument {
 		t.Fatalf("Expecting error, got %s", err)

cmd/erasure.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bpool"
 	"github.com/minio/minio/internal/dsync"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/sync/errgroup"
 )
@@ -174,33 +175,32 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 	for index := range disks {
 		index := index
 		g.Go(func() error {
-			diskEndpoint := endpoints[index].String()
+			di := madmin.Disk{
+				Endpoint:  endpoints[index].String(),
+				PoolIndex: endpoints[index].PoolIdx,
+				SetIndex:  endpoints[index].SetIdx,
+				DiskIndex: endpoints[index].DiskIdx,
+			}
 			if disks[index] == OfflineDisk {
-				logger.LogOnceIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]), "get-disks-info-offline-"+diskEndpoint)
-				disksInfo[index] = madmin.Disk{
-					State:    diskErrToDriveState(errDiskNotFound),
-					Endpoint: diskEndpoint,
-				}
+				logger.LogOnceIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]), "get-disks-info-offline-"+di.Endpoint)
+				di.State = diskErrToDriveState(errDiskNotFound)
+				disksInfo[index] = di
 				return nil
 			}
-			info, err := disks[index].DiskInfo(context.TODO())
-			di := madmin.Disk{
-				Endpoint:       diskEndpoint,
-				DrivePath:      info.MountPath,
-				TotalSpace:     info.Total,
-				UsedSpace:      info.Used,
-				AvailableSpace: info.Free,
-				UUID:           info.ID,
-				Major:          info.Major,
-				Minor:          info.Minor,
-				RootDisk:       info.RootDisk,
-				Healing:        info.Healing,
-				Scanning:       info.Scanning,
-				State:          diskErrToDriveState(err),
-				FreeInodes:     info.FreeInodes,
-				UsedInodes:     info.UsedInodes,
-			}
-			di.PoolIndex, di.SetIndex, di.DiskIndex = disks[index].GetDiskLoc()
+			info, err := disks[index].DiskInfo(context.TODO(), true)
+			di.DrivePath = info.MountPath
+			di.TotalSpace = info.Total
+			di.UsedSpace = info.Used
+			di.AvailableSpace = info.Free
+			di.UUID = info.ID
+			di.Major = info.Major
+			di.Minor = info.Minor
+			di.RootDisk = info.RootDisk
+			di.Healing = info.Healing
+			di.Scanning = info.Scanning
+			di.State = diskErrToDriveState(err)
+			di.FreeInodes = info.FreeInodes
+			di.UsedInodes = info.UsedInodes
 			if info.Healing {
 				if hi := disks[index].Healing(); hi != nil {
 					hd := hi.toHealingDisk()
@@ -208,8 +208,10 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				}
 			}
 			di.Metrics = &madmin.DiskMetrics{
-				LastMinute: make(map[string]madmin.TimedAction, len(info.Metrics.LastMinute)),
-				APICalls:   make(map[string]uint64, len(info.Metrics.APICalls)),
+				LastMinute:              make(map[string]madmin.TimedAction, len(info.Metrics.LastMinute)),
+				APICalls:                make(map[string]uint64, len(info.Metrics.APICalls)),
+				TotalErrorsAvailability: info.Metrics.TotalErrorsAvailability,
+				TotalErrorsTimeout:      info.Metrics.TotalErrorsTimeout,
 			}
 			for k, v := range info.Metrics.LastMinute {
 				if v.N > 0 {
@@ -290,7 +292,7 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 				return
 			}
 
-			di, err := disk.DiskInfo(context.Background())
+			di, err := disk.DiskInfo(context.Background(), false)
 			if err != nil || di.Healing {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.
@@ -308,6 +310,7 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 	}
 	wg.Wait()
 
+	var scanningDisks []StorageAPI
 	for i, info := range infos {
 		// Check if one of the drives in the set is being healed.
 		// this information is used by scanner to skip healing
@@ -316,9 +319,16 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 			healing = true
 			continue
 		}
-		newDisks = append(newDisks, disks[i])
+		if !info.Scanning {
+			newDisks = append(newDisks, disks[i])
+		} else {
+			scanningDisks = append(scanningDisks, disks[i])
+		}
 	}
 
+	// Prefer new disks over disks which are currently being scanned.
+	newDisks = append(newDisks, scanningDisks...)
+
 	return newDisks, healing
 }
 
@@ -333,10 +343,13 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
 				defer wg.Done()
 				diskPath := disk.Endpoint().Path
 				readDirFn(pathJoin(diskPath, minioMetaTmpDeletedBucket), func(ddir string, typ os.FileMode) error {
-					wait := deletedCleanupSleeper.Timer(ctx)
-					removeAll(pathJoin(diskPath, minioMetaTmpDeletedBucket, ddir))
-					wait()
-					return nil
+					w := xioutil.NewDeadlineWorker(diskMaxTimeout)
+					return w.Run(func() error {
+						wait := deletedCleanupSleeper.Timer(ctx)
+						removeAll(pathJoin(diskPath, minioMetaTmpDeletedBucket, ddir))
+						wait()
+						return nil
+					})
 				})
 			}(disk)
 		}

cmd/format-erasure.go

@@ -663,14 +663,14 @@ func closeStorageDisks(storageDisks ...StorageAPI) {
 
 // Initialize storage disks for each endpoint.
 // Errors are returned for each endpoint with matching index.
-func initStorageDisksWithErrors(endpoints Endpoints, healthCheck bool) ([]StorageAPI, []error) {
+func initStorageDisksWithErrors(endpoints Endpoints, opts storageOpts) ([]StorageAPI, []error) {
 	// Bootstrap disks.
 	storageDisks := make([]StorageAPI, len(endpoints))
 	g := errgroup.WithNErrs(len(endpoints))
 	for index := range endpoints {
 		index := index
 		g.Go(func() (err error) {
-			storageDisks[index], err = newStorageAPI(endpoints[index], healthCheck)
+			storageDisks[index], err = newStorageAPI(endpoints[index], opts)
 			return err
 		}, index)
 	}

cmd/format-erasure_test.go

@@ -36,9 +36,9 @@ func TestFixFormatV3(t *testing.T) {
 	for _, erasureDir := range erasureDirs {
 		defer os.RemoveAll(erasureDir)
 	}
-	endpoints := mustGetNewEndpoints(0, erasureDirs...)
+	endpoints := mustGetNewEndpoints(0, 8, erasureDirs...)
 
-	storageDisks, errs := initStorageDisksWithErrors(endpoints, false)
+	storageDisks, errs := initStorageDisksWithErrors(endpoints, storageOpts{cleanUp: false, healthCheck: false})
 	for _, err := range errs {
 		if err != nil && err != errDiskNotFound {
 			t.Fatal(err)
@@ -555,11 +555,12 @@ func benchmarkInitStorageDisksN(b *testing.B, nDisks int) {
 	if err != nil {
 		b.Fatal(err)
 	}
-	endpoints := mustGetNewEndpoints(0, fsDirs...)
+
+	endpoints := mustGetNewEndpoints(0, 16, fsDirs...)
 	b.RunParallel(func(pb *testing.PB) {
 		endpoints := endpoints
 		for pb.Next() {
-			initStorageDisksWithErrors(endpoints, false)
+			initStorageDisksWithErrors(endpoints, storageOpts{cleanUp: false, healthCheck: false})
 		}
 	})
 }

cmd/generic-handlers.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
+	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
 	xnet "github.com/minio/pkg/net"
 
@@ -206,7 +207,7 @@ func getRedirectLocation(r *http.Request) *xnet.URL {
 }
 
 // guessIsHealthCheckReq - returns true if incoming request looks
-// like healthcheck request
+// like healthCheck request
 func guessIsHealthCheckReq(req *http.Request) bool {
 	if req == nil {
 		return false
@@ -229,7 +230,8 @@ func guessIsMetricsReq(req *http.Request) bool {
 	return (aType == authTypeAnonymous || aType == authTypeJWT) &&
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsPathLegacy ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ClusterPath ||
-		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2NodePath
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2NodePath ||
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2BucketPath
 }
 
 // guessIsRPCReq - returns true if the request is for an RPC endpoint.
@@ -399,6 +401,17 @@ func setRequestValidityMiddleware(h http.Handler) http.Handler {
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL)
 				return
 			}
+		} else {
+			// Validate bucket names if it is not empty
+			if bucketName != "" && s3utils.CheckValidBucketNameStrict(bucketName) != nil {
+				if ok {
+					tc.FuncName = "handler.ValidRequest"
+					tc.ResponseRecorder.LogErrBody = true
+				}
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
+				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL)
+				return
+			}
 		}
 		// Deny SSE-C requests if not made over TLS
 		if !globalIsTLS && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
@@ -430,6 +443,11 @@ func setRequestValidityMiddleware(h http.Handler) http.Handler {
 // is obtained from centralized etcd configuration service.
 func setBucketForwardingMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if origin := w.Header().Get("Access-Control-Allow-Origin"); origin == "null" {
+			// This is a workaround change to ensure that "Origin: null"
+			// incoming request to a response back as "*" instead of "null"
+			w.Header().Set("Access-Control-Allow-Origin", "*")
+		}
 		if globalDNSConfig == nil || !globalBucketFederation ||
 			guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
 			guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) {

cmd/global-heal.go

@@ -192,7 +192,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		tracker.setObject("")
 		tracker.setBucket(bucket)
 		// Heal current bucket again in case if it is failed
-		// in the  being of erasure set healing
+		// in the beginning of erasure set healing
 		if _, err := er.HealBucket(ctx, bucket, madmin.HealOpts{
 			ScanMode: scanMode,
 		}); err != nil {
@@ -241,7 +241,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		}
 
 		// Collect updates to tracker from concurrent healEntry calls
-		results := make(chan healEntryResult)
+		results := make(chan healEntryResult, 1000)
 		go func() {
 			for res := range results {
 				if res.entryDone {
@@ -256,6 +256,15 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			}
 		}()
 
+		send := func(result healEntryResult) bool {
+			select {
+			case <-ctx.Done():
+				return false
+			case results <- result:
+				return true
+			}
+		}
+
 		// Note: updates from healEntry to tracker must be sent on results channel.
 		healEntry := func(bucket string, entry metaCacheEntry) {
 			if entry.name == "" && len(entry.metadata) == 0 {
@@ -302,12 +311,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					result = healEntrySuccess(0)
 				}
 
-				select {
-				case <-ctx.Done():
-					return
-				case results <- result:
-				}
-
+				send(result)
 				return
 			}
 
@@ -342,10 +346,8 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				}
 				bgSeq.logHeal(madmin.HealItemObject)
 
-				select {
-				case <-ctx.Done():
+				if !send(result) {
 					return
-				case results <- result:
 				}
 			}
 			// All versions resulted in 'ObjectNotFound'

cmd/globals.go

@@ -193,7 +193,7 @@ var (
 	globalBucketSSEConfigSys *BucketSSEConfigSys
 	globalBucketTargetSys    *BucketTargetSys
 	// globalAPIConfig controls S3 API requests throttling,
-	// healthcheck readiness deadlines and cors settings.
+	// healthCheck readiness deadlines and cors settings.
 	globalAPIConfig = apiConfig{listQuorum: "strict", rootAccess: true}
 
 	globalStorageClass storageclass.Config
@@ -346,7 +346,7 @@ var (
 
 	globalTierConfigMgr *TierConfigMgr
 
-	globalTierJournal *tierJournal
+	globalTierJournal *TierJournal
 
 	globalConsoleSrv *restapi.Server
 
@@ -397,6 +397,11 @@ var (
 	globalInternodeInterface     string
 	globalInternodeInterfaceOnce sync.Once
 
+	// Set last client perf extra time (get lock, and validate)
+	globalLastClientPerfExtraTime int64
+
+	// Captures all batch jobs metrics globally
+	globalBatchJobsMetrics batchJobMetrics
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -49,7 +49,7 @@ type apiConfig struct {
 	staleUploadsExpiry          time.Duration
 	staleUploadsCleanupInterval time.Duration
 	deleteCleanupInterval       time.Duration
-	disableODirect              bool
+	enableODirect               bool
 	gzipObjects                 bool
 	rootAccess                  bool
 	syncEvents                  bool
@@ -164,17 +164,17 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
 	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
 	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
-	t.disableODirect = cfg.DisableODirect
+	t.enableODirect = cfg.EnableODirect
 	t.gzipObjects = cfg.GzipObjects
 	t.rootAccess = cfg.RootAccess
 	t.syncEvents = cfg.SyncEvents
 }
 
-func (t *apiConfig) isDisableODirect() bool {
+func (t *apiConfig) odirectEnabled() bool {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
-	return t.disableODirect
+	return t.enableODirect
 }
 
 func (t *apiConfig) shouldGzipObjects() bool {

cmd/handler-utils.go

@@ -38,6 +38,7 @@ import (
 const (
 	copyDirective    = "COPY"
 	replaceDirective = "REPLACE"
+	accessDirective  = "ACCESS"
 )
 
 // Parses location constraint from the incoming reader.