add queue_dir to help message for logger/audit targets

There is some consistent confusion between the Community Helm Chart in this repo and the MinIO Kubernetes Operator Helm Chart.

This change seeks to clarify the differences between the two charts and which ones are community maintained vs MinIO maintained.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,3 +1,9 @@
+## Community Contribution License
+All community contributions in this pull request are licensed to the project maintainers
+under the terms of the [Apache 2 license] (https://www.apache.org/licenses/LICENSE-2.0). 
+By creating this pull request I represent that I have the right to license the 
+contributions to the project maintainers under the Apache 2 license.
+
 ## Description
 
 

.github/workflows/go-cross.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3

.github/workflows/go-fips.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v3

.github/workflows/go-healing.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v3

.github/workflows/go-lint.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v3

.github/workflows/go.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
     steps:
       - uses: actions/checkout@v3

.github/workflows/iam-integrations.yaml

@@ -61,7 +61,7 @@ jobs:
       # are turned off - i.e. if ldap="", then ldap server is not enabled for
       # the tests.
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         ldap: ["", "localhost:389"]
         etcd: ["", "http://localhost:2379"]
         openid: ["", "http://127.0.0.1:5556/dex"]
@@ -82,9 +82,9 @@ jobs:
           check-latest: true
       - name: Test LDAP/OpenID/Etcd combo
         env:
-          LDAP_TEST_SERVER: ${{ matrix.ldap }}
-          ETCD_SERVER: ${{ matrix.etcd }}
-          OPENID_TEST_SERVER: ${{ matrix.openid }}
+          _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
+          _MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
+          _MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
@@ -92,20 +92,20 @@ jobs:
       - name: Test with multiple OpenID providers
         if: matrix.openid == 'http://127.0.0.1:5556/dex'
         env:
-          LDAP_TEST_SERVER: ${{ matrix.ldap }}
-          ETCD_SERVER: ${{ matrix.etcd }}
-          OPENID_TEST_SERVER: ${{ matrix.openid }}
-          OPENID_TEST_SERVER_2: "http://127.0.0.1:5557/dex"
+          _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
+          _MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
+          _MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
+          _MINIO_OPENID_TEST_SERVER_2: "http://127.0.0.1:5557/dex"
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-iam
       - name: Test with Access Management Plugin enabled
         env:
-          LDAP_TEST_SERVER: ${{ matrix.ldap }}
-          ETCD_SERVER: ${{ matrix.etcd }}
-          OPENID_TEST_SERVER: ${{ matrix.openid }}
-          POLICY_PLUGIN_ENDPOINT: "http://127.0.0.1:8080"
+          _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }}
+          _MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }}
+          _MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }}
+          _MINIO_POLICY_PLUGIN_TEST_ENDPOINT: "http://127.0.0.1:8080"
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0

.github/workflows/mint.yml

@@ -29,7 +29,7 @@ jobs:
       - name: setup-go-step
         uses: actions/setup-go@v2
         with:
-          go-version: 1.20.x
+          go-version: 1.21.x
 
       - name: github sha short
         id: vars

.github/workflows/replication.yaml

@@ -21,7 +21,7 @@ jobs:
 
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/root-disable.yml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
 
     steps:

.github/workflows/upgrade-ci-cd.yaml

@@ -20,7 +20,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.20.x]
+        go-version: [1.21.x]
         os: [ubuntu-latest]
 
     steps:

.github/workflows/vulncheck.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.11
+          go-version: 1.21.0
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

CREDITS

@@ -2386,10 +2386,13 @@ SOFTWARE.
 
 ================================================================
 
-github.com/Shopify/sarama
-https://github.com/Shopify/sarama
+github.com/IBM/sarama
+https://github.com/IBM/sarama
 ----------------------------------------------------------------
+# MIT License
+
 Copyright (c) 2013 Shopify
+Copyright (c) 2023 IBM Corporation
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
@@ -2410,34 +2413,6 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
-================================================================
-
-github.com/Shopify/toxiproxy/v2
-https://github.com/Shopify/toxiproxy/v2
-----------------------------------------------------------------
-The MIT License (MIT)
-
-Copyright (c) 2014 Shopify
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-
 ================================================================
 
 github.com/alecthomas/participle

Dockerfile.release

@@ -38,6 +38,7 @@ RUN \
      curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /opt/bin/mc && \
      microdnf clean all && \
      chmod +x /opt/bin/minio && \
+     chmod +x /opt/bin/mc && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
      /usr/bin/verify-minio.sh && \

README.md

@@ -200,7 +200,7 @@ service iptables restart
 
 MinIO Server comes with an embedded web based object browser. Point your web browser to <http://127.0.0.1:9000> to ensure your server has started successfully.
 
-> NOTE: MinIO runs console on random port by default if you wish choose a specific port use `--console-address` to pick a specific interface and port.
+> NOTE: MinIO runs console on random port by default, if you wish to choose a specific port use `--console-address` to pick a specific interface and port.
 
 ### Things to consider
 
@@ -241,7 +241,7 @@ mc admin update <minio alias, e.g., myminio>
 ### Upgrade Checklist
 
 - Test all upgrades in a lower environment (DEV, QA, UAT) before applying to production. Performing blind upgrades in production environments carries significant risk.
-- Read the release notes for MinIO *before* performing any upgrade, there is no forced requirement to upgrade to latest releases upon every releases. Some releases may not be relevant to your setup, avoid upgrading production environments unnecessarily.
+- Read the release notes for MinIO *before* performing any upgrade, there is no forced requirement to upgrade to latest release upon every release. Some release may not be relevant to your setup, avoid upgrading production environments unnecessarily.
 - If you plan to use `mc admin update`, MinIO process must have write access to the parent directory where the binary is present on the host system.
 - `mc admin update` is not supported and should be avoided in kubernetes/container environments, please upgrade containers by upgrading relevant container images.
 - **We do not recommend upgrading one MinIO server at a time, the product is designed to support parallel upgrades please follow our recommended guidelines.**
@@ -259,6 +259,6 @@ Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/ma
 
 ## License
 
-- MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
-- MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+- MinIO source is licensed under the [GNU AGPLv3](https://github.com/minio/minio/blob/master/LICENSE).
+- MinIO [documentation](https://github.com/minio/minio/tree/master/docs) is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
 - [License Compliance](https://github.com/minio/minio/blob/master/COMPLIANCE.md)

buildscripts/resolve-right-versions.sh

@@ -3,6 +3,7 @@
 set -E
 set -o pipefail
 set -x
+set -e
 
 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"

cmd/admin-bucket-handlers.go

@@ -662,12 +662,31 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
-	bucketMap := make(map[string]struct{}, 1)
 	rpt := importMetaReport{
 		madmin.BucketMetaImportErrs{
 			Buckets: make(map[string]madmin.BucketStatus, len(zr.File)),
 		},
 	}
+
+	bucketMap := make(map[string]*BucketMetadata, len(zr.File))
+
+	updatedAt := UTCNow()
+
+	for _, file := range zr.File {
+		slc := strings.Split(file.Name, slashSeparator)
+		if len(slc) != 2 { // expecting bucket/configfile in the zipfile
+			rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/<config.json>"))
+			continue
+		}
+		bucket := slc[0]
+		meta, err := readBucketMetadata(ctx, objectAPI, bucket)
+		if err == nil {
+			bucketMap[bucket] = &meta
+		} else if err != errConfigNotFound {
+			rpt.SetStatus(bucket, "", err)
+		}
+	}
+
 	// import object lock config if any - order of import matters here.
 	for _, file := range zr.File {
 		slc := strings.Split(file.Name, slashSeparator)
@@ -704,7 +723,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 						continue
 					}
 				}
-				bucketMap[bucket] = struct{}{}
+				v := newBucketMetadata(bucket)
+				bucketMap[bucket] = &v
 			}
 
 			// Deny object locking configuration settings on existing buckets without object lock enabled.
@@ -713,27 +733,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData)
-			if err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].ObjectLockConfigXML = configData
+			bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
-
-			// Call site replication hook.
-			//
-			// We encode the xml bytes as base64 to ensure there are no encoding
-			// errors.
-			cfgStr := base64.StdEncoding.EncodeToString(configData)
-			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:             madmin.SRBucketMetaTypeObjectLockConfig,
-				Bucket:           bucket,
-				ObjectLockConfig: &cfgStr,
-				UpdatedAt:        updatedAt,
-			}); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
 		}
 	}
 
@@ -763,7 +765,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 						continue
 					}
 				}
-				bucketMap[bucket] = struct{}{}
+				v := newBucketMetadata(bucket)
+				bucketMap[bucket] = &v
 			}
 
 			if globalSiteReplicationSys.isEnabled() && v.Suspended() {
@@ -786,10 +789,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].VersioningConfigXML = configData
+			bucketMap[bucket].VersioningConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
 		}
 	}
@@ -807,6 +808,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			continue
 		}
 		bucket, fileName := slc[0], slc[1]
+
 		// create bucket if it does not exist yet.
 		if _, ok := bucketMap[bucket]; !ok {
 			err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{})
@@ -816,7 +818,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 					continue
 				}
 			}
-			bucketMap[bucket] = struct{}{}
+			v := newBucketMetadata(bucket)
+			bucketMap[bucket] = &v
 		}
 		if _, ok := bucketMap[bucket]; !ok {
 			continue
@@ -835,12 +838,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketNotificationConfig, configData); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
-			rulesMap := config.ToRulesMap()
-			globalEventNotifier.AddRulesMap(bucket, rulesMap)
+			bucketMap[bucket].NotificationConfigXML = configData
 			rpt.SetStatus(bucket, fileName, nil)
 		case bucketPolicyConfig:
 			// Error out if Content-Length is beyond allowed size.
@@ -873,22 +871,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData)
-			if err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].PolicyConfigJSON = configData
+			bucketMap[bucket].PolicyConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
-			// Call site replication hook.
-			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:      madmin.SRBucketMetaTypePolicy,
-				Bucket:    bucket,
-				Policy:    bucketPolicyBytes,
-				UpdatedAt: updatedAt,
-			}); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
 		case bucketLifecycleConfig:
 			bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz))
 			if err != nil {
@@ -914,10 +899,8 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].LifecycleConfigXML = configData
+			bucketMap[bucket].LifecycleConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
 		case bucketSSEConfig:
 			// Parse bucket encryption xml
@@ -952,29 +935,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			// Store the bucket encryption configuration in the object layer
-			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData)
-			if err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].EncryptionConfigXML = configData
+			bucketMap[bucket].EncryptionConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
-
-			// Call site replication hook.
-			//
-			// We encode the xml bytes as base64 to ensure there are no encoding
-			// errors.
-			cfgStr := base64.StdEncoding.EncodeToString(configData)
-			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:      madmin.SRBucketMetaTypeSSEConfig,
-				Bucket:    bucket,
-				SSEConfig: &cfgStr,
-				UpdatedAt: updatedAt,
-			}); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
-
 		case bucketTaggingConfig:
 			tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz))
 			if err != nil {
@@ -988,27 +951,9 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData)
-			if err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].TaggingConfigXML = configData
+			bucketMap[bucket].TaggingConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
-
-			// Call site replication hook.
-			//
-			// We encode the xml bytes as base64 to ensure there are no encoding
-			// errors.
-			cfgStr := base64.StdEncoding.EncodeToString(configData)
-			if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
-				Type:      madmin.SRBucketMetaTypeTags,
-				Bucket:    bucket,
-				Tags:      &cfgStr,
-				UpdatedAt: updatedAt,
-			}); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
 		case bucketQuotaConfigFile:
 			data, err := io.ReadAll(reader)
 			if err != nil {
@@ -1016,37 +961,49 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 
-			quotaConfig, err := parseBucketQuota(bucket, data)
+			_, err = parseBucketQuota(bucket, data)
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, err)
 				continue
 			}
 
-			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
-			if err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
+			bucketMap[bucket].QuotaConfigJSON = data
+			bucketMap[bucket].QuotaConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
-
-			bucketMeta := madmin.SRBucketMeta{
-				Type:      madmin.SRBucketMetaTypeQuotaConfig,
-				Bucket:    bucket,
-				Quota:     data,
-				UpdatedAt: updatedAt,
-			}
-			if quotaConfig.Quota == 0 {
-				bucketMeta.Quota = nil
-			}
-
-			// Call site replication hook.
-			if err = globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
 		}
 	}
 
+	enc := func(b []byte) *string {
+		if b == nil {
+			return nil
+		}
+		v := base64.StdEncoding.EncodeToString(b)
+		return &v
+	}
+
+	for bucket, meta := range bucketMap {
+		err := globalBucketMetadataSys.save(ctx, *meta)
+		if err != nil {
+			rpt.SetStatus(bucket, "", err)
+			continue
+		}
+		// Call site replication hook.
+		if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{
+			Bucket:           bucket,
+			Quota:            meta.QuotaConfigJSON,
+			Policy:           meta.PolicyConfigJSON,
+			Versioning:       enc(meta.VersioningConfigXML),
+			Tags:             enc(meta.TaggingConfigXML),
+			ObjectLockConfig: enc(meta.ObjectLockConfigXML),
+			SSEConfig:        enc(meta.EncryptionConfigXML),
+			UpdatedAt:        updatedAt,
+		}); err != nil {
+			rpt.SetStatus(bucket, "", err)
+			continue
+		}
+
+	}
+
 	rptData, err := json.Marshal(rpt.BucketMetaImportErrs)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)

cmd/admin-handlers-pools.go

@@ -30,7 +30,7 @@ import (
 )
 
 var (
-	errRebalanceDecommissionAlreadyRunning = errors.New("Rebalance cannot be started, decommission is aleady in progress")
+	errRebalanceDecommissionAlreadyRunning = errors.New("Rebalance cannot be started, decommission is already in progress")
 	errDecommissionRebalanceAlreadyRunning = errors.New("Decommission cannot be started, rebalance is already in progress")
 )
 

cmd/admin-handlers-site-replication.go

@@ -207,10 +207,15 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 		return
 	}
 
+	if item.Bucket == "" {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errSRInvalidRequest(errInvalidArgument)), r.URL)
+		return
+	}
+
 	var err error
 	switch item.Type {
 	default:
-		err = errSRInvalidRequest(errInvalidArgument)
+		err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, item)
 	case madmin.SRBucketMetaTypePolicy:
 		if item.Policy == nil {
 			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt)
@@ -236,7 +241,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 				return
 			}
 			if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig, item.UpdatedAt); err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 				return
 			}
 		}

cmd/admin-handlers-users_test.go

@@ -27,7 +27,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"os"
 	"runtime"
 	"strings"
 	"testing"
@@ -40,6 +39,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/signer"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/pkg/env"
 )
 
 const (
@@ -121,7 +121,7 @@ var iamTestSuites = func() []*TestSuiteIAM {
 }()
 
 const (
-	EnvTestEtcdBackend = "ETCD_SERVER"
+	EnvTestEtcdBackend = "_MINIO_ETCD_TEST_SERVER"
 )
 
 func (s *TestSuiteIAM) setUpEtcd(c *check, etcdServer string) {
@@ -144,7 +144,7 @@ func (s *TestSuiteIAM) setUpEtcd(c *check, etcdServer string) {
 func (s *TestSuiteIAM) SetUpSuite(c *check) {
 	// If etcd backend is specified and etcd server is not present, the test
 	// is skipped.
-	etcdServer := os.Getenv(EnvTestEtcdBackend)
+	etcdServer := env.Get(EnvTestEtcdBackend, "")
 	if s.withEtcdBackend && etcdServer == "" {
 		c.Skip("Skipping etcd backend IAM test as no etcd server is configured.")
 	}
@@ -983,9 +983,9 @@ func (s *TestSuiteIAM) SetUpAccMgmtPlugin(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()
 
-	pluginEndpoint := os.Getenv("POLICY_PLUGIN_ENDPOINT")
+	pluginEndpoint := env.Get("_MINIO_POLICY_PLUGIN_ENDPOINT", "")
 	if pluginEndpoint == "" {
-		c.Skip("POLICY_PLUGIN_ENDPOINT not given - skipping.")
+		c.Skip("_MINIO_POLICY_PLUGIN_ENDPOINT not given - skipping.")
 	}
 
 	configCmds := []string{

cmd/admin-handlers.go

@@ -41,6 +41,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -1024,7 +1025,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 {
 			clientToken := nh.clientToken
 			if globalIsDistErasure {
-				clientToken = fmt.Sprintf("%s@%d", nh.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
+				clientToken = fmt.Sprintf("%s:%d", nh.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
 			}
 			b, err := json.Marshal(madmin.HealStartSuccess{
 				ClientToken:   clientToken,
@@ -1146,6 +1147,7 @@ func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
 		return
 	}
+	ctx = lkctx.Context()
 	defer nsLock.Unlock(lkctx)
 
 	durationStr := r.Form.Get(peerRESTDuration)
@@ -1172,6 +1174,66 @@ func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request
 	}
 }
 
+// ClientDevNullExtraTime - return extratime for last devnull
+// [POST] /minio/admin/v3/speedtest/client/devnull/extratime
+func (a adminAPIHandlers) ClientDevNullExtraTime(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.BandwidthMonitorAction)
+	if objectAPI == nil {
+		return
+	}
+
+	enc := json.NewEncoder(w)
+	if err := enc.Encode(madmin.ClientPerfExtraTime{TimeSpent: globalLastClientPerfExtraTime}); err != nil {
+		return
+	}
+}
+
+// ClientDevNull - everything goes to io.Discard
+// [POST] /minio/admin/v3/speedtest/client/devnull
+func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	timeStart := time.Now()
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.BandwidthMonitorAction)
+	if objectAPI == nil {
+		return
+	}
+
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "client-perf")
+	lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
+		return
+	}
+	ctx = lkctx.Context()
+	defer nsLock.Unlock(lkctx)
+	timeEnd := time.Now()
+
+	atomic.SwapInt64(&globalLastClientPerfExtraTime, timeEnd.Sub(timeStart).Nanoseconds())
+
+	ctx, cancel := context.WithTimeout(ctx, madmin.MaxClientPerfTimeout)
+	defer cancel()
+	totalRx := int64(0)
+	connectTime := time.Now()
+	for {
+		n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			// would mean the network is not stable. Logging here will help in debugging network issues.
+			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
+				logger.LogIf(ctx, err)
+			}
+		}
+		totalRx += n
+		if err != nil || ctx.Err() != nil || totalRx > 100*humanize.GiByte {
+			break
+		}
+
+	}
+	w.WriteHeader(http.StatusOK)
+}
+
 // NetperfHandler - perform mesh style network throughput test
 func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
@@ -1192,6 +1254,7 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
 		return
 	}
+	ctx = lkctx.Context()
 	defer nsLock.Unlock(lkctx)
 
 	durationStr := r.Form.Get(peerRESTDuration)

cmd/admin-heal-ops.go

@@ -253,7 +253,7 @@ func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) {
 	} else {
 		clientToken := he.clientToken
 		if globalIsDistErasure {
-			clientToken = fmt.Sprintf("%s@%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
+			clientToken = fmt.Sprintf("%s:%d", he.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
 		}
 
 		hsp = madmin.HealStopSuccess{
@@ -327,7 +327,7 @@ func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLay
 
 	clientToken := h.clientToken
 	if globalIsDistErasure {
-		clientToken = fmt.Sprintf("%s@%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
+		clientToken = fmt.Sprintf("%s:%d", h.clientToken, GetProxyEndpointLocalIndex(globalProxyEndpoints))
 	}
 
 	b, err := json.Marshal(madmin.HealStartSuccess{

cmd/admin-router.go

@@ -36,6 +36,8 @@ const (
 	adminAPIVersionPrefix          = SlashSeparator + adminAPIVersion
 	adminAPISiteReplicationDevNull = "/site-replication/devnull"
 	adminAPISiteReplicationNetPerf = "/site-replication/netperf"
+	adminAPIClientDevNull          = "/speedtest/client/devnull"
+	adminAPIClientDevExtraTime     = "/speedtest/client/devnull/extratime"
 )
 
 var gzipHandler = func() func(http.Handler) http.HandlerFunc {
@@ -388,6 +390,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(adminMiddleware(adminAPI.DriveSpeedtestHandler, noGZFlag))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(adminMiddleware(adminAPI.NetperfHandler, noGZFlag))
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/site").HandlerFunc(adminMiddleware(adminAPI.SitePerfHandler, noGZFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPIClientDevNull).HandlerFunc(adminMiddleware(adminAPI.ClientDevNull, noGZFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPIClientDevExtraTime).HandlerFunc(adminMiddleware(adminAPI.ClientDevNullExtraTime, noGZFlag))
 
 		// HTTP Trace
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(adminMiddleware(adminAPI.TraceHandler, noObjLayerFlag))

cmd/api-errors.go

@@ -190,6 +190,7 @@ const (
 	ErrMaximumExpires
 	ErrSlowDownRead
 	ErrSlowDownWrite
+	ErrMaxVersionsExceeded
 	ErrInvalidPrefixMarker
 	ErrBadRequest
 	ErrKeyTooLongError
@@ -857,6 +858,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Resource requested is unwritable, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrMaxVersionsExceeded: {
+		Code:           "MaxVersionsExceeded",
+		Description:    "You've exceeded the limit on the number of versions you can create on this object",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidPrefixMarker: {
 		Code:           "InvalidPrefixMarker",
 		Description:    "Invalid marker prefix combination",
@@ -1397,7 +1403,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrAdminConfigIDPCfgNameAlreadyExists: {
 		Code:           "XMinioAdminConfigIDPCfgNameAlreadyExists",
-		Description:    "An IDP configuration with the given name aleady exists",
+		Description:    "An IDP configuration with the given name already exists",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrAdminConfigIDPCfgNameDoesNotExist: {
@@ -2110,6 +2116,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrSlowDownRead
 	case errErasureWriteQuorum:
 		apiErr = ErrSlowDownWrite
+	case errMaxVersionsExceeded:
+		apiErr = ErrMaxVersionsExceeded
 	// SSE errors
 	case errInvalidEncryptionParameters:
 		apiErr = ErrInvalidEncryptionParameters

cmd/api-headers.go

@@ -23,11 +23,11 @@ import (
 	"encoding/xml"
 	"fmt"
 	"net/http"
-	"net/url"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -140,9 +140,13 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 
 	// Set tag count if object has tags
 	if len(objInfo.UserTags) > 0 {
-		tags, _ := url.ParseQuery(objInfo.UserTags)
-		if len(tags) > 0 {
-			w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(len(tags))}
+		tags, _ := tags.ParseObjectTags(objInfo.UserTags)
+		if tags.Count() > 0 {
+			w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tags.Count())}
+			if opts.Tagging {
+				// This is MinIO only extension to return back tags along with the count.
+				w.Header()[xhttp.AmzObjectTagging] = []string{objInfo.UserTags}
+			}
 		}
 	}
 

cmd/api-router.go

@@ -545,12 +545,5 @@ func corsHandler(handler http.Handler) http.Handler {
 		ExposedHeaders:   commonS3Headers,
 		AllowCredentials: true,
 	}
-	for _, origin := range globalAPIConfig.getCorsAllowOrigins() {
-		if origin == "*" {
-			opts.AllowOriginFunc = nil
-			opts.AllowedOrigins = globalAPIConfig.getCorsAllowOrigins()
-			break
-		}
-	}
 	return cors.New(opts).Handler(handler)
 }

cmd/background-heal-ops.go

@@ -75,9 +75,9 @@ func waitForLowIO(maxIO int, maxWait time.Duration, currentIO func() int) {
 		if tmpMaxWait > 0 {
 			if tmpMaxWait < waitTick {
 				time.Sleep(tmpMaxWait)
-			} else {
-				time.Sleep(waitTick)
+				return
 			}
+			time.Sleep(waitTick)
 			tmpMaxWait -= waitTick
 		}
 		if tmpMaxWait <= 0 {

cmd/batch-handlers.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2022 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -42,230 +42,18 @@ import (
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
-	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
 	"github.com/minio/pkg/env"
 	iampolicy "github.com/minio/pkg/iam/policy"
-	"github.com/minio/pkg/wildcard"
 	"github.com/minio/pkg/workers"
 	"gopkg.in/yaml.v2"
 )
 
-// replicate:
-//   # source of the objects to be replicated
-//   source:
-//     type: "minio"
-//     bucket: "testbucket"
-//     prefix: "spark/"
-//
-//   # optional flags based filtering criteria
-//   # for source objects
-//   flags:
-//     filter:
-//       newerThan: "7d"
-//       olderThan: "7d"
-//       createdAfter: "date"
-//       createdBefore: "date"
-//       tags:
-//         - key: "name"
-//           value: "value*"
-//       metadata:
-//         - key: "content-type"
-//           value: "image/*"
-//     notify:
-//       endpoint: "https://splunk-hec.dev.com"
-//       token: "Splunk ..." # e.g. "Bearer token"
-//
-//   # target where the objects must be replicated
-//   target:
-//     type: "minio"
-//     bucket: "testbucket1"
-//     endpoint: "https://play.min.io"
-//     path: "on"
-//     credentials:
-//       accessKey: "minioadmin"
-//       secretKey: "minioadmin"
-//       sessionToken: ""
-
-// BatchJobReplicateKV is a datatype that holds key and values for filtering of objects
-// used by metadata filter as well as tags based filtering.
-type BatchJobReplicateKV struct {
-	Key   string `yaml:"key" json:"key"`
-	Value string `yaml:"value" json:"value"`
-}
-
-// Validate returns an error if key is empty
-func (kv BatchJobReplicateKV) Validate() error {
-	if kv.Key == "" {
-		return errInvalidArgument
-	}
-	return nil
-}
-
-// Empty indicates if kv is not set
-func (kv BatchJobReplicateKV) Empty() bool {
-	return kv.Key == "" && kv.Value == ""
-}
-
-// Match matches input kv with kv, value will be wildcard matched depending on the user input
-func (kv BatchJobReplicateKV) Match(ikv BatchJobReplicateKV) bool {
-	if kv.Empty() {
-		return true
-	}
-	if strings.EqualFold(kv.Key, ikv.Key) {
-		return wildcard.Match(kv.Value, ikv.Value)
-	}
-	return false
-}
-
-// BatchReplicateRetry datatype represents total retry attempts and delay between each retries.
-type BatchReplicateRetry struct {
-	Attempts int           `yaml:"attempts" json:"attempts"` // number of retry attempts
-	Delay    time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
-}
-
-// Validate validates input replicate retries.
-func (r BatchReplicateRetry) Validate() error {
-	if r.Attempts < 0 {
-		return errInvalidArgument
-	}
-
-	if r.Delay < 0 {
-		return errInvalidArgument
-	}
-
-	return nil
-}
-
-// BatchReplicateFilter holds all the filters currently supported for batch replication
-type BatchReplicateFilter struct {
-	NewerThan     time.Duration         `yaml:"newerThan,omitempty" json:"newerThan"`
-	OlderThan     time.Duration         `yaml:"olderThan,omitempty" json:"olderThan"`
-	CreatedAfter  time.Time             `yaml:"createdAfter,omitempty" json:"createdAfter"`
-	CreatedBefore time.Time             `yaml:"createdBefore,omitempty" json:"createdBefore"`
-	Tags          []BatchJobReplicateKV `yaml:"tags,omitempty" json:"tags"`
-	Metadata      []BatchJobReplicateKV `yaml:"metadata,omitempty" json:"metadata"`
-}
-
-// BatchReplicateNotification success or failure notification endpoint for each job attempts
-type BatchReplicateNotification struct {
-	Endpoint string `yaml:"endpoint" json:"endpoint"`
-	Token    string `yaml:"token" json:"token"`
-}
-
-// BatchJobReplicateFlags various configurations for replication job definition currently includes
-// - filter
-// - notify
-// - retry
-type BatchJobReplicateFlags struct {
-	Filter BatchReplicateFilter       `yaml:"filter" json:"filter"`
-	Notify BatchReplicateNotification `yaml:"notify" json:"notify"`
-	Retry  BatchReplicateRetry        `yaml:"retry" json:"retry"`
-}
-
-// BatchJobReplicateResourceType defines the type of batch jobs
-type BatchJobReplicateResourceType string
-
-// Validate validates if the replicate resource type is recognized and supported
-func (t BatchJobReplicateResourceType) Validate() error {
-	switch t {
-	case BatchJobReplicateResourceMinIO:
-	case BatchJobReplicateResourceS3:
-	default:
-		return errInvalidArgument
-	}
-	return nil
-}
-
-func (t BatchJobReplicateResourceType) isMinio() bool {
-	return t == BatchJobReplicateResourceMinIO
-}
-
-// Different types of batch jobs..
-const (
-	BatchJobReplicateResourceMinIO BatchJobReplicateResourceType = "minio"
-	BatchJobReplicateResourceS3    BatchJobReplicateResourceType = "s3"
-
-	// add future targets
-)
-
-// BatchJobReplicateCredentials access credentials for batch replication it may
-// be either for target or source.
-type BatchJobReplicateCredentials struct {
-	AccessKey    string `xml:"AccessKeyId" json:"accessKey,omitempty" yaml:"accessKey"`
-	SecretKey    string `xml:"SecretAccessKey" json:"secretKey,omitempty" yaml:"secretKey"`
-	SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"`
-}
-
-// Empty indicates if credentials are not set
-func (c BatchJobReplicateCredentials) Empty() bool {
-	return c.AccessKey == "" && c.SecretKey == "" && c.SessionToken == ""
-}
-
-// Validate validates if credentials are valid
-func (c BatchJobReplicateCredentials) Validate() error {
-	if !auth.IsAccessKeyValid(c.AccessKey) || !auth.IsSecretKeyValid(c.SecretKey) {
-		return errInvalidArgument
-	}
-	return nil
-}
-
-// BatchJobReplicateTarget describes target element of the replication job that receives
-// the filtered data from source
-type BatchJobReplicateTarget struct {
-	Type     BatchJobReplicateResourceType `yaml:"type" json:"type"`
-	Bucket   string                        `yaml:"bucket" json:"bucket"`
-	Prefix   string                        `yaml:"prefix" json:"prefix"`
-	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
-	Path     string                        `yaml:"path" json:"path"`
-	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
-}
-
-// ValidPath returns true if path is valid
-func (t BatchJobReplicateTarget) ValidPath() bool {
-	return t.Path == "on" || t.Path == "off" || t.Path == "auto" || t.Path == ""
-}
-
-// BatchJobReplicateSource describes source element of the replication job that is
-// the source of the data for the target
-type BatchJobReplicateSource struct {
-	Type     BatchJobReplicateResourceType `yaml:"type" json:"type"`
-	Bucket   string                        `yaml:"bucket" json:"bucket"`
-	Prefix   string                        `yaml:"prefix" json:"prefix"`
-	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
-	Path     string                        `yaml:"path" json:"path"`
-	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
-}
-
-// ValidPath returns true if path is valid
-func (s BatchJobReplicateSource) ValidPath() bool {
-	switch s.Path {
-	case "on", "off", "auto", "":
-		return true
-	default:
-		return false
-	}
-}
-
-// BatchJobReplicateV1 v1 of batch job replication
-type BatchJobReplicateV1 struct {
-	APIVersion string                  `yaml:"apiVersion" json:"apiVersion"`
-	Flags      BatchJobReplicateFlags  `yaml:"flags" json:"flags"`
-	Target     BatchJobReplicateTarget `yaml:"target" json:"target"`
-	Source     BatchJobReplicateSource `yaml:"source" json:"source"`
-
-	clnt *miniogo.Core `msg:"-"`
-}
-
-// RemoteToLocal returns true if source is remote and target is local
-func (r BatchJobReplicateV1) RemoteToLocal() bool {
-	return !r.Source.Creds.Empty()
-}
-
 // BatchJobRequest this is an internal data structure not for external consumption.
 type BatchJobRequest struct {
 	ID        string               `yaml:"-" json:"name"`
@@ -277,23 +65,28 @@ type BatchJobRequest struct {
 	ctx       context.Context      `msg:"-"`
 }
 
-// Notify notifies notification endpoint if configured regarding job failure or success.
-func (r BatchJobReplicateV1) Notify(ctx context.Context, body io.Reader) error {
-	if r.Flags.Notify.Endpoint == "" {
+func notifyEndpoint(ctx context.Context, ri *batchJobInfo, endpoint, token string) error {
+	if endpoint == "" {
 		return nil
 	}
 
+	buf, err := json.Marshal(ri)
+	if err != nil {
+		return err
+	}
+
 	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.Flags.Notify.Endpoint, body)
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(buf))
 	if err != nil {
 		return err
 	}
 
-	if r.Flags.Notify.Token != "" {
-		req.Header.Set("Authorization", r.Flags.Notify.Token)
+	if token != "" {
+		req.Header.Set("Authorization", token)
 	}
+	req.Header.Set("Content-Type", "application/json")
 
 	clnt := http.Client{Transport: getRemoteInstanceTransport}
 	resp, err := clnt.Do(req)
@@ -309,6 +102,11 @@ func (r BatchJobReplicateV1) Notify(ctx context.Context, body io.Reader) error {
 	return nil
 }
 
+// Notify notifies notification endpoint if configured regarding job failure or success.
+func (r BatchJobReplicateV1) Notify(ctx context.Context, ri *batchJobInfo) error {
+	return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token)
+}
+
 // ReplicateFromSource - this is not implemented yet where source is 'remote' and target is local.
 func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *miniogo.Core, srcObjInfo ObjectInfo, retry bool) error {
 	srcBucket := r.Source.Bucket
@@ -319,17 +117,17 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
 	}
 
-	versioned := globalBucketVersioningSys.PrefixEnabled(tgtBucket, tgtObject)
-	versionSuspended := globalBucketVersioningSys.PrefixSuspended(tgtBucket, tgtObject)
 	versionID := srcObjInfo.VersionID
 	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
 		versionID = ""
 	}
 	if srcObjInfo.DeleteMarker {
 		_, err := api.DeleteObject(ctx, tgtBucket, tgtObject, ObjectOptions{
-			VersionID:          versionID,
-			VersionSuspended:   versionSuspended,
-			Versioned:          versioned,
+			VersionID: versionID,
+			// Since we are preserving a delete marker, we have to make sure this is always true.
+			// regardless of the current configuration of the bucket we must preserve all versions
+			// on the pool being batch replicated from source.
+			Versioned:          true,
 			MTime:              srcObjInfo.ModTime,
 			DeleteMarker:       srcObjInfo.DeleteMarker,
 			ReplicationRequest: true,
@@ -338,12 +136,10 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 	}
 
 	opts := ObjectOptions{
-		VersionID:        srcObjInfo.VersionID,
-		Versioned:        versioned,
-		VersionSuspended: versionSuspended,
-		MTime:            srcObjInfo.ModTime,
-		PreserveETag:     srcObjInfo.ETag,
-		UserDefined:      srcObjInfo.UserDefined,
+		VersionID:    srcObjInfo.VersionID,
+		MTime:        srcObjInfo.ModTime,
+		PreserveETag: srcObjInfo.ETag,
+		UserDefined:  srcObjInfo.UserDefined,
 	}
 	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
 		opts.VersionID = ""
@@ -507,7 +303,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			}
 			for _, kv := range r.Flags.Filter.Tags {
 				for t, v := range tagMap {
-					if kv.Match(BatchJobReplicateKV{Key: t, Value: v}) {
+					if kv.Match(BatchJobKV{Key: t, Value: v}) {
 						return true
 					}
 				}
@@ -524,7 +320,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 						continue
 					}
 					// We only need to match x-amz-meta or standardHeaders
-					if kv.Match(BatchJobReplicateKV{Key: k, Value: v}) {
+					if kv.Match(BatchJobKV{Key: k, Value: v}) {
 						return true
 					}
 				}
@@ -645,8 +441,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		// persist in-memory state to disk.
 		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
-		buf, _ := json.Marshal(ri)
-		if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
+		if err := r.Notify(ctx, ri); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
 		}
 
@@ -1062,7 +857,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 			for _, kv := range r.Flags.Filter.Tags {
 				for t, v := range tagMap {
-					if kv.Match(BatchJobReplicateKV{Key: t, Value: v}) {
+					if kv.Match(BatchJobKV{Key: t, Value: v}) {
 						return true
 					}
 				}
@@ -1079,7 +874,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						continue
 					}
 					// We only need to match x-amz-meta or standardHeaders
-					if kv.Match(BatchJobReplicateKV{Key: k, Value: v}) {
+					if kv.Match(BatchJobKV{Key: k, Value: v}) {
 						return true
 					}
 				}
@@ -1193,8 +988,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		// persist in-memory state to disk.
 		logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
-		buf, _ := json.Marshal(ri)
-		if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
+		if err := r.Notify(ctx, ri); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
 		}
 
@@ -1406,7 +1200,6 @@ func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) {
 	case j.KeyRotate != nil:
 		deleteConfig(ctx, api, pathJoin(j.Location, batchKeyRotationName))
 	}
-	globalBatchJobsMetrics.delete(j.ID)
 	deleteConfig(ctx, api, j.Location)
 }
 
@@ -1520,14 +1313,14 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	id := r.Form.Get("jobId")
-	if id == "" {
+	jobID := r.Form.Get("jobId")
+	if jobID == "" {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL)
 		return
 	}
 
 	req := &BatchJobRequest{}
-	if err := req.load(ctx, objectAPI, pathJoin(batchJobPrefix, id)); err != nil {
+	if err := req.load(ctx, objectAPI, pathJoin(batchJobPrefix, jobID)); err != nil {
 		if !errors.Is(err, errNoSuchJob) {
 			logger.LogIf(ctx, err)
 		}
@@ -1555,7 +1348,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	buf, err := io.ReadAll(r.Body)
+	buf, err := io.ReadAll(ioutil.HardLimitReader(r.Body, humanize.MiByte*4))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1567,12 +1360,12 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 	}
 
 	job := &BatchJobRequest{}
-	if err = yaml.Unmarshal(buf, job); err != nil {
+	if err = yaml.UnmarshalStrict(buf, job); err != nil {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 
-	job.ID = shortuuid.New()
+	job.ID = fmt.Sprintf("%s:%d", shortuuid.New(), GetProxyEndpointLocalIndex(globalProxyEndpoints))
 	job.User = user
 	job.Started = time.Now()
 
@@ -1608,19 +1401,27 @@ func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request)
 	if objectAPI == nil {
 		return
 	}
+
 	jobID := r.Form.Get("id")
 	if jobID == "" {
 		writeErrorResponseJSON(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL)
 		return
 	}
+
+	if _, success := proxyRequestByToken(ctx, w, r, jobID); success {
+		return
+	}
+
 	if err := globalBatchJobPool.canceler(jobID, true); err != nil {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL)
 		return
 	}
+
 	j := BatchJobRequest{
 		ID:       jobID,
 		Location: pathJoin(batchJobPrefix, jobID),
 	}
+
 	j.delete(ctx, objectAPI)
 
 	writeSuccessNoContent(w)
@@ -1675,6 +1476,11 @@ func (j *BatchJobPool) resume() {
 			logger.LogIf(ctx, err)
 			continue
 		}
+		_, nodeIdx := parseRequestToken(req.ID)
+		if nodeIdx > -1 && GetProxyEndpointLocalIndex(globalProxyEndpoints) != nodeIdx {
+			// This job doesn't belong on this node.
+			continue
+		}
 		if err := j.queueJob(req); err != nil {
 			logger.LogIf(ctx, err)
 			continue
@@ -1795,10 +1601,6 @@ type batchJobMetrics struct {
 	metrics map[string]*batchJobInfo
 }
 
-var globalBatchJobsMetrics = batchJobMetrics{
-	metrics: make(map[string]*batchJobInfo),
-}
-
 //msgp:ignore batchJobMetric
 //go:generate stringer -type=batchJobMetric -trimprefix=batchJobMetric $GOFILE
 type batchJobMetric uint8
@@ -1838,9 +1640,17 @@ func (m *batchJobMetrics) report(jobID string) (metrics *madmin.BatchJobMetrics)
 	metrics = &madmin.BatchJobMetrics{CollectedAt: time.Now(), Jobs: make(map[string]madmin.JobMetric)}
 	m.RLock()
 	defer m.RUnlock()
+
+	match := true
 	for id, job := range m.metrics {
-		match := jobID != "" && id == jobID
-		metrics.Jobs[id] = madmin.JobMetric{
+		if jobID != "" {
+			match = id == jobID
+		}
+		if !match {
+			continue
+		}
+
+		m := madmin.JobMetric{
 			JobID:         job.JobID,
 			JobType:       job.JobType,
 			StartTime:     job.StartTime,
@@ -1848,28 +1658,58 @@ func (m *batchJobMetrics) report(jobID string) (metrics *madmin.BatchJobMetrics)
 			RetryAttempts: job.RetryAttempts,
 			Complete:      job.Complete,
 			Failed:        job.Failed,
-			Replicate: &madmin.ReplicateInfo{
+		}
+
+		switch job.JobType {
+		case string(madmin.BatchJobReplicate):
+			m.Replicate = &madmin.ReplicateInfo{
 				Bucket:           job.Bucket,
 				Object:           job.Object,
 				Objects:          job.Objects,
 				ObjectsFailed:    job.ObjectsFailed,
 				BytesTransferred: job.BytesTransferred,
 				BytesFailed:      job.BytesFailed,
-			},
-			KeyRotate: &madmin.KeyRotationInfo{
+			}
+		case string(madmin.BatchJobKeyRotate):
+			m.KeyRotate = &madmin.KeyRotationInfo{
 				Bucket:        job.Bucket,
 				Object:        job.Object,
 				Objects:       job.Objects,
 				ObjectsFailed: job.ObjectsFailed,
-			},
-		}
-		if match {
-			break
+			}
 		}
+
+		metrics.Jobs[id] = m
 	}
 	return metrics
 }
 
+// keep job metrics for some time after the job is completed
+// in-case some one wants to look at the older results.
+func (m *batchJobMetrics) purgeJobMetrics() {
+	t := time.NewTicker(6 * time.Hour)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-GlobalContext.Done():
+			return
+		case <-t.C:
+			var toDeleteJobMetrics []string
+			m.RLock()
+			for id, metrics := range m.metrics {
+				if time.Since(metrics.LastUpdate) > 24*time.Hour && (metrics.Complete || metrics.Failed) {
+					toDeleteJobMetrics = append(toDeleteJobMetrics, id)
+				}
+			}
+			m.RUnlock()
+			for _, jobID := range toDeleteJobMetrics {
+				m.delete(jobID)
+			}
+		}
+	}
+}
+
 func (m *batchJobMetrics) delete(jobID string) {
 	m.Lock()
 	defer m.Unlock()

cmd/batch-job-common-types.go

@@ -0,0 +1,83 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"strings"
+	"time"
+
+	"github.com/minio/pkg/wildcard"
+)
+
+//go:generate msgp -file $GOFILE
+
+// BatchJobKV is a key-value data type which supports wildcard matching
+type BatchJobKV struct {
+	Key   string `yaml:"key" json:"key"`
+	Value string `yaml:"value" json:"value"`
+}
+
+// Validate returns an error if key is empty
+func (kv BatchJobKV) Validate() error {
+	if kv.Key == "" {
+		return errInvalidArgument
+	}
+	return nil
+}
+
+// Empty indicates if kv is not set
+func (kv BatchJobKV) Empty() bool {
+	return kv.Key == "" && kv.Value == ""
+}
+
+// Match matches input kv with kv, value will be wildcard matched depending on the user input
+func (kv BatchJobKV) Match(ikv BatchJobKV) bool {
+	if kv.Empty() {
+		return true
+	}
+	if strings.EqualFold(kv.Key, ikv.Key) {
+		return wildcard.Match(kv.Value, ikv.Value)
+	}
+	return false
+}
+
+// BatchJobNotification stores notification endpoint and token information.
+// Used by batch jobs to notify of their status.
+type BatchJobNotification struct {
+	Endpoint string `yaml:"endpoint" json:"endpoint"`
+	Token    string `yaml:"token" json:"token"`
+}
+
+// BatchJobRetry stores retry configuration used in the event of failures.
+type BatchJobRetry struct {
+	Attempts int           `yaml:"attempts" json:"attempts"` // number of retry attempts
+	Delay    time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
+}
+
+// Validate validates input replicate retries.
+func (r BatchJobRetry) Validate() error {
+	if r.Attempts < 0 {
+		return errInvalidArgument
+	}
+
+	if r.Delay < 0 {
+		return errInvalidArgument
+	}
+
+	return nil
+}

cmd/batch-job-common-types_gen.go

@@ -0,0 +1,391 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobKV) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Key":
+			z.Key, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Key")
+				return
+			}
+		case "Value":
+			z.Value, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Value")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobKV) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Key"
+	err = en.Append(0x82, 0xa3, 0x4b, 0x65, 0x79)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Key)
+	if err != nil {
+		err = msgp.WrapError(err, "Key")
+		return
+	}
+	// write "Value"
+	err = en.Append(0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Value)
+	if err != nil {
+		err = msgp.WrapError(err, "Value")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobKV) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Key"
+	o = append(o, 0x82, 0xa3, 0x4b, 0x65, 0x79)
+	o = msgp.AppendString(o, z.Key)
+	// string "Value"
+	o = append(o, 0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
+	o = msgp.AppendString(o, z.Value)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobKV) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Key":
+			z.Key, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Key")
+				return
+			}
+		case "Value":
+			z.Value, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Value")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobKV) Msgsize() (s int) {
+	s = 1 + 4 + msgp.StringPrefixSize + len(z.Key) + 6 + msgp.StringPrefixSize + len(z.Value)
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobNotification) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Endpoint":
+			z.Endpoint, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "Token":
+			z.Token, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Token")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobNotification) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Endpoint"
+	err = en.Append(0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Endpoint)
+	if err != nil {
+		err = msgp.WrapError(err, "Endpoint")
+		return
+	}
+	// write "Token"
+	err = en.Append(0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Token)
+	if err != nil {
+		err = msgp.WrapError(err, "Token")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobNotification) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Endpoint"
+	o = append(o, 0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	o = msgp.AppendString(o, z.Endpoint)
+	// string "Token"
+	o = append(o, 0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e)
+	o = msgp.AppendString(o, z.Token)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobNotification) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Endpoint":
+			z.Endpoint, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Endpoint")
+				return
+			}
+		case "Token":
+			z.Token, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Token")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobNotification) Msgsize() (s int) {
+	s = 1 + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 6 + msgp.StringPrefixSize + len(z.Token)
+	return
+}
+
+// DecodeMsg implements msgp.Decodable
+func (z *BatchJobRetry) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Attempts":
+			z.Attempts, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "Attempts")
+				return
+			}
+		case "Delay":
+			z.Delay, err = dc.ReadDuration()
+			if err != nil {
+				err = msgp.WrapError(err, "Delay")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z BatchJobRetry) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 2
+	// write "Attempts"
+	err = en.Append(0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.Attempts)
+	if err != nil {
+		err = msgp.WrapError(err, "Attempts")
+		return
+	}
+	// write "Delay"
+	err = en.Append(0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
+	if err != nil {
+		return
+	}
+	err = en.WriteDuration(z.Delay)
+	if err != nil {
+		err = msgp.WrapError(err, "Delay")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z BatchJobRetry) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 2
+	// string "Attempts"
+	o = append(o, 0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
+	o = msgp.AppendInt(o, z.Attempts)
+	// string "Delay"
+	o = append(o, 0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
+	o = msgp.AppendDuration(o, z.Delay)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *BatchJobRetry) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "Attempts":
+			z.Attempts, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Attempts")
+				return
+			}
+		case "Delay":
+			z.Delay, bts, err = msgp.ReadDurationBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Delay")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z BatchJobRetry) Msgsize() (s int) {
+	s = 1 + 9 + msgp.IntSize + 6 + msgp.DurationSize
+	return
+}

cmd/batch-job-common-types_gen_test.go

@@ -0,0 +1,349 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalBatchJobKV(t *testing.T) {
+	v := BatchJobKV{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobKV(b *testing.B) {
+	v := BatchJobKV{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobKV(b *testing.B) {
+	v := BatchJobKV{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobKV(b *testing.B) {
+	v := BatchJobKV{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobKV(t *testing.T) {
+	v := BatchJobKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobKV Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobKV{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobKV(b *testing.B) {
+	v := BatchJobKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobKV(b *testing.B) {
+	v := BatchJobKV{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobNotification(t *testing.T) {
+	v := BatchJobNotification{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobNotification(b *testing.B) {
+	v := BatchJobNotification{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobNotification(b *testing.B) {
+	v := BatchJobNotification{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobNotification(b *testing.B) {
+	v := BatchJobNotification{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobNotification(t *testing.T) {
+	v := BatchJobNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobNotification Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobNotification{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobNotification(b *testing.B) {
+	v := BatchJobNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobNotification(b *testing.B) {
+	v := BatchJobNotification{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobRetry(t *testing.T) {
+	v := BatchJobRetry{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobRetry(b *testing.B) {
+	v := BatchJobRetry{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobRetry(b *testing.B) {
+	v := BatchJobRetry{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobRetry(b *testing.B) {
+	v := BatchJobRetry{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobRetry(t *testing.T) {
+	v := BatchJobRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobRetry Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobRetry{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobRetry(b *testing.B) {
+	v := BatchJobRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobRetry(b *testing.B) {
+	v := BatchJobRetry{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batch-replicate.go

@@ -0,0 +1,183 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"time"
+
+	miniogo "github.com/minio/minio-go/v7"
+
+	"github.com/minio/minio/internal/auth"
+)
+
+//go:generate msgp -file $GOFILE
+
+// replicate:
+//   # source of the objects to be replicated
+//   source:
+//     type: "minio"
+//     bucket: "testbucket"
+//     prefix: "spark/"
+//
+//   # optional flags based filtering criteria
+//   # for source objects
+//   flags:
+//     filter:
+//       newerThan: "7d"
+//       olderThan: "7d"
+//       createdAfter: "date"
+//       createdBefore: "date"
+//       tags:
+//         - key: "name"
+//           value: "value*"
+//       metadata:
+//         - key: "content-type"
+//           value: "image/*"
+//     notify:
+//       endpoint: "https://splunk-hec.dev.com"
+//       token: "Splunk ..." # e.g. "Bearer token"
+//
+//   # target where the objects must be replicated
+//   target:
+//     type: "minio"
+//     bucket: "testbucket1"
+//     endpoint: "https://play.min.io"
+//     path: "on"
+//     credentials:
+//       accessKey: "minioadmin"
+//       secretKey: "minioadmin"
+//       sessionToken: ""
+
+// BatchReplicateFilter holds all the filters currently supported for batch replication
+type BatchReplicateFilter struct {
+	NewerThan     time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
+	OlderThan     time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedAfter  time.Time     `yaml:"createdAfter,omitempty" json:"createdAfter"`
+	CreatedBefore time.Time     `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchJobKV  `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchJobKV  `yaml:"metadata,omitempty" json:"metadata"`
+}
+
+// BatchJobReplicateFlags various configurations for replication job definition currently includes
+// - filter
+// - notify
+// - retry
+type BatchJobReplicateFlags struct {
+	Filter BatchReplicateFilter `yaml:"filter" json:"filter"`
+	Notify BatchJobNotification `yaml:"notify" json:"notify"`
+	Retry  BatchJobRetry        `yaml:"retry" json:"retry"`
+}
+
+// BatchJobReplicateResourceType defines the type of batch jobs
+type BatchJobReplicateResourceType string
+
+// Validate validates if the replicate resource type is recognized and supported
+func (t BatchJobReplicateResourceType) Validate() error {
+	switch t {
+	case BatchJobReplicateResourceMinIO:
+	case BatchJobReplicateResourceS3:
+	default:
+		return errInvalidArgument
+	}
+	return nil
+}
+
+func (t BatchJobReplicateResourceType) isMinio() bool {
+	return t == BatchJobReplicateResourceMinIO
+}
+
+// Different types of batch jobs..
+const (
+	BatchJobReplicateResourceMinIO BatchJobReplicateResourceType = "minio"
+	BatchJobReplicateResourceS3    BatchJobReplicateResourceType = "s3"
+
+	// add future targets
+)
+
+// BatchJobReplicateCredentials access credentials for batch replication it may
+// be either for target or source.
+type BatchJobReplicateCredentials struct {
+	AccessKey    string `xml:"AccessKeyId" json:"accessKey,omitempty" yaml:"accessKey"`
+	SecretKey    string `xml:"SecretAccessKey" json:"secretKey,omitempty" yaml:"secretKey"`
+	SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"`
+}
+
+// Empty indicates if credentials are not set
+func (c BatchJobReplicateCredentials) Empty() bool {
+	return c.AccessKey == "" && c.SecretKey == "" && c.SessionToken == ""
+}
+
+// Validate validates if credentials are valid
+func (c BatchJobReplicateCredentials) Validate() error {
+	if !auth.IsAccessKeyValid(c.AccessKey) || !auth.IsSecretKeyValid(c.SecretKey) {
+		return errInvalidArgument
+	}
+	return nil
+}
+
+// BatchJobReplicateTarget describes target element of the replication job that receives
+// the filtered data from source
+type BatchJobReplicateTarget struct {
+	Type     BatchJobReplicateResourceType `yaml:"type" json:"type"`
+	Bucket   string                        `yaml:"bucket" json:"bucket"`
+	Prefix   string                        `yaml:"prefix" json:"prefix"`
+	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
+	Path     string                        `yaml:"path" json:"path"`
+	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
+}
+
+// ValidPath returns true if path is valid
+func (t BatchJobReplicateTarget) ValidPath() bool {
+	return t.Path == "on" || t.Path == "off" || t.Path == "auto" || t.Path == ""
+}
+
+// BatchJobReplicateSource describes source element of the replication job that is
+// the source of the data for the target
+type BatchJobReplicateSource struct {
+	Type     BatchJobReplicateResourceType `yaml:"type" json:"type"`
+	Bucket   string                        `yaml:"bucket" json:"bucket"`
+	Prefix   string                        `yaml:"prefix" json:"prefix"`
+	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
+	Path     string                        `yaml:"path" json:"path"`
+	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
+}
+
+// ValidPath returns true if path is valid
+func (s BatchJobReplicateSource) ValidPath() bool {
+	switch s.Path {
+	case "on", "off", "auto", "":
+		return true
+	default:
+		return false
+	}
+}
+
+// BatchJobReplicateV1 v1 of batch job replication
+type BatchJobReplicateV1 struct {
+	APIVersion string                  `yaml:"apiVersion" json:"apiVersion"`
+	Flags      BatchJobReplicateFlags  `yaml:"flags" json:"flags"`
+	Target     BatchJobReplicateTarget `yaml:"target" json:"target"`
+	Source     BatchJobReplicateSource `yaml:"source" json:"source"`
+
+	clnt *miniogo.Core `msg:"-"`
+}
+
+// RemoteToLocal returns true if source is remote and target is local
+func (r BatchJobReplicateV1) RemoteToLocal() bool {
+	return !r.Source.Creds.Empty()
+}

cmd/batch-replicate_gen_test.go

@@ -0,0 +1,688 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalBatchJobReplicateCredentials(t *testing.T) {
+	v := BatchJobReplicateCredentials{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobReplicateCredentials(b *testing.B) {
+	v := BatchJobReplicateCredentials{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobReplicateCredentials(b *testing.B) {
+	v := BatchJobReplicateCredentials{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobReplicateCredentials(b *testing.B) {
+	v := BatchJobReplicateCredentials{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobReplicateCredentials(t *testing.T) {
+	v := BatchJobReplicateCredentials{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobReplicateCredentials Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobReplicateCredentials{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobReplicateCredentials(b *testing.B) {
+	v := BatchJobReplicateCredentials{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobReplicateCredentials(b *testing.B) {
+	v := BatchJobReplicateCredentials{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobReplicateFlags(t *testing.T) {
+	v := BatchJobReplicateFlags{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobReplicateFlags(b *testing.B) {
+	v := BatchJobReplicateFlags{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobReplicateFlags(b *testing.B) {
+	v := BatchJobReplicateFlags{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobReplicateFlags(b *testing.B) {
+	v := BatchJobReplicateFlags{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobReplicateFlags(t *testing.T) {
+	v := BatchJobReplicateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobReplicateFlags Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobReplicateFlags{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobReplicateFlags(b *testing.B) {
+	v := BatchJobReplicateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobReplicateFlags(b *testing.B) {
+	v := BatchJobReplicateFlags{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobReplicateSource(t *testing.T) {
+	v := BatchJobReplicateSource{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobReplicateSource(b *testing.B) {
+	v := BatchJobReplicateSource{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobReplicateSource(b *testing.B) {
+	v := BatchJobReplicateSource{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobReplicateSource(b *testing.B) {
+	v := BatchJobReplicateSource{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobReplicateSource(t *testing.T) {
+	v := BatchJobReplicateSource{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobReplicateSource Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobReplicateSource{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobReplicateSource(b *testing.B) {
+	v := BatchJobReplicateSource{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobReplicateSource(b *testing.B) {
+	v := BatchJobReplicateSource{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobReplicateTarget(t *testing.T) {
+	v := BatchJobReplicateTarget{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobReplicateTarget(b *testing.B) {
+	v := BatchJobReplicateTarget{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobReplicateTarget(b *testing.B) {
+	v := BatchJobReplicateTarget{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobReplicateTarget(b *testing.B) {
+	v := BatchJobReplicateTarget{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobReplicateTarget(t *testing.T) {
+	v := BatchJobReplicateTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobReplicateTarget Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobReplicateTarget{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobReplicateTarget(b *testing.B) {
+	v := BatchJobReplicateTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobReplicateTarget(b *testing.B) {
+	v := BatchJobReplicateTarget{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchJobReplicateV1(t *testing.T) {
+	v := BatchJobReplicateV1{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchJobReplicateV1(b *testing.B) {
+	v := BatchJobReplicateV1{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchJobReplicateV1(b *testing.B) {
+	v := BatchJobReplicateV1{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchJobReplicateV1(b *testing.B) {
+	v := BatchJobReplicateV1{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchJobReplicateV1(t *testing.T) {
+	v := BatchJobReplicateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchJobReplicateV1 Msgsize() is inaccurate")
+	}
+
+	vn := BatchJobReplicateV1{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchJobReplicateV1(b *testing.B) {
+	v := BatchJobReplicateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchJobReplicateV1(b *testing.B) {
+	v := BatchJobReplicateV1{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestMarshalUnmarshalBatchReplicateFilter(t *testing.T) {
+	v := BatchReplicateFilter{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgBatchReplicateFilter(b *testing.B) {
+	v := BatchReplicateFilter{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgBatchReplicateFilter(b *testing.B) {
+	v := BatchReplicateFilter{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalBatchReplicateFilter(b *testing.B) {
+	v := BatchReplicateFilter{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeBatchReplicateFilter(t *testing.T) {
+	v := BatchReplicateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeBatchReplicateFilter Msgsize() is inaccurate")
+	}
+
+	vn := BatchReplicateFilter{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeBatchReplicateFilter(b *testing.B) {
+	v := BatchReplicateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeBatchReplicateFilter(b *testing.B) {
+	v := BatchReplicateFilter{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/batch-rotate.go

@@ -18,13 +18,9 @@
 package cmd
 
 import (
-	"bytes"
 	"context"
 	"encoding/base64"
-	"encoding/json"
-	"errors"
 	"fmt"
-	"io"
 	"math/rand"
 	"net/http"
 	"runtime"
@@ -39,7 +35,6 @@ import (
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/env"
-	"github.com/minio/pkg/wildcard"
 	"github.com/minio/pkg/workers"
 )
 
@@ -76,56 +71,6 @@ import (
 
 //go:generate msgp -file $GOFILE -unexported
 
-// BatchKeyRotateKV is a datatype that holds key and values for filtering of objects
-// used by metadata filter as well as tags based filtering.
-type BatchKeyRotateKV struct {
-	Key   string `yaml:"key" json:"key"`
-	Value string `yaml:"value" json:"value"`
-}
-
-// Validate returns an error if key is empty
-func (kv BatchKeyRotateKV) Validate() error {
-	if kv.Key == "" {
-		return errInvalidArgument
-	}
-	return nil
-}
-
-// Empty indicates if kv is not set
-func (kv BatchKeyRotateKV) Empty() bool {
-	return kv.Key == "" && kv.Value == ""
-}
-
-// Match matches input kv with kv, value will be wildcard matched depending on the user input
-func (kv BatchKeyRotateKV) Match(ikv BatchKeyRotateKV) bool {
-	if kv.Empty() {
-		return true
-	}
-	if strings.EqualFold(kv.Key, ikv.Key) {
-		return wildcard.Match(kv.Value, ikv.Value)
-	}
-	return false
-}
-
-// BatchKeyRotateRetry datatype represents total retry attempts and delay between each retries.
-type BatchKeyRotateRetry struct {
-	Attempts int           `yaml:"attempts" json:"attempts"` // number of retry attempts
-	Delay    time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
-}
-
-// Validate validates input replicate retries.
-func (r BatchKeyRotateRetry) Validate() error {
-	if r.Attempts < 0 {
-		return errInvalidArgument
-	}
-
-	if r.Delay < 0 {
-		return errInvalidArgument
-	}
-
-	return nil
-}
-
 // BatchKeyRotationType defines key rotation type
 type BatchKeyRotationType string
 
@@ -178,13 +123,13 @@ func (e BatchJobKeyRotateEncryption) Validate() error {
 
 // BatchKeyRotateFilter holds all the filters currently supported for batch replication
 type BatchKeyRotateFilter struct {
-	NewerThan     time.Duration      `yaml:"newerThan,omitempty" json:"newerThan"`
-	OlderThan     time.Duration      `yaml:"olderThan,omitempty" json:"olderThan"`
-	CreatedAfter  time.Time          `yaml:"createdAfter,omitempty" json:"createdAfter"`
-	CreatedBefore time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
-	Tags          []BatchKeyRotateKV `yaml:"tags,omitempty" json:"tags"`
-	Metadata      []BatchKeyRotateKV `yaml:"metadata,omitempty" json:"metadata"`
-	KMSKeyID      string             `yaml:"kmskeyid" json:"kmskey"`
+	NewerThan     time.Duration `yaml:"newerThan,omitempty" json:"newerThan"`
+	OlderThan     time.Duration `yaml:"olderThan,omitempty" json:"olderThan"`
+	CreatedAfter  time.Time     `yaml:"createdAfter,omitempty" json:"createdAfter"`
+	CreatedBefore time.Time     `yaml:"createdBefore,omitempty" json:"createdBefore"`
+	Tags          []BatchJobKV  `yaml:"tags,omitempty" json:"tags"`
+	Metadata      []BatchJobKV  `yaml:"metadata,omitempty" json:"metadata"`
+	KMSKeyID      string        `yaml:"kmskeyid" json:"kmskey"`
 }
 
 // BatchKeyRotateNotification success or failure notification endpoint for each job attempts
@@ -198,9 +143,9 @@ type BatchKeyRotateNotification struct {
 // - notify
 // - retry
 type BatchJobKeyRotateFlags struct {
-	Filter BatchKeyRotateFilter       `yaml:"filter" json:"filter"`
-	Notify BatchKeyRotateNotification `yaml:"notify" json:"notify"`
-	Retry  BatchKeyRotateRetry        `yaml:"retry" json:"retry"`
+	Filter BatchKeyRotateFilter `yaml:"filter" json:"filter"`
+	Notify BatchJobNotification `yaml:"notify" json:"notify"`
+	Retry  BatchJobRetry        `yaml:"retry" json:"retry"`
 }
 
 // BatchJobKeyRotateV1 v1 of batch key rotation job
@@ -214,35 +159,8 @@ type BatchJobKeyRotateV1 struct {
 }
 
 // Notify notifies notification endpoint if configured regarding job failure or success.
-func (r BatchJobKeyRotateV1) Notify(ctx context.Context, body io.Reader) error {
-	if r.Flags.Notify.Endpoint == "" {
-		return nil
-	}
-
-	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
-	defer cancel()
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.Flags.Notify.Endpoint, body)
-	if err != nil {
-		return err
-	}
-
-	if r.Flags.Notify.Token != "" {
-		req.Header.Set("Authorization", r.Flags.Notify.Token)
-	}
-
-	clnt := http.Client{Transport: getRemoteInstanceTransport}
-	resp, err := clnt.Do(req)
-	if err != nil {
-		return err
-	}
-
-	xhttp.DrainBody(resp.Body)
-	if resp.StatusCode != http.StatusOK {
-		return errors.New(resp.Status)
-	}
-
-	return nil
+func (r BatchJobKeyRotateV1) Notify(ctx context.Context, ri *batchJobInfo) error {
+	return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token)
 }
 
 // KeyRotate rotates encryption key of an object
@@ -330,7 +248,7 @@ const (
 	batchKeyRotateVersion              = batchKeyRotateVersionV1
 	batchKeyRotateAPIVersion           = "v1"
 	batchKeyRotateJobDefaultRetries    = 3
-	batchKeyRotateJobDefaultRetryDelay = 250 * time.Millisecond
+	batchKeyRotateJobDefaultRetryDelay = 25 * time.Millisecond
 )
 
 // Start the batch key rottion job, resumes if there was a pending job via "job.ID"
@@ -351,6 +269,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	if delay == 0 {
 		delay = batchKeyRotateJobDefaultRetryDelay
 	}
+
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
 	skip := func(info FileInfo) (ok bool) {
@@ -388,7 +307,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 			for _, kv := range r.Flags.Filter.Tags {
 				for t, v := range tagMap {
-					if kv.Match(BatchKeyRotateKV{Key: t, Value: v}) {
+					if kv.Match(BatchJobKV{Key: t, Value: v}) {
 						return true
 					}
 				}
@@ -405,7 +324,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						continue
 					}
 					// We only need to match x-amz-meta or standardHeaders
-					if kv.Match(BatchKeyRotateKV{Key: k, Value: v}) {
+					if kv.Match(BatchJobKV{Key: k, Value: v}) {
 						return true
 					}
 				}
@@ -475,6 +394,9 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 				if success {
 					break
 				}
+				if delay > 0 {
+					time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
+				}
 			}
 		}()
 	}
@@ -486,20 +408,11 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	// persist in-memory state to disk.
 	logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 
-	buf, _ := json.Marshal(ri)
-	if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
+	if err := r.Notify(ctx, ri); err != nil {
 		logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
 	}
 
 	cancel()
-	if ri.Failed {
-		ri.ObjectsFailed = 0
-		ri.Bucket = ""
-		ri.Object = ""
-		ri.Objects = 0
-		time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
-	}
-
 	return nil
 }
 

cmd/batch-rotate_gen.go

@@ -192,75 +192,17 @@ func (z *BatchJobKeyRotateFlags) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Notify":
-			var zb0002 uint32
-			zb0002, err = dc.ReadMapHeader()
+			err = z.Notify.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "Notify")
 				return
 			}
-			for zb0002 > 0 {
-				zb0002--
-				field, err = dc.ReadMapKeyPtr()
-				if err != nil {
-					err = msgp.WrapError(err, "Notify")
-					return
-				}
-				switch msgp.UnsafeString(field) {
-				case "Endpoint":
-					z.Notify.Endpoint, err = dc.ReadString()
-					if err != nil {
-						err = msgp.WrapError(err, "Notify", "Endpoint")
-						return
-					}
-				case "Token":
-					z.Notify.Token, err = dc.ReadString()
-					if err != nil {
-						err = msgp.WrapError(err, "Notify", "Token")
-						return
-					}
-				default:
-					err = dc.Skip()
-					if err != nil {
-						err = msgp.WrapError(err, "Notify")
-						return
-					}
-				}
-			}
 		case "Retry":
-			var zb0003 uint32
-			zb0003, err = dc.ReadMapHeader()
+			err = z.Retry.DecodeMsg(dc)
 			if err != nil {
 				err = msgp.WrapError(err, "Retry")
 				return
 			}
-			for zb0003 > 0 {
-				zb0003--
-				field, err = dc.ReadMapKeyPtr()
-				if err != nil {
-					err = msgp.WrapError(err, "Retry")
-					return
-				}
-				switch msgp.UnsafeString(field) {
-				case "Attempts":
-					z.Retry.Attempts, err = dc.ReadInt()
-					if err != nil {
-						err = msgp.WrapError(err, "Retry", "Attempts")
-						return
-					}
-				case "Delay":
-					z.Retry.Delay, err = dc.ReadDuration()
-					if err != nil {
-						err = msgp.WrapError(err, "Retry", "Delay")
-						return
-					}
-				default:
-					err = dc.Skip()
-					if err != nil {
-						err = msgp.WrapError(err, "Retry")
-						return
-					}
-				}
-			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -290,25 +232,9 @@ func (z *BatchJobKeyRotateFlags) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	// map header, size 2
-	// write "Endpoint"
-	err = en.Append(0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
+	err = z.Notify.EncodeMsg(en)
 	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Notify.Endpoint)
-	if err != nil {
-		err = msgp.WrapError(err, "Notify", "Endpoint")
-		return
-	}
-	// write "Token"
-	err = en.Append(0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Notify.Token)
-	if err != nil {
-		err = msgp.WrapError(err, "Notify", "Token")
+		err = msgp.WrapError(err, "Notify")
 		return
 	}
 	// write "Retry"
@@ -316,25 +242,9 @@ func (z *BatchJobKeyRotateFlags) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	// map header, size 2
-	// write "Attempts"
-	err = en.Append(0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
+	err = z.Retry.EncodeMsg(en)
 	if err != nil {
-		return
-	}
-	err = en.WriteInt(z.Retry.Attempts)
-	if err != nil {
-		err = msgp.WrapError(err, "Retry", "Attempts")
-		return
-	}
-	// write "Delay"
-	err = en.Append(0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
-	if err != nil {
-		return
-	}
-	err = en.WriteDuration(z.Retry.Delay)
-	if err != nil {
-		err = msgp.WrapError(err, "Retry", "Delay")
+		err = msgp.WrapError(err, "Retry")
 		return
 	}
 	return
@@ -353,22 +263,18 @@ func (z *BatchJobKeyRotateFlags) MarshalMsg(b []byte) (o []byte, err error) {
 	}
 	// string "Notify"
 	o = append(o, 0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79)
-	// map header, size 2
-	// string "Endpoint"
-	o = append(o, 0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
-	o = msgp.AppendString(o, z.Notify.Endpoint)
-	// string "Token"
-	o = append(o, 0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e)
-	o = msgp.AppendString(o, z.Notify.Token)
+	o, err = z.Notify.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Notify")
+		return
+	}
 	// string "Retry"
 	o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
-	// map header, size 2
-	// string "Attempts"
-	o = append(o, 0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
-	o = msgp.AppendInt(o, z.Retry.Attempts)
-	// string "Delay"
-	o = append(o, 0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
-	o = msgp.AppendDuration(o, z.Retry.Delay)
+	o, err = z.Retry.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Retry")
+		return
+	}
 	return
 }
 
@@ -397,75 +303,17 @@ func (z *BatchJobKeyRotateFlags) UnmarshalMsg(bts []byte) (o []byte, err error)
 				return
 			}
 		case "Notify":
-			var zb0002 uint32
-			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
+			bts, err = z.Notify.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Notify")
 				return
 			}
-			for zb0002 > 0 {
-				zb0002--
-				field, bts, err = msgp.ReadMapKeyZC(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Notify")
-					return
-				}
-				switch msgp.UnsafeString(field) {
-				case "Endpoint":
-					z.Notify.Endpoint, bts, err = msgp.ReadStringBytes(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Notify", "Endpoint")
-						return
-					}
-				case "Token":
-					z.Notify.Token, bts, err = msgp.ReadStringBytes(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Notify", "Token")
-						return
-					}
-				default:
-					bts, err = msgp.Skip(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Notify")
-						return
-					}
-				}
-			}
 		case "Retry":
-			var zb0003 uint32
-			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			bts, err = z.Retry.UnmarshalMsg(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Retry")
 				return
 			}
-			for zb0003 > 0 {
-				zb0003--
-				field, bts, err = msgp.ReadMapKeyZC(bts)
-				if err != nil {
-					err = msgp.WrapError(err, "Retry")
-					return
-				}
-				switch msgp.UnsafeString(field) {
-				case "Attempts":
-					z.Retry.Attempts, bts, err = msgp.ReadIntBytes(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Retry", "Attempts")
-						return
-					}
-				case "Delay":
-					z.Retry.Delay, bts, err = msgp.ReadDurationBytes(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Retry", "Delay")
-						return
-					}
-				default:
-					bts, err = msgp.Skip(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Retry")
-						return
-					}
-				}
-			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -480,7 +328,7 @@ func (z *BatchJobKeyRotateFlags) UnmarshalMsg(bts []byte) (o []byte, err error)
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobKeyRotateFlags) Msgsize() (s int) {
-	s = 1 + 7 + z.Filter.Msgsize() + 7 + 1 + 9 + msgp.StringPrefixSize + len(z.Notify.Endpoint) + 6 + msgp.StringPrefixSize + len(z.Notify.Token) + 6 + 1 + 9 + msgp.IntSize + 6 + msgp.DurationSize
+	s = 1 + 7 + z.Filter.Msgsize() + 7 + z.Notify.Msgsize() + 6 + z.Retry.Msgsize()
 	return
 }
 
@@ -509,11 +357,46 @@ func (z *BatchJobKeyRotateV1) DecodeMsg(dc *msgp.Reader) (err error) {
 				return
 			}
 		case "Flags":
-			err = z.Flags.DecodeMsg(dc)
+			var zb0002 uint32
+			zb0002, err = dc.ReadMapHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Flags")
 				return
 			}
+			for zb0002 > 0 {
+				zb0002--
+				field, err = dc.ReadMapKeyPtr()
+				if err != nil {
+					err = msgp.WrapError(err, "Flags")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "Filter":
+					err = z.Flags.Filter.DecodeMsg(dc)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Filter")
+						return
+					}
+				case "Notify":
+					err = z.Flags.Notify.DecodeMsg(dc)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Notify")
+						return
+					}
+				case "Retry":
+					err = z.Flags.Retry.DecodeMsg(dc)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Retry")
+						return
+					}
+				default:
+					err = dc.Skip()
+					if err != nil {
+						err = msgp.WrapError(err, "Flags")
+						return
+					}
+				}
+			}
 		case "Bucket":
 			z.Bucket, err = dc.ReadString()
 			if err != nil {
@@ -567,9 +450,35 @@ func (z *BatchJobKeyRotateV1) EncodeMsg(en *msgp.Writer) (err error) {
 	if err != nil {
 		return
 	}
-	err = z.Flags.EncodeMsg(en)
+	// map header, size 3
+	// write "Filter"
+	err = en.Append(0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
 	if err != nil {
-		err = msgp.WrapError(err, "Flags")
+		return
+	}
+	err = z.Flags.Filter.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Flags", "Filter")
+		return
+	}
+	// write "Notify"
+	err = en.Append(0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79)
+	if err != nil {
+		return
+	}
+	err = z.Flags.Notify.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Flags", "Notify")
+		return
+	}
+	// write "Retry"
+	err = en.Append(0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
+	if err != nil {
+		return
+	}
+	err = z.Flags.Retry.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "Flags", "Retry")
 		return
 	}
 	// write "Bucket"
@@ -624,9 +533,26 @@ func (z *BatchJobKeyRotateV1) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendString(o, z.APIVersion)
 	// string "Flags"
 	o = append(o, 0xa5, 0x46, 0x6c, 0x61, 0x67, 0x73)
-	o, err = z.Flags.MarshalMsg(o)
+	// map header, size 3
+	// string "Filter"
+	o = append(o, 0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72)
+	o, err = z.Flags.Filter.MarshalMsg(o)
 	if err != nil {
-		err = msgp.WrapError(err, "Flags")
+		err = msgp.WrapError(err, "Flags", "Filter")
+		return
+	}
+	// string "Notify"
+	o = append(o, 0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79)
+	o, err = z.Flags.Notify.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Flags", "Notify")
+		return
+	}
+	// string "Retry"
+	o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79)
+	o, err = z.Flags.Retry.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "Flags", "Retry")
 		return
 	}
 	// string "Bucket"
@@ -673,11 +599,46 @@ func (z *BatchJobKeyRotateV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				return
 			}
 		case "Flags":
-			bts, err = z.Flags.UnmarshalMsg(bts)
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadMapHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Flags")
 				return
 			}
+			for zb0002 > 0 {
+				zb0002--
+				field, bts, err = msgp.ReadMapKeyZC(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "Flags")
+					return
+				}
+				switch msgp.UnsafeString(field) {
+				case "Filter":
+					bts, err = z.Flags.Filter.UnmarshalMsg(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Filter")
+						return
+					}
+				case "Notify":
+					bts, err = z.Flags.Notify.UnmarshalMsg(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Notify")
+						return
+					}
+				case "Retry":
+					bts, err = z.Flags.Retry.UnmarshalMsg(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags", "Retry")
+						return
+					}
+				default:
+					bts, err = msgp.Skip(bts)
+					if err != nil {
+						err = msgp.WrapError(err, "Flags")
+						return
+					}
+				}
+			}
 		case "Bucket":
 			z.Bucket, bts, err = msgp.ReadStringBytes(bts)
 			if err != nil {
@@ -716,7 +677,7 @@ func (z *BatchJobKeyRotateV1) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobKeyRotateV1) Msgsize() (s int) {
-	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 6 + z.Flags.Msgsize() + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 11 + z.Encryption.Msgsize()
+	s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 6 + 1 + 7 + z.Flags.Filter.Msgsize() + 7 + z.Flags.Notify.Msgsize() + 6 + z.Flags.Retry.Msgsize() + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 11 + z.Encryption.Msgsize()
 	return
 }
 
@@ -772,91 +733,33 @@ func (z *BatchKeyRotateFilter) DecodeMsg(dc *msgp.Reader) (err error) {
 			if cap(z.Tags) >= int(zb0002) {
 				z.Tags = (z.Tags)[:zb0002]
 			} else {
-				z.Tags = make([]BatchKeyRotateKV, zb0002)
+				z.Tags = make([]BatchJobKV, zb0002)
 			}
 			for za0001 := range z.Tags {
-				var zb0003 uint32
-				zb0003, err = dc.ReadMapHeader()
+				err = z.Tags[za0001].DecodeMsg(dc)
 				if err != nil {
 					err = msgp.WrapError(err, "Tags", za0001)
 					return
 				}
-				for zb0003 > 0 {
-					zb0003--
-					field, err = dc.ReadMapKeyPtr()
-					if err != nil {
-						err = msgp.WrapError(err, "Tags", za0001)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "Key":
-						z.Tags[za0001].Key, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001, "Key")
-							return
-						}
-					case "Value":
-						z.Tags[za0001].Value, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001, "Value")
-							return
-						}
-					default:
-						err = dc.Skip()
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001)
-							return
-						}
-					}
-				}
 			}
 		case "Metadata":
-			var zb0004 uint32
-			zb0004, err = dc.ReadArrayHeader()
+			var zb0003 uint32
+			zb0003, err = dc.ReadArrayHeader()
 			if err != nil {
 				err = msgp.WrapError(err, "Metadata")
 				return
 			}
-			if cap(z.Metadata) >= int(zb0004) {
-				z.Metadata = (z.Metadata)[:zb0004]
+			if cap(z.Metadata) >= int(zb0003) {
+				z.Metadata = (z.Metadata)[:zb0003]
 			} else {
-				z.Metadata = make([]BatchKeyRotateKV, zb0004)
+				z.Metadata = make([]BatchJobKV, zb0003)
 			}
 			for za0002 := range z.Metadata {
-				var zb0005 uint32
-				zb0005, err = dc.ReadMapHeader()
+				err = z.Metadata[za0002].DecodeMsg(dc)
 				if err != nil {
 					err = msgp.WrapError(err, "Metadata", za0002)
 					return
 				}
-				for zb0005 > 0 {
-					zb0005--
-					field, err = dc.ReadMapKeyPtr()
-					if err != nil {
-						err = msgp.WrapError(err, "Metadata", za0002)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "Key":
-						z.Metadata[za0002].Key, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002, "Key")
-							return
-						}
-					case "Value":
-						z.Metadata[za0002].Value, err = dc.ReadString()
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002, "Value")
-							return
-						}
-					default:
-						err = dc.Skip()
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002)
-							return
-						}
-					}
-				}
 			}
 		case "KMSKeyID":
 			z.KMSKeyID, err = dc.ReadString()
@@ -929,25 +832,9 @@ func (z *BatchKeyRotateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 		return
 	}
 	for za0001 := range z.Tags {
-		// map header, size 2
-		// write "Key"
-		err = en.Append(0x82, 0xa3, 0x4b, 0x65, 0x79)
+		err = z.Tags[za0001].EncodeMsg(en)
 		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Tags[za0001].Key)
-		if err != nil {
-			err = msgp.WrapError(err, "Tags", za0001, "Key")
-			return
-		}
-		// write "Value"
-		err = en.Append(0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Tags[za0001].Value)
-		if err != nil {
-			err = msgp.WrapError(err, "Tags", za0001, "Value")
+			err = msgp.WrapError(err, "Tags", za0001)
 			return
 		}
 	}
@@ -962,25 +849,9 @@ func (z *BatchKeyRotateFilter) EncodeMsg(en *msgp.Writer) (err error) {
 		return
 	}
 	for za0002 := range z.Metadata {
-		// map header, size 2
-		// write "Key"
-		err = en.Append(0x82, 0xa3, 0x4b, 0x65, 0x79)
+		err = z.Metadata[za0002].EncodeMsg(en)
 		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Metadata[za0002].Key)
-		if err != nil {
-			err = msgp.WrapError(err, "Metadata", za0002, "Key")
-			return
-		}
-		// write "Value"
-		err = en.Append(0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-		if err != nil {
-			return
-		}
-		err = en.WriteString(z.Metadata[za0002].Value)
-		if err != nil {
-			err = msgp.WrapError(err, "Metadata", za0002, "Value")
+			err = msgp.WrapError(err, "Metadata", za0002)
 			return
 		}
 	}
@@ -1017,25 +888,21 @@ func (z *BatchKeyRotateFilter) MarshalMsg(b []byte) (o []byte, err error) {
 	o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(len(z.Tags)))
 	for za0001 := range z.Tags {
-		// map header, size 2
-		// string "Key"
-		o = append(o, 0x82, 0xa3, 0x4b, 0x65, 0x79)
-		o = msgp.AppendString(o, z.Tags[za0001].Key)
-		// string "Value"
-		o = append(o, 0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-		o = msgp.AppendString(o, z.Tags[za0001].Value)
+		o, err = z.Tags[za0001].MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Tags", za0001)
+			return
+		}
 	}
 	// string "Metadata"
 	o = append(o, 0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61)
 	o = msgp.AppendArrayHeader(o, uint32(len(z.Metadata)))
 	for za0002 := range z.Metadata {
-		// map header, size 2
-		// string "Key"
-		o = append(o, 0x82, 0xa3, 0x4b, 0x65, 0x79)
-		o = msgp.AppendString(o, z.Metadata[za0002].Key)
-		// string "Value"
-		o = append(o, 0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-		o = msgp.AppendString(o, z.Metadata[za0002].Value)
+		o, err = z.Metadata[za0002].MarshalMsg(o)
+		if err != nil {
+			err = msgp.WrapError(err, "Metadata", za0002)
+			return
+		}
 	}
 	// string "KMSKeyID"
 	o = append(o, 0xa8, 0x4b, 0x4d, 0x53, 0x4b, 0x65, 0x79, 0x49, 0x44)
@@ -1095,91 +962,33 @@ func (z *BatchKeyRotateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 			if cap(z.Tags) >= int(zb0002) {
 				z.Tags = (z.Tags)[:zb0002]
 			} else {
-				z.Tags = make([]BatchKeyRotateKV, zb0002)
+				z.Tags = make([]BatchJobKV, zb0002)
 			}
 			for za0001 := range z.Tags {
-				var zb0003 uint32
-				zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+				bts, err = z.Tags[za0001].UnmarshalMsg(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Tags", za0001)
 					return
 				}
-				for zb0003 > 0 {
-					zb0003--
-					field, bts, err = msgp.ReadMapKeyZC(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Tags", za0001)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "Key":
-						z.Tags[za0001].Key, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001, "Key")
-							return
-						}
-					case "Value":
-						z.Tags[za0001].Value, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001, "Value")
-							return
-						}
-					default:
-						bts, err = msgp.Skip(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Tags", za0001)
-							return
-						}
-					}
-				}
 			}
 		case "Metadata":
-			var zb0004 uint32
-			zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Metadata")
 				return
 			}
-			if cap(z.Metadata) >= int(zb0004) {
-				z.Metadata = (z.Metadata)[:zb0004]
+			if cap(z.Metadata) >= int(zb0003) {
+				z.Metadata = (z.Metadata)[:zb0003]
 			} else {
-				z.Metadata = make([]BatchKeyRotateKV, zb0004)
+				z.Metadata = make([]BatchJobKV, zb0003)
 			}
 			for za0002 := range z.Metadata {
-				var zb0005 uint32
-				zb0005, bts, err = msgp.ReadMapHeaderBytes(bts)
+				bts, err = z.Metadata[za0002].UnmarshalMsg(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "Metadata", za0002)
 					return
 				}
-				for zb0005 > 0 {
-					zb0005--
-					field, bts, err = msgp.ReadMapKeyZC(bts)
-					if err != nil {
-						err = msgp.WrapError(err, "Metadata", za0002)
-						return
-					}
-					switch msgp.UnsafeString(field) {
-					case "Key":
-						z.Metadata[za0002].Key, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002, "Key")
-							return
-						}
-					case "Value":
-						z.Metadata[za0002].Value, bts, err = msgp.ReadStringBytes(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002, "Value")
-							return
-						}
-					default:
-						bts, err = msgp.Skip(bts)
-						if err != nil {
-							err = msgp.WrapError(err, "Metadata", za0002)
-							return
-						}
-					}
-				}
 			}
 		case "KMSKeyID":
 			z.KMSKeyID, bts, err = msgp.ReadStringBytes(bts)
@@ -1203,144 +1012,16 @@ func (z *BatchKeyRotateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) {
 func (z *BatchKeyRotateFilter) Msgsize() (s int) {
 	s = 1 + 10 + msgp.DurationSize + 10 + msgp.DurationSize + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize
 	for za0001 := range z.Tags {
-		s += 1 + 4 + msgp.StringPrefixSize + len(z.Tags[za0001].Key) + 6 + msgp.StringPrefixSize + len(z.Tags[za0001].Value)
+		s += z.Tags[za0001].Msgsize()
 	}
 	s += 9 + msgp.ArrayHeaderSize
 	for za0002 := range z.Metadata {
-		s += 1 + 4 + msgp.StringPrefixSize + len(z.Metadata[za0002].Key) + 6 + msgp.StringPrefixSize + len(z.Metadata[za0002].Value)
+		s += z.Metadata[za0002].Msgsize()
 	}
 	s += 9 + msgp.StringPrefixSize + len(z.KMSKeyID)
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *BatchKeyRotateKV) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Key":
-			z.Key, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Key")
-				return
-			}
-		case "Value":
-			z.Value, err = dc.ReadString()
-			if err != nil {
-				err = msgp.WrapError(err, "Value")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z BatchKeyRotateKV) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
-	// write "Key"
-	err = en.Append(0x82, 0xa3, 0x4b, 0x65, 0x79)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Key)
-	if err != nil {
-		err = msgp.WrapError(err, "Key")
-		return
-	}
-	// write "Value"
-	err = en.Append(0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-	if err != nil {
-		return
-	}
-	err = en.WriteString(z.Value)
-	if err != nil {
-		err = msgp.WrapError(err, "Value")
-		return
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z BatchKeyRotateKV) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
-	// string "Key"
-	o = append(o, 0x82, 0xa3, 0x4b, 0x65, 0x79)
-	o = msgp.AppendString(o, z.Key)
-	// string "Value"
-	o = append(o, 0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65)
-	o = msgp.AppendString(o, z.Value)
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *BatchKeyRotateKV) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Key":
-			z.Key, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Key")
-				return
-			}
-		case "Value":
-			z.Value, bts, err = msgp.ReadStringBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Value")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z BatchKeyRotateKV) Msgsize() (s int) {
-	s = 1 + 4 + msgp.StringPrefixSize + len(z.Key) + 6 + msgp.StringPrefixSize + len(z.Value)
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *BatchKeyRotateNotification) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte
@@ -1469,134 +1150,6 @@ func (z BatchKeyRotateNotification) Msgsize() (s int) {
 	return
 }
 
-// DecodeMsg implements msgp.Decodable
-func (z *BatchKeyRotateRetry) DecodeMsg(dc *msgp.Reader) (err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, err = dc.ReadMapHeader()
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, err = dc.ReadMapKeyPtr()
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Attempts":
-			z.Attempts, err = dc.ReadInt()
-			if err != nil {
-				err = msgp.WrapError(err, "Attempts")
-				return
-			}
-		case "Delay":
-			z.Delay, err = dc.ReadDuration()
-			if err != nil {
-				err = msgp.WrapError(err, "Delay")
-				return
-			}
-		default:
-			err = dc.Skip()
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	return
-}
-
-// EncodeMsg implements msgp.Encodable
-func (z BatchKeyRotateRetry) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
-	// write "Attempts"
-	err = en.Append(0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
-	if err != nil {
-		return
-	}
-	err = en.WriteInt(z.Attempts)
-	if err != nil {
-		err = msgp.WrapError(err, "Attempts")
-		return
-	}
-	// write "Delay"
-	err = en.Append(0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
-	if err != nil {
-		return
-	}
-	err = en.WriteDuration(z.Delay)
-	if err != nil {
-		err = msgp.WrapError(err, "Delay")
-		return
-	}
-	return
-}
-
-// MarshalMsg implements msgp.Marshaler
-func (z BatchKeyRotateRetry) MarshalMsg(b []byte) (o []byte, err error) {
-	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
-	// string "Attempts"
-	o = append(o, 0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73)
-	o = msgp.AppendInt(o, z.Attempts)
-	// string "Delay"
-	o = append(o, 0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79)
-	o = msgp.AppendDuration(o, z.Delay)
-	return
-}
-
-// UnmarshalMsg implements msgp.Unmarshaler
-func (z *BatchKeyRotateRetry) UnmarshalMsg(bts []byte) (o []byte, err error) {
-	var field []byte
-	_ = field
-	var zb0001 uint32
-	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
-	if err != nil {
-		err = msgp.WrapError(err)
-		return
-	}
-	for zb0001 > 0 {
-		zb0001--
-		field, bts, err = msgp.ReadMapKeyZC(bts)
-		if err != nil {
-			err = msgp.WrapError(err)
-			return
-		}
-		switch msgp.UnsafeString(field) {
-		case "Attempts":
-			z.Attempts, bts, err = msgp.ReadIntBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Attempts")
-				return
-			}
-		case "Delay":
-			z.Delay, bts, err = msgp.ReadDurationBytes(bts)
-			if err != nil {
-				err = msgp.WrapError(err, "Delay")
-				return
-			}
-		default:
-			bts, err = msgp.Skip(bts)
-			if err != nil {
-				err = msgp.WrapError(err)
-				return
-			}
-		}
-	}
-	o = bts
-	return
-}
-
-// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
-func (z BatchKeyRotateRetry) Msgsize() (s int) {
-	s = 1 + 9 + msgp.IntSize + 6 + msgp.DurationSize
-	return
-}
-
 // DecodeMsg implements msgp.Decodable
 func (z *BatchKeyRotationType) DecodeMsg(dc *msgp.Reader) (err error) {
 	{

cmd/batch-rotate_gen_test.go

@@ -461,119 +461,6 @@ func BenchmarkDecodeBatchKeyRotateFilter(b *testing.B) {
 	}
 }
 
-func TestMarshalUnmarshalBatchKeyRotateKV(t *testing.T) {
-	v := BatchKeyRotateKV{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgBatchKeyRotateKV(b *testing.B) {
-	v := BatchKeyRotateKV{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgBatchKeyRotateKV(b *testing.B) {
-	v := BatchKeyRotateKV{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalBatchKeyRotateKV(b *testing.B) {
-	v := BatchKeyRotateKV{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodeBatchKeyRotateKV(t *testing.T) {
-	v := BatchKeyRotateKV{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateKV Msgsize() is inaccurate")
-	}
-
-	vn := BatchKeyRotateKV{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodeBatchKeyRotateKV(b *testing.B) {
-	v := BatchKeyRotateKV{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodeBatchKeyRotateKV(b *testing.B) {
-	v := BatchKeyRotateKV{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
 func TestMarshalUnmarshalBatchKeyRotateNotification(t *testing.T) {
 	v := BatchKeyRotateNotification{}
 	bts, err := v.MarshalMsg(nil)
@@ -686,116 +573,3 @@ func BenchmarkDecodeBatchKeyRotateNotification(b *testing.B) {
 		}
 	}
 }
-
-func TestMarshalUnmarshalBatchKeyRotateRetry(t *testing.T) {
-	v := BatchKeyRotateRetry{}
-	bts, err := v.MarshalMsg(nil)
-	if err != nil {
-		t.Fatal(err)
-	}
-	left, err := v.UnmarshalMsg(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
-	}
-
-	left, err = msgp.Skip(bts)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(left) > 0 {
-		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
-	}
-}
-
-func BenchmarkMarshalMsgBatchKeyRotateRetry(b *testing.B) {
-	v := BatchKeyRotateRetry{}
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.MarshalMsg(nil)
-	}
-}
-
-func BenchmarkAppendMsgBatchKeyRotateRetry(b *testing.B) {
-	v := BatchKeyRotateRetry{}
-	bts := make([]byte, 0, v.Msgsize())
-	bts, _ = v.MarshalMsg(bts[0:0])
-	b.SetBytes(int64(len(bts)))
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		bts, _ = v.MarshalMsg(bts[0:0])
-	}
-}
-
-func BenchmarkUnmarshalBatchKeyRotateRetry(b *testing.B) {
-	v := BatchKeyRotateRetry{}
-	bts, _ := v.MarshalMsg(nil)
-	b.ReportAllocs()
-	b.SetBytes(int64(len(bts)))
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		_, err := v.UnmarshalMsg(bts)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}
-
-func TestEncodeDecodeBatchKeyRotateRetry(t *testing.T) {
-	v := BatchKeyRotateRetry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-
-	m := v.Msgsize()
-	if buf.Len() > m {
-		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateRetry Msgsize() is inaccurate")
-	}
-
-	vn := BatchKeyRotateRetry{}
-	err := msgp.Decode(&buf, &vn)
-	if err != nil {
-		t.Error(err)
-	}
-
-	buf.Reset()
-	msgp.Encode(&buf, &v)
-	err = msgp.NewReader(&buf).Skip()
-	if err != nil {
-		t.Error(err)
-	}
-}
-
-func BenchmarkEncodeBatchKeyRotateRetry(b *testing.B) {
-	v := BatchKeyRotateRetry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	en := msgp.NewWriter(msgp.Nowhere)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		v.EncodeMsg(en)
-	}
-	en.Flush()
-}
-
-func BenchmarkDecodeBatchKeyRotateRetry(b *testing.B) {
-	v := BatchKeyRotateRetry{}
-	var buf bytes.Buffer
-	msgp.Encode(&buf, &v)
-	b.SetBytes(int64(buf.Len()))
-	rd := msgp.NewEndlessReader(buf.Bytes(), b)
-	dc := msgp.NewReader(rd)
-	b.ReportAllocs()
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		err := v.DecodeMsg(dc)
-		if err != nil {
-			b.Fatal(err)
-		}
-	}
-}

cmd/bitrot-streaming.go

@@ -94,16 +94,23 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
 	r, w := io.Pipe()
 	h := algo.New()
 
-	bw := &streamingBitrotWriter{iow: w, closeWithErr: w.CloseWithError, h: h, shardSize: shardSize, canClose: &sync.WaitGroup{}}
+	bw := &streamingBitrotWriter{
+		iow:          ioutil.NewDeadlineWriter(w, diskMaxTimeout),
+		closeWithErr: w.CloseWithError,
+		h:            h,
+		shardSize:    shardSize,
+		canClose:     &sync.WaitGroup{},
+	}
 	bw.canClose.Add(1)
 	go func() {
+		defer bw.canClose.Done()
+
 		totalFileSize := int64(-1) // For compressed objects length will be unknown (represented by length=-1)
 		if length != -1 {
 			bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
 			totalFileSize = bitrotSumsTotalSize + length
 		}
 		r.CloseWithError(disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r))
-		bw.canClose.Done()
 	}()
 	return bw
 }

cmd/bootstrap-messages.go

@@ -19,98 +19,51 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"sync"
-	"time"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/pubsub"
 )
 
-const bootstrapMsgsLimit = 4 << 10
+const bootstrapTraceLimit = 4 << 10
 
-type bootstrapInfo struct {
-	msg    string
-	ts     time.Time
-	source string
-}
 type bootstrapTracer struct {
-	mu         sync.RWMutex
-	idx        int
-	info       [bootstrapMsgsLimit]bootstrapInfo
-	lastUpdate time.Time
+	mu   sync.RWMutex
+	info []madmin.TraceInfo
 }
 
 var globalBootstrapTracer = &bootstrapTracer{}
 
-func (bs *bootstrapTracer) DropEvents() {
+func (bs *bootstrapTracer) Record(info madmin.TraceInfo) {
 	bs.mu.Lock()
 	defer bs.mu.Unlock()
 
-	if time.Now().UTC().Sub(bs.lastUpdate) > 24*time.Hour {
-		bs.info = [4096]bootstrapInfo{}
-		bs.idx = 0
+	if len(bs.info) > bootstrapTraceLimit {
+		return
 	}
-}
-
-func (bs *bootstrapTracer) Empty() bool {
-	var empty bool
-	bs.mu.RLock()
-	empty = bs.info[0].msg == ""
-	bs.mu.RUnlock()
-
-	return empty
-}
-
-func (bs *bootstrapTracer) Record(msg string, skip int) {
-	source := getSource(skip + 1)
-	bs.mu.Lock()
-	now := time.Now().UTC()
-	bs.info[bs.idx] = bootstrapInfo{
-		msg:    msg,
-		ts:     now,
-		source: source,
-	}
-	bs.lastUpdate = now
-	bs.idx = (bs.idx + 1) % bootstrapMsgsLimit
-	bs.mu.Unlock()
+	bs.info = append(bs.info, info)
 }
 
 func (bs *bootstrapTracer) Events() []madmin.TraceInfo {
-	traceInfo := make([]madmin.TraceInfo, 0, bootstrapMsgsLimit)
-
-	// Add all messages in order
-	addAll := func(info []bootstrapInfo) {
-		for _, msg := range info {
-			if msg.ts.IsZero() {
-				continue // skip empty events
-			}
-			traceInfo = append(traceInfo, madmin.TraceInfo{
-				TraceType: madmin.TraceBootstrap,
-				Time:      msg.ts,
-				NodeName:  globalLocalNodeName,
-				FuncName:  "BOOTSTRAP",
-				Message:   fmt.Sprintf("%s %s", msg.source, msg.msg),
-			})
-		}
-	}
+	traceInfo := make([]madmin.TraceInfo, 0, bootstrapTraceLimit)
 
 	bs.mu.RLock()
-	addAll(bs.info[bs.idx:])
-	addAll(bs.info[:bs.idx])
+	for _, i := range bs.info {
+		traceInfo = append(traceInfo, i)
+	}
 	bs.mu.RUnlock()
+
 	return traceInfo
 }
 
 func (bs *bootstrapTracer) Publish(ctx context.Context, trace *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) {
-	if bs.Empty() {
-		return
-	}
 	for _, bsEvent := range bs.Events() {
-		select {
-		case <-ctx.Done():
-		default:
-			trace.Publish(bsEvent)
+		if bsEvent.Message != "" {
+			select {
+			case <-ctx.Done():
+			default:
+				trace.Publish(bsEvent)
+			}
 		}
 	}
 }

cmd/bootstrap-messages_test.go

@@ -1,60 +0,0 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-	"time"
-)
-
-func TestBootstrap(t *testing.T) {
-	// Bootstrap events exceed bootstrap messages limit
-	bsTracer := &bootstrapTracer{}
-	for i := 0; i < bootstrapMsgsLimit+10; i++ {
-		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
-	}
-
-	traceInfos := bsTracer.Events()
-	if len(traceInfos) != bootstrapMsgsLimit {
-		t.Fatalf("Expected length of events %d but got %d", bootstrapMsgsLimit, len(traceInfos))
-	}
-
-	// Simulate the case where bootstrap events were updated a day ago
-	bsTracer.lastUpdate = time.Now().UTC().Add(-25 * time.Hour)
-	bsTracer.DropEvents()
-	if !bsTracer.Empty() {
-		t.Fatalf("Expected all bootstrap events to have been dropped, but found %d events", len(bsTracer.Events()))
-	}
-
-	// Fewer than 4K bootstrap events
-	for i := 0; i < 10; i++ {
-		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
-	}
-	events := bsTracer.Events()
-	if len(events) != 10 {
-		t.Fatalf("Expected length of events %d but got %d", 10, len(events))
-	}
-	for i, traceInfo := range bsTracer.Events() {
-		msg := fmt.Sprintf("msg-%d", i)
-		if !strings.HasSuffix(traceInfo.Message, msg) {
-			t.Fatalf("Expected %s but got %s", msg, traceInfo.Message)
-		}
-	}
-}

cmd/bootstrap-peer-server.go

@@ -221,7 +221,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 	for onlineServers < len(clnts)/2 {
 		for _, clnt := range clnts {
 			if err := clnt.Verify(ctx, srcCfg); err != nil {
-				bootstrapTrace(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
+				bootstrapTraceMsg(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
 				if !isNetworkError(err) {
 					logger.LogOnceIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err), clnt.String())
 					incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))

cmd/bucket-handlers_test.go

@@ -355,7 +355,7 @@ func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName s
 			maxUploads:         "0",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusNotFound,
+			expectedRespStatus: http.StatusBadRequest,
 			shouldPass:         false,
 		},
 		// Test case - 2.

cmd/bucket-lifecycle.go

@@ -280,8 +280,10 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
 			if err := transitionObject(t.ctx, objectAPI, task.objInfo, newLifecycleAuditEvent(task.src, task.event)); err != nil {
-				logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
-					task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
+				if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+					logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
+						task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
+				}
 			} else {
 				ts := tierStats{
 					TotalSize:   uint64(task.objInfo.Size),
@@ -403,13 +405,11 @@ func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *Ob
 		TierName:  oi.TransitionedObject.Tier,
 	}
 	if err := globalTierJournal.AddEntry(entry); err != nil {
-		logger.LogIf(ctx, err)
 		return err
 	}
 	// Delete metadata on source, now that data in remote tier has been
 	// marked for deletion.
 	if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
-		logger.LogIf(ctx, err)
 		return err
 	}
 

cmd/bucket-listobjects-handlers.go

@@ -231,7 +231,7 @@ func parseRequestToken(token string) (subToken string, nodeIndex int) {
 	if token == "" {
 		return token, -1
 	}
-	i := strings.Index(token, "@")
+	i := strings.Index(token, ":")
 	if i < 0 {
 		return token, -1
 	}

cmd/bucket-metadata-sys.go

@@ -151,14 +151,27 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 		return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile)
 	}
 
-	if err := meta.Save(ctx, objAPI); err != nil {
-		return updatedAt, err
+	err = sys.save(ctx, meta)
+	return updatedAt, err
+}
+
+func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error {
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return errServerNotInitialized
 	}
 
-	sys.Set(bucket, meta)
-	globalNotificationSys.LoadBucketMetadata(bgContext(ctx), bucket) // Do not use caller context here
+	if isMinioMetaBucketName(meta.Name) {
+		return errInvalidArgument
+	}
 
-	return updatedAt, nil
+	if err := meta.Save(ctx, objAPI); err != nil {
+		return err
+	}
+
+	sys.Set(meta.Name, meta)
+	globalNotificationSys.LoadBucketMetadata(bgContext(ctx), meta.Name) // Do not use caller context here
+	return nil
 }
 
 // Delete delete the bucket metadata for the specified bucket.

cmd/bucket-metadata.go

@@ -143,39 +143,38 @@ func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
 
 // Load - loads the metadata of bucket by name from ObjectLayer api.
 // If an error is returned the returned metadata will be default initialized.
-func (b *BucketMetadata) Load(ctx context.Context, api ObjectLayer, name string) error {
+func readBucketMetadata(ctx context.Context, api ObjectLayer, name string) (BucketMetadata, error) {
 	if name == "" {
 		logger.LogIf(ctx, errors.New("bucket name cannot be empty"))
-		return errInvalidArgument
+		return BucketMetadata{}, errInvalidArgument
 	}
+	b := newBucketMetadata(name)
 	configFile := path.Join(bucketMetaPrefix, name, bucketMetadataFile)
 	data, err := readConfig(ctx, api, configFile)
 	if err != nil {
-		return err
+		return b, err
 	}
 	if len(data) <= 4 {
-		return fmt.Errorf("loadBucketMetadata: no data")
+		return b, fmt.Errorf("loadBucketMetadata: no data")
 	}
 	// Read header
 	switch binary.LittleEndian.Uint16(data[0:2]) {
 	case bucketMetadataFormat:
 	default:
-		return fmt.Errorf("loadBucketMetadata: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+		return b, fmt.Errorf("loadBucketMetadata: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
 	}
 	switch binary.LittleEndian.Uint16(data[2:4]) {
 	case bucketMetadataVersion:
 	default:
-		return fmt.Errorf("loadBucketMetadata: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+		return b, fmt.Errorf("loadBucketMetadata: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
 	}
-	// OK, parse data.
 	_, err = b.UnmarshalMsg(data[4:])
-	b.Name = name // in-case parsing failed for some reason, make sure bucket name is not empty.
-	return err
+	return b, err
 }
 
 func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) {
-	b := newBucketMetadata(bucket)
-	err := b.Load(ctx, objectAPI, b.Name)
+	b, err := readBucketMetadata(ctx, objectAPI, bucket)
+	b.Name = bucket // in-case parsing failed for some reason, make sure bucket name is not empty.
 	if err != nil && !errors.Is(err, errConfigNotFound) {
 		return b, err
 	}

cmd/bucket-policy-handlers_test.go

@@ -289,7 +289,7 @@ func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			policyLen:          len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)),
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusNotFound,
+			expectedRespStatus: http.StatusBadRequest,
 		},
 		// Test case - 10.
 		// Existent bucket with policy with Version field empty.
@@ -463,7 +463,7 @@ func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string
 			accessKey:            credentials.AccessKey,
 			secretKey:            credentials.SecretKey,
 			expectedBucketPolicy: "",
-			expectedRespStatus:   http.StatusNotFound,
+			expectedRespStatus:   http.StatusBadRequest,
 		},
 	}
 	// Iterating over the cases, fetching the policy and validating the response.
@@ -686,7 +686,7 @@ func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName str
 			bucketName:         ".invalid-bucket-name",
 			accessKey:          credentials.AccessKey,
 			secretKey:          credentials.SecretKey,
-			expectedRespStatus: http.StatusNotFound,
+			expectedRespStatus: http.StatusBadRequest,
 		},
 	}
 	// Iterating over the cases and deleting the bucket policy and then asserting response.

cmd/bucket-replication-utils.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"net/http"
 	"net/url"
@@ -31,6 +32,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/logger"
 )
 
 //go:generate msgp -file=$GOFILE
@@ -180,6 +182,7 @@ type replicateTargetDecision struct {
 	Synchronous bool   // Synchronous replication configured.
 	Arn         string // ARN of replication target
 	ID          string
+	Tgt         *TargetClient
 }
 
 func (t *replicateTargetDecision) String() string {
@@ -288,7 +291,7 @@ var errInvalidReplicateDecisionFormat = fmt.Errorf("ReplicateDecision has invali
 
 // parse k-v pairs of target ARN to stringified ReplicateTargetDecision delimited by ',' into a
 // ReplicateDecision struct
-func parseReplicateDecision(s string) (r ReplicateDecision, err error) {
+func parseReplicateDecision(ctx context.Context, bucket, s string) (r ReplicateDecision, err error) {
 	r = ReplicateDecision{
 		targetsMap: make(map[string]replicateTargetDecision),
 	}
@@ -317,7 +320,13 @@ func parseReplicateDecision(s string) (r ReplicateDecision, err error) {
 		if err != nil {
 			return r, err
 		}
-		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: replicate, Synchronous: sync, Arn: tgt[2], ID: tgt[3]}
+		tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, slc[0])
+		if tgtClnt == nil {
+			// Skip stale targets if any and log them to be missing atleast once.
+			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, slc[0]), slc[0])
+			// We save the targetDecision even when its not configured or stale.
+		}
+		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: replicate, Synchronous: sync, Arn: tgt[2], ID: tgt[3], Tgt: tgtClnt}
 	}
 	return
 }
@@ -340,8 +349,6 @@ type ReplicationState struct {
 // Equal returns true if replication state is identical for version purge statuses and (replica)tion statuses.
 func (rs *ReplicationState) Equal(o ReplicationState) bool {
 	return rs.ReplicaStatus == o.ReplicaStatus &&
-		rs.ReplicaTimeStamp.Equal(o.ReplicaTimeStamp) &&
-		rs.ReplicationTimeStamp.Equal(o.ReplicationTimeStamp) &&
 		rs.ReplicationStatusInternal == o.ReplicationStatusInternal &&
 		rs.VersionPurgeStatusInternal == o.VersionPurgeStatusInternal
 }
@@ -357,7 +364,10 @@ func (rs *ReplicationState) CompositeReplicationStatus() (st replication.StatusT
 			replStatus := getCompositeReplicationStatus(rs.Targets)
 			// return REPLICA status if replica received timestamp is later than replication timestamp
 			// provided object replication completed for all targets.
-			if !rs.ReplicaTimeStamp.Equal(timeSentinel) && replStatus == replication.Completed && rs.ReplicaTimeStamp.After(rs.ReplicationTimeStamp) {
+			if rs.ReplicaTimeStamp.Equal(timeSentinel) || rs.ReplicaTimeStamp.IsZero() {
+				return replStatus
+			}
+			if replStatus == replication.Completed && rs.ReplicaTimeStamp.After(rs.ReplicationTimeStamp) {
 				return rs.ReplicaStatus
 			}
 			return replStatus
@@ -720,6 +730,7 @@ type TargetReplicationResyncStatus struct {
 	// Last bucket/object replicated.
 	Bucket string `json:"-" msg:"bkt"`
 	Object string `json:"-" msg:"obj"`
+	Error  error  `json:"-" msg:"-"`
 }
 
 // BucketReplicationResyncStatus captures current replication resync status

cmd/bucket-replication-utils_test.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"context"
 	"testing"
 
 	"github.com/minio/minio/internal/bucket/replication"
@@ -183,7 +184,7 @@ var parseReplicationDecisionTest = []struct {
 
 func TestParseReplicateDecision(t *testing.T) {
 	for i, test := range parseReplicationDecisionTest {
-		dsc, err := parseReplicateDecision(test.expDsc.String())
+		dsc, err := parseReplicateDecision(context.Background(), "bucket", test.expDsc.String())
 		if err != nil {
 			if test.expErr != err {
 				t.Errorf("Test%d (%s): Expected parse error got %t , want %t", i+1, test.name, err, test.expErr)

cmd/bucket-replication.go

@@ -27,9 +27,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
-	"os"
 	"path"
-	"path/filepath"
 	"reflect"
 	"strings"
 	"sync"
@@ -51,7 +49,6 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/pkg/workers"
 	"github.com/zeebo/xxh3"
 )
 
@@ -428,7 +425,7 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 
 	rcfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil || rcfg == nil {
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, fmt.Errorf("unable to obtain replication config for bucket: %s: err: %s", bucket, err), bucket)
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -443,9 +440,10 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		})
 		return
 	}
-	dsc, err := parseReplicateDecision(dobj.ReplicationState.ReplicateDecisionStr)
+	dsc, err := parseReplicateDecision(ctx, bucket, dobj.ReplicationState.ReplicateDecisionStr)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, fmt.Errorf("unable to parse replication decision parameters for bucket: %s, err: %s, decision: %s",
+			bucket, err, dobj.ReplicationState.ReplicateDecisionStr), dobj.ReplicationState.ReplicateDecisionStr)
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -467,7 +465,6 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		globalReplicationPool.queueMRFSave(dobj.ToMRFEntry())
-		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", dobj.ObjectName, bucket, dobj.TargetArn))
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -489,38 +486,23 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	var rinfos replicatedInfos
 	rinfos.Targets = make([]replicatedTargetInfo, len(dsc.targetsMap))
 	idx := -1
-	for tgtArn := range dsc.targetsMap {
+	for _, tgtEntry := range dsc.targetsMap {
 		idx++
-		tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, tgtArn)
-		if tgt == nil {
-			logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtArn))
-			sendEvent(eventArgs{
-				BucketName: bucket,
-				Object: ObjectInfo{
-					Bucket:       bucket,
-					Name:         dobj.ObjectName,
-					VersionID:    versionID,
-					DeleteMarker: dobj.DeleteMarker,
-				},
-				UserAgent: "Internal: [Replication]",
-				Host:      globalLocalNodeName,
-				EventName: event.ObjectReplicationNotTracked,
-			})
+		if tgtEntry.Tgt == nil {
 			continue
 		}
-		if tgt := dsc.targetsMap[tgtArn]; !tgt.Replicate {
+		if !tgtEntry.Replicate {
 			continue
 		}
 		// if dobj.TargetArn is not empty string, this is a case of specific target being re-synced.
-		if dobj.TargetArn != "" && dobj.TargetArn != tgt.ARN {
+		if dobj.TargetArn != "" && dobj.TargetArn != tgtEntry.Arn {
 			continue
 		}
 		wg.Add(1)
 		go func(index int, tgt *TargetClient) {
 			defer wg.Done()
-			rinfo := replicateDeleteToTarget(ctx, dobj, tgt)
-			rinfos.Targets[index] = rinfo
-		}(idx, tgt)
+			rinfos.Targets[index] = replicateDeleteToTarget(ctx, dobj, tgt)
+		}(idx, tgtEntry.Tgt)
 	}
 	wg.Wait()
 
@@ -650,6 +632,9 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 				return
 			}
 		default:
+			if err != nil && minio.IsNetworkOrHostDown(err, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
+				globalBucketTargetSys.markOffline(tgt.EndpointURL())
+			}
 			// mark delete marker replication as failed if target cluster not ready to receive
 			// this request yet (object version not replicated yet)
 			if err != nil && !toi.ReplicationReady {
@@ -674,6 +659,9 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 			rinfo.VersionPurgeStatus = Failed
 		}
 		logger.LogIf(ctx, fmt.Errorf("Unable to replicate delete marker to %s/%s(%s): %s", tgt.Bucket, dobj.ObjectName, versionID, rmErr))
+		if rmErr != nil && minio.IsNetworkOrHostDown(rmErr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
+			globalBucketTargetSys.markOffline(tgt.EndpointURL())
+		}
 	} else {
 		if dobj.VersionID == "" {
 			rinfo.ReplicationStatus = replication.Completed
@@ -1006,7 +994,6 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			Host:       globalLocalNodeName,
 		})
 		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
-		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, ri.TargetArn))
 		return
 	}
 	ctx = lkctx.Context()
@@ -1018,7 +1005,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	for i, tgtArn := range tgtArns {
 		tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, tgtArn)
 		if tgt == nil {
-			logger.LogIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtArn))
+			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtArn), tgtArn)
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
@@ -1157,7 +1144,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
-			logger.LogIf(ctx, fmt.Errorf("unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
+			logger.LogOnceIf(ctx, fmt.Errorf("unable to read source object %s/%s(%s): %w", bucket, object, objInfo.VersionID, err), object+":"+objInfo.VersionID)
 		}
 		return
 	}
@@ -1236,7 +1223,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 	r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts)
 	if objInfo.isMultipart() {
-		if err := replicateObjectWithMultipart(ctx, c, tgt.Bucket, object,
+		if err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object,
 			r, objInfo, putOpts); err != nil {
 			if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
@@ -1251,6 +1238,9 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			}
 		}
 	}
+	if err != nil && minio.IsNetworkOrHostDown(err, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
+		globalBucketTargetSys.markOffline(tgt.EndpointURL())
+	}
 	return
 }
 
@@ -1394,6 +1384,10 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 	// if target returns error other than NoSuchKey, defer replication attempt
 	if cerr != nil {
+		if minio.IsNetworkOrHostDown(cerr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
+			globalBucketTargetSys.markOffline(tgt.EndpointURL())
+		}
+
 		errResp := minio.ToErrorResponse(cerr)
 		switch errResp.Code {
 		case "NoSuchKey", "NoSuchVersion", "SlowDownRead":
@@ -1465,7 +1459,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		}
 		r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts)
 		if objInfo.isMultipart() {
-			if err := replicateObjectWithMultipart(ctx, c, tgt.Bucket, object,
+			if err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object,
 				r, objInfo, putOpts); err != nil {
 				if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
@@ -1484,6 +1478,9 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				}
 			}
 		}
+		if err != nil && minio.IsNetworkOrHostDown(err, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
+			globalBucketTargetSys.markOffline(tgt.EndpointURL())
+		}
 	}
 	return
 }
@@ -1492,7 +1489,9 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 	var uploadedParts []minio.CompletePart
 	// new multipart must not set mtime as it may lead to erroneous cleanups at various intervals.
 	opts.Internal.SourceMTime = time.Time{} // this value is saved properly in CompleteMultipartUpload()
-	uploadID, err := c.NewMultipartUpload(context.Background(), bucket, object, opts)
+	nctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+	defer cancel()
+	uploadID, err := c.NewMultipartUpload(nctx, bucket, object, opts)
 	if err != nil {
 		return err
 	}
@@ -1502,12 +1501,15 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 			// block and abort remote upload upon failure.
 			attempts := 1
 			for attempts <= 3 {
-				aerr := c.AbortMultipartUpload(ctx, bucket, object, uploadID)
+				actx, acancel := context.WithTimeout(ctx, time.Minute)
+				aerr := c.AbortMultipartUpload(actx, bucket, object, uploadID)
 				if aerr == nil {
+					acancel()
 					return
 				}
-				logger.LogIf(ctx,
-					fmt.Errorf("Trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
+				acancel()
+				logger.LogIf(actx,
+					fmt.Errorf("trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
 						humanize.Ordinal(attempts), uploadID, bucket, object, aerr))
 				attempts++
 				time.Sleep(time.Second)
@@ -1542,7 +1544,9 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 			ETag:       pInfo.ETag,
 		})
 	}
-	_, err = c.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{
+	cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute)
+	defer ccancel()
+	_, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{
 		Internal: minio.AdvancedPutOptions{
 			SourceMTime: objInfo.ModTime,
 			// always set this to distinguish between `mc mirror` replication and serverside
@@ -1641,10 +1645,12 @@ type ReplicationPool struct {
 	ctx      context.Context
 	priority string
 	mu       sync.RWMutex
+	mrfMU    sync.Mutex
 	resyncer *replicationResyncer
 
 	// workers:
 	workers         []chan ReplicationWorkerOperation
+	lrgworkers      []chan ReplicationWorkerOperation
 	existingWorkers chan ReplicationWorkerOperation
 
 	// mrf:
@@ -1653,7 +1659,6 @@ type ReplicationPool struct {
 	mrfSaveCh       chan MRFReplicateEntry
 	mrfStopCh       chan struct{}
 	mrfWorkerSize   int
-	saveStateCh     chan struct{}
 }
 
 // ReplicationWorkerOperation is a shared interface of replication operations.
@@ -1679,6 +1684,9 @@ const (
 
 	// MRFWorkerAutoDefault is default number of mrf workers for "auto" mode
 	MRFWorkerAutoDefault = 4
+
+	// LargeWorkerCount is default number of workers assigned to large uploads ( >= 128MiB)
+	LargeWorkerCount = 10
 )
 
 // NewReplicationPool creates a pool of replication workers of specified size
@@ -1702,18 +1710,20 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 
 	pool := &ReplicationPool{
 		workers:         make([]chan ReplicationWorkerOperation, 0, workers),
+		lrgworkers:      make([]chan ReplicationWorkerOperation, 0, LargeWorkerCount),
 		existingWorkers: make(chan ReplicationWorkerOperation, 100000),
+
 		mrfReplicaCh:    make(chan ReplicationWorkerOperation, 100000),
 		mrfWorkerKillCh: make(chan struct{}, failedWorkers),
 		resyncer:        newresyncer(),
 		mrfSaveCh:       make(chan MRFReplicateEntry, 100000),
 		mrfStopCh:       make(chan struct{}, 1),
-		saveStateCh:     make(chan struct{}, 1),
 		ctx:             ctx,
 		objLayer:        o,
 		priority:        priority,
 	}
 
+	pool.AddLargeWorkers()
 	pool.ResizeWorkers(workers, 0)
 	pool.ResizeFailedWorkers(failedWorkers)
 	go pool.AddWorker(pool.existingWorkers, nil)
@@ -1785,6 +1795,43 @@ func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opT
 	}
 }
 
+// AddLargeWorkers adds a static number of workers to handle large uploads
+func (p *ReplicationPool) AddLargeWorkers() {
+	for i := 0; i < LargeWorkerCount; i++ {
+		p.lrgworkers = append(p.lrgworkers, make(chan ReplicationWorkerOperation, 100000))
+		i := i
+		go p.AddLargeWorker(p.lrgworkers[i])
+	}
+	go func() {
+		<-p.ctx.Done()
+		for i := 0; i < LargeWorkerCount; i++ {
+			close(p.lrgworkers[i])
+		}
+	}()
+}
+
+// AddLargeWorker adds a replication worker to the static pool for large uploads.
+func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation) {
+	for {
+		select {
+		case <-p.ctx.Done():
+			return
+		case oi, ok := <-input:
+			if !ok {
+				return
+			}
+			switch v := oi.(type) {
+			case ReplicateObjectInfo:
+				replicateObject(p.ctx, v, p.objLayer)
+			case DeletedObjectReplicationInfo:
+				replicateDelete(p.ctx, v, p.objLayer)
+			default:
+				logger.LogOnceIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type")
+			}
+		}
+	}
+}
+
 // ActiveWorkers returns the number of active workers handling replication traffic.
 func (p *ReplicationPool) ActiveWorkers() int {
 	return int(atomic.LoadInt32(&p.activeWorkers))
@@ -1861,9 +1908,14 @@ func (p *ReplicationPool) ResizeFailedWorkers(n int) {
 	}
 }
 
+const (
+	minLargeObjSize = 128 * humanize.MiByte // 128MiB
+)
+
 // getWorkerCh gets a worker channel deterministically based on bucket and object names.
 // Must be able to grab read lock from p.
-func (p *ReplicationPool) getWorkerCh(bucket, object string) chan<- ReplicationWorkerOperation {
+
+func (p *ReplicationPool) getWorkerCh(bucket, object string, sz int64) chan<- ReplicationWorkerOperation {
 	h := xxh3.HashString(bucket + object)
 	p.mu.RLock()
 	defer p.mu.RUnlock()
@@ -1877,15 +1929,27 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 	if p == nil {
 		return
 	}
+	// if object is large, queue it to a static set of large workers
+	if ri.Size >= int64(minLargeObjSize) {
+		h := xxh3.HashString(ri.Bucket + ri.Name)
+		select {
+		case <-p.ctx.Done():
+		case p.lrgworkers[h%LargeWorkerCount] <- ri:
+		default:
+			globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
+		}
+		return
+	}
+
 	var ch, healCh chan<- ReplicationWorkerOperation
 	switch ri.OpType {
 	case replication.ExistingObjectReplicationType:
 		ch = p.existingWorkers
 	case replication.HealReplicationType:
 		ch = p.mrfReplicaCh
-		healCh = p.getWorkerCh(ri.Name, ri.Bucket)
+		healCh = p.getWorkerCh(ri.Name, ri.Bucket, ri.Size)
 	default:
-		ch = p.getWorkerCh(ri.Name, ri.Bucket)
+		ch = p.getWorkerCh(ri.Name, ri.Bucket, ri.Size)
 	}
 	if ch == nil && healCh == nil {
 		return
@@ -1945,7 +2009,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 	case replication.HealReplicationType:
 		fallthrough
 	default:
-		ch = p.getWorkerCh(doi.Bucket, doi.ObjectName)
+		ch = p.getWorkerCh(doi.Bucket, doi.ObjectName, 0)
 	}
 
 	select {
@@ -2403,12 +2467,12 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	objInfoCh := make(chan ObjectInfo)
 	cfg, err := getReplicationConfig(ctx, opts.bucket)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Replication resync of %s for arn %s failed with %w", opts.bucket, opts.arn, err))
+		logger.LogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed with %w", opts.bucket, opts.arn, err))
 		return
 	}
 	tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, opts.bucket)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Replication resync of %s for arn %s failed  %w", opts.bucket, opts.arn, err))
+		logger.LogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed  %w", opts.bucket, opts.arn, err))
 		return
 	}
 	rcfg := replicationConfig{
@@ -2421,12 +2485,12 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 			TargetArn: opts.arn,
 		})
 	if len(tgtArns) != 1 {
-		logger.LogIf(ctx, fmt.Errorf("Replication resync failed for %s - arn specified %s is missing in the replication config", opts.bucket, opts.arn))
+		logger.LogIf(ctx, fmt.Errorf("replication resync failed for %s - arn specified %s is missing in the replication config", opts.bucket, opts.arn))
 		return
 	}
 	tgt := globalBucketTargetSys.GetRemoteTargetClient(ctx, opts.arn)
 	if tgt == nil {
-		logger.LogIf(ctx, fmt.Errorf("Replication resync failed for %s - target could not be created for arn %s", opts.bucket, opts.arn))
+		logger.LogIf(ctx, fmt.Errorf("replication resync failed for %s - target could not be created for arn %s", opts.bucket, opts.arn))
 		return
 	}
 	// mark resync status as resync started
@@ -2449,7 +2513,83 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	if st.ResyncStatus == ResyncStarted || st.ResyncStatus == ResyncFailed {
 		lastCheckpoint = st.Object
 	}
-	workers, err := workers.New(resyncParallelRoutines)
+	workers := make([]chan ReplicateObjectInfo, resyncParallelRoutines)
+	resultCh := make(chan TargetReplicationResyncStatus, 1)
+	defer close(resultCh)
+	var wg sync.WaitGroup
+	for i := 0; i < resyncParallelRoutines; i++ {
+		wg.Add(1)
+		workers[i] = make(chan ReplicateObjectInfo, 100)
+		i := i
+		go func(ctx context.Context, idx int) {
+			defer wg.Done()
+			for roi := range workers[idx] {
+				select {
+				case <-ctx.Done():
+					return
+				case <-s.resyncCancelCh:
+				default:
+				}
+				traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
+				if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
+					versionID := ""
+					dmVersionID := ""
+					if roi.VersionPurgeStatus.Empty() {
+						dmVersionID = roi.VersionID
+					} else {
+						versionID = roi.VersionID
+					}
+
+					doi := DeletedObjectReplicationInfo{
+						DeletedObject: DeletedObject{
+							ObjectName:            roi.Name,
+							DeleteMarkerVersionID: dmVersionID,
+							VersionID:             versionID,
+							ReplicationState:      roi.getReplicationState(),
+							DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
+							DeleteMarker:          roi.DeleteMarker,
+						},
+						Bucket:    roi.Bucket,
+						OpType:    replication.ExistingObjectReplicationType,
+						EventType: ReplicateExistingDelete,
+					}
+					replicateDelete(ctx, doi, objectAPI)
+				} else {
+					roi.OpType = replication.ExistingObjectReplicationType
+					roi.EventType = ReplicateExisting
+					replicateObject(ctx, roi, objectAPI)
+				}
+				_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
+					VersionID: roi.VersionID,
+					Internal: minio.AdvancedGetOptions{
+						ReplicationProxyRequest: "false",
+					},
+				})
+				st := TargetReplicationResyncStatus{
+					Object: roi.Name,
+					Bucket: roi.Bucket,
+				}
+				if err != nil {
+					if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
+						st.ReplicatedCount++
+					} else {
+						st.FailedCount++
+					}
+				} else {
+					st.ReplicatedCount++
+					st.ReplicatedSize += roi.Size
+				}
+				traceFn(err)
+				select {
+				case <-ctx.Done():
+					return
+				case <-s.resyncCancelCh:
+					return
+				case resultCh <- st:
+				}
+			}
+		}(ctx, i)
+	}
 	for obj := range objInfoCh {
 		select {
 		case <-s.resyncCancelCh:
@@ -2463,71 +2603,30 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 			continue
 		}
 		lastCheckpoint = ""
-		obj := obj
-		workers.Take()
-		go func() {
-			defer workers.Give()
-			roi := getHealReplicateObjectInfo(obj, rcfg)
-			if !roi.ExistingObjResync.mustResync() {
-				return
-			}
-			traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
-			if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
-				versionID := ""
-				dmVersionID := ""
-				if roi.VersionPurgeStatus.Empty() {
-					dmVersionID = roi.VersionID
-				} else {
-					versionID = roi.VersionID
-				}
-
-				doi := DeletedObjectReplicationInfo{
-					DeletedObject: DeletedObject{
-						ObjectName:            roi.Name,
-						DeleteMarkerVersionID: dmVersionID,
-						VersionID:             versionID,
-						ReplicationState:      roi.getReplicationState(),
-						DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
-						DeleteMarker:          roi.DeleteMarker,
-					},
-					Bucket:    roi.Bucket,
-					OpType:    replication.ExistingObjectReplicationType,
-					EventType: ReplicateExistingDelete,
-				}
-				replicateDelete(ctx, doi, objectAPI)
-			} else {
-				roi.OpType = replication.ExistingObjectReplicationType
-				roi.EventType = ReplicateExisting
-				replicateObject(ctx, roi, objectAPI)
-			}
-			_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
-				VersionID: roi.VersionID,
-				Internal: minio.AdvancedGetOptions{
-					ReplicationProxyRequest: "false",
-				},
-			})
-			st := TargetReplicationResyncStatus{
-				Object: roi.Name,
-				Bucket: roi.Bucket,
-			}
-			success := true
-			if err != nil {
-				if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
-					st.ReplicatedCount++
-				} else {
-					st.FailedCount++
-					success = false
-				}
-			} else {
-				st.ReplicatedCount++
-				st.ReplicatedSize += roi.Size
-			}
-			s.incStats(st, opts)
-			traceFn(err)
-			globalSiteResyncMetrics.updateMetric(roi, success, opts.resyncID)
-		}()
+		roi := getHealReplicateObjectInfo(obj, rcfg)
+		if !roi.ExistingObjResync.mustResync() {
+			continue
+		}
+		select {
+		case <-s.resyncCancelCh:
+			return
+		case <-ctx.Done():
+			return
+		default:
+			h := xxh3.HashString(roi.Bucket + roi.Name)
+			workers[h%uint64(resyncParallelRoutines)] <- roi
+		}
 	}
-	workers.Wait()
+	for i := 0; i < resyncParallelRoutines; i++ {
+		close(workers[i])
+	}
+	go func() {
+		for r := range resultCh {
+			s.incStats(r, opts)
+			globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
+		}
+	}()
+	wg.Wait()
 	resyncStatus = ResyncCompleted
 }
 
@@ -2949,13 +3048,11 @@ func (p *ReplicationPool) persistMRF() {
 		return
 	}
 
-	var mu sync.Mutex
 	entries := make(map[string]MRFReplicateEntry)
 	mTimer := time.NewTimer(mrfSaveInterval)
 	defer mTimer.Stop()
+
 	saveMRFToDisk := func(drain bool) {
-		mu.Lock()
-		defer mu.Unlock()
 		if len(entries) == 0 {
 			return
 		}
@@ -2967,12 +3064,16 @@ func (p *ReplicationPool) persistMRF() {
 				entries[e.versionID] = e
 			}
 		}
+
 		// queue all entries for healing before overwriting the node mrf file
-		p.queueMRFHeal()
+		if !contextCanceled(p.ctx) {
+			p.queueMRFHeal()
+		}
 
 		if err := p.saveMRFEntries(cctx, entries); err != nil {
 			logger.LogOnceIf(p.ctx, fmt.Errorf("unable to persist replication failures to disk:%w", err), string(replicationSubsystem))
 		}
+
 		entries = make(map[string]MRFReplicateEntry)
 	}
 	for {
@@ -2985,21 +3086,14 @@ func (p *ReplicationPool) persistMRF() {
 			close(p.mrfSaveCh)
 			saveMRFToDisk(true)
 			return
-		case <-p.saveStateCh:
-			saveMRFToDisk(true)
-			return
 		case e, ok := <-p.mrfSaveCh:
 			if !ok {
 				return
 			}
-			var cnt int
-			mu.Lock()
-			entries[e.versionID] = e
-			cnt = len(entries)
-			mu.Unlock()
-			if cnt >= mrfMaxEntries {
+			if len(entries) >= mrfMaxEntries {
 				saveMRFToDisk(true)
 			}
+			entries[e.versionID] = e
 		}
 	}
 }
@@ -3033,6 +3127,7 @@ func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string
 	if len(entries) == 0 {
 		return nil
 	}
+
 	v := MRFReplicateEntries{
 		Entries: entries,
 		Version: mrfMetaVersionV1,
@@ -3048,41 +3143,19 @@ func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string
 		return err
 	}
 
-	for _, diskPath := range globalEndpoints.LocalDisksPaths() {
-		// write to first drive
-		mrfDir := filepath.Join(diskPath, minioMetaBucket, replicationMRFDir)
-		mrfFileName := filepath.Join(mrfDir, globalLocalNodeNameHex+".bin")
-		if err := os.MkdirAll(mrfDir, 0o777); err != nil {
-			return err
+	for _, localDrive := range globalLocalDrives {
+		if err := localDrive.WriteAll(ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), buf); err == nil {
+			break
 		}
-		file, err := OpenFile(mrfFileName, os.O_CREATE|os.O_WRONLY|writeMode, 0o666)
-		if err != nil {
-			continue
-		}
-		defer file.Close()
-		if _, err = file.Write(buf); err != nil {
-			return err
-		}
-		break
 	}
 	return nil
 }
 
 // load mrf entries from disk
-func (p *ReplicationPool) loadMRF(fileName string) (re MRFReplicateEntries, e error) {
+func (p *ReplicationPool) loadMRF(data []byte) (re MRFReplicateEntries, e error) {
 	if !p.initialized() {
 		return re, nil
 	}
-	file, err := Open(fileName)
-	if err != nil {
-		return re, err
-	}
-	defer file.Close()
-
-	data, err := io.ReadAll(file)
-	if err != nil {
-		return re, err
-	}
 	if len(data) == 0 {
 		// Seems to be empty.
 		return re, nil
@@ -3102,7 +3175,7 @@ func (p *ReplicationPool) loadMRF(fileName string) (re MRFReplicateEntries, e er
 		return re, fmt.Errorf("replication mrf: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
 	}
 	// OK, parse data.
-	if _, err = re.UnmarshalMsg(data[4:]); err != nil {
+	if _, err := re.UnmarshalMsg(data[4:]); err != nil {
 		return re, err
 	}
 
@@ -3147,29 +3220,40 @@ func (p *ReplicationPool) processMRF() {
 
 // process sends error logs to the heal channel for an attempt to heal replication.
 func (p *ReplicationPool) queueMRFHeal() error {
+	p.mrfMU.Lock()
+	defer p.mrfMU.Unlock()
+
 	if !p.initialized() {
 		return errServerNotInitialized
 	}
 
-	for _, diskPath := range globalEndpoints.LocalDisksPaths() {
-		fileName := filepath.Join(diskPath, minioMetaBucket, replicationMRFDir, globalLocalNodeNameHex+".bin")
-		mrfRec, err := p.loadMRF(fileName)
+	for _, localDrive := range globalLocalDrives {
+		buf, err := localDrive.ReadAll(p.ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"))
 		if err != nil {
-			return err
+			continue
 		}
+
+		mrfRec, err := p.loadMRF(buf)
+		if err != nil {
+			continue
+		}
+
 		// finally delete the file after processing mrf entries
-		os.Remove(fileName)
+		localDrive.Delete(p.ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), DeleteOptions{})
 
 		// queue replication heal in a goroutine to avoid holding up mrf save routine
 		go func(mrfRec MRFReplicateEntries) {
 			for vID, e := range mrfRec.Entries {
+				ctx, cancel := context.WithTimeout(p.ctx, time.Second) // Do not waste more than a second on this.
 
-				oi, err := p.objLayer.GetObjectInfo(p.ctx, e.Bucket, e.Object, ObjectOptions{
+				oi, err := p.objLayer.GetObjectInfo(ctx, e.Bucket, e.Object, ObjectOptions{
 					VersionID: vID,
 				})
+				cancel()
 				if err != nil {
 					continue
 				}
+
 				QueueReplicationHeal(p.ctx, e.Bucket, oi, e.RetryCount)
 			}
 		}(mrfRec)
@@ -3265,11 +3349,14 @@ func (p *ReplicationPool) getMRF(ctx context.Context, bucket string) (ch chan ma
 	mrfCh := make(chan madmin.ReplicationMRF, 100)
 	go func() {
 		defer close(mrfCh)
-		for _, diskPath := range globalEndpoints.LocalDisksPaths() {
-			file := filepath.Join(diskPath, minioMetaBucket, replicationMRFDir, globalLocalNodeNameHex+".bin")
-			mrfRec, err := p.loadMRF(file)
+		for _, localDrive := range globalLocalDrives {
+			buf, err := localDrive.ReadAll(ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"))
 			if err != nil {
-				break
+				continue
+			}
+			mrfRec, err := p.loadMRF(buf)
+			if err != nil {
+				continue
 			}
 			for vID, e := range mrfRec.Entries {
 				if e.Bucket != bucket && bucket != "" {

cmd/bucket-targets.go

@@ -58,7 +58,7 @@ type epHealth struct {
 }
 
 // isOffline returns current liveness result of remote target. Add endpoint to
-// healthcheck map if missing and default to online status
+// healthCheck map if missing and default to online status
 func (sys *BucketTargetSys) isOffline(ep *url.URL) bool {
 	sys.hMutex.RLock()
 	defer sys.hMutex.RUnlock()
@@ -69,6 +69,16 @@ func (sys *BucketTargetSys) isOffline(ep *url.URL) bool {
 	return false
 }
 
+// markOffline sets endpoint to offline if network i/o timeout seen.
+func (sys *BucketTargetSys) markOffline(ep *url.URL) {
+	sys.hMutex.Lock()
+	defer sys.hMutex.Unlock()
+	if h, ok := sys.hc[ep.Host]; ok {
+		h.Online = false
+		sys.hc[ep.Host] = h
+	}
+}
+
 func (sys *BucketTargetSys) initHC(ep *url.URL) {
 	sys.hMutex.Lock()
 	sys.hc[ep.Host] = epHealth{
@@ -126,7 +136,7 @@ func (sys *BucketTargetSys) heartBeat(ctx context.Context) {
 	}
 }
 
-// periodically rebuild the healthcheck map from list of targets to clear
+// periodically rebuild the healthCheck map from list of targets to clear
 // out stale endpoints
 func (sys *BucketTargetSys) reloadHealthCheckers(ctx context.Context) {
 	m := make(map[string]epHealth)
@@ -362,7 +372,7 @@ func NewBucketTargetSys(ctx context.Context) *BucketTargetSys {
 		hc:            make(map[string]epHealth),
 		hcClient:      newHCClient(),
 	}
-	// reload healthcheck endpoints map periodically to remove stale endpoints from the map.
+	// reload healthCheck endpoints map periodically to remove stale endpoints from the map.
 	go func() {
 		rTimer := time.NewTimer(defaultHealthCheckReloadDuration)
 		defer rTimer.Stop()

cmd/common-main.go

@@ -95,6 +95,9 @@ func init() {
 
 	initGlobalContext()
 
+	globalBatchJobsMetrics = batchJobMetrics{metrics: make(map[string]*batchJobInfo)}
+	go globalBatchJobsMetrics.purgeJobMetrics()
+
 	t, _ := minioVersionToReleaseTime(Version)
 	if !t.IsZero() {
 		globalVersionUnix = uint64(t.Unix())
@@ -102,22 +105,6 @@ func init() {
 
 	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != ""
 
-	// Call to refresh will refresh names in cache.
-	go func() {
-		// Baremetal setups set DNS refresh window to 10 minutes.
-		t := time.NewTicker(10 * time.Minute)
-		defer t.Stop()
-		for {
-			select {
-			case <-t.C:
-				globalDNSCache.Refresh()
-
-			case <-GlobalContext.Done():
-				return
-			}
-		}
-	}()
-
 	console.SetColor("Debug", fcolor.New())
 
 	gob.Register(StorageErr(""))
@@ -188,6 +175,10 @@ func minioConfigToConsoleFeatures() {
 	if value := env.Get(config.EnvBrowserLoginAnimation, "on"); value != "" {
 		os.Setenv("CONSOLE_ANIMATED_LOGIN", value)
 	}
+	// Pass on the session duration environment variable, else we will default to 12 hours
+	if value := env.Get(config.EnvBrowserSessionDuration, ""); value != "" {
+		os.Setenv("CONSOLE_STS_DURATION", value)
+	}
 
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
@@ -443,6 +434,28 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)}
 
 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
+
+	// Check if we have configured a custom DNS cache TTL.
+	dnsTTL := ctx.Duration("dns-cache-ttl")
+	if dnsTTL <= 0 {
+		dnsTTL = 10 * time.Minute
+	}
+
+	// Call to refresh will refresh names in cache.
+	go func() {
+		// Baremetal setups set DNS refresh window up to dnsTTL duration.
+		t := time.NewTicker(dnsTTL)
+		defer t.Stop()
+		for {
+			select {
+			case <-t.C:
+				globalDNSCache.Refresh()
+
+			case <-GlobalContext.Done():
+				return
+			}
+		}
+	}()
 }
 
 type envKV struct {

cmd/config-common.go

@@ -61,7 +61,8 @@ type objectDeleter interface {
 
 func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string) error {
 	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{
-		DeletePrefix: true,
+		DeletePrefix:       true,
+		DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
 	})
 	if err != nil && isErrObjectNotFound(err) {
 		return errConfigNotFound

cmd/config-current.go

@@ -386,7 +386,7 @@ func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, o
 		}
 	default:
 		if config.LoggerSubSystems.Contains(subSys) {
-			if err := logger.ValidateSubSysConfig(s, subSys); err != nil {
+			if err := logger.ValidateSubSysConfig(ctx, s, subSys); err != nil {
 				return err
 			}
 		}
@@ -440,7 +440,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err))
 	}
 	if err == nil && dnsURL != "" {
-		bootstrapTrace("initialize remote bucket DNS store")
+		bootstrapTraceMsg("initialize remote bucket DNS store")
 		globalDNSConfig, err = dns.NewOperatorDNS(dnsURL,
 			dns.Authentication(dnsUser, dnsPass),
 			dns.RootCAs(globalRootCAs))
@@ -455,7 +455,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 
 	if etcdCfg.Enabled {
-		bootstrapTrace("initialize etcd store")
+		bootstrapTraceMsg("initialize etcd store")
 		globalEtcdClient, err = etcd.New(etcdCfg)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err))
@@ -514,19 +514,19 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	transport := NewHTTPTransport()
 
-	bootstrapTrace("initialize the event notification targets")
+	bootstrapTraceMsg("initialize the event notification targets")
 	globalNotifyTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
 	}
 
-	bootstrapTrace("initialize the lambda targets")
+	bootstrapTraceMsg("initialize the lambda targets")
 	globalLambdaTargetList, err = lambda.FetchEnabledTargets(GlobalContext, s, transport)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err))
 	}
 
-	bootstrapTrace("applying the dynamic configuration")
+	bootstrapTraceMsg("applying the dynamic configuration")
 	// Apply dynamic config values
 	if err := applyDynamicConfig(ctx, objAPI, s); err != nil {
 		logger.LogIf(ctx, err)
@@ -575,7 +575,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		scannerCycle.Store(scannerCfg.Cycle)
 		logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
 	case config.LoggerWebhookSubSys:
-		loggerCfg, err := logger.LookupConfigForSubSys(s, config.LoggerWebhookSubSys)
+		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.LoggerWebhookSubSys)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to load logger webhook config: %w", err))
 		}
@@ -592,7 +592,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
 		}
 	case config.AuditWebhookSubSys:
-		loggerCfg, err := logger.LookupConfigForSubSys(s, config.AuditWebhookSubSys)
+		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.AuditWebhookSubSys)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to load audit webhook config: %w", err))
 		}
@@ -610,7 +610,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
 		}
 	case config.AuditKafkaSubSys:
-		loggerCfg, err := logger.LookupConfigForSubSys(s, config.AuditKafkaSubSys)
+		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.AuditKafkaSubSys)
 		if err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to load audit kafka config: %w", err))
 		}
@@ -786,13 +786,13 @@ func getValidConfig(objAPI ObjectLayer) (config.Config, error) {
 // from env if found and valid
 // data is optional. If nil it will be loaded from backend.
 func loadConfig(objAPI ObjectLayer, data []byte) error {
-	bootstrapTrace("load the configuration")
+	bootstrapTraceMsg("load the configuration")
 	srvCfg, err := readServerConfig(GlobalContext, objAPI, data)
 	if err != nil {
 		return err
 	}
 
-	bootstrapTrace("lookup the configuration")
+	bootstrapTraceMsg("lookup the configuration")
 	// Override any values from ENVs.
 	lookupConfigs(srvCfg, objAPI)
 

cmd/config-migrate.go

@@ -2423,7 +2423,7 @@ func migrateV27ToV28() error {
 // Migrates ${HOME}/.minio/config.json to '<export_path>/.minio.sys/config/config.json'
 // if etcd is configured then migrates /config/config.json to '<export_path>/.minio.sys/config/config.json'
 func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
-	bootstrapTrace("migrate config to .minio.sys/config/config.json")
+	bootstrapTraceMsg("migrate config to .minio.sys/config/config.json")
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 

cmd/config.go

@@ -97,7 +97,8 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
 func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) error {
 	historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix)
 	_, err := objAPI.DeleteObject(ctx, minioMetaBucket, historyFile, ObjectOptions{
-		DeletePrefix: true,
+		DeletePrefix:       true,
+		DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
 	})
 	return err
 }
@@ -196,10 +197,10 @@ func NewConfigSys() *ConfigSys {
 
 // Initialize and load config from remote etcd or local config directory
 func initConfig(objAPI ObjectLayer) (err error) {
-	bootstrapTrace("load the configuration")
+	bootstrapTraceMsg("load the configuration")
 	defer func() {
 		if err != nil {
-			bootstrapTrace(fmt.Sprintf("loading configuration failed: %v", err))
+			bootstrapTraceMsg(fmt.Sprintf("loading configuration failed: %v", err))
 		}
 	}()
 
@@ -212,7 +213,7 @@ func initConfig(objAPI ObjectLayer) (err error) {
 		return err
 	}
 
-	bootstrapTrace("lookup the configuration")
+	bootstrapTraceMsg("lookup the configuration")
 
 	// Override any values from ENVs.
 	lookupConfigs(srvCfg, objAPI)

cmd/data-scanner-metric.go

@@ -142,14 +142,14 @@ func (p *scannerMetrics) incTime(s scannerMetric, d time.Duration) {
 // timeILM times an ILM action.
 // lifecycle.NoneAction is ignored.
 // Use for s < scannerMetricLastRealtime
-func (p *scannerMetrics) timeILM(a lifecycle.Action) func() {
+func (p *scannerMetrics) timeILM(a lifecycle.Action) func(versions uint64) {
 	if a == lifecycle.NoneAction || a >= lifecycle.ActionCount {
-		return func() {}
+		return func(_ uint64) {}
 	}
 	startTime := time.Now()
-	return func() {
+	return func(versions uint64) {
 		duration := time.Since(startTime)
-		atomic.AddUint64(&p.actions[a], 1)
+		atomic.AddUint64(&p.actions[a], versions)
 		p.actionsLatency[a].add(duration)
 	}
 }

cmd/data-scanner.go

@@ -550,7 +550,6 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				dst = &dataUsageEntry{Compacted: false}
 			}
 			if err := f.scanFolder(ctx, folder, dst); err != nil {
-				logger.LogIf(ctx, err)
 				return
 			}
 			if !into.Compacted {
@@ -930,7 +929,15 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q Initial scan: %v\n", i.objectPath(), lcEvt.Action)
 		}
 	}
-	defer globalScannerMetrics.timeILM(lcEvt.Action)()
+	defer func() {
+		if applied {
+			numVersions := uint64(1)
+			if lcEvt.Action == lifecycle.DeleteAllVersionsAction {
+				numVersions = uint64(oi.NumVersions)
+			}
+			globalScannerMetrics.timeILM(lcEvt.Action)(numVersions)
+		}
+	}()
 
 	switch lcEvt.Action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction, lifecycle.DeleteAllVersionsAction:
@@ -990,7 +997,6 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 
 	versioned := vcfg != nil && vcfg.Versioned(i.objectPath())
 
-	// current version + most recent lim noncurrent versions
 	objectInfos := make([]ObjectInfo, 0, len(fivs))
 
 	if i.lifeCycle == nil {
@@ -1010,6 +1016,10 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 	}
 
 	overflowVersions := fivs[lim+1:]
+	// Retain the current version + most recent lim noncurrent versions
+	for _, fi := range fivs[:lim+1] {
+		objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
+	}
 
 	toDel := make([]ObjectToDelete, 0, len(overflowVersions))
 	for _, fi := range overflowVersions {
@@ -1178,10 +1188,10 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 	if lcEvent.Action.DeleteVersioned() {
 		opts.VersionID = obj.VersionID
 	}
-	if opts.VersionID == "" {
-		opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
-		opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
-	}
+
+	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name)
+	opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name)
+
 	if lcEvent.Action.DeleteAll() {
 		opts.DeletePrefix = true
 	}

cmd/data-scanner_test.go

@@ -0,0 +1,137 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"encoding/xml"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/minio/minio/internal/bucket/lifecycle"
+	"github.com/minio/minio/internal/bucket/versioning"
+)
+
+func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) {
+	objAPI, disks, err := prepareErasure(context.Background(), 8)
+	if err != nil {
+		t.Fatalf("Failed to initialize object layer: %v", err)
+	}
+	defer removeRoots(disks)
+	setObjectLayer(objAPI)
+	globalBucketMetadataSys = NewBucketMetadataSys()
+	globalBucketObjectLockSys = &BucketObjectLockSys{}
+	globalBucketVersioningSys = &BucketVersioningSys{}
+	globalExpiryState = newExpiryState()
+	var wg sync.WaitGroup
+	wg.Add(1)
+	expired := make([]ObjectToDelete, 0, 5)
+	go func() {
+		defer wg.Done()
+		for t := range globalExpiryState.byNewerNoncurrentCh {
+			expired = append(expired, t.versions...)
+		}
+	}()
+	lc := lifecycle.Lifecycle{
+		Rules: []lifecycle.Rule{
+			{
+				ID:     "max-versions",
+				Status: "Enabled",
+				NoncurrentVersionExpiration: lifecycle.NoncurrentVersionExpiration{
+					NewerNoncurrentVersions: 1,
+				},
+			},
+		},
+	}
+	lcXML, err := xml.Marshal(lc)
+	if err != nil {
+		t.Fatalf("Failed to marshal lifecycle config: %v", err)
+	}
+	vcfg := versioning.Versioning{
+		Status: "Enabled",
+	}
+	vcfgXML, err := xml.Marshal(vcfg)
+	if err != nil {
+		t.Fatalf("Failed to marshal versioning config: %v", err)
+	}
+
+	bucket := "bucket"
+	obj := "obj-1"
+	now := time.Now()
+	meta := BucketMetadata{
+		Name:                      bucket,
+		Created:                   now,
+		LifecycleConfigXML:        lcXML,
+		VersioningConfigXML:       vcfgXML,
+		VersioningConfigUpdatedAt: now,
+		LifecycleConfigUpdatedAt:  now,
+		lifecycleConfig:           &lc,
+		versioningConfig:          &vcfg,
+	}
+	globalBucketMetadataSys.Set(bucket, meta)
+	item := scannerItem{
+		Path:       obj,
+		bucket:     bucket,
+		prefix:     "",
+		objectName: obj,
+		lifeCycle:  &lc,
+	}
+
+	modTime := time.Now()
+	uuids := make([]uuid.UUID, 5)
+	for i := range uuids {
+		uuids[i] = uuid.UUID([16]byte{15: uint8(i + 1)})
+	}
+	fivs := make([]FileInfo, 5)
+	for i := 0; i < 5; i++ {
+		fivs[i] = FileInfo{
+			Volume:      bucket,
+			Name:        obj,
+			VersionID:   uuids[i].String(),
+			IsLatest:    i == 0,
+			ModTime:     modTime.Add(-1 * time.Duration(i) * time.Minute),
+			Size:        1 << 10,
+			NumVersions: 5,
+		}
+	}
+	versioned := vcfg.Status == "Enabled"
+	wants := make([]ObjectInfo, 2)
+	for i, fi := range fivs[:2] {
+		wants[i] = fi.ToObjectInfo(bucket, obj, versioned)
+	}
+	gots, err := item.applyNewerNoncurrentVersionLimit(context.TODO(), objAPI, fivs)
+	if err != nil {
+		t.Fatalf("Failed with err: %v", err)
+	}
+	if len(gots) != len(wants) {
+		t.Fatalf("Expected %d objects but got %d", len(wants), len(gots))
+	}
+
+	// Close expiry state's channel to inspect object versions enqueued for expiration
+	close(globalExpiryState.byNewerNoncurrentCh)
+	wg.Wait()
+	for _, obj := range expired {
+		switch obj.ObjectV.VersionID {
+		case uuids[2].String(), uuids[3].String(), uuids[4].String():
+		default:
+			t.Errorf("Unexpected versionID being expired: %#v\n", obj)
+		}
+	}
+}

cmd/disk-cache-backend.go

@@ -231,7 +231,7 @@ func newDiskCache(ctx context.Context, dir string, config cache.Config) (*diskCa
 // stops when disk usage drops to 56%
 func (c *diskCache) diskUsageLow() bool {
 	gcStopPct := c.quotaPct * c.lowWatermark / 100
-	di, err := disk.GetInfo(c.dir)
+	di, err := disk.GetInfo(c.dir, false)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
 		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
@@ -254,7 +254,7 @@ func (c *diskCache) diskSpaceAvailable(size int64) bool {
 	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
 
 	gcTriggerPct := c.quotaPct * c.highWatermark / 100
-	di, err := disk.GetInfo(c.dir)
+	di, err := disk.GetInfo(c.dir, false)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		return false
@@ -288,7 +288,7 @@ func (c *diskCache) queueGC() {
 // toClear returns how many bytes should be cleared to reach the low watermark quota.
 // returns 0 if below quota.
 func (c *diskCache) toClear() uint64 {
-	di, err := disk.GetInfo(c.dir)
+	di, err := disk.GetInfo(c.dir, false)
 	if err != nil {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("cachePath", c.dir)
 		ctx := logger.SetReqInfo(GlobalContext, reqInfo)

cmd/endpoint-ellipses.go

@@ -360,7 +360,7 @@ func createServerEndpoints(serverAddr string, args ...string) (
 			return nil, -1, err
 		}
 		for i := range endpointList {
-			endpointList[i].SetPool(0)
+			endpointList[i].SetPoolIndex(0)
 		}
 		endpointServerPools = append(endpointServerPools, PoolEndpoints{
 			Legacy:       true,

cmd/endpoint.go

@@ -69,8 +69,9 @@ type Node struct {
 // Endpoint - any type of endpoint.
 type Endpoint struct {
 	*url.URL
-	Pool    int
 	IsLocal bool
+
+	PoolIdx, SetIdx, DiskIdx int
 }
 
 func (endpoint Endpoint) String() string {
@@ -106,9 +107,19 @@ func (endpoint *Endpoint) UpdateIsLocal() (err error) {
 	return nil
 }
 
-// SetPool sets a specific pool number to this node
-func (endpoint *Endpoint) SetPool(i int) {
-	endpoint.Pool = i
+// SetPoolIndex sets a specific pool number to this node
+func (endpoint *Endpoint) SetPoolIndex(i int) {
+	endpoint.PoolIdx = i
+}
+
+// SetSetIndex sets a specific set number to this node
+func (endpoint *Endpoint) SetSetIndex(i int) {
+	endpoint.SetIdx = i
+}
+
+// SetDiskIndex sets a specific disk number to this node
+func (endpoint *Endpoint) SetDiskIndex(i int) {
+	endpoint.DiskIdx = i
 }
 
 // NewEndpoint - returns new endpoint based on given arguments.
@@ -205,6 +216,9 @@ func NewEndpoint(arg string) (ep Endpoint, e error) {
 	return Endpoint{
 		URL:     u,
 		IsLocal: isLocal,
+		PoolIdx: -1,
+		SetIdx:  -1,
+		DiskIdx: -1,
 	}, nil
 }
 
@@ -236,8 +250,8 @@ func (l EndpointServerPools) GetNodes() (nodes []Node) {
 					Host:   ep.Host,
 				}
 			}
-			if !slices.Contains(node.Pools, ep.Pool) {
-				node.Pools = append(node.Pools, ep.Pool)
+			if !slices.Contains(node.Pools, ep.PoolIdx) {
+				node.Pools = append(node.Pools, ep.PoolIdx)
 			}
 			nodesMap[ep.Host] = node
 		}
@@ -811,7 +825,9 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 			return nil, setupType, config.ErrInvalidEndpoint(nil).Msg("use path style endpoint for single node setup")
 		}
 
-		endpoint.SetPool(0)
+		endpoint.SetPoolIndex(0)
+		endpoint.SetSetIndex(0)
+		endpoint.SetDiskIndex(0)
 
 		var endpoints Endpoints
 		endpoints = append(endpoints, endpoint)
@@ -828,7 +844,7 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 
 	for poolIdx, args := range poolArgs {
 		var endpoints Endpoints
-		for _, iargs := range args {
+		for setIdx, iargs := range args {
 			// Convert args to endpoints
 			eps, err := NewEndpoints(iargs...)
 			if err != nil {
@@ -840,8 +856,10 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 				return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 			}
 
-			for i := range eps {
-				eps[i].SetPool(poolIdx)
+			for diskIdx := range eps {
+				eps[diskIdx].SetPoolIndex(poolIdx)
+				eps[diskIdx].SetSetIndex(setIdx)
+				eps[diskIdx].SetDiskIndex(diskIdx)
 			}
 
 			endpoints = append(endpoints, eps...)
@@ -1022,6 +1040,11 @@ func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType,
 		if endpoint.Type() != PathEndpointType {
 			return endpoints, setupType, config.ErrInvalidEndpoint(nil).Msg("use path style endpoint for single node setup")
 		}
+
+		endpoint.SetPoolIndex(0)
+		endpoint.SetSetIndex(0)
+		endpoint.SetDiskIndex(0)
+
 		endpoints = append(endpoints, endpoint)
 		setupType = ErasureSDSetupType
 
@@ -1033,7 +1056,7 @@ func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType,
 		return endpoints, setupType, nil
 	}
 
-	for _, iargs := range args {
+	for setIdx, iargs := range args {
 		// Convert args to endpoints
 		eps, err := NewEndpoints(iargs...)
 		if err != nil {
@@ -1045,6 +1068,11 @@ func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType,
 			return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 		}
 
+		for diskIdx := range eps {
+			eps[diskIdx].SetSetIndex(setIdx)
+			eps[diskIdx].SetDiskIndex(diskIdx)
+		}
+
 		endpoints = append(endpoints, eps...)
 	}
 

cmd/endpoint_test.go

@@ -37,9 +37,9 @@ func TestNewEndpoint(t *testing.T) {
 		expectedType     EndpointType
 		expectedErr      error
 	}{
-		{"/foo", Endpoint{URL: &url.URL{Path: rootSlashFoo}, IsLocal: true}, PathEndpointType, nil},
-		{"https://example.org/path", Endpoint{URL: u2, IsLocal: false}, URLEndpointType, nil},
-		{"http://192.168.253.200/path", Endpoint{URL: u4, IsLocal: false}, URLEndpointType, nil},
+		{"/foo", Endpoint{&url.URL{Path: rootSlashFoo}, true, -1, -1, -1}, PathEndpointType, nil},
+		{"https://example.org/path", Endpoint{u2, false, -1, -1, -1}, URLEndpointType, nil},
+		{"http://192.168.253.200/path", Endpoint{u4, false, -1, -1, -1}, URLEndpointType, nil},
 		{"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{SlashSeparator, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},
 		{`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")},

cmd/erasure-common.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"io"
 	"math/rand"
 	"sync"
 	"time"
@@ -41,7 +42,7 @@ func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 			if disks[i] == nil {
 				return
 			}
-			di, err := disks[i].DiskInfo(context.Background())
+			di, err := disks[i].DiskInfo(context.Background(), false)
 			if err != nil || di.Healing {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.
@@ -99,7 +100,7 @@ func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 			if disks[i] == nil {
 				return
 			}
-			di, err := disks[i].DiskInfo(context.Background())
+			di, err := disks[i].DiskInfo(context.Background(), false)
 			if err != nil || di.Healing {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.
@@ -188,18 +189,21 @@ func readMultipleFiles(ctx context.Context, disks []StorageAPI, req ReadMultiple
 		dataArray = append(dataArray, toAdd)
 	}
 
+	ignoredErrs := []error{
+		errFileNotFound,
+		errVolumeNotFound,
+		errFileVersionNotFound,
+		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
+		io.EOF,              // some times we would read without locks, ignore these errors
+	}
+	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)
+
 	errs := g.Wait()
 	for index, err := range errs {
 		if err == nil {
 			continue
 		}
-		if !IsErr(err, []error{
-			errFileNotFound,
-			errVolumeNotFound,
-			errFileVersionNotFound,
-			errDiskNotFound,
-			errUnformattedDisk,
-		}...) {
+		if !IsErr(err, ignoredErrs...) {
 			logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
 				disks[index], req.Bucket, req.Prefix, err),
 				disks[index].String())

cmd/erasure-decode.go

@@ -289,12 +289,15 @@ func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.Read
 }
 
 // Heal reads from readers, reconstruct shards and writes the data to the writers.
-func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.ReaderAt, totalLength int64) (derr error) {
+func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.ReaderAt, totalLength int64, prefer []bool) (derr error) {
 	if len(writers) != e.parityBlocks+e.dataBlocks {
 		return errInvalidArgument
 	}
 
 	reader := newParallelReader(readers, e, 0, totalLength)
+	if len(readers) == len(prefer) {
+		reader.preferReaders(prefer)
+	}
 
 	startBlock := int64(0)
 	endBlock := totalLength / e.blockSize

cmd/erasure-heal_test.go

@@ -132,7 +132,7 @@ func TestErasureHeal(t *testing.T) {
 		}
 
 		// test case setup is complete - now call Heal()
-		err = erasure.Heal(context.Background(), staleWriters, readers, test.size)
+		err = erasure.Heal(context.Background(), staleWriters, readers, test.size, nil)
 		closeBitrotReaders(readers)
 		closeBitrotWriters(staleWriters)
 		if err != nil && !test.shouldFail {

cmd/erasure-healing-common_test.go

@@ -20,7 +20,6 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -426,7 +425,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 			}
 
 			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", true)
-			_, err = getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
+			fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
 			}
@@ -484,11 +483,6 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 				}
 
 			}
-			partsMetadata, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", true)
-			fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
-			if !errors.Is(err, errErasureReadQuorum) {
-				t.Fatalf("Failed to getLatestFileInfo, expected %v, got %v", errErasureReadQuorum, err)
-			}
 
 			rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
 			onlineDisks, modTime, _ := listOnlineDisks(erasureDisks, partsMetadata, test.errs, rQuorum)

cmd/erasure-healing.go

@@ -661,6 +661,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			partChecksums := latestMeta.Parts[partIndex].Checksums
 			tillOffset := erasure.ShardFileOffset(0, partSize, partSize)
 			readers := make([]io.ReaderAt, len(latestDisks))
+			prefer := make([]bool, len(latestDisks))
 			checksumAlgo := erasureInfo.GetChecksumInfo(partNumber).Algorithm
 			for i, disk := range latestDisks {
 				if disk == OfflineDisk {
@@ -670,6 +671,8 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 				partPath := pathJoin(object, srcDataDir, fmt.Sprintf("part.%d", partNumber))
 				readers[i] = newBitrotReader(disk, copyPartsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo,
 					checksumInfo.Hash, erasure.ShardSize())
+				prefer[i] = disk.Hostname() == ""
+
 			}
 			writers := make([]io.Writer, len(outDatedDisks))
 			for i, disk := range outDatedDisks {
@@ -689,7 +692,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 			// Heal each part. erasure.Heal() will write the healed
 			// part to .minio/tmp/uuid/ which needs to be renamed
 			// later to the final location.
-			err = erasure.Heal(ctx, writers, readers, partSize)
+			err = erasure.Heal(ctx, writers, readers, partSize, prefer)
 			closeBitrotReaders(readers)
 			closeBitrotWriters(writers)
 			if err != nil {

cmd/erasure-healing_test.go

@@ -1065,8 +1065,8 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 	}
 
 	for i := 0; i < (nfi.Erasure.DataBlocks + nfi.Erasure.ParityBlocks); i++ {
-		_, err = erasureDisks[i].StatInfoFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
-		if err == nil {
+		stats, _ := erasureDisks[i].StatInfoFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		if len(stats) != 0 {
 			t.Errorf("Expected xl.meta file to be not present, but succeeeded")
 		}
 	}

cmd/erasure-metadata-utils.go

@@ -170,11 +170,10 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 		errFileNotFound,
 		errVolumeNotFound,
 		errFileVersionNotFound,
-		errDiskNotFound,
-		errUnformattedDisk,
 		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
 		io.EOF,              // some times we would read without locks, ignore these errors
 	}
+	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)
 	errs := g.Wait()
 	for index, err := range errs {
 		if err == nil {

cmd/erasure-metadata.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
-	"net/http"
 	"sort"
 	"strconv"
 	"strings"
@@ -365,8 +364,6 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 			// Server-side replication fields
 			fmt.Fprintf(h, "%v", meta.MarkDeleted)
 			fmt.Fprint(h, meta.Metadata[string(meta.ReplicationState.ReplicaStatus)])
-			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.ReplicationTimeStamp.Format(http.TimeFormat)])
-			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.ReplicaTimeStamp.Format(http.TimeFormat)])
 			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.ReplicationStatusInternal])
 			fmt.Fprint(h, meta.Metadata[meta.ReplicationState.VersionPurgeStatusInternal])
 

cmd/erasure-multipart.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -36,6 +36,7 @@ import (
 	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/mimedb"
 	"github.com/minio/pkg/sync/errgroup"
@@ -100,7 +101,6 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 
 	// Pick one from the first valid metadata.
 	fi, err = pickValidFileInfo(ctx, partsMetadata, modTime, etag, quorum)
-
 	return fi, partsMetadata, err
 }
 
@@ -205,25 +205,31 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 			if err != nil {
 				return nil
 			}
-			wait := deletedCleanupSleeper.Timer(ctx)
-			if now.Sub(fi.ModTime) > expiry {
-				removeAll(pathJoin(diskPath, minioMetaMultipartBucket, uploadIDPath))
-			}
-			wait()
-			return nil
+			w := xioutil.NewDeadlineWorker(diskMaxTimeout)
+			return w.Run(func() error {
+				wait := deletedCleanupSleeper.Timer(ctx)
+				if now.Sub(fi.ModTime) > expiry {
+					removeAll(pathJoin(diskPath, minioMetaMultipartBucket, uploadIDPath))
+				}
+				wait()
+				return nil
+			})
 		})
 		vi, err := disk.StatVol(ctx, pathJoin(minioMetaMultipartBucket, shaDir))
 		if err != nil {
 			return nil
 		}
-		wait := deletedCleanupSleeper.Timer(ctx)
-		if now.Sub(vi.Created) > expiry {
-			// We are not deleting shaDir recursively here, if shaDir is empty
-			// and its older then we can happily delete it.
-			Remove(pathJoin(diskPath, minioMetaMultipartBucket, shaDir))
-		}
-		wait()
-		return nil
+		w := xioutil.NewDeadlineWorker(diskMaxTimeout)
+		return w.Run(func() error {
+			wait := deletedCleanupSleeper.Timer(ctx)
+			if now.Sub(vi.Created) > expiry {
+				// We are not deleting shaDir recursively here, if shaDir is empty
+				// and its older then we can happily delete it.
+				Remove(pathJoin(diskPath, minioMetaMultipartBucket, shaDir))
+			}
+			wait()
+			return nil
+		})
 	})
 
 	readDirFn(pathJoin(diskPath, minioMetaTmpBucket), func(tmpDir string, typ os.FileMode) error {
@@ -234,12 +240,15 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		if err != nil {
 			return nil
 		}
-		wait := deletedCleanupSleeper.Timer(ctx)
-		if now.Sub(vi.Created) > expiry {
-			removeAll(pathJoin(diskPath, minioMetaTmpBucket, tmpDir))
-		}
-		wait()
-		return nil
+		w := xioutil.NewDeadlineWorker(diskMaxTimeout)
+		return w.Run(func() error {
+			wait := deletedCleanupSleeper.Timer(ctx)
+			if now.Sub(vi.Created) > expiry {
+				removeAll(pathJoin(diskPath, minioMetaTmpBucket, tmpDir))
+			}
+			wait()
+			return nil
+		})
 	})
 }
 
@@ -300,7 +309,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 		populatedUploadIds.Add(uploadID)
 		uploads = append(uploads, MultipartInfo{
 			Object:    object,
-			UploadID:  uploadID,
+			UploadID:  base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID, uploadID))),
 			Initiated: fi.ModTime,
 		})
 	}
@@ -405,7 +414,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		wg.Add(1)
 		go func(disk StorageAPI) {
 			defer wg.Done()
-			di, err := disk.DiskInfo(ctx)
+			di, err := disk.DiskInfo(ctx, false)
 			if err != nil || di.ID == "" {
 				atomicOfflineDrives.Inc()
 				atomicParityDrives.Inc()
@@ -635,9 +644,8 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	tmpPartPath := pathJoin(tmpPart, partSuffix)
 
 	// Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete.
-	var online int
 	defer func() {
-		if online != len(onlineDisks) {
+		if countOnlineDisks(onlineDisks) != len(onlineDisks) {
 			er.deleteAll(context.Background(), minioMetaTmpBucket, tmpPart)
 		}
 	}()
@@ -944,10 +952,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	if err != nil {
 		return oi, err
 	}
-	wctx := wlkctx.Context()
+	ctx = wlkctx.Context()
 	defer uploadIDLock.Unlock(wlkctx)
 
-	fi, partsMetadata, err := er.checkUploadIDExists(wctx, bucket, object, uploadID, true)
+	fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
 	if err != nil {
 		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
@@ -1205,6 +1213,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			partsMetadata[index].Metadata = fi.Metadata
 			partsMetadata[index].Parts = fi.Parts
 			partsMetadata[index].Checksum = fi.Checksum
+			partsMetadata[index].Versioned = opts.Versioned || opts.VersionSuspended
 		}
 	}
 
@@ -1217,14 +1226,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	ctx = lkctx.Context()
 	defer lk.Unlock(lkctx)
 
-	// Write final `xl.meta` at uploadID location
-	onlineDisks, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum)
-	if err != nil {
-		return oi, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
-	}
-
 	// Remove parts that weren't present in CompleteMultipartUpload request.
 	for _, curpart := range currentFI.Parts {
+		// Remove part.meta which is not needed anymore.
+		er.removePartMeta(bucket, object, uploadID, currentFI.DataDir, curpart.Number)
+
 		if objectPartIndex(fi.Parts, curpart.Number) == -1 {
 			// Delete the missing part files. e.g,
 			// Request 1: NewMultipart
@@ -1232,14 +1238,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			// Request 3: PutObjectPart 2
 			// Request 4: CompleteMultipartUpload --part 2
 			// N.B. 1st part is not present. This part should be removed from the storage.
-			er.removeObjectPart(bucket, object, uploadID, fi.DataDir, curpart.Number)
+			er.removeObjectPart(bucket, object, uploadID, currentFI.DataDir, curpart.Number)
 		}
 	}
 
-	// Remove part.meta which is not needed anymore.
-	for _, part := range fi.Parts {
-		er.removePartMeta(bucket, object, uploadID, fi.DataDir, part.Number)
-	}
+	defer er.deleteAll(context.Background(), minioMetaMultipartBucket, uploadIDPath)
 
 	// Rename the multipart object to final location.
 	onlineDisks, versionsDisparity, err := renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,

cmd/erasure-object.go

@@ -412,9 +412,14 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 				switch scan {
 				case madmin.HealNormalScan, madmin.HealDeepScan:
 					healOnce.Do(func() {
-						if _, healing := er.getOnlineDisksWithHealing(); !healing {
-							go healObject(bucket, object, fi.VersionID, scan)
-						}
+						globalMRFState.addPartialOp(partialOperation{
+							bucket:    bucket,
+							object:    object,
+							versionID: fi.VersionID,
+							queued:    time.Now(),
+							setIndex:  er.setIndex,
+							poolIndex: er.poolIndex,
+						})
 					})
 					// Healing is triggered and we have written
 					// successfully the content to client for
@@ -571,10 +576,10 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		errFileNameTooLong,
 		errVolumeNotFound,
 		errFileVersionNotFound,
-		errDiskNotFound,
 		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
 		io.EOF,              // some times we would read without locks, ignore these errors
 	}
+	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)
 
 	errs := g.Wait()
 	for index, err := range errs {
@@ -732,9 +737,14 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	// additionally do not heal delete markers inline, let them be
 	// healed upon regular heal process.
 	if !fi.Deleted && missingBlocks > 0 && missingBlocks < readQuorum {
-		if _, healing := er.getOnlineDisksWithHealing(); !healing {
-			go healObject(bucket, object, fi.VersionID, madmin.HealNormalScan)
-		}
+		globalMRFState.addPartialOp(partialOperation{
+			bucket:    bucket,
+			object:    object,
+			versionID: fi.VersionID,
+			queued:    time.Now(),
+			setIndex:  er.setIndex,
+			poolIndex: er.poolIndex,
+		})
 	}
 
 	return fi, metaArr, onlineDisks, nil
@@ -1102,7 +1112,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			wg.Add(1)
 			go func(disk StorageAPI) {
 				defer wg.Done()
-				di, err := disk.DiskInfo(ctx)
+				di, err := disk.DiskInfo(ctx, false)
 				if err != nil || di.ID == "" {
 					atomicOfflineDrives.Inc()
 					atomicParityDrives.Inc()
@@ -1186,15 +1196,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	partName := "part.1"
 	tempErasureObj := pathJoin(uniqueID, fi.DataDir, partName)
 
-	// Delete temporary object in the event of failure.
-	// If PutObject succeeded there would be no temporary
-	// object to delete.
-	var online int
-	defer func() {
-		if online != len(onlineDisks) {
-			er.deleteAll(context.Background(), minioMetaTmpBucket, tempObj)
-		}
-	}()
+	defer er.deleteAll(context.Background(), minioMetaTmpBucket, tempObj)
 
 	shardFileSize := erasure.ShardFileSize(data.Size())
 	writers := make([]io.Writer, len(onlineDisks))
@@ -1299,6 +1301,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			Algorithm:  DefaultBitrotAlgorithm,
 			Hash:       bitrotWriterSum(w),
 		})
+		partsMetadata[i].Versioned = opts.Versioned || opts.VersionSuspended
 	}
 
 	userDefined["etag"] = r.MD5CurrentHexString()
@@ -1379,7 +1382,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}
 
 	fi.ReplicationState = opts.PutReplicationState()
-	online = countOnlineDisks(onlineDisks)
 
 	// we are adding a new version to this object under the namespace lock, so this is the latest version.
 	fi.IsLatest = true
@@ -1959,8 +1961,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
 }
 
-// updateObjectMeta will update the metadata of a file.
-func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object string, fi FileInfo, onlineDisks []StorageAPI) error {
+func (er erasureObjects) updateObjectMetaWithOpts(ctx context.Context, bucket, object string, fi FileInfo, onlineDisks []StorageAPI, opts UpdateMetadataOpts) error {
 	if len(fi.Metadata) == 0 {
 		return nil
 	}
@@ -1974,7 +1975,7 @@ func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object st
 			if onlineDisks[index] == nil {
 				return errDiskNotFound
 			}
-			return onlineDisks[index].UpdateMetadata(ctx, bucket, object, fi)
+			return onlineDisks[index].UpdateMetadata(ctx, bucket, object, fi, opts)
 		}, index)
 	}
 
@@ -1984,6 +1985,11 @@ func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object st
 	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, er.defaultWQuorum())
 }
 
+// updateObjectMeta will update the metadata of a file.
+func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object string, fi FileInfo, onlineDisks []StorageAPI) error {
+	return er.updateObjectMetaWithOpts(ctx, bucket, object, fi, onlineDisks, UpdateMetadataOpts{})
+}
+
 // DeleteObjectTags - delete object tags from an existing object
 func (er erasureObjects) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 	return er.PutObjectTags(ctx, bucket, object, "", opts)

cmd/erasure-server-pool-decom.go

@@ -696,7 +696,9 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 	// Check if bucket is object locked.
 	lr, _ := globalBucketObjectLockSys.Get(bi.Name)
 
-	for _, set := range pool.sets {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	for setIdx, set := range pool.sets {
 		set := set
 
 		filterLifecycle := func(bucket, object string, fi FileInfo) bool {
@@ -895,16 +897,26 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 		}
 
 		wk.Take()
-		go func() {
+		go func(setIdx int) {
 			defer wk.Give()
-			err := set.listObjectsToDecommission(ctx, bi,
-				func(entry metaCacheEntry) {
-					wk.Take()
-					go decommissionEntry(entry)
-				},
-			)
-			logger.LogIf(ctx, err)
-		}()
+			// We will perpetually retry listing if it fails, since we cannot
+			// possibly give up in this matter
+			for {
+				err := set.listObjectsToDecommission(ctx, bi,
+					func(entry metaCacheEntry) {
+						wk.Take()
+						go decommissionEntry(entry)
+					},
+				)
+				if err == nil {
+					break
+				}
+				setN := humanize.Ordinal(setIdx + 1)
+				retryDur := time.Duration(r.Float64() * float64(5*time.Second))
+				logger.LogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v, retrying in %v", setN, err, retryDur), "decom-listing-failed"+setN)
+				time.Sleep(retryDur)
+			}
+		}(setIdx)
 	}
 	wk.Wait()
 	return nil

cmd/erasure-server-pool.go

@@ -263,8 +263,8 @@ type serverPoolsAvailableSpace []poolAvailableSpace
 
 type poolAvailableSpace struct {
 	Index      int
-	Available  uint64
-	MaxUsedPct int // Used disk percentage of most filled disk, rounded down.
+	Available  uint64 // in bytes
+	MaxUsedPct int    // Used disk percentage of most filled disk, rounded down.
 }
 
 // TotalAvailable - total available space
@@ -286,7 +286,7 @@ func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
 	}
 	var ok bool
 	for _, z := range p {
-		if z.MaxUsedPct < max {
+		if z.Available > 0 && z.MaxUsedPct < max {
 			ok = true
 			break
 		}
@@ -299,7 +299,7 @@ func (p serverPoolsAvailableSpace) FilterMaxUsed(max int) {
 
 	// Remove entries that are above.
 	for i, z := range p {
-		if z.MaxUsedPct < max {
+		if z.Available > 0 && z.MaxUsedPct < max {
 			continue
 		}
 		p[i].Available = 0
@@ -365,7 +365,7 @@ func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, b
 		}
 		var available uint64
 		if !isMinioMetaBucketName(bucket) {
-			if avail, err := hasSpaceFor(zinfo, size); err != nil && !avail {
+			if avail, err := hasSpaceFor(zinfo, size); err != nil || !avail {
 				serverPools[i] = poolAvailableSpace{Index: i}
 				continue
 			}
@@ -1254,6 +1254,11 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 		return loi, err
 	}
 	defer merged.truncate(0) // Release when returning
+
+	if contextCanceled(ctx) {
+		return ListObjectVersionsInfo{}, ctx.Err()
+	}
+
 	if versionMarker == "" {
 		o := listPathOptions{Marker: marker}
 		// If we are not looking for a specific version skip it.
@@ -1341,6 +1346,10 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 			loi.Objects = append(loi.Objects, objInfo)
 			return loi, nil
 		}
+
+		if contextCanceled(ctx) {
+			return ListObjectsInfo{}, ctx.Err()
+		}
 	}
 
 	merged, err := z.listPath(ctx, &opts)
@@ -1354,6 +1363,10 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	merged.forwardPast(opts.Marker)
 	defer merged.truncate(0) // Release when returning
 
+	if contextCanceled(ctx) {
+		return ListObjectsInfo{}, ctx.Err()
+	}
+
 	// Default is recursive, if delimiter is set then list non recursive.
 	objects := merged.fileInfos(bucket, prefix, delimiter)
 	loi.IsTruncated = err == nil && len(objects) > 0
@@ -1475,7 +1488,7 @@ func (z *erasureServerPools) CopyObjectPart(ctx context.Context, srcBucket, srcO
 
 // PutObjectPart - writes part of an object to hashedSet based on the object name.
 func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (PartInfo, error) {
-	if err := checkPutObjectPartArgs(ctx, bucket, object, z); err != nil {
+	if err := checkPutObjectPartArgs(ctx, bucket, object, uploadID, z); err != nil {
 		return PartInfo{}, err
 	}
 
@@ -1507,7 +1520,7 @@ func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object,
 }
 
 func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (MultipartInfo, error) {
-	if err := checkListPartsArgs(ctx, bucket, object, z); err != nil {
+	if err := checkListPartsArgs(ctx, bucket, object, uploadID, z); err != nil {
 		return MultipartInfo{}, err
 	}
 
@@ -1538,7 +1551,7 @@ func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, objec
 
 // ListObjectParts - lists all uploaded parts to an object in hashedSet.
 func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (ListPartsInfo, error) {
-	if err := checkListPartsArgs(ctx, bucket, object, z); err != nil {
+	if err := checkListPartsArgs(ctx, bucket, object, uploadID, z); err != nil {
 		return ListPartsInfo{}, err
 	}
 
@@ -1567,7 +1580,7 @@ func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object
 
 // Aborts an in-progress multipart operation on hashedSet based on the object name.
 func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error {
-	if err := checkAbortMultipartArgs(ctx, bucket, object, z); err != nil {
+	if err := checkAbortMultipartArgs(ctx, bucket, object, uploadID, z); err != nil {
 		return err
 	}
 
@@ -1598,7 +1611,7 @@ func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, o
 
 // CompleteMultipartUpload - completes a pending multipart transaction, on hashedSet based on object name.
 func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	if err = checkCompleteMultipartArgs(ctx, bucket, object, z); err != nil {
+	if err = checkCompleteMultipartArgs(ctx, bucket, object, uploadID, z); err != nil {
 		return objInfo, err
 	}
 
@@ -1822,45 +1835,86 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 						return
 					}
 
+					send := func(objInfo ObjectInfo) bool {
+						select {
+						case <-ctx.Done():
+							return false
+						case results <- objInfo:
+							return true
+						}
+					}
+
+					askDisks := getListQuorum(opts.WalkAskDisks, set.setDriveCount)
+					var fallbackDisks []StorageAPI
+
+					// Special case: ask all disks if the drive count is 4
+					if set.setDriveCount == 4 || askDisks > len(disks) {
+						askDisks = len(disks) // use all available drives
+					}
+
+					if askDisks > 0 && len(disks) > askDisks {
+						fallbackDisks = disks[askDisks:]
+						disks = disks[:askDisks]
+					}
+
+					requestedVersions := 0
+					if opts.WalkLatestOnly {
+						requestedVersions = 1
+					}
 					loadEntry := func(entry metaCacheEntry) {
 						if entry.isDir() {
 							return
 						}
 
-						fivs, err := entry.fileInfoVersions(bucket)
-						if err != nil {
-							cancel()
-							return
-						}
-
-						versionsSorter(fivs.Versions).reverse()
-
-						for _, version := range fivs.Versions {
-							send := true
-							if opts.WalkFilter != nil && !opts.WalkFilter(version) {
-								send = false
-							}
-
-							if !send {
-								continue
-							}
-
-							versioned := vcfg != nil && vcfg.Versioned(version.Name)
-							objInfo := version.ToObjectInfo(bucket, version.Name, versioned)
-
-							select {
-							case <-ctx.Done():
+						if opts.WalkLatestOnly {
+							fi, err := entry.fileInfo(bucket)
+							if err != nil {
+								cancel()
 								return
-							case results <- objInfo:
+							}
+							if opts.WalkFilter != nil {
+								if opts.WalkFilter(fi) {
+									if !send(fi.ToObjectInfo(bucket, fi.Name, vcfg != nil && vcfg.Versioned(fi.Name))) {
+										return
+									}
+								}
+							} else {
+								if !send(fi.ToObjectInfo(bucket, fi.Name, vcfg != nil && vcfg.Versioned(fi.Name))) {
+									return
+								}
+							}
+
+						} else {
+							fivs, err := entry.fileInfoVersions(bucket)
+							if err != nil {
+								cancel()
+								return
+							}
+
+							versionsSorter(fivs.Versions).reverse()
+
+							for _, version := range fivs.Versions {
+								if opts.WalkFilter != nil {
+									if opts.WalkFilter(version) {
+										if !send(version.ToObjectInfo(bucket, version.Name, vcfg != nil && vcfg.Versioned(version.Name))) {
+											return
+										}
+									}
+								} else {
+									if !send(version.ToObjectInfo(bucket, version.Name, vcfg != nil && vcfg.Versioned(version.Name))) {
+										return
+									}
+								}
 							}
 						}
 					}
 
 					// How to resolve partial results.
 					resolver := metadataResolutionParams{
-						dirQuorum: 1,
-						objQuorum: 1,
-						bucket:    bucket,
+						dirQuorum:         1,
+						objQuorum:         1,
+						bucket:            bucket,
+						requestedVersions: requestedVersions,
 					}
 
 					path := baseDirFromPrefix(prefix)
@@ -1871,6 +1925,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 
 					lopts := listPathRawOptions{
 						disks:          disks,
+						fallbackDisks:  fallbackDisks,
 						bucket:         bucket,
 						path:           path,
 						filterPrefix:   filterPrefix,
@@ -2060,9 +2115,13 @@ type HealthOptions struct {
 // additionally with any specific heuristic information which
 // was queried
 type HealthResult struct {
-	Healthy       bool
-	HealingDrives int
-	PoolID, SetID int
+	Healthy        bool
+	HealingDrives  int
+	UnhealthyPools []struct {
+		Maintenance   bool
+		PoolID, SetID int
+		WriteQuorum   int
+	}
 	WriteQuorum   int
 	UsingDefaults bool
 }
@@ -2164,24 +2223,6 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 		usingDefaults = true
 	}
 
-	for poolIdx := range erasureSetUpCount {
-		for setIdx := range erasureSetUpCount[poolIdx] {
-			if erasureSetUpCount[poolIdx][setIdx] < poolWriteQuorums[poolIdx] {
-				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
-					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
-						poolIdx, setIdx, poolWriteQuorums[poolIdx]))
-				return HealthResult{
-					Healthy:       false,
-					HealingDrives: len(aggHealStateResult.HealDisks),
-					PoolID:        poolIdx,
-					SetID:         setIdx,
-					WriteQuorum:   poolWriteQuorums[poolIdx],
-					UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
-				}
-			}
-		}
-	}
-
 	var maximumWriteQuorum int
 	for _, writeQuorum := range poolWriteQuorums {
 		if maximumWriteQuorum == 0 {
@@ -2192,6 +2233,35 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 		}
 	}
 
+	result := HealthResult{
+		HealingDrives: len(aggHealStateResult.HealDisks),
+		WriteQuorum:   maximumWriteQuorum,
+		UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node.
+	}
+
+	for poolIdx := range erasureSetUpCount {
+		for setIdx := range erasureSetUpCount[poolIdx] {
+			if erasureSetUpCount[poolIdx][setIdx] < poolWriteQuorums[poolIdx] {
+				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
+					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
+						poolIdx, setIdx, poolWriteQuorums[poolIdx]))
+				result.UnhealthyPools = append(result.UnhealthyPools, struct {
+					Maintenance                bool
+					PoolID, SetID, WriteQuorum int
+				}{
+					Maintenance: opts.Maintenance,
+					SetID:       setIdx,
+					PoolID:      poolIdx,
+					WriteQuorum: poolWriteQuorums[poolIdx],
+				})
+			}
+		}
+		if len(result.UnhealthyPools) > 0 {
+			// We have unhealthy pools return error.
+			return result
+		}
+	}
+
 	// when maintenance is not specified we don't have
 	// to look at the healing side of the code.
 	if !opts.Maintenance {

cmd/erasure-sets.go

@@ -116,7 +116,10 @@ func (s *erasureSets) getDiskMap() map[Endpoint]StorageAPI {
 // Initializes a new StorageAPI from the endpoint argument, returns
 // StorageAPI and also `format` which exists on the disk.
 func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
-	disk, err := newStorageAPI(endpoint, false)
+	disk, err := newStorageAPI(endpoint, storageOpts{
+		cleanUp:     false,
+		healthCheck: false,
+	})
 	if err != nil {
 		return nil, nil, err
 	}
@@ -124,7 +127,7 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 	format, err := loadFormatErasure(disk)
 	if err != nil {
 		if errors.Is(err, errUnformattedDisk) {
-			info, derr := disk.DiskInfo(context.TODO())
+			info, derr := disk.DiskInfo(context.TODO(), false)
 			if derr != nil && info.RootDisk {
 				return nil, nil, fmt.Errorf("Drive: %s is a root drive", disk)
 			}
@@ -132,6 +135,15 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 		return nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error
 	}
 
+	disk.Close()
+	disk, err = newStorageAPI(endpoint, storageOpts{
+		cleanUp:     true,
+		healthCheck: true,
+	})
+	if err != nil {
+		return nil, nil, err
+	}
+
 	return disk, format, nil
 }
 
@@ -241,21 +253,10 @@ func (s *erasureSets) connectDisks() {
 				}
 				s.erasureDisks[setIndex][diskIndex].Close()
 			}
-			if disk.IsLocal() {
-				disk.SetDiskID(format.Erasure.This)
-				s.erasureDisks[setIndex][diskIndex] = disk
-			} else {
-				// Enable healthcheck disk for remote endpoint.
-				disk, err = newStorageAPI(endpoint, true)
-				if err != nil {
-					printEndpointError(endpoint, err, false)
-					s.erasureDisksMu.Unlock()
-					return
-				}
-				disk.SetDiskID(format.Erasure.This)
-				s.erasureDisks[setIndex][diskIndex] = disk
-			}
+
+			disk.SetDiskID(format.Erasure.This)
 			disk.SetDiskLoc(s.poolIndex, setIndex, diskIndex)
+			s.erasureDisks[setIndex][diskIndex] = disk
 			s.erasureDisksMu.Unlock()
 		}(endpoint)
 	}
@@ -775,7 +776,7 @@ func (s *erasureSets) deletePrefix(ctx context.Context, bucket string, prefix st
 
 // DeleteObject - deletes an object from the hashedSet based on the object name.
 func (s *erasureSets) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) {
-	if opts.DeletePrefix {
+	if opts.DeletePrefix && !opts.DeletePrefixObject {
 		err := s.deletePrefix(ctx, bucket, object)
 		return ObjectInfo{}, err
 	}
@@ -1025,7 +1026,7 @@ func getHealDiskInfos(storageDisks []StorageAPI, errs []error) ([]DiskInfo, []er
 				return errDiskNotFound
 			}
 			var err error
-			infos[index], err = storageDisks[index].DiskInfo(context.TODO())
+			infos[index], err = storageDisks[index].DiskInfo(context.TODO(), false)
 			return err
 		}, index)
 	}
@@ -1055,7 +1056,10 @@ func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
 
 // HealFormat - heals missing `format.json` on fresh unformatted disks.
 func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
-	storageDisks, _ := initStorageDisksWithErrors(s.endpoints.Endpoints, false)
+	storageDisks, _ := initStorageDisksWithErrors(s.endpoints.Endpoints, storageOpts{
+		cleanUp:     true,
+		healthCheck: true,
+	})
 
 	defer func(storageDisks []StorageAPI) {
 		if err != nil {

cmd/erasure-sets_test.go

@@ -173,7 +173,7 @@ func TestNewErasureSets(t *testing.T) {
 		defer os.RemoveAll(disk)
 	}
 
-	endpoints := mustGetNewEndpoints(0, erasureDisks...)
+	endpoints := mustGetNewEndpoints(0, 16, erasureDisks...)
 	_, _, err := waitForFormatErasure(true, endpoints, 1, 0, 16, "", "")
 	if err != errInvalidArgument {
 		t.Fatalf("Expecting error, got %s", err)

cmd/erasure.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bpool"
 	"github.com/minio/minio/internal/dsync"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/sync/errgroup"
 )
@@ -174,33 +175,32 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 	for index := range disks {
 		index := index
 		g.Go(func() error {
-			diskEndpoint := endpoints[index].String()
+			di := madmin.Disk{
+				Endpoint:  endpoints[index].String(),
+				PoolIndex: endpoints[index].PoolIdx,
+				SetIndex:  endpoints[index].SetIdx,
+				DiskIndex: endpoints[index].DiskIdx,
+			}
 			if disks[index] == OfflineDisk {
-				logger.LogOnceIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]), "get-disks-info-offline-"+diskEndpoint)
-				disksInfo[index] = madmin.Disk{
-					State:    diskErrToDriveState(errDiskNotFound),
-					Endpoint: diskEndpoint,
-				}
+				logger.LogOnceIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]), "get-disks-info-offline-"+di.Endpoint)
+				di.State = diskErrToDriveState(errDiskNotFound)
+				disksInfo[index] = di
 				return nil
 			}
-			info, err := disks[index].DiskInfo(context.TODO())
-			di := madmin.Disk{
-				Endpoint:       diskEndpoint,
-				DrivePath:      info.MountPath,
-				TotalSpace:     info.Total,
-				UsedSpace:      info.Used,
-				AvailableSpace: info.Free,
-				UUID:           info.ID,
-				Major:          info.Major,
-				Minor:          info.Minor,
-				RootDisk:       info.RootDisk,
-				Healing:        info.Healing,
-				Scanning:       info.Scanning,
-				State:          diskErrToDriveState(err),
-				FreeInodes:     info.FreeInodes,
-				UsedInodes:     info.UsedInodes,
-			}
-			di.PoolIndex, di.SetIndex, di.DiskIndex = disks[index].GetDiskLoc()
+			info, err := disks[index].DiskInfo(context.TODO(), true)
+			di.DrivePath = info.MountPath
+			di.TotalSpace = info.Total
+			di.UsedSpace = info.Used
+			di.AvailableSpace = info.Free
+			di.UUID = info.ID
+			di.Major = info.Major
+			di.Minor = info.Minor
+			di.RootDisk = info.RootDisk
+			di.Healing = info.Healing
+			di.Scanning = info.Scanning
+			di.State = diskErrToDriveState(err)
+			di.FreeInodes = info.FreeInodes
+			di.UsedInodes = info.UsedInodes
 			if info.Healing {
 				if hi := disks[index].Healing(); hi != nil {
 					hd := hi.toHealingDisk()
@@ -208,8 +208,10 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				}
 			}
 			di.Metrics = &madmin.DiskMetrics{
-				LastMinute: make(map[string]madmin.TimedAction, len(info.Metrics.LastMinute)),
-				APICalls:   make(map[string]uint64, len(info.Metrics.APICalls)),
+				LastMinute:              make(map[string]madmin.TimedAction, len(info.Metrics.LastMinute)),
+				APICalls:                make(map[string]uint64, len(info.Metrics.APICalls)),
+				TotalErrorsAvailability: info.Metrics.TotalErrorsAvailability,
+				TotalErrorsTimeout:      info.Metrics.TotalErrorsTimeout,
 			}
 			for k, v := range info.Metrics.LastMinute {
 				if v.N > 0 {
@@ -290,7 +292,7 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 				return
 			}
 
-			di, err := disk.DiskInfo(context.Background())
+			di, err := disk.DiskInfo(context.Background(), false)
 			if err != nil || di.Healing {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.
@@ -308,6 +310,7 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 	}
 	wg.Wait()
 
+	var scanningDisks []StorageAPI
 	for i, info := range infos {
 		// Check if one of the drives in the set is being healed.
 		// this information is used by scanner to skip healing
@@ -316,9 +319,16 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 			healing = true
 			continue
 		}
-		newDisks = append(newDisks, disks[i])
+		if !info.Scanning {
+			newDisks = append(newDisks, disks[i])
+		} else {
+			scanningDisks = append(scanningDisks, disks[i])
+		}
 	}
 
+	// Prefer new disks over disks which are currently being scanned.
+	newDisks = append(newDisks, scanningDisks...)
+
 	return newDisks, healing
 }
 
@@ -333,10 +343,13 @@ func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
 				defer wg.Done()
 				diskPath := disk.Endpoint().Path
 				readDirFn(pathJoin(diskPath, minioMetaTmpDeletedBucket), func(ddir string, typ os.FileMode) error {
-					wait := deletedCleanupSleeper.Timer(ctx)
-					removeAll(pathJoin(diskPath, minioMetaTmpDeletedBucket, ddir))
-					wait()
-					return nil
+					w := xioutil.NewDeadlineWorker(diskMaxTimeout)
+					return w.Run(func() error {
+						wait := deletedCleanupSleeper.Timer(ctx)
+						removeAll(pathJoin(diskPath, minioMetaTmpDeletedBucket, ddir))
+						wait()
+						return nil
+					})
 				})
 			}(disk)
 		}

cmd/format-erasure.go

@@ -663,14 +663,14 @@ func closeStorageDisks(storageDisks ...StorageAPI) {
 
 // Initialize storage disks for each endpoint.
 // Errors are returned for each endpoint with matching index.
-func initStorageDisksWithErrors(endpoints Endpoints, healthCheck bool) ([]StorageAPI, []error) {
+func initStorageDisksWithErrors(endpoints Endpoints, opts storageOpts) ([]StorageAPI, []error) {
 	// Bootstrap disks.
 	storageDisks := make([]StorageAPI, len(endpoints))
 	g := errgroup.WithNErrs(len(endpoints))
 	for index := range endpoints {
 		index := index
 		g.Go(func() (err error) {
-			storageDisks[index], err = newStorageAPI(endpoints[index], healthCheck)
+			storageDisks[index], err = newStorageAPI(endpoints[index], opts)
 			return err
 		}, index)
 	}

cmd/format-erasure_test.go

@@ -36,9 +36,9 @@ func TestFixFormatV3(t *testing.T) {
 	for _, erasureDir := range erasureDirs {
 		defer os.RemoveAll(erasureDir)
 	}
-	endpoints := mustGetNewEndpoints(0, erasureDirs...)
+	endpoints := mustGetNewEndpoints(0, 8, erasureDirs...)
 
-	storageDisks, errs := initStorageDisksWithErrors(endpoints, false)
+	storageDisks, errs := initStorageDisksWithErrors(endpoints, storageOpts{cleanUp: false, healthCheck: false})
 	for _, err := range errs {
 		if err != nil && err != errDiskNotFound {
 			t.Fatal(err)
@@ -555,11 +555,12 @@ func benchmarkInitStorageDisksN(b *testing.B, nDisks int) {
 	if err != nil {
 		b.Fatal(err)
 	}
-	endpoints := mustGetNewEndpoints(0, fsDirs...)
+
+	endpoints := mustGetNewEndpoints(0, 16, fsDirs...)
 	b.RunParallel(func(pb *testing.PB) {
 		endpoints := endpoints
 		for pb.Next() {
-			initStorageDisksWithErrors(endpoints, false)
+			initStorageDisksWithErrors(endpoints, storageOpts{cleanUp: false, healthCheck: false})
 		}
 	})
 }

cmd/generic-handlers.go

@@ -29,6 +29,7 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
+	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
 	xnet "github.com/minio/pkg/net"
 
@@ -206,7 +207,7 @@ func getRedirectLocation(r *http.Request) *xnet.URL {
 }
 
 // guessIsHealthCheckReq - returns true if incoming request looks
-// like healthcheck request
+// like healthCheck request
 func guessIsHealthCheckReq(req *http.Request) bool {
 	if req == nil {
 		return false
@@ -229,7 +230,8 @@ func guessIsMetricsReq(req *http.Request) bool {
 	return (aType == authTypeAnonymous || aType == authTypeJWT) &&
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsPathLegacy ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ClusterPath ||
-		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2NodePath
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2NodePath ||
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2BucketPath
 }
 
 // guessIsRPCReq - returns true if the request is for an RPC endpoint.
@@ -399,6 +401,17 @@ func setRequestValidityMiddleware(h http.Handler) http.Handler {
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL)
 				return
 			}
+		} else {
+			// Validate bucket names if it is not empty
+			if bucketName != "" && s3utils.CheckValidBucketNameStrict(bucketName) != nil {
+				if ok {
+					tc.FuncName = "handler.ValidRequest"
+					tc.ResponseRecorder.LogErrBody = true
+				}
+				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
+				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL)
+				return
+			}
 		}
 		// Deny SSE-C requests if not made over TLS
 		if !globalIsTLS && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) {
@@ -430,6 +443,11 @@ func setRequestValidityMiddleware(h http.Handler) http.Handler {
 // is obtained from centralized etcd configuration service.
 func setBucketForwardingMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if origin := w.Header().Get("Access-Control-Allow-Origin"); origin == "null" {
+			// This is a workaround change to ensure that "Origin: null"
+			// incoming request to a response back as "*" instead of "null"
+			w.Header().Set("Access-Control-Allow-Origin", "*")
+		}
 		if globalDNSConfig == nil || !globalBucketFederation ||
 			guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
 			guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) {

cmd/global-heal.go

@@ -192,7 +192,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		tracker.setObject("")
 		tracker.setBucket(bucket)
 		// Heal current bucket again in case if it is failed
-		// in the  being of erasure set healing
+		// in the beginning of erasure set healing
 		if _, err := er.HealBucket(ctx, bucket, madmin.HealOpts{
 			ScanMode: scanMode,
 		}); err != nil {
@@ -241,7 +241,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		}
 
 		// Collect updates to tracker from concurrent healEntry calls
-		results := make(chan healEntryResult)
+		results := make(chan healEntryResult, 1000)
 		go func() {
 			for res := range results {
 				if res.entryDone {
@@ -256,6 +256,15 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			}
 		}()
 
+		send := func(result healEntryResult) bool {
+			select {
+			case <-ctx.Done():
+				return false
+			case results <- result:
+				return true
+			}
+		}
+
 		// Note: updates from healEntry to tracker must be sent on results channel.
 		healEntry := func(bucket string, entry metaCacheEntry) {
 			if entry.name == "" && len(entry.metadata) == 0 {
@@ -302,12 +311,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					result = healEntrySuccess(0)
 				}
 
-				select {
-				case <-ctx.Done():
-					return
-				case results <- result:
-				}
-
+				send(result)
 				return
 			}
 
@@ -342,10 +346,8 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				}
 				bgSeq.logHeal(madmin.HealItemObject)
 
-				select {
-				case <-ctx.Done():
+				if !send(result) {
 					return
-				case results <- result:
 				}
 			}
 			// All versions resulted in 'ObjectNotFound'

cmd/globals.go

@@ -193,7 +193,7 @@ var (
 	globalBucketSSEConfigSys *BucketSSEConfigSys
 	globalBucketTargetSys    *BucketTargetSys
 	// globalAPIConfig controls S3 API requests throttling,
-	// healthcheck readiness deadlines and cors settings.
+	// healthCheck readiness deadlines and cors settings.
 	globalAPIConfig = apiConfig{listQuorum: "strict", rootAccess: true}
 
 	globalStorageClass storageclass.Config
@@ -346,7 +346,7 @@ var (
 
 	globalTierConfigMgr *TierConfigMgr
 
-	globalTierJournal *tierJournal
+	globalTierJournal *TierJournal
 
 	globalConsoleSrv *restapi.Server
 
@@ -397,6 +397,11 @@ var (
 	globalInternodeInterface     string
 	globalInternodeInterfaceOnce sync.Once
 
+	// Set last client perf extra time (get lock, and validate)
+	globalLastClientPerfExtraTime int64
+
+	// Captures all batch jobs metrics globally
+	globalBatchJobsMetrics batchJobMetrics
 	// Add new variable global values here.
 )
 

cmd/handler-api.go

@@ -49,7 +49,7 @@ type apiConfig struct {
 	staleUploadsExpiry          time.Duration
 	staleUploadsCleanupInterval time.Duration
 	deleteCleanupInterval       time.Duration
-	disableODirect              bool
+	enableODirect               bool
 	gzipObjects                 bool
 	rootAccess                  bool
 	syncEvents                  bool
@@ -164,17 +164,17 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 	t.staleUploadsExpiry = cfg.StaleUploadsExpiry
 	t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval
 	t.deleteCleanupInterval = cfg.DeleteCleanupInterval
-	t.disableODirect = cfg.DisableODirect
+	t.enableODirect = cfg.EnableODirect
 	t.gzipObjects = cfg.GzipObjects
 	t.rootAccess = cfg.RootAccess
 	t.syncEvents = cfg.SyncEvents
 }
 
-func (t *apiConfig) isDisableODirect() bool {
+func (t *apiConfig) odirectEnabled() bool {
 	t.mu.RLock()
 	defer t.mu.RUnlock()
 
-	return t.disableODirect
+	return t.enableODirect
 }
 
 func (t *apiConfig) shouldGzipObjects() bool {

cmd/handler-utils.go

@@ -38,6 +38,7 @@ import (
 const (
 	copyDirective    = "COPY"
 	replaceDirective = "REPLACE"
+	accessDirective  = "ACCESS"
 )
 
 // Parses location constraint from the incoming reader.

cmd/healthcheck-handler.go

@@ -28,22 +28,17 @@ import (
 
 const unavailable = "offline"
 
-func isServerNotInitialized() bool {
-	return newObjectLayerFn() == nil
-}
-
 // ClusterCheckHandler returns if the server is ready for requests.
 func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClusterCheckHandler")
 
-	if isServerNotInitialized() {
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
 
-	objLayer := newObjectLayerFn()
-
 	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline())
 	defer cancel()
 
@@ -52,16 +47,13 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 		DeploymentType: r.Form.Get("deployment-type"),
 	}
 	result := objLayer.Health(ctx, opts)
-	if result.WriteQuorum > 0 {
-		w.Header().Set(xhttp.MinIOWriteQuorum, strconv.Itoa(result.WriteQuorum))
-	}
+	w.Header().Set(xhttp.MinIOWriteQuorum, strconv.Itoa(result.WriteQuorum))
 	w.Header().Set(xhttp.MinIOStorageClassDefaults, strconv.FormatBool(result.UsingDefaults))
-
+	// return how many drives are being healed if any
+	if result.HealingDrives > 0 {
+		w.Header().Set(xhttp.MinIOHealingDrives, strconv.Itoa(result.HealingDrives))
+	}
 	if !result.Healthy {
-		// return how many drives are being healed if any
-		if result.HealingDrives > 0 {
-			w.Header().Set(xhttp.MinIOHealingDrives, strconv.Itoa(result.HealingDrives))
-		}
 		// As a maintenance call we are purposefully asked to be taken
 		// down, this is for orchestrators to know if we can safely
 		// take this server down, return appropriate error.
@@ -79,14 +71,13 @@ func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) {
 func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ClusterReadCheckHandler")
 
-	if isServerNotInitialized() {
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 		writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone)
 		return
 	}
 
-	objLayer := newObjectLayerFn()
-
 	ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline())
 	defer cancel()
 
@@ -106,17 +97,17 @@ func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) {
 
 // LivenessCheckHandler - Checks if the process is up. Always returns success.
 func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) {
-	peerCall := r.Header.Get("x-minio-from-peer") != ""
-
-	if peerCall {
-		return
-	}
-
-	if isServerNotInitialized() {
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
 		// Service not initialized yet
 		w.Header().Set(xhttp.MinIOServerStatus, unavailable)
 	}
 
+	peerCall := r.Header.Get(xhttp.MinIOPeerCall) != ""
+	if peerCall {
+		return
+	}
+
 	if int(globalHTTPStats.loadRequestsInQueue()) > globalAPIConfig.getRequestsPoolCapacity() {
 		apiErr := getAPIError(ErrBusy)
 		switch r.Method {

cmd/http-stats.go

@@ -127,6 +127,14 @@ func (bh *bucketHTTPStats) updateHTTPStats(bucket, api string, w *xhttp.Response
 		return
 	}
 
+	if w != nil {
+		// Increment the prometheus http request response histogram with API, Bucket
+		bucketHTTPRequestsDuration.With(prometheus.Labels{
+			"api":    api,
+			"bucket": bucket,
+		}).Observe(w.TimeToFirstByte.Seconds())
+	}
+
 	bh.Lock()
 	defer bh.Unlock()
 

cmd/iam-object-store.go

@@ -384,10 +384,12 @@ func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[str
 
 // Assumes cache is locked by caller.
 func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache) error {
-	bootstrapTrace("loading all IAM items")
 	if iamOS.objAPI == nil {
 		return errServerNotInitialized
 	}
+
+	bootstrapTraceMsg("loading all IAM items")
+
 	listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
 	if err != nil {
 		return fmt.Errorf("unable to list IAM data: %w", err)
@@ -395,7 +397,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 
 	// Loads things in the same order as `LoadIAMCache()`
 
-	bootstrapTrace("loading policy documents")
+	bootstrapTraceMsg("loading policy documents")
 
 	policiesList := listedConfigItems[policiesListKey]
 	for _, item := range policiesList {
@@ -407,7 +409,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
 
 	if iamOS.usersSysType == MinIOUsersSysType {
-		bootstrapTrace("loading regular IAM users")
+		bootstrapTraceMsg("loading regular IAM users")
 		regUsersList := listedConfigItems[usersListKey]
 		for _, item := range regUsersList {
 			userName := path.Dir(item)
@@ -416,7 +418,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 			}
 		}
 
-		bootstrapTrace("loading regular IAM groups")
+		bootstrapTraceMsg("loading regular IAM groups")
 		groupsList := listedConfigItems[groupsListKey]
 		for _, item := range groupsList {
 			group := path.Dir(item)
@@ -426,7 +428,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 		}
 	}
 
-	bootstrapTrace("loading user policy mapping")
+	bootstrapTraceMsg("loading user policy mapping")
 	userPolicyMappingsList := listedConfigItems[policyDBUsersListKey]
 	for _, item := range userPolicyMappingsList {
 		userName := strings.TrimSuffix(item, ".json")
@@ -435,7 +437,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 		}
 	}
 
-	bootstrapTrace("loading group policy mapping")
+	bootstrapTraceMsg("loading group policy mapping")
 	groupPolicyMappingsList := listedConfigItems[policyDBGroupsListKey]
 	for _, item := range groupPolicyMappingsList {
 		groupName := strings.TrimSuffix(item, ".json")
@@ -444,7 +446,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 		}
 	}
 
-	bootstrapTrace("loading service accounts")
+	bootstrapTraceMsg("loading service accounts")
 	svcAccList := listedConfigItems[svcAccListKey]
 	for _, item := range svcAccList {
 		userName := path.Dir(item)
@@ -453,7 +455,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 		}
 	}
 
-	bootstrapTrace("loading STS users")
+	bootstrapTraceMsg("loading STS users")
 	stsUsersList := listedConfigItems[stsListKey]
 	for _, item := range stsUsersList {
 		userName := path.Dir(item)
@@ -462,7 +464,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 		}
 	}
 
-	bootstrapTrace("loading STS policy mapping")
+	bootstrapTraceMsg("loading STS policy mapping")
 	stsPolicyMappingsList := listedConfigItems[policyDBSTSUsersListKey]
 	for _, item := range stsPolicyMappingsList {
 		stsName := strings.TrimSuffix(item, ".json")

cmd/iam-store.go

@@ -109,7 +109,7 @@ func getUserIdentityPath(user string, userType IAMUserType) string {
 }
 
 func saveIAMFormat(ctx context.Context, store IAMStorageAPI) error {
-	bootstrapTrace("Load IAM format file")
+	bootstrapTraceMsg("Load IAM format file")
 	var iamFmt iamFormat
 	path := getIAMFormatFilePath()
 	if err := store.loadIAMConfig(ctx, &iamFmt, path); err != nil {
@@ -127,7 +127,7 @@ func saveIAMFormat(ctx context.Context, store IAMStorageAPI) error {
 		return nil
 	}
 
-	bootstrapTrace("Write IAM format file")
+	bootstrapTraceMsg("Write IAM format file")
 	// Save iam format to version 1.
 	if err := store.saveIAMConfig(ctx, newIAMFormatVersion1(), path); err != nil {
 		logger.LogIf(ctx, err)
@@ -462,7 +462,7 @@ func setDefaultCannedPolicies(policies map[string]PolicyDoc) {
 // LoadIAMCache reads all IAM items and populates a new iamCache object and
 // replaces the in-memory cache object.
 func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
-	bootstrapTrace("loading IAM data")
+	bootstrapTraceMsg("loading IAM data")
 
 	newCache := newIamCache()
 
@@ -475,7 +475,7 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
 		}
 	} else {
 
-		bootstrapTrace("loading policy documents")
+		bootstrapTraceMsg("loading policy documents")
 		if err := store.loadPolicyDocs(ctx, newCache.iamPolicyDocsMap); err != nil {
 			return err
 		}
@@ -484,41 +484,41 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
 		setDefaultCannedPolicies(newCache.iamPolicyDocsMap)
 
 		if store.getUsersSysType() == MinIOUsersSysType {
-			bootstrapTrace("loading regular users")
+			bootstrapTraceMsg("loading regular users")
 			if err := store.loadUsers(ctx, regUser, newCache.iamUsersMap); err != nil {
 				return err
 			}
-			bootstrapTrace("loading regular groups")
+			bootstrapTraceMsg("loading regular groups")
 			if err := store.loadGroups(ctx, newCache.iamGroupsMap); err != nil {
 				return err
 			}
 		}
 
-		bootstrapTrace("loading user policy mapping")
+		bootstrapTraceMsg("loading user policy mapping")
 		// load polices mapped to users
 		if err := store.loadMappedPolicies(ctx, regUser, false, newCache.iamUserPolicyMap); err != nil {
 			return err
 		}
 
-		bootstrapTrace("loading group policy mapping")
+		bootstrapTraceMsg("loading group policy mapping")
 		// load policies mapped to groups
 		if err := store.loadMappedPolicies(ctx, regUser, true, newCache.iamGroupPolicyMap); err != nil {
 			return err
 		}
 
-		bootstrapTrace("loading service accounts")
+		bootstrapTraceMsg("loading service accounts")
 		// load service accounts
 		if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil {
 			return err
 		}
 
-		bootstrapTrace("loading STS users")
+		bootstrapTraceMsg("loading STS users")
 		// load STS temp users
 		if err := store.loadUsers(ctx, stsUser, newCache.iamUsersMap); err != nil {
 			return err
 		}
 
-		bootstrapTrace("loading STS policy mapping")
+		bootstrapTraceMsg("loading STS policy mapping")
 		// load STS policy mappings
 		if err := store.loadMappedPolicies(ctx, stsUser, false, newCache.iamUserPolicyMap); err != nil {
 			return err

cmd/iam.go

@@ -187,7 +187,7 @@ func (sys *IAMSys) Initialized() bool {
 }
 
 // Load - loads all credentials, policies and policy mappings.
-func (sys *IAMSys) Load(ctx context.Context) error {
+func (sys *IAMSys) Load(ctx context.Context, firstTime bool) error {
 	loadStartTime := time.Now()
 	err := sys.store.LoadIAMCache(ctx)
 	if err != nil {
@@ -200,6 +200,10 @@ func (sys *IAMSys) Load(ctx context.Context) error {
 	atomic.StoreUint64(&sys.LastRefreshTimeUnixNano, uint64(loadStartTime.Add(loadDuration).UnixNano()))
 	atomic.AddUint64(&sys.TotalRefreshSuccesses, 1)
 
+	if firstTime {
+		bootstrapTraceMsg(fmt.Sprintf("globalIAMSys.Load(): (duration: %s)", loadDuration))
+	}
+
 	select {
 	case <-sys.configLoaded:
 	default:
@@ -210,7 +214,7 @@ func (sys *IAMSys) Load(ctx context.Context) error {
 
 // Init - initializes config system by reading entries from config/iam
 func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, iamRefreshInterval time.Duration) {
-	bootstrapTrace("IAM initialization started")
+	bootstrapTraceMsg("IAM initialization started")
 	globalServerConfigMu.RLock()
 	s := globalServerConfig
 	globalServerConfigMu.RUnlock()
@@ -300,7 +304,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 
 	// Load IAM data from storage.
 	for {
-		if err := sys.Load(retryCtx); err != nil {
+		if err := sys.Load(retryCtx, true); err != nil {
 			if configRetriableErrors(err) {
 				logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v)", err)
 				time.Sleep(time.Duration(r.Float64() * float64(5*time.Second)))
@@ -313,8 +317,6 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		break
 	}
 
-	bootstrapTrace("finishing IAM loading")
-
 	refreshInterval := sys.iamRefreshInterval
 
 	// Set up polling for expired accounts and credentials purging.
@@ -371,6 +373,8 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	}
 
 	sys.printIAMRoles()
+
+	bootstrapTraceMsg("finishing IAM loading")
 }
 
 func (sys *IAMSys) validateAndAddRolePolicyMappings(ctx context.Context, m map[arn.ARN]string) {
@@ -446,7 +450,7 @@ func (sys *IAMSys) watch(ctx context.Context) {
 		select {
 		case <-timer.C:
 			refreshStart := time.Now()
-			if err := sys.Load(ctx); err != nil {
+			if err := sys.Load(ctx, false); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (took %.2fs): %v", time.Since(refreshStart).Seconds(), err))
 			} else {
 				took := time.Since(refreshStart).Seconds()
@@ -590,12 +594,7 @@ func (sys *IAMSys) ListPolicies(ctx context.Context, bucketName string) (map[str
 		return nil, errServerNotInitialized
 	}
 
-	select {
-	case <-sys.configLoaded:
-		return sys.store.ListPolicies(ctx, bucketName)
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	}
+	return sys.store.ListPolicies(ctx, bucketName)
 }
 
 // ListPolicyDocs - lists all canned policy docs.
@@ -604,12 +603,7 @@ func (sys *IAMSys) ListPolicyDocs(ctx context.Context, bucketName string) (map[s
 		return nil, errServerNotInitialized
 	}
 
-	select {
-	case <-sys.configLoaded:
-		return sys.store.ListPolicyDocs(ctx, bucketName)
-	case <-ctx.Done():
-		return nil, ctx.Err()
-	}
+	return sys.store.ListPolicyDocs(ctx, bucketName)
 }
 
 // SetPolicy - sets a new named policy.

cmd/kms-handlers.go

@@ -20,7 +20,9 @@ package cmd
 import (
 	"crypto/subtle"
 	"encoding/json"
+	"io"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/minio/kes-go"
@@ -258,15 +260,26 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
 		return
 	}
-	keys, err := manager.ListKeys(ctx, r.Form.Get("pattern"))
+	keys, err := manager.ListKeys(ctx)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	values, err := keys.Values(0)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
+
+	pattern := r.Form.Get("pattern")
+	if !strings.Contains(pattern, "*") {
+		pattern += "*"
+	}
+
+	var values []kes.KeyInfo
+	for name, err := keys.SeekTo(ctx, pattern); err != io.EOF; name, err = keys.Next(ctx) {
+		if err != nil {
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+			return
+		}
+		values = append(values, kes.KeyInfo{
+			Name: name,
+		})
 	}
 	if res, err := json.Marshal(values); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
@@ -454,37 +467,6 @@ func (a kmsAPIHandlers) KMSAssignPolicyHandler(w http.ResponseWriter, r *http.Re
 	writeSuccessResponseHeadersOnly(w)
 }
 
-// KMSSetPolicyHandler - POST /minio/kms/v1/policy/policy?policy=<policy>
-func (a kmsAPIHandlers) KMSSetPolicyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "KMSSetPolicy")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSSetPolicyAction)
-	if objectAPI == nil {
-		return
-	}
-
-	if GlobalKMS == nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL)
-		return
-	}
-	manager, ok := GlobalKMS.(kms.PolicyManager)
-	if !ok {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
-	}
-	var policy kes.Policy
-	if err := json.NewDecoder(r.Body).Decode(&policy); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	if err := manager.SetPolicy(ctx, r.Form.Get("policy"), &policy); err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-	writeSuccessResponseHeadersOnly(w)
-}
-
 // KMSDeletePolicyHandler - DELETE /minio/kms/v1/policy/delete?policy=<policy>
 func (a kmsAPIHandlers) KMSDeletePolicyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSDeletePolicy")
@@ -531,15 +513,26 @@ func (a kmsAPIHandlers) KMSListPoliciesHandler(w http.ResponseWriter, r *http.Re
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
-	policies, err := manager.ListPolicies(ctx, r.Form.Get("pattern"))
+	policies, err := manager.ListPolicies(ctx)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	values, err := policies.Values(0)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
+
+	pattern := r.Form.Get("pattern")
+	if !strings.Contains(pattern, "*") {
+		pattern += "*"
+	}
+
+	var values []kes.PolicyInfo
+	for name, err := policies.SeekTo(ctx, pattern); err != io.EOF; name, err = policies.Next(ctx) {
+		if err != nil {
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+			return
+		}
+		values = append(values, kes.PolicyInfo{
+			Name: name,
+		})
 	}
 	if res, err := json.Marshal(values); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
@@ -707,15 +700,26 @@ func (a kmsAPIHandlers) KMSListIdentitiesHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
 		return
 	}
-	identities, err := manager.ListIdentities(ctx, r.Form.Get("pattern"))
+	identities, err := manager.ListIdentities(ctx)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	values, err := identities.Values(0)
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
+
+	pattern := r.Form.Get("pattern")
+	if !strings.Contains(pattern, "*") {
+		pattern += "*"
+	}
+
+	var values []kes.IdentityInfo
+	for name, err := identities.SeekTo(ctx, pattern); err != io.EOF; name, err = identities.Next(ctx) {
+		if err != nil {
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+			return
+		}
+		values = append(values, kes.IdentityInfo{
+			Identity: name,
+		})
 	}
 	if res, err := json.Marshal(values); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)

cmd/kms-router.go

@@ -63,7 +63,6 @@ func registerKMSRouter(router *mux.Router) {
 		kmsRouter.Methods(http.MethodGet).Path(version + "/key/status").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSKeyStatusHandler)))
 
 		// KMS Policy APIs
-		kmsRouter.Methods(http.MethodPost).Path(version+"/policy/set").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSSetPolicyHandler))).Queries("policy", "{policy:.*}")
 		kmsRouter.Methods(http.MethodPost).Path(version+"/policy/assign").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSAssignPolicyHandler))).Queries("policy", "{policy:.*}")
 		kmsRouter.Methods(http.MethodGet).Path(version+"/policy/describe").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSDescribePolicyHandler))).Queries("policy", "{policy:.*}")
 		kmsRouter.Methods(http.MethodGet).Path(version+"/policy/get").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSGetPolicyHandler))).Queries("policy", "{policy:.*}")

cmd/main.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
+	"github.com/minio/pkg/env"
 	"github.com/minio/pkg/trie"
 	"github.com/minio/pkg/words"
 )
@@ -191,7 +192,7 @@ func Main(args []string) {
 	// Set the minio app name.
 	appName := filepath.Base(args[0])
 
-	if os.Getenv("_MINIO_DEBUG_NO_EXIT") != "" {
+	if env.Get("_MINIO_DEBUG_NO_EXIT", "") != "" {
 		freeze := func(_ int) {
 			// Infinite blocking op
 			<-make(chan struct{})

cmd/metacache-server-pool.go

@@ -129,7 +129,8 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 				return entries, io.EOF
 			}
 			if !errors.Is(err, context.DeadlineExceeded) {
-				o.debugln("listPath: got error", err)
+				// Report error once per bucket, but continue listing.
+				logger.LogOnceIf(ctx, err, "GetMetacacheListing:"+o.Bucket)
 			}
 			o.Transient = true
 			o.Create = false

cmd/metacache-set.go

@@ -436,31 +436,29 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				break
 			}
 		}
-
+		retryWait := func() {
+			retries++
+			if retries == 1 {
+				time.Sleep(retryDelay)
+			} else {
+				time.Sleep(retryDelay250)
+			}
+		}
 		// Load first part metadata...
 		// Read metadata associated with the object from all disks.
 		fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(0), ObjectOptions{}, true)
 		if err != nil {
 			switch toObjectErr(err, minioMetaBucket, o.objectPath(0)).(type) {
-			case ObjectNotFound:
-				retries++
-				if retries == 1 {
-					time.Sleep(retryDelay)
-				} else {
-					time.Sleep(retryDelay250)
-				}
+			case ObjectNotFound, InsufficientReadQuorum:
+				retryWait()
 				continue
-			case InsufficientReadQuorum:
-				retries++
-				if retries == 1 {
-					time.Sleep(retryDelay)
-				} else {
-					time.Sleep(retryDelay250)
-				}
-				continue
-			default:
-				return entries, fmt.Errorf("reading first part metadata: %w", err)
 			}
+			// Allow one fast retry for other errors.
+			if retries > 0 {
+				return entries, fmt.Errorf("reading first part metadata: %v", err)
+			}
+			retryWait()
+			continue
 		}
 
 		partN, err := o.findFirstPart(fi)
@@ -474,8 +472,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				}
 				retries = -1
 			}
-			retries++
-			time.Sleep(retryDelay250)
+			retryWait()
 			continue
 		case errors.Is(err, io.EOF):
 			return entries, io.EOF
@@ -597,14 +594,11 @@ func getListQuorum(quorum string, driveCount int) int {
 		return 1
 	case "reduced":
 		return 2
-	case "strict":
-		return driveCount
-	}
-	// Defaults to (driveCount+1)/2 drives per set, defaults to "optimal" value
-	if driveCount > 0 {
+	case "optimal":
 		return (driveCount + 1) / 2
-	} // "3" otherwise.
-	return 3
+	}
+	// defaults to 'strict'
+	return driveCount
 }
 
 // Will return io.EOF if continuing would not yield more results.
@@ -791,7 +785,7 @@ func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCache
 			for k, v := range meta {
 				fi.Metadata[k] = v
 			}
-			err := er.updateObjectMeta(ctx, minioMetaBucket, o.objectPath(0), fi, er.getDisks())
+			err := er.updateObjectMetaWithOpts(ctx, minioMetaBucket, o.objectPath(0), fi, er.getDisks(), UpdateMetadataOpts{NoPersistence: true})
 			if err == nil {
 				break
 			}

cmd/metacache-walk.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -69,16 +69,22 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		return err
 	}
 
-	// Stat a volume entry.
-	if err = Access(volumeDir); err != nil {
-		if osIsNotExist(err) {
-			return errVolumeNotFound
-		} else if isSysErrIO(err) {
-			return errFaultyDisk
+	if !skipAccessChecks(opts.Bucket) {
+		// Stat a volume entry.
+		if err = Access(volumeDir); err != nil {
+			if osIsNotExist(err) {
+				return errVolumeNotFound
+			} else if isSysErrIO(err) {
+				return errFaultyDisk
+			}
+			return err
 		}
-		return err
 	}
 
+	s.RLock()
+	legacy := s.formatLegacy
+	s.RUnlock()
+
 	// Use a small block size to start sending quickly
 	w := newMetacacheWriter(wr, 16<<10)
 	w.reuseBlocks = true // We are not sharing results, so reuse buffers.
@@ -98,6 +104,15 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			objsReturned++
 		}
 	}
+	send := func(entry metaCacheEntry) error {
+		objReturned(entry.metadata)
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		case out <- entry:
+		}
+		return nil
+	}
 
 	// Fast exit track to check if we are listing an object with
 	// a trailing slash, this will avoid to list the object content.
@@ -105,15 +120,17 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		metadata, err := s.readMetadata(ctx, pathJoin(volumeDir,
 			opts.BaseDir[:len(opts.BaseDir)-1]+globalDirSuffix,
 			xlStorageFormatFile))
+		diskHealthCheckOK(ctx, err)
 		if err == nil {
 			// if baseDir is already a directory object, consider it
 			// as part of the list call, this is AWS S3 specific
 			// behavior.
-			out <- metaCacheEntry{
+			if err := send(metaCacheEntry{
 				name:     opts.BaseDir,
 				metadata: metadata,
+			}); err != nil {
+				return err
 			}
-			objReturned(metadata)
 		} else {
 			st, sterr := Lstat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile))
 			if sterr == nil && st.Mode().IsRegular() {
@@ -143,19 +160,25 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			return nil
 		}
 
-		s.walkMu.Lock()
+		if s.walkMu != nil {
+			s.walkMu.Lock()
+		}
 		entries, err := s.ListDir(ctx, opts.Bucket, current, -1)
-		s.walkMu.Unlock()
+		if s.walkMu != nil {
+			s.walkMu.Unlock()
+		}
 		if err != nil {
 			// Folder could have gone away in-between
 			if err != errVolumeNotFound && err != errFileNotFound {
 				logger.LogOnceIf(ctx, err, "metacache-walk-scan-dir")
 			}
 			if opts.ReportNotFound && err == errFileNotFound && current == opts.BaseDir {
-				return errFileNotFound
+				err = errFileNotFound
+			} else {
+				err = nil
 			}
-			// Forward some errors?
-			return nil
+			diskHealthCheckOK(ctx, err)
+			return err
 		}
 		diskHealthCheckOK(ctx, err)
 		if len(entries) == 0 {
@@ -202,9 +225,13 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 			// If root was an object return it as such.
 			if HasSuffix(entry, xlStorageFormatFile) {
 				var meta metaCacheEntry
-				s.walkReadMu.Lock()
+				if s.walkReadMu != nil {
+					s.walkReadMu.Lock()
+				}
 				meta.metadata, err = s.readMetadata(ctx, pathJoinBuf(&sb, volumeDir, current, entry))
-				s.walkReadMu.Unlock()
+				if s.walkReadMu != nil {
+					s.walkReadMu.Unlock()
+				}
 				diskHealthCheckOK(ctx, err)
 				if err != nil {
 					// It is totally possible that xl.meta was overwritten
@@ -219,17 +246,15 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				meta.name = strings.TrimSuffix(meta.name, SlashSeparator)
 				meta.name = pathJoinBuf(&sb, current, meta.name)
 				meta.name = decodeDirObject(meta.name)
-
-				objReturned(meta.metadata)
-				out <- meta
+				if err := send(meta); err != nil {
+					return err
+				}
 				return nil
 			}
 			// Check legacy.
-			if HasSuffix(entry, xlStorageFormatFileV1) {
+			if HasSuffix(entry, xlStorageFormatFileV1) && legacy {
 				var meta metaCacheEntry
-				s.walkReadMu.Lock()
 				meta.metadata, err = xioutil.ReadFile(pathJoinBuf(&sb, volumeDir, current, entry))
-				s.walkReadMu.Unlock()
 				diskHealthCheckOK(ctx, err)
 				if err != nil {
 					if !IsErrIgnored(err, io.EOF, io.ErrUnexpectedEOF) {
@@ -240,9 +265,9 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				meta.name = strings.TrimSuffix(entry, xlStorageFormatFileV1)
 				meta.name = strings.TrimSuffix(meta.name, SlashSeparator)
 				meta.name = pathJoinBuf(&sb, current, meta.name)
-				objReturned(meta.metadata)
-
-				out <- meta
+				if err := send(meta); err != nil {
+					return err
+				}
 				return nil
 			}
 			// Skip all other files.
@@ -295,9 +320,13 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				meta.name = meta.name[:len(meta.name)-1] + globalDirSuffixWithSlash
 			}
 
-			s.walkReadMu.Lock()
+			if s.walkReadMu != nil {
+				s.walkReadMu.Lock()
+			}
 			meta.metadata, err = s.readMetadata(ctx, pathJoinBuf(&sb, volumeDir, meta.name, xlStorageFormatFile))
-			s.walkReadMu.Unlock()
+			if s.walkReadMu != nil {
+				s.walkReadMu.Unlock()
+			}
 			diskHealthCheckOK(ctx, err)
 			switch {
 			case err == nil:
@@ -305,18 +334,20 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 				if isDirObj {
 					meta.name = strings.TrimSuffix(meta.name, globalDirSuffixWithSlash) + slashSeparator
 				}
-				objReturned(meta.metadata)
-
-				out <- meta
+				if err := send(meta); err != nil {
+					return err
+				}
 			case osIsNotExist(err), isSysErrIsDir(err):
-				meta.metadata, err = xioutil.ReadFile(pathJoinBuf(&sb, volumeDir, meta.name, xlStorageFormatFileV1))
-				diskHealthCheckOK(ctx, err)
-				if err == nil {
-					// It was an object
-					objReturned(meta.metadata)
-
-					out <- meta
-					continue
+				if legacy {
+					meta.metadata, err = xioutil.ReadFile(pathJoinBuf(&sb, volumeDir, meta.name, xlStorageFormatFileV1))
+					diskHealthCheckOK(ctx, err)
+					if err == nil {
+						// It was an object
+						if err := send(meta); err != nil {
+							return err
+						}
+						continue
+					}
 				}
 
 				// NOT an object, append to stack (with slash)

cmd/metrics-realtime.go

@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"context"
+	"net/http"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
@@ -38,8 +39,14 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 		return
 	}
 
+	byHostName := globalMinioAddr
 	if len(opts.hosts) > 0 {
-		if _, ok := opts.hosts[globalMinioAddr]; !ok {
+		server := getLocalServerProperty(globalEndpoints, &http.Request{
+			Host: globalLocalNodeName,
+		})
+		if _, ok := opts.hosts[server.Endpoint]; ok {
+			byHostName = server.Endpoint
+		} else {
 			return
 		}
 	}
@@ -85,8 +92,8 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 	// Add types...
 
 	// ByHost is a shallow reference, so careful about sharing.
-	m.ByHost = map[string]madmin.Metrics{globalMinioAddr: m.Aggregated}
-	m.Hosts = append(m.Hosts, globalMinioAddr)
+	m.ByHost = map[string]madmin.Metrics{byHostName: m.Aggregated}
+	m.Hosts = append(m.Hosts, byHostName)
 
 	return m
 }