fix incorrect disk io stats in k8s environment (#19016)

The previous logic of calculating per second values for disk io stats
divides the stats by the host uptime. This doesn't work in k8s
environment as the uptime is of the pod, but the stats (from
/proc/diskstats) are from the host.

Fix this by storing the initial values of uptime and the stats at the
timme of server startup, and using the difference between current and
initial values when calculating the per second values.

.github/workflows/multipart/migrate.sh

@@ -26,7 +26,7 @@ fi
 
 (
 	cd /tmp
-	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@master
 )
 
 export RELEASE=RELEASE.2023-08-29T23-07-35Z

Makefile

@@ -46,7 +46,7 @@ lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 check: test
 test: verifiers build build-debugging ## builds minio, runs linters, tests
 	@echo "Running unit tests"
-	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...
+	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -v -tags kqueue ./...
 
 test-root-disable: install-race
 	@echo "Running minio root lockdown tests"
@@ -116,7 +116,6 @@ verify-healing: ## verify healing and replacing disks with minio binary
 	@echo "Verify healing build with race"
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
-	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
 	@(env bash $(PWD)/buildscripts/heal-inconsistent-versions.sh)
 
 verify-healing-with-root-disks: ## verify healing root disks
@@ -149,7 +148,7 @@ hotfix-vars:
 
 hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags
 	@wget -q -c https://github.com/minio/pkger/releases/download/v2.2.0/pkger_2.2.0_linux_amd64.deb
-	@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.0.0/linux-systemd/distributed/minio.service
+	@wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.0.1/linux-systemd/distributed/minio.service
 	@sudo apt install ./pkger_2.2.0_linux_amd64.deb --yes
 	@mkdir -p minio-release/$(GOOS)-$(GOARCH)/archive
 	@cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio

buildscripts/rewrite-old-new.sh

@@ -87,7 +87,7 @@ function verify_rewrite() {
 		exit 1
 	fi
 
-	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@master
 	if ! s3-check-md5 \
 		-debug \
 		-versions \

buildscripts/unaligned-healing.sh

@@ -1,177 +0,0 @@
-#!/bin/bash -e
-#
-
-set -E
-set -o pipefail
-set -x
-
-if [ ! -x "$PWD/minio" ]; then
-	echo "minio executable binary not found in current directory"
-	exit 1
-fi
-
-WORK_DIR="$PWD/.verify-$RANDOM"
-MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO_OLD=("$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server)
-MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
-
-function download_old_release() {
-	if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
-		curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
-		chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
-	fi
-}
-
-function start_minio_16drive() {
-	start_port=$1
-
-	export MINIO_ROOT_USER=minio
-	export MINIO_ROOT_PASSWORD=minio123
-	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
-	unset MINIO_KMS_AUTO_ENCRYPTION         # do not auto-encrypt objects
-	export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
-	export MINIO_CI_CD=1
-
-	MC_BUILD_DIR="mc-$RANDOM"
-	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-		echo "failed to download https://github.com/minio/mc"
-		purge "${MC_BUILD_DIR}"
-		exit 1
-	fi
-
-	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
-
-	# remove mc source.
-	purge "${MC_BUILD_DIR}"
-
-	"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
-	pid=$!
-	disown $pid
-	sleep 30
-
-	if ! ps -p ${pid} 1>&2 >/dev/null; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
-	"${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
-	"${WORK_DIR}/mc" cp \
-		"${WORK_DIR}/unaligned" \
-		minio/healing-shard-bucket/unaligned \
-		--disable-multipart --quiet
-
-	## "unaligned" object name gets consistently distributed
-	## to disks in following distribution order
-	##
-	## NOTE: if you change the name make sure to change the
-	## distribution order present here
-	##
-	## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
-
-	## make sure to remove the "last" data shard
-	rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
-	sleep 10
-	## Heal the shard
-	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
-	## then remove any other data shard let's pick first disk
-	## - 1st data shard.
-	rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
-	sleep 10
-
-	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
-	if ! s3-check-md5 \
-		-debug \
-		-access-key minio \
-		-secret-key minio123 \
-		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	pkill minio
-	sleep 3
-
-	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
-	pid=$!
-	disown $pid
-	sleep 30
-
-	if ! ps -p ${pid} 1>&2 >/dev/null; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	if ! s3-check-md5 \
-		-debug \
-		-access-key minio \
-		-secret-key minio123 \
-		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		mkdir -p inspects
-		(
-			cd inspects
-			"${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**
-		)
-
-		"${WORK_DIR}/mc" mb play/inspects
-		"${WORK_DIR}/mc" mirror inspects play/inspects
-
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
-
-	if ! s3-check-md5 \
-		-debug \
-		-access-key minio \
-		-secret-key minio123 \
-		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
-		echo "server1 log:"
-		cat "${WORK_DIR}/server1.log"
-		echo "FAILED"
-		mkdir -p inspects
-		(
-			cd inspects
-			"${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**
-		)
-
-		"${WORK_DIR}/mc" mb play/inspects
-		"${WORK_DIR}/mc" mirror inspects play/inspects
-
-		purge "$WORK_DIR"
-		exit 1
-	fi
-
-	pkill minio
-	sleep 3
-}
-
-function main() {
-	download_old_release
-
-	start_port=$(shuf -i 10000-65000 -n 1)
-
-	start_minio_16drive ${start_port}
-}
-
-function purge() {
-	rm -rf "$1"
-}
-
-(main "$@")
-rv=$?
-purge "$WORK_DIR"
-exit "$rv"

cmd/admin-bucket-handlers.go

@@ -680,24 +680,17 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			if _, ok := bucketMap[bucket]; !ok {
 				opts := MakeBucketOptions{
 					LockEnabled: config.Enabled(),
+					ForceCreate: true, // ignore if it already exists
 				}
 				err = objectAPI.MakeBucket(ctx, bucket, opts)
 				if err != nil {
-					if _, ok := err.(BucketExists); !ok {
-						rpt.SetStatus(bucket, fileName, err)
-						continue
-					}
+					rpt.SetStatus(bucket, fileName, err)
+					continue
 				}
 				v := newBucketMetadata(bucket)
 				bucketMap[bucket] = &v
 			}
 
-			// Deny object locking configuration settings on existing buckets without object lock enabled.
-			if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
-				rpt.SetStatus(bucket, fileName, err)
-				continue
-			}
-
 			bucketMap[bucket].ObjectLockConfigXML = configData
 			bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt
 			rpt.SetStatus(bucket, fileName, nil)
@@ -724,11 +717,11 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 			if _, ok := bucketMap[bucket]; !ok {
-				if err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{}); err != nil {
-					if _, ok := err.(BucketExists); !ok {
-						rpt.SetStatus(bucket, fileName, err)
-						continue
-					}
+				if err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{
+					ForceCreate: true, // ignore if it already exists
+				}); err != nil {
+					rpt.SetStatus(bucket, fileName, err)
+					continue
 				}
 				v := newBucketMetadata(bucket)
 				bucketMap[bucket] = &v
@@ -776,12 +769,12 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 
 		// create bucket if it does not exist yet.
 		if _, ok := bucketMap[bucket]; !ok {
-			err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{})
+			err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{
+				ForceCreate: true, // ignore if it already exists
+			})
 			if err != nil {
-				if _, ok := err.(BucketExists); !ok {
-					rpt.SetStatus(bucket, "", err)
-					continue
-				}
+				rpt.SetStatus(bucket, "", err)
+				continue
 			}
 			v := newBucketMetadata(bucket)
 			bucketMap[bucket] = &v

cmd/admin-handlers-config-kv.go

@@ -58,7 +58,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		logger.LogIf(ctx, err, logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -162,7 +162,7 @@ func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		logger.LogIf(ctx, err, logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
@@ -443,7 +443,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 	password := cred.SecretKey
 	kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		logger.LogIf(ctx, err, logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}

cmd/admin-handlers-idp-config.go

@@ -60,7 +60,7 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		logger.LogIf(ctx, err, logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}

cmd/admin-handlers-idp-ldap.go

@@ -132,7 +132,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		logger.LogIf(ctx, err, logger.Application)
+		logger.LogIf(ctx, err, logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}

cmd/admin-handlers-users.go

@@ -27,6 +27,7 @@ import (
 	"net/http"
 	"os"
 	"sort"
+	"strconv"
 	"time"
 
 	"github.com/klauspost/compress/zip"
@@ -774,25 +775,6 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if !globalIAMSys.IsAllowed(policy.Args{
-		AccountName:     cred.AccessKey,
-		Groups:          cred.Groups,
-		Action:          policy.UpdateServiceAccountAdminAction,
-		ConditionValues: getConditionValues(r, "", cred),
-		IsOwner:         owner,
-		Claims:          cred.Claims,
-	}) {
-		requestUser := cred.AccessKey
-		if cred.ParentUser != "" {
-			requestUser = cred.ParentUser
-		}
-
-		if requestUser != svcAccount.ParentUser {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
-	}
-
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
@@ -813,6 +795,31 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	condValues := getConditionValues(r, "", cred)
+	addExpirationToCondValues(updateReq.NewExpiration, condValues)
+
+	// Permission checks:
+	//
+	// 1. Any type of account (i.e. access keys (previously/still called service
+	// accounts), STS accounts, internal IDP accounts, etc) with the
+	// policy.UpdateServiceAccountAdminAction permission can update any service
+	// account.
+	//
+	// 2. We would like to let a user update their own access keys, however it
+	// is currently blocked pending a re-design. Users are still able to delete
+	// and re-create them.
+	if !globalIAMSys.IsAllowed(policy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          policy.UpdateServiceAccountAdminAction,
+		ConditionValues: condValues,
+		IsOwner:         owner,
+		Claims:          cred.Claims,
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
+	}
+
 	var sp *policy.Policy
 	if len(updateReq.NewPolicy) > 0 {
 		sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
@@ -2414,6 +2421,13 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 	}
 }
 
+func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) {
+	if exp == nil {
+		return
+	}
+	condValues["DurationSeconds"] = []string{strconv.FormatInt(int64(exp.Sub(time.Now()).Seconds()), 10)}
+}
+
 func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) {
 	ctx := r.Context()
 
@@ -2469,13 +2483,16 @@ func commonAddServiceAccount(r *http.Request) (context.Context, auth.Credentials
 		claims:      make(map[string]interface{}),
 	}
 
+	condValues := getConditionValues(r, "", cred)
+	addExpirationToCondValues(createReq.Expiration, condValues)
+
 	// Check if action is allowed if creating access key for another user
 	// Check if action is explicitly denied if for self
 	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          policy.CreateServiceAccountAdminAction,
-		ConditionValues: getConditionValues(r, "", cred),
+		ConditionValues: condValues,
 		IsOwner:         owner,
 		Claims:          cred.Claims,
 		DenyOnly:        (targetUser == cred.AccessKey || targetUser == cred.ParentUser),

cmd/admin-handlers-users_test.go

@@ -206,7 +206,9 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
 				suite.TestCannedPolicies(c)
 				suite.TestGroupAddRemove(c)
 				suite.TestServiceAccountOpsByAdmin(c)
+				suite.TestServiceAccountPrivilegeEscalationBug(c)
 				suite.TestServiceAccountOpsByUser(c)
+				suite.TestServiceAccountDurationSecondsCondition(c)
 				suite.TestAddServiceAccountPerms(c)
 				suite.TearDownSuite(c)
 			},
@@ -896,14 +898,6 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 	// 3. Check S3 access
 	c.assertSvcAccS3Access(ctx, s, cr, bucket)
 
-	// 4. Check that svc account can restrict the policy, and that the
-	// session policy can be updated.
-	c.assertSvcAccSessionPolicyUpdate(ctx, s, userAdmClient, accessKey, bucket)
-
-	// 4. Check that service account's secret key and account status can be
-	// updated.
-	c.assertSvcAccSecretKeyAndStatusUpdate(ctx, s, userAdmClient, accessKey, bucket)
-
 	// 5. Check that service account can be deleted.
 	c.assertSvcAccDeletion(ctx, s, userAdmClient, accessKey, bucket)
 
@@ -911,6 +905,93 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 	c.mustNotCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient)
 }
 
+func (s *TestSuiteIAM) TestServiceAccountDurationSecondsCondition(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// Create policy, user and associate policy
+	policy := "mypolicy"
+	policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Deny",
+   "Action": [
+     "admin:CreateServiceAccount",
+     "admin:UpdateServiceAccount"
+   ],
+   "Condition": {"NumericGreaterThan": {"svc:DurationSeconds": "3600"}}
+  },
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+
+	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+
+	// Create an madmin client with user creds
+	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  credentials.NewStaticV4(accessKey, secretKey, ""),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating user admin client: %v", err)
+	}
+	userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	distantExpiration := time.Now().Add(30 * time.Minute)
+	cr, err := userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+		TargetUser: accessKey,
+		AccessKey:  "svc-accesskey",
+		SecretKey:  "svc-secretkey",
+		Expiration: &distantExpiration,
+	})
+	if err != nil {
+		c.Fatalf("Unable to create svc acc: %v", err)
+	}
+
+	c.assertSvcAccS3Access(ctx, s, cr, bucket)
+
+	closeExpiration := time.Now().Add(2 * time.Hour)
+	_, err = userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+		TargetUser: accessKey,
+		AccessKey:  "svc-accesskey",
+		SecretKey:  "svc-secretkey",
+		Expiration: &closeExpiration,
+	})
+	if err == nil {
+		c.Fatalf("Creating a svc acc with distant expiration should fail")
+	}
+}
+
 func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()
@@ -979,6 +1060,95 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 	c.assertSvcAccDeletion(ctx, s, s.adm, accessKey, bucket)
 }
 
+func (s *TestSuiteIAM) TestServiceAccountPrivilegeEscalationBug(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	err := s.client.MakeBucket(ctx, "public", minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	err = s.client.MakeBucket(ctx, "private", minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	pubPolicyBytes := []byte(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:*"
+   ],
+   "Resource": [
+    "arn:aws:s3:::public",
+    "arn:aws:s3:::public/*"
+   ]
+  }
+ ]
+}`)
+
+	fullS3PolicyBytes := []byte(`{
+  "Version": "2012-10-17",
+  "Statement": [
+    {
+      "Effect": "Allow",
+      "Action": [
+        "s3:*"
+      ],
+      "Resource": [
+        "arn:aws:s3:::*"
+      ]
+    }
+  ]
+}
+`)
+
+	// Create a service account for the root user.
+	cr, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
+		TargetUser: globalActiveCred.AccessKey,
+		Policy:     pubPolicyBytes,
+	})
+	if err != nil {
+		c.Fatalf("admin should be able to create service account for themselves %s", err)
+	}
+
+	svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "")
+
+	// Check that the service account can access the public bucket.
+	buckets, err := svcClient.ListBuckets(ctx)
+	if err != nil {
+		c.Fatalf("err fetching buckets %s", err)
+	}
+	if len(buckets) != 1 || buckets[0].Name != "public" {
+		c.Fatalf("service account should only have access to public bucket")
+	}
+
+	// Create an madmin client with the service account creds.
+	svcAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
+		Creds:  credentials.NewStaticV4(cr.AccessKey, cr.SecretKey, ""),
+		Secure: s.secure,
+	})
+	if err != nil {
+		c.Fatalf("Err creating svcacct admin client: %v", err)
+	}
+	svcAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport)
+
+	// Attempt to update the policy on the service account.
+	err = svcAdmClient.UpdateServiceAccount(ctx, cr.AccessKey,
+		madmin.UpdateServiceAccountReq{
+			NewPolicy: fullS3PolicyBytes,
+		})
+
+	if err == nil {
+		c.Fatalf("service account should not be able to update policy on itself")
+	} else if !strings.Contains(err.Error(), "Access Denied") {
+		c.Fatalf("unexpected error: %v", err)
+	}
+}
+
 func (s *TestSuiteIAM) SetUpAccMgmtPlugin(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()

cmd/admin-handlers.go

@@ -50,6 +50,7 @@ import (
 	"github.com/minio/madmin-go/v3/estream"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/dsync"
+	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
@@ -79,6 +80,213 @@ const (
 	mgmtForceStop   = "forceStop"
 )
 
+// ServerUpdateV2Handler - POST /minio/admin/v3/update?updateURL={updateURL}&type=2
+// ----------
+// updates all minio servers and restarts them gracefully.
+func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerUpdateAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if globalInplaceUpdateDisabled || currentReleaseTime.IsZero() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
+		return
+	}
+
+	vars := mux.Vars(r)
+	updateURL := vars["updateURL"]
+	dryRun := r.Form.Get("dry-run") == "true"
+
+	mode := getMinioMode()
+	if updateURL == "" {
+		updateURL = minioReleaseInfoURL
+		if runtime.GOOS == globalWindowsOSName {
+			updateURL = minioReleaseWindowsInfoURL
+		}
+	}
+
+	u, err := url.Parse(updateURL)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	content, err := downloadReleaseURL(u, updateTimeout, mode)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	sha256Sum, lrTime, releaseInfo, err := parseReleaseData(content)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
+	// Download Binary Once
+	binC, bin, err := downloadBinary(u, mode)
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	updateStatus := madmin.ServerUpdateStatusV2{DryRun: dryRun}
+	peerResults := make(map[string]madmin.ServerPeerUpdateStatus)
+
+	local := globalLocalNodeName
+	if local == "" {
+		local = "127.0.0.1"
+	}
+
+	failedClients := make(map[int]struct{})
+
+	if globalIsDistErasure {
+		// Push binary to other servers
+		for idx, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, binC) {
+			if nerr.Err != nil {
+				peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
+					Host:           nerr.Host.String(),
+					Err:            nerr.Err.Error(),
+					CurrentVersion: Version,
+				}
+				failedClients[idx] = struct{}{}
+			} else {
+				peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
+					Host:           nerr.Host.String(),
+					CurrentVersion: Version,
+					UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+				}
+			}
+		}
+	}
+
+	if lrTime.Sub(currentReleaseTime) > 0 {
+		if err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin)); err != nil {
+			peerResults[local] = madmin.ServerPeerUpdateStatus{
+				Host:           local,
+				Err:            err.Error(),
+				CurrentVersion: Version,
+			}
+		} else {
+			peerResults[local] = madmin.ServerPeerUpdateStatus{
+				Host:           local,
+				CurrentVersion: Version,
+				UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+			}
+		}
+	} else {
+		peerResults[local] = madmin.ServerPeerUpdateStatus{
+			Host:           local,
+			Err:            fmt.Sprintf("server is already running the latest version: %s", Version),
+			CurrentVersion: Version,
+		}
+	}
+
+	if !dryRun {
+		if globalIsDistErasure {
+			ng := WithNPeers(len(globalNotificationSys.peerClients))
+			for idx, client := range globalNotificationSys.peerClients {
+				_, ok := failedClients[idx]
+				if ok {
+					continue
+				}
+				client := client
+				ng.Go(ctx, func() error {
+					return client.CommitBinary(ctx)
+				}, idx, *client.host)
+			}
+
+			for _, nerr := range ng.Wait() {
+				if nerr.Err != nil {
+					prs, ok := peerResults[nerr.Host.String()]
+					if ok {
+						prs.Err = nerr.Err.Error()
+						peerResults[nerr.Host.String()] = prs
+					} else {
+						peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{
+							Host:           nerr.Host.String(),
+							Err:            nerr.Err.Error(),
+							CurrentVersion: Version,
+							UpdatedVersion: lrTime.Format(minioReleaseTagTimeLayout),
+						}
+					}
+				}
+			}
+		}
+		prs := peerResults[local]
+		if prs.Err == "" {
+			if err = commitBinary(); err != nil {
+				prs.Err = err.Error()
+			}
+			peerResults[local] = prs
+		}
+	}
+
+	prs, ok := peerResults[local]
+	if ok {
+		prs.WaitingDrives = waitingDrivesNode()
+		peerResults[local] = prs
+	}
+
+	if globalIsDistErasure {
+		// Notify all other MinIO peers signal service.
+		ng := WithNPeers(len(globalNotificationSys.peerClients))
+		for idx, client := range globalNotificationSys.peerClients {
+			_, ok := failedClients[idx]
+			if ok {
+				continue
+			}
+			client := client
+			ng.Go(ctx, func() error {
+				prs, ok := peerResults[client.String()]
+				if ok && prs.CurrentVersion != prs.UpdatedVersion && prs.UpdatedVersion != "" {
+					return client.SignalService(serviceRestart, "", dryRun)
+				}
+				return nil
+			}, idx, *client.host)
+		}
+
+		for _, nerr := range ng.Wait() {
+			if nerr.Err != nil {
+				waitingDrives := map[string]madmin.DiskMetrics{}
+				jerr := json.Unmarshal([]byte(nerr.Err.Error()), &waitingDrives)
+				if jerr == nil {
+					prs, ok := peerResults[nerr.Host.String()]
+					if ok {
+						prs.WaitingDrives = waitingDrives
+						peerResults[nerr.Host.String()] = prs
+					}
+					continue
+				}
+			}
+		}
+	}
+
+	for _, pr := range peerResults {
+		updateStatus.Results = append(updateStatus.Results, pr)
+	}
+
+	// Marshal API response
+	jsonBytes, err := json.Marshal(updateStatus)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	writeSuccessResponseJSON(w, jsonBytes)
+
+	if !dryRun {
+		if lrTime.Sub(currentReleaseTime) > 0 {
+			globalServiceSignalCh <- serviceRestart
+		}
+	}
+}
+
 // ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL}
 // ----------
 // updates all minio servers and restarts them gracefully.
@@ -90,7 +298,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if globalInplaceUpdateDisabled {
+	if globalInplaceUpdateDisabled || currentReleaseTime.IsZero() {
 		// if MINIO_UPDATE=off - inplace update is disabled, mostly in containers.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL)
 		return
@@ -124,14 +332,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
-	crTime, err := GetCurrentReleaseTime()
-	if err != nil {
-		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
-		return
-	}
-
-	if lrTime.Sub(crTime) <= 0 {
+	if lrTime.Sub(currentReleaseTime) <= 0 {
 		updateStatus := madmin.ServerUpdateStatus{
 			CurrentVersion: Version,
 			UpdatedVersion: Version,
@@ -148,8 +349,10 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
+	u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo
+
 	// Download Binary Once
-	reader, err := downloadBinary(u, mode)
+	binC, bin, err := downloadBinary(u, mode)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -157,7 +360,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 	}
 
 	// Push binary to other servers
-	for _, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, reader) {
+	for _, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, binC) {
 		if nerr.Err != nil {
 			err := AdminError{
 				Code:       AdminUpdateApplyFailure,
@@ -171,7 +374,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
-	err = verifyBinary(u, sha256Sum, releaseInfo, mode, reader)
+	err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin))
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("server update failed with %w", err))
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -248,7 +451,7 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	case madmin.ServiceActionUnfreeze:
 		serviceSig = serviceUnFreeze
 	default:
-		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.Application)
+		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
 		return
 	}
@@ -287,6 +490,131 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
+type servicePeerResult struct {
+	Host          string                        `json:"host"`
+	Err           string                        `json:"err,omitempty"`
+	WaitingDrives map[string]madmin.DiskMetrics `json:"waitingDrives,omitempty"`
+}
+
+type serviceResult struct {
+	Action  madmin.ServiceAction `json:"action"`
+	DryRun  bool                 `json:"dryRun"`
+	Results []servicePeerResult  `json:"results,omitempty"`
+}
+
+// ServiceV2Handler - POST /minio/admin/v3/service?action={action}&type=2
+// ----------
+// Supports following actions:
+// - restart (restarts all the MinIO instances in a setup)
+// - stop (stops all the MinIO instances in a setup)
+// - freeze (freezes all incoming S3 API calls)
+// - unfreeze (unfreezes previously frozen S3 API calls)
+//
+// This newer API now returns back status per remote peer and local regarding
+// if a "restart/stop" was successful or not. Service signal now supports
+// a dry-run that helps skip the nodes that may have hung drives. By default
+// restart/stop will ignore the servers that are hung on drives. You can use
+// 'force' param to force restart even with hung drives if needed.
+func (a adminAPIHandlers) ServiceV2Handler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	vars := mux.Vars(r)
+	action := vars["action"]
+	dryRun := r.Form.Get("dry-run") == "true"
+
+	var serviceSig serviceSignal
+	act := madmin.ServiceAction(action)
+	switch act {
+	case madmin.ServiceActionRestart:
+		serviceSig = serviceRestart
+	case madmin.ServiceActionStop:
+		serviceSig = serviceStop
+	case madmin.ServiceActionFreeze:
+		serviceSig = serviceFreeze
+	case madmin.ServiceActionUnfreeze:
+		serviceSig = serviceUnFreeze
+	default:
+		logger.LogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL)
+		return
+	}
+
+	var objectAPI ObjectLayer
+	switch serviceSig {
+	case serviceRestart:
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceRestartAdminAction)
+	case serviceStop:
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceStopAdminAction)
+	case serviceFreeze, serviceUnFreeze:
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceFreezeAdminAction)
+	}
+	if objectAPI == nil {
+		return
+	}
+
+	// Notify all other MinIO peers signal service.
+	srvResult := serviceResult{Action: act, Results: []servicePeerResult{}}
+
+	process := act == madmin.ServiceActionRestart || act == madmin.ServiceActionStop
+	if process {
+		localhost := globalLocalNodeName
+		if globalLocalNodeName == "" {
+			localhost = "127.0.0.1"
+		}
+		waitingDrives := waitingDrivesNode()
+		srvResult.Results = append(srvResult.Results, servicePeerResult{
+			Host:          localhost,
+			WaitingDrives: waitingDrives,
+		})
+	}
+
+	if globalIsDistErasure {
+		for _, nerr := range globalNotificationSys.SignalServiceV2(serviceSig, dryRun) {
+			if nerr.Err != nil && process {
+				waitingDrives := map[string]madmin.DiskMetrics{}
+				jerr := json.Unmarshal([]byte(nerr.Err.Error()), &waitingDrives)
+				if jerr == nil {
+					srvResult.Results = append(srvResult.Results, servicePeerResult{
+						Host:          nerr.Host.String(),
+						WaitingDrives: waitingDrives,
+					})
+					continue
+				}
+			}
+			errStr := ""
+			if nerr.Err != nil {
+				errStr = nerr.Err.Error()
+			}
+			srvResult.Results = append(srvResult.Results, servicePeerResult{
+				Host: nerr.Host.String(),
+				Err:  errStr,
+			})
+		}
+	}
+
+	srvResult.DryRun = dryRun
+
+	buf, err := json.Marshal(srvResult)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	// Reply to the client before restarting, stopping MinIO server.
+	writeSuccessResponseJSON(w, buf)
+
+	switch serviceSig {
+	case serviceFreeze:
+		freezeServices()
+	case serviceUnFreeze:
+		unfreezeServices()
+	case serviceRestart, serviceStop:
+		if !dryRun {
+			globalServiceSignalCh <- serviceSig
+		}
+	}
+}
+
 // ServerProperties holds some server information such as, version, region
 // uptime, etc..
 type ServerProperties struct {
@@ -477,7 +805,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// DataUsageInfoHandler - GET /minio/admin/v3/datausage
+// DataUsageInfoHandler - GET /minio/admin/v3/datausage?capacity={true}
 // ----------
 // Get server/cluster data usage info
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
@@ -500,6 +828,16 @@ func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	// Get capacity info when asked.
+	if r.Form.Get("capacity") == "true" {
+		sinfo := objectAPI.StorageInfo(ctx, false)
+		dataUsageInfo.TotalCapacity = GetTotalUsableCapacity(sinfo.Disks, sinfo)
+		dataUsageInfo.TotalFreeCapacity = GetTotalUsableCapacityFree(sinfo.Disks, sinfo)
+		if dataUsageInfo.TotalCapacity > dataUsageInfo.TotalFreeCapacity {
+			dataUsageInfo.TotalUsedCapacity = dataUsageInfo.TotalCapacity - dataUsageInfo.TotalFreeCapacity
+		}
+	}
+
 	writeSuccessResponseJSON(w, dataUsageInfoJSON)
 }
 
@@ -901,7 +1239,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 	if hip.clientToken == "" {
 		jerr := json.NewDecoder(r).Decode(&hip.hs)
 		if jerr != nil {
-			logger.LogIf(GlobalContext, jerr, logger.Application)
+			logger.LogIf(GlobalContext, jerr, logger.ErrorKind)
 			err = ErrRequestBodyParse
 			return
 		}
@@ -1314,7 +1652,7 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 		duration = time.Second * 10
 	}
 
-	storageInfo := objectAPI.StorageInfo(ctx, true)
+	storageInfo := objectAPI.StorageInfo(ctx, false)
 
 	sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune)
 	if !sufficientCapacity {
@@ -1591,16 +1929,13 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	setEventStreamHeaders(w)
 
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
-	// Use buffered channel to take care of burst sends or slow w.Write()
-
-	// Keep 100k buffered channel, should be sufficient to ensure we do not lose any events.
-	traceCh := make(chan madmin.TraceInfo, 100000)
-
+	// Keep 100k buffered channel.
+	// If receiver cannot keep up with that we drop events.
+	traceCh := make(chan []byte, 100000)
 	peers, _ := newPeerRestClients(globalEndpoints)
-
-	err = globalTrace.Subscribe(traceOpts.TraceTypes(), traceCh, ctx.Done(), func(entry madmin.TraceInfo) bool {
+	err = globalTrace.SubscribeJSON(traceOpts.TraceTypes(), traceCh, ctx.Done(), func(entry madmin.TraceInfo) bool {
 		return shouldTrace(entry, traceOpts)
-	})
+	}, nil)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1615,19 +1950,19 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		if peer == nil {
 			continue
 		}
-		peer.Trace(traceCh, ctx.Done(), traceOpts)
+		peer.Trace(ctx, traceCh, traceOpts)
 	}
 
 	keepAliveTicker := time.NewTicker(time.Second)
 	defer keepAliveTicker.Stop()
 
-	enc := json.NewEncoder(w)
 	for {
 		select {
 		case entry := <-traceCh:
-			if err := enc.Encode(entry); err != nil {
+			if _, err := w.Write(entry); err != nil {
 				return
 			}
+			grid.PutByteBuffer(entry)
 			if len(traceCh) == 0 {
 				// Flush if nothing is queued
 				w.(http.Flusher).Flush()
@@ -1866,7 +2201,10 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]
 		return nil, errServerNotInitialized
 	}
 
-	z, _ := objectAPI.(*erasureServerPools)
+	z, ok := objectAPI.(*erasureServerPools)
+	if !ok {
+		return nil, errServerNotInitialized
+	}
 
 	poolsInfo := make(map[int]map[int]madmin.ErasureSetInfo)
 	for _, d := range allDisks {
@@ -1897,7 +2235,7 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]
 	return poolsInfo, nil
 }
 
-func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request) madmin.InfoMessage {
+func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) madmin.InfoMessage {
 	ldap := madmin.LDAP{}
 	if globalIAMSys.LDAPConfig.Enabled() {
 		ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
@@ -1918,8 +2256,8 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 	// Get the notification target info
 	notifyTarget := fetchLambdaInfo()
 
-	local := getLocalServerProperty(globalEndpoints, r)
-	servers := globalNotificationSys.ServerInfo()
+	local := getLocalServerProperty(globalEndpoints, r, metrics)
+	servers := globalNotificationSys.ServerInfo(metrics)
 	servers = append(servers, local)
 
 	assignPoolNumbers(servers)
@@ -1973,7 +2311,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 			DrivesPerSet:     backendInfo.DrivesPerSet,
 		}
 
-		if poolsInfoEnabled {
+		if pools {
 			poolsInfo, _ = getPoolsInfo(ctx, allDisks)
 		}
 	}
@@ -2019,7 +2357,7 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {
 	}
 
 	client := &http.Client{
-		Transport: NewHTTPTransport(),
+		Transport: globalHealthChkTransport,
 		Timeout:   10 * time.Second,
 	}
 
@@ -2339,7 +2677,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 	}
 
 	go func() {
-		defer close(healthInfoCh)
+		defer xioutil.SafeClose(healthInfoCh)
 
 		partialWrite(healthInfo) // Write first message with only version and deployment id populated
 		getAndWritePlatformInfo()
@@ -2355,7 +2693,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 		getAndWriteSysConfig()
 
 		if query.Get("minioinfo") == "true" {
-			infoMessage := getServerInfo(healthCtx, false, nil)
+			infoMessage := getServerInfo(healthCtx, false, true, nil)
 			servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers))
 			for _, server := range infoMessage.Servers {
 				anonEndpoint := anonAddr(server.Endpoint)
@@ -2543,8 +2881,10 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
+	metrics := r.Form.Get(peerRESTMetrics) == "true"
+
 	// Marshal API response
-	jsonBytes, err := json.Marshal(getServerInfo(ctx, true, r))
+	jsonBytes, err := json.Marshal(getServerInfo(ctx, true, metrics, r))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -2728,14 +3068,14 @@ func getClusterMetaInfo(ctx context.Context) []byte {
 	resultCh := make(chan madmin.ClusterRegistrationInfo)
 
 	go func() {
-		defer close(resultCh)
+		defer xioutil.SafeClose(resultCh)
 
 		ci := madmin.ClusterRegistrationInfo{}
 		ci.Info.NoOfServerPools = len(globalEndpoints)
-		ci.Info.NoOfServers = len(globalEndpoints.Hostnames())
+		ci.Info.NoOfServers = totalNodeCount()
 		ci.Info.MinioVersion = Version
 
-		si := objectAPI.StorageInfo(ctx, true)
+		si := objectAPI.StorageInfo(ctx, false)
 
 		ci.Info.NoOfDrives = len(si.Disks)
 		for _, disk := range si.Disks {
@@ -2972,7 +3312,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	// save the format.json as part of inspect by default
-	if volume != minioMetaBucket && file != formatConfigFile {
+	if !(volume == minioMetaBucket && file == formatConfigFile) {
 		err = o.GetRawData(ctx, minioMetaBucket, formatConfigFile, rawDataFn)
 	}
 	if !errors.Is(err, errFileNotFound) {

cmd/admin-handlers_test.go

@@ -168,6 +168,7 @@ func testServiceSignalReceiver(cmd cmdType, t *testing.T) {
 func getServiceCmdRequest(cmd cmdType, cred auth.Credentials) (*http.Request, error) {
 	queryVal := url.Values{}
 	queryVal.Set("action", string(cmd.toServiceAction()))
+	queryVal.Set("type", "2")
 	resource := adminPathPrefix + adminAPIVersionPrefix + "/service?" + queryVal.Encode()
 	req, err := newTestRequest(http.MethodPost, resource, 0, nil)
 	if err != nil {
@@ -220,12 +221,18 @@ func testServicesCmdHandler(cmd cmdType, t *testing.T) {
 	rec := httptest.NewRecorder()
 	adminTestBed.router.ServeHTTP(rec, req)
 
+	resp, _ := io.ReadAll(rec.Body)
 	if rec.Code != http.StatusOK {
-		resp, _ := io.ReadAll(rec.Body)
 		t.Errorf("Expected to receive %d status code but received %d. Body (%s)",
 			http.StatusOK, rec.Code, string(resp))
 	}
 
+	result := &serviceResult{}
+	if err := json.Unmarshal(resp, result); err != nil {
+		t.Error(err)
+	}
+	_ = result
+
 	// Wait until testServiceSignalReceiver() called in a goroutine quits.
 	wg.Wait()
 }

cmd/admin-heal-ops.go

@@ -28,6 +28,7 @@ import (
 	"time"
 
 	"github.com/minio/madmin-go/v3"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -132,7 +133,13 @@ func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) {
 func (ahs *allHealState) updateHealStatus(tracker *healingTracker) {
 	ahs.Lock()
 	defer ahs.Unlock()
-	ahs.healStatus[tracker.ID] = *tracker
+
+	tracker.mu.RLock()
+	t := *tracker
+	t.QueuedBuckets = append(make([]string, 0, len(tracker.QueuedBuckets)), tracker.QueuedBuckets...)
+	t.HealedBuckets = append(make([]string, 0, len(tracker.HealedBuckets)), tracker.HealedBuckets...)
+	ahs.healStatus[tracker.ID] = t
+	tracker.mu.RUnlock()
 }
 
 // Sort by zone, set and disk index
@@ -800,7 +807,7 @@ func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
 func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) {
 	bucketsOnly := false // Heals buckets and objects also.
 	h.traverseAndHealDoneCh <- h.healItems(objAPI, bucketsOnly)
-	close(h.traverseAndHealDoneCh)
+	xioutil.SafeClose(h.traverseAndHealDoneCh)
 }
 
 // healMinioSysMeta - heals all files under a given meta prefix, returns a function

cmd/admin-router.go

@@ -154,9 +154,16 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 	}
 
 	for _, adminVersion := range adminVersions {
-		// Restart and stop MinIO service.
+		// Restart and stop MinIO service type=2
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(adminMiddleware(adminAPI.ServiceV2Handler, traceAllFlag)).Queries("action", "{action:.*}", "type", "2")
+
+		// Deprecated: Restart and stop MinIO service.
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(adminMiddleware(adminAPI.ServiceHandler, traceAllFlag)).Queries("action", "{action:.*}")
-		// Update MinIO servers.
+
+		// Update all MinIO servers type=2
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(adminMiddleware(adminAPI.ServerUpdateV2Handler, traceAllFlag)).Queries("updateURL", "{updateURL:.*}", "type", "2")
+
+		// Deprecated: Update MinIO servers.
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(adminMiddleware(adminAPI.ServerUpdateHandler, traceAllFlag)).Queries("updateURL", "{updateURL:.*}")
 
 		// Info operations

cmd/admin-server-info.go

@@ -34,7 +34,7 @@ import (
 
 // getLocalServerProperty - returns madmin.ServerProperties for only the
 // local endpoints from given list of endpoints
-func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request) madmin.ServerProperties {
+func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request, metrics bool) madmin.ServerProperties {
 	addr := globalLocalNodeName
 	if r != nil {
 		addr = r.Host
@@ -141,7 +141,7 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 
 	objLayer := newObjectLayerFn()
 	if objLayer != nil {
-		storageInfo := objLayer.LocalStorageInfo(GlobalContext, true)
+		storageInfo := objLayer.LocalStorageInfo(GlobalContext, metrics)
 		props.State = string(madmin.ItemOnline)
 		props.Disks = storageInfo.Disks
 	} else {

cmd/api-errors.go

@@ -24,6 +24,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"os"
 	"strconv"
 	"strings"
 
@@ -297,7 +298,6 @@ const (
 	ErrAdminConfigLDAPValidation
 	ErrAdminConfigIDPCfgNameAlreadyExists
 	ErrAdminConfigIDPCfgNameDoesNotExist
-	ErrAdminCredentialsMismatch
 	ErrInsecureClientRequest
 	ErrObjectTampered
 
@@ -433,6 +433,9 @@ const (
 	// New Codes for GetObjectAttributes and GetObjectVersionAttributes
 	ErrInvalidAttributeName
 
+	ErrAdminNoAccessKey
+	ErrAdminNoSecretKey
+
 	apiErrCodeEnd // This is used only for the testing code
 )
 
@@ -1359,6 +1362,16 @@ var errorCodes = errorCodeMap{
 		Description:    "The secret key is invalid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrAdminNoAccessKey: {
+		Code:           "XMinioAdminNoAccessKey",
+		Description:    "No access key was provided.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrAdminNoSecretKey: {
+		Code:           "XMinioAdminNoSecretKey",
+		Description:    "No secret key was provided.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminConfigNoQuorum: {
 		Code:           "XMinioAdminConfigNoQuorum",
 		Description:    "Configuration update failed because server quorum was not met",
@@ -1425,11 +1438,6 @@ var errorCodes = errorCodeMap{
 		Description:    "Unable to perform the requested operation because profiling is not enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrAdminCredentialsMismatch: {
-		Code:           "XMinioAdminCredentialsMismatch",
-		Description:    "Credentials in config mismatch with server environment variables",
-		HTTPStatusCode: http.StatusServiceUnavailable,
-	},
 	ErrAdminBucketQuotaExceeded: {
 		Code:           "XMinioAdminBucketQuotaExceeded",
 		Description:    "Bucket quota exceeded",
@@ -2082,6 +2090,11 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		return ErrNone
 	}
 
+	// Errors that are generated by net.Conn and any context errors must be handled here.
+	if errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) {
+		return ErrRequestTimedout
+	}
+
 	// Only return ErrClientDisconnected if the provided context is actually canceled.
 	// This way downstream context.Canceled will still report ErrRequestTimedout
 	if contextCanceled(ctx) && errors.Is(ctx.Err(), context.Canceled) {
@@ -2124,6 +2137,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminInvalidAccessKey
 	case auth.ErrInvalidSecretKeyLength:
 		apiErr = ErrAdminInvalidSecretKey
+	case auth.ErrNoAccessKeyWithSecretKey:
+		apiErr = ErrAdminNoAccessKey
+	case auth.ErrNoSecretKeyWithAccessKey:
+		apiErr = ErrAdminNoSecretKey
 	case errInvalidStorageClass:
 		apiErr = ErrInvalidStorageClass
 	case errErasureReadQuorum:
@@ -2351,31 +2368,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case dns.ErrBucketConflict:
 		apiErr = ErrBucketAlreadyExists
 	default:
-		if _, ok := err.(tags.Error); ok {
-			// tag errors are not exported, so we check their custom interface to avoid logging.
-			// The correct type is inserted by toAPIError.
-			apiErr = ErrInternalError
-			break
-		}
-		var ie, iw int
-		// This work-around is to handle the issue golang/go#30648
-		//nolint:gocritic
-		if _, ferr := fmt.Fscanf(strings.NewReader(err.Error()),
-			"request declared a Content-Length of %d but only wrote %d bytes",
-			&ie, &iw); ferr != nil {
-			apiErr = ErrInternalError
-			// Make sure to log the errors which we cannot translate
-			// to a meaningful S3 API errors. This is added to aid in
-			// debugging unexpected/unhandled errors.
-			logger.LogIf(ctx, err)
-		} else if ie > iw {
+		if strings.Contains(err.Error(), "request declared a Content-Length") {
 			apiErr = ErrIncompleteBody
 		} else {
 			apiErr = ErrInternalError
-			// Make sure to log the errors which we cannot translate
-			// to a meaningful S3 API errors. This is added to aid in
-			// debugging unexpected/unhandled errors.
-			logger.LogIf(ctx, err)
 		}
 	}
 
@@ -2519,6 +2515,13 @@ func toAPIError(ctx context.Context, err error) APIError {
 		}
 	}
 
+	if apiErr.Code == "InternalError" {
+		// Make sure to log the errors which we cannot translate
+		// to a meaningful S3 API errors. This is added to aid in
+		// debugging unexpected/unhandled errors.
+		logger.LogIf(ctx, err)
+	}
+
 	return apiErr
 }
 

cmd/api-response.go

@@ -946,11 +946,13 @@ func writeSuccessResponseHeadersOnly(w http.ResponseWriter) {
 
 // writeErrorRespone writes error headers
 func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) {
-	switch err.Code {
-	case "SlowDown", "XMinioServerNotInitialized", "XMinioReadQuorum", "XMinioWriteQuorum":
+	if err.HTTPStatusCode == http.StatusServiceUnavailable {
 		// Set retry-after header to indicate user-agents to retry request after 120secs.
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set(xhttp.RetryAfter, "120")
+	}
+
+	switch err.Code {
 	case "InvalidRegion":
 		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalSite.Region)
 	case "AuthorizationHeaderMalformed":

cmd/api-router.go

@@ -227,7 +227,7 @@ func registerAPIRouter(router *mux.Router) {
 		// HeadObject
 		router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler)))))
-		// GetObjectAttribytes
+		// GetObjectAttributes
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("getobjectattributes", maxClients(gz(httpTraceHdrs(api.GetObjectAttributesHandler))))).Queries("attributes", "")
 		// CopyObjectPart

cmd/auth-handler.go

@@ -162,7 +162,8 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 	s3Err := ErrAccessDenied
 	if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
 		getRequestAuthType(r) == authTypeSigned {
-		// We only support admin credentials to access admin APIs.
+
+		// Get credential information from the request.
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {
 			return cred, owner, s3Err
@@ -256,7 +257,7 @@ func getClaimsFromTokenWithSecret(token, secret string) (map[string]interface{},
 		if err != nil {
 			// Base64 decoding fails, we should log to indicate
 			// something is malforming the request sent by client.
-			logger.LogIf(GlobalContext, err, logger.Application)
+			logger.LogIf(GlobalContext, err, logger.ErrorKind)
 			return nil, errAuthentication
 		}
 		claims.MapClaims[sessionPolicyNameExtracted] = string(spBytes)
@@ -338,7 +339,7 @@ func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action po
 
 func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) {
 	if logger.GetReqInfo(ctx) == nil {
-		logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.Minio)
+		logger.LogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind)
 		return ErrAccessDenied
 	}
 
@@ -377,7 +378,7 @@ func authenticateRequest(ctx context.Context, r *http.Request, action policy.Act
 		// To extract region from XML in request body, get copy of request body.
 		payload, err := io.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize))
 		if err != nil {
-			logger.LogIf(ctx, err, logger.Application)
+			logger.LogIf(ctx, err, logger.ErrorKind)
 			return ErrMalformedXML
 		}
 

cmd/background-newdisks-heal-ops.go

@@ -376,7 +376,7 @@ func getLocalDisksToHeal() (disksToHeal Endpoints) {
 var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second)
 
 func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error {
-	disk, format, err := connectEndpoint(endpoint)
+	disk, format, _, err := connectEndpoint(endpoint)
 	if err != nil {
 		return fmt.Errorf("Error: %w, %s", err, endpoint)
 	}
@@ -490,6 +490,10 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	for _, disk := range disks {
+		if disk == nil {
+			continue
+		}
+
 		t, err := loadHealingTracker(ctx, disk)
 		if err != nil {
 			if !errors.Is(err, errFileNotFound) {

cmd/batch-expire.go

@@ -32,6 +32,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/versioning"
 	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/wildcard"
@@ -648,7 +649,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 		case expireCh <- toDel:
 		}
 	}
-	close(expireCh)
+	xioutil.SafeClose(expireCh)
 
 	wk.Wait() // waits for all expire goroutines to complete
 
@@ -658,7 +659,7 @@ func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJo
 
 	// Close the saverQuitCh - this also triggers saving in-memory state
 	// immediately one last time before we exit this method.
-	close(saverQuitCh)
+	xioutil.SafeClose(saverQuitCh)
 
 	// Notify expire jobs final status to the configured endpoint
 	buf, _ := json.Marshal(ri)

cmd/batch-handlers.go

@@ -47,6 +47,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/ioutil"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	"github.com/minio/pkg/v2/env"
@@ -545,7 +546,7 @@ func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLa
 	}
 
 	go func() {
-		defer close(input)
+		defer xioutil.SafeClose(input)
 
 		for _, entry := range entries {
 			gr, err := objAPI.GetObjectNInfo(ctx, r.Source.Bucket,
@@ -924,6 +925,21 @@ func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo,
 	ri.countItem(info.Size, info.DeleteMarker, success)
 }
 
+func (ri *batchJobInfo) trackCurrentBucketBatch(bucket string, batch []ObjectInfo) {
+	if ri == nil {
+		return
+	}
+
+	ri.mu.Lock()
+	defer ri.mu.Unlock()
+
+	ri.Bucket = bucket
+	for i := range batch {
+		ri.Object = batch[i].Name
+		ri.countItem(batch[i].Size, batch[i].DeleteMarker, true)
+	}
+}
+
 // Start start the batch replication job, resumes if there was a pending job via "job.ID"
 func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
 	ri := &batchJobInfo{
@@ -1038,7 +1054,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 
 	if !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() {
 		go func() {
-			defer close(slowCh)
+			defer xioutil.SafeClose(slowCh)
 
 			// Snowball currently needs the high level minio-go Client, not the Core one
 			cl, err := miniogo.New(u.Host, &miniogo.Options{
@@ -1086,6 +1102,11 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 						for _, b := range batch {
 							slowCh <- b
 						}
+					} else {
+						ri.trackCurrentBucketBatch(r.Source.Bucket, batch)
+						globalBatchJobsMetrics.save(job.ID, ri)
+						// persist in-memory state to disk after every 10secs.
+						logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 					}
 					batch = batch[:0]
 				}
@@ -1809,7 +1830,7 @@ func (j *BatchJobPool) queueJob(req *BatchJobRequest) error {
 	select {
 	case <-j.ctx.Done():
 		j.once.Do(func() {
-			close(j.jobCh)
+			xioutil.SafeClose(j.jobCh)
 		})
 	case j.jobCh <- req:
 	default:

cmd/batch-job-common-types.go

@@ -213,11 +213,14 @@ func (b BatchJobSnowball) Validate() error {
 		}
 	}
 	_, err := humanize.ParseBytes(*b.SmallerThan)
-	return BatchJobYamlErr{
-		line: b.line,
-		col:  b.col,
-		msg:  err.Error(),
+	if err != nil {
+		return BatchJobYamlErr{
+			line: b.line,
+			col:  b.col,
+			msg:  err.Error(),
+		}
 	}
+	return nil
 }
 
 // BatchJobSizeFilter supports size based filters - LesserThan and GreaterThan

cmd/bitrot-streaming.go

@@ -20,16 +20,12 @@ package cmd
 import (
 	"bytes"
 	"context"
-	"encoding/hex"
-	"fmt"
 	"hash"
 	"io"
-	"strings"
 	"sync"
 
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
 )
 
 // Calculates bitrot in chunks and writes the hash into the stream.
@@ -90,7 +86,7 @@ func newStreamingBitrotWriterBuffer(w io.Writer, algo BitrotAlgorithm, shardSize
 }
 
 // Returns streaming bitrot writer implementation.
-func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
+func newStreamingBitrotWriter(disk StorageAPI, origvolume, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
 	r, w := io.Pipe()
 	h := algo.New()
 
@@ -110,7 +106,7 @@ func newStreamingBitrotWriter(disk StorageAPI, volume, filePath string, length i
 			bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums.
 			totalFileSize = bitrotSumsTotalSize + length
 		}
-		r.CloseWithError(disk.CreateFile(context.TODO(), volume, filePath, totalFileSize, r))
+		r.CloseWithError(disk.CreateFile(context.TODO(), origvolume, volume, filePath, totalFileSize, r))
 	}()
 	return bw
 }
@@ -154,29 +150,12 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		// Can never happen unless there are programmer bugs
 		return 0, errUnexpected
 	}
-	ignoredErrs := []error{
-		errDiskNotFound,
-	}
-	if strings.HasPrefix(b.volume, minioMetaBucket) {
-		ignoredErrs = append(ignoredErrs,
-			errFileNotFound,
-			errVolumeNotFound,
-			errFileVersionNotFound,
-		)
-	}
 	if b.rc == nil {
 		// For the first ReadAt() call we need to open the stream for reading.
 		b.currOffset = offset
 		streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset
 		if len(b.data) == 0 && b.tillOffset != streamOffset {
 			b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
-			if err != nil {
-				if !IsErr(err, ignoredErrs...) {
-					logger.LogOnceIf(GlobalContext,
-						fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
-							b.disk, b.volume, b.filePath, err), "bitrot-read-file-stream-"+b.volume+"-"+b.filePath)
-				}
-			}
 		} else {
 			b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset)
 		}
@@ -198,10 +177,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 		return 0, err
 	}
 	b.h.Write(buf)
-
 	if !bytes.Equal(b.h.Sum(nil), b.hashBytes) {
-		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s  -> %s/%s - content hash does not match - expected %s, got %s",
-			b.disk, b.volume, b.filePath, hex.EncodeToString(b.hashBytes), hex.EncodeToString(b.h.Sum(nil))))
 		return 0, errFileCorrupt
 	}
 	b.currOffset += int64(len(buf))

cmd/bitrot-whole.go

@@ -19,11 +19,8 @@ package cmd
 
 import (
 	"context"
-	"fmt"
 	"hash"
 	"io"
-
-	"github.com/minio/minio/internal/logger"
 )
 
 // Implementation to calculate bitrot for the whole file.
@@ -38,12 +35,10 @@ type wholeBitrotWriter struct {
 func (b *wholeBitrotWriter) Write(p []byte) (int, error) {
 	err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p)
 	if err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	_, err = b.Hash.Write(p)
 	if err != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s returned %w", b.disk, err))
 		return 0, err
 	}
 	return len(p), nil
@@ -72,12 +67,10 @@ func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error)
 	if b.buf == nil {
 		b.buf = make([]byte, b.tillOffset-offset)
 		if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil {
-			logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, err))
 			return 0, err
 		}
 	}
 	if len(b.buf) < len(buf) {
-		logger.LogIf(GlobalContext, fmt.Errorf("Drive: %s -> %s/%s returned %w", b.disk, b.volume, b.filePath, errLessData))
 		return 0, errLessData
 	}
 	n = copy(buf, b.buf)

cmd/bitrot.go

@@ -102,9 +102,9 @@ func BitrotAlgorithmFromString(s string) (a BitrotAlgorithm) {
 	return
 }
 
-func newBitrotWriter(disk StorageAPI, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
+func newBitrotWriter(disk StorageAPI, origvolume, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer {
 	if algo == HighwayHash256S {
-		return newStreamingBitrotWriter(disk, volume, filePath, length, algo, shardSize)
+		return newStreamingBitrotWriter(disk, origvolume, volume, filePath, length, algo, shardSize)
 	}
 	return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize)
 }

cmd/bitrot_test.go

@@ -36,7 +36,7 @@ func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) {
 
 	disk.MakeVol(context.Background(), volume)
 
-	writer := newBitrotWriter(disk, volume, filePath, 35, bitrotAlgo, 10)
+	writer := newBitrotWriter(disk, "", volume, filePath, 35, bitrotAlgo, 10)
 
 	_, err = writer.Write([]byte("aaaaaaaaaa"))
 	if err != nil {

cmd/bootstrap-peer-server.go

@@ -19,71 +19,47 @@ package cmd
 
 import (
 	"context"
-	"encoding/json"
 	"errors"
 	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"os"
+	"math/rand"
 	"reflect"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/set"
-	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/rest"
-	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/env"
 )
 
-const (
-	bootstrapRESTVersion       = "v1"
-	bootstrapRESTVersionPrefix = SlashSeparator + bootstrapRESTVersion
-	bootstrapRESTPrefix        = minioReservedBucketPath + "/bootstrap"
-	bootstrapRESTPath          = bootstrapRESTPrefix + bootstrapRESTVersionPrefix
-)
-
-const (
-	bootstrapRESTMethodHealth = "/health"
-	bootstrapRESTMethodVerify = "/verify"
-)
-
 // To abstract a node over network.
 type bootstrapRESTServer struct{}
 
+//go:generate msgp -file=$GOFILE
+
 // ServerSystemConfig - captures information about server configuration.
 type ServerSystemConfig struct {
-	MinioEndpoints EndpointServerPools
-	MinioEnv       map[string]string
+	NEndpoints int
+	CmdLines   []string
+	MinioEnv   map[string]string
 }
 
 // Diff - returns error on first difference found in two configs.
-func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
-	if s1.MinioEndpoints.NEndpoints() != s2.MinioEndpoints.NEndpoints() {
-		return fmt.Errorf("Expected number of endpoints %d, seen %d", s1.MinioEndpoints.NEndpoints(),
-			s2.MinioEndpoints.NEndpoints())
+func (s1 *ServerSystemConfig) Diff(s2 *ServerSystemConfig) error {
+	ns1 := s1.NEndpoints
+	ns2 := s2.NEndpoints
+	if ns1 != ns2 {
+		return fmt.Errorf("Expected number of endpoints %d, seen %d", ns1, ns2)
 	}
 
-	for i, ep := range s1.MinioEndpoints {
-		if ep.CmdLine != s2.MinioEndpoints[i].CmdLine {
-			return fmt.Errorf("Expected command line argument %s, seen %s", ep.CmdLine,
-				s2.MinioEndpoints[i].CmdLine)
-		}
-		if ep.SetCount != s2.MinioEndpoints[i].SetCount {
-			return fmt.Errorf("Expected set count %d, seen %d", ep.SetCount,
-				s2.MinioEndpoints[i].SetCount)
-		}
-		if ep.DrivesPerSet != s2.MinioEndpoints[i].DrivesPerSet {
-			return fmt.Errorf("Expected drives pet set %d, seen %d", ep.DrivesPerSet,
-				s2.MinioEndpoints[i].DrivesPerSet)
-		}
-		if ep.Platform != s2.MinioEndpoints[i].Platform {
-			return fmt.Errorf("Expected platform '%s', found to be on '%s'",
-				ep.Platform, s2.MinioEndpoints[i].Platform)
+	for i, cmdLine := range s1.CmdLines {
+		if cmdLine != s2.CmdLines[i] {
+			return fmt.Errorf("Expected command line argument %s, seen %s", cmdLine,
+				s2.CmdLines[i])
 		}
 	}
+
 	if reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
 		return nil
 	}
@@ -131,7 +107,7 @@ var skipEnvs = map[string]struct{}{
 	"MINIO_SECRET_KEY":    {},
 }
 
-func getServerSystemCfg() ServerSystemConfig {
+func getServerSystemCfg() *ServerSystemConfig {
 	envs := env.List("MINIO_")
 	envValues := make(map[string]string, len(envs))
 	for _, envK := range envs {
@@ -144,120 +120,104 @@ func getServerSystemCfg() ServerSystemConfig {
 		}
 		envValues[envK] = logger.HashString(env.Get(envK, ""))
 	}
-	return ServerSystemConfig{
-		MinioEndpoints: globalEndpoints,
-		MinioEnv:       envValues,
+	scfg := &ServerSystemConfig{NEndpoints: globalEndpoints.NEndpoints(), MinioEnv: envValues}
+	var cmdLines []string
+	for _, ep := range globalEndpoints {
+		cmdLines = append(cmdLines, ep.CmdLine)
 	}
+	scfg.CmdLines = cmdLines
+	return scfg
 }
 
-func (b *bootstrapRESTServer) writeErrorResponse(w http.ResponseWriter, err error) {
-	w.WriteHeader(http.StatusForbidden)
-	w.Write([]byte(err.Error()))
+func (s *bootstrapRESTServer) VerifyHandler(params *grid.MSS) (*ServerSystemConfig, *grid.RemoteErr) {
+	return getServerSystemCfg(), nil
 }
 
-// HealthHandler returns success if request is valid
-func (b *bootstrapRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) {}
-
-func (b *bootstrapRESTServer) VerifyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "VerifyHandler")
-
-	if err := storageServerRequestValidate(r); err != nil {
-		b.writeErrorResponse(w, err)
-		return
-	}
-
-	cfg := getServerSystemCfg()
-	logger.LogIf(ctx, json.NewEncoder(w).Encode(&cfg))
-}
+var serverVerifyHandler = grid.NewSingleHandler[*grid.MSS, *ServerSystemConfig](grid.HandlerServerVerify, grid.NewMSS, func() *ServerSystemConfig { return &ServerSystemConfig{} })
 
 // registerBootstrapRESTHandlers - register bootstrap rest router.
-func registerBootstrapRESTHandlers(router *mux.Router) {
-	h := func(f http.HandlerFunc) http.HandlerFunc {
-		return collectInternodeStats(httpTraceHdrs(f))
-	}
-
+func registerBootstrapRESTHandlers(gm *grid.Manager) {
 	server := &bootstrapRESTServer{}
-	subrouter := router.PathPrefix(bootstrapRESTPrefix).Subrouter()
-
-	subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodHealth).HandlerFunc(
-		h(server.HealthHandler))
-
-	subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodVerify).HandlerFunc(
-		h(server.VerifyHandler))
+	logger.FatalIf(serverVerifyHandler.Register(gm, server.VerifyHandler), "unable to register handler")
 }
 
 // client to talk to bootstrap NEndpoints.
 type bootstrapRESTClient struct {
-	endpoint   Endpoint
-	restClient *rest.Client
+	gridConn *grid.Connection
 }
 
-// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected
-// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints()
-// after verifying format.json
-func (client *bootstrapRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) {
-	if values == nil {
-		values = make(url.Values)
+// Verify function verifies the server config.
+func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg *ServerSystemConfig) (err error) {
+	if newObjectLayerFn() != nil {
+		return nil
 	}
 
-	respBody, err = client.restClient.Call(ctx, method, values, body, length)
-	if err == nil {
-		return respBody, nil
+	recvCfg, err := serverVerifyHandler.Call(ctx, client.gridConn, grid.NewMSS())
+	if err != nil {
+		return err
 	}
+	// We do not need the response after returning.
+	defer serverVerifyHandler.PutResponse(recvCfg)
 
-	return nil, err
+	return srcCfg.Diff(recvCfg)
 }
 
 // Stringer provides a canonicalized representation of node.
 func (client *bootstrapRESTClient) String() string {
-	return client.endpoint.String()
+	return client.gridConn.String()
 }
 
-// Verify - fetches system server config.
-func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg ServerSystemConfig) (err error) {
-	if newObjectLayerFn() != nil {
-		return nil
-	}
-	respBody, err := client.callWithContext(ctx, bootstrapRESTMethodVerify, nil, nil, -1)
-	if err != nil {
-		return
-	}
-	defer xhttp.DrainBody(respBody)
-	recvCfg := ServerSystemConfig{}
-	if err = json.NewDecoder(respBody).Decode(&recvCfg); err != nil {
-		return err
-	}
-	return srcCfg.Diff(recvCfg)
-}
-
-func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointServerPools) error {
+func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointServerPools, gm *grid.Manager) error {
 	srcCfg := getServerSystemCfg()
-	clnts := newBootstrapRESTClients(endpointServerPools)
+	clnts := newBootstrapRESTClients(endpointServerPools, gm)
 	var onlineServers int
 	var offlineEndpoints []error
 	var incorrectConfigs []error
 	var retries int
+	var mu sync.Mutex
 	for onlineServers < len(clnts)/2 {
+		var wg sync.WaitGroup
+		wg.Add(len(clnts))
+		onlineServers = 0
 		for _, clnt := range clnts {
-			if err := clnt.Verify(ctx, srcCfg); err != nil {
-				bootstrapTraceMsg(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
-				if !isNetworkError(err) {
-					logger.LogOnceIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err), clnt.String())
-					incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
-				} else {
-					offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt.String(), err))
+			go func(clnt *bootstrapRESTClient) {
+				defer wg.Done()
+
+				if clnt.gridConn.State() != grid.StateConnected {
+					mu.Lock()
+					offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt, grid.ErrDisconnected))
+					mu.Unlock()
+					return
 				}
-				continue
-			}
-			onlineServers++
+
+				ctx, cancel := context.WithTimeout(ctx, 2*time.Second)
+				defer cancel()
+
+				err := clnt.Verify(ctx, srcCfg)
+				mu.Lock()
+				if err != nil {
+					bootstrapTraceMsg(fmt.Sprintf("clnt.Verify: %v, endpoint: %s", err, clnt))
+					if !isNetworkError(err) {
+						logger.LogOnceIf(context.Background(), fmt.Errorf("%s has incorrect configuration: %w", clnt, err), "incorrect_"+clnt.String())
+						incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt, err))
+					} else {
+						offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt, err))
+					}
+				} else {
+					onlineServers++
+				}
+				mu.Unlock()
+			}(clnt)
 		}
+		wg.Wait()
+
 		select {
 		case <-ctx.Done():
 			return ctx.Err()
 		default:
-			// Sleep for a while - so that we don't go into
-			// 100% CPU when half the endpoints are offline.
-			time.Sleep(100 * time.Millisecond)
+			// Sleep and stagger to avoid blocked CPU and thundering
+			// herd upon start up sequence.
+			time.Sleep(25*time.Millisecond + time.Duration(rand.Int63n(int64(100*time.Millisecond))))
 			retries++
 			// after 20 retries start logging that servers are not reachable yet
 			if retries >= 20 {
@@ -268,7 +228,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 				if len(incorrectConfigs) > 0 {
 					logger.Info(fmt.Sprintf("Following servers have mismatching configuration %s", incorrectConfigs))
 				}
-				retries = 0 // reset to log again after 5 retries.
+				retries = 0 // reset to log again after 20 retries.
 			}
 			offlineEndpoints = nil
 			incorrectConfigs = nil
@@ -277,39 +237,20 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 	return nil
 }
 
-func newBootstrapRESTClients(endpointServerPools EndpointServerPools) []*bootstrapRESTClient {
-	seenHosts := set.NewStringSet()
+func newBootstrapRESTClients(endpointServerPools EndpointServerPools, gm *grid.Manager) []*bootstrapRESTClient {
+	seenClient := set.NewStringSet()
 	var clnts []*bootstrapRESTClient
 	for _, ep := range endpointServerPools {
 		for _, endpoint := range ep.Endpoints {
-			if seenHosts.Contains(endpoint.Host) {
+			if endpoint.IsLocal {
 				continue
 			}
-			seenHosts.Add(endpoint.Host)
-
-			// Only proceed for remote endpoints.
-			if !endpoint.IsLocal {
-				cl := newBootstrapRESTClient(endpoint)
-				if serverDebugLog {
-					cl.restClient.TraceOutput = os.Stdout
-				}
-				clnts = append(clnts, cl)
+			if seenClient.Contains(endpoint.Host) {
+				continue
 			}
+			seenClient.Add(endpoint.Host)
+			clnts = append(clnts, &bootstrapRESTClient{gm.Connection(endpoint.GridHost())})
 		}
 	}
 	return clnts
 }
-
-// Returns a new bootstrap client.
-func newBootstrapRESTClient(endpoint Endpoint) *bootstrapRESTClient {
-	serverURL := &url.URL{
-		Scheme: endpoint.Scheme,
-		Host:   endpoint.Host,
-		Path:   bootstrapRESTPath,
-	}
-
-	restClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken())
-	restClient.HealthCheckFn = nil
-
-	return &bootstrapRESTClient{endpoint: endpoint, restClient: restClient}
-}

cmd/bootstrap-peer-server_gen.go

@@ -0,0 +1,270 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"github.com/tinylib/msgp/msgp"
+)
+
+// DecodeMsg implements msgp.Decodable
+func (z *ServerSystemConfig) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "NEndpoints":
+			z.NEndpoints, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "NEndpoints")
+				return
+			}
+		case "CmdLines":
+			var zb0002 uint32
+			zb0002, err = dc.ReadArrayHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "CmdLines")
+				return
+			}
+			if cap(z.CmdLines) >= int(zb0002) {
+				z.CmdLines = (z.CmdLines)[:zb0002]
+			} else {
+				z.CmdLines = make([]string, zb0002)
+			}
+			for za0001 := range z.CmdLines {
+				z.CmdLines[za0001], err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "CmdLines", za0001)
+					return
+				}
+			}
+		case "MinioEnv":
+			var zb0003 uint32
+			zb0003, err = dc.ReadMapHeader()
+			if err != nil {
+				err = msgp.WrapError(err, "MinioEnv")
+				return
+			}
+			if z.MinioEnv == nil {
+				z.MinioEnv = make(map[string]string, zb0003)
+			} else if len(z.MinioEnv) > 0 {
+				for key := range z.MinioEnv {
+					delete(z.MinioEnv, key)
+				}
+			}
+			for zb0003 > 0 {
+				zb0003--
+				var za0002 string
+				var za0003 string
+				za0002, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "MinioEnv")
+					return
+				}
+				za0003, err = dc.ReadString()
+				if err != nil {
+					err = msgp.WrapError(err, "MinioEnv", za0002)
+					return
+				}
+				z.MinioEnv[za0002] = za0003
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *ServerSystemConfig) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 3
+	// write "NEndpoints"
+	err = en.Append(0x83, 0xaa, 0x4e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.NEndpoints)
+	if err != nil {
+		err = msgp.WrapError(err, "NEndpoints")
+		return
+	}
+	// write "CmdLines"
+	err = en.Append(0xa8, 0x43, 0x6d, 0x64, 0x4c, 0x69, 0x6e, 0x65, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteArrayHeader(uint32(len(z.CmdLines)))
+	if err != nil {
+		err = msgp.WrapError(err, "CmdLines")
+		return
+	}
+	for za0001 := range z.CmdLines {
+		err = en.WriteString(z.CmdLines[za0001])
+		if err != nil {
+			err = msgp.WrapError(err, "CmdLines", za0001)
+			return
+		}
+	}
+	// write "MinioEnv"
+	err = en.Append(0xa8, 0x4d, 0x69, 0x6e, 0x69, 0x6f, 0x45, 0x6e, 0x76)
+	if err != nil {
+		return
+	}
+	err = en.WriteMapHeader(uint32(len(z.MinioEnv)))
+	if err != nil {
+		err = msgp.WrapError(err, "MinioEnv")
+		return
+	}
+	for za0002, za0003 := range z.MinioEnv {
+		err = en.WriteString(za0002)
+		if err != nil {
+			err = msgp.WrapError(err, "MinioEnv")
+			return
+		}
+		err = en.WriteString(za0003)
+		if err != nil {
+			err = msgp.WrapError(err, "MinioEnv", za0002)
+			return
+		}
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *ServerSystemConfig) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 3
+	// string "NEndpoints"
+	o = append(o, 0x83, 0xaa, 0x4e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73)
+	o = msgp.AppendInt(o, z.NEndpoints)
+	// string "CmdLines"
+	o = append(o, 0xa8, 0x43, 0x6d, 0x64, 0x4c, 0x69, 0x6e, 0x65, 0x73)
+	o = msgp.AppendArrayHeader(o, uint32(len(z.CmdLines)))
+	for za0001 := range z.CmdLines {
+		o = msgp.AppendString(o, z.CmdLines[za0001])
+	}
+	// string "MinioEnv"
+	o = append(o, 0xa8, 0x4d, 0x69, 0x6e, 0x69, 0x6f, 0x45, 0x6e, 0x76)
+	o = msgp.AppendMapHeader(o, uint32(len(z.MinioEnv)))
+	for za0002, za0003 := range z.MinioEnv {
+		o = msgp.AppendString(o, za0002)
+		o = msgp.AppendString(o, za0003)
+	}
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *ServerSystemConfig) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "NEndpoints":
+			z.NEndpoints, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "NEndpoints")
+				return
+			}
+		case "CmdLines":
+			var zb0002 uint32
+			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "CmdLines")
+				return
+			}
+			if cap(z.CmdLines) >= int(zb0002) {
+				z.CmdLines = (z.CmdLines)[:zb0002]
+			} else {
+				z.CmdLines = make([]string, zb0002)
+			}
+			for za0001 := range z.CmdLines {
+				z.CmdLines[za0001], bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "CmdLines", za0001)
+					return
+				}
+			}
+		case "MinioEnv":
+			var zb0003 uint32
+			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "MinioEnv")
+				return
+			}
+			if z.MinioEnv == nil {
+				z.MinioEnv = make(map[string]string, zb0003)
+			} else if len(z.MinioEnv) > 0 {
+				for key := range z.MinioEnv {
+					delete(z.MinioEnv, key)
+				}
+			}
+			for zb0003 > 0 {
+				var za0002 string
+				var za0003 string
+				zb0003--
+				za0002, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "MinioEnv")
+					return
+				}
+				za0003, bts, err = msgp.ReadStringBytes(bts)
+				if err != nil {
+					err = msgp.WrapError(err, "MinioEnv", za0002)
+					return
+				}
+				z.MinioEnv[za0002] = za0003
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *ServerSystemConfig) Msgsize() (s int) {
+	s = 1 + 11 + msgp.IntSize + 9 + msgp.ArrayHeaderSize
+	for za0001 := range z.CmdLines {
+		s += msgp.StringPrefixSize + len(z.CmdLines[za0001])
+	}
+	s += 9 + msgp.MapHeaderSize
+	if z.MinioEnv != nil {
+		for za0002, za0003 := range z.MinioEnv {
+			_ = za0003
+			s += msgp.StringPrefixSize + len(za0002) + msgp.StringPrefixSize + len(za0003)
+		}
+	}
+	return
+}

cmd/bootstrap-peer-server_gen_test.go

@@ -0,0 +1,123 @@
+package cmd
+
+// Code generated by github.com/tinylib/msgp DO NOT EDIT.
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/tinylib/msgp/msgp"
+)
+
+func TestMarshalUnmarshalServerSystemConfig(t *testing.T) {
+	v := ServerSystemConfig{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgServerSystemConfig(b *testing.B) {
+	v := ServerSystemConfig{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgServerSystemConfig(b *testing.B) {
+	v := ServerSystemConfig{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalServerSystemConfig(b *testing.B) {
+	v := ServerSystemConfig{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeServerSystemConfig(t *testing.T) {
+	v := ServerSystemConfig{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeServerSystemConfig Msgsize() is inaccurate")
+	}
+
+	vn := ServerSystemConfig{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeServerSystemConfig(b *testing.B) {
+	v := ServerSystemConfig{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeServerSystemConfig(b *testing.B) {
+	v := ServerSystemConfig{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}

cmd/bucket-handlers.go

@@ -467,13 +467,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// Ignore errors here to preserve the S3 error behavior of GetBucketInfo()
 	checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "")
 
-	// Before proceeding validate if bucket exists.
-	_, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{})
-	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-
 	deleteObjectsFn := objectAPI.DeleteObjects
 
 	// Return Malformed XML as S3 spec if the number of objects is empty
@@ -611,6 +604,12 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		VersionSuspended: vc.Suspended(),
 	})
 
+	// Are all objects saying bucket not found?
+	if isAllBucketsNotFound(errs) {
+		writeErrorResponse(ctx, w, toAPIError(ctx, errs[0]), r.URL)
+		return
+	}
+
 	for i := range errs {
 		// DeleteMarkerVersionID is not used specifically to avoid
 		// lookup errors, since DeleteMarkerVersionID is only
@@ -618,7 +617,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		// specify a versionID.
 		objToDel := ObjectToDelete{
 			ObjectV: ObjectV{
-				ObjectName: dObjects[i].ObjectName,
+				ObjectName: decodeDirObject(dObjects[i].ObjectName),
 				VersionID:  dObjects[i].VersionID,
 			},
 			VersionPurgeStatus:            dObjects[i].VersionPurgeStatus(),
@@ -1395,7 +1394,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			// Notify object created events.
 			sendEvent(eventArgsList[i])
 
-			if eventArgsList[i].Object.NumVersions > dataScannerExcessiveVersionsThreshold {
+			if eventArgsList[i].Object.NumVersions > int(scannerExcessObjectVersions.Load()) {
 				// Send events for excessive versions.
 				sendEvent(eventArgs{
 					EventName:    event.ObjectManyVersions,
@@ -1406,6 +1405,15 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 					UserAgent:    r.UserAgent() + " " + "MinIO-Fan-Out",
 					Host:         handlers.GetSourceIP(r),
 				})
+
+				auditLogInternal(context.Background(), AuditLogOptions{
+					Event:     "scanner:manyversions",
+					APIName:   "PostPolicyBucket",
+					Bucket:    eventArgsList[i].Object.Bucket,
+					Object:    eventArgsList[i].Object.Name,
+					VersionID: eventArgsList[i].Object.VersionID,
+					Status:    http.StatusText(http.StatusOK),
+				})
 			}
 		}
 
@@ -1461,7 +1469,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		Host:         handlers.GetSourceIP(r),
 	})
 
-	if objInfo.NumVersions > dataScannerExcessiveVersionsThreshold {
+	if objInfo.NumVersions > int(scannerExcessObjectVersions.Load()) {
 		defer sendEvent(eventArgs{
 			EventName:    event.ObjectManyVersions,
 			BucketName:   objInfo.Bucket,
@@ -1471,6 +1479,15 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 			UserAgent:    r.UserAgent(),
 			Host:         handlers.GetSourceIP(r),
 		})
+
+		auditLogInternal(context.Background(), AuditLogOptions{
+			Event:     "scanner:manyversions",
+			APIName:   "PostPolicyBucket",
+			Bucket:    objInfo.Bucket,
+			Object:    objInfo.Name,
+			VersionID: objInfo.VersionID,
+			Status:    http.StatusText(http.StatusOK),
+		})
 	}
 
 	if redirectURL != nil { // success_action_redirect is valid and set.

cmd/bucket-handlers_test.go

@@ -880,6 +880,15 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 			expectedContent:    encodedAnonResponseWithPartialPublicAccess,
 			expectedRespStatus: http.StatusOK,
 		},
+		// Test case - 7.
+		// Bucket does not exist.
+		7: {
+			bucket:             "unknown-bucket-name",
+			objects:            successRequest0,
+			accessKey:          credentials.AccessKey,
+			secretKey:          credentials.SecretKey,
+			expectedRespStatus: http.StatusNotFound,
+		},
 	}
 
 	for i, testCase := range testCases {
@@ -888,13 +897,12 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 
 		// Generate a signed or anonymous request based on the testCase
 		if testCase.accessKey != "" {
-			req, err = newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", bucketName),
+			req, err = newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", testCase.bucket),
 				int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey, nil)
 		} else {
-			req, err = newTestRequest(http.MethodPost, getDeleteMultipleObjectsURL("", bucketName),
+			req, err = newTestRequest(http.MethodPost, getDeleteMultipleObjectsURL("", testCase.bucket),
 				int64(len(testCase.objects)), bytes.NewReader(testCase.objects))
 		}
-
 		if err != nil {
 			t.Fatalf("Failed to create HTTP request for DeleteMultipleObjects: <ERROR> %v", err)
 		}

cmd/bucket-lifecycle.go

@@ -39,6 +39,7 @@ import (
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/event"
 	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
 	"github.com/minio/pkg/v2/env"
@@ -118,8 +119,8 @@ func (es *expiryState) PendingTasks() int {
 // close closes work channels exactly once.
 func (es *expiryState) close() {
 	es.once.Do(func() {
-		close(es.byDaysCh)
-		close(es.byNewerNoncurrentCh)
+		xioutil.SafeClose(es.byDaysCh)
+		xioutil.SafeClose(es.byNewerNoncurrentCh)
 	})
 }
 

cmd/bucket-metadata-sys.go

@@ -22,6 +22,7 @@ import (
 	"encoding/xml"
 	"errors"
 	"fmt"
+	"math/rand"
 	"sync"
 	"time"
 
@@ -482,11 +483,11 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 	for index := range buckets {
 		index := index
 		g.Go(func() error {
-			_, _ = sys.objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{
-				// Ensure heal opts for bucket metadata be deep healed all the time.
-				ScanMode: madmin.HealDeepScan,
-				Recreate: true,
-			})
+			// Sleep and stagger to avoid blocked CPU and thundering
+			// herd upon start up sequence.
+			time.Sleep(25*time.Millisecond + time.Duration(rand.Int63n(int64(100*time.Millisecond))))
+
+			_, _ = sys.objAPI.HealBucket(ctx, buckets[index].Name, madmin.HealOpts{Recreate: true})
 			meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[index].Name)
 			if err != nil {
 				return err

cmd/bucket-metadata.go

@@ -182,26 +182,34 @@ func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket
 		b.defaultTimestamps()
 	}
 
-	configs, err := b.getAllLegacyConfigs(ctx, objectAPI)
-	if err != nil {
-		return b, err
+	// If bucket metadata is missing look for legacy files,
+	// since we only ever had b.Created as non-zero when
+	// migration was complete in 2020-May release. So this
+	// a check to avoid migrating for buckets that already
+	// have this field set.
+	if b.Created.IsZero() {
+		configs, err := b.getAllLegacyConfigs(ctx, objectAPI)
+		if err != nil {
+			return b, err
+		}
+
+		if len(configs) > 0 {
+			// Old bucket without bucket metadata. Hence we migrate existing settings.
+			if err = b.convertLegacyConfigs(ctx, objectAPI, configs); err != nil {
+				return b, err
+			}
+		}
 	}
 
-	if len(configs) == 0 {
-		if parse {
-			// nothing to update, parse and proceed.
-			err = b.parseAllConfigs(ctx, objectAPI)
+	if parse {
+		// nothing to update, parse and proceed.
+		if err = b.parseAllConfigs(ctx, objectAPI); err != nil {
+			return b, err
 		}
-	} else {
-		// Old bucket without bucket metadata. Hence we migrate existing settings.
-		err = b.convertLegacyConfigs(ctx, objectAPI, configs)
-	}
-	if err != nil {
-		return b, err
 	}
 
 	// migrate unencrypted remote targets
-	if err := b.migrateTargetConfig(ctx, objectAPI); err != nil {
+	if err = b.migrateTargetConfig(ctx, objectAPI); err != nil {
 		return b, err
 	}
 
@@ -331,7 +339,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
 	for _, legacyFile := range legacyConfigs {
 		configFile := path.Join(bucketMetaPrefix, b.Name, legacyFile)
 
-		configData, err := readConfig(ctx, objectAPI, configFile)
+		configData, info, err := readConfigWithMetadata(ctx, objectAPI, configFile, ObjectOptions{})
 		if err != nil {
 			if _, ok := err.(ObjectExistsAsDirectory); ok {
 				// in FS mode it possible that we have actual
@@ -346,6 +354,7 @@ func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI Obje
 			return nil, err
 		}
 		configs[legacyFile] = configData
+		b.Created = info.ModTime
 	}
 
 	return configs, nil

cmd/bucket-policy.go

@@ -66,6 +66,14 @@ func NewPolicySys() *PolicySys {
 	return &PolicySys{}
 }
 
+func getSTSConditionValues(r *http.Request, lc string, cred auth.Credentials) map[string][]string {
+	m := make(map[string][]string)
+	if d := r.Form.Get("DurationSeconds"); d != "" {
+		m["DurationSeconds"] = []string{d}
+	}
+	return m
+}
+
 func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[string][]string {
 	currTime := UTCNow()
 

cmd/bucket-replication-metrics.go

@@ -379,3 +379,145 @@ func (s *SMA) simpleMovingAvg() float64 {
 const (
 	defaultWindowSize = 10
 )
+
+type proxyStatsCache struct {
+	srProxyStats ProxyMetric
+	bucketStats  map[string]ProxyMetric
+	sync.RWMutex // mutex for proxy stats
+}
+
+func newProxyStatsCache() proxyStatsCache {
+	return proxyStatsCache{
+		bucketStats: make(map[string]ProxyMetric),
+	}
+}
+
+func (p *proxyStatsCache) inc(bucket string, api replProxyAPI, isErr bool) {
+	p.Lock()
+	defer p.Unlock()
+	v, ok := p.bucketStats[bucket]
+	if !ok {
+		v = ProxyMetric{}
+	}
+	switch api {
+	case putObjectTaggingAPI:
+		if !isErr {
+			atomic.AddUint64(&v.PutTagTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.PutTagTotal, 1)
+		} else {
+			atomic.AddUint64(&v.PutTagFailedTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.PutTagFailedTotal, 1)
+		}
+	case getObjectTaggingAPI:
+		if !isErr {
+			atomic.AddUint64(&v.GetTagTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.GetTagTotal, 1)
+		} else {
+			atomic.AddUint64(&v.GetTagFailedTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.GetTagFailedTotal, 1)
+		}
+	case removeObjectTaggingAPI:
+		if !isErr {
+			atomic.AddUint64(&v.RmvTagTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.RmvTagTotal, 1)
+		} else {
+			atomic.AddUint64(&v.RmvTagFailedTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.RmvTagFailedTotal, 1)
+		}
+	case headObjectAPI:
+		if !isErr {
+			atomic.AddUint64(&v.HeadTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.HeadTotal, 1)
+		} else {
+			atomic.AddUint64(&v.HeadFailedTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.HeadFailedTotal, 1)
+		}
+	case getObjectAPI:
+		if !isErr {
+			atomic.AddUint64(&v.GetTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.GetTotal, 1)
+		} else {
+			atomic.AddUint64(&v.GetFailedTotal, 1)
+			atomic.AddUint64(&p.srProxyStats.GetFailedTotal, 1)
+		}
+	default:
+		return
+	}
+	p.bucketStats[bucket] = v
+}
+
+func (p *proxyStatsCache) getBucketStats(bucket string) ProxyMetric {
+	p.RLock()
+	defer p.RUnlock()
+	v, ok := p.bucketStats[bucket]
+
+	if !ok {
+		return ProxyMetric{}
+	}
+	return ProxyMetric{
+		PutTagTotal: atomic.LoadUint64(&v.PutTagTotal),
+		GetTagTotal: atomic.LoadUint64(&v.GetTagTotal),
+		RmvTagTotal: atomic.LoadUint64(&v.RmvTagTotal),
+		HeadTotal:   atomic.LoadUint64(&v.HeadTotal),
+		GetTotal:    atomic.LoadUint64(&v.GetTotal),
+
+		PutTagFailedTotal: atomic.LoadUint64(&v.PutTagFailedTotal),
+		GetTagFailedTotal: atomic.LoadUint64(&v.GetTagFailedTotal),
+		RmvTagFailedTotal: atomic.LoadUint64(&v.RmvTagFailedTotal),
+		HeadFailedTotal:   atomic.LoadUint64(&v.HeadFailedTotal),
+		GetFailedTotal:    atomic.LoadUint64(&v.GetFailedTotal),
+	}
+}
+
+func (p *proxyStatsCache) getSiteStats() ProxyMetric {
+	v := p.srProxyStats
+	return ProxyMetric{
+		PutTagTotal:       atomic.LoadUint64(&v.PutTagTotal),
+		GetTagTotal:       atomic.LoadUint64(&v.GetTagTotal),
+		RmvTagTotal:       atomic.LoadUint64(&v.RmvTagTotal),
+		HeadTotal:         atomic.LoadUint64(&v.HeadTotal),
+		GetTotal:          atomic.LoadUint64(&v.GetTotal),
+		PutTagFailedTotal: atomic.LoadUint64(&v.PutTagFailedTotal),
+		GetTagFailedTotal: atomic.LoadUint64(&v.GetTagFailedTotal),
+		RmvTagFailedTotal: atomic.LoadUint64(&v.RmvTagFailedTotal),
+		HeadFailedTotal:   atomic.LoadUint64(&v.HeadFailedTotal),
+		GetFailedTotal:    atomic.LoadUint64(&v.GetFailedTotal),
+	}
+}
+
+type replProxyAPI string
+
+const (
+	putObjectTaggingAPI    replProxyAPI = "PutObjectTagging"
+	getObjectTaggingAPI    replProxyAPI = "GetObjectTagging"
+	removeObjectTaggingAPI replProxyAPI = "RemoveObjectTagging"
+	headObjectAPI          replProxyAPI = "HeadObject"
+	getObjectAPI           replProxyAPI = "GetObject"
+)
+
+// ProxyMetric holds stats for replication proxying
+type ProxyMetric struct {
+	PutTagTotal       uint64 `json:"putTaggingProxyTotal" msg:"ptc"`
+	GetTagTotal       uint64 `json:"getTaggingProxyTotal" msg:"gtc"`
+	RmvTagTotal       uint64 `json:"removeTaggingProxyTotal" msg:"rtc"`
+	GetTotal          uint64 `json:"getProxyTotal" msg:"gc"`
+	HeadTotal         uint64 `json:"headProxyTotal" msg:"hc"`
+	PutTagFailedTotal uint64 `json:"putTaggingProxyFailed" msg:"ptf"`
+	GetTagFailedTotal uint64 `json:"getTaggingProxyFailed" msg:"gtf"`
+	RmvTagFailedTotal uint64 `json:"removeTaggingProxyFailed" msg:"rtf"`
+	GetFailedTotal    uint64 `json:"getProxyFailed" msg:"gf"`
+	HeadFailedTotal   uint64 `json:"headProxyFailed" msg:"hf"`
+}
+
+func (p *ProxyMetric) add(p2 ProxyMetric) {
+	atomic.AddUint64(&p.GetTotal, p2.GetTotal)
+	atomic.AddUint64(&p.HeadTotal, p2.HeadTotal)
+	atomic.AddUint64(&p.GetTagTotal, p2.GetTagTotal)
+	atomic.AddUint64(&p.PutTagTotal, p2.PutTagTotal)
+	atomic.AddUint64(&p.RmvTagTotal, p2.RmvTagTotal)
+	atomic.AddUint64(&p.GetFailedTotal, p2.GetFailedTotal)
+	atomic.AddUint64(&p.HeadFailedTotal, p2.HeadFailedTotal)
+	atomic.AddUint64(&p.GetTagFailedTotal, p2.GetTagFailedTotal)
+	atomic.AddUint64(&p.PutTagFailedTotal, p2.PutTagFailedTotal)
+	atomic.AddUint64(&p.RmvTagFailedTotal, p2.RmvTagFailedTotal)
+}

cmd/bucket-replication-metrics_gen.go

@@ -635,6 +635,334 @@ func (z InQueueStats) Msgsize() (s int) {
 	return
 }
 
+// DecodeMsg implements msgp.Decodable
+func (z *ProxyMetric) DecodeMsg(dc *msgp.Reader) (err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, err = dc.ReadMapHeader()
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, err = dc.ReadMapKeyPtr()
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ptc":
+			z.PutTagTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "PutTagTotal")
+				return
+			}
+		case "gtc":
+			z.GetTagTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "GetTagTotal")
+				return
+			}
+		case "rtc":
+			z.RmvTagTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "RmvTagTotal")
+				return
+			}
+		case "gc":
+			z.GetTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "GetTotal")
+				return
+			}
+		case "hc":
+			z.HeadTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "HeadTotal")
+				return
+			}
+		case "ptf":
+			z.PutTagFailedTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "PutTagFailedTotal")
+				return
+			}
+		case "gtf":
+			z.GetTagFailedTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "GetTagFailedTotal")
+				return
+			}
+		case "rtf":
+			z.RmvTagFailedTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "RmvTagFailedTotal")
+				return
+			}
+		case "gf":
+			z.GetFailedTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "GetFailedTotal")
+				return
+			}
+		case "hf":
+			z.HeadFailedTotal, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "HeadFailedTotal")
+				return
+			}
+		default:
+			err = dc.Skip()
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	return
+}
+
+// EncodeMsg implements msgp.Encodable
+func (z *ProxyMetric) EncodeMsg(en *msgp.Writer) (err error) {
+	// map header, size 10
+	// write "ptc"
+	err = en.Append(0x8a, 0xa3, 0x70, 0x74, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.PutTagTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "PutTagTotal")
+		return
+	}
+	// write "gtc"
+	err = en.Append(0xa3, 0x67, 0x74, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.GetTagTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "GetTagTotal")
+		return
+	}
+	// write "rtc"
+	err = en.Append(0xa3, 0x72, 0x74, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.RmvTagTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "RmvTagTotal")
+		return
+	}
+	// write "gc"
+	err = en.Append(0xa2, 0x67, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.GetTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "GetTotal")
+		return
+	}
+	// write "hc"
+	err = en.Append(0xa2, 0x68, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.HeadTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "HeadTotal")
+		return
+	}
+	// write "ptf"
+	err = en.Append(0xa3, 0x70, 0x74, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.PutTagFailedTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "PutTagFailedTotal")
+		return
+	}
+	// write "gtf"
+	err = en.Append(0xa3, 0x67, 0x74, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.GetTagFailedTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "GetTagFailedTotal")
+		return
+	}
+	// write "rtf"
+	err = en.Append(0xa3, 0x72, 0x74, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.RmvTagFailedTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "RmvTagFailedTotal")
+		return
+	}
+	// write "gf"
+	err = en.Append(0xa2, 0x67, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.GetFailedTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "GetFailedTotal")
+		return
+	}
+	// write "hf"
+	err = en.Append(0xa2, 0x68, 0x66)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.HeadFailedTotal)
+	if err != nil {
+		err = msgp.WrapError(err, "HeadFailedTotal")
+		return
+	}
+	return
+}
+
+// MarshalMsg implements msgp.Marshaler
+func (z *ProxyMetric) MarshalMsg(b []byte) (o []byte, err error) {
+	o = msgp.Require(b, z.Msgsize())
+	// map header, size 10
+	// string "ptc"
+	o = append(o, 0x8a, 0xa3, 0x70, 0x74, 0x63)
+	o = msgp.AppendUint64(o, z.PutTagTotal)
+	// string "gtc"
+	o = append(o, 0xa3, 0x67, 0x74, 0x63)
+	o = msgp.AppendUint64(o, z.GetTagTotal)
+	// string "rtc"
+	o = append(o, 0xa3, 0x72, 0x74, 0x63)
+	o = msgp.AppendUint64(o, z.RmvTagTotal)
+	// string "gc"
+	o = append(o, 0xa2, 0x67, 0x63)
+	o = msgp.AppendUint64(o, z.GetTotal)
+	// string "hc"
+	o = append(o, 0xa2, 0x68, 0x63)
+	o = msgp.AppendUint64(o, z.HeadTotal)
+	// string "ptf"
+	o = append(o, 0xa3, 0x70, 0x74, 0x66)
+	o = msgp.AppendUint64(o, z.PutTagFailedTotal)
+	// string "gtf"
+	o = append(o, 0xa3, 0x67, 0x74, 0x66)
+	o = msgp.AppendUint64(o, z.GetTagFailedTotal)
+	// string "rtf"
+	o = append(o, 0xa3, 0x72, 0x74, 0x66)
+	o = msgp.AppendUint64(o, z.RmvTagFailedTotal)
+	// string "gf"
+	o = append(o, 0xa2, 0x67, 0x66)
+	o = msgp.AppendUint64(o, z.GetFailedTotal)
+	// string "hf"
+	o = append(o, 0xa2, 0x68, 0x66)
+	o = msgp.AppendUint64(o, z.HeadFailedTotal)
+	return
+}
+
+// UnmarshalMsg implements msgp.Unmarshaler
+func (z *ProxyMetric) UnmarshalMsg(bts []byte) (o []byte, err error) {
+	var field []byte
+	_ = field
+	var zb0001 uint32
+	zb0001, bts, err = msgp.ReadMapHeaderBytes(bts)
+	if err != nil {
+		err = msgp.WrapError(err)
+		return
+	}
+	for zb0001 > 0 {
+		zb0001--
+		field, bts, err = msgp.ReadMapKeyZC(bts)
+		if err != nil {
+			err = msgp.WrapError(err)
+			return
+		}
+		switch msgp.UnsafeString(field) {
+		case "ptc":
+			z.PutTagTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PutTagTotal")
+				return
+			}
+		case "gtc":
+			z.GetTagTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "GetTagTotal")
+				return
+			}
+		case "rtc":
+			z.RmvTagTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "RmvTagTotal")
+				return
+			}
+		case "gc":
+			z.GetTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "GetTotal")
+				return
+			}
+		case "hc":
+			z.HeadTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "HeadTotal")
+				return
+			}
+		case "ptf":
+			z.PutTagFailedTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "PutTagFailedTotal")
+				return
+			}
+		case "gtf":
+			z.GetTagFailedTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "GetTagFailedTotal")
+				return
+			}
+		case "rtf":
+			z.RmvTagFailedTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "RmvTagFailedTotal")
+				return
+			}
+		case "gf":
+			z.GetFailedTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "GetFailedTotal")
+				return
+			}
+		case "hf":
+			z.HeadFailedTotal, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "HeadFailedTotal")
+				return
+			}
+		default:
+			bts, err = msgp.Skip(bts)
+			if err != nil {
+				err = msgp.WrapError(err)
+				return
+			}
+		}
+	}
+	o = bts
+	return
+}
+
+// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
+func (z *ProxyMetric) Msgsize() (s int) {
+	s = 1 + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size
+	return
+}
+
 // DecodeMsg implements msgp.Decodable
 func (z *QStat) DecodeMsg(dc *msgp.Reader) (err error) {
 	var field []byte

cmd/bucket-replication-metrics_gen_test.go

@@ -348,6 +348,119 @@ func BenchmarkDecodeInQueueStats(b *testing.B) {
 	}
 }
 
+func TestMarshalUnmarshalProxyMetric(t *testing.T) {
+	v := ProxyMetric{}
+	bts, err := v.MarshalMsg(nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	left, err := v.UnmarshalMsg(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
+	}
+
+	left, err = msgp.Skip(bts)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(left) > 0 {
+		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
+	}
+}
+
+func BenchmarkMarshalMsgProxyMetric(b *testing.B) {
+	v := ProxyMetric{}
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.MarshalMsg(nil)
+	}
+}
+
+func BenchmarkAppendMsgProxyMetric(b *testing.B) {
+	v := ProxyMetric{}
+	bts := make([]byte, 0, v.Msgsize())
+	bts, _ = v.MarshalMsg(bts[0:0])
+	b.SetBytes(int64(len(bts)))
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		bts, _ = v.MarshalMsg(bts[0:0])
+	}
+}
+
+func BenchmarkUnmarshalProxyMetric(b *testing.B) {
+	v := ProxyMetric{}
+	bts, _ := v.MarshalMsg(nil)
+	b.ReportAllocs()
+	b.SetBytes(int64(len(bts)))
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		_, err := v.UnmarshalMsg(bts)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func TestEncodeDecodeProxyMetric(t *testing.T) {
+	v := ProxyMetric{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+
+	m := v.Msgsize()
+	if buf.Len() > m {
+		t.Log("WARNING: TestEncodeDecodeProxyMetric Msgsize() is inaccurate")
+	}
+
+	vn := ProxyMetric{}
+	err := msgp.Decode(&buf, &vn)
+	if err != nil {
+		t.Error(err)
+	}
+
+	buf.Reset()
+	msgp.Encode(&buf, &v)
+	err = msgp.NewReader(&buf).Skip()
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func BenchmarkEncodeProxyMetric(b *testing.B) {
+	v := ProxyMetric{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	en := msgp.NewWriter(msgp.Nowhere)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		v.EncodeMsg(en)
+	}
+	en.Flush()
+}
+
+func BenchmarkDecodeProxyMetric(b *testing.B) {
+	v := ProxyMetric{}
+	var buf bytes.Buffer
+	msgp.Encode(&buf, &v)
+	b.SetBytes(int64(buf.Len()))
+	rd := msgp.NewEndlessReader(buf.Bytes(), b)
+	dc := msgp.NewReader(rd)
+	b.ReportAllocs()
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		err := v.DecodeMsg(dc)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
 func TestMarshalUnmarshalQStat(t *testing.T) {
 	v := QStat{}
 	bts, err := v.MarshalMsg(nil)

cmd/bucket-replication-stats.go

@@ -45,6 +45,8 @@ type ReplicationStats struct {
 	workers *ActiveWorkerStat
 	// queue stats cache
 	qCache queueCache
+
+	pCache proxyStatsCache
 	// mrf backlog stats
 	mrfStats ReplicationMRFStats
 	// for bucket replication, continue to use existing cache
@@ -305,6 +307,7 @@ func (r *ReplicationStats) getSRMetricsForNode() SRMetricsSummary {
 		Queued:        r.qCache.getSiteStats(),
 		ActiveWorkers: r.ActiveWorkers(),
 		Metrics:       r.srStats.get(),
+		Proxied:       r.pCache.getSiteStats(),
 		ReplicaSize:   atomic.LoadInt64(&r.srStats.ReplicaSize),
 		ReplicaCount:  atomic.LoadInt64(&r.srStats.ReplicaCount),
 	}
@@ -333,6 +336,7 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
 	rs := ReplicationStats{
 		Cache:           make(map[string]*BucketReplicationStats),
 		qCache:          newQueueCache(r),
+		pCache:          newProxyStatsCache(),
 		srStats:         newSRStats(),
 		movingAvgTicker: time.NewTicker(2 * time.Second),
 		wTimer:          time.NewTicker(2 * time.Second),
@@ -371,6 +375,7 @@ func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, bucket
 				Stats: make(map[string]*BucketReplicationStat),
 			},
 			QueueStats: ReplicationQueueStats{},
+			ProxyStats: ProxyMetric{},
 		}
 		return bs
 	}
@@ -430,11 +435,16 @@ func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, bucket
 	for _, bs := range bucketStats {
 		qs.Nodes = append(qs.Nodes, bs.QueueStats.Nodes...)
 	}
-
 	qs.Uptime = UTCNow().Unix() - globalBootTime.Unix()
+
+	var ps ProxyMetric
+	for _, bs := range bucketStats {
+		ps.add(bs.ProxyStats)
+	}
 	bs = BucketStats{
 		ReplicationStats: s,
 		QueueStats:       qs,
+		ProxyStats:       ps,
 	}
 	r.mostRecentStatsMu.Lock()
 	if len(r.mostRecentStats.Stats) == 0 {
@@ -482,3 +492,12 @@ func (r *ReplicationStats) decQ(bucket string, sz int64, isDelMarker bool, opTyp
 	atomic.AddInt64(&r.qCache.srQueueStats.nowBytes, -1*sz)
 	atomic.AddInt64(&r.qCache.srQueueStats.nowCount, -1)
 }
+
+// incProxy increments proxy metrics for proxied calls
+func (r *ReplicationStats) incProxy(bucket string, api replProxyAPI, isErr bool) {
+	r.pCache.inc(bucket, api, isErr)
+}
+
+func (r *ReplicationStats) getProxyStats(bucket string) ProxyMetric {
+	return r.pCache.getBucketStats(bucket)
+}

cmd/bucket-replication.go

@@ -47,6 +47,7 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
 	"github.com/zeebo/xxh3"
@@ -178,7 +179,7 @@ func checkRemoteEndpoint(ctx context.Context, epURL *url.URL) error {
 	}
 
 	client := &http.Client{
-		Transport: NewHTTPTransport(),
+		Transport: globalRemoteTargetTransport,
 		Timeout:   10 * time.Second,
 	}
 
@@ -683,7 +684,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		} else {
 			rinfo.VersionPurgeStatus = Failed
 		}
-		logger.LogIf(ctx, fmt.Errorf("Unable to replicate delete marker to %s/%s(%s): %s", tgt.Bucket, dobj.ObjectName, versionID, rmErr))
+		logger.LogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr))
 		if rmErr != nil && minio.IsNetworkOrHostDown(rmErr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
 			globalBucketTargetSys.markOffline(tgt.EndpointURL())
 		}
@@ -1204,7 +1205,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 
 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
+		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty for target %s", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1230,7 +1231,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 
 	putOpts, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s err:%w", bucket, err))
+		logger.LogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1265,14 +1266,14 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			r, objInfo, putOpts); rinfo.Err != nil {
 			if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s (target: %s)", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err, tgt.EndpointURL()))
 			}
 		}
 	} else {
 		if _, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); rinfo.Err != nil {
 			if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s (target: %s)", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err, tgt.EndpointURL()))
 			}
 		}
 	}
@@ -1336,7 +1337,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
-			logger.LogIf(ctx, fmt.Errorf("unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
+			logger.LogIf(ctx, fmt.Errorf("unable to replicate to target %s for %s/%s(%s): %w", tgt.EndpointURL(), bucket, object, objInfo.VersionID, err))
 		}
 		return
 	}
@@ -1367,7 +1368,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 
 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
+		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s) to %s, target bucket is missing", objInfo.Name, objInfo.VersionID, tgt.EndpointURL()))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1397,7 +1398,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		if rAction == replicateNone {
 			if ri.OpType == replication.ExistingObjectReplicationType &&
 				objInfo.ModTime.Unix() > oi.LastModified.Unix() && objInfo.VersionID == nullVersionID {
-				logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target", bucket, object))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target %s", bucket, object, tgt.EndpointURL()))
 				sendEvent(eventArgs{
 					EventName:  event.ObjectReplicationNotTracked,
 					BucketName: bucket,
@@ -1487,13 +1488,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		}
 		if _, rinfo.Err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); rinfo.Err != nil {
 			rinfo.ReplicationStatus = replication.Failed
-			logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
+			logger.LogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
 		}
 	} else {
 		var putOpts minio.PutObjectOptions
 		putOpts, err = putReplicationOpts(ctx, tgt.StorageClass, objInfo)
 		if err != nil {
-			logger.LogIf(ctx, fmt.Errorf("failed to get target for replication bucket:%s err:%w", bucket, err))
+			logger.LogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err))
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
@@ -1527,7 +1528,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				r, objInfo, putOpts); rinfo.Err != nil {
 				if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
+					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
 				} else {
 					rinfo.ReplicationStatus = replication.Completed
 				}
@@ -1536,7 +1537,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			if _, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); rinfo.Err != nil {
 				if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" {
 					rinfo.ReplicationStatus = replication.Failed
-					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, rinfo.Err))
+					logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err))
 				} else {
 					rinfo.ReplicationStatus = replication.Completed
 				}
@@ -1894,7 +1895,7 @@ func (p *ReplicationPool) AddLargeWorkers() {
 	go func() {
 		<-p.ctx.Done()
 		for i := 0; i < LargeWorkerCount; i++ {
-			close(p.lrgworkers[i])
+			xioutil.SafeClose(p.lrgworkers[i])
 		}
 	}()
 }
@@ -1953,7 +1954,7 @@ func (p *ReplicationPool) ResizeWorkers(n, checkOld int) {
 	for len(p.workers) > n {
 		worker := p.workers[len(p.workers)-1]
 		p.workers = p.workers[:len(p.workers)-1]
-		close(worker)
+		xioutil.SafeClose(worker)
 	}
 }
 
@@ -2234,6 +2235,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 	if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header
 		return nil, oi, proxy
 	}
+	var perr error
 	for _, t := range proxyTargets.Targets {
 		tgt = globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn)
 		if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
@@ -2263,6 +2265,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 
 		objInfo, err := tgt.StatObject(ctx, t.TargetBucket, object, gopts)
 		if err != nil {
+			perr = err
 			if isErrInvalidRange(ErrorRespToObjectError(err, bucket, object)) {
 				return nil, oi, proxyResult{Err: err}
 			}
@@ -2299,6 +2302,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 		}
 		return tgt, oi, proxyResult{Proxy: true}
 	}
+	proxy.Err = perr
 	return nil, oi, proxy
 }
 
@@ -2405,8 +2409,7 @@ func proxyTaggingToRepTarget(ctx context.Context, bucket, object string, tags *t
 			taggedCount++
 			continue
 		}
-		errCode := minio.ToErrorResponse(err).Code
-		if errCode != "NoSuchKey" && errCode != "NoSuchVersion" {
+		if err != nil {
 			terr = err
 		}
 	}
@@ -2466,6 +2469,9 @@ func proxyGetTaggingToRepTarget(ctx context.Context, bucket, object string, opts
 			tgs, _ = tags.MapToObjectTags(tagSlc[idx])
 		}
 	}
+	if len(errs) == 1 {
+		proxy.Err = errs[0]
+	}
 	return tgs, proxy
 }
 
@@ -2619,7 +2625,7 @@ func (s *replicationResyncer) PersistToDisk(ctx context.Context, objectAPI Objec
 				}
 				if updt {
 					if err := saveResyncStatus(ctx, bucket, brs, objectAPI); err != nil {
-						logger.LogIf(ctx, fmt.Errorf("Could not save resync metadata to drive for %s - %w", bucket, err))
+						logger.LogIf(ctx, fmt.Errorf("could not save resync metadata to drive for %s - %w", bucket, err))
 					} else {
 						lastResyncStatusSave[bucket] = brs.LastUpdate
 					}
@@ -2755,7 +2761,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	}
 	workers := make([]chan ReplicateObjectInfo, resyncParallelRoutines)
 	resultCh := make(chan TargetReplicationResyncStatus, 1)
-	defer close(resultCh)
+	defer xioutil.SafeClose(resultCh)
 	go func() {
 		for r := range resultCh {
 			s.incStats(r, opts)
@@ -2867,7 +2873,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 		}
 	}
 	for i := 0; i < resyncParallelRoutines; i++ {
-		close(workers[i])
+		xioutil.SafeClose(workers[i])
 	}
 	wg.Wait()
 	resyncStatus = ResyncCompleted
@@ -3123,7 +3129,7 @@ func getReplicationDiff(ctx context.Context, objAPI ObjectLayer, bucket string,
 	}
 	diffCh := make(chan madmin.DiffInfo, 4000)
 	go func() {
-		defer close(diffCh)
+		defer xioutil.SafeClose(diffCh)
 		for obj := range objInfoCh {
 			if contextCanceled(ctx) {
 				// Just consume input...
@@ -3212,6 +3218,9 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 		return roi
 	}
 
+	if isVeeamSOSAPIObject(oi.Name) {
+		return roi
+	}
 	if rcfg.Config == nil || rcfg.remotes == nil {
 		return roi
 	}
@@ -3316,7 +3325,7 @@ func (p *ReplicationPool) persistMRF() {
 			mTimer.Reset(mrfSaveInterval)
 		case <-p.ctx.Done():
 			p.mrfStopCh <- struct{}{}
-			close(p.mrfSaveCh)
+			xioutil.SafeClose(p.mrfSaveCh)
 			// We try to save if possible, but we don't care beyond that.
 			saveMRFToDisk()
 			return
@@ -3389,7 +3398,7 @@ func (p *ReplicationPool) persistToDrive(ctx context.Context, v MRFReplicateEntr
 
 	for _, localDrive := range localDrives {
 		r := newReader()
-		err := localDrive.CreateFile(ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), -1, r)
+		err := localDrive.CreateFile(ctx, "", minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), -1, r)
 		r.Close()
 		if err == nil {
 			break
@@ -3551,7 +3560,7 @@ func (p *ReplicationPool) getMRF(ctx context.Context, bucket string) (ch <-chan
 
 	mrfCh := make(chan madmin.ReplicationMRF, 100)
 	go func() {
-		defer close(mrfCh)
+		defer xioutil.SafeClose(mrfCh)
 		for vID, e := range mrfRec.Entries {
 			if bucket != "" && e.Bucket != bucket {
 				continue

cmd/bucket-stats.go

@@ -154,6 +154,7 @@ type BucketStats struct {
 	Uptime           int64                  `json:"uptime"`
 	ReplicationStats BucketReplicationStats `json:"currStats"`  // current replication stats since cluster startup
 	QueueStats       ReplicationQueueStats  `json:"queueStats"` // replication queue stats
+	ProxyStats       ProxyMetric            `json:"proxyStats"`
 }
 
 // BucketReplicationStats represents inline replication statistics

cmd/bucket-stats_gen.go

@@ -1142,6 +1142,12 @@ func (z *BucketStats) DecodeMsg(dc *msgp.Reader) (err error) {
 					}
 				}
 			}
+		case "ProxyStats":
+			err = z.ProxyStats.DecodeMsg(dc)
+			if err != nil {
+				err = msgp.WrapError(err, "ProxyStats")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -1155,9 +1161,9 @@ func (z *BucketStats) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z *BucketStats) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 3
+	// map header, size 4
 	// write "Uptime"
-	err = en.Append(0x83, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65)
+	err = en.Append(0x84, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -1209,15 +1215,25 @@ func (z *BucketStats) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "QueueStats", "Uptime")
 		return
 	}
+	// write "ProxyStats"
+	err = en.Append(0xaa, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73)
+	if err != nil {
+		return
+	}
+	err = z.ProxyStats.EncodeMsg(en)
+	if err != nil {
+		err = msgp.WrapError(err, "ProxyStats")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketStats) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 3
+	// map header, size 4
 	// string "Uptime"
-	o = append(o, 0x83, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65)
+	o = append(o, 0x84, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65)
 	o = msgp.AppendInt64(o, z.Uptime)
 	// string "ReplicationStats"
 	o = append(o, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73)
@@ -1242,6 +1258,13 @@ func (z *BucketStats) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Uptime"
 	o = append(o, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65)
 	o = msgp.AppendInt64(o, z.QueueStats.Uptime)
+	// string "ProxyStats"
+	o = append(o, 0xaa, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73)
+	o, err = z.ProxyStats.MarshalMsg(o)
+	if err != nil {
+		err = msgp.WrapError(err, "ProxyStats")
+		return
+	}
 	return
 }
 
@@ -1323,6 +1346,12 @@ func (z *BucketStats) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					}
 				}
 			}
+		case "ProxyStats":
+			bts, err = z.ProxyStats.UnmarshalMsg(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "ProxyStats")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -1341,7 +1370,7 @@ func (z *BucketStats) Msgsize() (s int) {
 	for za0001 := range z.QueueStats.Nodes {
 		s += z.QueueStats.Nodes[za0001].Msgsize()
 	}
-	s += 7 + msgp.Int64Size
+	s += 7 + msgp.Int64Size + 11 + z.ProxyStats.Msgsize()
 	return
 }
 

cmd/common-main.go

@@ -69,7 +69,7 @@ import (
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
 
-var shardDiskTimeDelta time.Duration
+var currentReleaseTime time.Time
 
 func init() {
 	if runtime.GOOS == "windows" {
@@ -105,18 +105,15 @@ func init() {
 	gob.Register(StorageErr(""))
 	gob.Register(madmin.TimeInfo{})
 	gob.Register(madmin.XFSErrorConfigs{})
+	gob.Register(map[string]string{})
 	gob.Register(map[string]interface{}{})
 
-	var err error
-	shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m"))
-	if err != nil {
-		shardDiskTimeDelta = 1 * time.Minute
-	}
-
 	// All minio-go and madmin-go API operations shall be performed only once,
 	// another way to look at this is we are turning off retries.
 	minio.MaxRetry = 1
 	madmin.MaxRetry = 1
+
+	currentReleaseTime, _ = GetCurrentReleaseTime()
 }
 
 const consolePrefix = "CONSOLE_"
@@ -301,9 +298,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	// Its OK to ignore any errors during doUpdate() here.
-	crTime, err := GetCurrentReleaseTime()
-	if err != nil {
+	if currentReleaseTime.IsZero() {
 		return
 	}
 
@@ -314,8 +309,8 @@ func checkUpdate(mode string) {
 
 	var older time.Duration
 	var downloadURL string
-	if lrTime.After(crTime) {
-		older = lrTime.Sub(crTime)
+	if lrTime.After(currentReleaseTime) {
+		older = lrTime.Sub(currentReleaseTime)
 		downloadURL = getDownloadURL(releaseTimeToReleaseTag(lrTime))
 	}
 
@@ -324,7 +319,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	logger.Info(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
+	logger.Info(prepareUpdateMessage("Run `mc admin update ALIAS`", lrTime.Sub(currentReleaseTime)))
 }
 
 func newConfigDir(dir string, dirSet bool, getDefaultDir func() string) (*ConfigDir, error) {
@@ -373,7 +368,6 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 
 	// Check "no-compat" flag from command line argument.
 	ctxt.StrictS3Compat = !(ctx.IsSet("no-compat") || ctx.GlobalIsSet("no-compat"))
-	ctxt.PreAllocate = ctx.IsSet("pre-allocate") || ctx.GlobalIsSet("pre-allocate")
 
 	switch {
 	case ctx.IsSet("config-dir"):
@@ -400,10 +394,12 @@ func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) {
 	ctxt.UserTimeout = ctx.Duration("conn-user-timeout")
 	ctxt.ConnReadDeadline = ctx.Duration("conn-read-deadline")
 	ctxt.ConnWriteDeadline = ctx.Duration("conn-write-deadline")
+	ctxt.ConnClientReadDeadline = ctx.Duration("conn-client-read-deadline")
 
 	ctxt.ShutdownTimeout = ctx.Duration("shutdown-timeout")
 	ctxt.IdleTimeout = ctx.Duration("idle-timeout")
 	ctxt.ReadHeaderTimeout = ctx.Duration("read-header-timeout")
+	ctxt.MaxIdleConnsPerHost = ctx.Int("max-idle-conns-per-host")
 
 	if conf := ctx.String("config"); len(conf) > 0 {
 		err = mergeServerCtxtFromConfigFile(conf, ctxt)
@@ -1040,7 +1036,7 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 		}
 		if err = manager.AddCertificate(certFile, keyFile); err != nil {
 			err = fmt.Errorf("Unable to load TLS certificate '%s,%s': %w", certFile, keyFile, err)
-			logger.LogIf(GlobalContext, err, logger.Minio)
+			logger.LogIf(GlobalContext, err, logger.ErrorKind)
 		}
 	}
 	secureConn = true

cmd/config-current.go

@@ -602,6 +602,8 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		// update dynamic scanner values.
 		scannerIdleMode.Store(scannerCfg.IdleMode)
 		scannerCycle.Store(scannerCfg.Cycle)
+		scannerExcessObjectVersions.Store(scannerCfg.ExcessVersions)
+		scannerExcessFolders.Store(scannerCfg.ExcessFolders)
 		logger.LogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait))
 	case config.LoggerWebhookSubSys:
 		loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.LoggerWebhookSubSys)

cmd/consolelogger.go

@@ -196,7 +196,7 @@ func (sys *HTTPConsoleLoggerSys) Send(ctx context.Context, entry interface{}) er
 	sys.logBuf.Value = lg
 	sys.logBuf = sys.logBuf.Next()
 	sys.Unlock()
-	err := sys.console.Send(entry, string(logger.All))
+	err := sys.console.Send(entry)
 	if err != nil {
 		atomic.AddInt64(&sys.failedMessages, 1)
 	}

cmd/data-scanner.go

@@ -40,6 +40,7 @@ import (
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/heal"
 	"github.com/minio/minio/internal/event"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	uatomic "go.uber.org/atomic"
@@ -56,18 +57,17 @@ const (
 
 	healDeleteDangling   = true
 	healObjectSelectProb = 1024 // Overall probability of a file being scanned; one in n.
-
-	dataScannerExcessiveVersionsThreshold = 100   // Issue a warning when a single object has more versions than this
-	dataScannerExcessiveFoldersThreshold  = 50000 // Issue a warning when a folder has more subfolders than this in a *set*
 )
 
 var (
 	globalHealConfig heal.Config
 
 	// Sleeper values are updated when config is loaded.
-	scannerSleeper  = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
-	scannerCycle    = uatomic.NewDuration(dataScannerStartDelay)
-	scannerIdleMode = uatomic.NewInt32(0) // default is throttled when idle
+	scannerSleeper              = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
+	scannerCycle                = uatomic.NewDuration(dataScannerStartDelay)
+	scannerIdleMode             = uatomic.NewInt32(0) // default is throttled when idle
+	scannerExcessObjectVersions = uatomic.NewInt64(100)
+	scannerExcessFolders        = uatomic.NewInt64(50000)
 )
 
 // initDataScanner will start the scanner in the background.
@@ -529,8 +529,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders ||
 			len(existingFolders)+len(newFolders) >= dataScannerForceCompactAtFolders
 
-		if len(existingFolders)+len(newFolders) > dataScannerExcessiveFoldersThreshold {
-			// Notify object accessed via a GET request.
+		if len(existingFolders)+len(newFolders) > int(scannerExcessFolders.Load()) {
 			sendEvent(eventArgs{
 				EventName:  event.PrefixManyFolders,
 				BucketName: f.root,
@@ -1100,7 +1099,7 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 	}
 
 	// Check if we have many versions after applyNewerNoncurrentVersionLimit.
-	if len(objInfos) > dataScannerExcessiveVersionsThreshold {
+	if len(objInfos) > int(scannerExcessObjectVersions.Load()) {
 		// Notify object accessed via a GET request.
 		sendEvent(eventArgs{
 			EventName:  event.ObjectManyVersions,
@@ -1112,6 +1111,16 @@ func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fi
 			Host:         globalLocalNodeName,
 			RespElements: map[string]string{"x-minio-versions": strconv.Itoa(len(fivs))},
 		})
+
+		auditLogInternal(context.Background(), AuditLogOptions{
+			Event:   "scanner:manyversions",
+			APIName: ILMExpiry,
+			Bucket:  i.bucket,
+			Object:  i.objectPath(),
+			Tags: map[string]interface{}{
+				"x-minio-versions": strconv.Itoa(len(fivs)),
+			},
+		})
 	}
 
 	return objInfos, nil
@@ -1479,7 +1488,7 @@ func (d *dynamicSleeper) Update(factor float64, maxWait time.Duration) error {
 		return nil
 	}
 	// Update values and cycle waiting.
-	close(d.cycle)
+	xioutil.SafeClose(d.cycle)
 	d.factor = factor
 	d.maxSleep = maxWait
 	d.cycle = make(chan struct{})

cmd/data-usage-utils.go

@@ -71,6 +71,10 @@ type BucketUsageInfo struct {
 
 // DataUsageInfo represents data usage stats of the underlying Object API
 type DataUsageInfo struct {
+	TotalCapacity     uint64 `json:"capacity,omitempty"`
+	TotalUsedCapacity uint64 `json:"usedCapacity,omitempty"`
+	TotalFreeCapacity uint64 `json:"freeCapacity,omitempty"`
+
 	// LastUpdate is the timestamp of when the data usage info was last updated.
 	// This does not indicate a full scan.
 	LastUpdate time.Time `json:"lastUpdate"`
@@ -87,6 +91,7 @@ type DataUsageInfo struct {
 	// Objects total size across all buckets
 	ObjectsTotalSize uint64                           `json:"objectsTotalSize"`
 	ReplicationInfo  map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
+
 	// Total number of buckets in this cluster
 	BucketsCount uint64 `json:"bucketsCount"`
 

cmd/data-usage.go

@@ -155,7 +155,9 @@ func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsag
 			bui.ReplicationFailedSizeV1 > 0 || bui.ReplicationPendingCountV1 > 0 {
 			cfg, _ := getReplicationConfig(GlobalContext, bucket)
 			if cfg != nil && cfg.RoleArn != "" {
-				dataUsageInfo.ReplicationInfo = make(map[string]BucketTargetUsageInfo)
+				if dataUsageInfo.ReplicationInfo == nil {
+					dataUsageInfo.ReplicationInfo = make(map[string]BucketTargetUsageInfo)
+				}
 				dataUsageInfo.ReplicationInfo[cfg.RoleArn] = BucketTargetUsageInfo{
 					ReplicationFailedSize:   bui.ReplicationFailedSizeV1,
 					ReplicationFailedCount:  bui.ReplicationFailedCountV1,

cmd/endpoint.go

@@ -62,8 +62,9 @@ type ProxyEndpoint struct {
 // Node holds information about a node in this cluster
 type Node struct {
 	*url.URL
-	Pools   []int
-	IsLocal bool
+	Pools    []int
+	IsLocal  bool
+	GridHost string
 }
 
 // Endpoint - any type of endpoint.
@@ -254,6 +255,7 @@ func (l EndpointServerPools) GetNodes() (nodes []Node) {
 					Scheme: ep.Scheme,
 					Host:   ep.Host,
 				}
+				node.GridHost = ep.GridHost()
 			}
 			if !slices.Contains(node.Pools, ep.PoolIdx) {
 				node.Pools = append(node.Pools, ep.PoolIdx)
@@ -386,6 +388,95 @@ func (l EndpointServerPools) NEndpoints() (count int) {
 	return count
 }
 
+// GridHosts will return all peers, including local.
+// in websocket grid compatible format, The local peer
+// is returned as a separate string.
+func (l EndpointServerPools) GridHosts() (gridHosts []string, gridLocal string) {
+	seenHosts := set.NewStringSet()
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			u := endpoint.GridHost()
+			if seenHosts.Contains(u) {
+				continue
+			}
+			seenHosts.Add(u)
+
+			// Set local endpoint
+			if endpoint.IsLocal {
+				gridLocal = u
+			}
+
+			gridHosts = append(gridHosts, u)
+		}
+	}
+
+	return gridHosts, gridLocal
+}
+
+// FindGridHostsFromPeerPool will return a matching peerPool from provided peer (as string)
+func (l EndpointServerPools) FindGridHostsFromPeerPool(peer string) []int {
+	if peer == "" {
+		return nil
+	}
+
+	var pools []int
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				continue
+			}
+
+			if !slices.Contains(pools, endpoint.PoolIdx) {
+				pools = append(pools, endpoint.PoolIdx)
+			}
+		}
+	}
+
+	return pools
+}
+
+// FindGridHostsFromPeerStr will return a matching peer from provided peer (as string)
+func (l EndpointServerPools) FindGridHostsFromPeerStr(peer string) (peerGrid string) {
+	if peer == "" {
+		return ""
+	}
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				continue
+			}
+
+			if endpoint.Host == peer {
+				return endpoint.GridHost()
+			}
+		}
+	}
+	return ""
+}
+
+// FindGridHostsFromPeer will return a matching peer from provided peer.
+func (l EndpointServerPools) FindGridHostsFromPeer(peer *xnet.Host) (peerGrid string) {
+	if peer == nil {
+		return ""
+	}
+	for _, ep := range l {
+		for _, endpoint := range ep.Endpoints {
+			if endpoint.IsLocal {
+				continue
+			}
+			host, err := xnet.ParseHost(endpoint.Host)
+			if err != nil {
+				continue
+			}
+
+			if host.String() == peer.String() {
+				return endpoint.GridHost()
+			}
+		}
+	}
+	return ""
+}
+
 // Hostnames - returns list of unique hostnames
 func (l EndpointServerPools) Hostnames() []string {
 	foundSet := set.NewStringSet()
@@ -435,7 +526,9 @@ func (l EndpointServerPools) peers() (peers []string, local string) {
 			peer := endpoint.Host
 			if endpoint.IsLocal {
 				if _, port := mustSplitHostPort(peer); port == globalMinioPort {
-					local = peer
+					if local == "" {
+						local = peer
+					}
 				}
 			}
 
@@ -480,7 +573,7 @@ func hostResolveToLocalhost(endpoint Endpoint) bool {
 			endpoint.Hostname(),
 		)
 		ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-		logger.LogOnceIf(ctx, err, endpoint.Hostname(), logger.Application)
+		logger.LogOnceIf(ctx, err, endpoint.Hostname(), logger.ErrorKind)
 		return false
 	}
 	var loopback int
@@ -552,7 +645,7 @@ func (endpoints Endpoints) UpdateIsLocal() error {
 						ctx := logger.SetReqInfo(GlobalContext,
 							reqInfo)
 						logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
-							endpoints[i].Hostname()), endpoints[i].Hostname(), logger.Application)
+							endpoints[i].Hostname()), endpoints[i].Hostname(), logger.ErrorKind)
 					}
 
 					continue
@@ -581,7 +674,7 @@ func (endpoints Endpoints) UpdateIsLocal() error {
 							))
 						ctx := logger.SetReqInfo(GlobalContext,
 							reqInfo)
-						logger.LogOnceIf(ctx, err, endpoints[i].Hostname(), logger.Application)
+						logger.LogOnceIf(ctx, err, endpoints[i].Hostname(), logger.ErrorKind)
 					}
 				} else {
 					resolvedList[i] = true
@@ -746,7 +839,7 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 							ctx := logger.SetReqInfo(GlobalContext,
 								reqInfo)
 							logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
-								endpoint.Hostname()), endpoint.Hostname(), logger.Application)
+								endpoint.Hostname()), endpoint.Hostname(), logger.ErrorKind)
 						}
 						continue
 					}
@@ -774,7 +867,7 @@ func (p PoolEndpointList) UpdateIsLocal() error {
 								))
 							ctx := logger.SetReqInfo(GlobalContext,
 								reqInfo)
-							logger.LogOnceIf(ctx, err, endpoint.Hostname(), logger.Application)
+							logger.LogOnceIf(ctx, err, endpoint.Hostname(), logger.ErrorKind)
 						}
 					} else {
 						resolvedList[endpoint] = true

cmd/erasure-coding.go

@@ -80,11 +80,9 @@ func (e *Erasure) EncodeData(ctx context.Context, data []byte) ([][]byte, error)
 	}
 	encoded, err := e.encoder().Split(data)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		return nil, err
 	}
 	if err = e.encoder().Encode(encoded); err != nil {
-		logger.LogIf(ctx, err)
 		return nil, err
 	}
 	return encoded, nil
@@ -111,11 +109,7 @@ func (e *Erasure) DecodeDataBlocks(data [][]byte) error {
 // DecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it.
 // It returns an error if the decoding failed.
 func (e *Erasure) DecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error {
-	if err := e.encoder().Reconstruct(data); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-	return nil
+	return e.encoder().Reconstruct(data)
 }
 
 // ShardSize - returns actual shared size from erasure blockSize.

cmd/erasure-common.go

@@ -42,7 +42,7 @@ func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 			if disks[i] == nil {
 				return
 			}
-			di, err := disks[i].DiskInfo(context.Background(), false)
+			di, err := disks[i].DiskInfo(context.Background(), DiskInfoOptions{})
 			if err != nil || di.Healing {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.

cmd/erasure-decode.go

@@ -25,7 +25,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"github.com/minio/minio/internal/logger"
+	xioutil "github.com/minio/minio/internal/ioutil"
 )
 
 // Reads in parallel from readers.
@@ -118,7 +118,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 	}
 
 	readTriggerCh := make(chan bool, len(p.readers))
-	defer close(readTriggerCh) // close the channel upon return
+	defer xioutil.SafeClose(readTriggerCh) // close the channel upon return
 
 	for i := 0; i < p.dataBlocks; i++ {
 		// Setup read triggers for p.dataBlocks number of reads so that it reads in parallel.
@@ -210,11 +210,9 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 // A set of preferred drives can be supplied. In that case they will be used and the data reconstructed.
 func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.ReaderAt, offset, length, totalLength int64, prefer []bool) (written int64, derr error) {
 	if offset < 0 || length < 0 {
-		logger.LogIf(ctx, errInvalidArgument)
 		return -1, errInvalidArgument
 	}
 	if offset+length > totalLength {
-		logger.LogIf(ctx, errInvalidArgument)
 		return -1, errInvalidArgument
 	}
 
@@ -268,7 +266,6 @@ func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.Read
 		}
 
 		if err = e.DecodeDataBlocks(bufs); err != nil {
-			logger.LogIf(ctx, err)
 			return -1, err
 		}
 
@@ -281,7 +278,6 @@ func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.Read
 	}
 
 	if bytesWritten != length {
-		logger.LogIf(ctx, errLessData)
 		return bytesWritten, errLessData
 	}
 
@@ -320,7 +316,6 @@ func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.Rea
 		}
 
 		if err = e.DecodeDataAndParityBlocks(ctx, bufs); err != nil {
-			logger.LogOnceIf(ctx, err, "erasure-heal-decode")
 			return err
 		}
 
@@ -331,7 +326,6 @@ func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.Rea
 		}
 
 		if err = w.Write(ctx, bufs); err != nil {
-			logger.LogOnceIf(ctx, err, "erasure-heal-write")
 			return err
 		}
 	}

cmd/erasure-decode_test.go

@@ -106,7 +106,7 @@ func TestErasureDecode(t *testing.T) {
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
 		writers := make([]io.Writer, len(disks))
 		for i, disk := range disks {
-			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(test.data), writeAlgorithm, erasure.ShardSize())
+			writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(test.data), writeAlgorithm, erasure.ShardSize())
 		}
 		n, err := erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
@@ -228,7 +228,7 @@ func TestErasureDecodeRandomOffsetLength(t *testing.T) {
 		if disk == nil {
 			continue
 		}
-		writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(length), DefaultBitrotAlgorithm, erasure.ShardSize())
+		writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(length), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	// 10000 iterations with random offsets and lengths.
@@ -297,7 +297,7 @@ func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64,
 		if disk == nil {
 			continue
 		}
-		writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
+		writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	content := make([]byte, size)

cmd/erasure-encode.go

@@ -22,9 +22,6 @@ import (
 	"fmt"
 	"io"
 	"sync"
-
-	"github.com/minio/minio/internal/hash"
-	"github.com/minio/minio/internal/logger"
 )
 
 // Writes in parallel to writers
@@ -92,29 +89,26 @@ func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []io.Writer
 				io.EOF,
 				io.ErrUnexpectedEOF,
 			}...) {
-				if !hash.IsChecksumMismatch(err) {
-					logger.LogIf(ctx, err)
-				}
 				return 0, err
 			}
 		}
+
 		eof := err == io.EOF || err == io.ErrUnexpectedEOF
 		if n == 0 && total != 0 {
 			// Reached EOF, nothing more to be done.
 			break
 		}
+
 		// We take care of the situation where if n == 0 and total == 0 by creating empty data and parity files.
 		blocks, err = e.EncodeData(ctx, buf[:n])
 		if err != nil {
-			logger.LogIf(ctx, err)
-
 			return 0, err
 		}
 
 		if err = writer.Write(ctx, blocks); err != nil {
-			logger.LogIf(ctx, err)
 			return 0, err
 		}
+
 		total += int64(n)
 		if eof {
 			break

cmd/erasure-encode_test.go

@@ -41,7 +41,7 @@ func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset
 	return nil, errFaultyDisk
 }
 
-func (a badDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
+func (a badDisk) CreateFile(ctx context.Context, origvolume, volume, path string, size int64, reader io.Reader) error {
 	return errFaultyDisk
 }
 
@@ -103,7 +103,7 @@ func TestErasureEncode(t *testing.T) {
 			if disk == OfflineDisk {
 				continue
 			}
-			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
+			writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
 		}
 		n, err := erasure.Encode(context.Background(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
@@ -127,7 +127,7 @@ func TestErasureEncode(t *testing.T) {
 				if disk == nil {
 					continue
 				}
-				writers[i] = newBitrotWriter(disk, "testbucket", "object2", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
+				writers[i] = newBitrotWriter(disk, "", "testbucket", "object2", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize())
 			}
 			for j := range disks[:test.offDisks] {
 				switch w := writers[j].(type) {
@@ -192,7 +192,7 @@ func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64,
 				Recursive: false,
 				Immediate: false,
 			})
-			writers[i] = newBitrotWriter(disk, "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
+			writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize())
 		}
 		_, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)

cmd/erasure-heal_test.go

@@ -86,7 +86,7 @@ func TestErasureHeal(t *testing.T) {
 		buffer := make([]byte, test.blocksize, 2*test.blocksize)
 		writers := make([]io.Writer, len(disks))
 		for i, disk := range disks {
-			writers[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
+			writers[i] = newBitrotWriter(disk, "", "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
 		}
 		_, err = erasure.Encode(context.Background(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1)
 		closeBitrotWriters(writers)
@@ -128,7 +128,7 @@ func TestErasureHeal(t *testing.T) {
 				continue
 			}
 			os.Remove(pathJoin(disk.String(), "testbucket", "testobject"))
-			staleWriters[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
+			staleWriters[i] = newBitrotWriter(disk, "", "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
 		}
 
 		// test case setup is complete - now call Heal()

cmd/erasure-healing-common.go

@@ -191,19 +191,6 @@ func filterOnlineDisksInplace(fi FileInfo, partsMetadata []FileInfo, onlineDisks
 	}
 }
 
-// Extracts list of disk mtimes from FileInfo slice and returns, skips
-// slice elements that have errors.
-func listObjectDiskMtimes(partsMetadata []FileInfo) (diskMTimes []time.Time) {
-	diskMTimes = bootModtimes(len(partsMetadata))
-	for index, metadata := range partsMetadata {
-		if metadata.IsValid() {
-			// Once the file is found, save the disk mtime saved on disk.
-			diskMTimes[index] = metadata.DiskMTime
-		}
-	}
-	return diskMTimes
-}
-
 // Notes:
 // There are 5 possible states a disk could be in,
 // 1. __online__             - has the latest copy of xl.meta - returned by listOnlineDisks
@@ -277,13 +264,6 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 	errs []error, latestMeta FileInfo, bucket, object string,
 	scanMode madmin.HealScanMode) ([]StorageAPI, []error, time.Time,
 ) {
-	var diskMTime time.Time
-	var shardFix bool
-	if !latestMeta.DataShardFixed() {
-		diskMTime = pickValidDiskTimeWithQuorum(partsMetadata,
-			latestMeta.Erasure.DataBlocks)
-	}
-
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	dataErrs := make([]error, len(onlineDisks))
 
@@ -351,23 +331,6 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			}
 		}
 
-		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
-			if !partsMetadata[i].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
-				// not with in acceptable delta, skip.
-				// If disk mTime mismatches it is considered outdated
-				// https://github.com/minio/minio/pull/13803
-				//
-				// This check only is active if we could find maximally
-				// occurring disk mtimes that are somewhat same across
-				// the quorum. Allowing to skip those shards which we
-				// might think are wrong.
-				shardFix = true
-				partsMetadata[i] = FileInfo{}
-				dataErrs[i] = errFileCorrupt
-				continue
-			}
-		}
-
 		// Always check data, if we got it.
 		if (len(meta.Data) > 0 || meta.Size == 0) && len(meta.Parts) > 0 {
 			checksumInfo := meta.Erasure.GetChecksumInfo(meta.Parts[0].Number)
@@ -410,10 +373,5 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		}
 	}
 
-	if shardFix {
-		// Only when shard is fixed return an appropriate disk mtime value.
-		return availableDisks, dataErrs, diskMTime
-	} // else return timeSentinel for disk mtime
-
 	return availableDisks, dataErrs, timeSentinel
 }

cmd/erasure-healing-common_test.go

@@ -246,7 +246,7 @@ func TestListOnlineDisks(t *testing.T) {
 				t.Fatalf("Failed to putObject %v", err)
 			}
 
-			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 			fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
@@ -424,7 +424,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 				t.Fatalf("Failed to putObject %v", err)
 			}
 
-			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", true, true)
+			partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", true, true)
 			fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs)
 			if err != nil {
 				t.Fatalf("Failed to getLatestFileInfo %v", err)
@@ -437,7 +437,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 				partsMetadata[j].ModTime = test.modTimes[j]
 			}
 
-			if erasureDisks, err = writeUniqueFileInfo(ctx, erasureDisks, bucket, object, partsMetadata, diskCount(erasureDisks)); err != nil {
+			if erasureDisks, err = writeUniqueFileInfo(ctx, erasureDisks, "", bucket, object, partsMetadata, diskCount(erasureDisks)); err != nil {
 				t.Fatal(ctx, err)
 			}
 
@@ -534,7 +534,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	_, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	_, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	readQuorum := len(erasureDisks) / 2
 	if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
 		t.Fatalf("Failed to read xl meta data %v", reducedErr)
@@ -542,7 +542,7 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	// Test 1: Test that all disks are returned without any failures with
 	// unmodified meta data
-	partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	if err != nil {
 		t.Fatalf("Failed to read xl meta data %v", err)
 	}

cmd/erasure-healing.go

@@ -33,8 +33,6 @@ import (
 	"github.com/minio/pkg/v2/sync/errgroup"
 )
 
-const reservedMetadataPrefixLowerDataShardFix = ReservedMetadataPrefixLower + "data-shard-fix"
-
 //go:generate stringer -type=healingMetric -trimprefix=healingMetric $GOFILE
 
 type healingMetric uint8
@@ -45,29 +43,6 @@ const (
 	healingMetricCheckAbandonedParts
 )
 
-// AcceptableDelta returns 'true' if the fi.DiskMTime is under
-// acceptable delta of "delta" duration with maxTime.
-//
-// This code is primarily used for heuristic detection of
-// incorrect shards, as per https://github.com/minio/minio/pull/13803
-//
-// This check only is active if we could find maximally
-// occurring disk mtimes that are somewhat same across
-// the quorum. Allowing to skip those shards which we
-// might think are wrong.
-func (fi FileInfo) AcceptableDelta(maxTime time.Time, delta time.Duration) bool {
-	diff := maxTime.Sub(fi.DiskMTime)
-	if diff < 0 {
-		diff = -diff
-	}
-	return diff < delta
-}
-
-// DataShardFixed - data shard fixed?
-func (fi FileInfo) DataShardFixed() bool {
-	return fi.Metadata[reservedMetadataPrefixLowerDataShardFix] == "true"
-}
-
 func (er erasureObjects) listAndHeal(bucket, prefix string, scanMode madmin.HealScanMode, healEntry func(string, metaCacheEntry, madmin.HealScanMode) error) error {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -203,7 +178,10 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 	return false
 }
 
-const xMinIOHealing = ReservedMetadataPrefix + "healing"
+const (
+	xMinIOHealing = ReservedMetadataPrefix + "healing"
+	xMinIODataMov = ReservedMetadataPrefix + "data-mov"
+)
 
 // SetHealing marks object (version) as being healed.
 // Note: this is to be used only from healObject
@@ -221,6 +199,21 @@ func (fi FileInfo) Healing() bool {
 	return ok
 }
 
+// SetDataMov marks object (version) as being currently
+// in movement, such as decommissioning or rebalance.
+func (fi *FileInfo) SetDataMov() {
+	if fi.Metadata == nil {
+		fi.Metadata = make(map[string]string)
+	}
+	fi.Metadata[xMinIODataMov] = "true"
+}
+
+// DataMov returns true if object is being in movement
+func (fi FileInfo) DataMov() bool {
+	_, ok := fi.Metadata[xMinIODataMov]
+	return ok
+}
+
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func (er *erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) {
 	dryRun := opts.DryRun
@@ -255,7 +248,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	}
 
 	// Re-read when we have lock...
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, object, versionID, true, true)
+	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, "", bucket, object, versionID, true, true)
 	if isAllNotFound(errs) {
 		err := errFileNotFound
 		if versionID != "" {
@@ -311,7 +304,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	// used here for reconstruction. This is done to ensure that
 	// we do not skip drives that have inconsistent metadata to be
 	// skipped from purging when they are stale.
-	availableDisks, dataErrs, diskMTime := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+	availableDisks, dataErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
 		errs, latestMeta, bucket, object, scanMode)
 
 	var erasure Erasure
@@ -543,7 +536,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 					inlineBuffers[i] = bytes.NewBuffer(make([]byte, 0, erasure.ShardFileSize(latestMeta.Size)+32))
 					writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize())
 				} else {
-					writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, partPath,
+					writers[i] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, partPath,
 						tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 				}
 			}
@@ -627,20 +620,6 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 		}
 	}
 
-	if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
-		// Update metadata to indicate special fix.
-		_, err = er.PutObjectMetadata(ctx, bucket, object, ObjectOptions{
-			NoLock: true,
-			UserDefined: map[string]string{
-				reservedMetadataPrefixLowerDataShardFix: "true",
-				// another reserved metadata to capture original disk-mtime
-				// captured for this version of the object, to be used
-				// possibly in future to heal other versions if possible.
-				ReservedMetadataPrefixLower + "disk-mtime": diskMTime.String(),
-			},
-		})
-	}
-
 	return result, nil
 }
 
@@ -835,7 +814,7 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
 		}
 		index := index
 		g.Go(func() error {
-			entries, err := storageDisks[index].ListDir(ctx, bucket, prefix, 1)
+			entries, err := storageDisks[index].ListDir(ctx, "", bucket, prefix, 1)
 			if err != nil {
 				return err
 			}
@@ -849,6 +828,10 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
 	return g.Wait()
 }
 
+func isAllVolumeNotFound(errs []error) bool {
+	return countErrs(errs, errVolumeNotFound) == len(errs)
+}
+
 // isAllNotFound will return if any element of the error slice is not
 // errFileNotFound, errFileVersionNotFound or errVolumeNotFound.
 // A 0 length slice will always return false.
@@ -878,8 +861,12 @@ func isAllBucketsNotFound(errs []error) bool {
 	}
 	notFoundCount := 0
 	for _, err := range errs {
-		if err != nil && errors.Is(err, errVolumeNotFound) {
-			notFoundCount++
+		if err != nil {
+			if errors.Is(err, errVolumeNotFound) {
+				notFoundCount++
+			} else if isErrBucketNotFound(err) {
+				notFoundCount++
+			}
 		}
 	}
 	return len(errs) == notFoundCount
@@ -917,38 +904,28 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	// We can consider an object data not reliable
 	// when xl.meta is not found in read quorum disks.
 	// or when xl.meta is not readable in read quorum disks.
-	danglingErrsCount := func(cerrs []error) (int, int, int) {
+	danglingErrsCount := func(cerrs []error) (int, int) {
 		var (
-			notFoundCount     int
-			corruptedCount    int
-			diskNotFoundCount int
+			notFoundCount      int
+			nonActionableCount int
 		)
 		for _, readErr := range cerrs {
+			if readErr == nil {
+				continue
+			}
 			switch {
 			case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound):
 				notFoundCount++
-			case errors.Is(readErr, errFileCorrupt):
-				corruptedCount++
-			case errors.Is(readErr, errDiskNotFound):
-				diskNotFoundCount++
+			default:
+				// All other errors are non-actionable
+				nonActionableCount++
 			}
 		}
-		return notFoundCount, corruptedCount, diskNotFoundCount
+		return notFoundCount, nonActionableCount
 	}
 
-	ndataErrs := make([]error, len(dataErrs))
-	for i := range dataErrs {
-		if errs[i] != dataErrs[i] {
-			// Only count part errors, if the error is not
-			// same as xl.meta error. This is to avoid
-			// double counting when both parts and xl.meta
-			// are not available.
-			ndataErrs[i] = dataErrs[i]
-		}
-	}
-
-	notFoundMetaErrs, corruptedMetaErrs, driveNotFoundMetaErrs := danglingErrsCount(errs)
-	notFoundPartsErrs, corruptedPartsErrs, driveNotFoundPartsErrs := danglingErrsCount(ndataErrs)
+	notFoundMetaErrs, nonActionableMetaErrs := danglingErrsCount(errs)
+	notFoundPartsErrs, nonActionablePartsErrs := danglingErrsCount(dataErrs)
 
 	for _, m := range metaArr {
 		if m.IsValid() {
@@ -958,10 +935,9 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	}
 
 	if !validMeta.IsValid() {
-		// validMeta is invalid because notFoundPartsErrs is
-		// greater than parity blocks, thus invalidating the FileInfo{}
-		// every dataErrs[i], metaArr[i] is an empty FileInfo{}
-		dataBlocks := (len(ndataErrs) + 1) / 2
+		// validMeta is invalid because all xl.meta is missing apparently
+		// we should figure out if dataDirs are also missing > dataBlocks.
+		dataBlocks := (len(dataErrs) + 1) / 2
 		if notFoundPartsErrs > dataBlocks {
 			// Not using parity to ensure that we do not delete
 			// any valid content, if any is recoverable. But if
@@ -978,25 +954,34 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 		return validMeta, false
 	}
 
-	if driveNotFoundMetaErrs > 0 || driveNotFoundPartsErrs > 0 {
+	if nonActionableMetaErrs > 0 || nonActionablePartsErrs > 0 {
 		return validMeta, false
 	}
 
 	if validMeta.Deleted {
 		// notFoundPartsErrs is ignored since
 		// - delete marker does not have any parts
-		return validMeta, corruptedMetaErrs+notFoundMetaErrs > len(errs)/2
+		dataBlocks := (len(errs) + 1) / 2
+		return validMeta, notFoundMetaErrs > dataBlocks
 	}
 
-	totalErrs := notFoundMetaErrs + corruptedMetaErrs + notFoundPartsErrs + corruptedPartsErrs
-	if validMeta.IsRemote() {
-		// notFoundPartsErrs is ignored since
-		// - transition status of complete has no parts
-		totalErrs = notFoundMetaErrs + corruptedMetaErrs
+	// TODO: It is possible to replay the object via just single
+	// xl.meta file, considering quorum number of data-dirs are still
+	// present on other drives.
+	//
+	// However this requires a bit of a rewrite, leave this up for
+	// future work.
+	if notFoundMetaErrs > 0 && notFoundMetaErrs > validMeta.Erasure.ParityBlocks {
+		// All xl.meta is beyond data blocks missing, this is dangling
+		return validMeta, true
 	}
 
-	// We have valid meta, now verify if we have enough files with parity blocks.
-	return validMeta, totalErrs > validMeta.Erasure.ParityBlocks
+	if !validMeta.IsRemote() && notFoundPartsErrs > 0 && notFoundPartsErrs > validMeta.Erasure.ParityBlocks {
+		// All data-dir is beyond data blocks missing, this is dangling
+		return validMeta, true
+	}
+
+	return validMeta, false
 }
 
 // HealObject - heal the given object, automatically deletes the object if stale/corrupted if `remove` is true.
@@ -1028,7 +1013,7 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 
 	// Perform quick read without lock.
 	// This allows to quickly check if all is ok or all are missing.
-	_, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false, false)
+	_, errs := readAllFileInfo(healCtx, storageDisks, "", bucket, object, versionID, false, false)
 	if isAllNotFound(errs) {
 		err := errFileNotFound
 		if versionID != "" {

cmd/erasure-healing_test.go

@@ -41,6 +41,10 @@ func TestIsObjectDangling(t *testing.T) {
 	fi := newFileInfo("test-object", 2, 2)
 	fi.Erasure.Index = 1
 
+	ifi := newFileInfo("test-object", 2, 2)
+	ifi.SetInlineData()
+	ifi.Erasure.Index = 1
+
 	testCases := []struct {
 		name             string
 		metaArr          []FileInfo
@@ -129,13 +133,77 @@ func TestIsObjectDangling(t *testing.T) {
 			expectedMeta:     FileInfo{},
 			expectedDangling: false,
 		},
+		{
+			name: "FileInfoUnDecided-case4",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				ifi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				errFileCorrupt,
+				nil,
+			},
+			dataErrs:         nil,
+			expectedMeta:     ifi,
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoUnDecided-case5-(ignore errFileCorrupt error)",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				nil,
+				nil,
+			},
+			dataErrs: []error{
+				errFileCorrupt,
+				errFileNotFound,
+				nil,
+				errFileCorrupt,
+			},
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
+		{
+			name: "FileInfoUnDecided-case6-(data-dir intact)",
+			metaArr: []FileInfo{
+				{},
+				{},
+				{},
+				fi,
+			},
+			errs: []error{
+				errFileNotFound,
+				errFileNotFound,
+				errFileNotFound,
+				nil,
+			},
+			dataErrs: []error{
+				errFileNotFound,
+				errFileCorrupt,
+				nil,
+				nil,
+			},
+			expectedMeta:     fi,
+			expectedDangling: false,
+		},
 		{
 			name: "FileInfoDecided-case1",
 			metaArr: []FileInfo{
 				{},
 				{},
 				{},
-				fi,
+				ifi,
 			},
 			errs: []error{
 				errFileNotFound,
@@ -144,25 +212,7 @@ func TestIsObjectDangling(t *testing.T) {
 				nil,
 			},
 			dataErrs:         nil,
-			expectedMeta:     fi,
-			expectedDangling: true,
-		},
-		{
-			name: "FileInfoDecided-case2",
-			metaArr: []FileInfo{
-				{},
-				{},
-				{},
-				fi,
-			},
-			errs: []error{
-				errFileNotFound,
-				errFileCorrupt,
-				errFileCorrupt,
-				nil,
-			},
-			dataErrs:         nil,
-			expectedMeta:     fi,
+			expectedMeta:     ifi,
 			expectedDangling: true,
 		},
 		{
@@ -175,8 +225,8 @@ func TestIsObjectDangling(t *testing.T) {
 			},
 			errs: []error{
 				errFileNotFound,
-				errFileCorrupt,
-				errFileCorrupt,
+				errFileNotFound,
+				errFileNotFound,
 				nil,
 			},
 			dataErrs:         nil,
@@ -184,26 +234,26 @@ func TestIsObjectDangling(t *testing.T) {
 			expectedDangling: true,
 		},
 		{
-			name: "FileInfoDecided-case2-(duplicate data errors)",
+			name: "FileInfoDecided-case3-(enough data-dir missing)",
 			metaArr: []FileInfo{
 				{},
 				{},
 				{},
-				{Deleted: true},
+				fi,
 			},
 			errs: []error{
 				errFileNotFound,
-				errFileCorrupt,
-				errFileCorrupt,
+				errFileNotFound,
+				nil,
 				nil,
 			},
 			dataErrs: []error{
 				errFileNotFound,
-				errFileCorrupt,
-				nil,
+				errFileNotFound,
 				nil,
+				errFileNotFound,
 			},
-			expectedMeta:     FileInfo{Deleted: true},
+			expectedMeta:     fi,
 			expectedDangling: true,
 		},
 		// Add new cases as seen
@@ -266,7 +316,7 @@ func TestHealing(t *testing.T) {
 	}
 
 	disk := er.getDisks()[0]
-	fileInfoPreHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal, err := disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -289,7 +339,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err := disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal, err := disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -308,7 +358,7 @@ func TestHealing(t *testing.T) {
 	// gone down when an object was replaced by a new object.
 	fileInfoOutDated := fileInfoPreHeal
 	fileInfoOutDated.ModTime = time.Now()
-	err = disk.WriteMetadata(context.Background(), bucket, object, fileInfoOutDated)
+	err = disk.WriteMetadata(context.Background(), "", bucket, object, fileInfoOutDated)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -318,7 +368,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -426,11 +476,11 @@ func TestHealingVersioned(t *testing.T) {
 	}
 
 	disk := er.getDisks()[0]
-	fileInfoPreHeal1, err := disk.ReadVersion(context.Background(), bucket, object, oi1.VersionID, ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal1, err := disk.ReadVersion(context.Background(), "", bucket, object, oi1.VersionID, ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
-	fileInfoPreHeal2, err := disk.ReadVersion(context.Background(), bucket, object, oi2.VersionID, ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal2, err := disk.ReadVersion(context.Background(), "", bucket, object, oi2.VersionID, ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -453,11 +503,11 @@ func TestHealingVersioned(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal1, err := disk.ReadVersion(context.Background(), bucket, object, oi1.VersionID, ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal1, err := disk.ReadVersion(context.Background(), "", bucket, object, oi1.VersionID, ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
-	fileInfoPostHeal2, err := disk.ReadVersion(context.Background(), bucket, object, oi2.VersionID, ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal2, err := disk.ReadVersion(context.Background(), "", bucket, object, oi2.VersionID, ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -479,7 +529,7 @@ func TestHealingVersioned(t *testing.T) {
 	// gone down when an object was replaced by a new object.
 	fileInfoOutDated := fileInfoPreHeal1
 	fileInfoOutDated.ModTime = time.Now()
-	err = disk.WriteMetadata(context.Background(), bucket, object, fileInfoOutDated)
+	err = disk.WriteMetadata(context.Background(), "", bucket, object, fileInfoOutDated)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -489,7 +539,7 @@ func TestHealingVersioned(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal1, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal1, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -499,7 +549,7 @@ func TestHealingVersioned(t *testing.T) {
 		t.Fatal("HealObject failed")
 	}
 
-	fileInfoPostHeal2, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal2, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -637,7 +687,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	// Restore...
 	setDisks(orgDisks[:4]...)
 
-	fileInfoPreHeal, err := disks[0].ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal, err := disks[0].ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -654,7 +704,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	fileInfoPostHeal, err := disks[0].ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal, err := disks[0].ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -684,7 +734,7 @@ func TestHealingDanglingObject(t *testing.T) {
 
 	setDisks(orgDisks[:4]...)
 	disk := getDisk(0)
-	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -702,7 +752,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	disk = getDisk(0)
-	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -733,7 +783,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	setDisks(orgDisks[:4]...)
 
 	disk = getDisk(0)
-	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPreHeal, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -751,7 +801,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 
 	disk = getDisk(0)
-	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), bucket, object, "", ReadOptions{ReadData: false, Healing: true})
+	fileInfoPostHeal, err = disk.ReadVersion(context.Background(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true})
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -836,7 +886,7 @@ func TestHealCorrectQuorum(t *testing.T) {
 		er := set.sets[0]
 		erasureDisks := er.getDisks()
 
-		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 		nfi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 		if errors.Is(err, errFileNotFound) {
 			continue
@@ -858,12 +908,12 @@ func TestHealCorrectQuorum(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 		if countErrs(errs, nil) != len(fileInfos) {
 			t.Fatal("Expected all xl.meta healed, but partial heal detected")
 		}
 
-		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, minioMetaBucket, cfgFile, "", false, true)
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", minioMetaBucket, cfgFile, "", false, true)
 		nfi, err = getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 		if errors.Is(err, errFileNotFound) {
 			continue
@@ -885,7 +935,7 @@ func TestHealCorrectQuorum(t *testing.T) {
 			t.Fatal(err)
 		}
 
-		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, minioMetaBucket, cfgFile, "", false, true)
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", minioMetaBucket, cfgFile, "", false, true)
 		if countErrs(errs, nil) != len(fileInfos) {
 			t.Fatal("Expected all xl.meta healed, but partial heal detected")
 		}
@@ -970,7 +1020,7 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
 
-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
@@ -998,14 +1048,12 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		t.Errorf("Expected nil but received %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	nfi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	fi.DiskMTime = time.Time{}
-	nfi.DiskMTime = time.Time{}
 	if !reflect.DeepEqual(fi, nfi) {
 		t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
 	}
@@ -1029,14 +1077,12 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 		t.Errorf("Expected nil but received %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	nfi, err = getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	fi.DiskMTime = time.Time{}
-	nfi.DiskMTime = time.Time{}
 	if !reflect.DeepEqual(fi, nfi) {
 		t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi)
 	}
@@ -1133,7 +1179,7 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 	firstDisk := erasureDisks[0]
 
 	// Test 1: Remove the object backend files from the first disk.
-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
@@ -1156,14 +1202,12 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	nfi1, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	fi.DiskMTime = time.Time{}
-	nfi1.DiskMTime = time.Time{}
 	if !reflect.DeepEqual(fi, nfi1) {
 		t.Fatalf("FileInfo not equal after healing")
 	}
@@ -1179,14 +1223,12 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 		t.Errorf("Expected nil but received %v", err)
 	}
 
-	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	nfi2, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
 	}
 
-	fi.DiskMTime = time.Time{}
-	nfi2.DiskMTime = time.Time{}
 	if !reflect.DeepEqual(fi, nfi2) {
 		t.Fatalf("FileInfo not equal after healing")
 	}
@@ -1277,7 +1319,7 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 	firstDisk := erasureDisks[0]
 	secondDisk := erasureDisks[1]
 
-	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false, true)
+	fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true)
 	fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs)
 	if err != nil {
 		t.Fatalf("Failed to getLatestFileInfo - %v", err)
@@ -1603,7 +1645,6 @@ func TestHealLastDataShard(t *testing.T) {
 
 			_, err = obj.PutObject(ctx, bucket, object,
 				mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
-
 			if err != nil {
 				t.Fatal(err)
 			}

cmd/erasure-metadata-utils.go

@@ -20,9 +20,7 @@ package cmd
 import (
 	"context"
 	"errors"
-	"fmt"
 	"hash/crc32"
-	"io"
 
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
@@ -148,29 +146,9 @@ func hashOrder(key string, cardinality int) []int {
 	return nums
 }
 
-var readFileInfoIgnoredErrs = append(objectOpIgnoredErrs,
-	errFileNotFound,
-	errVolumeNotFound,
-	errFileVersionNotFound,
-	io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
-	io.EOF,              // some times we would read without locks, ignore these errors
-)
-
-func readFileInfo(ctx context.Context, disk StorageAPI, bucket, object, versionID string, opts ReadOptions) (FileInfo, error) {
-	fi, err := disk.ReadVersion(ctx, bucket, object, versionID, opts)
-
-	if err != nil && !IsErr(err, readFileInfoIgnoredErrs...) {
-		logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
-			disk.String(), bucket, object, err),
-			disk.String())
-	}
-
-	return fi, err
-}
-
 // Reads all `xl.meta` metadata as a FileInfo slice.
 // Returns error slice indicating the failed metadata reads.
-func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, versionID string, readData, healing bool) ([]FileInfo, []error) {
+func readAllFileInfo(ctx context.Context, disks []StorageAPI, origbucket string, bucket, object, versionID string, readData, healing bool) ([]FileInfo, []error) {
 	metadataArray := make([]FileInfo, len(disks))
 
 	opts := ReadOptions{
@@ -186,7 +164,7 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			metadataArray[index], err = readFileInfo(ctx, disks[index], bucket, object, versionID, opts)
+			metadataArray[index], err = disks[index].ReadVersion(ctx, origbucket, bucket, object, versionID, opts)
 			return err
 		}, index)
 	}
@@ -330,15 +308,12 @@ var (
 // returns error if totalSize is -1, partSize is 0, partIndex is 0.
 func calculatePartSizeFromIdx(ctx context.Context, totalSize int64, partSize int64, partIndex int) (currPartSize int64, err error) {
 	if totalSize < -1 {
-		logger.LogIf(ctx, errInvalidArgument)
 		return 0, errInvalidArgument
 	}
 	if partSize == 0 {
-		logger.LogIf(ctx, errPartSizeZero)
 		return 0, errPartSizeZero
 	}
 	if partIndex < 1 {
-		logger.LogIf(ctx, errPartSizeIndex)
 		return 0, errPartSizeIndex
 	}
 	if totalSize == -1 {

cmd/erasure-metadata.go

@@ -374,17 +374,6 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 	return FileInfo{}, errErasureReadQuorum
 }
 
-func pickValidDiskTimeWithQuorum(metaArr []FileInfo, quorum int) time.Time {
-	diskMTimes := listObjectDiskMtimes(metaArr)
-
-	diskMTime, diskMaxima := commonTimeAndOccurence(diskMTimes, shardDiskTimeDelta)
-	if diskMaxima >= quorum {
-		return diskMTime
-	}
-
-	return timeSentinel
-}
-
 // pickValidFileInfo - picks one valid FileInfo content and returns from a
 // slice of FileInfo.
 func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Time, etag string, quorum int) (FileInfo, error) {
@@ -392,7 +381,7 @@ func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Tim
 }
 
 // writeUniqueFileInfo - writes unique `xl.meta` content for each disk concurrently.
-func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
+func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) {
 	g := errgroup.WithNErrs(len(disks))
 
 	// Start writing `xl.meta` to all disks in parallel.
@@ -406,7 +395,7 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
 			fi := files[index]
 			fi.Erasure.Index = index + 1
 			if fi.IsValid() {
-				return disks[index].WriteMetadata(ctx, bucket, prefix, fi)
+				return disks[index].WriteMetadata(ctx, origbucket, bucket, prefix, fi)
 			}
 			return errCorruptedFormat
 		}, index)

cmd/erasure-multipart.go

@@ -61,7 +61,7 @@ func (er erasureObjects) getMultipartSHADir(bucket, object string) string {
 // checkUploadIDExists - verify if a given uploadID exists and is valid.
 func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string, write bool) (fi FileInfo, metArr []FileInfo, err error) {
 	defer func() {
-		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
+		if errors.Is(err, errFileNotFound) {
 			err = errUploadIDNotFound
 		}
 	}()
@@ -71,7 +71,7 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	storageDisks := er.getDisks()
 
 	// Read metadata associated with the object from all disks.
-	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, minioMetaMultipartBucket,
+	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, minioMetaMultipartBucket,
 		uploadIDPath, "", false, false)
 
 	readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
@@ -87,15 +87,14 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 	// List all online disks.
 	_, modTime, etag := listOnlineDisks(storageDisks, partsMetadata, errs, quorum)
 
+	var reducedErr error
 	if write {
-		reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-		if errors.Is(reducedErr, errErasureWriteQuorum) {
-			return fi, nil, reducedErr
-		}
+		reducedErr = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
 	} else {
-		if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
-			return fi, nil, reducedErr
-		}
+		reducedErr = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum)
+	}
+	if reducedErr != nil {
+		return fi, nil, reducedErr
 	}
 
 	// Pick one from the first valid metadata.
@@ -200,7 +199,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 	readDirFn(pathJoin(diskPath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
 		readDirFn(pathJoin(diskPath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error {
 			uploadIDPath := pathJoin(shaDir, uploadIDDir)
-			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{})
+			fi, err := disk.ReadVersion(ctx, "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{})
 			if err != nil {
 				return nil
 			}
@@ -281,15 +280,14 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 		if !disk.IsOnline() {
 			continue
 		}
-		uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
+		uploadIDs, err = disk.ListDir(ctx, bucket, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
 		if err != nil {
 			if errors.Is(err, errDiskNotFound) {
 				continue
 			}
-			if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
+			if errors.Is(err, errFileNotFound) {
 				return result, nil
 			}
-			logger.LogIf(ctx, err)
 			return result, toObjectErr(err, bucket, object)
 		}
 		break
@@ -303,10 +301,10 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 	// to read the metadata entry.
 	var uploads []MultipartInfo
 
-	populatedUploadIds := set.NewStringSet()
+	populatedUploadIDs := set.NewStringSet()
 
 	for _, uploadID := range uploadIDs {
-		if populatedUploadIds.Contains(uploadID) {
+		if populatedUploadIDs.Contains(uploadID) {
 			continue
 		}
 		// If present, use time stored in ID.
@@ -323,7 +321,7 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 			UploadID:  base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID(), uploadID))),
 			Initiated: startTime,
 		})
-		populatedUploadIds.Add(uploadID)
+		populatedUploadIDs.Add(uploadID)
 	}
 
 	sort.Slice(uploads, func(i int, j int) bool {
@@ -486,8 +484,8 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadUUID)
 
 	// Write updated `xl.meta` to all disks.
-	if _, err := writeUniqueFileInfo(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
-		return nil, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
+	if _, err := writeUniqueFileInfo(ctx, onlineDisks, bucket, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil {
+		return nil, toObjectErr(err, bucket, object)
 	}
 	return &NewMultipartUploadResult{
 		UploadID:     uploadID,
@@ -582,6 +580,13 @@ func writeAllDisks(ctx context.Context, disks []StorageAPI, dstBucket, dstEntry
 func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) {
 	auditObjectErasureSet(ctx, object, &er)
 
+	data := r.Reader
+	// Validate input data size and it can never be less than zero.
+	if data.Size() < -1 {
+		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
+		return pi, toObjectErr(errInvalidArgument)
+	}
+
 	// Read lock for upload id.
 	// Only held while reading the upload metadata.
 	uploadIDRLock := er.NewNSLock(bucket, pathJoin(object, uploadID))
@@ -596,16 +601,12 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	// Validates if upload ID exists.
 	fi, _, err := er.checkUploadIDExists(rctx, bucket, object, uploadID, true)
 	if err != nil {
+		if errors.Is(err, errVolumeNotFound) {
+			return pi, toObjectErr(err, bucket)
+		}
 		return pi, toObjectErr(err, bucket, object, uploadID)
 	}
 
-	data := r.Reader
-	// Validate input data size and it can never be less than zero.
-	if data.Size() < -1 {
-		logger.LogIf(rctx, errInvalidArgument, logger.Application)
-		return pi, toObjectErr(errInvalidArgument)
-	}
-
 	// Write lock for this part ID, only hold it if we are planning to read from the
 	// streamto avoid any concurrent updates.
 	//
@@ -681,7 +682,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		if disk == nil {
 			continue
 		}
-		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tmpPartPath, erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize())
+		writers[i] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, tmpPartPath, erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	toEncode := io.Reader(data)
@@ -794,6 +795,9 @@ func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, u
 
 	fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false)
 	if err != nil {
+		if errors.Is(err, errVolumeNotFound) {
+			return result, toObjectErr(err, bucket)
+		}
 		return result, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -979,6 +983,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 
 	fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true)
 	if err != nil {
+		if errors.Is(err, errVolumeNotFound) {
+			return oi, toObjectErr(err, bucket)
+		}
 		return oi, toObjectErr(err, bucket, object, uploadID)
 	}
 
@@ -1237,6 +1244,10 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	// Save the consolidated actual size.
 	fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)
 
+	if opts.DataMovement {
+		fi.SetDataMov()
+	}
+
 	// Update all erasure metadata, make sure to not modify fields like
 	// checksum which are different on each disks.
 	for index := range partsMetadata {
@@ -1337,6 +1348,9 @@ func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, objec
 
 	// Validates if upload ID exists.
 	if _, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false); err != nil {
+		if errors.Is(err, errVolumeNotFound) {
+			return toObjectErr(err, bucket)
+		}
 		return toObjectErr(err, bucket, object, uploadID)
 	}
 

cmd/erasure-object.go

@@ -46,7 +46,6 @@ import (
 	"github.com/minio/pkg/v2/mimedb"
 	"github.com/minio/pkg/v2/sync/errgroup"
 	"github.com/minio/pkg/v2/wildcard"
-	"github.com/tinylib/msgp/msgp"
 )
 
 // list all errors which can be ignored in object operations.
@@ -91,7 +90,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 
 	// Read metadata associated with the object from all disks.
 	if srcOpts.VersionID != "" {
-		metaArr, errs = readAllFileInfo(ctx, storageDisks, srcBucket, srcObject, srcOpts.VersionID, true, false)
+		metaArr, errs = readAllFileInfo(ctx, storageDisks, "", srcBucket, srcObject, srcOpts.VersionID, true, false)
 	} else {
 		metaArr, errs = readAllXL(ctx, storageDisks, srcBucket, srcObject, true, false, true)
 	}
@@ -179,7 +178,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	}
 
 	// Write unique `xl.meta` for each disk.
-	if _, err = writeUniqueFileInfo(ctx, onlineDisks, srcBucket, srcObject, metaArr, writeQuorum); err != nil {
+	if _, err = writeUniqueFileInfo(ctx, onlineDisks, "", srcBucket, srcObject, metaArr, writeQuorum); err != nil {
 		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
 
@@ -227,30 +226,6 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		return nil, toObjectErr(err, bucket, object)
 	}
 
-	if !fi.DataShardFixed() {
-		diskMTime := pickValidDiskTimeWithQuorum(metaArr, fi.Erasure.DataBlocks)
-		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
-			for index := range onlineDisks {
-				if onlineDisks[index] == OfflineDisk {
-					continue
-				}
-				if !metaArr[index].IsValid() {
-					continue
-				}
-				if !metaArr[index].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
-					// If disk mTime mismatches it is considered outdated
-					// https://github.com/minio/minio/pull/13803
-					//
-					// This check only is active if we could find maximally
-					// occurring disk mtimes that are somewhat same across
-					// the quorum. Allowing to skip those shards which we
-					// might think are wrong.
-					onlineDisks[index] = OfflineDisk
-				}
-			}
-		}
-	}
-
 	objInfo := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
 	if objInfo.DeleteMarker {
 		if opts.VersionID == "" {
@@ -511,7 +486,7 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		// count the number of offline disks
 		offline := 0
 		for i := 0; i < max(len(errs), len(dataErrs)); i++ {
-			if i < len(errs) && errs[i] == errDiskNotFound || i < len(dataErrs) && dataErrs[i] == errDiskNotFound {
+			if i < len(errs) && errors.Is(errs[i], errDiskNotFound) || i < len(dataErrs) && errors.Is(dataErrs[i], errDiskNotFound) {
 				offline++
 			}
 		}
@@ -570,29 +545,6 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 	return m, err
 }
 
-var readRawFileInfoErrs = append(objectOpIgnoredErrs,
-	errFileNotFound,
-	errFileNameTooLong,
-	errVolumeNotFound,
-	errFileVersionNotFound,
-	io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
-	io.EOF,              // some times we would read without locks, ignore these errors
-	msgp.ErrShortBytes,
-	context.DeadlineExceeded,
-	context.Canceled,
-)
-
-func readRawFileInfo(ctx context.Context, disk StorageAPI, bucket, object string, readData bool) (RawFileInfo, error) {
-	rf, err := disk.ReadXL(ctx, bucket, object, readData)
-
-	if err != nil && !IsErr(err, readRawFileInfoErrs...) {
-		logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
-			disk.String(), bucket, object, err),
-			disk.String())
-	}
-	return rf, err
-}
-
 func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVers, allParts bool) (FileInfo, error) {
 	var xl xlMetaV2
 	if err := xl.LoadOrConvert(ri.Buf); err != nil {
@@ -622,7 +574,6 @@ func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVe
 		fileInfo.Data = xl.data.find(versionID)
 	}
 
-	fileInfo.DiskMTime = ri.DiskMTime
 	return fileInfo, nil
 }
 
@@ -635,7 +586,7 @@ func readAllRawFileInfo(ctx context.Context, disks []StorageAPI, bucket, object
 			if disks[index] == nil {
 				return errDiskNotFound
 			}
-			rf, err := readRawFileInfo(ctx, disks[index], bucket, object, readData)
+			rf, err := disks[index].ReadXL(ctx, bucket, object, readData)
 			if err != nil {
 				return err
 			}
@@ -673,9 +624,7 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 			continue
 		}
 		metadataArray[index] = &xl
-		metaFileInfos[index] = FileInfo{
-			DiskMTime: rf.DiskMTime,
-		}
+		metaFileInfos[index] = FileInfo{}
 	}
 
 	for index := range metadataArray {
@@ -715,7 +664,6 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 		}
 
 		// make sure to preserve this for diskmtime based healing bugfix.
-		diskMTime := metaFileInfos[index].DiskMTime
 		metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID, inclFreeVers, allParts)
 		if errs[index] != nil {
 			continue
@@ -724,7 +672,6 @@ func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo,
 		if readData {
 			metaFileInfos[index].Data = metadataArray[index].data.find(versionID)
 		}
-		metaFileInfos[index].DiskMTime = diskMTime
 	}
 	if !readData {
 		for i := range v2bufs {
@@ -768,8 +715,6 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 }
 
 func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (FileInfo, []FileInfo, []StorageAPI, error) {
-	var mu sync.Mutex
-
 	rawArr := make([]RawFileInfo, er.setDriveCount)
 	metaArr := make([]FileInfo, er.setDriveCount)
 	errs := make([]error, er.setDriveCount)
@@ -780,14 +725,16 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	done := make(chan bool, er.setDriveCount)
 	disks := er.getDisks()
 
-	mrfCheck := make(chan FileInfo)
-	defer close(mrfCheck)
-
 	ropts := ReadOptions{
 		ReadData: readData,
 		Healing:  false,
 	}
 
+	mrfCheck := make(chan FileInfo)
+	defer xioutil.SafeClose(mrfCheck)
+
+	var rw sync.Mutex
+
 	// Ask for all disks first;
 	go func() {
 		ctx, cancel := context.WithCancel(ctx)
@@ -799,56 +746,69 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 				done <- false
 				continue
 			}
+			if !disk.IsOnline() {
+				done <- false
+				continue
+			}
 			wg.Add(1)
 			go func(i int, disk StorageAPI) {
 				defer wg.Done()
-				var fi FileInfo
-				var err error
+
+				var (
+					fi  FileInfo
+					rfi RawFileInfo
+					err error
+				)
+
 				if opts.VersionID != "" {
 					// Read a specific version ID
-					fi, err = readFileInfo(ctx, disk, bucket, object, opts.VersionID, ropts)
-					mu.Lock()
-					metaArr[i], errs[i] = fi, err
-					mu.Unlock()
+					fi, err = disk.ReadVersion(ctx, "", bucket, object, opts.VersionID, ropts)
 				} else {
 					// Read the latest version
-					var ri RawFileInfo
-					ri, err = readRawFileInfo(ctx, disk, bucket, object, readData)
-					mu.Lock()
-					rawArr[i], errs[i] = ri, err
-					mu.Unlock()
+					rfi, err = disk.ReadXL(ctx, bucket, object, readData)
 					if err == nil {
-						fi, err = fileInfoFromRaw(ri, bucket, object, readData, opts.InclFreeVersions, true)
-						mu.Lock()
-						metaArr[i], errs[i] = fi, err
-						mu.Unlock()
+						fi, err = fileInfoFromRaw(rfi, bucket, object, readData, opts.InclFreeVersions, true)
 					}
 				}
+
+				rw.Lock()
+				rawArr[i] = rfi
+				metaArr[i], errs[i] = fi, err
+				rw.Unlock()
+
 				done <- err == nil
 			}(i, disk)
 		}
 
 		wg.Wait()
-		close(done)
+		xioutil.SafeClose(done)
 
 		fi, ok := <-mrfCheck
 		if !ok {
 			return
 		}
+
 		if fi.Deleted {
 			return
 		}
+
 		// if one of the disk is offline, return right here no need
 		// to attempt a heal on the object.
 		if countErrs(errs, errDiskNotFound) > 0 {
 			return
 		}
+
 		var missingBlocks int
 		for i := range errs {
-			if errors.Is(errs[i], errFileNotFound) {
+			if IsErr(errs[i],
+				errFileNotFound,
+				errFileVersionNotFound,
+				errFileCorrupt,
+			) {
 				missingBlocks++
 			}
 		}
+
 		// if missing metadata can be reconstructed, attempt to reconstruct.
 		// additionally do not heal delete markers inline, let them be
 		// healed upon regular heal process.
@@ -879,13 +839,12 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		minDisks = er.setDriveCount - er.defaultParityCount
 	}
 
-	calcQuorum := func() (FileInfo, []FileInfo, []StorageAPI, time.Time, string, error) {
+	calcQuorum := func(metaArr []FileInfo, errs []error) (FileInfo, []FileInfo, []StorageAPI, time.Time, string, error) {
 		readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount)
 		if err != nil {
 			return FileInfo{}, nil, nil, time.Time{}, "", err
 		}
-		err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum)
-		if err != nil {
+		if err := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); err != nil {
 			return FileInfo{}, nil, nil, time.Time{}, "", err
 		}
 		onlineDisks, modTime, etag := listOnlineDisks(disks, metaArr, errs, readQuorum)
@@ -895,7 +854,11 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 		}
 
 		onlineMeta := make([]FileInfo, len(metaArr))
-		copy(onlineMeta, metaArr)
+		for i, disk := range onlineDisks {
+			if disk != nil {
+				onlineMeta[i] = metaArr[i]
+			}
+		}
 
 		return fi, onlineMeta, onlineDisks, modTime, etag, nil
 	}
@@ -922,20 +885,24 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 				continue
 			}
 		}
+
+		rw.Lock()
 		if opts.VersionID == "" && totalResp == er.setDriveCount {
-			// Disks cannot agree about the latest version, pass this to a more advanced code
-			metaArr, errs = pickLatestQuorumFilesInfo(ctx, rawArr, errs, bucket, object, readData, opts.InclFreeVersions, true)
+			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(pickLatestQuorumFilesInfo(ctx,
+				rawArr, errs, bucket, object, readData, opts.InclFreeVersions, true))
+		} else {
+			fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(metaArr, errs)
 		}
-		mu.Lock()
-		fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum()
-		mu.Unlock()
+		rw.Unlock()
 		if err == nil && fi.InlineData() {
 			break
 		}
 	}
 
 	if err != nil {
-		if shouldCheckForDangling(err, errs, bucket) {
+		// We can only look for dangling if we received all the responses, if we did
+		// not we simply ignore it, since we can't tell for sure if its dangling object.
+		if totalResp == er.setDriveCount && shouldCheckForDangling(err, errs, bucket) {
 			_, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts)
 			if derr != nil {
 				err = derr
@@ -1122,7 +1089,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 
 	// Validate input data size and it can never be less than zero.
 	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument, logger.Application)
+		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
 		return ObjectInfo{}, toObjectErr(errInvalidArgument)
 	}
 
@@ -1227,7 +1194,7 @@ func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *
 		}
 	}
 
-	if _, err = writeUniqueFileInfo(ctx, onlineDisks, minioMetaBucket, key, partsMetadata, writeQuorum); err != nil {
+	if _, err = writeUniqueFileInfo(ctx, onlineDisks, "", minioMetaBucket, key, partsMetadata, writeQuorum); err != nil {
 		return ObjectInfo{}, toObjectErr(err, minioMetaBucket, key)
 	}
 
@@ -1309,7 +1276,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 	// Validate input data size and it can never be less than -1.
 	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument, logger.Application)
+		logger.LogIf(ctx, errInvalidArgument, logger.ErrorKind)
 		return ObjectInfo{}, toObjectErr(errInvalidArgument)
 	}
 
@@ -1457,7 +1424,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			continue
 		}
 
-		writers[i] = newBitrotWriter(disk, minioMetaTmpBucket, tempErasureObj, shardFileSize, DefaultBitrotAlgorithm, erasure.ShardSize())
+		writers[i] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, tempErasureObj, shardFileSize, DefaultBitrotAlgorithm, erasure.ShardSize())
 	}
 
 	toEncode := io.Reader(data)
@@ -1477,7 +1444,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	n, erasureErr := erasure.Encode(ctx, toEncode, writers, buffer, writeQuorum)
 	closeBitrotWriters(writers)
 	if erasureErr != nil {
-		return ObjectInfo{}, toObjectErr(erasureErr, minioMetaTmpBucket, tempErasureObj)
+		return ObjectInfo{}, toObjectErr(erasureErr, bucket, object)
 	}
 
 	// Should return IncompleteBody{} error when reader has fewer bytes
@@ -1544,13 +1511,12 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		partsMetadata[index].Metadata = userDefined
 		partsMetadata[index].Size = n
 		partsMetadata[index].ModTime = modTime
-	}
-
-	if len(inlineBuffers) > 0 {
-		// Set an additional header when data is inlined.
-		for index := range partsMetadata {
+		if len(inlineBuffers) > 0 {
 			partsMetadata[index].SetInlineData()
 		}
+		if opts.DataMovement {
+			partsMetadata[index].SetDataMov()
+		}
 	}
 
 	// Rename the successfully written temporary object to final location.
@@ -2065,7 +2031,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 
 	// Read metadata associated with the object from all disks.
 	if opts.VersionID != "" {
-		metaArr, errs = readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false, false)
+		metaArr, errs = readAllFileInfo(ctx, disks, "", bucket, object, opts.VersionID, false, false)
 	} else {
 		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false, true)
 	}
@@ -2138,7 +2104,7 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 
 	// Read metadata associated with the object from all disks.
 	if opts.VersionID != "" {
-		metaArr, errs = readAllFileInfo(ctx, disks, bucket, object, opts.VersionID, false, false)
+		metaArr, errs = readAllFileInfo(ctx, disks, "", bucket, object, opts.VersionID, false, false)
 	} else {
 		metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false, true)
 	}
@@ -2205,7 +2171,7 @@ func (er erasureObjects) updateObjectMetaWithOpts(ctx context.Context, bucket, o
 	// Wait for all the routines.
 	mErrs := g.Wait()
 
-	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, er.defaultWQuorum())
+	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, fi.WriteQuorum(er.defaultWQuorum()))
 }
 
 // updateObjectMeta will update the metadata of a file.
@@ -2466,7 +2432,7 @@ func (er erasureObjects) DecomTieredObject(ctx context.Context, bucket, object s
 	var onlineDisks []StorageAPI
 	onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi)
 
-	if _, err := writeUniqueFileInfo(ctx, onlineDisks, bucket, object, partsMetadata, writeQuorum); err != nil {
+	if _, err := writeUniqueFileInfo(ctx, onlineDisks, "", bucket, object, partsMetadata, writeQuorum); err != nil {
 		return toObjectErr(err, bucket, object)
 	}
 

cmd/erasure-object_test.go

@@ -521,7 +521,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 	}
 
 	for _, disk := range xl.getDisks() {
-		files, _ := disk.ListDir(ctx, bucket, object, -1)
+		files, _ := disk.ListDir(ctx, "", bucket, object, -1)
 		for _, file := range files {
 			if file != "xl.meta" {
 				disk.Delete(ctx, bucket, pathJoin(object, file), DeleteOptions{
@@ -625,7 +625,7 @@ func TestHeadObjectNoQuorum(t *testing.T) {
 		t.Fatal(err)
 	}
 	for _, disk := range xl.getDisks() {
-		files, _ := disk.ListDir(ctx, bucket, object, -1)
+		files, _ := disk.ListDir(ctx, "", bucket, object, -1)
 		for _, file := range files {
 			if file != "xl.meta" {
 				disk.Delete(ctx, bucket, pathJoin(object, file), DeleteOptions{
@@ -908,7 +908,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts1, errs1 := readAllFileInfo(ctx, erasureDisks, bucket, object1, "", false, false)
+	parts1, errs1 := readAllFileInfo(ctx, erasureDisks, "", bucket, object1, "", false, false)
 	parts1SC := globalStorageClass
 
 	// Object for test case 2 - No StorageClass defined, MetaData in PutObject requesting RRS Class
@@ -920,7 +920,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts2, errs2 := readAllFileInfo(ctx, erasureDisks, bucket, object2, "", false, false)
+	parts2, errs2 := readAllFileInfo(ctx, erasureDisks, "", bucket, object2, "", false, false)
 	parts2SC := globalStorageClass
 
 	// Object for test case 3 - No StorageClass defined, MetaData in PutObject requesting Standard Storage Class
@@ -932,7 +932,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts3, errs3 := readAllFileInfo(ctx, erasureDisks, bucket, object3, "", false, false)
+	parts3, errs3 := readAllFileInfo(ctx, erasureDisks, "", bucket, object3, "", false, false)
 	parts3SC := globalStorageClass
 
 	// Object for test case 4 - Standard StorageClass defined as Parity 6, MetaData in PutObject requesting Standard Storage Class
@@ -950,7 +950,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts4, errs4 := readAllFileInfo(ctx, erasureDisks, bucket, object4, "", false, false)
+	parts4, errs4 := readAllFileInfo(ctx, erasureDisks, "", bucket, object4, "", false, false)
 	parts4SC := storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 6,
@@ -973,7 +973,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts5, errs5 := readAllFileInfo(ctx, erasureDisks, bucket, object5, "", false, false)
+	parts5, errs5 := readAllFileInfo(ctx, erasureDisks, "", bucket, object5, "", false, false)
 	parts5SC := globalStorageClass
 
 	// Object for test case 6 - RRS StorageClass defined as Parity 2, MetaData in PutObject requesting Standard Storage Class
@@ -994,7 +994,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts6, errs6 := readAllFileInfo(ctx, erasureDisks, bucket, object6, "", false, false)
+	parts6, errs6 := readAllFileInfo(ctx, erasureDisks, "", bucket, object6, "", false, false)
 	parts6SC := storageclass.Config{
 		RRS: storageclass.StorageClass{
 			Parity: 2,
@@ -1017,7 +1017,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 		t.Fatalf("Failed to putObject %v", err)
 	}
 
-	parts7, errs7 := readAllFileInfo(ctx, erasureDisks, bucket, object7, "", false, false)
+	parts7, errs7 := readAllFileInfo(ctx, erasureDisks, "", bucket, object7, "", false, false)
 	parts7SC := storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 5,

cmd/erasure-server-pool-decom.go

@@ -653,7 +653,8 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 			}
 		}
 		_, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, parts, ObjectOptions{
-			MTime: objInfo.ModTime,
+			DataMovement: true,
+			MTime:        objInfo.ModTime,
 		})
 		if err != nil {
 			err = fmt.Errorf("decommissionObject: CompleteMultipartUpload() %w", err)
@@ -665,11 +666,13 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 	if err != nil {
 		return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 	}
+
 	_, err = z.PutObject(ctx,
 		bucket,
 		objInfo.Name,
 		NewPutObjReader(hr),
 		ObjectOptions{
+			DataMovement: true,
 			VersionID:    objInfo.VersionID,
 			MTime:        objInfo.ModTime,
 			UserDefined:  objInfo.UserDefined,
@@ -936,7 +939,8 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					bi.Name,
 					encodeDirObject(entry.name),
 					ObjectOptions{
-						DeletePrefix: true, // use prefix delete to delete all versions at once.
+						DeletePrefix:       true, // use prefix delete to delete all versions at once.
+						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
 					},
 				)
 				stopFn(err)

cmd/erasure-server-pool-rebalance.go

@@ -34,6 +34,7 @@ import (
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/hash"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 )
@@ -114,12 +115,60 @@ func (z *erasureServerPools) loadRebalanceMeta(ctx context.Context) error {
 	}
 
 	z.rebalMu.Lock()
-	z.rebalMeta = r
+	if len(r.PoolStats) == len(z.serverPools) {
+		z.rebalMeta = r
+	} else {
+		z.updateRebalanceStats(ctx)
+	}
 	z.rebalMu.Unlock()
 
 	return nil
 }
 
+// updates rebalance.bin from let's say 2 pool setup in the middle
+// of a rebalance, was expanded can cause z.rebalMeta to be outdated
+// due to a missing new pool. This function tries to handle this
+// scenario, albeit rare it seems to have occurred in the wild.
+//
+// since we do not explicitly disallow it, but it is okay for them
+// expand and then we continue to rebalance.
+func (z *erasureServerPools) updateRebalanceStats(ctx context.Context) error {
+	var ok bool
+	for i := range z.serverPools {
+		if z.findIndex(i) == -1 {
+			// Also ensure to initialize rebalanceStats to indicate
+			// its a new pool that can receive rebalanced data.
+			z.rebalMeta.PoolStats = append(z.rebalMeta.PoolStats, &rebalanceStats{})
+			ok = true
+		}
+	}
+	if ok {
+		lock := z.serverPools[0].NewNSLock(minioMetaBucket, rebalMetaName)
+		lkCtx, err := lock.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			logger.LogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err))
+			return err
+		}
+		defer lock.Unlock(lkCtx)
+
+		ctx = lkCtx.Context()
+
+		noLockOpts := ObjectOptions{NoLock: true}
+		return z.rebalMeta.saveWithOpts(ctx, z.serverPools[0], noLockOpts)
+	}
+
+	return nil
+}
+
+func (z *erasureServerPools) findIndex(index int) int {
+	for i := 0; i < len(z.rebalMeta.PoolStats); i++ {
+		if i == index {
+			return index
+		}
+	}
+	return -1
+}
+
 // initRebalanceMeta initializes rebalance metadata for a new rebalance
 // operation and saves it in the object store.
 func (z *erasureServerPools) initRebalanceMeta(ctx context.Context, buckets []string) (arn string, err error) {
@@ -324,7 +373,7 @@ func (z *erasureServerPools) IsPoolRebalancing(poolIndex int) bool {
 
 func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int) (err error) {
 	doneCh := make(chan struct{})
-	defer close(doneCh)
+	defer xioutil.SafeClose(doneCh)
 
 	// Save rebalance.bin periodically.
 	go func() {
@@ -510,7 +559,6 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 
 				// Apply lifecycle rules on the objects that are expired.
 				if filterLifecycle(bucket, version.Name, version) {
-					rebalanced++
 					expired++
 					continue
 				}
@@ -609,7 +657,8 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 					bucket,
 					encodeDirObject(entry.name),
 					ObjectOptions{
-						DeletePrefix: true, // use prefix delete to delete all versions at once.
+						DeletePrefix:       true, // use prefix delete to delete all versions at once.
+						DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls)
 					},
 				)
 				stopFn(err)
@@ -695,8 +744,7 @@ func (z *erasureServerPools) saveRebalanceStats(ctx context.Context, poolIdx int
 	}
 	z.rebalMeta = r
 
-	err = z.rebalMeta.saveWithOpts(ctx, z.serverPools[0], noLockOpts)
-	return err
+	return z.rebalMeta.saveWithOpts(ctx, z.serverPools[0], noLockOpts)
 }
 
 func auditLogRebalance(ctx context.Context, apiName, bucket, object, versionID string, err error) {
@@ -761,7 +809,8 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 			}
 		}
 		_, err = z.CompleteMultipartUpload(ctx, bucket, oi.Name, res.UploadID, parts, ObjectOptions{
-			MTime: oi.ModTime,
+			DataMovement: true,
+			MTime:        oi.ModTime,
 		})
 		if err != nil {
 			err = fmt.Errorf("rebalanceObject: CompleteMultipartUpload() %w", err)
@@ -773,11 +822,13 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 	if err != nil {
 		return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err)
 	}
+
 	_, err = z.PutObject(ctx,
 		bucket,
 		oi.Name,
 		NewPutObjReader(hr),
 		ObjectOptions{
+			DataMovement: true,
 			VersionID:    oi.VersionID,
 			MTime:        oi.ModTime,
 			UserDefined:  oi.UserDefined,

cmd/erasure-server-pool.go

@@ -40,6 +40,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bpool"
 	"github.com/minio/minio/internal/config/storageclass"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/sync/errgroup"
 	"github.com/minio/pkg/v2/wildcard"
@@ -71,32 +72,38 @@ func (z *erasureServerPools) SinglePool() bool {
 func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServerPools) (ObjectLayer, error) {
 	var (
 		deploymentID       string
-		distributionAlgo   string
 		commonParityDrives int
 		err                error
 
 		formats      = make([]*formatErasureV3, len(endpointServerPools))
 		storageDisks = make([][]StorageAPI, len(endpointServerPools))
 		z            = &erasureServerPools{
-			serverPools: make([]*erasureSets, len(endpointServerPools)),
-			s3Peer:      NewS3PeerSys(endpointServerPools),
+			serverPools:      make([]*erasureSets, len(endpointServerPools)),
+			s3Peer:           NewS3PeerSys(endpointServerPools),
+			distributionAlgo: formatErasureVersionV3DistributionAlgoV3,
 		}
 	)
 
 	// Maximum number of reusable buffers per node at any given point in time.
-	n := 1024 // single node single/multiple drives set this to 1024 entries
+	n := uint64(1024) // single node single/multiple drives set this to 1024 entries
 
 	if globalIsDistErasure {
 		n = 2048
 	}
 
+	if globalIsCICD {
+		n = 256 // 256MiB for CI/CD environments is sufficient
+	}
+
+	// Avoid allocating more than half of the available memory
+	if maxN := availableMemory() / (blockSizeV2 * 2); n > maxN {
+		n = maxN
+	}
+
 	// Initialize byte pool once for all sets, bpool size is set to
 	// setCount * setDriveCount with each memory upto blockSizeV2.
 	globalBytePoolCap = bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2)
-
-	if globalServerCtxt.PreAllocate {
-		globalBytePoolCap.Populate()
-	}
+	globalBytePoolCap.Populate()
 
 	var localDrives []StorageAPI
 	local := endpointServerPools.FirstLocal()
@@ -117,8 +124,10 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			return nil, fmt.Errorf("parity validation returned an error: %w <- (%d, %d), for pool(%s)", err, commonParityDrives, ep.DrivesPerSet, humanize.Ordinal(i+1))
 		}
 
-		storageDisks[i], formats[i], err = waitForFormatErasure(local, ep.Endpoints, i+1,
-			ep.SetCount, ep.DrivesPerSet, deploymentID, distributionAlgo)
+		bootstrapTrace("waitForFormatErasure: loading disks", func() {
+			storageDisks[i], formats[i], err = waitForFormatErasure(local, ep.Endpoints, i+1,
+				ep.SetCount, ep.DrivesPerSet, deploymentID)
+		})
 		if err != nil {
 			return nil, err
 		}
@@ -128,16 +137,14 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			deploymentID = formats[i].ID
 		}
 
-		if distributionAlgo == "" {
-			distributionAlgo = formats[i].Erasure.DistributionAlgo
-		}
-
 		// Validate if users brought different DeploymentID pools.
 		if deploymentID != formats[i].ID {
 			return nil, fmt.Errorf("all pools must have same deployment ID - expected %s, got %s for pool(%s)", deploymentID, formats[i].ID, humanize.Ordinal(i+1))
 		}
 
-		z.serverPools[i], err = newErasureSets(ctx, ep, storageDisks[i], formats[i], commonParityDrives, i)
+		bootstrapTrace(fmt.Sprintf("newErasureSets: initializing %s pool", humanize.Ordinal(i+1)), func() {
+			z.serverPools[i], err = newErasureSets(ctx, ep, storageDisks[i], formats[i], commonParityDrives, i)
+		})
 		if err != nil {
 			return nil, err
 		}
@@ -146,10 +153,6 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			z.deploymentID = uuid.MustParse(deploymentID)
 		}
 
-		if distributionAlgo != "" && z.distributionAlgo == "" {
-			z.distributionAlgo = distributionAlgo
-		}
-
 		for _, storageDisk := range storageDisks[i] {
 			if storageDisk != nil && storageDisk.IsLocal() {
 				localDrives = append(localDrives, storageDisk)
@@ -175,8 +178,12 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 	setObjectLayer(z)
 
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	attempt := 1
 	for {
-		err := z.Init(ctx) // Initializes all pools.
+		var err error
+		bootstrapTrace(fmt.Sprintf("poolMeta.Init: loading pool metadata, attempt: %d", attempt), func() {
+			err = z.Init(ctx) // Initializes all pools.
+		})
 		if err != nil {
 			if !configRetriableErrors(err) {
 				logger.Fatal(err, "Unable to initialize backend")
@@ -184,6 +191,7 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ
 			retry := time.Duration(r.Float64() * float64(5*time.Second))
 			logger.LogIf(ctx, fmt.Errorf("Unable to initialize backend: %w, retrying in %s", err, retry))
 			time.Sleep(retry)
+			attempt++
 			continue
 		}
 		break
@@ -643,7 +651,7 @@ func (z *erasureServerPools) StorageInfo(ctx context.Context, metrics bool) Stor
 
 func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error {
 	// Updates must be closed before we return.
-	defer close(updates)
+	defer xioutil.SafeClose(updates)
 
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
@@ -670,7 +678,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataU
 			results = append(results, dataUsageCache{})
 			go func(i int, erObj *erasureObjects) {
 				updates := make(chan dataUsageCache, 1)
-				defer close(updates)
+				defer xioutil.SafeClose(updates)
 				// Start update collector.
 				go func() {
 					defer wg.Done()
@@ -729,7 +737,7 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataU
 				return
 			case v := <-updateCloser:
 				update()
-				close(v)
+				xioutil.SafeClose(v)
 				return
 			case <-updateTicker.C:
 				update()
@@ -1296,7 +1304,10 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
-		return loi, err
+		if !isErrBucketNotFound(err) {
+			logger.LogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
+		}
+		return loi, toObjectErr(err, bucket)
 	}
 	defer merged.truncate(0) // Release when returning
 
@@ -1445,6 +1456,12 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 			}
 			return loi, nil
 		}
+		if isErrBucketNotFound(err) {
+			return loi, err
+		}
+		if contextCanceled(ctx) {
+			return ListObjectsInfo{}, ctx.Err()
+		}
 	}
 
 	if len(prefix) > 0 && maxKeys == 1 && marker == "" {
@@ -1470,7 +1487,9 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 			loi.Objects = append(loi.Objects, objInfo)
 			return loi, nil
 		}
-
+		if isErrBucketNotFound(err) {
+			return ListObjectsInfo{}, err
+		}
 		if contextCanceled(ctx) {
 			return ListObjectsInfo{}, ctx.Err()
 		}
@@ -1481,7 +1500,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		if !isErrBucketNotFound(err) {
 			logger.LogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
 		}
-		return loi, err
+		return loi, toObjectErr(err, bucket)
 	}
 
 	merged.forwardPast(opts.Marker)
@@ -1526,7 +1545,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 }
 
 func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) {
-	if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, z); err != nil {
+	if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil {
 		return ListMultipartsInfo{}, err
 	}
 
@@ -1813,12 +1832,6 @@ func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, op
 		}
 	}
 
-	if err != nil && !isErrBucketNotFound(err) {
-		if !opts.NoRecreate {
-			z.s3Peer.MakeBucket(ctx, bucket, MakeBucketOptions{})
-		}
-	}
-
 	if err == nil {
 		// Purge the entire bucket metadata entirely.
 		z.deleteAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket))
@@ -1931,11 +1944,7 @@ func (z *erasureServerPools) HealFormat(ctx context.Context, dryRun bool) (madmi
 }
 
 func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
-	// Attempt heal on the bucket metadata, ignore any failures
-	hopts := opts
-	hopts.Recreate = false
-	defer z.HealObject(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, bucket, bucketMetadataFile), "", hopts)
-
+	// .metadata.bin healing is not needed here, it is automatically healed via read() call.
 	return z.s3Peer.HealBucket(ctx, bucket, opts)
 }
 
@@ -1945,9 +1954,9 @@ func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts
 // error walker returns error. Optionally if context.Done() is received
 // then Walk() stops the walker.
 func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo, opts WalkOptions) error {
-	if err := checkListObjsArgs(ctx, bucket, prefix, "", z); err != nil {
+	if err := checkListObjsArgs(ctx, bucket, prefix, ""); err != nil {
 		// Upon error close the channel.
-		close(results)
+		xioutil.SafeClose(results)
 		return err
 	}
 
@@ -1956,7 +1965,7 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 	ctx, cancel := context.WithCancel(ctx)
 	go func() {
 		defer cancel()
-		defer close(results)
+		defer xioutil.SafeClose(results)
 
 		for _, erasureSet := range z.serverPools {
 			var wg sync.WaitGroup
@@ -2275,6 +2284,7 @@ type HealthResult struct {
 	ESHealth      []struct {
 		Maintenance   bool
 		PoolID, SetID int
+		Healthy       bool
 		HealthyDrives int
 		HealingDrives int
 		ReadQuorum    int
@@ -2398,23 +2408,25 @@ func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) Hea
 			result.ESHealth = append(result.ESHealth, struct {
 				Maintenance                  bool
 				PoolID, SetID                int
+				Healthy                      bool
 				HealthyDrives, HealingDrives int
 				ReadQuorum, WriteQuorum      int
 			}{
 				Maintenance:   opts.Maintenance,
 				SetID:         setIdx,
 				PoolID:        poolIdx,
+				Healthy:       erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx],
 				HealthyDrives: erasureSetUpCount[poolIdx][setIdx].online,
 				HealingDrives: erasureSetUpCount[poolIdx][setIdx].healing,
 				ReadQuorum:    poolReadQuorums[poolIdx],
 				WriteQuorum:   poolWriteQuorums[poolIdx],
 			})
 
-			if erasureSetUpCount[poolIdx][setIdx].online < poolWriteQuorums[poolIdx] {
+			result.Healthy = erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx]
+			if !result.Healthy {
 				logger.LogIf(logger.SetReqInfo(ctx, reqInfo),
 					fmt.Errorf("Write quorum may be lost on pool: %d, set: %d, expected write quorum: %d",
 						poolIdx, setIdx, poolWriteQuorums[poolIdx]))
-				result.Healthy = false
 			}
 		}
 	}

cmd/erasure-sets.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/dsync"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	"github.com/minio/pkg/v2/sync/errgroup"
@@ -114,26 +115,26 @@ func (s *erasureSets) getDiskMap() map[Endpoint]StorageAPI {
 
 // Initializes a new StorageAPI from the endpoint argument, returns
 // StorageAPI and also `format` which exists on the disk.
-func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
+func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, []byte, error) {
 	disk, err := newStorageAPI(endpoint, storageOpts{
 		cleanUp:     false,
 		healthCheck: false,
 	})
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
-	format, err := loadFormatErasure(disk)
+	format, formatData, err := loadFormatErasureWithData(disk, false)
 	if err != nil {
 		if errors.Is(err, errUnformattedDisk) {
-			info, derr := disk.DiskInfo(context.TODO(), false)
+			info, derr := disk.DiskInfo(context.TODO(), DiskInfoOptions{})
 			if derr != nil && info.RootDisk {
 				disk.Close()
-				return nil, nil, fmt.Errorf("Drive: %s is a root drive", disk)
+				return nil, nil, nil, fmt.Errorf("Drive: %s is a root drive", disk)
 			}
 		}
 		disk.Close()
-		return nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error
+		return nil, nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error
 	}
 
 	disk.Close()
@@ -142,10 +143,10 @@ func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) {
 		healthCheck: true,
 	})
 	if err != nil {
-		return nil, nil, err
+		return nil, nil, nil, err
 	}
 
-	return disk, format, nil
+	return disk, format, formatData, nil
 }
 
 // findDiskIndex - returns the i,j'th position of the input `diskID` against the reference
@@ -226,7 +227,7 @@ func (s *erasureSets) connectDisks() {
 		wg.Add(1)
 		go func(endpoint Endpoint) {
 			defer wg.Done()
-			disk, format, err := connectEndpoint(endpoint)
+			disk, format, formatData, err := connectEndpoint(endpoint)
 			if err != nil {
 				if endpoint.IsLocal && errors.Is(err, errUnformattedDisk) {
 					globalBackgroundHealState.pushHealLocalDisks(endpoint)
@@ -261,6 +262,7 @@ func (s *erasureSets) connectDisks() {
 
 			disk.SetDiskID(format.Erasure.This)
 			disk.SetDiskLoc(s.poolIndex, setIndex, diskIndex)
+			disk.SetFormatData(formatData)
 			s.erasureDisks[setIndex][diskIndex] = disk
 			s.erasureDisksMu.Unlock()
 
@@ -383,19 +385,27 @@ func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks [
 		}
 	}
 
+	var wg sync.WaitGroup
+	var lk sync.Mutex
 	for i := 0; i < setCount; i++ {
 		lockerEpSet := set.NewStringSet()
 		for j := 0; j < setDriveCount; j++ {
-			endpoint := endpoints.Endpoints[i*setDriveCount+j]
-			// Only add lockers only one per endpoint and per erasure set.
-			if locker, ok := erasureLockers[endpoint.Host]; ok && !lockerEpSet.Contains(endpoint.Host) {
-				lockerEpSet.Add(endpoint.Host)
-				s.erasureLockers[i] = append(s.erasureLockers[i], locker)
-			}
+			wg.Add(1)
+			go func(i int, endpoint Endpoint) {
+				defer wg.Done()
+
+				lk.Lock()
+				// Only add lockers only one per endpoint and per erasure set.
+				if locker, ok := erasureLockers[endpoint.Host]; ok && !lockerEpSet.Contains(endpoint.Host) {
+					lockerEpSet.Add(endpoint.Host)
+					s.erasureLockers[i] = append(s.erasureLockers[i], locker)
+				}
+				lk.Unlock()
+			}(i, endpoints.Endpoints[i*setDriveCount+j])
 		}
 	}
+	wg.Wait()
 
-	var wg sync.WaitGroup
 	for i := 0; i < setCount; i++ {
 		wg.Add(1)
 		go func(i int) {
@@ -658,10 +668,10 @@ func (s *erasureSets) Shutdown(ctx context.Context) error {
 	select {
 	case _, ok := <-s.setReconnectEvent:
 		if ok {
-			close(s.setReconnectEvent)
+			xioutil.SafeClose(s.setReconnectEvent)
 		}
 	default:
-		close(s.setReconnectEvent)
+		xioutil.SafeClose(s.setReconnectEvent)
 	}
 	return nil
 }
@@ -723,7 +733,7 @@ func listDeletedBuckets(ctx context.Context, storageDisks []StorageAPI, delBucke
 				// we ignore disk not found errors
 				return nil
 			}
-			volsInfo, err := storageDisks[index].ListDir(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix), -1)
+			volsInfo, err := storageDisks[index].ListDir(ctx, "", minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix), -1)
 			if err != nil {
 				if errors.Is(err, errFileNotFound) {
 					return nil
@@ -1020,52 +1030,10 @@ func formatsToDrivesInfo(endpoints Endpoints, formats []*formatErasureV3, sErrs
 	return beforeDrives
 }
 
-func getHealDiskInfos(storageDisks []StorageAPI, errs []error) ([]DiskInfo, []error) {
-	infos := make([]DiskInfo, len(storageDisks))
-	g := errgroup.WithNErrs(len(storageDisks))
-	for index := range storageDisks {
-		index := index
-		g.Go(func() error {
-			if errs[index] != nil && errs[index] != errUnformattedDisk {
-				return errs[index]
-			}
-			if storageDisks[index] == nil {
-				return errDiskNotFound
-			}
-			var err error
-			infos[index], err = storageDisks[index].DiskInfo(context.TODO(), false)
-			return err
-		}, index)
-	}
-	return infos, g.Wait()
-}
-
-// Mark root disks as down so as not to heal them.
-func markRootDisksAsDown(storageDisks []StorageAPI, errs []error) {
-	if globalIsCICD {
-		// Do nothing
-		return
-	}
-	infos, ierrs := getHealDiskInfos(storageDisks, errs)
-	for i := range storageDisks {
-		if ierrs[i] != nil && ierrs[i] != errUnformattedDisk {
-			storageDisks[i] = nil
-			continue
-		}
-		if storageDisks[i] != nil && infos[i].RootDisk {
-			// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
-			// defective drive we should not heal a path on the root disk.
-			logger.LogIf(GlobalContext, fmt.Errorf("Drive `%s` is part of root drive, will not be used", storageDisks[i]))
-			storageDisks[i].Close()
-			storageDisks[i] = nil
-		}
-	}
-}
-
 // HealFormat - heals missing `format.json` on fresh unformatted disks.
 func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) {
 	storageDisks, _ := initStorageDisksWithErrors(s.endpoints.Endpoints, storageOpts{
-		cleanUp:     true,
+		cleanUp:     false,
 		healthCheck: false,
 	})
 
@@ -1080,9 +1048,6 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 		return madmin.HealResultItem{}, err
 	}
 
-	// Mark all root disks down
-	markRootDisksAsDown(storageDisks, sErrs)
-
 	refFormat, err := getFormatErasureInQuorum(formats)
 	if err != nil {
 		return res, err
@@ -1171,14 +1136,13 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 					xldisk, ok := disk.(*xlStorageDiskIDCheck)
 					if ok {
-						if driveQuorum {
-							commonWrites, commonDeletes := calcCommonWritesDeletes(currentDisksInfo[m], (s.setDriveCount+1)/2)
-							xldisk.totalWrites.Store(commonWrites)
-							xldisk.totalDeletes.Store(commonDeletes)
-							xldisk.storage.setWriteAttribute(commonWrites)
-							xldisk.storage.setDeleteAttribute(commonDeletes)
+						_, commonDeletes := calcCommonWritesDeletes(currentDisksInfo[m], (s.setDriveCount+1)/2)
+						xldisk.totalDeletes.Store(commonDeletes)
+						xldisk.storage.setDeleteAttribute(commonDeletes)
+
+						if globalDriveMonitoring {
+							go xldisk.monitorDiskWritable(xldisk.diskCtx)
 						}
-						go xldisk.monitorDiskWritable(xldisk.diskCtx)
 					}
 				} else {
 					disk.Close() // Close the remote storage client, re-initialize with healthchecks.
@@ -1191,9 +1155,18 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 
 				s.erasureDisks[m][n] = disk
 
-				if disk.IsLocal() && globalIsDistErasure {
+				if disk.IsLocal() {
 					globalLocalDrivesMu.Lock()
-					globalLocalSetDrives[s.poolIndex][m][n] = disk
+					if globalIsDistErasure {
+						globalLocalSetDrives[s.poolIndex][m][n] = disk
+					}
+					for i, ldisk := range globalLocalDrives {
+						_, k, l := ldisk.GetDiskLoc()
+						if k == m && l == n {
+							globalLocalDrives[i] = disk
+							break
+						}
+					}
 					globalLocalDrivesMu.Unlock()
 				}
 			}

cmd/erasure-sets_test.go

@@ -174,18 +174,18 @@ func TestNewErasureSets(t *testing.T) {
 	}
 
 	endpoints := mustGetNewEndpoints(0, 16, erasureDisks...)
-	_, _, err := waitForFormatErasure(true, endpoints, 1, 0, 16, "", "")
+	_, _, err := waitForFormatErasure(true, endpoints, 1, 0, 16, "")
 	if err != errInvalidArgument {
 		t.Fatalf("Expecting error, got %s", err)
 	}
 
-	_, _, err = waitForFormatErasure(true, nil, 1, 1, 16, "", "")
+	_, _, err = waitForFormatErasure(true, nil, 1, 1, 16, "")
 	if err != errInvalidArgument {
 		t.Fatalf("Expecting error, got %s", err)
 	}
 
 	// Initializes all erasure disks
-	storageDisks, format, err := waitForFormatErasure(true, endpoints, 1, 1, 16, "", "")
+	storageDisks, format, err := waitForFormatErasure(true, endpoints, 1, 1, 16, "")
 	if err != nil {
 		t.Fatalf("Unable to format drives for erasure, %s", err)
 	}

cmd/erasure-utils.go

@@ -25,7 +25,6 @@ import (
 	"strings"
 
 	"github.com/klauspost/reedsolomon"
-	"github.com/minio/minio/internal/logger"
 )
 
 // getDataBlockLen - get length of data blocks from encoded blocks.
@@ -43,19 +42,16 @@ func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int {
 func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) {
 	// Offset and out size cannot be negative.
 	if offset < 0 || length < 0 {
-		logger.LogIf(ctx, errUnexpected)
 		return 0, errUnexpected
 	}
 
 	// Do we have enough blocks?
 	if len(enBlocks) < dataBlocks {
-		logger.LogIf(ctx, fmt.Errorf("diskBlocks(%d)/dataBlocks(%d) - %w", len(enBlocks), dataBlocks, reedsolomon.ErrTooFewShards))
 		return 0, reedsolomon.ErrTooFewShards
 	}
 
 	// Do we have enough data?
 	if int64(getDataBlockLen(enBlocks, dataBlocks)) < length {
-		logger.LogIf(ctx, fmt.Errorf("getDataBlockLen(enBlocks, dataBlocks)(%d)/length(%d) - %w", getDataBlockLen(enBlocks, dataBlocks), length, reedsolomon.ErrShortData))
 		return 0, reedsolomon.ErrShortData
 	}
 
@@ -85,11 +81,6 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 		if write < int64(len(block)) {
 			n, err := dst.Write(block[:write])
 			if err != nil {
-				// The writer will be closed in case of range queries, which will emit ErrClosedPipe.
-				// The reader pipe might be closed at ListObjects io.EOF ignore it.
-				if err != io.ErrClosedPipe && err != io.EOF {
-					logger.LogIf(ctx, err)
-				}
 				return 0, err
 			}
 			totalWritten += int64(n)
@@ -99,11 +90,6 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 		// Copy the block.
 		n, err := dst.Write(block)
 		if err != nil {
-			// The writer will be closed in case of range queries, which will emit ErrClosedPipe.
-			// The reader pipe might be closed at ListObjects io.EOF ignore it.
-			if err != io.ErrClosedPipe && err != io.EOF {
-				logger.LogIf(ctx, err)
-			}
 			return 0, err
 		}
 

cmd/erasure.go

@@ -178,7 +178,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint, metrics bool) (disks
 				disksInfo[index] = di
 				return nil
 			}
-			info, err := disks[index].DiskInfo(context.TODO(), metrics)
+			info, err := disks[index].DiskInfo(context.TODO(), DiskInfoOptions{Metrics: metrics})
 			di.DrivePath = info.MountPath
 			di.TotalSpace = info.Total
 			di.UsedSpace = info.Used
@@ -203,6 +203,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint, metrics bool) (disks
 				APICalls:                make(map[string]uint64, len(info.Metrics.APICalls)),
 				TotalErrorsAvailability: info.Metrics.TotalErrorsAvailability,
 				TotalErrorsTimeout:      info.Metrics.TotalErrorsTimeout,
+				TotalWaiting:            info.Metrics.TotalWaiting,
 			}
 			for k, v := range info.Metrics.LastMinute {
 				if v.N > 0 {
@@ -288,7 +289,7 @@ func (er erasureObjects) getOnlineDisksWithHealingAndInfo(inclHealing bool) (new
 				return
 			}
 
-			di, err := disk.DiskInfo(context.Background(), false)
+			di, err := disk.DiskInfo(context.Background(), DiskInfoOptions{})
 			infos[i] = di
 			if err != nil {
 				// - Do not consume disks which are not reachable
@@ -421,7 +422,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 			bucketCh <- b
 		}
 	}
-	close(bucketCh)
+	xioutil.SafeClose(bucketCh)
 
 	bucketResults := make(chan dataUsageEntryInfo, len(disks))
 
@@ -495,7 +496,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				cache.Info.SkipHealing = healing
 				cache.Info.NextCycle = wantCycle
 				if cache.Info.Name != bucket.Name {
-					logger.LogIf(ctx, fmt.Errorf("cache name mismatch: %s != %s", cache.Info.Name, bucket.Name))
 					cache.Info = dataUsageCacheInfo{
 						Name:       bucket.Name,
 						LastUpdate: time.Time{},
@@ -558,7 +558,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 		}(i)
 	}
 	wg.Wait()
-	close(bucketResults)
+	xioutil.SafeClose(bucketResults)
 	saverWg.Wait()
 
 	return nil

cmd/event-notification.go

@@ -225,7 +225,11 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 		},
 	}
 
-	if args.EventName != event.ObjectRemovedDelete && args.EventName != event.ObjectRemovedDeleteMarkerCreated {
+	isRemovedEvent := args.EventName == event.ObjectRemovedDelete ||
+		args.EventName == event.ObjectRemovedDeleteMarkerCreated ||
+		args.EventName == event.ObjectRemovedNoOP
+
+	if !isRemovedEvent {
 		newEvent.S3.Object.ETag = args.Object.ETag
 		newEvent.S3.Object.Size = args.Object.Size
 		newEvent.S3.Object.ContentType = args.Object.ContentType

cmd/format-erasure.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -28,6 +28,7 @@ import (
 	"sync"
 
 	"github.com/dustin/go-humanize"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/storageclass"
@@ -325,20 +326,16 @@ func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasur
 			if storageDisks[index] == nil {
 				return errDiskNotFound
 			}
-			format, err := loadFormatErasure(storageDisks[index])
+			format, formatData, err := loadFormatErasureWithData(storageDisks[index], heal)
 			if err != nil {
 				return err
 			}
-			info, err := storageDisks[index].DiskInfo(context.Background(), false)
-			if err != nil {
-				return err
-			}
-			format.Info = info
 			formats[index] = format
 			if !heal {
 				// If no healing required, make the disks valid and
 				// online.
 				storageDisks[index].SetDiskID(format.Erasure.This)
+				storageDisks[index].SetFormatData(formatData)
 			}
 			return nil
 		}, index)
@@ -354,7 +351,7 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, healID string)
 	}
 
 	// Marshal and write to disk.
-	formatBytes, err := json.Marshal(format)
+	formatData, err := json.Marshal(format)
 	if err != nil {
 		return err
 	}
@@ -368,7 +365,7 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, healID string)
 	})
 
 	// write to unique file.
-	if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, formatBytes); err != nil {
+	if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, formatData); err != nil {
 		return err
 	}
 
@@ -378,6 +375,7 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, healID string)
 	}
 
 	disk.SetDiskID(format.Erasure.This)
+	disk.SetFormatData(formatData)
 	if healID != "" {
 		ctx := context.Background()
 		ht := initHealingTracker(disk, healID)
@@ -386,14 +384,38 @@ func saveFormatErasure(disk StorageAPI, format *formatErasureV3, healID string)
 	return nil
 }
 
+// loadFormatErasureWithData - loads format.json from disk.
+func loadFormatErasureWithData(disk StorageAPI, heal bool) (format *formatErasureV3, data []byte, err error) {
+	data, err = disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile)
+	if err != nil {
+		// 'file not found' and 'volume not found' as
+		// same. 'volume not found' usually means its a fresh disk.
+		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) {
+			return nil, nil, errUnformattedDisk
+		}
+		return nil, nil, err
+	}
+
+	// Try to decode format json into formatConfigV1 struct.
+	format = &formatErasureV3{}
+	if err = json.Unmarshal(data, format); err != nil {
+		return nil, nil, err
+	}
+
+	if heal {
+		info, err := disk.DiskInfo(context.Background(), DiskInfoOptions{NoOp: heal})
+		if err != nil {
+			return nil, nil, err
+		}
+		format.Info = info
+	}
+
+	// Success.
+	return format, data, nil
+}
+
 // loadFormatErasure - loads format.json from disk.
 func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
-	// Ensure that the grid is online.
-	if _, err := disk.DiskInfo(context.Background(), false); err != nil {
-		if errors.Is(err, errDiskNotFound) {
-			return nil, err
-		}
-	}
 	buf, err := disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile)
 	if err != nil {
 		// 'file not found' and 'volume not found' as
@@ -404,6 +426,8 @@ func loadFormatErasure(disk StorageAPI) (format *formatErasureV3, err error) {
 		return nil, err
 	}
 
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
+
 	// Try to decode format json into formatConfigV1 struct.
 	format = &formatErasureV3{}
 	if err = json.Unmarshal(buf, format); err != nil {
@@ -480,9 +504,7 @@ func formatErasureGetDeploymentID(refFormat *formatErasureV3, formats []*formatE
 }
 
 // formatErasureFixDeploymentID - Add deployment id if it is not present.
-func formatErasureFixDeploymentID(endpoints Endpoints, storageDisks []StorageAPI, refFormat *formatErasureV3) (err error) {
-	// Attempt to load all `format.json` from all disks.
-	formats, _ := loadFormatErasureAll(storageDisks, false)
+func formatErasureFixDeploymentID(endpoints Endpoints, storageDisks []StorageAPI, refFormat *formatErasureV3, formats []*formatErasureV3) (err error) {
 	for index := range formats {
 		// If the Erasure sets do not match, set those formats to nil,
 		// We do not have to update the ID on those format.json file.
@@ -739,7 +761,7 @@ func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats
 }
 
 // initFormatErasure - save Erasure format configuration on all disks.
-func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, setDriveCount int, deploymentID, distributionAlgo string, sErrs []error) (*formatErasureV3, error) {
+func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, setDriveCount int, deploymentID string, sErrs []error) (*formatErasureV3, error) {
 	format := newFormatErasureV3(setCount, setDriveCount)
 	formats := make([]*formatErasureV3, len(storageDisks))
 	wantAtMost, err := ecDrivesNoConfig(setDriveCount)
@@ -756,9 +778,6 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 			}
 			newFormat := format.Clone()
 			newFormat.Erasure.This = format.Erasure.Sets[i][j]
-			if distributionAlgo != "" {
-				newFormat.Erasure.DistributionAlgo = distributionAlgo
-			}
 			if deploymentID != "" {
 				newFormat.ID = deploymentID
 			}
@@ -781,15 +800,18 @@ func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount,
 						logger.Info("   - Drive: %s", disk.String())
 					}
 				})
-				logger.Info(color.Yellow("WARNING:")+" Host %v has more than %v drives of set. "+
-					"A host failure will result in data becoming unavailable.", host, wantAtMost)
+				var warning string
+				if wantAtMost == 0 {
+					warning = fmt.Sprintf("Host %v has all drives of set. ", host)
+				} else {
+					warning = fmt.Sprintf("Host %v has more than %v drives of set. ", host, wantAtMost)
+				}
+				logger.Info(color.Yellow("WARNING: ") + warning +
+					"A host failure will result in data becoming unavailable.")
 			}
 		}
 	}
 
-	// Mark all root disks down
-	markRootDisksAsDown(storageDisks, sErrs)
-
 	// Save formats `format.json` across all disks.
 	if err := saveFormatErasureAll(ctx, storageDisks, formats); err != nil {
 		return nil, err
@@ -831,7 +853,9 @@ func newHealFormatSets(refFormat *formatErasureV3, setCount, setDriveCount int,
 				newFormats[i][j].Erasure.DistributionAlgo = refFormat.Erasure.DistributionAlgo
 			}
 			if format := formats[i*setDriveCount+j]; format != nil && (errs[i*setDriveCount+j] == nil) {
-				currentDisksInfo[i][j] = format.Info
+				if format.Info.Endpoint != "" {
+					currentDisksInfo[i][j] = format.Info
+				}
 			}
 		}
 	}

cmd/ftp-server-driver.go

@@ -32,6 +32,7 @@ import (
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/internal/auth"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	ftp "goftp.io/server/v2"
 )
@@ -386,7 +387,7 @@ func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, path string) (err error) {
 
 	// Send object names that are needed to be removed to objectsCh
 	go func() {
-		defer close(objectsCh)
+		defer xioutil.SafeClose(objectsCh)
 		opts := minio.ListObjectsOptions{
 			Prefix:    prefix,
 			Recursive: true,

cmd/global-heal.go

@@ -28,6 +28,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config/storageclass"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	"github.com/minio/pkg/v2/wildcard"
@@ -156,7 +157,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		}
 	}
 
-	info, err := tracker.disk.DiskInfo(ctx, false)
+	info, err := tracker.disk.DiskInfo(ctx, DiskInfoOptions{})
 	if err != nil {
 		return fmt.Errorf("unable to get disk information before healing it: %w", err)
 	}
@@ -443,7 +444,7 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			finished: nil,
 		})
 		jt.Wait() // synchronize all the concurrent heal jobs
-		close(results)
+		xioutil.SafeClose(results)
 		if err != nil {
 			// Set this such that when we return this function
 			// we let the caller retry this disk again for the

cmd/globals.go

@@ -34,6 +34,7 @@ import (
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/browser"
+	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/kms"
 	"go.uber.org/atomic"
@@ -56,6 +57,7 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/pubsub"
 	"github.com/minio/pkg/v2/certs"
+	"github.com/minio/pkg/v2/env"
 	xnet "github.com/minio/pkg/v2/net"
 )
 
@@ -129,6 +131,11 @@ const (
 	tlsClientSessionCacheSize = 100
 )
 
+func init() {
+	// Injected to prevent circular dependency.
+	pubsub.GetByteBuffer = grid.GetByteBuffer
+}
+
 type poolDisksLayout struct {
 	cmdline string
 	layout  [][]string
@@ -143,7 +150,6 @@ type serverCtxt struct {
 	JSON, Quiet               bool
 	Anonymous                 bool
 	StrictS3Compat            bool
-	PreAllocate               bool
 	Addr, ConsoleAddr         string
 	ConfigDir, CertsDir       string
 	configDirSet, certsDirSet bool
@@ -154,13 +160,15 @@ type serverCtxt struct {
 	FTP  []string
 	SFTP []string
 
-	UserTimeout       time.Duration
-	ConnReadDeadline  time.Duration
-	ConnWriteDeadline time.Duration
+	UserTimeout            time.Duration
+	ConnReadDeadline       time.Duration
+	ConnWriteDeadline      time.Duration
+	ConnClientReadDeadline time.Duration
 
-	ShutdownTimeout   time.Duration
-	IdleTimeout       time.Duration
-	ReadHeaderTimeout time.Duration
+	ShutdownTimeout     time.Duration
+	IdleTimeout         time.Duration
+	ReadHeaderTimeout   time.Duration
+	MaxIdleConnsPerHost int
 
 	// The layout of disks as interpreted
 	Layout disksLayout
@@ -389,6 +397,8 @@ var (
 
 	globalRemoteTargetTransport http.RoundTripper
 
+	globalHealthChkTransport http.RoundTripper
+
 	globalDNSCache = &dnscache.Resolver{
 		Timeout: 5 * time.Second,
 	}
@@ -409,10 +419,12 @@ var (
 	globalServiceFreezeMu  sync.Mutex // Updates.
 
 	// List of local drives to this node, this is only set during server startup,
-	// and should never be mutated. Hold globalLocalDrivesMu to access.
+	// and is only mutated by HealFormat. Hold globalLocalDrivesMu to access.
 	globalLocalDrives   []StorageAPI
 	globalLocalDrivesMu sync.RWMutex
 
+	globalDriveMonitoring = env.Get("_MINIO_DRIVE_ACTIVE_MONITORING", config.EnableOn) == config.EnableOn
+
 	// Is MINIO_CI_CD set?
 	globalIsCICD bool
 

cmd/grid.go

@@ -22,7 +22,6 @@ import (
 	"crypto/tls"
 	"sync/atomic"
 
-	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/fips"
 	"github.com/minio/minio/internal/grid"
 	xhttp "github.com/minio/minio/internal/http"
@@ -36,28 +35,11 @@ var globalGrid atomic.Pointer[grid.Manager]
 var globalGridStart = make(chan struct{})
 
 func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error {
-	seenHosts := set.NewStringSet()
-	var hosts []string
-	var local string
-	for _, ep := range eps {
-		for _, endpoint := range ep.Endpoints {
-			u := endpoint.GridHost()
-			if seenHosts.Contains(u) {
-				continue
-			}
-			seenHosts.Add(u)
-
-			// Set local endpoint
-			if endpoint.IsLocal {
-				local = u
-			}
-			hosts = append(hosts, u)
-		}
-	}
 	lookupHost := globalDNSCache.LookupHost
 	if IsKubernetes() || IsDocker() {
 		lookupHost = nil
 	}
+	hosts, local := eps.GridHosts()
 	g, err := grid.NewManager(ctx, grid.ManagerOptions{
 		Dialer:       grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions))),
 		Local:        local,

cmd/handler-api.go

@@ -56,16 +56,31 @@ type apiConfig struct {
 	syncEvents                  bool
 }
 
-const cgroupLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
+const (
+	cgroupV1MemLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes"
+	cgroupV2MemLimitFile = "/sys/fs/cgroup/memory.max"
+	cgroupMemNoLimit     = 9223372036854771712
+)
 
-func cgroupLimit(limitFile string) (limit uint64) {
-	buf, err := os.ReadFile(limitFile)
+func cgroupMemLimit() (limit uint64) {
+	buf, err := os.ReadFile(cgroupV2MemLimitFile)
 	if err != nil {
-		return 9223372036854771712
+		buf, err = os.ReadFile(cgroupV1MemLimitFile)
+	}
+	if err != nil {
+		return 0
 	}
 	limit, err = strconv.ParseUint(string(buf), 10, 64)
 	if err != nil {
-		return 9223372036854771712
+		// The kernel can return valid but non integer values
+		// but still, no need to interpret more
+		return 0
+	}
+	if limit == cgroupMemNoLimit {
+		// No limit set, It's the highest positive signed 64-bit
+		// integer (2^63-1), rounded down to multiples of 4096 (2^12),
+		// the most common page size on x86 systems - for cgroup_limits.
+		return 0
 	}
 	return limit
 }
@@ -74,23 +89,19 @@ func availableMemory() (available uint64) {
 	available = 8 << 30 // Default to 8 GiB when we can't find the limits.
 
 	if runtime.GOOS == "linux" {
-		available = cgroupLimit(cgroupLimitFile)
-
-		// No limit set, It's the highest positive signed 64-bit
-		// integer (2^63-1), rounded down to multiples of 4096 (2^12),
-		// the most common page size on x86 systems - for cgroup_limits.
-		if available != 9223372036854771712 {
-			// This means cgroup memory limit is configured.
+		// Useful in container mode
+		limit := cgroupMemLimit()
+		if limit > 0 {
+			// A valid value is found
+			available = limit
 			return
-		} // no-limit set proceed to set the limits based on virtual memory.
-
+		}
 	} // for all other platforms limits are based on virtual memory.
 
 	memStats, err := mem.VirtualMemory()
 	if err != nil {
 		return
 	}
-
 	available = memStats.Available / 2
 	return
 }
@@ -133,8 +144,8 @@ func (t *apiConfig) init(cfg api.Config, setDriveCounts []int) {
 		}
 	} else {
 		apiRequestsMaxPerNode = cfg.RequestsMax
-		if len(globalEndpoints.Hostnames()) > 0 {
-			apiRequestsMaxPerNode /= len(globalEndpoints.Hostnames())
+		if n := totalNodeCount(); n > 0 {
+			apiRequestsMaxPerNode /= n
 		}
 	}
 

cmd/iam-object-store.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 )
@@ -575,7 +576,7 @@ func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix stri
 	ch := make(chan itemOrErr)
 
 	go func() {
-		defer close(ch)
+		defer xioutil.SafeClose(ch)
 
 		// Allocate new results channel to receive ObjectInfo.
 		objInfoCh := make(chan ObjectInfo)

cmd/iam-store.go

@@ -1017,18 +1017,37 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 	newPolicyMapping.UpdatedAt = UTCNow()
 	addedOrRemoved = policiesToUpdate.ToSlice()
 
-	if err = store.saveMappedPolicy(ctx, name, userType, isGroup, newPolicyMapping); err != nil {
-		return
-	}
-	if !isGroup {
-		if userType == stsUser {
-			cache.iamSTSPolicyMap[name] = newPolicyMapping
+	// In case of detach operation, it is possible that no policies are mapped -
+	// in this case, we delete the mapping from the store.
+	if len(newPolicies) == 0 {
+		if err = store.deleteMappedPolicy(ctx, name, userType, isGroup); err != nil && !errors.Is(err, errNoSuchPolicy) {
+			return
+		}
+		if !isGroup {
+			if userType == stsUser {
+				delete(cache.iamSTSPolicyMap, name)
+			} else {
+				delete(cache.iamUserPolicyMap, name)
+			}
 		} else {
-			cache.iamUserPolicyMap[name] = newPolicyMapping
+			delete(cache.iamGroupPolicyMap, name)
 		}
 	} else {
-		cache.iamGroupPolicyMap[name] = newPolicyMapping
+
+		if err = store.saveMappedPolicy(ctx, name, userType, isGroup, newPolicyMapping); err != nil {
+			return
+		}
+		if !isGroup {
+			if userType == stsUser {
+				cache.iamSTSPolicyMap[name] = newPolicyMapping
+			} else {
+				cache.iamUserPolicyMap[name] = newPolicyMapping
+			}
+		} else {
+			cache.iamGroupPolicyMap[name] = newPolicyMapping
+		}
 	}
+
 	cache.updatedAt = UTCNow()
 	return cache.updatedAt, addedOrRemoved, newPolicies, nil
 }
@@ -1147,8 +1166,11 @@ func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy
 	return err
 }
 
-// DeletePolicy - deletes policy from storage and cache.
-func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string) error {
+// DeletePolicy - deletes policy from storage and cache. When this called in
+// response to a notification (i.e. isFromNotification = true), it skips the
+// validation of policy usage and the attempt to delete in the backend as well
+// (as this is already done by the notifying node).
+func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string, isFromNotification bool) error {
 	if policy == "" {
 		return errInvalidArgument
 	}
@@ -1156,42 +1178,44 @@ func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string) error
 	cache := store.lock()
 	defer store.unlock()
 
-	// Check if policy is mapped to any existing user or group. If so, we do not
-	// allow deletion of the policy. If the policy is mapped to an STS account,
-	// we do allow deletion.
-	users := []string{}
-	groups := []string{}
-	for u, mp := range cache.iamUserPolicyMap {
-		pset := mp.policySet()
-		if store.getUsersSysType() == MinIOUsersSysType {
-			if _, ok := cache.iamUsersMap[u]; !ok {
-				// This case can happen when a temporary account is
-				// deleted or expired - remove it from userPolicyMap.
-				delete(cache.iamUserPolicyMap, u)
-				continue
+	if !isFromNotification {
+		// Check if policy is mapped to any existing user or group. If so, we do not
+		// allow deletion of the policy. If the policy is mapped to an STS account,
+		// we do allow deletion.
+		users := []string{}
+		groups := []string{}
+		for u, mp := range cache.iamUserPolicyMap {
+			pset := mp.policySet()
+			if store.getUsersSysType() == MinIOUsersSysType {
+				if _, ok := cache.iamUsersMap[u]; !ok {
+					// This case can happen when a temporary account is
+					// deleted or expired - remove it from userPolicyMap.
+					delete(cache.iamUserPolicyMap, u)
+					continue
+				}
+			}
+			if pset.Contains(policy) {
+				users = append(users, u)
 			}
 		}
-		if pset.Contains(policy) {
-			users = append(users, u)
+		for g, mp := range cache.iamGroupPolicyMap {
+			pset := mp.policySet()
+			if pset.Contains(policy) {
+				groups = append(groups, g)
+			}
 		}
-	}
-	for g, mp := range cache.iamGroupPolicyMap {
-		pset := mp.policySet()
-		if pset.Contains(policy) {
-			groups = append(groups, g)
+		if len(users) != 0 || len(groups) != 0 {
+			return errPolicyInUse
 		}
-	}
-	if len(users) != 0 || len(groups) != 0 {
-		return errPolicyInUse
-	}
 
-	err := store.deletePolicyDoc(ctx, policy)
-	if errors.Is(err, errNoSuchPolicy) {
-		// Ignore error if policy is already deleted.
-		err = nil
-	}
-	if err != nil {
-		return err
+		err := store.deletePolicyDoc(ctx, policy)
+		if errors.Is(err, errNoSuchPolicy) {
+			// Ignore error if policy is already deleted.
+			err = nil
+		}
+		if err != nil {
+			return err
+		}
 	}
 
 	delete(cache.iamPolicyDocsMap, policy)

cmd/iam.go

@@ -45,6 +45,7 @@ import (
 	"github.com/minio/minio/internal/config/policy/opa"
 	polplugin "github.com/minio/minio/internal/config/policy/plugin"
 	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/policy"
@@ -207,7 +208,7 @@ func (sys *IAMSys) Load(ctx context.Context, firstTime bool) error {
 	select {
 	case <-sys.configLoaded:
 	default:
-		close(sys.configLoaded)
+		xioutil.SafeClose(sys.configLoaded)
 	}
 	return nil
 }
@@ -269,16 +270,13 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	setGlobalAuthZPlugin(polplugin.New(authZPluginCfg))
 
 	sys.Lock()
-	defer sys.Unlock()
-
 	sys.LDAPConfig = ldapConfig
 	sys.OpenIDConfig = openidConfig
 	sys.STSTLSConfig = stsTLSConfig
-
 	sys.iamRefreshInterval = iamRefreshInterval
-
 	// Initialize IAM store
 	sys.initStore(objAPI, etcdClient)
+	sys.Unlock()
 
 	retryCtx, cancel := context.WithCancel(ctx)
 
@@ -528,7 +526,9 @@ func (sys *IAMSys) GetRolePolicy(arnStr string) (arn.ARN, string, error) {
 	return roleArn, rolePolicy, nil
 }
 
-// DeletePolicy - deletes a canned policy from backend or etcd.
+// DeletePolicy - deletes a canned policy from backend. `notifyPeers` is true
+// whenever this is called via the API. It is false when called via a
+// notification from another peer. This is to avoid infinite loops.
 func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPeers bool) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
@@ -542,7 +542,7 @@ func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPe
 		}
 	}
 
-	err := sys.store.DeletePolicy(ctx, policyName)
+	err := sys.store.DeletePolicy(ctx, policyName, !notifyPeers)
 	if err != nil {
 		return err
 	}
@@ -940,6 +940,13 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 		return auth.Credentials{}, time.Time{}, errInvalidArgument
 	}
 
+	if len(opts.accessKey) > 0 && len(opts.secretKey) == 0 {
+		return auth.Credentials{}, time.Time{}, auth.ErrNoSecretKeyWithAccessKey
+	}
+	if len(opts.secretKey) > 0 && len(opts.accessKey) == 0 {
+		return auth.Credentials{}, time.Time{}, auth.ErrNoAccessKeyWithSecretKey
+	}
+
 	var policyBuf []byte
 	if opts.sessionPolicy != nil {
 		err := opts.sessionPolicy.Validate()
@@ -973,7 +980,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 		m[iamPolicyClaimNameSA()] = inheritedPolicyType
 	}
 
-	// Add all the necessary claims for the service accounts.
+	// Add all the necessary claims for the service account.
 	for k, v := range opts.claims {
 		_, ok := m[k]
 		if !ok {
@@ -983,7 +990,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 
 	var accessKey, secretKey string
 	var err error
-	if len(opts.accessKey) > 0 {
+	if len(opts.accessKey) > 0 || len(opts.secretKey) > 0 {
 		accessKey, secretKey = opts.accessKey, opts.secretKey
 	} else {
 		accessKey, secretKey, err = auth.GenerateCredentials()
@@ -1847,37 +1854,14 @@ func (sys *IAMSys) IsAllowedServiceAccount(args policy.Args, parentUser string)
 		return isOwnerDerived || combinedPolicy.IsAllowed(parentArgs)
 	}
 
-	// Now check if we have a sessionPolicy.
-	spolicy, ok := args.Claims[sessionPolicyNameExtracted]
-	if !ok {
-		return false
+	// 3. If an inline session-policy is present, evaluate it.
+	hasSessionPolicy, isAllowedSP := isAllowedBySessionPolicyForServiceAccount(args)
+	if hasSessionPolicy {
+		return isAllowedSP && (isOwnerDerived || combinedPolicy.IsAllowed(parentArgs))
 	}
 
-	spolicyStr, ok := spolicy.(string)
-	if !ok {
-		// Sub policy if set, should be a string reject
-		// malformed/malicious requests.
-		return false
-	}
-
-	// Check if policy is parseable.
-	subPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
-	if err != nil {
-		// Log any error in input session policy config.
-		logger.LogIf(GlobalContext, err)
-		return false
-	}
-
-	// This can only happen if policy was set but with an empty JSON.
-	if subPolicy.Version == "" && len(subPolicy.Statements) == 0 {
-		return isOwnerDerived || combinedPolicy.IsAllowed(parentArgs)
-	}
-
-	if subPolicy.Version == "" {
-		return false
-	}
-
-	return subPolicy.IsAllowed(parentArgs) && (isOwnerDerived || combinedPolicy.IsAllowed(parentArgs))
+	// Sub policy not set. Evaluate only the parent policies.
+	return (isOwnerDerived || combinedPolicy.IsAllowed(parentArgs))
 }
 
 // IsAllowedSTS is meant for STS based temporary credentials,
@@ -1967,6 +1951,67 @@ func (sys *IAMSys) IsAllowedSTS(args policy.Args, parentUser string) bool {
 	return isOwnerDerived || combinedPolicy.IsAllowed(args)
 }
 
+func isAllowedBySessionPolicyForServiceAccount(args policy.Args) (hasSessionPolicy bool, isAllowed bool) {
+	hasSessionPolicy = false
+	isAllowed = false
+
+	// Now check if we have a sessionPolicy.
+	spolicy, ok := args.Claims[sessionPolicyNameExtracted]
+	if !ok {
+		return
+	}
+
+	hasSessionPolicy = true
+
+	spolicyStr, ok := spolicy.(string)
+	if !ok {
+		// Sub policy if set, should be a string reject
+		// malformed/malicious requests.
+		return
+	}
+
+	// Check if policy is parseable.
+	subPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
+	if err != nil {
+		// Log any error in input session policy config.
+		logger.LogIf(GlobalContext, err)
+		return
+	}
+
+	// SPECIAL CASE: For service accounts, any valid JSON is allowed as a
+	// policy, regardless of whether the number of statements is 0, this
+	// includes `null`, `{}` and `{"Statement": null}`. In fact, MinIO Console
+	// sends `null` when no policy is set and the intended behavior is that the
+	// service account should inherit parent policy.
+	//
+	// However, for a policy like `{"Statement":[]}`, the intention is to not
+	// provide any permissions via the session policy - i.e. the service account
+	// can do nothing (such a JSON could be generated by an external application
+	// as the policy for the service account). Inheriting the parent policy in
+	// such a case, is a security issue. Ideally, we should not allow such
+	// behavior, but for compatibility with the Console, we currently allow it.
+	//
+	// TODO:
+	//
+	// 1. fix console behavior and allow this inheritance for service accounts
+	// created before a certain (TBD) future date.
+	//
+	// 2. do not allow empty statement policies for service accounts.
+	if subPolicy.Version == "" && subPolicy.Statements == nil && subPolicy.ID == "" {
+		hasSessionPolicy = false
+		return
+	}
+
+	// As the session policy exists, even if the parent is the root account, it
+	// must be restricted by it. So, we set `.IsOwner` to false here
+	// unconditionally.
+	sessionPolicyArgs := args
+	sessionPolicyArgs.IsOwner = false
+
+	// Sub policy is set and valid.
+	return hasSessionPolicy, subPolicy.IsAllowed(sessionPolicyArgs)
+}
+
 func isAllowedBySessionPolicy(args policy.Args) (hasSessionPolicy bool, isAllowed bool) {
 	hasSessionPolicy = false
 	isAllowed = false
@@ -1999,8 +2044,14 @@ func isAllowedBySessionPolicy(args policy.Args) (hasSessionPolicy bool, isAllowe
 		return
 	}
 
+	// As the session policy exists, even if the parent is the root account, it
+	// must be restricted by it. So, we set `.IsOwner` to false here
+	// unconditionally.
+	sessionPolicyArgs := args
+	sessionPolicyArgs.IsOwner = false
+
 	// Sub policy is set and valid.
-	return hasSessionPolicy, subPolicy.IsAllowed(args)
+	return hasSessionPolicy, subPolicy.IsAllowed(sessionPolicyArgs)
 }
 
 // GetCombinedPolicy returns a combined policy combining all policies
@@ -2009,6 +2060,13 @@ func (sys *IAMSys) GetCombinedPolicy(policies ...string) policy.Policy {
 	return policy
 }
 
+// doesPolicyAllow - checks if the given policy allows the passed action with given args. This is rarely needed.
+// Notice there is no account name involved, so this is a dangerous function.
+func (sys *IAMSys) doesPolicyAllow(policy string, args policy.Args) bool {
+	// Policies were found, evaluate all of them.
+	return sys.GetCombinedPolicy(policy).IsAllowed(args)
+}
+
 // IsAllowed - checks given policy args is allowed to continue the Rest API.
 func (sys *IAMSys) IsAllowed(args policy.Args) bool {
 	// If opa is configured, use OPA always.

cmd/listen-notification-handlers.go

@@ -18,12 +18,14 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"net/http"
 	"strconv"
 	"time"
 
 	"github.com/minio/minio/internal/event"
+	"github.com/minio/minio/internal/grid"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/pubsub"
 	"github.com/minio/mux"
@@ -116,11 +118,32 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 
 	// Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	listenCh := make(chan event.Event, globalAPIConfig.getRequestsPoolCapacity()*len(globalEndpoints.Hostnames()))
+	mergeCh := make(chan []byte, globalAPIConfig.getRequestsPoolCapacity()*len(globalEndpoints.Hostnames()))
+	localCh := make(chan event.Event, globalAPIConfig.getRequestsPoolCapacity())
 
+	// Convert local messages to JSON and send to mergeCh
+	go func() {
+		buf := bytes.NewBuffer(grid.GetByteBuffer()[:0])
+		enc := json.NewEncoder(buf)
+		tmpEvt := struct{ Records []event.Event }{[]event.Event{{}}}
+		for {
+			select {
+			case ev := <-localCh:
+				buf.Reset()
+				tmpEvt.Records[0] = ev
+				if err := enc.Encode(tmpEvt); err != nil {
+					logger.LogOnceIf(ctx, err, "event: Encode failed")
+					continue
+				}
+				mergeCh <- append(grid.GetByteBuffer()[:0], buf.Bytes()...)
+			case <-ctx.Done():
+				grid.PutByteBuffer(buf.Bytes())
+				return
+			}
+		}
+	}()
 	peers, _ := newPeerRestClients(globalEndpoints)
-
-	err := globalHTTPListen.Subscribe(mask, listenCh, ctx.Done(), func(ev event.Event) bool {
+	err := globalHTTPListen.Subscribe(mask, localCh, ctx.Done(), func(ev event.Event) bool {
 		if ev.S3.Bucket.Name != "" && bucketName != "" {
 			if ev.S3.Bucket.Name != bucketName {
 				return false
@@ -139,7 +162,7 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 		if peer == nil {
 			continue
 		}
-		peer.Listen(listenCh, ctx.Done(), values)
+		peer.Listen(ctx, mergeCh, values)
 	}
 
 	var (
@@ -170,14 +193,16 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 	enc := json.NewEncoder(w)
 	for {
 		select {
-		case ev := <-listenCh:
-			if err := enc.Encode(struct{ Records []event.Event }{[]event.Event{ev}}); err != nil {
+		case ev := <-mergeCh:
+			_, err := w.Write(ev)
+			if err != nil {
 				return
 			}
-			if len(listenCh) == 0 {
+			if len(mergeCh) == 0 {
 				// Flush if nothing is queued
 				w.(http.Flusher).Flush()
 			}
+			grid.PutByteBuffer(ev)
 		case <-emptyEventTicker:
 			if err := enc.Encode(struct{ Records []event.Event }{}); err != nil {
 				return

cmd/metacache-entries.go

@@ -26,6 +26,7 @@ import (
 	"sort"
 	"strings"
 
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 )
@@ -659,7 +660,7 @@ func (m *metaCacheEntriesSorted) forwardPast(s string) {
 // If the context is canceled the function will return the error,
 // otherwise the function will return nil.
 func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<- metaCacheEntry, readQuorum int) error {
-	defer close(out)
+	defer xioutil.SafeClose(out)
 	top := make([]*metaCacheEntry, len(in))
 	nDone := 0
 	ctxDone := ctx.Done()

cmd/metacache-server-pool.go

@@ -28,6 +28,7 @@ import (
 	"sync"
 	"time"
 
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 )
 
@@ -57,9 +58,10 @@ func renameAllBucketMetacache(epPath string) error {
 // Other important fields are Limit, Marker.
 // List ID always derived from the Marker.
 func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (entries metaCacheEntriesSorted, err error) {
-	if err := checkListObjsArgs(ctx, o.Bucket, o.Prefix, o.Marker, z); err != nil {
+	if err := checkListObjsArgs(ctx, o.Bucket, o.Prefix, o.Marker); err != nil {
 		return entries, err
 	}
+
 	// Marker points to before the prefix, just ignore it.
 	if o.Marker < o.Prefix {
 		o.Marker = ""
@@ -303,6 +305,15 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 
 	cancelList()
 	wg.Wait()
+
+	// we should return 'errs' from per disk
+	if isAllNotFound(errs) {
+		if isAllVolumeNotFound(errs) {
+			return errVolumeNotFound
+		}
+		return nil
+	}
+
 	if err != nil {
 		return err
 	}
@@ -311,10 +322,6 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 		return ctx.Err()
 	}
 
-	if isAllNotFound(errs) {
-		return nil
-	}
-
 	for _, err := range errs {
 		if errors.Is(err, io.EOF) {
 			continue
@@ -339,7 +346,7 @@ func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions,
 // When 'in' is closed or the context is canceled the
 // function closes 'out' and exits.
 func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCacheEntry, out chan<- metaCacheEntry) {
-	defer close(out)
+	defer xioutil.SafeClose(out)
 
 	for {
 		var obj metaCacheEntry
@@ -472,16 +479,16 @@ func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions
 				funcReturnedMu.Unlock()
 				outCh <- entry
 				if returned {
-					close(outCh)
+					xioutil.SafeClose(outCh)
 				}
 			}
 			entry.reusable = returned
 			saveCh <- entry
 		}
 		if !returned {
-			close(outCh)
+			xioutil.SafeClose(outCh)
 		}
-		close(saveCh)
+		xioutil.SafeClose(saveCh)
 	}()
 
 	return filteredResults()

cmd/metacache-set.go

@@ -37,6 +37,7 @@ import (
 	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/hash"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 )
@@ -426,7 +427,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 				if !disk.IsOnline() {
 					continue
 				}
-				_, err := disk.ReadVersion(ctx, minioMetaBucket,
+				_, err := disk.ReadVersion(ctx, "", minioMetaBucket,
 					o.objectPath(0), "", ReadOptions{})
 				if err != nil {
 					time.Sleep(retryDelay250)
@@ -503,7 +504,7 @@ func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOpt
 						if !disk.IsOnline() {
 							continue
 						}
-						_, err := disk.ReadVersion(ctx, minioMetaBucket,
+						_, err := disk.ReadVersion(ctx, "", minioMetaBucket,
 							o.objectPath(partN), "", ReadOptions{})
 						if err != nil {
 							time.Sleep(retryDelay250)
@@ -679,7 +680,7 @@ func getQuorumDisks(disks []StorageAPI, infos []DiskInfo, readQuorum int) (newDi
 
 // Will return io.EOF if continuing would not yield more results.
 func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) (err error) {
-	defer close(results)
+	defer xioutil.SafeClose(results)
 	o.debugf(color.Green("listPath:")+" with options: %#v", o)
 
 	// get prioritized non-healing disks for listing
@@ -1056,7 +1057,7 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 	for {
 		// Get the top entry from each
 		var current metaCacheEntry
-		var atEOF, fnf, hasErr, agree int
+		var atEOF, fnf, vnf, hasErr, agree int
 		for i := range topEntries {
 			topEntries[i] = metaCacheEntry{}
 		}
@@ -1082,6 +1083,11 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 					errDiskNotFound.Error():
 					atEOF++
 					fnf++
+					// This is a special case, to handle bucket does
+					// not exist situations.
+					if errors.Is(err, errVolumeNotFound) {
+						vnf++
+					}
 					continue
 				}
 				hasErr++
@@ -1139,6 +1145,10 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 			return errors.New(strings.Join(combinedErr, ", "))
 		}
 
+		if vnf == len(readers) {
+			return errVolumeNotFound
+		}
+
 		// Break if all at EOF or error.
 		if atEOF+hasErr == len(readers) {
 			if hasErr > 0 && opts.finished != nil {
@@ -1146,9 +1156,11 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 			}
 			break
 		}
+
 		if fnf == len(readers) {
 			return errFileNotFound
 		}
+
 		if agree == len(readers) {
 			// Everybody agreed
 			for _, r := range readers {

cmd/metacache-stream.go

@@ -27,6 +27,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/s2"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
 	"github.com/valyala/bytebufferpool"
@@ -550,7 +551,7 @@ func (r *metacacheReader) readAll(ctx context.Context, dst chan<- metaCacheEntry
 	if r.err != nil {
 		return r.err
 	}
-	defer close(dst)
+	defer xioutil.SafeClose(dst)
 	if r.current.name != "" {
 		select {
 		case <-ctx.Done():

cmd/metacache-walk.go

@@ -73,12 +73,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	if !skipAccessChecks(opts.Bucket) {
 		// Stat a volume entry.
 		if err = Access(volumeDir); err != nil {
-			if osIsNotExist(err) {
-				return errVolumeNotFound
-			} else if isSysErrIO(err) {
-				return errFaultyDisk
-			}
-			return err
+			return convertAccessError(err, errVolumeAccessDenied)
 		}
 	}
 
@@ -94,7 +89,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 	if err != nil {
 		return err
 	}
-	defer close(out)
+	defer xioutil.SafeClose(out)
 	var objsReturned int
 
 	objReturned := func(metadata []byte) {
@@ -169,7 +164,7 @@ func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writ
 		if s.walkMu != nil {
 			s.walkMu.Lock()
 		}
-		entries, err := s.ListDir(ctx, opts.Bucket, current, -1)
+		entries, err := s.ListDir(ctx, "", opts.Bucket, current, -1)
 		if s.walkMu != nil {
 			s.walkMu.Unlock()
 		}
@@ -407,12 +402,12 @@ func (client *storageRESTClient) WalkDir(ctx context.Context, opts WalkDirOption
 	opts.DiskID = client.diskID
 	b, err := opts.MarshalMsg(grid.GetByteBuffer()[:0])
 	if err != nil {
-		return err
+		return toStorageErr(err)
 	}
 
 	st, err := client.gridConn.NewStream(ctx, grid.HandlerWalkDir, b)
 	if err != nil {
-		return err
+		return toStorageErr(err)
 	}
 	return toStorageErr(st.Results(func(in []byte) error {
 		_, err := wr.Write(in)

cmd/metrics-resource.go

@@ -83,6 +83,11 @@ var (
 	// resourceMetricsHelpMap maps metric name to its help string
 	resourceMetricsHelpMap map[MetricName]string
 	resourceMetricsGroups  []*MetricsGroup
+	// initial values for drives (at the time  of server startup)
+	// used for calculating avg values for drive metrics
+	initialDriveStats   map[string]madmin.DiskIOStats
+	initialDriveStatsMu sync.RWMutex
+	initialUptime       uint64
 )
 
 // PeerResourceMetrics represents the resource metrics
@@ -214,41 +219,58 @@ func updateResourceMetrics(subSys MetricSubsystem, name MetricName, val float64,
 	resourceMetricsMap[subSys] = subsysMetrics
 }
 
-func collectDriveMetrics(m madmin.RealtimeMetrics) {
-	upt, _ := host.Uptime()
-	kib := 1 << 10
+// updateDriveIOStats - Updates the drive IO stats by calculating the difference between the current
+// and initial values. We cannot rely on host.Uptime here as it will not work in k8s environments, where
+// it will return the pod's uptime but the disk metrics are always from the host (/proc/diskstats)
+func updateDriveIOStats(currentStats madmin.DiskIOStats, initialStats madmin.DiskIOStats, labels map[string]string) {
 	sectorSize := uint64(512)
+	kib := float64(1 << 10)
+	uptime, _ := host.Uptime()
+	uptimeDiff := float64(uptime - initialUptime)
+	if uptimeDiff == 0 {
+		// too soon to update the stats
+		return
+	}
+	diffStats := madmin.DiskIOStats{
+		ReadIOs:      currentStats.ReadIOs - initialStats.ReadIOs,
+		WriteIOs:     currentStats.WriteIOs - initialStats.WriteIOs,
+		ReadTicks:    currentStats.ReadTicks - initialStats.ReadTicks,
+		WriteTicks:   currentStats.WriteTicks - initialStats.WriteTicks,
+		TotalTicks:   currentStats.TotalTicks - initialStats.TotalTicks,
+		ReadSectors:  currentStats.ReadSectors - initialStats.ReadSectors,
+		WriteSectors: currentStats.WriteSectors - initialStats.WriteSectors,
+	}
 
+	updateResourceMetrics(driveSubsystem, readsPerSec, float64(diffStats.ReadIOs)/uptimeDiff, labels, false)
+	readKib := float64(diffStats.ReadSectors*sectorSize) / kib
+	updateResourceMetrics(driveSubsystem, readsKBPerSec, readKib/uptimeDiff, labels, false)
+
+	updateResourceMetrics(driveSubsystem, writesPerSec, float64(diffStats.WriteIOs)/uptimeDiff, labels, false)
+	writeKib := float64(diffStats.WriteSectors*sectorSize) / kib
+	updateResourceMetrics(driveSubsystem, writesKBPerSec, writeKib/uptimeDiff, labels, false)
+
+	rdAwait := 0.0
+	if diffStats.ReadIOs > 0 {
+		rdAwait = float64(diffStats.ReadTicks) / float64(diffStats.ReadIOs)
+	}
+	updateResourceMetrics(driveSubsystem, readsAwait, rdAwait, labels, false)
+
+	wrAwait := 0.0
+	if diffStats.WriteIOs > 0 {
+		wrAwait = float64(diffStats.WriteTicks) / float64(diffStats.WriteIOs)
+	}
+	updateResourceMetrics(driveSubsystem, writesAwait, wrAwait, labels, false)
+	updateResourceMetrics(driveSubsystem, percUtil, float64(diffStats.TotalTicks)/(uptimeDiff*10), labels, false)
+}
+
+func collectDriveMetrics(m madmin.RealtimeMetrics) {
 	for d, dm := range m.ByDisk {
-		stats := dm.IOStats
 		labels := map[string]string{"drive": d}
-		updateResourceMetrics(driveSubsystem, readsPerSec, float64(stats.ReadIOs)/float64(upt), labels, false)
-
-		readBytes := stats.ReadSectors * sectorSize
-		readKib := float64(readBytes) / float64(kib)
-		readKibPerSec := readKib / float64(upt)
-		updateResourceMetrics(driveSubsystem, readsKBPerSec, readKibPerSec, labels, false)
-
-		updateResourceMetrics(driveSubsystem, writesPerSec, float64(stats.WriteIOs)/float64(upt), labels, false)
-
-		writeBytes := stats.WriteSectors * sectorSize
-		writeKib := float64(writeBytes) / float64(kib)
-		writeKibPerSec := writeKib / float64(upt)
-		updateResourceMetrics(driveSubsystem, writesKBPerSec, writeKibPerSec, labels, false)
-
-		rdAwait := 0.0
-		if stats.ReadIOs > 0 {
-			rdAwait = float64(stats.ReadTicks) / float64(stats.ReadIOs)
+		initialStats, ok := initialDriveStats[d]
+		if !ok {
+			continue
 		}
-		updateResourceMetrics(driveSubsystem, readsAwait, rdAwait, labels, false)
-
-		wrAwait := 0.0
-		if stats.WriteIOs > 0 {
-			wrAwait = float64(stats.WriteTicks) / float64(stats.WriteIOs)
-		}
-		updateResourceMetrics(driveSubsystem, writesAwait, wrAwait, labels, false)
-
-		updateResourceMetrics(driveSubsystem, percUtil, float64(stats.TotalTicks)/float64(upt*10), labels, false)
+		updateDriveIOStats(dm.IOStats, initialStats, labels)
 	}
 
 	globalLocalDrivesMu.RLock()
@@ -256,8 +278,8 @@ func collectDriveMetrics(m madmin.RealtimeMetrics) {
 	globalLocalDrivesMu.RUnlock()
 
 	for _, d := range localDrives {
-		labels := map[string]string{"drive": d.Endpoint().RawPath}
-		di, err := d.DiskInfo(GlobalContext, false)
+		di, err := d.DiskInfo(GlobalContext, DiskInfoOptions{})
+		labels := map[string]string{"drive": di.Endpoint}
 		if err == nil {
 			updateResourceMetrics(driveSubsystem, usedBytes, float64(di.Used), labels, false)
 			updateResourceMetrics(driveSubsystem, totalBytes, float64(di.Total), labels, false)
@@ -339,8 +361,30 @@ func collectLocalResourceMetrics() {
 	collectDriveMetrics(m)
 }
 
+// populateInitialValues - populates the initial values
+// for drive stats and host uptime
+func populateInitialValues() {
+	initialDriveStatsMu.Lock()
+
+	m := collectLocalMetrics(madmin.MetricsDisk, collectMetricsOpts{
+		hosts: map[string]struct{}{
+			globalLocalNodeName: {},
+		},
+	})
+
+	initialDriveStats = map[string]madmin.DiskIOStats{}
+	for d, dm := range m.ByDisk {
+		initialDriveStats[d] = dm.IOStats
+	}
+
+	initialUptime, _ = host.Uptime()
+	initialDriveStatsMu.Unlock()
+}
+
 // startResourceMetricsCollection - starts the job for collecting resource metrics
 func startResourceMetricsCollection() {
+	populateInitialValues()
+
 	resourceMetricsMapMu.Lock()
 	resourceMetricsMap = map[MetricSubsystem]ResourceMetrics{}
 	resourceMetricsMapMu.Unlock()

cmd/metrics-v2.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2023 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -32,6 +32,7 @@ import (
 	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
 	"github.com/minio/minio/internal/rest"
@@ -59,13 +60,13 @@ func init() {
 		getClusterHealthMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
 		getIAMNodeMetrics(MetricsGroupOpts{dependGlobalAuthNPlugin: true, dependGlobalIAMSys: true}),
 		getReplicationSiteMetrics(MetricsGroupOpts{dependGlobalSiteReplicationSys: true}),
+		getBatchJobsMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
 	}
 
 	peerMetricsGroups = []*MetricsGroup{
 		getGoMetrics(),
 		getHTTPMetrics(MetricsGroupOpts{}),
 		getNotificationMetrics(MetricsGroupOpts{dependGlobalLambdaTargetList: true}),
-		getLocalStorageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
 		getMinioProcMetrics(),
 		getMinioVersionMetrics(),
 		getNetworkMetrics(),
@@ -76,7 +77,7 @@ func init() {
 		getKMSNodeMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependGlobalKMS: true}),
 		getMinioHealingMetrics(MetricsGroupOpts{dependGlobalBackgroundHealState: true}),
 		getWebhookMetrics(),
-		getReplicationClusterMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependBucketTargetSys: true}),
+		getTierMetrics(),
 	}
 
 	allMetricsGroups := func() (allMetrics []*MetricsGroup) {
@@ -96,13 +97,13 @@ func init() {
 		getDistLockMetrics(MetricsGroupOpts{dependGlobalIsDistErasure: true, dependGlobalLockServer: true}),
 		getIAMNodeMetrics(MetricsGroupOpts{dependGlobalAuthNPlugin: true, dependGlobalIAMSys: true}),
 		getLocalStorageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
+		getReplicationNodeMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependBucketTargetSys: true}),
 	}
 
 	bucketMetricsGroups := []*MetricsGroup{
 		getBucketUsageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
 		getHTTPMetrics(MetricsGroupOpts{bucketOnly: true}),
 		getBucketTTFBMetric(),
-		getBatchJobsMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}),
 	}
 
 	bucketPeerMetricsGroups = []*MetricsGroup{
@@ -224,10 +225,20 @@ const (
 	linkOfflineDuration       MetricName = "link_offline_duration_seconds"
 	linkDowntimeTotalDuration MetricName = "link_downtime_duration_seconds"
 
-	avgInQueueCount MetricName = "average_queued_count"
-	avgInQueueBytes MetricName = "average_queued_bytes"
-	maxInQueueCount MetricName = "max_queued_count"
-	maxInQueueBytes MetricName = "max_queued_bytes"
+	avgInQueueCount                     MetricName = "average_queued_count"
+	avgInQueueBytes                     MetricName = "average_queued_bytes"
+	maxInQueueCount                     MetricName = "max_queued_count"
+	maxInQueueBytes                     MetricName = "max_queued_bytes"
+	proxiedGetRequestsTotal             MetricName = "proxied_get_requests_total"
+	proxiedHeadRequestsTotal            MetricName = "proxied_head_requests_total"
+	proxiedPutTaggingRequestsTotal      MetricName = "proxied_put_tagging_requests_total"
+	proxiedGetTaggingRequestsTotal      MetricName = "proxied_get_tagging_requests_total"
+	proxiedDeleteTaggingRequestsTotal   MetricName = "proxied_delete_tagging_requests_total"
+	proxiedGetRequestsFailures          MetricName = "proxied_get_requests_failures"
+	proxiedHeadRequestsFailures         MetricName = "proxied_head_requests_failures"
+	proxiedPutTaggingRequestFailures    MetricName = "proxied_put_tagging_requests_failures"
+	proxiedGetTaggingRequestFailures    MetricName = "proxied_get_tagging_requests_failures"
+	proxiedDeleteTaggingRequestFailures MetricName = "proxied_delete_tagging_requests_failures"
 
 	freeBytes       MetricName = "free_bytes"
 	readBytes       MetricName = "read_bytes"
@@ -256,6 +267,7 @@ const (
 	startTime        = "starttime_seconds"
 	upTime           = "uptime_seconds"
 	memory           = "resident_memory_bytes"
+	vmemory          = "virtual_memory_bytes"
 	cpu              = "cpu_total_seconds"
 
 	expiryPendingTasks     MetricName = "expiry_pending_tasks"
@@ -519,7 +531,7 @@ func getNodeDriveTimeoutErrorsMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: driveSubsystem,
 		Name:      "errors_timeout",
-		Help:      "Total number of timeout errors since server start",
+		Help:      "Total number of drive timeout errors since server start",
 		Type:      counterMetric,
 	}
 }
@@ -529,7 +541,17 @@ func getNodeDriveAvailablityErrorsMD() MetricDescription {
 		Namespace: nodeMetricNamespace,
 		Subsystem: driveSubsystem,
 		Name:      "errors_availability",
-		Help:      "Total number of I/O errors, permission denied and timeouts since server start",
+		Help:      "Total number of drive I/O errors, permission denied and timeouts since server start",
+		Type:      counterMetric,
+	}
+}
+
+func getNodeDriveWaitingIOMD() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: driveSubsystem,
+		Name:      "io_waiting",
+		Help:      "Total number I/O operations waiting on drive",
 		Type:      counterMetric,
 	}
 }
@@ -1102,6 +1124,106 @@ func getClusterReplMaxTransferRateMD() MetricDescription {
 	}
 }
 
+func getClusterReplProxiedGetOperationsMD(ns MetricNamespace) MetricDescription {
+	return MetricDescription{
+		Namespace: ns,
+		Subsystem: replicationSubsystem,
+		Name:      proxiedGetRequestsTotal,
+		Help:      "Number of GET requests proxied to replication target",
+		Type:      counterMetric,
+	}
+}
+
+func getClusterReplProxiedHeadOperationsMD(ns MetricNamespace) MetricDescription {
+	return MetricDescription{
+		Namespace: ns,
+		Subsystem: replicationSubsystem,
+		Name:      proxiedHeadRequestsTotal,
+		Help:      "Number of HEAD requests proxied to replication target",
+		Type:      counterMetric,
+	}
+}
+
+func getClusterReplProxiedPutTaggingOperationsMD(ns MetricNamespace) MetricDescription {
+	return MetricDescription{
+		Namespace: ns,
+		Subsystem: replicationSubsystem,
+		Name:      proxiedPutTaggingRequestsTotal,
+		Help:      "Number of PUT tagging requests proxied to replication target",
+		Type:      counterMetric,
+	}
+}
+
+func getClusterReplProxiedGetTaggingOperationsMD(ns MetricNamespace) MetricDescription {
+	return MetricDescription{
+		Namespace: ns,
+		Subsystem: replicationSubsystem,
+		Name:      proxiedGetTaggingRequestsTotal,
+		Help:      "Number of GET tagging requests proxied to replication target",
+		Type:      counterMetric,
+	}
+}
+
+func getClusterReplProxiedRmvTaggingOperationsMD(ns MetricNamespace) MetricDescription {
+	return MetricDescription{
+		Namespace: ns,
+		Subsystem: replicationSubsystem,
+		Name:      proxiedDeleteTaggingRequestsTotal,
+		Help:      "Number of DELETE tagging requests proxied to replication target",
+		Type:      counterMetric,
+	}
+}
+
+func getClusterReplProxiedGetFailedOperationsMD(ns MetricNamespace) MetricDescription {
+	return MetricDescription{
+		Namespace: ns,
+		Subsystem: replicationSubsystem,
+		Name:      proxiedGetRequestsFailures,
+		Help:      "Number of failures in GET requests proxied to replication target",
+		Type:      counterMetric,
+	}
+}
+
+func getClusterReplProxiedHeadFailedOperationsMD(ns MetricNamespace) MetricDescription {
+	return MetricDescription{
+		Namespace: ns,
+		Subsystem: replicationSubsystem,
+		Name:      proxiedHeadRequestsFailures,
+		Help:      "Number of failures in HEAD requests proxied to replication target",
+		Type:      counterMetric,
+	}
+}
+
+func getClusterReplProxiedPutTaggingFailedOperationsMD(ns MetricNamespace) MetricDescription {
+	return MetricDescription{
+		Namespace: ns,
+		Subsystem: replicationSubsystem,
+		Name:      proxiedPutTaggingRequestFailures,
+		Help:      "Number of failures in PUT tagging proxy requests to replication target",
+		Type:      counterMetric,
+	}
+}
+
+func getClusterReplProxiedGetTaggingFailedOperationsMD(ns MetricNamespace) MetricDescription {
+	return MetricDescription{
+		Namespace: ns,
+		Subsystem: replicationSubsystem,
+		Name:      proxiedGetTaggingRequestFailures,
+		Help:      "Number of failures in GET tagging proxy requests to replication target",
+		Type:      counterMetric,
+	}
+}
+
+func getClusterReplProxiedRmvTaggingFailedOperationsMD(ns MetricNamespace) MetricDescription {
+	return MetricDescription{
+		Namespace: ns,
+		Subsystem: replicationSubsystem,
+		Name:      proxiedDeleteTaggingRequestFailures,
+		Help:      "Number of  failures in DELETE tagging proxy requests to replication target",
+		Type:      counterMetric,
+	}
+}
+
 func getBucketObjectDistributionMD() MetricDescription {
 	return MetricDescription{
 		Namespace: bucketMetricNamespace,
@@ -1532,6 +1654,16 @@ func getMinIOProcessResidentMemory() MetricDescription {
 	}
 }
 
+func getMinIOProcessVirtualMemory() MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: processSubsystem,
+		Name:      memory,
+		Help:      "Virtual memory size in bytes",
+		Type:      gaugeMetric,
+	}
+}
+
 func getMinIOProcessCPUTime() MetricDescription {
 	return MetricDescription{
 		Namespace: nodeMetricNamespace,
@@ -1654,6 +1786,14 @@ func getMinioProcMetrics() *MetricsGroup {
 				})
 		}
 
+		if stat.VirtualMemory() > 0 {
+			metrics = append(metrics,
+				Metric{
+					Description: getMinIOProcessVirtualMemory(),
+					Value:       float64(stat.VirtualMemory()),
+				})
+		}
+
 		if stat.CPUTime() > 0 {
 			metrics = append(metrics,
 				Metric{
@@ -1685,7 +1825,7 @@ func getGoMetrics() *MetricsGroup {
 func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription) []Metric {
 	ch := make(chan prometheus.Metric)
 	go func() {
-		defer close(ch)
+		defer xioutil.SafeClose(ch)
 		// Collects prometheus metrics from hist and sends it over ch
 		hist.Collect(ch)
 	}()
@@ -2097,7 +2237,7 @@ func getIAMNodeMetrics(opts MetricsGroupOpts) *MetricsGroup {
 }
 
 // replication metrics for each node - published to the cluster endpoint with nodename as label
-func getReplicationClusterMetrics(opts MetricsGroupOpts) *MetricsGroup {
+func getReplicationNodeMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	mg := &MetricsGroup{
 		cacheInterval:    1 * time.Minute,
 		metricsGroupOpts: opts,
@@ -2114,58 +2254,45 @@ func getReplicationClusterMetrics(opts MetricsGroupOpts) *MetricsGroup {
 		if globalReplicationStats != nil {
 			qs := globalReplicationStats.getNodeQueueStatsSummary()
 			activeWorkersCount := Metric{
-				Description:    getClusterReplActiveWorkersCountMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplActiveWorkersCountMD(),
 			}
 			avgActiveWorkersCount := Metric{
-				Description:    getClusterReplAvgActiveWorkersCountMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplAvgActiveWorkersCountMD(),
 			}
 			maxActiveWorkersCount := Metric{
-				Description:    getClusterReplMaxActiveWorkersCountMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplMaxActiveWorkersCountMD(),
 			}
 			currInQueueCount := Metric{
-				Description:    getClusterReplCurrQueuedOperationsMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplCurrQueuedOperationsMD(),
 			}
 			currInQueueBytes := Metric{
-				Description:    getClusterReplCurrQueuedBytesMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplCurrQueuedBytesMD(),
 			}
 
 			currTransferRate := Metric{
-				Description:    getClusterReplCurrentTransferRateMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplCurrentTransferRateMD(),
 			}
 			avgQueueCount := Metric{
-				Description:    getClusterReplAvgQueuedOperationsMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplAvgQueuedOperationsMD(),
 			}
 			avgQueueBytes := Metric{
-				Description:    getClusterReplAvgQueuedBytesMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplAvgQueuedBytesMD(),
 			}
 			maxQueueCount := Metric{
-				Description:    getClusterReplMaxQueuedOperationsMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplMaxQueuedOperationsMD(),
 			}
 			maxQueueBytes := Metric{
-				Description:    getClusterReplMaxQueuedBytesMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplMaxQueuedBytesMD(),
 			}
 			avgTransferRate := Metric{
-				Description:    getClusterReplAvgTransferRateMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplAvgTransferRateMD(),
 			}
 			maxTransferRate := Metric{
-				Description:    getClusterReplMaxTransferRateMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Description: getClusterReplMaxTransferRateMD(),
 			}
 			mrfCount := Metric{
-				Description:    getClusterReplMRFFailedOperationsMD(),
-				VariableLabels: map[string]string{serverName: qs.NodeName},
-				Value:          float64(qs.MRFStats.LastFailedCount),
+				Description: getClusterReplMRFFailedOperationsMD(),
+				Value:       float64(qs.MRFStats.LastFailedCount),
 			}
 
 			if qs.QStats.Avg.Count > 0 || qs.QStats.Curr.Count > 0 {
@@ -2209,7 +2336,6 @@ func getReplicationClusterMetrics(opts MetricsGroupOpts) *MetricsGroup {
 				Description: getClusterRepLinkLatencyCurrMD(),
 				VariableLabels: map[string]string{
 					"endpoint": ep,
-					serverName: globalLocalNodeName,
 				},
 			}
 			m.Value = float64(health.latency.curr / time.Millisecond)
@@ -2220,7 +2346,6 @@ func getReplicationClusterMetrics(opts MetricsGroupOpts) *MetricsGroup {
 				Description: getClusterRepLinkLatencyAvgMD(),
 				VariableLabels: map[string]string{
 					"endpoint": ep,
-					serverName: globalLocalNodeName,
 				},
 			}
 			m.Value = float64(health.latency.avg / time.Millisecond)
@@ -2231,7 +2356,6 @@ func getReplicationClusterMetrics(opts MetricsGroupOpts) *MetricsGroup {
 				Description: getClusterRepLinkLatencyMaxMD(),
 				VariableLabels: map[string]string{
 					"endpoint": ep,
-					serverName: globalLocalNodeName,
 				},
 			}
 			m.Value = float64(health.latency.peak / time.Millisecond)
@@ -2241,7 +2365,6 @@ func getReplicationClusterMetrics(opts MetricsGroupOpts) *MetricsGroup {
 				Description: getClusterRepLinkOnlineMD(),
 				VariableLabels: map[string]string{
 					"endpoint": ep,
-					serverName: globalLocalNodeName,
 				},
 			}
 			online := Offline
@@ -2254,7 +2377,6 @@ func getReplicationClusterMetrics(opts MetricsGroupOpts) *MetricsGroup {
 				Description: getClusterRepLinkCurrOfflineDurationMD(),
 				VariableLabels: map[string]string{
 					"endpoint": ep,
-					serverName: globalLocalNodeName,
 				},
 			}
 			currDowntime := time.Duration(0)
@@ -2268,7 +2390,6 @@ func getReplicationClusterMetrics(opts MetricsGroupOpts) *MetricsGroup {
 				Description: getClusterRepLinkTotalOfflineDurationMD(),
 				VariableLabels: map[string]string{
 					"endpoint": ep,
-					serverName: globalLocalNodeName,
 				},
 			}
 			dwntime := currDowntime
@@ -2360,6 +2481,46 @@ func getReplicationSiteMetrics(opts MetricsGroupOpts) *MetricsGroup {
 					})
 				}
 			}
+			ml = append(ml, Metric{
+				Description: getClusterReplProxiedGetOperationsMD(clusterMetricNamespace),
+				Value:       float64(m.Proxied.GetTotal),
+			})
+			ml = append(ml, Metric{
+				Description: getClusterReplProxiedHeadOperationsMD(clusterMetricNamespace),
+				Value:       float64(m.Proxied.HeadTotal),
+			})
+			ml = append(ml, Metric{
+				Description: getClusterReplProxiedPutTaggingOperationsMD(clusterMetricNamespace),
+				Value:       float64(m.Proxied.PutTagTotal),
+			})
+			ml = append(ml, Metric{
+				Description: getClusterReplProxiedGetTaggingOperationsMD(clusterMetricNamespace),
+				Value:       float64(m.Proxied.GetTagTotal),
+			})
+			ml = append(ml, Metric{
+				Description: getClusterReplProxiedRmvTaggingOperationsMD(clusterMetricNamespace),
+				Value:       float64(m.Proxied.RmvTagTotal),
+			})
+			ml = append(ml, Metric{
+				Description: getClusterReplProxiedGetFailedOperationsMD(clusterMetricNamespace),
+				Value:       float64(m.Proxied.GetFailedTotal),
+			})
+			ml = append(ml, Metric{
+				Description: getClusterReplProxiedHeadFailedOperationsMD(clusterMetricNamespace),
+				Value:       float64(m.Proxied.HeadFailedTotal),
+			})
+			ml = append(ml, Metric{
+				Description: getClusterReplProxiedPutTaggingFailedOperationsMD(clusterMetricNamespace),
+				Value:       float64(m.Proxied.PutTagFailedTotal),
+			})
+			ml = append(ml, Metric{
+				Description: getClusterReplProxiedGetTaggingFailedOperationsMD(clusterMetricNamespace),
+				Value:       float64(m.Proxied.GetTagFailedTotal),
+			})
+			ml = append(ml, Metric{
+				Description: getClusterReplProxiedRmvTaggingFailedOperationsMD(clusterMetricNamespace),
+				Value:       float64(m.Proxied.RmvTagFailedTotal),
+			})
 		}
 
 		return ml
@@ -2900,6 +3061,9 @@ func getClusterUsageMetrics(opts MetricsGroupOpts) *MetricsGroup {
 	}
 	mg.RegisterRead(func(ctx context.Context) (metrics []Metric) {
 		objLayer := newObjectLayerFn()
+		if objLayer == nil {
+			return
+		}
 
 		metrics = make([]Metric, 0, 50)
 		dataUsageInfo, err := loadDataUsageFromBackend(ctx, objLayer)
@@ -3074,6 +3238,51 @@ func getBucketUsageMetrics(opts MetricsGroupOpts) *MetricsGroup {
 						Value:          float64(stats.ReplicaCount),
 						VariableLabels: map[string]string{"bucket": bucket},
 					})
+					metrics = append(metrics, Metric{
+						Description:    getClusterReplProxiedGetOperationsMD(bucketMetricNamespace),
+						Value:          float64(s.ProxyStats.GetTotal),
+						VariableLabels: map[string]string{"bucket": bucket},
+					})
+					metrics = append(metrics, Metric{
+						Description:    getClusterReplProxiedHeadOperationsMD(bucketMetricNamespace),
+						Value:          float64(s.ProxyStats.HeadTotal),
+						VariableLabels: map[string]string{"bucket": bucket},
+					})
+					metrics = append(metrics, Metric{
+						Description:    getClusterReplProxiedPutTaggingOperationsMD(bucketMetricNamespace),
+						Value:          float64(s.ProxyStats.PutTagTotal),
+						VariableLabels: map[string]string{"bucket": bucket},
+					})
+					metrics = append(metrics, Metric{
+						Description:    getClusterReplProxiedGetTaggingOperationsMD(bucketMetricNamespace),
+						Value:          float64(s.ProxyStats.GetTagTotal),
+						VariableLabels: map[string]string{"bucket": bucket},
+					})
+					metrics = append(metrics, Metric{
+						Description:    getClusterReplProxiedRmvTaggingOperationsMD(bucketMetricNamespace),
+						Value:          float64(s.ProxyStats.RmvTagTotal),
+						VariableLabels: map[string]string{"bucket": bucket},
+					})
+					metrics = append(metrics, Metric{
+						Description: getClusterReplProxiedGetFailedOperationsMD(bucketMetricNamespace),
+						Value:       float64(s.ProxyStats.GetFailedTotal),
+					})
+					metrics = append(metrics, Metric{
+						Description: getClusterReplProxiedHeadFailedOperationsMD(bucketMetricNamespace),
+						Value:       float64(s.ProxyStats.HeadFailedTotal),
+					})
+					metrics = append(metrics, Metric{
+						Description: getClusterReplProxiedPutTaggingFailedOperationsMD(bucketMetricNamespace),
+						Value:       float64(s.ProxyStats.PutTagFailedTotal),
+					})
+					metrics = append(metrics, Metric{
+						Description: getClusterReplProxiedGetTaggingFailedOperationsMD(bucketMetricNamespace),
+						Value:       float64(s.ProxyStats.GetTagFailedTotal),
+					})
+					metrics = append(metrics, Metric{
+						Description: getClusterReplProxiedRmvTaggingFailedOperationsMD(bucketMetricNamespace),
+						Value:       float64(s.ProxyStats.RmvTagFailedTotal),
+					})
 				}
 				if stats.hasReplicationUsage() {
 					for arn, stat := range stats.Stats {
@@ -3132,20 +3341,20 @@ func getBucketUsageMetrics(opts MetricsGroupOpts) *MetricsGroup {
 						}
 					}
 				}
-				metrics = append(metrics, Metric{
-					Description:          getBucketObjectDistributionMD(),
-					Histogram:            usage.ObjectSizesHistogram,
-					HistogramBucketLabel: "range",
-					VariableLabels:       map[string]string{"bucket": bucket},
-				})
-
-				metrics = append(metrics, Metric{
-					Description:          getBucketObjectVersionsMD(),
-					Histogram:            usage.ObjectVersionsHistogram,
-					HistogramBucketLabel: "range",
-					VariableLabels:       map[string]string{"bucket": bucket},
-				})
 			}
+			metrics = append(metrics, Metric{
+				Description:          getBucketObjectDistributionMD(),
+				Histogram:            usage.ObjectSizesHistogram,
+				HistogramBucketLabel: "range",
+				VariableLabels:       map[string]string{"bucket": bucket},
+			})
+
+			metrics = append(metrics, Metric{
+				Description:          getBucketObjectVersionsMD(),
+				Histogram:            usage.ObjectVersionsHistogram,
+				HistogramBucketLabel: "range",
+				VariableLabels:       map[string]string{"bucket": bucket},
+			})
 		}
 		return
 	})
@@ -3260,6 +3469,12 @@ func getLocalStorageMetrics(opts MetricsGroupOpts) *MetricsGroup {
 					VariableLabels: map[string]string{"drive": disk.DrivePath},
 				})
 
+				metrics = append(metrics, Metric{
+					Description:    getNodeDriveWaitingIOMD(),
+					Value:          float64(disk.Metrics.TotalWaiting),
+					VariableLabels: map[string]string{"drive": disk.DrivePath},
+				})
+
 				for apiName, latency := range disk.Metrics.LastMinute {
 					metrics = append(metrics, Metric{
 						Description:    getNodeDriveAPILatencyMD(),
@@ -3320,6 +3535,16 @@ func getClusterHealthStatusMD() MetricDescription {
 	}
 }
 
+func getClusterErasureSetHealthStatusMD() MetricDescription {
+	return MetricDescription{
+		Namespace: clusterMetricNamespace,
+		Subsystem: "health",
+		Name:      "erasure_set_status",
+		Help:      "Get current health status for this erasure set",
+		Type:      gaugeMetric,
+	}
+}
+
 func getClusterErasureSetReadQuorumMD() MetricDescription {
 	return MetricDescription{
 		Namespace: clusterMetricNamespace,
@@ -3413,6 +3638,17 @@ func getClusterHealthMetrics(opts MetricsGroupOpts) *MetricsGroup {
 				VariableLabels: labels,
 				Value:          float64(h.HealingDrives),
 			})
+
+			health := 1
+			if !h.Healthy {
+				health = 0
+			}
+
+			metrics = append(metrics, Metric{
+				Description:    getClusterErasureSetHealthStatusMD(),
+				VariableLabels: labels,
+				Value:          float64(health),
+			})
 		}
 
 		return
@@ -3827,7 +4063,7 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
 func ReportMetrics(ctx context.Context, metricsGroups []*MetricsGroup) <-chan Metric {
 	ch := make(chan Metric)
 	go func() {
-		defer close(ch)
+		defer xioutil.SafeClose(ch)
 		populateAndPublish(metricsGroups, func(m Metric) bool {
 			if m.VariableLabels == nil {
 				m.VariableLabels = make(map[string]string)

cmd/metrics.go

@@ -381,7 +381,7 @@ func storageMetricsPrometheus(ch chan<- prometheus.Metric) {
 
 	server := getLocalServerProperty(globalEndpoints, &http.Request{
 		Host: globalLocalNodeName,
-	})
+	}, true)
 
 	onlineDisks, offlineDisks := getOnlineOfflineDisksStats(server.Disks)
 	totalDisks := offlineDisks.Merge(onlineDisks)

cmd/mrf.go

@@ -47,7 +47,7 @@ type mrfState struct {
 	ctx   context.Context
 	pools *erasureServerPools
 
-	mu   sync.Mutex
+	mu   sync.RWMutex
 	opCh chan partialOperation
 }
 
@@ -72,6 +72,9 @@ func (m *mrfState) addPartialOp(op partialOperation) {
 		return
 	}
 
+	m.mu.RLock()
+	defer m.mu.RUnlock()
+
 	select {
 	case m.opCh <- op:
 	default:

cmd/naughty-disk_test.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2024 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -108,6 +108,10 @@ func (d *naughtyDisk) GetDiskID() (string, error) {
 	return d.disk.GetDiskID()
 }
 
+func (d *naughtyDisk) SetFormatData(b []byte) {
+	d.disk.SetFormatData(b)
+}
+
 func (d *naughtyDisk) SetDiskID(id string) {
 	d.disk.SetDiskID(id)
 }
@@ -119,11 +123,11 @@ func (d *naughtyDisk) NSScanner(ctx context.Context, cache dataUsageCache, updat
 	return d.disk.NSScanner(ctx, cache, updates, scanMode, weSleep)
 }
 
-func (d *naughtyDisk) DiskInfo(ctx context.Context, metrics bool) (info DiskInfo, err error) {
+func (d *naughtyDisk) DiskInfo(ctx context.Context, opts DiskInfoOptions) (info DiskInfo, err error) {
 	if err := d.calcError(); err != nil {
 		return info, err
 	}
-	return d.disk.DiskInfo(ctx, metrics)
+	return d.disk.DiskInfo(ctx, opts)
 }
 
 func (d *naughtyDisk) MakeVolBulk(ctx context.Context, volumes ...string) (err error) {
@@ -168,11 +172,11 @@ func (d *naughtyDisk) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Wr
 	return d.disk.WalkDir(ctx, opts, wr)
 }
 
-func (d *naughtyDisk) ListDir(ctx context.Context, volume, dirPath string, count int) (entries []string, err error) {
+func (d *naughtyDisk) ListDir(ctx context.Context, origvolume, volume, dirPath string, count int) (entries []string, err error) {
 	if err := d.calcError(); err != nil {
 		return []string{}, err
 	}
-	return d.disk.ListDir(ctx, volume, dirPath, count)
+	return d.disk.ListDir(ctx, origvolume, volume, dirPath, count)
 }
 
 func (d *naughtyDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) {
@@ -189,11 +193,11 @@ func (d *naughtyDisk) ReadFileStream(ctx context.Context, volume, path string, o
 	return d.disk.ReadFileStream(ctx, volume, path, offset, length)
 }
 
-func (d *naughtyDisk) CreateFile(ctx context.Context, volume, path string, size int64, reader io.Reader) error {
+func (d *naughtyDisk) CreateFile(ctx context.Context, origvolume, volume, path string, size int64, reader io.Reader) error {
 	if err := d.calcError(); err != nil {
 		return err
 	}
-	return d.disk.CreateFile(ctx, volume, path, size, reader)
+	return d.disk.CreateFile(ctx, origvolume, volume, path, size, reader)
 }
 
 func (d *naughtyDisk) AppendFile(ctx context.Context, volume string, path string, buf []byte) error {
@@ -242,11 +246,11 @@ func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, version
 	return d.disk.DeleteVersions(ctx, volume, versions, opts)
 }
 
-func (d *naughtyDisk) WriteMetadata(ctx context.Context, volume, path string, fi FileInfo) (err error) {
+func (d *naughtyDisk) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) (err error) {
 	if err := d.calcError(); err != nil {
 		return err
 	}
-	return d.disk.WriteMetadata(ctx, volume, path, fi)
+	return d.disk.WriteMetadata(ctx, origvolume, volume, path, fi)
 }
 
 func (d *naughtyDisk) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) (err error) {
@@ -263,11 +267,11 @@ func (d *naughtyDisk) DeleteVersion(ctx context.Context, volume, path string, fi
 	return d.disk.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts)
 }
 
-func (d *naughtyDisk) ReadVersion(ctx context.Context, volume, path, versionID string, opts ReadOptions) (fi FileInfo, err error) {
+func (d *naughtyDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (fi FileInfo, err error) {
 	if err := d.calcError(); err != nil {
 		return FileInfo{}, err
 	}
-	return d.disk.ReadVersion(ctx, volume, path, versionID, opts)
+	return d.disk.ReadVersion(ctx, origvolume, volume, path, versionID, opts)
 }
 
 func (d *naughtyDisk) WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) {

cmd/notification.go

@@ -18,6 +18,7 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
@@ -25,14 +26,17 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"runtime"
 	"sync"
 	"time"
 
 	"github.com/cespare/xxhash/v2"
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/madmin-go/v3"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	xnet "github.com/minio/pkg/v2/net"
 	"github.com/minio/pkg/v2/sync/errgroup"
+	"github.com/minio/pkg/v2/workers"
 
 	"github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/logger"
@@ -58,7 +62,7 @@ type NotificationPeerErr struct {
 //
 // A zero NotificationGroup is valid and does not cancel on error.
 type NotificationGroup struct {
-	wg         sync.WaitGroup
+	workers    *workers.Workers
 	errs       []NotificationPeerErr
 	retryCount int
 }
@@ -66,7 +70,22 @@ type NotificationGroup struct {
 // WithNPeers returns a new NotificationGroup with length of errs slice upto nerrs,
 // upon Wait() errors are returned collected from all tasks.
 func WithNPeers(nerrs int) *NotificationGroup {
-	return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs), retryCount: 3}
+	if nerrs <= 0 {
+		nerrs = 1
+	}
+	wk, _ := workers.New(nerrs)
+	return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs), workers: wk, retryCount: 3}
+}
+
+// WithNPeersThrottled returns a new NotificationGroup with length of errs slice upto nerrs,
+// upon Wait() errors are returned collected from all tasks, optionally allows for X workers
+// only "per" parallel task.
+func WithNPeersThrottled(nerrs, wks int) *NotificationGroup {
+	if nerrs <= 0 {
+		nerrs = 1
+	}
+	wk, _ := workers.New(wks)
+	return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs), workers: wk, retryCount: 3}
 }
 
 // WithRetries sets the retry count for all function calls from the Go method.
@@ -80,7 +99,7 @@ func (g *NotificationGroup) WithRetries(retryCount int) *NotificationGroup {
 // Wait blocks until all function calls from the Go method have returned, then
 // returns the slice of errors from all function calls.
 func (g *NotificationGroup) Wait() []NotificationPeerErr {
-	g.wg.Wait()
+	g.workers.Wait()
 	return g.errs
 }
 
@@ -91,10 +110,11 @@ func (g *NotificationGroup) Wait() []NotificationPeerErr {
 func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, addr xnet.Host) {
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	g.wg.Add(1)
+	g.workers.Take()
 
 	go func() {
-		defer g.wg.Done()
+		defer g.workers.Give()
+
 		g.errs[index] = NotificationPeerErr{
 			Host: addr,
 		}
@@ -106,7 +126,7 @@ func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, a
 				if i == g.retryCount-1 {
 					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
 					ctx := logger.SetReqInfo(ctx, reqInfo)
-					logger.LogIf(ctx, err)
+					logger.LogOnceIf(ctx, err, addr.String())
 				}
 				// Wait for a minimum of 100ms and dynamically increase this based on number of attempts.
 				if i < g.retryCount-1 {
@@ -338,15 +358,34 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 }
 
 // VerifyBinary - asks remote peers to verify the checksum
-func (sys *NotificationSys) VerifyBinary(ctx context.Context, u *url.URL, sha256Sum []byte, releaseInfo string, reader []byte) []NotificationPeerErr {
-	ng := WithNPeers(len(sys.peerClients))
+func (sys *NotificationSys) VerifyBinary(ctx context.Context, u *url.URL, sha256Sum []byte, releaseInfo string, bin []byte) []NotificationPeerErr {
+	// FIXME: network calls made in this manner such as one goroutine per node,
+	// can easily eat into the internode bandwidth. This function would be mostly
+	// TX saturating, however there are situations where a RX might also saturate.
+	// To avoid these problems we must split the work at scale. With 1000 node
+	// setup becoming a reality we must try to shard the work properly such as
+	// pick 10 nodes that precisely can send those 100 requests the first node
+	// in the 10 node shard would coordinate between other 9 shards to get the
+	// rest of the `99*9` requests.
+	//
+	// This essentially splits the workload properly and also allows for network
+	// utilization to be optimal, instead of blindly throttling the way we are
+	// doing below. However the changes that are needed here are a bit involved,
+	// further discussion advised. Remove this comment and remove the worker model
+	// for this function in future.
+	maxWorkers := runtime.GOMAXPROCS(0) / 2
+	if maxWorkers > len(sys.peerClients) {
+		maxWorkers = len(sys.peerClients)
+	}
+
+	ng := WithNPeersThrottled(len(sys.peerClients), maxWorkers)
 	for idx, client := range sys.peerClients {
 		if client == nil {
 			continue
 		}
 		client := client
 		ng.Go(ctx, func() error {
-			return client.VerifyBinary(ctx, u, sha256Sum, releaseInfo, reader)
+			return client.VerifyBinary(ctx, u, sha256Sum, releaseInfo, bytes.NewReader(bin))
 		}, idx, *client.host)
 	}
 	return ng.Wait()
@@ -376,7 +415,7 @@ func (sys *NotificationSys) SignalConfigReload(subSys string) []NotificationPeer
 		}
 		client := client
 		ng.Go(GlobalContext, func() error {
-			return client.SignalService(serviceReloadDynamic, subSys)
+			return client.SignalService(serviceReloadDynamic, subSys, false)
 		}, idx, *client.host)
 	}
 	return ng.Wait()
@@ -391,7 +430,23 @@ func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerE
 		}
 		client := client
 		ng.Go(GlobalContext, func() error {
-			return client.SignalService(sig, "")
+			// force == true preserves the current behavior
+			return client.SignalService(sig, "", false)
+		}, idx, *client.host)
+	}
+	return ng.Wait()
+}
+
+// SignalServiceV2 - calls signal service RPC call on all peers with v2 API
+func (sys *NotificationSys) SignalServiceV2(sig serviceSignal, dryRun bool) []NotificationPeerErr {
+	ng := WithNPeers(len(sys.peerClients))
+	for idx, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		client := client
+		ng.Go(GlobalContext, func() error {
+			return client.SignalService(sig, "", dryRun)
 		}, idx, *client.host)
 	}
 	return ng.Wait()
@@ -405,6 +460,7 @@ func (sys *NotificationSys) GetLocks(ctx context.Context, r *http.Request) []*Pe
 	g := errgroup.WithNErrs(len(sys.peerClients))
 	for index, client := range sys.peerClients {
 		index := index
+		client := client
 		g.Go(func() error {
 			if client == nil {
 				return errPeerNotReachable
@@ -448,7 +504,7 @@ func (sys *NotificationSys) LoadBucketMetadata(ctx context.Context, bucketName s
 	for _, nErr := range ng.Wait() {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
 		if nErr.Err != nil {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+			logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
 }
@@ -478,7 +534,7 @@ func (sys *NotificationSys) DeleteBucketMetadata(ctx context.Context, bucketName
 	for _, nErr := range ng.Wait() {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
 		if nErr.Err != nil {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+			logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
 }
@@ -505,7 +561,7 @@ func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []Buck
 	for _, nErr := range ng.Wait() {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
 		if nErr.Err != nil {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+			logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
 
@@ -517,6 +573,7 @@ func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []Buck
 	for k, replicationStats := range replicationStatsList {
 		bucketStatsMap.Stats[k] = BucketStats{
 			ReplicationStats: replicationStats,
+			ProxyStats:       globalReplicationStats.getProxyStats(k),
 		}
 	}
 
@@ -546,12 +603,13 @@ func (sys *NotificationSys) GetClusterBucketStats(ctx context.Context, bucketNam
 	for _, nErr := range ng.Wait() {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
 		if nErr.Err != nil {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+			logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
 	bucketStats = append(bucketStats, BucketStats{
 		ReplicationStats: globalReplicationStats.Get(bucketName),
 		QueueStats:       ReplicationQueueStats{Nodes: []ReplQNodeStats{globalReplicationStats.getNodeQueueStats(bucketName)}},
+		ProxyStats:       globalReplicationStats.getProxyStats(bucketName),
 	})
 	return bucketStats
 }
@@ -578,7 +636,7 @@ func (sys *NotificationSys) GetClusterSiteMetrics(ctx context.Context) []SRMetri
 	for _, nErr := range ng.Wait() {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
 		if nErr.Err != nil {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+			logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
 	siteStats = append(siteStats, globalReplicationStats.getSRMetricsForNode())
@@ -600,7 +658,7 @@ func (sys *NotificationSys) ReloadPoolMeta(ctx context.Context) {
 	for _, nErr := range ng.Wait() {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
 		if nErr.Err != nil {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+			logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
 }
@@ -621,7 +679,7 @@ func (sys *NotificationSys) StopRebalance(ctx context.Context) {
 	for _, nErr := range ng.Wait() {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
 		if nErr.Err != nil {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+			logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
 
@@ -630,6 +688,7 @@ func (sys *NotificationSys) StopRebalance(ctx context.Context) {
 		logger.LogIf(ctx, errServerNotInitialized)
 		return
 	}
+
 	if pools, ok := objAPI.(*erasureServerPools); ok {
 		pools.StopRebalance()
 	}
@@ -652,7 +711,7 @@ func (sys *NotificationSys) LoadRebalanceMeta(ctx context.Context, startRebalanc
 	for _, nErr := range ng.Wait() {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
 		if nErr.Err != nil {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+			logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
 }
@@ -673,7 +732,7 @@ func (sys *NotificationSys) LoadTransitionTierConfig(ctx context.Context) {
 	for _, nErr := range ng.Wait() {
 		reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String())
 		if nErr.Err != nil {
-			logger.LogIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err)
+			logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String())
 		}
 	}
 }
@@ -1021,7 +1080,7 @@ func (sys *NotificationSys) StorageInfo(objLayer ObjectLayer, metrics bool) Stor
 }
 
 // ServerInfo - calls ServerInfo RPC call on all peers.
-func (sys *NotificationSys) ServerInfo() []madmin.ServerProperties {
+func (sys *NotificationSys) ServerInfo(metrics bool) []madmin.ServerProperties {
 	reply := make([]madmin.ServerProperties, len(sys.peerClients))
 	var wg sync.WaitGroup
 	for i, client := range sys.peerClients {
@@ -1031,7 +1090,7 @@ func (sys *NotificationSys) ServerInfo() []madmin.ServerProperties {
 		wg.Add(1)
 		go func(client *peerRESTClient, idx int) {
 			defer wg.Done()
-			info, err := client.ServerInfo()
+			info, err := client.ServerInfo(metrics)
 			if err != nil {
 				info.Endpoint = client.host.String()
 				info.State = string(madmin.ItemOffline)
@@ -1209,7 +1268,7 @@ func (sys *NotificationSys) collectPeerMetrics(ctx context.Context, peerChannels
 	}
 	go func(wg *sync.WaitGroup, ch chan Metric) {
 		wg.Wait()
-		close(ch)
+		xioutil.SafeClose(ch)
 	}(&wg, ch)
 	return ch
 }
@@ -1275,7 +1334,7 @@ func (sys *NotificationSys) ServiceFreeze(ctx context.Context, freeze bool) []No
 		}
 		client := client
 		ng.Go(GlobalContext, func() error {
-			return client.SignalService(serviceSig, "")
+			return client.SignalService(serviceSig, "", false)
 		}, idx, *client.host)
 	}
 	nerrs := ng.Wait()
@@ -1419,7 +1478,7 @@ func (sys *NotificationSys) DriveSpeedTest(ctx context.Context, opts madmin.Driv
 
 			reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", client.host.String())
 			ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-			logger.LogIf(ctx, err)
+			logger.LogOnceIf(ctx, err, client.host.String())
 		}(client)
 	}
 
@@ -1434,7 +1493,7 @@ func (sys *NotificationSys) DriveSpeedTest(ctx context.Context, opts madmin.Driv
 
 	go func(wg *sync.WaitGroup, ch chan madmin.DriveSpeedTestResult) {
 		wg.Wait()
-		close(ch)
+		xioutil.SafeClose(ch)
 	}(&wg, ch)
 
 	return ch
@@ -1562,7 +1621,7 @@ func (sys *NotificationSys) GetReplicationMRF(ctx context.Context, bucket, node
 	}(mrfCh)
 	go func(wg *sync.WaitGroup) {
 		wg.Wait()
-		close(mrfCh)
+		xioutil.SafeClose(mrfCh)
 	}(&wg)
 	return mrfCh, nil
 }

cmd/object-api-input-checks.go

@@ -40,7 +40,7 @@ func checkDelObjArgs(ctx context.Context, bucket, object string) error {
 // Checks bucket and object name validity, returns nil if both are valid.
 func checkBucketAndObjectNames(ctx context.Context, bucket, object string) error {
 	// Verify if bucket is valid.
-	if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketName(bucket) != nil {
+	if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil {
 		return BucketNameInvalid{Bucket: bucket}
 	}
 	// Verify if object is valid.
@@ -58,15 +58,12 @@ func checkBucketAndObjectNames(ctx context.Context, bucket, object string) error
 }
 
 // Checks for all ListObjects arguments validity.
-func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string, obj getBucketInfoI) error {
-	// Verify if bucket exists before validating object name.
-	// This is done on purpose since the order of errors is
-	// important here bucket does not exist error should
-	// happen before we return an error for invalid object name.
-	// FIXME: should be moved to handler layer.
-	if err := checkBucketExist(ctx, bucket, obj); err != nil {
-		return err
+func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string) error {
+	// Verify if bucket is valid.
+	if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil {
+		return BucketNameInvalid{Bucket: bucket}
 	}
+
 	// Validates object prefix validity after bucket exists.
 	if !IsValidObjectPrefix(prefix) {
 		logger.LogIf(ctx, ObjectNameInvalid{
@@ -82,8 +79,8 @@ func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string, obj g
 }
 
 // Checks for all ListMultipartUploads arguments validity.
-func checkListMultipartArgs(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, obj ObjectLayer) error {
-	if err := checkListObjsArgs(ctx, bucket, prefix, keyMarker, obj); err != nil {
+func checkListMultipartArgs(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string) error {
+	if err := checkListObjsArgs(ctx, bucket, prefix, keyMarker); err != nil {
 		return err
 	}
 	if uploadIDMarker != "" {
@@ -143,14 +140,11 @@ func checkAbortMultipartArgs(ctx context.Context, bucket, object, uploadID strin
 	return checkMultipartObjectArgs(ctx, bucket, object, uploadID, obj)
 }
 
-// Checks Object arguments validity, also validates if bucket exists.
+// Checks Object arguments validity.
 func checkObjectArgs(ctx context.Context, bucket, object string, obj ObjectLayer) error {
-	// Verify if bucket exists before validating object name.
-	// This is done on purpose since the order of errors is
-	// important here bucket does not exist error should
-	// happen before we return an error for invalid object name.
-	if err := checkBucketExist(ctx, bucket, obj); err != nil {
-		return err
+	// Verify if bucket is valid.
+	if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil {
+		return BucketNameInvalid{Bucket: bucket}
 	}
 
 	if err := checkObjectNameForLengthAndSlash(bucket, object); err != nil {
@@ -168,8 +162,13 @@ func checkObjectArgs(ctx context.Context, bucket, object string, obj ObjectLayer
 	return nil
 }
 
-// Checks for PutObject arguments validity, also validates if bucket exists.
+// Checks for PutObject arguments validity.
 func checkPutObjectArgs(ctx context.Context, bucket, object string) error {
+	// Verify if bucket is valid.
+	if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil {
+		return BucketNameInvalid{Bucket: bucket}
+	}
+
 	if err := checkObjectNameForLengthAndSlash(bucket, object); err != nil {
 		return err
 	}
@@ -182,16 +181,3 @@ func checkPutObjectArgs(ctx context.Context, bucket, object string) error {
 	}
 	return nil
 }
-
-type getBucketInfoI interface {
-	GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error)
-}
-
-// Checks whether bucket exists and returns appropriate error if not.
-func checkBucketExist(ctx context.Context, bucket string, obj getBucketInfoI) error {
-	_, err := obj.GetBucketInfo(ctx, bucket, BucketOptions{})
-	if err != nil {
-		return err
-	}
-	return nil
-}

cmd/object-api-interface.go

@@ -109,6 +109,8 @@ type ObjectOptions struct {
 	// participating in a rebalance operation. Typically set for 'write' operations.
 	SkipRebalancing bool
 
+	DataMovement bool // indicates an going decommisionning or rebalacing
+
 	PrefixEnabledFn func(prefix string) bool // function which returns true if versioning is enabled on prefix
 
 	// IndexCB will return any index created but the compression.

cmd/object-api-listobjects_test.go

@@ -155,16 +155,21 @@ func testListObjectsVersionedFolders(obj ObjectLayer, instanceType string, t1 Te
 	for i, testCase := range testCases {
 		testCase := testCase
 		t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) {
-			t.Log("ListObjects, bucket:", testCase.bucketName, "prefix:",
-				testCase.prefix, "marker:", testCase.marker, "delimiter:",
-				testCase.delimiter, "maxkeys:", testCase.maxKeys)
 			var err error
 			var resultL ListObjectsInfo
 			var resultV ListObjectVersionsInfo
 			if testCase.versioned {
+				t.Log("ListObjectVersions, bucket:", testCase.bucketName, "prefix:",
+					testCase.prefix, "marker:", testCase.marker, "delimiter:",
+					testCase.delimiter, "maxkeys:", testCase.maxKeys)
+
 				resultV, err = obj.ListObjectVersions(context.Background(), testCase.bucketName,
 					testCase.prefix, testCase.marker, "", testCase.delimiter, testCase.maxKeys)
 			} else {
+				t.Log("ListObjects, bucket:", testCase.bucketName, "prefix:",
+					testCase.prefix, "marker:", testCase.marker, "delimiter:",
+					testCase.delimiter, "maxkeys:", testCase.maxKeys)
+
 				resultL, err = obj.ListObjects(context.Background(), testCase.bucketName,
 					testCase.prefix, testCase.marker, testCase.delimiter, testCase.maxKeys)
 			}
@@ -819,16 +824,14 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		shouldPass bool
 	}{
 		// Test cases with invalid bucket names ( Test number 1-4 ).
-		{".test", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: ".test"}, false},
-		{"Test", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "Test"}, false},
-		{"---", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "---"}, false},
-		{"ad", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "ad"}, false},
-		// Using an existing file for bucket name, but its not a directory (5).
-		{"simple-file.txt", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "simple-file.txt"}, false},
+		{".test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
 		// Valid bucket names, but they do not exist (6-8).
-		{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
-		{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
-		{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		{"volatile-bucket-1", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
 		// If marker is *after* the last possible object from the prefix it should return an empty list.
 		{"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true},
 		// If the marker is *before* the first possible object from the prefix it should return the first object.
@@ -1564,16 +1567,14 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
 		shouldPass bool
 	}{
 		// Test cases with invalid bucket names ( Test number 1-4).
-		{".test", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: ".test"}, false},
-		{"Test", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "Test"}, false},
-		{"---", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "---"}, false},
-		{"ad", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "ad"}, false},
-		// Using an existing file for bucket name, but its not a directory (5).
-		{"simple-file.txt", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "simple-file.txt"}, false},
+		{".test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
 		// Valid bucket names, but they do not exist (6-8).
-		{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
-		{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
-		{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		{"volatile-bucket-1", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
 		// If marker is *after* the last possible object from the prefix it should return an empty list.
 		{"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true},
 		// Setting a non-existing directory to be prefix (10-11).

cmd/object-api-multipart_test.go

@@ -113,7 +113,7 @@ func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t Test
 		uploadID        string
 		expectedErrType error
 	}{
-		{"--", object, uploadID, BucketNotFound{}},
+		{"--", object, uploadID, BucketNameInvalid{}},
 		{"foo", object, uploadID, BucketNotFound{}},
 		{bucket, object, "foo-foo", InvalidUploadID{}},
 		{bucket, object, uploadID, nil},
@@ -194,6 +194,19 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 		// Failed to create NewMultipartUpload, abort.
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
+
+	err = obj.MakeBucket(context.Background(), "abc", MakeBucketOptions{})
+	if err != nil {
+		// Failed to create newbucket, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
+	resN, err := obj.NewMultipartUpload(context.Background(), "abc", "def", opts)
+	if err != nil {
+		// Failed to create NewMultipartUpload, abort.
+		t.Fatalf("%s : %s", instanceType, err.Error())
+	}
+
 	uploadID := res.UploadID
 	// Creating a dummy bucket for tests.
 	err = obj.MakeBucket(context.Background(), "unused-bucket", MakeBucketOptions{})
@@ -202,6 +215,8 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 		t.Fatalf("%s : %s", instanceType, err.Error())
 	}
 
+	obj.DeleteBucket(context.Background(), "abc", DeleteBucketOptions{})
+
 	// Collection of non-exhaustive PutObjectPart test cases, valid errors
 	// and success responses.
 	testCases := []struct {
@@ -221,19 +236,19 @@ func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrH
 	}{
 		// Test case  1-4.
 		// Cases with invalid bucket name.
-		{bucketName: ".test", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket not found: .test")},
-		{bucketName: "------", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket not found: ------")},
+		{bucketName: ".test", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket name invalid: .test")},
+		{bucketName: "------", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket name invalid: ------")},
 		{
 			bucketName: "$this-is-not-valid-too", objName: "obj", PartID: 1,
-			expectedError: fmt.Errorf("%s", "Bucket not found: $this-is-not-valid-too"),
+			expectedError: fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too"),
 		},
-		{bucketName: "a", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket not found: a")},
+		{bucketName: "a", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket name invalid: a")},
 		// Test case - 5.
 		// Case with invalid object names.
 		{bucketName: bucket, PartID: 1, expectedError: fmt.Errorf("%s", "Object name invalid: minio-bucket/")},
 		// Test case - 6.
 		// Valid object and bucket names but non-existent bucket.
-		{bucketName: "abc", objName: "def", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket not found: abc")},
+		{bucketName: "abc", objName: "def", uploadID: resN.UploadID, PartID: 1, expectedError: fmt.Errorf("%s", "Bucket not found: abc")},
 		// Test Case - 7.
 		// Existing bucket, but using a bucket on which NewMultipartUpload is not Initiated.
 		{bucketName: "unused-bucket", objName: "def", uploadID: "xyz", PartID: 1, expectedError: fmt.Errorf("%s", "Invalid upload id xyz")},
@@ -1041,10 +1056,10 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 		shouldPass bool
 	}{
 		// Test cases with invalid bucket names ( Test number 1-4 ).
-		{".test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: ".test"}, false},
-		{"Test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "Test"}, false},
-		{"---", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "---"}, false},
-		{"ad", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "ad"}, false},
+		{".test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
 		// Valid bucket names, but they do not exist (Test number 5-7).
 		{"volatile-bucket-1", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
 		{"volatile-bucket-2", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
@@ -1357,15 +1372,15 @@ func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks
 		shouldPass bool
 	}{
 		// Test cases with invalid bucket names (Test number 1-4).
-		{".test", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: ".test"}, false},
-		{"Test", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "Test"}, false},
-		{"---", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "---"}, false},
-		{"ad", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "ad"}, false},
+		{".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
 		// Test cases for listing uploadID with single part.
 		// Valid bucket names, but they do not exist (Test number 5-7).
-		{"volatile-bucket-1", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
-		{"volatile-bucket-2", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
-		{"volatile-bucket-3", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		{"volatile-bucket-1", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
 		// Test case for Asserting for invalid objectName (Test number 8).
 		{bucketNames[0], "", "", 0, 0, ListPartsInfo{}, ObjectNameInvalid{Bucket: bucketNames[0]}, false},
 		// Asserting for Invalid UploadID (Test number 9).
@@ -1594,15 +1609,15 @@ func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler)
 		shouldPass bool
 	}{
 		// Test cases with invalid bucket names (Test number 1-4).
-		{".test", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: ".test"}, false},
-		{"Test", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "Test"}, false},
-		{"---", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "---"}, false},
-		{"ad", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "ad"}, false},
+		{".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false},
 		// Test cases for listing uploadID with single part.
 		// Valid bucket names, but they do not exist (Test number 5-7).
-		{"volatile-bucket-1", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
-		{"volatile-bucket-2", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
-		{"volatile-bucket-3", "", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		{"volatile-bucket-1", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
 		// Test case for Asserting for invalid objectName (Test number 8).
 		{bucketNames[0], "", "", 0, 0, ListPartsInfo{}, ObjectNameInvalid{Bucket: bucketNames[0]}, false},
 		// Asserting for Invalid UploadID (Test number 9).
@@ -1809,15 +1824,15 @@ func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t T
 		shouldPass bool
 	}{
 		// Test cases with invalid bucket names (Test number 1-4).
-		{".test", "", "", []CompletePart{}, "", BucketNotFound{Bucket: ".test"}, false},
-		{"Test", "", "", []CompletePart{}, "", BucketNotFound{Bucket: "Test"}, false},
-		{"---", "", "", []CompletePart{}, "", BucketNotFound{Bucket: "---"}, false},
-		{"ad", "", "", []CompletePart{}, "", BucketNotFound{Bucket: "ad"}, false},
+		{".test", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: ".test"}, false},
+		{"Test", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "Test"}, false},
+		{"---", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "---"}, false},
+		{"ad", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "ad"}, false},
 		// Test cases for listing uploadID with single part.
 		// Valid bucket names, but they do not exist (Test number 5-7).
-		{"volatile-bucket-1", "", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-1"}, false},
-		{"volatile-bucket-2", "", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-2"}, false},
-		{"volatile-bucket-3", "", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-3"}, false},
+		{"volatile-bucket-1", "test1", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-1"}, false},
+		{"volatile-bucket-2", "test1", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-2"}, false},
+		{"volatile-bucket-3", "test1", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-3"}, false},
 		// Test case for Asserting for invalid objectName (Test number 8).
 		{bucketNames[0], "", "", []CompletePart{}, "", ObjectNameInvalid{Bucket: bucketNames[0]}, false},
 		// Asserting for Invalid UploadID (Test number 9).