admin-info: add DeleteMarkers count (#17659)

Also return Bucket Not Found error in the same use case.

.github/workflows/helm-lint.yml

@@ -0,0 +1,30 @@
+name: Helm Chart linting
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  release:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+
+      - name: Run helm lint
+        run: |
+          cd helm/minio
+          helm lint .

.github/workflows/mint.yml

@@ -3,7 +3,7 @@ name: Mint Tests
 on:
   pull_request:
     branches:
-    - master
+      - master
 
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -37,8 +37,7 @@ jobs:
 
       - name: build-minio
         run: |
-          make install
-          docker build . -t "minio/minio:${{ steps.vars.outputs.sha_short }}"
+          TAG="minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
 
       - name: compress and encrypt
         run: |
@@ -51,7 +50,16 @@ jobs:
       - name: standalone erasure
         run: |
           ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
+
+      - name: The job must cleanup
+        if: ${{ always() }}
+        run: |
+          export JOB_NAME=${{ steps.vars.outputs.sha_short }}
+          for mode in $(echo compress-encrypt pools erasure); do
+             docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true
+             docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true
+          done
           docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }}
-
-
-
+          docker system prune -f || true
+          docker volume prune -f || true
+          docker volume rm $(docker volume ls -q -f dangling=true) || true

.github/workflows/run-mint.sh

@@ -18,9 +18,17 @@ cd .github/workflows/mint
 docker-compose -f minio-${MODE}.yaml up -d
 sleep 5m
 
-docker run --rm --net=host \
+docker system prune -f || true
+docker volume prune -f || true
+docker volume rm $(docker volume ls -q -f dangling=true) || true
+
+# Stop two nodes, one of each pool, to check that all S3 calls work while quorum is still there
+[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio2
+[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio6
+
+docker run --rm --net=mint_default \
 	--name="mint-${MODE}-${JOB_NAME}" \
-	-e SERVER_ENDPOINT="127.0.0.1:9000" \
+	-e SERVER_ENDPOINT="nginx:9000" \
 	-e ACCESS_KEY="${ACCESS_KEY}" \
 	-e SECRET_KEY="${SECRET_KEY}" \
 	-e ENABLE_HTTPS=0 \
@@ -32,7 +40,7 @@ sleep 10s
 
 docker system prune -f || true
 docker volume prune -f || true
-docker volume rm $(docker volume ls -f dangling=true) || true
+docker volume rm $(docker volume ls -q -f dangling=true) || true
 
 ## change working directory
 cd ../../../

.github/workflows/vulncheck.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.19.10
+          go-version: 1.19.11
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

.gitignore

@@ -38,4 +38,6 @@ docs/debugging/s3-check-md5/s3-check-md5
 docs/debugging/hash-set/hash-set
 docs/debugging/healing-bin/healing-bin
 docs/debugging/inspect/inspect
+docs/debugging/pprofgoparser/pprofgoparser
 .bin/
+*.gz

CREDITS

@@ -2920,7 +2920,7 @@ SOFTWARE.
 github.com/beevik/ntp
 https://github.com/beevik/ntp
 ----------------------------------------------------------------
-Copyright 2015-2023 Brett Vickers. All rights reserved.
+Copyright © 2015-2023 Brett Vickers. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions
@@ -16022,6 +16022,34 @@ For more information on this, and how to apply and follow the GNU AGPL, see
 
 ================================================================
 
+github.com/minio/dnscache
+https://github.com/minio/dnscache
+----------------------------------------------------------------
+MIT License
+
+Copyright (c) 2023 MinIO, Inc.
+Copyright (c) 2018 Olivier Poitrey
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================
+
 github.com/minio/dperf
 https://github.com/minio/dperf
 ----------------------------------------------------------------
@@ -25164,33 +25192,6 @@ THE SOFTWARE.
 
 ================================================================
 
-github.com/rs/dnscache
-https://github.com/rs/dnscache
-----------------------------------------------------------------
-MIT License
-
-Copyright (c) 2018 Olivier Poitrey
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-================================================================
-
 github.com/rs/xid
 https://github.com/rs/xid
 ----------------------------------------------------------------

Dockerfile.release

@@ -36,7 +36,6 @@ RUN \
      curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
      curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
      curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /opt/bin/mc && \
-     gzip /opt/bin/mc && \
      microdnf clean all && \
      chmod +x /opt/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \

cmd/admin-bucket-handlers.go

@@ -56,9 +56,7 @@ const (
 // specified in the quota configuration will be applied by default
 // to enforce total quota for the specified bucket.
 func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "PutBucketQuotaConfig")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SetBucketQuotaAdminAction)
 	if objectAPI == nil {
@@ -110,9 +108,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 
 // GetBucketQuotaConfigHandler - gets bucket quota configuration
 func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetBucketQuotaConfig")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetBucketQuotaAdminAction)
 	if objectAPI == nil {
@@ -145,9 +141,8 @@ func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *
 
 // SetRemoteTargetHandler - sets a remote target for bucket
 func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetBucketTarget")
+	ctx := r.Context()
 
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 	update := r.Form.Get("update") == "true"
@@ -289,9 +284,8 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 // ListRemoteTargetsHandler - lists remote target(s) for a bucket or gets a target
 // for a particular ARN type
 func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListBucketTargets")
+	ctx := r.Context()
 
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 	arnType := vars["type"]
@@ -324,9 +318,8 @@ func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *htt
 
 // RemoveRemoteTargetHandler - removes a remote target for bucket with specified ARN
 func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RemoveBucketTarget")
+	ctx := r.Context()
 
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := pathClean(vars["bucket"])
 	arn := vars["arn"]
@@ -368,8 +361,7 @@ func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *ht
 
 // ExportBucketMetadataHandler - exports all bucket metadata as a zipped file
 func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ExportBucketMetadata")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	bucket := pathClean(r.Form.Get("bucket"))
 	// Get current object layer instance.
@@ -652,9 +644,7 @@ func (i *importMetaReport) SetStatus(bucket, fname string, err error) {
 // 2. Replication config - is omitted from import as remote target credentials are not available from exported data for security reasons.
 // 3. lifecycle config - if transition rules are present, tier name needs to have been defined.
 func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ImportBucketMetadata")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ImportBucketMetadataAction)
@@ -1069,8 +1059,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 // ReplicationDiffHandler - POST returns info on unreplicated versions for a remote target ARN
 // to the connected HTTP client.
 func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ReplicationDiff")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
@@ -1129,3 +1118,62 @@ func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.
 		}
 	}
 }
+
+// ReplicationMRFHandler - POST returns info on entries in the MRF backlog for a node or all nodes
+func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ReplicationDiff)
+	if objectAPI == nil {
+		return
+	}
+
+	// Check if bucket exists.
+	if bucket != "" {
+		if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+	}
+
+	q := r.Form
+	node := q.Get("node")
+
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
+
+	mrfCh, err := globalNotificationSys.GetReplicationMRF(ctx, bucket, node)
+	if err != nil {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	enc := json.NewEncoder(w)
+	for {
+		select {
+		case entry, ok := <-mrfCh:
+			if !ok {
+				return
+			}
+			if err := enc.Encode(entry); err != nil {
+				return
+			}
+			if len(mrfCh) == 0 {
+				// Flush if nothing is queued
+				w.(http.Flusher).Flush()
+			}
+		case <-keepAliveTicker.C:
+			if len(mrfCh) > 0 {
+				continue
+			}
+			if _, err := w.Write([]byte(" ")); err != nil {
+				return
+			}
+			w.(http.Flusher).Flush()
+		case <-ctx.Done():
+			return
+		}
+	}
+}

cmd/admin-handlers-config-kv.go

@@ -43,9 +43,7 @@ import (
 
 // DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "DeleteConfigKV")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
@@ -149,9 +147,7 @@ type setConfigResult struct {
 
 // SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetConfigKV")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
@@ -244,9 +240,7 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
 //
 // This is a reporting API and config secrets are redacted in the response.
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetConfigKV")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
@@ -292,9 +286,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 }
 
 func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ClearConfigHistoryKV")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
@@ -327,9 +319,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 
 // RestoreConfigHistoryKVHandler - restores a config with KV settings for the given KV id.
 func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RestoreConfigHistoryKV")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
@@ -375,9 +365,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 
 // ListConfigHistoryKVHandler - lists all the KV ids.
 func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListConfigHistoryKV")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
@@ -415,9 +403,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 
 // HelpConfigKVHandler - GET /minio/admin/v3/help-config-kv?subSys={subSys}&key={key}
 func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "HelpConfigKV")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
@@ -442,9 +428,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 
 // SetConfigHandler - PUT /minio/admin/v3/config
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetConfig")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
@@ -496,9 +480,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 // This endpoint is mainly for exporting and backing up the configuration.
 // Secrets are not redacted.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetConfig")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {

cmd/admin-handlers-idp-config.go

@@ -37,7 +37,7 @@ import (
 	"github.com/minio/pkg/ldap"
 )
 
-func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
+func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
@@ -198,30 +198,27 @@ func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isU
 //
 // PUT <admin-prefix>/idp-cfg/openid/_ -> create (default) named config `_`
 func (a adminAPIHandlers) AddIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AddIdentityProviderCfg")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
-	a.addOrUpdateIDPHandler(ctx, w, r, false)
+	addOrUpdateIDPHandler(ctx, w, r, false)
 }
 
 // UpdateIdentityProviderCfg: updates an existing IDP config for openid/ldap.
 //
-// PATCH <admin-prefix>/idp-cfg/openid/dex1 -> update named config `dex1`
+// POST <admin-prefix>/idp-cfg/openid/dex1 -> update named config `dex1`
 //
-// PATCH <admin-prefix>/idp-cfg/openid/_ -> update (default) named config `_`
+// POST <admin-prefix>/idp-cfg/openid/_ -> update (default) named config `_`
 func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "UpdateIdentityProviderCfg")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
-	a.addOrUpdateIDPHandler(ctx, w, r, true)
+	addOrUpdateIDPHandler(ctx, w, r, true)
 }
 
 // ListIdentityProviderCfg:
 //
 // GET <admin-prefix>/idp-cfg/openid -> lists openid provider configs.
 func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListIdentityProviderCfg")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
@@ -274,8 +271,7 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
 //
 // GET <admin-prefix>/idp-cfg/openid/dex_test
 func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetIdentityProviderCfg")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
@@ -334,9 +330,7 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
 //
 // DELETE <admin-prefix>/idp-cfg/openid/dex_test
 func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "DeleteIdentityProviderCfg")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
 	if objectAPI == nil {

cmd/admin-handlers-idp-ldap.go

@@ -45,9 +45,7 @@ import (
 //
 // When all query parameters are omitted, returns mappings for all policies.
 func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListLDAPPolicyMappingEntities")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Check authorization.
 
@@ -94,9 +92,7 @@ func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r
 //
 // POST <admin-prefix>/idp/ldap/policy/{operation}
 func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AttachDetachPolicyLDAP")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Check authorization.
 

cmd/admin-handlers-pools.go

@@ -35,9 +35,7 @@ var (
 )
 
 func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "StartDecommission")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
 	if objectAPI == nil {
@@ -113,9 +111,7 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 }
 
 func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "CancelDecommission")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
 	if objectAPI == nil {
@@ -161,9 +157,7 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
 }
 
 func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "StatusPool")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
 	if objectAPI == nil {
@@ -204,9 +198,7 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListPools")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
 	if objectAPI == nil {
@@ -239,8 +231,7 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
 }
 
 func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RebalanceStart")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
 	if objectAPI == nil {
@@ -311,8 +302,7 @@ func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request)
 }
 
 func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RebalanceStatus")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
 	if objectAPI == nil {
@@ -352,8 +342,7 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
 }
 
 func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RebalanceStop")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
 	if objectAPI == nil {

cmd/admin-handlers-site-replication.go

@@ -20,25 +20,26 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/gob"
 	"encoding/json"
+	"errors"
 	"io"
 	"net/http"
 	"strings"
+	"sync/atomic"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
-	"github.com/minio/mux"
-
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"
 	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 // SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add
 func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SiteReplicationAdd")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
 	if objectAPI == nil {
@@ -72,9 +73,7 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 // used internally to tell current cluster to enable SR with
 // the provided peer clusters and service account.
 func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRPeerJoin")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
 	if objectAPI == nil {
@@ -96,9 +95,7 @@ func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
 
 // SRPeerBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
 func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRPeerBucketOps")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
 	if objectAPI == nil {
@@ -145,9 +142,7 @@ func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request
 
 // SRPeerReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
 func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRPeerReplicateIAMItem")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
 	if objectAPI == nil {
@@ -199,9 +194,7 @@ func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.
 
 // SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
 func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRPeerReplicateBucketItem")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationOperationAction)
 	if objectAPI == nil {
@@ -265,9 +258,7 @@ func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *ht
 
 // SiteReplicationInfo - GET /minio/admin/v3/site-replication/info
 func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SiteReplicationInfo")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
 	if objectAPI == nil {
@@ -287,9 +278,7 @@ func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Req
 }
 
 func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SiteReplicationGetIDPSettings")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
 	if objectAPI == nil {
@@ -326,9 +315,7 @@ func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptio
 
 // SiteReplicationStatus - GET /minio/admin/v3/site-replication/status
 func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SiteReplicationStatus")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
 	if objectAPI == nil {
@@ -357,9 +344,7 @@ func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.R
 
 // SiteReplicationMetaInfo - GET /minio/admin/v3/site-replication/metainfo
 func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SiteReplicationMetaInfo")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationInfoAction)
 	if objectAPI == nil {
@@ -381,8 +366,7 @@ func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http
 
 // SiteReplicationEdit - PUT /minio/admin/v3/site-replication/edit
 func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SiteReplicationEdit")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
 	if objectAPI == nil {
@@ -413,8 +397,7 @@ func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Req
 //
 // used internally to tell current cluster to update endpoint for peer
 func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRPeerEdit")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationAddAction)
 	if objectAPI == nil {
@@ -448,9 +431,7 @@ func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) {
 
 // SiteReplicationRemove - PUT /minio/admin/v3/site-replication/remove
 func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SiteReplicationRemove")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationRemoveAction)
 	if objectAPI == nil {
@@ -481,8 +462,7 @@ func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.R
 //
 // used internally to tell current cluster to update endpoint for peer
 func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRPeerRemove")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationRemoveAction)
 	if objectAPI == nil {
@@ -504,9 +484,7 @@ func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) {
 
 // SiteReplicationResyncOp - PUT /minio/admin/v3/site-replication/resync/op
 func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SiteReplicationResyncOp")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.SiteReplicationResyncAction)
 	if objectAPI == nil {
@@ -543,3 +521,45 @@ func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http
 	}
 	writeSuccessResponseJSON(w, body)
 }
+
+// SiteReplicationDevNull - everything goes to io.Discard
+// [POST] /minio/admin/v3/site-replication/devnull
+func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	globalSiteNetPerfRX.Connect()
+	defer globalSiteNetPerfRX.Disconnect()
+
+	connectTime := time.Now()
+	for {
+		n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
+		atomic.AddUint64(&globalSiteNetPerfRX.RX, uint64(n))
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
+			// would mean the network is not stable. Logging here will help in debugging network issues.
+			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
+				logger.LogIf(ctx, err)
+			}
+		}
+		if err != nil {
+			if errors.Is(err, io.EOF) {
+				w.WriteHeader(http.StatusNoContent)
+			} else {
+				w.WriteHeader(http.StatusBadRequest)
+			}
+			break
+		}
+	}
+}
+
+// SiteReplicationNetPerf - everything goes to io.Discard
+// [POST] /minio/admin/v3/site-replication/netperf
+func (a adminAPIHandlers) SiteReplicationNetPerf(w http.ResponseWriter, r *http.Request) {
+	durationStr := r.Form.Get(peerRESTDuration)
+	duration, _ := time.ParseDuration(durationStr)
+	if duration < globalNetPerfMinDuration {
+		duration = globalNetPerfMinDuration
+	}
+	result := siteNetperf(r.Context(), duration)
+	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
+}

cmd/admin-handlers-users.go

@@ -39,9 +39,7 @@ import (
 
 // RemoveUser - DELETE /minio/admin/v3/remove-user?accessKey=<access_key>
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RemoveUser")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
 	if objectAPI == nil {
@@ -85,9 +83,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 
 // ListBucketUsers - GET /minio/admin/v3/list-users?bucket={bucket}
 func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListBucketUsers")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
 	if objectAPI == nil {
@@ -121,9 +117,7 @@ func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request
 
 // ListUsers - GET /minio/admin/v3/list-users
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListUsers")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
 	if objectAPI == nil {
@@ -167,9 +161,7 @@ func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 
 // GetUserInfo - GET /minio/admin/v3/user-info
 func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetUserInfo")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	vars := mux.Vars(r)
 	name := vars["accessKey"]
@@ -224,9 +216,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 
 // UpdateGroupMembers - PUT /minio/admin/v3/update-group-members
 func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "UpdateGroupMembers")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
 	if objectAPI == nil {
@@ -296,9 +286,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 
 // GetGroup - /minio/admin/v3/group?group=mygroup1
 func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "GetGroup")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetGroupAdminAction)
 	if objectAPI == nil {
@@ -325,9 +313,7 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 
 // ListGroups - GET /minio/admin/v3/groups
 func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListGroups")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
 	if objectAPI == nil {
@@ -351,9 +337,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 
 // SetGroupStatus - PUT /minio/admin/v3/set-group-status?group=mygroup1&status=enabled
 func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetGroupStatus")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
 	if objectAPI == nil {
@@ -396,9 +380,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 
 // SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetUserStatus")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
 	if objectAPI == nil {
@@ -436,9 +418,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 
 // AddUser - PUT /minio/admin/v3/add-user?accessKey=<access_key>
 func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AddUser")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	vars := mux.Vars(r)
 	accessKey := vars["accessKey"]
@@ -543,9 +523,7 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 
 // TemporaryAccountInfo - GET /minio/admin/v3/temporary-account-info
 func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "TemporaryAccountInfo")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -628,9 +606,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 
 // AddServiceAccount - PUT /minio/admin/v3/add-service-account
 func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AddServiceAccount")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -861,9 +837,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 
 // UpdateServiceAccount - POST /minio/admin/v3/update-service-account
 func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "UpdateServiceAccount")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -975,9 +949,7 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 
 // InfoServiceAccount - GET /minio/admin/v3/info-service-account
 func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "InfoServiceAccount")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -1074,9 +1046,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 
 // ListServiceAccounts - GET /minio/admin/v3/list-service-accounts
 func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListServiceAccounts")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -1153,9 +1123,7 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 
 // DeleteServiceAccount - DELETE /minio/admin/v3/delete-service-account
 func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "DeleteServiceAccount")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -1226,9 +1194,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 
 // AccountInfoHandler returns usage
 func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AccountInfo")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
@@ -1449,9 +1415,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 // timestamps along with the policy JSON. Both versions are supported for now,
 // for smooth transition to new API.
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "InfoCannedPolicy")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
 	if objectAPI == nil {
@@ -1497,9 +1461,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 
 // ListBucketPolicies - GET /minio/admin/v3/list-canned-policies?bucket={bucket}
 func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListBucketPolicies")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
@@ -1530,9 +1492,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 
 // ListCannedPolicies - GET /minio/admin/v3/list-canned-policies
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListCannedPolicies")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
@@ -1562,9 +1522,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 
 // RemoveCannedPolicy - DELETE /minio/admin/v3/remove-canned-policy?name=<policy_name>
 func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "RemoveCannedPolicy")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
 	if objectAPI == nil {
@@ -1590,9 +1548,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 
 // AddCannedPolicy - PUT /minio/admin/v3/add-canned-policy?name=<policy_name>
 func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AddCannedPolicy")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
 	if objectAPI == nil {
@@ -1656,9 +1612,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 
 // SetPolicyForUserOrGroup - PUT /minio/admin/v3/set-policy?policy=xxx&user-or-group=?[&is-group]
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SetPolicyForUserOrGroup")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
 	if objectAPI == nil {
@@ -1730,9 +1684,7 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 
 // ListPolicyMappingEntities - GET /minio/admin/v3/idp/builtin/polciy-entities?policy=xxx&user=xxx&group=xxx
 func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListPolicyMappingEntities")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Check authorization.
 	objectAPI, cred := validateAdminReq(ctx, w, r,
@@ -1772,9 +1724,7 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht
 
 // AttachDetachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/policy/{operation}
 func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AttachPolicyBuiltin")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction,
 		iampolicy.AttachPolicyAdminAction)
@@ -1873,8 +1823,7 @@ const (
 
 // ExportIAMHandler - exports all iam info as a zipped file
 func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ExportIAM")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Get current object layer instance.
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportIAMAction)
@@ -2112,9 +2061,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 
 // ImportIAM - imports all IAM info into MinIO
 func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ImportIAM")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()

cmd/admin-handlers.go

@@ -78,9 +78,7 @@ const (
 // ----------
 // updates all minio servers and restarts them gracefully.
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ServerUpdate")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction)
 	if objectAPI == nil {
@@ -229,9 +227,7 @@ func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Req
 // - freeze (freezes all incoming S3 API calls)
 // - unfreeze (unfreezes previously frozen S3 API calls)
 func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "Service")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	vars := mux.Vars(r)
 	action := vars["action"]
@@ -297,15 +293,12 @@ type ServerProperties struct {
 	SQSARN       []string `json:"sqsARN"`
 }
 
-// ServerConnStats holds transferred bytes from/to the server
-type ServerConnStats struct {
-	TotalInputBytes  uint64 `json:"transferred"`
-	TotalOutputBytes uint64 `json:"received"`
-	Throughput       uint64 `json:"throughput,omitempty"`
-	S3InputBytes     uint64 `json:"transferredS3"`
-	S3OutputBytes    uint64 `json:"receivedS3"`
-	AdminInputBytes  uint64 `json:"transferredAdmin"`
-	AdminOutputBytes uint64 `json:"receivedAdmin"`
+// serverConnStats holds transferred bytes from/to the server
+type serverConnStats struct {
+	internodeInputBytes  uint64
+	internodeOutputBytes uint64
+	s3InputBytes         uint64
+	s3OutputBytes        uint64
 }
 
 // ServerHTTPAPIStats holds total number of HTTP operations from/to the server,
@@ -335,9 +328,7 @@ type ServerHTTPStats struct {
 // ----------
 // Get server information
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "StorageInfo")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
 	if objectAPI == nil {
@@ -376,9 +367,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 // ----------
 // Get realtime server metrics
 func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "Metrics")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
 	if objectAPI == nil {
@@ -487,9 +476,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 // ----------
 // Get server/cluster data usage info
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "DataUsageInfo")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
@@ -572,9 +559,7 @@ type PeerLocks struct {
 
 // ForceUnlockHandler force unlocks requested resource
 func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ForceUnlock")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ForceUnlockAdminAction)
 	if objectAPI == nil {
@@ -609,9 +594,7 @@ func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Requ
 
 // TopLocksHandler Get list of locks in use
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "TopLocks")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
 	if objectAPI == nil {
@@ -661,9 +644,7 @@ type StartProfilingResult struct {
 // ----------
 // Enable server profiling
 func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "StartProfiling")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
@@ -748,9 +729,7 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 // ----------
 // Enable server profiling
 func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "Profile")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
@@ -845,9 +824,7 @@ func (f dummyFileInfo) Sys() interface{}   { return f.sys }
 // ----------
 // Download profiling information of all nodes in a zip format - deprecated API
 func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "DownloadProfiling")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
@@ -946,9 +923,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 // sequence. However, if the force-start flag is provided, the server
 // aborts the running heal sequence and starts a new one.
 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "Heal")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
@@ -1132,9 +1107,7 @@ func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmi
 }
 
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "HealBackgroundStatus")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
 	if objectAPI == nil {
@@ -1153,11 +1126,55 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 	}
 }
 
+// SitePerfHandler -  measures network throughput between site replicated setups
+func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+
+	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	if objectAPI == nil {
+		return
+	}
+
+	if !globalSiteReplicationSys.isEnabled() {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	nsLock := objectAPI.NewNSLock(minioMetaBucket, "site-net-perf")
+	lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
+		return
+	}
+	defer nsLock.Unlock(lkctx)
+
+	durationStr := r.Form.Get(peerRESTDuration)
+	duration, err := time.ParseDuration(durationStr)
+	if err != nil {
+		duration = globalNetPerfMinDuration
+	}
+
+	if duration < globalNetPerfMinDuration {
+		// We need sample size of minimum 10 secs.
+		duration = globalNetPerfMinDuration
+	}
+
+	duration = duration.Round(time.Second)
+
+	results, err := globalSiteReplicationSys.Netperf(ctx, duration)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL)
+		return
+	}
+	enc := json.NewEncoder(w)
+	if err := enc.Encode(results); err != nil {
+		return
+	}
+}
+
 // NetperfHandler - perform mesh style network throughput test
 func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "NetperfHandler")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -1197,19 +1214,12 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 	}
 }
 
-// SpeedtestHandler - Deprecated. See ObjectSpeedTestHandler
-func (a adminAPIHandlers) SpeedTestHandler(w http.ResponseWriter, r *http.Request) {
-	a.ObjectSpeedTestHandler(w, r)
-}
-
 // ObjectSpeedTestHandler - reports maximum speed of a cluster by performing PUT and
 // GET operations on the server, supports auto tuning by default by automatically
 // increasing concurrency and stopping when we have reached the limits on the
 // system.
 func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ObjectSpeedTestHandler")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -1368,18 +1378,9 @@ func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo,
 	return true, autotune, ""
 }
 
-// NetSpeedtestHandler - reports maximum network throughput
-func (a adminAPIHandlers) NetSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "NetSpeedtestHandler")
-
-	writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-}
-
 // DriveSpeedtestHandler - reports throughput of drives available in the cluster
 func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "DriveSpeedtestHandler")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -1499,7 +1500,7 @@ func extractTraceOptions(r *http.Request) (opts madmin.ServiceTraceOpts, err err
 // ----------
 // The handler sends http trace to the connected HTTP client.
 func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "HTTPTrace")
+	ctx := r.Context()
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "")
@@ -1571,9 +1572,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 
 // The ConsoleLogHandler handler sends console logs to the connected HTTP client.
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ConsoleLog")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
 	if objectAPI == nil {
@@ -1654,8 +1653,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 
 // KMSCreateKeyHandler - POST /minio/admin/v3/kms/key/create?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "KMSCreateKey")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction)
 	if objectAPI == nil {
@@ -1676,8 +1674,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 
 // KMSKeyStatusHandler - GET /minio/admin/v3/kms/status
 func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "KMSStatus")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
@@ -1714,9 +1711,7 @@ func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Reques
 
 // KMSKeyStatusHandler - GET /minio/admin/v3/kms/key/status?key-id=<master-key-id>
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "KMSKeyStatus")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
@@ -1811,6 +1806,7 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]
 				dataUsageInfo := cache.dui(dataUsageRoot, nil)
 				erasureSet.ObjectsCount = dataUsageInfo.ObjectsTotalCount
 				erasureSet.VersionsCount = dataUsageInfo.VersionsTotalCount
+				erasureSet.DeleteMarkersCount = dataUsageInfo.DeleteMarkersTotalCount
 				erasureSet.Usage = dataUsageInfo.ObjectsTotalSize
 			}
 		}
@@ -1826,8 +1822,6 @@ func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]
 }
 
 func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request) madmin.InfoMessage {
-	kmsStat := fetchKMSStatus()
-
 	ldap := madmin.LDAP{}
 	if globalIAMSys.LDAPConfig.Enabled() {
 		ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect()
@@ -1862,6 +1856,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 	buckets := madmin.Buckets{}
 	objects := madmin.Objects{}
 	versions := madmin.Versions{}
+	deleteMarkers := madmin.DeleteMarkers{}
 	usage := madmin.Usage{}
 
 	objectAPI := newObjectLayerFn()
@@ -1874,10 +1869,12 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 			buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount}
 			objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount}
 			versions = madmin.Versions{Count: dataUsageInfo.VersionsTotalCount}
+			deleteMarkers = madmin.DeleteMarkers{Count: dataUsageInfo.DeleteMarkersTotalCount}
 			usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize}
 		} else {
 			buckets = madmin.Buckets{Error: err.Error()}
 			objects = madmin.Objects{Error: err.Error()}
+			deleteMarkers = madmin.DeleteMarkers{Error: err.Error()}
 			usage = madmin.Usage{Error: err.Error()}
 		}
 
@@ -1907,7 +1904,8 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 
 	domain := globalDomainNames
 	services := madmin.Services{
-		KMS:           kmsStat,
+		KMS:           fetchKMSStatus(),
+		KMSStatus:     fetchKMSStatusV2(ctx),
 		LDAP:          ldap,
 		Logger:        log,
 		Audit:         audit,
@@ -1915,19 +1913,20 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 	}
 
 	return madmin.InfoMessage{
-		Mode:         string(mode),
-		Domain:       domain,
-		Region:       globalSite.Region,
-		SQSARN:       globalEventNotifier.GetARNList(false),
-		DeploymentID: globalDeploymentID,
-		Buckets:      buckets,
-		Objects:      objects,
-		Versions:     versions,
-		Usage:        usage,
-		Services:     services,
-		Backend:      backend,
-		Servers:      servers,
-		Pools:        poolsInfo,
+		Mode:          string(mode),
+		Domain:        domain,
+		Region:        globalSite.Region,
+		SQSARN:        globalEventNotifier.GetARNList(false),
+		DeploymentID:  globalDeploymentID,
+		Buckets:       buckets,
+		Objects:       objects,
+		Versions:      versions,
+		DeleteMarkers: deleteMarkers,
+		Usage:         usage,
+		Services:      services,
+		Backend:       backend,
+		Servers:       servers,
+		Pools:         poolsInfo,
 	}
 }
 
@@ -2312,8 +2311,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 // ----------
 // Get server health info
 func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "HealthInfo")
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
 	if objectAPI == nil {
@@ -2419,9 +2417,7 @@ func getTLSInfo() madmin.TLSInfo {
 // ----------
 // Get server information
 func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ServerInfo")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
@@ -2527,6 +2523,28 @@ func fetchKMSStatus() madmin.KMS {
 	return kmsStat
 }
 
+// fetchKMSStatusV2 fetches KMS-related status information for all instances
+func fetchKMSStatusV2(ctx context.Context) []madmin.KMS {
+	if GlobalKMS == nil {
+		return []madmin.KMS{}
+	}
+
+	results := GlobalKMS.Verify(ctx)
+
+	stats := []madmin.KMS{}
+	for _, result := range results {
+		stats = append(stats, madmin.KMS{
+			Status:   result.Status,
+			Endpoint: result.Endpoint,
+			Encrypt:  result.Encrypt,
+			Decrypt:  result.Decrypt,
+			Version:  result.Version,
+		})
+	}
+
+	return stats
+}
+
 // fetchLoggerDetails return log info
 func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
 	var loggerInfo []madmin.Logger
@@ -2651,7 +2669,7 @@ type getRawDataer interface {
 // ----------
 // Download file from all nodes in a zip format
 func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "InspectData")
+	ctx := r.Context()
 
 	// Validate request signature.
 	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.InspectDataAction, "")
@@ -2659,7 +2677,6 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
 	}
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
 	objLayer := newObjectLayerFn()
 	o, ok := objLayer.(getRawDataer)
@@ -2848,6 +2865,36 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	}
 	sb.WriteString("\n")
 	logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes()))
+
+	// save MinIO start script to inspect command
+	var scrb bytes.Buffer
+	scrb.WriteString(`#!/usr/bin/env bash
+
+function main() {
+	for file in $(ls -1); do
+		dest_file=$(echo "$file" | cut -d ":" -f1)
+		mv "$file" "$dest_file"
+	done
+
+	# Read content of inspect-input.txt
+	MINIO_OPTS=$(grep "Server command line args" <./inspect-input.txt | sed "s/Server command line args: //g" | sed -r "s#https:\/\/#\.\/#g")
+
+	# Start MinIO instance using the options
+	START_CMD="CI=on MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server ${MINIO_OPTS} &"
+	echo
+	echo "Starting MinIO instance: ${START_CMD}"
+	echo
+	eval "$START_CMD"
+	MINIO_SRVR_PID="$!"
+	echo "MinIO Server PID: ${MINIO_SRVR_PID}"
+	echo
+	echo "Waiting for MinIO instance to get ready!"
+	sleep 10
+}
+
+main "$@"`,
+	)
+	logger.LogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes()))
 }
 
 func getSubnetAdminPublicKey() []byte {

cmd/admin-handlers_test.go

@@ -73,7 +73,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 	// Initialize boot time
 	globalBootTime = UTCNow()
 
-	globalEndpoints = mustGetPoolEndpoints(erasureDirs...)
+	globalEndpoints = mustGetPoolEndpoints(0, erasureDirs...)
 
 	initAllSubsystems(ctx)
 
@@ -108,7 +108,7 @@ func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error)
 	if err != nil {
 		return nil, nil, err
 	}
-	endpoints := mustGetPoolEndpoints(erasureDirs...)
+	endpoints := mustGetPoolEndpoints(0, erasureDirs...)
 	globalPolicySys = NewPolicySys()
 	objLayer, err := newErasureServerPools(ctx, endpoints)
 	if err != nil {

cmd/admin-router.go

@@ -19,6 +19,9 @@ package cmd
 
 import (
 	"net/http"
+	"reflect"
+	"runtime"
+	"strings"
 
 	"github.com/klauspost/compress/gzhttp"
 	"github.com/klauspost/compress/gzip"
@@ -28,11 +31,113 @@ import (
 )
 
 const (
-	adminPathPrefix       = minioReservedBucketPath + "/admin"
-	adminAPIVersion       = madmin.AdminAPIVersion
-	adminAPIVersionPrefix = SlashSeparator + adminAPIVersion
+	adminPathPrefix                = minioReservedBucketPath + "/admin"
+	adminAPIVersion                = madmin.AdminAPIVersion
+	adminAPIVersionPrefix          = SlashSeparator + adminAPIVersion
+	adminAPISiteReplicationDevNull = "/site-replication/devnull"
+	adminAPISiteReplicationNetPerf = "/site-replication/netperf"
 )
 
+var gzipHandler = func() func(http.Handler) http.HandlerFunc {
+	gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
+	if err != nil {
+		// Static params, so this is very unlikely.
+		logger.Fatal(err, "Unable to initialize server")
+	}
+	return gz
+}()
+
+// Set of handler options as bit flags
+type hFlag uint8
+
+const (
+	// this flag disables gzip compression of responses
+	noGZFlag = 1 << iota
+
+	// this flag enables tracing body and headers instead of just headers
+	traceAllFlag
+
+	// pass this flag to skip checking if object layer is available
+	noObjLayerFlag
+)
+
+// Has checks if the the given flag is enabled in `h`.
+func (h hFlag) Has(flag hFlag) bool {
+	// Use bitwise-AND and check if the result is non-zero.
+	return h&flag != 0
+}
+
+func getHandlerName(f http.HandlerFunc) string {
+	name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()
+	name = strings.TrimPrefix(name, "github.com/minio/minio/cmd.adminAPIHandlers.")
+	name = strings.TrimSuffix(name, "Handler-fm")
+	name = strings.TrimSuffix(name, "-fm")
+	return name
+}
+
+// adminMiddleware performs some common admin handler functionality for all
+// handlers:
+//
+// - updates request context with `logger.ReqInfo` and api name based on the
+// name of the function handler passed (this handler must be a method of
+// `adminAPIHandlers`).
+//
+// - sets up call to send AuditLog
+//
+// Note that, while this is a middleware function (i.e. it takes a handler
+// function and returns one), due to flags being passed based on required
+// conditions, it is done per-"handler function registration" in the router.
+//
+// When no flags are passed, gzip compression, http tracing of headers and
+// checking of object layer availability are all enabled. Use flags to modify
+// this behavior.
+func adminMiddleware(f http.HandlerFunc, flags ...hFlag) http.HandlerFunc {
+	// Collect all flags with bitwise-OR and assign operator
+	var handlerFlags hFlag
+	for _, flag := range flags {
+		handlerFlags |= flag
+	}
+
+	// Get name of the handler using reflection. NOTE: The passed in handler
+	// function must be a method of `adminAPIHandlers` for this extraction to
+	// work as expected.
+	handlerName := getHandlerName(f)
+
+	var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) {
+		// Update request context with `logger.ReqInfo`.
+		r = r.WithContext(newContext(r, w, handlerName))
+
+		defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
+
+		// Check if object layer is available, if not return error early.
+		if !handlerFlags.Has(noObjLayerFlag) {
+			objectAPI := newObjectLayerFn()
+			if objectAPI == nil || globalNotificationSys == nil {
+				writeErrorResponseJSON(r.Context(), w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+				return
+			}
+		}
+
+		// Apply http tracing "middleware" based on presence of flag.
+		var f2 http.HandlerFunc
+		if handlerFlags.Has(traceAllFlag) {
+			f2 = httpTraceAll(f)
+		} else {
+			f2 = httpTraceHdrs(f)
+		}
+
+		// call the final handler
+		f2(w, r)
+	}
+
+	// Enable compression of responses based on presence of flag.
+	if !handlerFlags.Has(noGZFlag) {
+		handler = gzipHandler(handler)
+	}
+
+	return handler
+}
+
 // adminAPIHandlers provides HTTP handlers for MinIO admin API.
 type adminAPIHandlers struct{}
 
@@ -46,262 +151,262 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 		adminAPIVersionPrefix,
 	}
 
-	gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
-	if err != nil {
-		// Static params, so this is very unlikely.
-		logger.Fatal(err, "Unable to initialize server")
-	}
-
 	for _, adminVersion := range adminVersions {
 		// Restart and stop MinIO service.
-		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(gz(httpTraceAll(adminAPI.ServiceHandler))).Queries("action", "{action:.*}")
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(adminMiddleware(adminAPI.ServiceHandler, traceAllFlag)).Queries("action", "{action:.*}")
 		// Update MinIO servers.
-		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(gz(httpTraceAll(adminAPI.ServerUpdateHandler))).Queries("updateURL", "{updateURL:.*}")
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(adminMiddleware(adminAPI.ServerUpdateHandler, traceAllFlag)).Queries("updateURL", "{updateURL:.*}")
 
 		// Info operations
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(gz(httpTraceAll(adminAPI.ServerInfoHandler)))
-		adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(httpTraceAll(adminAPI.InspectDataHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(adminMiddleware(adminAPI.ServerInfoHandler, traceAllFlag, noObjLayerFlag))
+		adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(adminMiddleware(adminAPI.InspectDataHandler, noGZFlag, traceAllFlag))
 
 		// StorageInfo operations
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(adminMiddleware(adminAPI.StorageInfoHandler, traceAllFlag))
 		// DataUsageInfo operations
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.DataUsageInfoHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(adminMiddleware(adminAPI.DataUsageInfoHandler, traceAllFlag))
 		// Metrics operation
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(gz(httpTraceAll(adminAPI.MetricsHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(adminMiddleware(adminAPI.MetricsHandler, traceAllFlag))
 
 		if globalIsDistErasure || globalIsErasure {
 			// Heal operations
 
 			// Heal processing endpoint.
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(gz(httpTraceAll(adminAPI.BackgroundHealStatusHandler)))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(adminMiddleware(adminAPI.HealHandler, traceAllFlag))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}").HandlerFunc(adminMiddleware(adminAPI.HealHandler, traceAllFlag))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(adminMiddleware(adminAPI.HealHandler, traceAllFlag))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(adminMiddleware(adminAPI.BackgroundHealStatusHandler, traceAllFlag))
 
 			// Pool operations
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/pools/list").HandlerFunc(gz(httpTraceAll(adminAPI.ListPools)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/pools/status").HandlerFunc(gz(httpTraceAll(adminAPI.StatusPool))).Queries("pool", "{pool:.*}")
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/pools/list").HandlerFunc(adminMiddleware(adminAPI.ListPools, traceAllFlag))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/pools/status").HandlerFunc(adminMiddleware(adminAPI.StatusPool, traceAllFlag)).Queries("pool", "{pool:.*}")
 
-			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/decommission").HandlerFunc(gz(httpTraceAll(adminAPI.StartDecommission))).Queries("pool", "{pool:.*}")
-			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(gz(httpTraceAll(adminAPI.CancelDecommission))).Queries("pool", "{pool:.*}")
+			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/decommission").HandlerFunc(adminMiddleware(adminAPI.StartDecommission, traceAllFlag)).Queries("pool", "{pool:.*}")
+			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(adminMiddleware(adminAPI.CancelDecommission, traceAllFlag)).Queries("pool", "{pool:.*}")
 
 			// Rebalance operations
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/start").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStart)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/rebalance/status").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStatus)))
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/stop").HandlerFunc(gz(httpTraceAll(adminAPI.RebalanceStop)))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/start").HandlerFunc(adminMiddleware(adminAPI.RebalanceStart, traceAllFlag))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/rebalance/status").HandlerFunc(adminMiddleware(adminAPI.RebalanceStatus, traceAllFlag))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/stop").HandlerFunc(adminMiddleware(adminAPI.RebalanceStop, traceAllFlag))
 		}
 
 		// Profiling operations - deprecated API
-		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(gz(httpTraceAll(adminAPI.StartProfilingHandler))).
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(adminMiddleware(adminAPI.StartProfilingHandler, traceAllFlag, noObjLayerFlag)).
 			Queries("profilerType", "{profilerType:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(gz(httpTraceAll(adminAPI.DownloadProfilingHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(adminMiddleware(adminAPI.DownloadProfilingHandler, traceAllFlag, noObjLayerFlag))
 		// Profiling operations
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/profile").HandlerFunc(gz(httpTraceAll(adminAPI.ProfileHandler)))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/profile").HandlerFunc(adminMiddleware(adminAPI.ProfileHandler, traceAllFlag, noObjLayerFlag))
 
 		// Config KV operations.
 		if enableConfigOps {
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-config-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetConfigKVHandler))).Queries("key", "{key:.*}")
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/set-config-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetConfigKVHandler)))
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/del-config-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.DelConfigKVHandler)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-config-kv").HandlerFunc(adminMiddleware(adminAPI.GetConfigKVHandler)).Queries("key", "{key:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/set-config-kv").HandlerFunc(adminMiddleware(adminAPI.SetConfigKVHandler))
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/del-config-kv").HandlerFunc(adminMiddleware(adminAPI.DelConfigKVHandler))
 		}
 
 		// Enable config help in all modes.
-		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/help-config-kv").HandlerFunc(gz(httpTraceAll(adminAPI.HelpConfigKVHandler))).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/help-config-kv").HandlerFunc(adminMiddleware(adminAPI.HelpConfigKVHandler, traceAllFlag)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
 
 		// Config KV history operations.
 		if enableConfigOps {
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-config-history-kv").HandlerFunc(gz(httpTraceAll(adminAPI.ListConfigHistoryKVHandler))).Queries("count", "{count:[0-9]+}")
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/clear-config-history-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler))).Queries("restoreId", "{restoreId:.*}")
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/restore-config-history-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler))).Queries("restoreId", "{restoreId:.*}")
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-config-history-kv").HandlerFunc(adminMiddleware(adminAPI.ListConfigHistoryKVHandler, traceAllFlag)).Queries("count", "{count:[0-9]+}")
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/clear-config-history-kv").HandlerFunc(adminMiddleware(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/restore-config-history-kv").HandlerFunc(adminMiddleware(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
 		}
 
 		// Config import/export bulk operations
 		if enableConfigOps {
 			// Get config
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/config").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetConfigHandler)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/config").HandlerFunc(adminMiddleware(adminAPI.GetConfigHandler))
 			// Set config
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/config").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetConfigHandler)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/config").HandlerFunc(adminMiddleware(adminAPI.SetConfigHandler))
 		}
 
 		// -- IAM APIs --
 
 		// Add policy IAM
-		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(gz(httpTraceAll(adminAPI.AddCannedPolicy))).Queries("name", "{name:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(adminMiddleware(adminAPI.AddCannedPolicy, traceAllFlag)).Queries("name", "{name:.*}")
 
 		// Add user IAM
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(gz(httpTraceAll(adminAPI.AccountInfoHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(adminMiddleware(adminAPI.AccountInfoHandler, traceAllFlag))
 
-		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddUser))).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(adminMiddleware(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
 
-		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetUserStatus))).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(adminMiddleware(adminAPI.SetUserStatus)).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
 		// Service accounts ops
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddServiceAccount)))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateServiceAccount))).Queries("accessKey", "{accessKey:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.InfoServiceAccount))).Queries("accessKey", "{accessKey:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListServiceAccounts)))
-		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteServiceAccount))).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(adminMiddleware(adminAPI.AddServiceAccount))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update-service-account").HandlerFunc(adminMiddleware(adminAPI.UpdateServiceAccount)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-service-account").HandlerFunc(adminMiddleware(adminAPI.InfoServiceAccount)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(adminMiddleware(adminAPI.ListServiceAccounts))
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(adminMiddleware(adminAPI.DeleteServiceAccount)).Queries("accessKey", "{accessKey:.*}")
 
 		// STS accounts ops
-		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/temporary-account-info").HandlerFunc(gz(httpTraceHdrs(adminAPI.TemporaryAccountInfo))).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/temporary-account-info").HandlerFunc(adminMiddleware(adminAPI.TemporaryAccountInfo)).Queries("accessKey", "{accessKey:.*}")
 
 		// Info policy IAM latest
-		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(gz(httpTraceHdrs(adminAPI.InfoCannedPolicy))).Queries("name", "{name:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(adminMiddleware(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
 		// List policies latest
-		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListBucketPolicies))).Queries("bucket", "{bucket:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListCannedPolicies)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(adminMiddleware(adminAPI.ListBucketPolicies)).Queries("bucket", "{bucket:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(adminMiddleware(adminAPI.ListCannedPolicies))
 
 		// Builtin IAM policy associations
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/builtin/policy-entities").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListPolicyMappingEntities)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/builtin/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListPolicyMappingEntities))
 
 		// Remove policy IAM
-		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveCannedPolicy))).Queries("name", "{name:.*}")
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(adminMiddleware(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
 
 		// Set user or group policy
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-or-group-policy").
-			HandlerFunc(gz(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup))).
+			HandlerFunc(adminMiddleware(adminAPI.SetPolicyForUserOrGroup)).
 			Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
 
 		// Attach/Detach policies to/from user or group
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/{operation}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachDetachPolicyBuiltin)))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/{operation}").HandlerFunc(adminMiddleware(adminAPI.AttachDetachPolicyBuiltin))
 
 		// Remove user IAM
-		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveUser))).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(adminMiddleware(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
 
 		// List users
-		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-users").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListBucketUsers))).Queries("bucket", "{bucket:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListUsers)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-users").HandlerFunc(adminMiddleware(adminAPI.ListBucketUsers)).Queries("bucket", "{bucket:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(adminMiddleware(adminAPI.ListUsers))
 
 		// User info
-		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetUserInfo))).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(adminMiddleware(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
 		// Add/Remove members from group
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateGroupMembers)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(adminMiddleware(adminAPI.UpdateGroupMembers))
 
 		// Get Group
-		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/group").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetGroup))).Queries("group", "{group:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/group").HandlerFunc(adminMiddleware(adminAPI.GetGroup)).Queries("group", "{group:.*}")
 
 		// List Groups
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/groups").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListGroups)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/groups").HandlerFunc(adminMiddleware(adminAPI.ListGroups))
 
 		// Set Group Status
-		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetGroupStatus))).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(adminMiddleware(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
 
 		// Export IAM info to zipped file
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-iam").HandlerFunc(httpTraceHdrs(adminAPI.ExportIAM))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-iam").HandlerFunc(adminMiddleware(adminAPI.ExportIAM, noGZFlag))
 
 		// Import IAM info
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(httpTraceHdrs(adminAPI.ImportIAM))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(adminMiddleware(adminAPI.ImportIAM, noGZFlag))
 
 		// IDentity Provider configuration APIs
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddIdentityProviderCfg)))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateIdentityProviderCfg)))
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListIdentityProviderCfg)))
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetIdentityProviderCfg)))
-		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteIdentityProviderCfg)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.AddIdentityProviderCfg))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.UpdateIdentityProviderCfg))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}").HandlerFunc(adminMiddleware(adminAPI.ListIdentityProviderCfg))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.GetIdentityProviderCfg))
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.DeleteIdentityProviderCfg))
 
 		// LDAP IAM operations
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListLDAPPolicyMappingEntities)))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachDetachPolicyLDAP)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListLDAPPolicyMappingEntities))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(adminMiddleware(adminAPI.AttachDetachPolicyLDAP))
 		// -- END IAM APIs --
 
 		// GetBucketQuotaConfig
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
+			adminMiddleware(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
 		// PutBucketQuotaConfig
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
+			adminMiddleware(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
 
 		// Bucket replication operations
 		// GetBucketTargetHandler
 		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.ListRemoteTargetsHandler))).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+			adminMiddleware(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
 		// SetRemoteTargetHandler
 		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.SetRemoteTargetHandler))).Queries("bucket", "{bucket:.*}")
+			adminMiddleware(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
 		// RemoveRemoteTargetHandler
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler))).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
+			adminMiddleware(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
 		// ReplicationDiff - MinIO extension API
 		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/replication/diff").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.ReplicationDiffHandler))).Queries("bucket", "{bucket:.*}")
+			adminMiddleware(adminAPI.ReplicationDiffHandler)).Queries("bucket", "{bucket:.*}")
+		// ReplicationMRFHandler - MinIO extension API
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/replication/mrf").HandlerFunc(
+			adminMiddleware(adminAPI.ReplicationMRFHandler)).Queries("bucket", "{bucket:.*}")
 
 		// Batch job operations
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/start-job").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.StartBatchJob)))
+			adminMiddleware(adminAPI.StartBatchJob))
 
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-jobs").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.ListBatchJobs)))
+			adminMiddleware(adminAPI.ListBatchJobs))
 
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.DescribeBatchJob)))
+			adminMiddleware(adminAPI.DescribeBatchJob))
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/cancel-job").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.CancelBatchJob)))
+			adminMiddleware(adminAPI.CancelBatchJob))
 
 		// Bucket migration operations
 		// ExportBucketMetaHandler
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-bucket-metadata").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.ExportBucketMetadataHandler)))
+			adminMiddleware(adminAPI.ExportBucketMetadataHandler))
 		// ImportBucketMetaHandler
 		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-bucket-metadata").HandlerFunc(
-			gz(httpTraceHdrs(adminAPI.ImportBucketMetadataHandler)))
+			adminMiddleware(adminAPI.ImportBucketMetadataHandler))
 
 		// Remote Tier management operations
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler)))
-		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveTierHandler)))
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.VerifyTierHandler)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(adminMiddleware(adminAPI.AddTierHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(adminMiddleware(adminAPI.EditTierHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(adminMiddleware(adminAPI.ListTierHandler))
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/tier/{tier}").HandlerFunc(adminMiddleware(adminAPI.RemoveTierHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier/{tier}").HandlerFunc(adminMiddleware(adminAPI.VerifyTierHandler))
 		// Tier stats
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(gz(httpTraceHdrs(adminAPI.TierStatsHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(adminMiddleware(adminAPI.TierStatsHandler))
 
 		// Cluster Replication APIs
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationRemove)))
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationMetaInfo)))
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationStatus)))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationAdd))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/remove").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationRemove))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationInfo))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationMetaInfo))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationStatus))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPISiteReplicationDevNull).HandlerFunc(adminMiddleware(adminAPI.SiteReplicationDevNull, noObjLayerFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPISiteReplicationNetPerf).HandlerFunc(adminMiddleware(adminAPI.SiteReplicationNetPerf, noObjLayerFlag))
 
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerJoin)))
-		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationEdit)))
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerEdit)))
-		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerRemove)))
-		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/resync/op").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationResyncOp))).Queries("operation", "{operation:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(adminMiddleware(adminAPI.SRPeerEdit))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(adminMiddleware(adminAPI.SRPeerRemove))
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/resync/op").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationResyncOp)).Queries("operation", "{operation:.*}")
 
 		if globalIsDistErasure {
 			// Top locks
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(gz(httpTraceHdrs(adminAPI.TopLocksHandler)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(adminMiddleware(adminAPI.TopLocksHandler))
 			// Force unlocks paths
 			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/force-unlock").
-				Queries("paths", "{paths:.*}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ForceUnlockHandler)))
+				Queries("paths", "{paths:.*}").HandlerFunc(adminMiddleware(adminAPI.ForceUnlockHandler))
 		}
 
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(httpTraceHdrs(adminAPI.SpeedTestHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(httpTraceHdrs(adminAPI.ObjectSpeedTestHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(httpTraceHdrs(adminAPI.DriveSpeedtestHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(httpTraceHdrs(adminAPI.NetperfHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(adminMiddleware(adminAPI.ObjectSpeedTestHandler, noGZFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(adminMiddleware(adminAPI.ObjectSpeedTestHandler, noGZFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(adminMiddleware(adminAPI.DriveSpeedtestHandler, noGZFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(adminMiddleware(adminAPI.NetperfHandler, noGZFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/site").HandlerFunc(adminMiddleware(adminAPI.SitePerfHandler, noGZFlag))
 
 		// HTTP Trace
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(gz(http.HandlerFunc(adminAPI.TraceHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(adminMiddleware(adminAPI.TraceHandler, noObjLayerFlag))
 
 		// Console Logs
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/log").HandlerFunc(gz(httpTraceAll(adminAPI.ConsoleLogHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/log").HandlerFunc(adminMiddleware(adminAPI.ConsoleLogHandler, traceAllFlag))
 
 		// -- KMS APIs --
 		//
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/kms/status").HandlerFunc(gz(httpTraceAll(adminAPI.KMSStatusHandler)))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(gz(httpTraceAll(adminAPI.KMSCreateKeyHandler))).Queries("key-id", "{key-id:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(gz(httpTraceAll(adminAPI.KMSKeyStatusHandler)))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/kms/status").HandlerFunc(adminMiddleware(adminAPI.KMSStatusHandler, traceAllFlag))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(adminMiddleware(adminAPI.KMSCreateKeyHandler, traceAllFlag)).Queries("key-id", "{key-id:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(adminMiddleware(adminAPI.KMSKeyStatusHandler, traceAllFlag))
 
 		// Keep obdinfo for backward compatibility with mc
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
-			HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
+			HandlerFunc(adminMiddleware(adminAPI.HealthInfoHandler))
 		// -- Health API --
 		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
-			HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
+			HandlerFunc(adminMiddleware(adminAPI.HealthInfoHandler))
 	}
 
 	// If none of the routes match add default error handler routes

cmd/api-errors.go

@@ -138,6 +138,8 @@ const (
 	ErrReplicationDenyEditError
 	ErrRemoteTargetDenyAddError
 	ErrReplicationNoExistingObjects
+	ErrReplicationValidationError
+	ErrReplicationPermissionCheckError
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
 	ErrNoSuchUpload
@@ -1015,6 +1017,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Versioning must be 'Enabled' on the bucket to add a replication target",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrReplicationValidationError: {
+		Code:           "InvalidRequest",
+		Description:    "Replication validation failed on target",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrReplicationPermissionCheckError: {
+		Code:           "ReplicationPermissionCheck",
+		Description:    "X-Minio-Source-Replication-Check cannot be specified in request. Request cannot be completed",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrNoSuchObjectLockConfiguration: {
 		Code:           "NoSuchObjectLockConfiguration",
 		Description:    "The specified object does not have a ObjectLock configuration",

cmd/api-headers.go

@@ -153,7 +153,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 			continue
 		}
 
-		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+		if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
 			// Do not need to send any internal metadata
 			// values to client.
 			continue
@@ -166,7 +166,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 
 		var isSet bool
 		for _, userMetadataPrefix := range userMetadataKeyPrefixes {
-			if !strings.HasPrefix(strings.ToLower(k), strings.ToLower(userMetadataPrefix)) {
+			if !stringsHasPrefixFold(k, userMetadataPrefix) {
 				continue
 			}
 			w.Header()[strings.ToLower(k)] = []string{v}

cmd/api-response.go

@@ -550,7 +550,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 			}
 			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
-				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+				if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
 					// Do not need to send any internal metadata
 					// values to client.
 					continue
@@ -693,7 +693,7 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 					content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 				}
 				for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
-					if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+					if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
 						// Do not need to send any internal metadata
 						// values to client.
 						continue

cmd/api-router.go

@@ -464,6 +464,9 @@ func registerAPIRouter(router *mux.Router) {
 		// GetBucketReplicationMetrics
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "")
+		// ValidateBucketReplicationCreds
+		router.Methods(http.MethodGet).HandlerFunc(
+			collectAPIStats("checkbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.ValidateBucketReplicationCredsHandler))))).Queries("replication-check", "")
 
 		// Register rejected bucket APIs
 		for _, r := range rejectedBucketAPIs {
@@ -520,14 +523,9 @@ func corsHandler(handler http.Handler) http.Handler {
 		"x-amz*",
 		"*",
 	}
-
-	return cors.New(cors.Options{
+	opts := cors.Options{
 		AllowOriginFunc: func(origin string) bool {
-			allowedOrigins := globalAPIConfig.getCorsAllowOrigins()
-			if len(allowedOrigins) == 0 {
-				allowedOrigins = []string{"*"}
-			}
-			for _, allowedOrigin := range allowedOrigins {
+			for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() {
 				if wildcard.MatchSimple(allowedOrigin, origin) {
 					return true
 				}
@@ -546,5 +544,13 @@ func corsHandler(handler http.Handler) http.Handler {
 		AllowedHeaders:   commonS3Headers,
 		ExposedHeaders:   commonS3Headers,
 		AllowCredentials: true,
-	}).Handler(handler)
+	}
+	for _, origin := range globalAPIConfig.getCorsAllowOrigins() {
+		if origin == "*" {
+			opts.AllowOriginFunc = nil
+			opts.AllowedOrigins = globalAPIConfig.getCorsAllowOrigins()
+			break
+		}
+	}
+	return cors.New(opts).Handler(handler)
 }

cmd/auth-handler.go

@@ -595,8 +595,8 @@ func isSupportedS3AuthType(aType authType) bool {
 	return ok
 }
 
-// setAuthHandler to validate authorization header for the incoming request.
-func setAuthHandler(h http.Handler) http.Handler {
+// setAuthMiddleware to validate authorization header for the incoming request.
+func setAuthMiddleware(h http.Handler) http.Handler {
 	// handler for validating incoming authorization headers.
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)

cmd/background-newdisks-heal-ops.go

@@ -485,7 +485,12 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 	}
 
 	// Remove .healing.bin from all disks with similar heal-id
-	for _, disk := range z.serverPools[poolIdx].sets[setIdx].getDisks() {
+	disks, err := z.GetDisks(poolIdx, setIdx)
+	if err != nil {
+		return err
+	}
+
+	for _, disk := range disks {
 		t, err := loadHealingTracker(ctx, disk)
 		if err != nil {
 			if !errors.Is(err, errFileNotFound) {

cmd/batch-handlers.go

@@ -520,7 +520,7 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		if len(r.Flags.Filter.Metadata) > 0 {
 			for _, kv := range r.Flags.Filter.Metadata {
 				for k, v := range oi.UserDefined {
-					if !strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") && !isStandardHeader(k) {
+					if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
 						continue
 					}
 					// We only need to match x-amz-meta or standardHeaders
@@ -1075,7 +1075,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		if len(r.Flags.Filter.Metadata) > 0 {
 			for _, kv := range r.Flags.Filter.Metadata {
 				for k, v := range info.Metadata {
-					if !strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") && !isStandardHeader(k) {
+					if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
 						continue
 					}
 					// We only need to match x-amz-meta or standardHeaders
@@ -1463,9 +1463,7 @@ func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (p
 // ListBatchJobs - lists all currently active batch jobs, optionally takes {jobType}
 // input to list only active batch jobs of 'jobType'
 func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "ListBatchJobs")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListBatchJobsAction)
 	if objectAPI == nil {
@@ -1515,9 +1513,7 @@ var errNoSuchJob = errors.New("no such job")
 
 // DescribeBatchJob returns the currently active batch job definition
 func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "DescribeBatchJob")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DescribeBatchJobAction)
 	if objectAPI == nil {
@@ -1552,9 +1548,7 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 
 // StarBatchJob queue a new job for execution
 func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "StartBatchJob")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.StartBatchJobAction)
 	if objectAPI == nil {
@@ -1608,9 +1602,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 
 // CancelBatchJob cancels a job in progress
 func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "CancelBatchJob")
-
-	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+	ctx := r.Context()
 
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CancelBatchJobAction)
 	if objectAPI == nil {

cmd/batch-rotate.go

@@ -289,7 +289,7 @@ func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, ob
 	)
 	encMetadata := make(map[string]string)
 	for k, v := range oi.UserDefined {
-		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+		if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
 			encMetadata[k] = v
 		}
 	}
@@ -401,7 +401,7 @@ func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		if len(r.Flags.Filter.Metadata) > 0 {
 			for _, kv := range r.Flags.Filter.Metadata {
 				for k, v := range info.Metadata {
-					if !strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") && !isStandardHeader(k) {
+					if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
 						continue
 					}
 					// We only need to match x-amz-meta or standardHeaders

cmd/bitrot-streaming.go

@@ -167,7 +167,7 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 				if !IsErr(err, ignoredErrs...) {
 					logger.LogOnceIf(GlobalContext,
 						fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
-							b.disk, b.volume, b.filePath, err), "bitrot-read-file-stream"+b.volume+b.filePath)
+							b.disk, b.volume, b.filePath, err), "bitrot-read-file-stream-"+b.volume+"-"+b.filePath)
 				}
 			}
 		} else {

cmd/bootstrap-peer-server.go

@@ -153,14 +153,18 @@ func (b *bootstrapRESTServer) VerifyHandler(w http.ResponseWriter, r *http.Reque
 
 // registerBootstrapRESTHandlers - register bootstrap rest router.
 func registerBootstrapRESTHandlers(router *mux.Router) {
+	h := func(f http.HandlerFunc) http.HandlerFunc {
+		return collectInternodeStats(httpTraceHdrs(f))
+	}
+
 	server := &bootstrapRESTServer{}
 	subrouter := router.PathPrefix(bootstrapRESTPrefix).Subrouter()
 
 	subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodHealth).HandlerFunc(
-		httpTraceHdrs(server.HealthHandler))
+		h(server.HealthHandler))
 
 	subrouter.Methods(http.MethodPost).Path(bootstrapRESTVersionPrefix + bootstrapRESTMethodVerify).HandlerFunc(
-		httpTraceHdrs(server.VerifyHandler))
+		h(server.VerifyHandler))
 }
 
 // client to talk to bootstrap NEndpoints.

cmd/bucket-handlers.go

@@ -20,7 +20,9 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"crypto/md5"
 	"encoding/base64"
+	"encoding/hex"
 	"encoding/json"
 	"encoding/xml"
 	"errors"
@@ -578,8 +580,8 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			}
 		}
 		if object.VersionID != "" && hasLockEnabled {
-			if apiErrCode := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); apiErrCode != ErrNone {
-				apiErr := errorCodes.ToAPIErr(apiErrCode)
+			if err := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); err != nil {
+				apiErr := toAPIError(ctx, err)
 				deleteResults[index].errInfo = DeleteError{
 					Code:      apiErr.Code,
 					Message:   apiErr.Description,
@@ -1290,15 +1292,20 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		buf := bytebufferpool.Get()
 		defer bytebufferpool.Put(buf)
 
+		md5w := md5.New()
+
 		// Maximum allowed fan-out object size.
 		const maxFanOutSize = 16 << 20
 
-		n, err := io.Copy(buf, ioutil.HardLimitReader(pReader, maxFanOutSize))
+		n, err := io.Copy(io.MultiWriter(buf, md5w), ioutil.HardLimitReader(pReader, maxFanOutSize))
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
 
+		// Set the correct hex md5sum for the fan-out stream.
+		fanOutOpts.MD5Hex = hex.EncodeToString(md5w.Sum(nil))
+
 		concurrentSize := 100
 		if runtime.GOMAXPROCS(0) < concurrentSize {
 			concurrentSize = runtime.GOMAXPROCS(0)
@@ -1614,10 +1621,14 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 	}
 
-	deleteBucket := objectAPI.DeleteBucket
+	// Return an error if the bucket does not exist
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil && !forceDelete {
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
 
 	// Attempt to delete bucket.
-	if err := deleteBucket(ctx, bucket, DeleteBucketOptions{
+	if err := objectAPI.DeleteBucket(ctx, bucket, DeleteBucketOptions{
 		Force:      forceDelete,
 		SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()),
 	}); err != nil {

cmd/bucket-lifecycle.go

@@ -691,7 +691,7 @@ func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo O
 
 	if rreq.Type == SelectRestoreRequest {
 		for _, v := range rreq.OutputLocation.S3.UserMetadata {
-			if !strings.HasPrefix(strings.ToLower(v.Name), "x-amz-meta") {
+			if !stringsHasPrefixFold(v.Name, "x-amz-meta") {
 				meta["x-amz-meta-"+v.Name] = v.Value
 				continue
 			}

cmd/bucket-metadata-sys.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"runtime"
 	"sync"
 	"time"
 
@@ -489,6 +488,8 @@ func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []Buck
 func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
 	const bucketMetadataRefresh = 15 * time.Minute
 
+	sleeper := newDynamicSleeper(2, 150*time.Millisecond, false)
+
 	t := time.NewTimer(bucketMetadataRefresh)
 	defer t.Stop()
 	for {
@@ -512,13 +513,16 @@ func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) {
 			sys.RemoveStaleBuckets(diskBuckets)
 
 			for _, bucket := range buckets {
+				wait := sleeper.Timer(ctx)
+
 				err := sys.loadBucketMetadata(ctx, bucket)
 				if err != nil {
 					logger.LogIf(ctx, err)
+					wait() // wait to proceed to next entry.
 					continue
 				}
-				// Check if there is a spare procs, wait 100ms instead
-				waitForLowIO(runtime.GOMAXPROCS(0), 100*time.Millisecond, currentHTTPIO)
+
+				wait() // wait to proceed to next entry.
 			}
 
 			t.Reset(bucketMetadataRefresh)

cmd/bucket-object-lock.go

@@ -82,24 +82,24 @@ func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locke
 // For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR
 // governance bypass headers are set and user has governance bypass permissions.
 // Objects in "Compliance" mode can be overwritten only if retention date is past.
-func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, oi ObjectInfo, gerr error) APIErrorCode {
+func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, oi ObjectInfo, gerr error) error {
 	if gerr != nil { // error from GetObjectInfo
 		if _, ok := gerr.(MethodNotAllowed); ok {
 			// This happens usually for a delete marker
 			if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 				// Delete marker should be present and valid.
-				return ErrNone
+				return nil
 			}
 		}
 		if isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr) {
-			return ErrNone
+			return nil
 		}
-		return toAPIErrorCode(ctx, gerr)
+		return gerr
 	}
 
 	lhold := objectlock.GetObjectLegalHoldMeta(oi.UserDefined)
 	if lhold.Status.Valid() && lhold.Status == objectlock.LegalHoldOn {
-		return ErrObjectLocked
+		return ObjectLocked{}
 	}
 
 	ret := objectlock.GetObjectRetentionMeta(oi.UserDefined)
@@ -115,13 +115,13 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 			t, err := objectlock.UTCNowNTP()
 			if err != nil {
 				logger.LogIf(ctx, err)
-				return ErrObjectLocked
+				return ObjectLocked{}
 			}
 
 			if !ret.RetainUntilDate.Before(t) {
-				return ErrObjectLocked
+				return ObjectLocked{}
 			}
-			return ErrNone
+			return nil
 		case objectlock.RetGovernance:
 			// In governance mode, users can't overwrite or delete an object
 			// version or alter its lock settings unless they have special
@@ -141,22 +141,22 @@ func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucke
 				t, err := objectlock.UTCNowNTP()
 				if err != nil {
 					logger.LogIf(ctx, err)
-					return ErrObjectLocked
+					return ObjectLocked{}
 				}
 
 				if !ret.RetainUntilDate.Before(t) {
-					return ErrObjectLocked
+					return ObjectLocked{}
 				}
-				return ErrNone
+				return nil
 			}
 			// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes
 			// If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention, the operation will succeed.
 			if checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName) != ErrNone {
-				return ErrAccessDenied
+				return errAuthentication
 			}
 		}
 	}
-	return ErrNone
+	return nil
 }
 
 // enforceRetentionBypassForPut enforces whether an existing object under governance can be overwritten

cmd/bucket-quota.go

@@ -65,10 +65,10 @@ func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, error) {
 	v, err := sys.bucketStorageCache.Get()
 	if err != nil && v != nil {
-		logger.LogIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err))
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
 	}
 	if v == nil {
-		logger.LogIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"))
+		logger.LogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
 	}
 
 	var bui BucketUsageInfo

cmd/bucket-replication-handlers.go

@@ -18,13 +18,18 @@
 package cmd
 
 import (
+	"bytes"
 	"encoding/json"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
+	"path"
 	"time"
 
+	"github.com/minio/minio-go/v7"
+	objectlock "github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
@@ -466,3 +471,148 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 	// Write success response.
 	writeSuccessResponseJSON(w, data)
 }
+
+// ValidateBucketReplicationCredsHandler - validate replication credentials for a bucket.
+// ----------
+func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "ValidateBucketReplicationCreds")
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	vars := mux.Vars(r)
+	bucket := vars["bucket"]
+	objectAPI := api.ObjectAPI()
+	if objectAPI == nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+		return
+	}
+	if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
+		return
+	}
+	// Check if bucket exists.
+	if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL)
+		return
+	}
+
+	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
+		return
+	}
+	replicationConfig, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
+	if err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationConfigurationNotFoundError, err), r.URL)
+		return
+	}
+
+	lockEnabled := false
+	lcfg, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
+	if err != nil {
+		if !errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucket}) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL)
+			return
+		}
+	}
+	if lcfg != nil {
+		lockEnabled = lcfg.Enabled()
+	}
+
+	sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, true)
+	if apiErr != noError {
+		writeErrorResponse(ctx, w, apiErr, r.URL)
+		return
+	}
+
+	// Validate the bucket replication config
+	if err = replicationConfig.Validate(bucket, sameTarget); err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL)
+		return
+	}
+	buf := bytes.Repeat([]byte("a"), 8)
+	for _, rule := range replicationConfig.Rules {
+		if rule.Status == replication.Disabled {
+			continue
+		}
+		clnt := globalBucketTargetSys.GetRemoteTargetClient(ctx, rule.Destination.Bucket)
+		if clnt == nil {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetNotFoundError, fmt.Errorf("replication config with rule ID %s has a stale target", rule.ID)), r.URL)
+			return
+		}
+		if lockEnabled {
+			lock, _, _, _, err := clnt.GetObjectLockConfig(ctx, clnt.Bucket)
+			if err != nil {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL)
+				return
+			}
+			if lock != objectlock.Enabled {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationDestinationMissingLock, fmt.Errorf("target bucket %s is not object lock enabled", clnt.Bucket)), r.URL)
+				return
+			}
+		}
+		vcfg, err := clnt.GetBucketVersioning(ctx, clnt.Bucket)
+		if err != nil {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL)
+			return
+		}
+		if !vcfg.Enabled() {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetNotVersionedError, fmt.Errorf("target bucket %s is not versioned", clnt.Bucket)), r.URL)
+			return
+		}
+		if sameTarget && bucket == clnt.Bucket {
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL)
+			return
+		}
+
+		reader := bytes.NewReader(buf)
+		// fake a PutObject and RemoveObject call to validate permissions
+		c := &minio.Core{Client: clnt.Client}
+		putOpts := minio.PutObjectOptions{
+			Internal: minio.AdvancedPutOptions{
+				SourceVersionID:          mustGetUUID(),
+				ReplicationStatus:        minio.ReplicationStatusReplica,
+				SourceMTime:              time.Now(),
+				ReplicationRequest:       true, // always set this to distinguish between `mc mirror` replication and serverside
+				ReplicationValidityCheck: true, // set this to validate the replication config
+			},
+		}
+		obj := path.Join(minioReservedBucket, globalLocalNodeNameHex, "deleteme")
+		ui, err := c.PutObject(ctx, clnt.Bucket, obj, reader, int64(len(buf)), "", "", putOpts)
+		if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateObject permissions missing for replication user: %w", err)), r.URL)
+			return
+		}
+
+		err = c.RemoveObject(ctx, clnt.Bucket, obj, minio.RemoveObjectOptions{
+			VersionID: ui.VersionID,
+			Internal: minio.AdvancedRemoveOptions{
+				ReplicationDeleteMarker:  true,
+				ReplicationMTime:         time.Now(),
+				ReplicationStatus:        minio.ReplicationStatusReplica,
+				ReplicationRequest:       true, // always set this to distinguish between `mc mirror` replication and serverside
+				ReplicationValidityCheck: true, // set this to validate the replication config
+			},
+		})
+		if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateDelete permissions missing for replication user: %w", err)), r.URL)
+			return
+		}
+		// fake a versioned delete - to ensure deny policies are not in place
+		err = c.RemoveObject(ctx, clnt.Bucket, obj, minio.RemoveObjectOptions{
+			VersionID: ui.VersionID,
+			Internal: minio.AdvancedRemoveOptions{
+				ReplicationDeleteMarker:  false,
+				ReplicationMTime:         time.Now(),
+				ReplicationStatus:        minio.ReplicationStatusReplica,
+				ReplicationRequest:       true, // always set this to distinguish between `mc mirror` replication and serverside
+				ReplicationValidityCheck: true, // set this to validate the replication config
+			},
+		})
+		if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateDelete/s3:DeleteObject permissions missing for replication user: %w", err)), r.URL)
+			return
+		}
+	}
+
+	// Write success response.
+	writeSuccessResponseHeadersOnly(w)
+}

cmd/bucket-replication-utils.go

@@ -536,12 +536,18 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 	}
 }
 
+func (ri *ReplicateObjectInfo) getReplicationState() ReplicationState {
+	rs := ri.ObjectInfo.getReplicationState()
+	rs.ReplicateDecisionStr = ri.Dsc.String()
+	return rs
+}
+
 // vID here represents the versionID client specified in request - need to distinguish between delete marker and delete marker deletion
-func (o *ObjectInfo) getReplicationState(dsc string, vID string, heal bool) ReplicationState {
+func (o *ObjectInfo) getReplicationState() ReplicationState {
 	rs := ReplicationState{
 		ReplicationStatusInternal:  o.ReplicationStatusInternal,
 		VersionPurgeStatusInternal: o.VersionPurgeStatusInternal,
-		ReplicateDecisionStr:       dsc,
+		ReplicateDecisionStr:       o.replicationDecision,
 		Targets:                    make(map[string]replication.StatusType),
 		PurgeTargets:               make(map[string]VersionPurgeStatusType),
 		ResetStatusesMap:           make(map[string]string),
@@ -779,9 +785,10 @@ const (
 
 // MRFReplicateEntry mrf entry to save to disk
 type MRFReplicateEntry struct {
-	Bucket    string `json:"bucket" msg:"b"`
-	Object    string `json:"object" msg:"o"`
-	versionID string `json:"-"`
+	Bucket     string `json:"bucket" msg:"b"`
+	Object     string `json:"object" msg:"o"`
+	versionID  string `json:"-"`
+	RetryCount int    `json:"retryCount" msg:"rc"`
 }
 
 // MRFReplicateEntries has the map of MRF entries to save to disk
@@ -793,9 +800,10 @@ type MRFReplicateEntries struct {
 // ToMRFEntry returns the relevant info needed by MRF
 func (ri ReplicateObjectInfo) ToMRFEntry() MRFReplicateEntry {
 	return MRFReplicateEntry{
-		Bucket:    ri.Bucket,
-		Object:    ri.Name,
-		versionID: ri.VersionID,
+		Bucket:     ri.Bucket,
+		Object:     ri.Name,
+		versionID:  ri.VersionID,
+		RetryCount: int(ri.RetryCount),
 	}
 }
 

cmd/bucket-replication-utils_gen.go

@@ -327,6 +327,12 @@ func (z *MRFReplicateEntries) DecodeMsg(dc *msgp.Reader) (err error) {
 							err = msgp.WrapError(err, "Entries", za0001, "Object")
 							return
 						}
+					case "rc":
+						za0002.RetryCount, err = dc.ReadInt()
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "RetryCount")
+							return
+						}
 					default:
 						err = dc.Skip()
 						if err != nil {
@@ -373,9 +379,9 @@ func (z *MRFReplicateEntries) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "Entries")
 			return
 		}
-		// map header, size 2
+		// map header, size 3
 		// write "b"
-		err = en.Append(0x82, 0xa1, 0x62)
+		err = en.Append(0x83, 0xa1, 0x62)
 		if err != nil {
 			return
 		}
@@ -394,6 +400,16 @@ func (z *MRFReplicateEntries) EncodeMsg(en *msgp.Writer) (err error) {
 			err = msgp.WrapError(err, "Entries", za0001, "Object")
 			return
 		}
+		// write "rc"
+		err = en.Append(0xa2, 0x72, 0x63)
+		if err != nil {
+			return
+		}
+		err = en.WriteInt(za0002.RetryCount)
+		if err != nil {
+			err = msgp.WrapError(err, "Entries", za0001, "RetryCount")
+			return
+		}
 	}
 	// write "v"
 	err = en.Append(0xa1, 0x76)
@@ -417,13 +433,16 @@ func (z *MRFReplicateEntries) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.AppendMapHeader(o, uint32(len(z.Entries)))
 	for za0001, za0002 := range z.Entries {
 		o = msgp.AppendString(o, za0001)
-		// map header, size 2
+		// map header, size 3
 		// string "b"
-		o = append(o, 0x82, 0xa1, 0x62)
+		o = append(o, 0x83, 0xa1, 0x62)
 		o = msgp.AppendString(o, za0002.Bucket)
 		// string "o"
 		o = append(o, 0xa1, 0x6f)
 		o = msgp.AppendString(o, za0002.Object)
+		// string "rc"
+		o = append(o, 0xa2, 0x72, 0x63)
+		o = msgp.AppendInt(o, za0002.RetryCount)
 	}
 	// string "v"
 	o = append(o, 0xa1, 0x76)
@@ -498,6 +517,12 @@ func (z *MRFReplicateEntries) UnmarshalMsg(bts []byte) (o []byte, err error) {
 							err = msgp.WrapError(err, "Entries", za0001, "Object")
 							return
 						}
+					case "rc":
+						za0002.RetryCount, bts, err = msgp.ReadIntBytes(bts)
+						if err != nil {
+							err = msgp.WrapError(err, "Entries", za0001, "RetryCount")
+							return
+						}
 					default:
 						bts, err = msgp.Skip(bts)
 						if err != nil {
@@ -532,7 +557,7 @@ func (z *MRFReplicateEntries) Msgsize() (s int) {
 	if z.Entries != nil {
 		for za0001, za0002 := range z.Entries {
 			_ = za0002
-			s += msgp.StringPrefixSize + len(za0001) + 1 + 2 + msgp.StringPrefixSize + len(za0002.Bucket) + 2 + msgp.StringPrefixSize + len(za0002.Object)
+			s += msgp.StringPrefixSize + len(za0001) + 1 + 2 + msgp.StringPrefixSize + len(za0002.Bucket) + 2 + msgp.StringPrefixSize + len(za0002.Object) + 3 + msgp.IntSize
 		}
 	}
 	s += 2 + msgp.IntSize
@@ -569,6 +594,12 @@ func (z *MRFReplicateEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Object")
 				return
 			}
+		case "rc":
+			z.RetryCount, err = dc.ReadInt()
+			if err != nil {
+				err = msgp.WrapError(err, "RetryCount")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -582,9 +613,9 @@ func (z *MRFReplicateEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 
 // EncodeMsg implements msgp.Encodable
 func (z MRFReplicateEntry) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 2
+	// map header, size 3
 	// write "b"
-	err = en.Append(0x82, 0xa1, 0x62)
+	err = en.Append(0x83, 0xa1, 0x62)
 	if err != nil {
 		return
 	}
@@ -603,19 +634,32 @@ func (z MRFReplicateEntry) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Object")
 		return
 	}
+	// write "rc"
+	err = en.Append(0xa2, 0x72, 0x63)
+	if err != nil {
+		return
+	}
+	err = en.WriteInt(z.RetryCount)
+	if err != nil {
+		err = msgp.WrapError(err, "RetryCount")
+		return
+	}
 	return
 }
 
 // MarshalMsg implements msgp.Marshaler
 func (z MRFReplicateEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 2
+	// map header, size 3
 	// string "b"
-	o = append(o, 0x82, 0xa1, 0x62)
+	o = append(o, 0x83, 0xa1, 0x62)
 	o = msgp.AppendString(o, z.Bucket)
 	// string "o"
 	o = append(o, 0xa1, 0x6f)
 	o = msgp.AppendString(o, z.Object)
+	// string "rc"
+	o = append(o, 0xa2, 0x72, 0x63)
+	o = msgp.AppendInt(o, z.RetryCount)
 	return
 }
 
@@ -649,6 +693,12 @@ func (z *MRFReplicateEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Object")
 				return
 			}
+		case "rc":
+			z.RetryCount, bts, err = msgp.ReadIntBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "RetryCount")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -663,7 +713,7 @@ func (z *MRFReplicateEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z MRFReplicateEntry) Msgsize() (s int) {
-	s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object)
+	s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.IntSize
 	return
 }
 

cmd/bucket-replication.go

@@ -27,7 +27,9 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"os"
 	"path"
+	"path/filepath"
 	"reflect"
 	"strings"
 	"sync"
@@ -605,7 +607,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		return
 	}
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete"+tgt.ARN)
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN)
 		sendEvent(eventArgs{
 			BucketName: dobj.Bucket,
 			Object: ObjectInfo{
@@ -634,13 +636,20 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 				IsReplicationReadyForDeleteMarker: true,
 			},
 		})
-		if isErrMethodNotAllowed(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
-			if dobj.VersionID == "" {
+		switch {
+		case isErrMethodNotAllowed(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)):
+			// delete marker already replicated
+			if dobj.VersionID == "" && rinfo.VersionPurgeStatus.Empty() {
 				rinfo.ReplicationStatus = replication.Completed
 				return
 			}
-		}
-		if !isErrObjectNotFound(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
+		case isErrObjectNotFound(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)):
+			// version being purged is already not found on target.
+			if !rinfo.VersionPurgeStatus.Empty() {
+				rinfo.VersionPurgeStatus = Complete
+				return
+			}
+		default:
 			// mark delete marker replication as failed if target cluster not ready to receive
 			// this request yet (object version not replicated yet)
 			if err != nil && !toi.ReplicationReady {
@@ -649,7 +658,6 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 			}
 		}
 	}
-
 	rmErr := tgt.RemoveObject(ctx, tgt.Bucket, dobj.ObjectName, minio.RemoveObjectOptions{
 		VersionID: versionID,
 		Internal: minio.AdvancedRemoveOptions{
@@ -679,7 +687,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 func getCopyObjMetadata(oi ObjectInfo, sc string) map[string]string {
 	meta := make(map[string]string, len(oi.UserDefined))
 	for k, v := range oi.UserDefined {
-		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+		if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
 			continue
 		}
 
@@ -744,7 +752,7 @@ func (m caseInsensitiveMap) Lookup(key string) (string, bool) {
 func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, err error) {
 	meta := make(map[string]string)
 	for k, v := range objInfo.UserDefined {
-		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+		if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
 			continue
 		}
 		if isStandardHeader(k) {
@@ -909,7 +917,7 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
 	for k, v := range oi1.UserDefined {
 		var found bool
 		for _, prefix := range compareKeys {
-			if !strings.HasPrefix(strings.ToLower(k), strings.ToLower(prefix)) {
+			if !stringsHasPrefixFold(k, prefix) {
 				continue
 			}
 			found = true
@@ -924,7 +932,7 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
 	for k, v := range oi2.Metadata {
 		var found bool
 		for _, prefix := range compareKeys {
-			if !strings.HasPrefix(strings.ToLower(k), strings.ToLower(prefix)) {
+			if !stringsHasPrefixFold(k, prefix) {
 				continue
 			}
 			found = true
@@ -970,7 +978,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
-		logger.LogOnceIf(ctx, err, "get-replication-config"+bucket)
+		logger.LogOnceIf(ctx, err, "get-replication-config-"+bucket)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1048,7 +1056,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		popts := ObjectOptions{
 			MTime:     objInfo.ModTime,
 			VersionID: objInfo.VersionID,
-			EvalMetadataFn: func(oi *ObjectInfo) error {
+			EvalMetadataFn: func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = newReplStatusInternal
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 				oi.UserDefined[xhttp.AmzBucketReplicationStatus] = string(rinfos.ReplicationStatus())
@@ -1060,7 +1068,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 				if objInfo.UserTags != "" {
 					oi.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
 				}
-				return nil
+				return dsc, nil
 			},
 		}
 
@@ -1121,7 +1129,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN), "replication-target-offline-obj"+tgt.ARN)
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1271,7 +1279,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN), "replication-target-offline-all"+tgt.ARN)
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline-heal"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -2478,7 +2486,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 						ObjectName:            roi.Name,
 						DeleteMarkerVersionID: dmVersionID,
 						VersionID:             versionID,
-						ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
+						ReplicationState:      roi.getReplicationState(),
 						DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 						DeleteMarker:          roi.DeleteMarker,
 					},
@@ -2661,7 +2669,9 @@ func (p *ReplicationPool) loadResync(ctx context.Context, buckets []BucketInfo,
 	defer cancel()
 
 	for index := range buckets {
-		meta, err := loadBucketResyncMetadata(ctx, buckets[index].Name, objAPI)
+		bucket := buckets[index].Name
+
+		meta, err := loadBucketResyncMetadata(ctx, bucket, objAPI)
 		if err != nil {
 			if !errors.Is(err, errVolumeNotFound) {
 				logger.LogIf(ctx, err)
@@ -2670,18 +2680,10 @@ func (p *ReplicationPool) loadResync(ctx context.Context, buckets []BucketInfo,
 		}
 
 		p.resyncer.Lock()
-		p.resyncer.statusMap[buckets[index].Name] = meta
+		p.resyncer.statusMap[bucket] = meta
 		p.resyncer.Unlock()
-	}
-	for index := range buckets {
-		bucket := buckets[index].Name
-		var tgts map[string]TargetReplicationResyncStatus
-		p.resyncer.RLock()
-		m, ok := p.resyncer.statusMap[bucket]
-		if ok {
-			tgts = m.cloneTgtStats()
-		}
-		p.resyncer.RUnlock()
+
+		tgts := meta.cloneTgtStats()
 		for arn, st := range tgts {
 			switch st.ResyncStatus {
 			case ResyncFailed, ResyncStarted, ResyncPending:
@@ -2847,7 +2849,7 @@ func getReplicationDiff(ctx context.Context, objAPI ObjectLayer, bucket string,
 }
 
 // QueueReplicationHeal is a wrapper for queueReplicationHeal
-func QueueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo) {
+func QueueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, retryCount int) {
 	// un-versioned or a prefix
 	if oi.VersionID == "" || oi.ModTime.IsZero() {
 		return
@@ -2857,12 +2859,12 @@ func QueueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo) {
 	queueReplicationHeal(ctx, bucket, oi, replicationConfig{
 		Config:  rcfg,
 		remotes: tgts,
-	})
+	}, retryCount)
 }
 
 // queueReplicationHeal enqueues objects that failed replication OR eligible for resyncing through
 // an ongoing resync operation or via existing objects replication configuration setting.
-func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcfg replicationConfig) (roi ReplicateObjectInfo) {
+func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcfg replicationConfig, retryCount int) (roi ReplicateObjectInfo) {
 	// un-versioned or a prefix
 	if oi.VersionID == "" || oi.ModTime.IsZero() {
 		return roi
@@ -2872,6 +2874,7 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 		return roi
 	}
 	roi = getHealReplicateObjectInfo(oi, rcfg)
+	roi.RetryCount = uint32(retryCount)
 	if !roi.Dsc.ReplicateAny() {
 		return
 	}
@@ -2895,7 +2898,7 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 				ObjectName:            roi.Name,
 				DeleteMarkerVersionID: dmVersionID,
 				VersionID:             versionID,
-				ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
+				ReplicationState:      roi.getReplicationState(),
 				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 				DeleteMarker:          roi.DeleteMarker,
 			},
@@ -2933,7 +2936,13 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 	return
 }
 
-const mrfTimeInterval = 5 * time.Minute
+const (
+	mrfSaveInterval  = 5 * time.Minute
+	mrfQueueInterval = 6 * time.Minute
+
+	mrfRetryLimit = 3 // max number of retries before letting scanner catch up on this object version
+	mrfMaxEntries = 1000000
+)
 
 func (p *ReplicationPool) persistMRF() {
 	if !p.initialized() {
@@ -2942,7 +2951,7 @@ func (p *ReplicationPool) persistMRF() {
 
 	var mu sync.Mutex
 	entries := make(map[string]MRFReplicateEntry)
-	mTimer := time.NewTimer(mrfTimeInterval)
+	mTimer := time.NewTimer(mrfSaveInterval)
 	defer mTimer.Stop()
 	saveMRFToDisk := func(drain bool) {
 		mu.Lock()
@@ -2958,8 +2967,11 @@ func (p *ReplicationPool) persistMRF() {
 				entries[e.versionID] = e
 			}
 		}
+		// queue all entries for healing before overwriting the node mrf file
+		p.queueMRFHeal()
+
 		if err := p.saveMRFEntries(cctx, entries); err != nil {
-			logger.LogOnceIf(p.ctx, fmt.Errorf("Unable to persist replication failures to disk:%w", err), string(replicationSubsystem))
+			logger.LogOnceIf(p.ctx, fmt.Errorf("unable to persist replication failures to disk:%w", err), string(replicationSubsystem))
 		}
 		entries = make(map[string]MRFReplicateEntry)
 	}
@@ -2967,7 +2979,7 @@ func (p *ReplicationPool) persistMRF() {
 		select {
 		case <-mTimer.C:
 			saveMRFToDisk(false)
-			mTimer.Reset(mrfTimeInterval)
+			mTimer.Reset(mrfSaveInterval)
 		case <-p.ctx.Done():
 			p.mrfStopCh <- struct{}{}
 			close(p.mrfSaveCh)
@@ -2985,7 +2997,7 @@ func (p *ReplicationPool) persistMRF() {
 			entries[e.versionID] = e
 			cnt = len(entries)
 			mu.Unlock()
-			if cnt >= cap(p.mrfSaveCh) || len(p.mrfSaveCh) >= int(0.8*float32(cap(p.mrfSaveCh))) {
+			if cnt >= mrfMaxEntries {
 				saveMRFToDisk(true)
 			}
 		}
@@ -2996,6 +3008,9 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
 	if !p.initialized() {
 		return
 	}
+	if entry.RetryCount > mrfRetryLimit {
+		return
+	}
 	select {
 	case <-GlobalContext.Done():
 		return
@@ -3009,7 +3024,7 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
 	}
 }
 
-// save mrf entries to mrf_<uuid>.bin
+// save mrf entries to nodenamehex.bin
 func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string]MRFReplicateEntry) error {
 	if !p.initialized() {
 		return nil
@@ -3025,17 +3040,32 @@ func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string
 	data := make([]byte, 4, v.Msgsize()+4)
 
 	// Initialize the resync meta header.
-	binary.LittleEndian.PutUint16(data[0:2], resyncMetaFormat)
-	binary.LittleEndian.PutUint16(data[2:4], resyncMetaVersion)
+	binary.LittleEndian.PutUint16(data[0:2], mrfMetaFormat)
+	binary.LittleEndian.PutUint16(data[2:4], mrfMetaVersion)
 
 	buf, err := v.MarshalMsg(data)
 	if err != nil {
 		return err
 	}
 
-	configFile := path.Join(replicationMRFDir, mustGetUUID()+".bin")
-	err = saveConfig(ctx, p.objLayer, configFile, buf)
-	return err
+	for _, diskPath := range globalEndpoints.LocalDisksPaths() {
+		// write to first drive
+		mrfDir := filepath.Join(diskPath, minioMetaBucket, replicationMRFDir)
+		mrfFileName := filepath.Join(mrfDir, globalLocalNodeNameHex+".bin")
+		if err := os.MkdirAll(mrfDir, 0o777); err != nil {
+			return err
+		}
+		file, err := OpenFile(mrfFileName, os.O_CREATE|os.O_WRONLY|writeMode, 0o666)
+		if err != nil {
+			continue
+		}
+		defer file.Close()
+		if _, err = file.Write(buf); err != nil {
+			return err
+		}
+		break
+	}
+	return nil
 }
 
 // load mrf entries from disk
@@ -3043,9 +3073,14 @@ func (p *ReplicationPool) loadMRF(fileName string) (re MRFReplicateEntries, e er
 	if !p.initialized() {
 		return re, nil
 	}
+	file, err := Open(fileName)
+	if err != nil {
+		return re, err
+	}
+	defer file.Close()
 
-	data, err := readConfig(p.ctx, p.objLayer, fileName)
-	if err != nil && err != errConfigNotFound {
+	data, err := io.ReadAll(file)
+	if err != nil {
 		return re, err
 	}
 	if len(data) == 0 {
@@ -3083,7 +3118,7 @@ func (p *ReplicationPool) processMRF() {
 	if !p.initialized() {
 		return
 	}
-	pTimer := time.NewTimer(mrfTimeInterval)
+	pTimer := time.NewTimer(mrfQueueInterval)
 	defer pTimer.Stop()
 	for {
 		select {
@@ -3097,24 +3132,13 @@ func (p *ReplicationPool) processMRF() {
 				}
 			}
 			if len(tgts) == offlineCnt {
-				pTimer.Reset(mrfTimeInterval)
+				pTimer.Reset(mrfQueueInterval)
 				continue
 			}
-			objCh := make(chan ObjectInfo)
-			cctx, cancelFn := context.WithCancel(p.ctx)
-			if err := p.objLayer.Walk(cctx, minioMetaBucket, replicationMRFDir, objCh, ObjectOptions{}); err != nil {
-				pTimer.Reset(mrfTimeInterval)
-				cancelFn()
+			if err := p.queueMRFHeal(); err != nil && !osIsNotExist(err) {
 				logger.LogIf(p.ctx, err)
-				continue
 			}
-			for item := range objCh {
-				if err := p.queueMRFHeal(item.Name); err == nil {
-					p.objLayer.DeleteObject(p.ctx, minioMetaBucket, item.Name, ObjectOptions{})
-				}
-			}
-			pTimer.Reset(mrfTimeInterval)
-			cancelFn()
+			pTimer.Reset(mrfQueueInterval)
 		case <-p.ctx.Done():
 			return
 		}
@@ -3122,24 +3146,36 @@ func (p *ReplicationPool) processMRF() {
 }
 
 // process sends error logs to the heal channel for an attempt to heal replication.
-func (p *ReplicationPool) queueMRFHeal(file string) error {
+func (p *ReplicationPool) queueMRFHeal() error {
 	if !p.initialized() {
 		return errServerNotInitialized
 	}
 
-	mrfRec, err := p.loadMRF(file)
-	if err != nil {
-		return err
-	}
-	for vID, e := range mrfRec.Entries {
-		oi, err := p.objLayer.GetObjectInfo(p.ctx, e.Bucket, e.Object, ObjectOptions{
-			VersionID: vID,
-		})
+	for _, diskPath := range globalEndpoints.LocalDisksPaths() {
+		fileName := filepath.Join(diskPath, minioMetaBucket, replicationMRFDir, globalLocalNodeNameHex+".bin")
+		mrfRec, err := p.loadMRF(fileName)
 		if err != nil {
-			continue
+			return err
 		}
-		QueueReplicationHeal(p.ctx, e.Bucket, oi)
+		// finally delete the file after processing mrf entries
+		os.Remove(fileName)
+
+		// queue replication heal in a goroutine to avoid holding up mrf save routine
+		go func(mrfRec MRFReplicateEntries) {
+			for vID, e := range mrfRec.Entries {
+
+				oi, err := p.objLayer.GetObjectInfo(p.ctx, e.Bucket, e.Object, ObjectOptions{
+					VersionID: vID,
+				})
+				if err != nil {
+					continue
+				}
+				QueueReplicationHeal(p.ctx, e.Bucket, oi, e.RetryCount)
+			}
+		}(mrfRec)
+		break
 	}
+
 	return nil
 }
 
@@ -3223,3 +3259,36 @@ func (p *ReplicationPool) saveStats(ctx context.Context) error {
 	}
 	return saveConfig(ctx, p.objLayer, getReplicationStatsPath(), data)
 }
+
+// getMRF returns MRF entries for this node.
+func (p *ReplicationPool) getMRF(ctx context.Context, bucket string) (ch chan madmin.ReplicationMRF, err error) {
+	mrfCh := make(chan madmin.ReplicationMRF, 100)
+	go func() {
+		defer close(mrfCh)
+		for _, diskPath := range globalEndpoints.LocalDisksPaths() {
+			file := filepath.Join(diskPath, minioMetaBucket, replicationMRFDir, globalLocalNodeNameHex+".bin")
+			mrfRec, err := p.loadMRF(file)
+			if err != nil {
+				break
+			}
+			for vID, e := range mrfRec.Entries {
+				if e.Bucket != bucket && bucket != "" {
+					continue
+				}
+				select {
+				case mrfCh <- madmin.ReplicationMRF{
+					NodeName:   globalLocalNodeName,
+					Object:     e.Object,
+					VersionID:  vID,
+					Bucket:     e.Bucket,
+					RetryCount: e.RetryCount,
+				}:
+				case <-ctx.Done():
+					return
+				}
+			}
+		}
+	}()
+
+	return mrfCh, nil
+}

cmd/common-main.go

@@ -66,7 +66,6 @@ import (
 	"github.com/minio/pkg/ellipses"
 	"github.com/minio/pkg/env"
 	xnet "github.com/minio/pkg/net"
-	"github.com/rs/dnscache"
 )
 
 // serverDebugLog will enable debug printing
@@ -96,11 +95,6 @@ func init() {
 
 	initGlobalContext()
 
-	options := dnscache.ResolverRefreshOptions{
-		ClearUnused:      true,
-		PersistOnFailure: false,
-	}
-
 	t, _ := minioVersionToReleaseTime(Version)
 	if !t.IsZero() {
 		globalVersionUnix = uint64(t.Unix())
@@ -108,24 +102,16 @@ func init() {
 
 	globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != ""
 
-	containers := IsKubernetes() || IsDocker() || IsBOSH() || IsDCOS() || IsPCFTile()
-
-	// Call to refresh will refresh names in cache. If you pass true, it will also
-	// remove cached names not looked up since the last call to Refresh. It is a good idea
-	// to call this method on a regular interval.
+	// Call to refresh will refresh names in cache.
 	go func() {
-		var t *time.Ticker
-		if containers {
-			// k8s DNS TTL is 30s (Attempt a refresh only after)
-			t = time.NewTicker(30 * time.Second)
-		} else {
-			t = time.NewTicker(10 * time.Minute)
-		}
+		// Baremetal setups set DNS refresh window to 10 minutes.
+		t := time.NewTicker(10 * time.Minute)
 		defer t.Stop()
 		for {
 			select {
 			case <-t.C:
-				globalDNSCache.RefreshWithOptions(options)
+				globalDNSCache.Refresh()
+
 			case <-GlobalContext.Done():
 				return
 			}

cmd/crossdomain-xml-handler.go

@@ -30,7 +30,7 @@ const crossDomainXMLEntity = "/crossdomain.xml"
 // When clients request content hosted on a particular source domain and that content make requests
 // directed towards a domain other than its own, the remote domain needs to host a cross-domain
 // policy file that grants access to the source domain, allowing the client to continue the transaction.
-func setCrossDomainPolicy(h http.Handler) http.Handler {
+func setCrossDomainPolicyMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Look for 'crossdomain.xml' in the incoming request.
 		if r.URL.Path == crossDomainXMLEntity {

cmd/crossdomain-xml-handler_test.go

@@ -29,7 +29,7 @@ import (
 func TestCrossXMLHandler(t *testing.T) {
 	// Server initialization.
 	router := mux.NewRouter().SkipClean(true).UseEncodedPath()
-	handler := setCrossDomainPolicy(router)
+	handler := setCrossDomainPolicyMiddleware(router)
 	srv := httptest.NewServer(handler)
 
 	resp, err := http.Get(srv.URL + crossDomainXMLEntity)

cmd/data-scanner-metric.go

@@ -1,3 +1,20 @@
+// Copyright (c) 2015-2022 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package cmd
 
 import (

cmd/data-scanner.go

@@ -33,7 +33,6 @@ import (
 	"sync"
 	"time"
 
-	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/object/lock"
@@ -206,7 +205,7 @@ func runDataScanner(ctx context.Context, objAPI ObjectLayer) {
 			go storeDataUsageInBackend(ctx, objAPI, results)
 			err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode)
 			logger.LogIf(ctx, err)
-			res := map[string]string{"cycle": fmt.Sprint(cycleInfo.current)}
+			res := map[string]string{"cycle": strconv.FormatUint(cycleInfo.current, 10)}
 			if err != nil {
 				res["error"] = err.Error()
 			}
@@ -302,7 +301,7 @@ type folderScanner struct {
 // The returned cache will always be valid, but may not be updated from the existing.
 // Before each operation sleepDuration is called which can be used to temporarily halt the scanner.
 // If the supplied context is canceled the function will return at the first chance.
-func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
+func scanDataFolder(ctx context.Context, disks []StorageAPI, basePath string, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode) (dataUsageCache, error) {
 	switch cache.Info.Name {
 	case "", dataUsageRoot:
 		return cache, errors.New("internal error: root scan attempted")
@@ -321,20 +320,8 @@ func scanDataFolder(ctx context.Context, poolIdx, setIdx int, basePath string, c
 		scanMode:              scanMode,
 		updates:               cache.Info.updates,
 		updateCurrentPath:     updatePath,
-	}
-
-	// Add disks for set healing.
-	if poolIdx >= 0 && setIdx >= 0 {
-		objAPI, ok := newObjectLayerFn().(*erasureServerPools)
-		if ok {
-			if poolIdx < len(objAPI.serverPools) && setIdx < len(objAPI.serverPools[poolIdx].sets) {
-				// Pass the disks belonging to the set.
-				s.disks = objAPI.serverPools[poolIdx].sets[setIdx].getDisks()
-				s.disksQuorum = len(s.disks) / 2
-			} else {
-				logger.LogIf(ctx, fmt.Errorf("Matching pool %s, set %s not found", humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1)))
-			}
-		}
+		disks:                 disks,
+		disksQuorum:           len(disks) / 2,
 	}
 
 	// Enable healing in XL mode.
@@ -403,7 +390,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			abandonedChildren = f.oldCache.findChildrenCopy(thisHash)
 		}
 
-		// If there are lifecycle rules for the prefix, remove the filter.
+		// If there are lifecycle rules for the prefix.
 		_, prefix := path2BucketObjectWithBasePath(f.root, folder.name)
 		var activeLifeCycle *lifecycle.Lifecycle
 		if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix) {
@@ -412,7 +399,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			}
 			activeLifeCycle = f.oldCache.Info.lifeCycle
 		}
-		// If there are replication rules for the prefix, remove the filter.
+		// If there are replication rules for the prefix.
 		var replicationCfg replicationConfig
 		if !f.oldCache.Info.replication.Empty() && f.oldCache.Info.replication.Config.HasActiveRules(prefix, true) {
 			replicationCfg = f.oldCache.Info.replication
@@ -649,8 +636,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			break
 		}
 
-		objAPI, ok := newObjectLayerFn().(*erasureServerPools)
-		if !ok || len(f.disks) == 0 || f.disksQuorum == 0 {
+		if len(f.disks) == 0 || f.disksQuorum == 0 {
 			break
 		}
 
@@ -688,7 +674,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				// Bucket might be missing as well with abandoned children.
 				// make sure it is created first otherwise healing won't proceed
 				// for objects.
-				_, _ = objAPI.HealBucket(ctx, bucket, madmin.HealOpts{})
+				bgSeq.queueHealTask(healSource{
+					bucket: bucket,
+				}, madmin.HealItemBucket)
 			}
 
 			resolver.bucket = bucket
@@ -813,11 +801,11 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			f.newCache.deleteRecursive(thisHash)
 			f.newCache.replaceHashed(thisHash, folder.parent, *flat)
 			total := map[string]string{
-				"objects": fmt.Sprint(flat.Objects),
-				"size":    fmt.Sprint(flat.Size),
+				"objects": strconv.FormatUint(flat.Objects, 10),
+				"size":    strconv.FormatInt(flat.Size, 10),
 			}
 			if flat.Versions > 0 {
-				total["versions"] = fmt.Sprint(flat.Versions)
+				total["versions"] = strconv.FormatUint(flat.Versions, 10)
 			}
 			stop(total)
 		}
@@ -857,6 +845,7 @@ type scannerItem struct {
 type sizeSummary struct {
 	totalSize       int64
 	versions        uint64
+	deleteMarkers   uint64
 	replicatedSize  int64
 	pendingSize     int64
 	failedSize      int64
@@ -944,7 +933,7 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 	defer globalScannerMetrics.timeILM(lcEvt.Action)()
 
 	switch lcEvt.Action {
-	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction, lifecycle.DeleteAllVersionsAction:
 		return applyLifecycleAction(lcEvt, lcEventSrc_Scanner, oi), 0
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
 		return applyLifecycleAction(lcEvt, lcEventSrc_Scanner, oi), size
@@ -1044,7 +1033,7 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 		if time.Now().UTC().Before(lifecycle.ExpectedExpiryTime(obj.SuccessorModTime, event.NoncurrentDays)) {
 			// add this version back to remaining versions for
 			// subsequent lifecycle policy applications
-			fivs = append(fivs, fi)
+			objectInfos = append(objectInfos, obj)
 			continue
 		}
 
@@ -1133,6 +1122,12 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr loc
 		return event
 	}
 
+	if obj.IsLatest && event.Action == lifecycle.DeleteAllVersionsAction {
+		if lr.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
+			return lifecycle.Event{Action: lifecycle.NoneAction}
+		}
+	}
+
 	switch event.Action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction:
 		// Defensive code, should never happen
@@ -1232,7 +1227,8 @@ func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) bool
 func applyLifecycleAction(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) (success bool) {
 	switch action := event.Action; action {
 	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction,
-		lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction,
+		lifecycle.DeleteAllVersionsAction:
 		success = applyExpiryRule(event, src, obj)
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
 		success = applyTransitionRule(event, src, obj)
@@ -1253,7 +1249,7 @@ func (i *scannerItem) healReplication(ctx context.Context, o ObjectLayer, oi Obj
 	if i.replication.Config == nil {
 		return
 	}
-	roi := queueReplicationHeal(ctx, oi.Bucket, oi, i.replication)
+	roi := queueReplicationHeal(ctx, oi.Bucket, oi, i.replication, 0)
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		return
 	}

cmd/data-usage-cache.go

@@ -57,6 +57,7 @@ type dataUsageEntry struct {
 	Size             int64                `msg:"sz"`
 	Objects          uint64               `msg:"os"`
 	Versions         uint64               `msg:"vs"` // Versions that are not delete markers.
+	DeleteMarkers    uint64               `msg:"dms"`
 	ObjSizes         sizeHistogram        `msg:"szs"`
 	ObjVersions      versionsHistogram    `msg:"vh"`
 	ReplicationStats *replicationAllStats `msg:"rs,omitempty"`
@@ -328,6 +329,7 @@ type dataUsageCacheInfo struct {
 func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.Size += summary.totalSize
 	e.Versions += summary.versions
+	e.DeleteMarkers += summary.deleteMarkers
 	e.ObjSizes.add(summary.totalSize)
 	e.ObjVersions.add(summary.versions)
 
@@ -366,6 +368,7 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 func (e *dataUsageEntry) merge(other dataUsageEntry) {
 	e.Objects += other.Objects
 	e.Versions += other.Versions
+	e.DeleteMarkers += other.DeleteMarkers
 	e.Size += other.Size
 	if other.ReplicationStats != nil {
 		if e.ReplicationStats == nil {
@@ -531,13 +534,14 @@ func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo {
 	}
 	flat := d.flatten(*e)
 	dui := DataUsageInfo{
-		LastUpdate:         d.Info.LastUpdate,
-		ObjectsTotalCount:  flat.Objects,
-		VersionsTotalCount: flat.Versions,
-		ObjectsTotalSize:   uint64(flat.Size),
-		BucketsCount:       uint64(len(e.Children)),
-		BucketsUsage:       d.bucketsUsageInfo(buckets),
-		TierStats:          d.tiersUsageInfo(buckets),
+		LastUpdate:              d.Info.LastUpdate,
+		ObjectsTotalCount:       flat.Objects,
+		VersionsTotalCount:      flat.Versions,
+		DeleteMarkersTotalCount: flat.DeleteMarkers,
+		ObjectsTotalSize:        uint64(flat.Size),
+		BucketsCount:            uint64(len(e.Children)),
+		BucketsUsage:            d.bucketsUsageInfo(buckets),
+		TierStats:               d.tiersUsageInfo(buckets),
 	}
 	return dui
 }
@@ -805,6 +809,7 @@ func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]Bucke
 			Size:                    uint64(flat.Size),
 			VersionsCount:           flat.Versions,
 			ObjectsCount:            flat.Objects,
+			DeleteMarkersCount:      flat.DeleteMarkers,
 			ObjectSizesHistogram:    flat.ObjSizes.toMap(),
 			ObjectVersionsHistogram: flat.ObjVersions.toMap(),
 		}

cmd/data-usage-cache_gen.go

@@ -1522,6 +1522,12 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Versions")
 				return
 			}
+		case "dms":
+			z.DeleteMarkers, err = dc.ReadUint64()
+			if err != nil {
+				err = msgp.WrapError(err, "DeleteMarkers")
+				return
+			}
 		case "szs":
 			var zb0002 uint32
 			zb0002, err = dc.ReadArrayHeader()
@@ -1614,16 +1620,16 @@ func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 	// omitempty: check for empty values
-	zb0001Len := uint32(9)
-	var zb0001Mask uint16 /* 9 bits */
+	zb0001Len := uint32(10)
+	var zb0001Mask uint16 /* 10 bits */
 	_ = zb0001Mask
 	if z.ReplicationStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x40
+		zb0001Mask |= 0x80
 	}
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x80
+		zb0001Mask |= 0x100
 	}
 	// variable map header, size zb0001Len
 	err = en.Append(0x80 | uint8(zb0001Len))
@@ -1673,6 +1679,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Versions")
 		return
 	}
+	// write "dms"
+	err = en.Append(0xa3, 0x64, 0x6d, 0x73)
+	if err != nil {
+		return
+	}
+	err = en.WriteUint64(z.DeleteMarkers)
+	if err != nil {
+		err = msgp.WrapError(err, "DeleteMarkers")
+		return
+	}
 	// write "szs"
 	err = en.Append(0xa3, 0x73, 0x7a, 0x73)
 	if err != nil {
@@ -1707,7 +1723,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
-	if (zb0001Mask & 0x40) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not empty
 		// write "rs"
 		err = en.Append(0xa2, 0x72, 0x73)
 		if err != nil {
@@ -1726,7 +1742,7 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x80) == 0 { // if not empty
+	if (zb0001Mask & 0x100) == 0 { // if not empty
 		// write "ats"
 		err = en.Append(0xa3, 0x61, 0x74, 0x73)
 		if err != nil {
@@ -1762,16 +1778,16 @@ func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) {
 func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
 	// omitempty: check for empty values
-	zb0001Len := uint32(9)
-	var zb0001Mask uint16 /* 9 bits */
+	zb0001Len := uint32(10)
+	var zb0001Mask uint16 /* 10 bits */
 	_ = zb0001Mask
 	if z.ReplicationStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x40
+		zb0001Mask |= 0x80
 	}
 	if z.AllTierStats == nil {
 		zb0001Len--
-		zb0001Mask |= 0x80
+		zb0001Mask |= 0x100
 	}
 	// variable map header, size zb0001Len
 	o = append(o, 0x80|uint8(zb0001Len))
@@ -1794,6 +1810,9 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "vs"
 	o = append(o, 0xa2, 0x76, 0x73)
 	o = msgp.AppendUint64(o, z.Versions)
+	// string "dms"
+	o = append(o, 0xa3, 0x64, 0x6d, 0x73)
+	o = msgp.AppendUint64(o, z.DeleteMarkers)
 	// string "szs"
 	o = append(o, 0xa3, 0x73, 0x7a, 0x73)
 	o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen))
@@ -1806,7 +1825,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 	for za0002 := range z.ObjVersions {
 		o = msgp.AppendUint64(o, z.ObjVersions[za0002])
 	}
-	if (zb0001Mask & 0x40) == 0 { // if not empty
+	if (zb0001Mask & 0x80) == 0 { // if not empty
 		// string "rs"
 		o = append(o, 0xa2, 0x72, 0x73)
 		if z.ReplicationStats == nil {
@@ -1819,7 +1838,7 @@ func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) {
 			}
 		}
 	}
-	if (zb0001Mask & 0x80) == 0 { // if not empty
+	if (zb0001Mask & 0x100) == 0 { // if not empty
 		// string "ats"
 		o = append(o, 0xa3, 0x61, 0x74, 0x73)
 		if z.AllTierStats == nil {
@@ -1880,6 +1899,12 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "Versions")
 				return
 			}
+		case "dms":
+			z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "DeleteMarkers")
+				return
+			}
 		case "szs":
 			var zb0002 uint32
 			zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts)
@@ -1970,7 +1995,7 @@ func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) {
 
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *dataUsageEntry) Msgsize() (s int) {
-	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
+	s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 3
 	if z.ReplicationStats == nil {
 		s += msgp.NilSize
 	} else {

cmd/data-usage-utils.go

@@ -62,6 +62,7 @@ type BucketUsageInfo struct {
 	ObjectSizesHistogram    map[string]uint64                `json:"objectsSizesHistogram"`
 	ObjectVersionsHistogram map[string]uint64                `json:"objectsVersionsHistogram"`
 	VersionsCount           uint64                           `json:"versionsCount"`
+	DeleteMarkersCount      uint64                           `json:"deleteMarkersCount"`
 	ReplicaSize             uint64                           `json:"objectReplicaTotalSize"`
 	ReplicationInfo         map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`
 }
@@ -75,9 +76,12 @@ type DataUsageInfo struct {
 	// Objects total count across all buckets
 	ObjectsTotalCount uint64 `json:"objectsCount"`
 
-	// Objects total count across all buckets
+	// Versions total count across all buckets
 	VersionsTotalCount uint64 `json:"versionsCount"`
 
+	// Delete markers total count across all buckets
+	DeleteMarkersTotalCount uint64 `json:"deleteMarkersCount"`
+
 	// Objects total size across all buckets
 	ObjectsTotalSize uint64                           `json:"objectsTotalSize"`
 	ReplicationInfo  map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"`

cmd/data-usage_test.go

@@ -62,7 +62,7 @@ func TestDataUsageUpdate(t *testing.T) {
 		return
 	}
 
-	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0)
+	got, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -173,7 +173,7 @@ func TestDataUsageUpdate(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize, 0)
+		got, err = scanDataFolder(context.Background(), nil, base, got, getSize, 0)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -280,7 +280,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 		}
 		return
 	}
-	got, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize, 0)
+	got, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize, 0)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -414,7 +414,7 @@ func TestDataUsageUpdatePrefix(t *testing.T) {
 	}
 	// Changed dir must be picked up in this many cycles.
 	for i := 0; i < dataUsageUpdateDirCycles; i++ {
-		got, err = scanDataFolder(context.Background(), 0, 0, base, got, getSize, 0)
+		got, err = scanDataFolder(context.Background(), nil, base, got, getSize, 0)
 		got.Info.NextCycle++
 		if err != nil {
 			t.Fatal(err)
@@ -562,7 +562,7 @@ func TestDataUsageCacheSerialize(t *testing.T) {
 		}
 		return
 	}
-	want, err := scanDataFolder(context.Background(), 0, 0, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0)
+	want, err := scanDataFolder(context.Background(), nil, base, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0)
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/disk-cache.go

@@ -139,14 +139,14 @@ func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *disk
 	bkMeta := make(map[string]string, len(bkObjectInfo.UserDefined))
 	cacheMeta := make(map[string]string, len(cacheObjInfo.UserDefined))
 	for k, v := range bkObjectInfo.UserDefined {
-		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+		if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
 			// Do not need to send any internal metadata
 			continue
 		}
 		bkMeta[http.CanonicalHeaderKey(k)] = v
 	}
 	for k, v := range cacheObjInfo.UserDefined {
-		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+		if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
 			// Do not need to send any internal metadata
 			continue
 		}

cmd/endpoint_test.go

@@ -372,7 +372,7 @@ func TestGetLocalPeer(t *testing.T) {
 	}
 
 	for i, testCase := range testCases {
-		zendpoints := mustGetPoolEndpoints(testCase.endpointArgs...)
+		zendpoints := mustGetPoolEndpoints(0, testCase.endpointArgs...)
 		if !zendpoints[0].Endpoints[0].IsLocal {
 			if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)
@@ -405,7 +405,7 @@ func TestGetRemotePeers(t *testing.T) {
 	}
 
 	for _, testCase := range testCases {
-		zendpoints := mustGetPoolEndpoints(testCase.endpointArgs...)
+		zendpoints := mustGetPoolEndpoints(0, testCase.endpointArgs...)
 		if !zendpoints[0].Endpoints[0].IsLocal {
 			if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil {
 				t.Errorf("error: expected = <nil>, got = %v", err)

cmd/erasure-common.go

@@ -20,7 +20,9 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"math/rand"
 	"sync"
+	"time"
 
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/sync/errgroup"
@@ -30,15 +32,16 @@ func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 	disks := er.getDisks()
 	var wg sync.WaitGroup
 	var mu sync.Mutex
-	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	for _, i := range r.Perm(len(disks)) {
 		i := i
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			if disks[i-1] == nil {
+			if disks[i] == nil {
 				return
 			}
-			di, err := disks[i-1].DiskInfo(context.Background())
+			di, err := disks[i].DiskInfo(context.Background())
 			if err != nil || di.Healing {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.
@@ -50,7 +53,7 @@ func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 			}
 
 			mu.Lock()
-			newDisks = append(newDisks, disks[i-1])
+			newDisks = append(newDisks, disks[i])
 			mu.Unlock()
 		}()
 	}
@@ -61,9 +64,10 @@ func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	disks := er.getDisks()
 	// Based on the random shuffling return back randomized disks.
-	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
-		if disks[i-1] != nil && disks[i-1].IsLocal() {
-			newDisks = append(newDisks, disks[i-1])
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	for _, i := range r.Perm(len(disks)) {
+		if disks[i] != nil && disks[i].IsLocal() {
+			newDisks = append(newDisks, disks[i])
 		}
 	}
 	return newDisks
@@ -74,10 +78,11 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 	disks := er.getDisks()
 
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 	if !optimized {
 		var newDisks []StorageAPI
-		for _, i := range hashOrder(UTCNow().String(), len(disks)) {
-			newDisks = append(newDisks, disks[i-1])
+		for _, i := range r.Perm(len(disks)) {
+			newDisks = append(newDisks, disks[i])
 		}
 		return newDisks
 	}
@@ -86,15 +91,15 @@ func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 	var mu sync.Mutex
 	newDisks := map[uint64][]StorageAPI{}
 	// Based on the random shuffling return back randomized disks.
-	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+	for _, i := range r.Perm(len(disks)) {
 		i := i
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			if disks[i-1] == nil {
+			if disks[i] == nil {
 				return
 			}
-			di, err := disks[i-1].DiskInfo(context.Background())
+			di, err := disks[i].DiskInfo(context.Background())
 			if err != nil || di.Healing {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.
@@ -107,7 +112,7 @@ func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
 
 			mu.Lock()
 			// Capture disks usage wise upto resolution of MiB
-			newDisks[di.Used/1024/1024] = append(newDisks[di.Used/1024/1024], disks[i-1])
+			newDisks[di.Used/1024/1024] = append(newDisks[di.Used/1024/1024], disks[i])
 			mu.Unlock()
 		}()
 	}

cmd/erasure-healing-common_test.go

@@ -233,7 +233,12 @@ func TestListOnlineDisks(t *testing.T) {
 	object := "object"
 	data := bytes.Repeat([]byte("a"), smallFileThreshold*16)
 	z := obj.(*erasureServerPools)
-	erasureDisks := z.serverPools[0].sets[0].getDisks()
+
+	erasureDisks, err := z.GetDisks(0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	for i, test := range testCases {
 		test := test
 		t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) {
@@ -405,7 +410,12 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 	object := "object"
 	data := bytes.Repeat([]byte("a"), smallFileThreshold/2)
 	z := obj.(*erasureServerPools)
-	erasureDisks := z.serverPools[0].sets[0].getDisks()
+
+	erasureDisks, err := z.GetDisks(0, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	for i, test := range testCases {
 		test := test
 		t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) {

cmd/erasure-healing_test.go

@@ -584,7 +584,7 @@ func TestHealingDanglingObject(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Everything is fine, should return nil
-	objLayer, disks, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	objLayer, disks, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -776,8 +776,8 @@ func TestHealCorrectQuorum(t *testing.T) {
 
 	defer removeRoots(fsDirs)
 
-	pools := mustGetPoolEndpoints(fsDirs[:16]...)
-	pools = append(pools, mustGetPoolEndpoints(fsDirs[16:]...)...)
+	pools := mustGetPoolEndpoints(0, fsDirs[:16]...)
+	pools = append(pools, mustGetPoolEndpoints(1, fsDirs[16:]...)...)
 
 	// Everything is fine, should return nil
 	objLayer, _, err := initObjectLayer(ctx, pools)
@@ -907,8 +907,8 @@ func TestHealObjectCorruptedPools(t *testing.T) {
 
 	defer removeRoots(fsDirs)
 
-	pools := mustGetPoolEndpoints(fsDirs[:16]...)
-	pools = append(pools, mustGetPoolEndpoints(fsDirs[16:]...)...)
+	pools := mustGetPoolEndpoints(0, fsDirs[:16]...)
+	pools = append(pools, mustGetPoolEndpoints(1, fsDirs[16:]...)...)
 
 	// Everything is fine, should return nil
 	objLayer, _, err := initObjectLayer(ctx, pools)
@@ -1088,7 +1088,7 @@ func TestHealObjectCorruptedXLMeta(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Everything is fine, should return nil
-	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1231,7 +1231,7 @@ func TestHealObjectCorruptedParts(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Everything is fine, should return nil
-	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1388,7 +1388,7 @@ func TestHealObjectErasure(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Everything is fine, should return nil
-	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1486,7 +1486,7 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 	defer removeRoots(fsDirs)
 
 	// Everything is fine, should return nil
-	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -1582,7 +1582,7 @@ func TestHealLastDataShard(t *testing.T) {
 
 			defer removeRoots(fsDirs)
 
-			obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+			obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...))
 			if err != nil {
 				t.Fatal(err)
 			}

cmd/erasure-metadata-utils_test.go

@@ -156,7 +156,7 @@ func TestShuffleDisks(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(disks...))
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
 	if err != nil {
 		removeRoots(disks)
 		t.Fatal(err)
@@ -204,7 +204,7 @@ func TestEvalDisks(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(disks...))
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
 	if err != nil {
 		removeRoots(disks)
 		t.Fatal(err)

cmd/erasure-metadata.go

@@ -350,8 +350,11 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 			for _, part := range meta.Parts {
 				fmt.Fprintf(h, "part.%d", part.Number)
 			}
-			fmt.Fprintf(h, "%v+%v", meta.Erasure.DataBlocks, meta.Erasure.ParityBlocks)
-			fmt.Fprintf(h, "%v", meta.Erasure.Distribution)
+
+			if !meta.Deleted && meta.Size != 0 {
+				fmt.Fprintf(h, "%v+%v", meta.Erasure.DataBlocks, meta.Erasure.ParityBlocks)
+				fmt.Fprintf(h, "%v", meta.Erasure.Distribution)
+			}
 
 			// ILM transition fields
 			fmt.Fprint(h, meta.TransitionStatus)
@@ -499,7 +502,8 @@ func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int)
 			parities[index] = -1
 			continue
 		}
-		if metadata.Deleted {
+		// Delete marker or zero byte objects take highest parity.
+		if metadata.Deleted || metadata.Size == 0 {
 			parities[index] = len(partsMetadata) / 2
 		} else {
 			parities[index] = metadata.Erasure.ParityBlocks

cmd/erasure-object.go

@@ -278,6 +278,11 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		return gr.WithCleanupFuncs(nsUnlocker), nil
 	}
 
+	if objInfo.Size == 0 {
+		// Zero byte objects don't even need to further initialize pipes etc.
+		return NewGetObjectReaderFromReader(bytes.NewReader(nil), objInfo, opts)
+	}
+
 	fn, off, length, err := NewGetObjectReader(rs, objInfo, opts)
 	if err != nil {
 		return nil, err
@@ -527,6 +532,10 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 	metadataArray := make([]*xlMetaV2, len(disks))
 	metaFileInfos := make([]FileInfo, len(metadataArray))
 	metadataShallowVersions := make([][]xlMetaV2ShallowVersion, len(disks))
+	var v2bufs [][]byte
+	if !readData {
+		v2bufs = make([][]byte, len(disks))
+	}
 
 	g := errgroup.WithNErrs(len(disks))
 	// Read `xl.meta` in parallel across disks.
@@ -540,6 +549,10 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 			if err != nil {
 				return err
 			}
+			if !readData {
+				// Save the buffer so we can reuse it.
+				v2bufs[index] = rf.Buf
+			}
 
 			var xl xlMetaV2
 			if err = xl.LoadOrConvert(rf.Buf); err != nil {
@@ -623,6 +636,11 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		}
 		metaFileInfos[index].DiskMTime = diskMTime
 	}
+	if !readData {
+		for i := range v2bufs {
+			metaDataPoolPut(v2bufs[i])
+		}
+	}
 
 	// Return all the metadata.
 	return metaFileInfos, errs
@@ -1322,7 +1340,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		if errors.Is(err, errFileNotFound) {
 			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
 		}
-		logger.LogOnceIf(ctx, err, "erasure-object-rename"+bucket+object)
+		logger.LogOnceIf(ctx, err, "erasure-object-rename-"+bucket+"-"+object)
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
 
@@ -1644,6 +1662,22 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 			return objInfo, gerr
 		}
 	}
+	if opts.EvalMetadataFn != nil {
+		dsc, err := opts.EvalMetadataFn(&goi, err)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		if dsc.ReplicateAny() {
+			opts.SetDeleteReplicationState(dsc, opts.VersionID)
+			goi.replicationDecision = opts.DeleteReplication.ReplicateDecisionStr
+		}
+	}
+
+	if opts.EvalRetentionBypassFn != nil {
+		if err := opts.EvalRetentionBypassFn(goi, gerr); err != nil {
+			return ObjectInfo{}, err
+		}
+	}
 
 	if opts.Expiration.Expire {
 		if gerr == nil {
@@ -1752,7 +1786,9 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 		if err = er.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker); err != nil {
 			return objInfo, toObjectErr(err, bucket, object)
 		}
-		return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil
+		oi := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
+		oi.replicationDecision = goi.replicationDecision
+		return oi, nil
 	}
 
 	// Delete the object version on all disks.
@@ -1842,7 +1878,7 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 
 	objInfo := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)
 	if opts.EvalMetadataFn != nil {
-		if err := opts.EvalMetadataFn(&objInfo); err != nil {
+		if _, err := opts.EvalMetadataFn(&objInfo, err); err != nil {
 			return ObjectInfo{}, err
 		}
 	}

cmd/erasure-object_test.go

@@ -1076,7 +1076,7 @@ func TestGetObjectInlineNotInline(t *testing.T) {
 		fsDirs = append(fsDirs, filepath.Join(path, fmt.Sprintf("disk%d", i)))
 	}
 
-	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+	objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...))
 	if err != nil {
 		removeRoots(fsDirs)
 		t.Fatal(err)

cmd/erasure-server-pool-decom.go

@@ -245,6 +245,7 @@ func (p *poolMeta) QueueBuckets(idx int, buckets []decomBucketInfo) {
 var (
 	errDecommissionAlreadyRunning = errors.New("decommission is already in progress")
 	errDecommissionComplete       = errors.New("decommission is complete, please remove the servers from command-line")
+	errDecommissionNotStarted     = errors.New("decommission is not in progress")
 )
 
 func (p *poolMeta) Decommission(idx int, pi poolSpaceInfo) error {
@@ -1183,7 +1184,11 @@ func (z *erasureServerPools) Status(ctx context.Context, idx int) (PoolStatus, e
 	poolInfo := z.poolMeta.Pools[idx]
 	if poolInfo.Decommission != nil {
 		poolInfo.Decommission.TotalSize = pi.Total
-		poolInfo.Decommission.CurrentSize = poolInfo.Decommission.StartSize + poolInfo.Decommission.BytesDone
+		if poolInfo.Decommission.Failed || poolInfo.Decommission.Canceled {
+			poolInfo.Decommission.CurrentSize = pi.Free
+		} else {
+			poolInfo.Decommission.CurrentSize = poolInfo.Decommission.StartSize + poolInfo.Decommission.BytesDone
+		}
 	} else {
 		poolInfo.Decommission = &PoolDecommissionInfo{
 			TotalSize:   pi.Total,
@@ -1219,15 +1224,21 @@ func (z *erasureServerPools) DecommissionCancel(ctx context.Context, idx int) (e
 	z.poolMetaMutex.Lock()
 	defer z.poolMetaMutex.Unlock()
 
+	fn := z.decommissionCancelers[idx]
+	if fn == nil {
+		// canceling a decommission before it started return an error.
+		return errDecommissionNotStarted
+	}
+
+	defer fn() // cancel any active thread.
+
 	if z.poolMeta.DecommissionCancel(idx) {
-		if fn := z.decommissionCancelers[idx]; fn != nil {
-			defer fn() // cancel any active thread.
-		}
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
 		globalNotificationSys.ReloadPoolMeta(ctx)
 	}
+
 	return nil
 }
 
@@ -1245,8 +1256,9 @@ func (z *erasureServerPools) DecommissionFailed(ctx context.Context, idx int) (e
 
 	if z.poolMeta.DecommissionFailed(idx) {
 		if fn := z.decommissionCancelers[idx]; fn != nil {
-			defer fn() // cancel any active thread.
-		}
+			defer fn()
+		} // cancel any active thread.
+
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
@@ -1269,8 +1281,9 @@ func (z *erasureServerPools) CompleteDecommission(ctx context.Context, idx int)
 
 	if z.poolMeta.DecommissionComplete(idx) {
 		if fn := z.decommissionCancelers[idx]; fn != nil {
-			defer fn() // cancel any active thread.
-		}
+			defer fn()
+		} // cancel any active thread.
+
 		if err = z.poolMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}

cmd/erasure-server-pool-decom_test.go

@@ -29,8 +29,8 @@ func prepareErasurePools() (ObjectLayer, []string, error) {
 		return nil, nil, err
 	}
 
-	pools := mustGetPoolEndpoints(fsDirs[:16]...)
-	pools = append(pools, mustGetPoolEndpoints(fsDirs[16:]...)...)
+	pools := mustGetPoolEndpoints(0, fsDirs[:16]...)
+	pools = append(pools, mustGetPoolEndpoints(1, fsDirs[16:]...)...)
 
 	// Everything is fine, should return nil
 	objLayer, err := newErasureServerPools(context.Background(), pools)

cmd/erasure-server-pool.go

@@ -242,6 +242,14 @@ func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string
 	return nil
 }
 
+// Return the disks belonging to the poolIdx, and setIdx.
+func (z *erasureServerPools) GetDisks(poolIdx, setIdx int) ([]StorageAPI, error) {
+	if poolIdx < len(z.serverPools) && setIdx < len(z.serverPools[poolIdx].sets) {
+		return z.serverPools[poolIdx].sets[setIdx].getDisks(), nil
+	}
+	return nil, fmt.Errorf("Matching pool %s, set %s not found", humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1))
+}
+
 // Return the count of disks in each pool
 func (z *erasureServerPools) SetDriveCounts() []int {
 	setDriveCounts := make([]int, len(z.serverPools))
@@ -630,11 +638,6 @@ func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataU
 		return nil
 	}
 
-	// Scanner latest allBuckets first.
-	sort.Slice(allBuckets, func(i, j int) bool {
-		return allBuckets[i].Created.After(allBuckets[j].Created)
-	})
-
 	// Collect for each set in serverPools.
 	for _, z := range z.serverPools {
 		for _, erObj := range z.sets {
@@ -1343,7 +1346,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
 		if !isErrBucketNotFound(err) {
-			logger.LogOnceIf(ctx, err, "erasure-list-objects-path"+bucket)
+			logger.LogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket)
 		}
 		return loi, err
 	}

cmd/erasure-sets_test.go

@@ -173,7 +173,7 @@ func TestNewErasureSets(t *testing.T) {
 		defer os.RemoveAll(disk)
 	}
 
-	endpoints := mustGetNewEndpoints(erasureDisks...)
+	endpoints := mustGetNewEndpoints(0, erasureDisks...)
 	_, _, err := waitForFormatErasure(true, endpoints, 1, 0, 16, "", "")
 	if err != errInvalidArgument {
 		t.Fatalf("Expecting error, got %s", err)

cmd/erasure.go

@@ -176,7 +176,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 		g.Go(func() error {
 			diskEndpoint := endpoints[index].String()
 			if disks[index] == OfflineDisk {
-				logger.LogOnceIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]), "get-disks-info-offline"+diskEndpoint)
+				logger.LogOnceIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]), "get-disks-info-offline-"+diskEndpoint)
 				disksInfo[index] = madmin.Disk{
 					State:    diskErrToDriveState(errDiskNotFound),
 					Endpoint: diskEndpoint,
@@ -277,31 +277,33 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 	var wg sync.WaitGroup
 	disks := er.getDisks()
 	infos := make([]DiskInfo, len(disks))
-	for _, i := range hashOrder(UTCNow().String(), len(disks)) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	for _, i := range r.Perm(len(disks)) {
 		i := i
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
 
-			disk := disks[i-1]
-
+			disk := disks[i]
 			if disk == nil {
-				infos[i-1].Error = "nil drive"
+				infos[i].Error = "offline drive"
 				return
 			}
 
 			di, err := disk.DiskInfo(context.Background())
-			if err != nil {
+			if err != nil || di.Healing {
 				// - Do not consume disks which are not reachable
 				//   unformatted or simply not accessible for some reason.
 				//
 				//
 				// - Future: skip busy disks
-				infos[i-1].Error = err.Error()
+				if err != nil {
+					infos[i].Error = err.Error()
+				}
 				return
 			}
 
-			infos[i-1] = di
+			infos[i] = di
 		}()
 	}
 	wg.Wait()
@@ -373,23 +375,30 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 
 	// Put all buckets into channel.
 	bucketCh := make(chan BucketInfo, len(buckets))
+
+	// Shuffle buckets to ensure total randomness of buckets, being scanned.
+	// Otherwise same set of buckets get scanned across erasure sets always.
+	// at any given point in time. This allows different buckets to be scanned
+	// in different order per erasure set, this wider spread is needed when
+	// there are lots of buckets with different order of objects in them.
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	permutes := r.Perm(len(buckets))
 	// Add new buckets first
-	for _, b := range buckets {
-		if oldCache.find(b.Name) == nil {
+	for _, idx := range permutes {
+		b := buckets[idx]
+		if e := oldCache.find(b.Name); e == nil {
 			bucketCh <- b
 		}
 	}
-
-	// Add existing buckets.
-	for _, b := range buckets {
-		e := oldCache.find(b.Name)
-		if e != nil {
+	for _, idx := range permutes {
+		b := buckets[idx]
+		if e := oldCache.find(b.Name); e != nil {
 			cache.replace(b.Name, dataUsageRoot, *e)
 			bucketCh <- b
 		}
 	}
-
 	close(bucketCh)
+
 	bucketResults := make(chan dataUsageEntryInfo, len(disks))
 
 	// Start async collector/saver.
@@ -428,11 +437,6 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 		}
 	}()
 
-	// Shuffle disks to ensure a total randomness of bucket/disk association to ensure
-	// that objects that are not present in all disks are accounted and ILM applied.
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	r.Shuffle(len(disks), func(i, j int) { disks[i], disks[j] = disks[j], disks[i] })
-
 	// Restrict parallelism for disk usage scanner
 	// upto GOMAXPROCS if GOMAXPROCS is < len(disks)
 	maxProcs := runtime.GOMAXPROCS(0)

cmd/event-notification.go

@@ -268,7 +268,7 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 		newEvent.S3.Object.ContentType = args.Object.ContentType
 		newEvent.S3.Object.UserMetadata = make(map[string]string, len(args.Object.UserDefined))
 		for k, v := range args.Object.UserDefined {
-			if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+			if stringsHasPrefixFold(strings.ToLower(k), ReservedMetadataPrefixLower) {
 				continue
 			}
 			newEvent.S3.Object.UserMetadata[k] = v

cmd/format-erasure_test.go

@@ -36,9 +36,9 @@ func TestFixFormatV3(t *testing.T) {
 	for _, erasureDir := range erasureDirs {
 		defer os.RemoveAll(erasureDir)
 	}
-	endpoints := mustGetNewEndpoints(erasureDirs...)
+	endpoints := mustGetNewEndpoints(0, erasureDirs...)
 
-	storageDisks, errs := initStorageDisksWithErrors(endpoints, true)
+	storageDisks, errs := initStorageDisksWithErrors(endpoints, false)
 	for _, err := range errs {
 		if err != nil && err != errDiskNotFound {
 			t.Fatal(err)
@@ -555,11 +555,11 @@ func benchmarkInitStorageDisksN(b *testing.B, nDisks int) {
 	if err != nil {
 		b.Fatal(err)
 	}
-	endpoints := mustGetNewEndpoints(fsDirs...)
+	endpoints := mustGetNewEndpoints(0, fsDirs...)
 	b.RunParallel(func(pb *testing.PB) {
 		endpoints := endpoints
 		for pb.Next() {
-			initStorageDisksWithErrors(endpoints, true)
+			initStorageDisksWithErrors(endpoints, false)
 		}
 	})
 }

cmd/ftp-server-driver.go

@@ -322,7 +322,7 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 		return minio.New(driver.endpoint, &minio.Options{
 			Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
 			Secure:    globalIsTLS,
-			Transport: globalRemoteTargetTransport,
+			Transport: globalRemoteFTPClientTransport,
 		})
 	}
 
@@ -336,7 +336,7 @@ func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error)
 	return minio.New(driver.endpoint, &minio.Options{
 		Creds:     credentials.NewStaticV4(ui.Credentials.AccessKey, ui.Credentials.SecretKey, ""),
 		Secure:    globalIsTLS,
-		Transport: globalRemoteTargetTransport,
+		Transport: globalRemoteFTPClientTransport,
 	})
 }
 

cmd/ftp-server.go

@@ -23,17 +23,19 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"os"
 	"strconv"
 	"strings"
 
 	"github.com/minio/cli"
-	"github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/pkg/sftp"
 	ftp "goftp.io/server/v2"
 	"golang.org/x/crypto/ssh"
 )
 
+var globalRemoteFTPClientTransport = NewRemoteTargetHTTPTransport(true)()
+
 // minioLogger use an instance of this to log in a standard format
 type minioLogger struct{}
 
@@ -115,7 +117,7 @@ func startSFTPServer(c *cli.Context) {
 		logger.Fatal(fmt.Errorf("invalid arguments passed, private key file is mandatory for --sftp='ssh-private-key=path/to/id_ecdsa'"), "unable to start SFTP server")
 	}
 
-	privateBytes, err := ioutil.ReadFile(sshPrivateKey)
+	privateBytes, err := os.ReadFile(sshPrivateKey)
 	if err != nil {
 		logger.Fatal(fmt.Errorf("invalid arguments passed, private key file is not accessible: %v", err), "unable to start SFTP server")
 	}

cmd/generic-handlers.go

@@ -71,7 +71,7 @@ const (
 // and must not set by clients
 func containsReservedMetadata(header http.Header) bool {
 	for key := range header {
-		if strings.HasPrefix(strings.ToLower(key), ReservedMetadataPrefixLower) {
+		if stringsHasPrefixFold(key, ReservedMetadataPrefix) {
 			return true
 		}
 	}
@@ -87,7 +87,7 @@ func isHTTPHeaderSizeTooLarge(header http.Header) bool {
 		length := len(key) + len(header.Get(key))
 		size += length
 		for _, prefix := range userMetadataKeyPrefixes {
-			if strings.HasPrefix(strings.ToLower(key), prefix) {
+			if stringsHasPrefixFold(key, prefix) {
 				usersize += length
 				break
 			}
@@ -100,7 +100,7 @@ func isHTTPHeaderSizeTooLarge(header http.Header) bool {
 }
 
 // Limits body and header to specific allowed maximum limits as per S3/MinIO API requirements.
-func setRequestLimitHandler(h http.Handler) http.Handler {
+func setRequestLimitMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
 
@@ -147,7 +147,7 @@ func guessIsBrowserReq(r *http.Request) bool {
 		globalBrowserEnabled && aType == authTypeAnonymous
 }
 
-func setBrowserRedirectHandler(h http.Handler) http.Handler {
+func setBrowserRedirectMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		read := r.Method == http.MethodGet || r.Method == http.MethodHead
 		// Re-direction is handled specifically for browser requests.
@@ -325,7 +325,7 @@ func hasMultipleAuth(r *http.Request) bool {
 
 // requestValidityHandler validates all the incoming paths for
 // any malicious requests.
-func setRequestValidityHandler(h http.Handler) http.Handler {
+func setRequestValidityMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
 
@@ -425,10 +425,10 @@ func setRequestValidityHandler(h http.Handler) http.Handler {
 	})
 }
 
-// setBucketForwardingHandler middleware forwards the path style requests
+// setBucketForwardingMiddleware middleware forwards the path style requests
 // on a bucket to the right bucket location, bucket to IP configuration
 // is obtained from centralized etcd configuration service.
-func setBucketForwardingHandler(h http.Handler) http.Handler {
+func setBucketForwardingMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if globalDNSConfig == nil || !globalBucketFederation ||
 			guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||
@@ -493,13 +493,12 @@ func setBucketForwardingHandler(h http.Handler) http.Handler {
 	})
 }
 
-// addCustomHeaders adds various HTTP(S) response headers.
+// addCustomHeadersMiddleware adds various HTTP(S) response headers.
 // Security Headers enable various security protections behaviors in the client's browser.
-func addCustomHeaders(h http.Handler) http.Handler {
+func addCustomHeadersMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		header := w.Header()
 		header.Set("X-XSS-Protection", "1; mode=block")                                // Prevents against XSS attacks
-		header.Set("Content-Security-Policy", "block-all-mixed-content")               // prevent mixed (HTTP / HTTPS content)
 		header.Set("X-Content-Type-Options", "nosniff")                                // Prevent mime-sniff
 		header.Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains") // HSTS mitigates variants of MITM attacks
 
@@ -540,9 +539,9 @@ func setCriticalErrorHandler(h http.Handler) http.Handler {
 	})
 }
 
-// setUploadForwardingHandler middleware forwards multiparts requests
+// setUploadForwardingMiddleware middleware forwards multiparts requests
 // in a site replication setup to peer that initiated the upload
-func setUploadForwardingHandler(h http.Handler) http.Handler {
+func setUploadForwardingMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		if !globalSiteReplicationSys.isEnabled() ||
 			guessIsHealthCheckReq(r) || guessIsMetricsReq(r) ||

cmd/generic-handlers_test.go

@@ -155,7 +155,7 @@ func TestSSETLSHandler(t *testing.T) {
 		r.Header = test.Header
 		r.URL = test.URL
 
-		h := setRequestValidityHandler(okHandler)
+		h := setRequestValidityMiddleware(okHandler)
 		h.ServeHTTP(w, r)
 
 		switch {

cmd/globals.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/minio/console/restapi"
+	"github.com/minio/dnscache"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/set"
@@ -34,7 +35,6 @@ import (
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/kms"
-	"github.com/rs/dnscache"
 
 	"github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/auth"
@@ -370,6 +370,7 @@ var (
 	// Used for collecting stats for netperf
 	globalNetPerfMinDuration     = time.Second * 10
 	globalNetPerfRX              netPerfRX
+	globalSiteNetPerfRX          netPerfRX
 	globalObjectPerfBucket       = "minio-perf-test-tmp-bucket"
 	globalObjectPerfUserMetadata = "X-Amz-Meta-Minio-Object-Perf" // Clients can set this to bypass S3 API service freeze. Used by object pref tests.
 
@@ -392,6 +393,10 @@ var (
 	// Is _MINIO_DISABLE_API_FREEZE_ON_BOOT set?
 	globalDisableFreezeOnBoot bool
 
+	// Contains NIC interface name used for internode communication
+	globalInternodeInterface     string
+	globalInternodeInterfaceOnce sync.Once
+
 	// Add new variable global values here.
 )
 

cmd/handler-utils.go

@@ -184,7 +184,7 @@ func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[
 
 	for key := range v {
 		for _, prefix := range userMetadataKeyPrefixes {
-			if !strings.HasPrefix(strings.ToLower(key), strings.ToLower(prefix)) {
+			if !stringsHasPrefixFold(key, prefix) {
 				continue
 			}
 			value, ok := nv[http.CanonicalHeaderKey(key)]
@@ -267,6 +267,20 @@ func trimAwsChunkedContentEncoding(contentEnc string) (trimmedContentEnc string)
 	return strings.Join(newEncs, ",")
 }
 
+func collectInternodeStats(f http.HandlerFunc) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		f.ServeHTTP(w, r)
+
+		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
+		if !ok || tc == nil {
+			return
+		}
+
+		globalConnStats.incInternodeInputBytes(int64(tc.RequestRecorder.Size()))
+		globalConnStats.incInternodeOutputBytes(int64(tc.ResponseRecorder.Size()))
+	}
+}
+
 func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
@@ -297,21 +311,6 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 		}
 
 		if tc != nil {
-			if strings.HasPrefix(r.URL.Path, storageRESTPrefix) ||
-				strings.HasPrefix(r.URL.Path, peerRESTPrefix) ||
-				strings.HasPrefix(r.URL.Path, peerS3Prefix) ||
-				strings.HasPrefix(r.URL.Path, lockRESTPrefix) {
-				globalConnStats.incInputBytes(int64(tc.RequestRecorder.Size()))
-				globalConnStats.incOutputBytes(int64(tc.ResponseRecorder.Size()))
-				return
-			}
-
-			if strings.HasPrefix(r.URL.Path, minioReservedBucketPath) {
-				globalConnStats.incAdminInputBytes(int64(tc.RequestRecorder.Size()))
-				globalConnStats.incAdminOutputBytes(int64(tc.ResponseRecorder.Size()))
-				return
-			}
-
 			globalHTTPStats.updateStats(api, tc.ResponseRecorder)
 			globalConnStats.incS3InputBytes(int64(tc.RequestRecorder.Size()))
 			globalConnStats.incS3OutputBytes(int64(tc.ResponseRecorder.Size()))

cmd/http-stats.go

@@ -26,93 +26,69 @@ import (
 	"github.com/prometheus/client_golang/prometheus"
 )
 
-// ConnStats - Network statistics
+// connStats - Network statistics
 // Count total input/output transferred bytes during
 // the server's life.
-type ConnStats struct {
-	totalInputBytes  uint64
-	totalOutputBytes uint64
-	s3InputBytes     uint64
-	s3OutputBytes    uint64
-	adminInputBytes  uint64
-	adminOutputBytes uint64
+type connStats struct {
+	internodeInputBytes  uint64
+	internodeOutputBytes uint64
+	s3InputBytes         uint64
+	s3OutputBytes        uint64
 }
 
 // Increase internode total input bytes
-func (s *ConnStats) incInputBytes(n int64) {
-	atomic.AddUint64(&s.totalInputBytes, uint64(n))
+func (s *connStats) incInternodeInputBytes(n int64) {
+	atomic.AddUint64(&s.internodeInputBytes, uint64(n))
 }
 
 // Increase internode total output bytes
-func (s *ConnStats) incOutputBytes(n int64) {
-	atomic.AddUint64(&s.totalOutputBytes, uint64(n))
+func (s *connStats) incInternodeOutputBytes(n int64) {
+	atomic.AddUint64(&s.internodeOutputBytes, uint64(n))
 }
 
 // Return internode total input bytes
-func (s *ConnStats) getTotalInputBytes() uint64 {
-	return atomic.LoadUint64(&s.totalInputBytes)
+func (s *connStats) getInternodeInputBytes() uint64 {
+	return atomic.LoadUint64(&s.internodeInputBytes)
 }
 
 // Return total output bytes
-func (s *ConnStats) getTotalOutputBytes() uint64 {
-	return atomic.LoadUint64(&s.totalOutputBytes)
+func (s *connStats) getInternodeOutputBytes() uint64 {
+	return atomic.LoadUint64(&s.internodeOutputBytes)
 }
 
 // Increase S3 total input bytes
-func (s *ConnStats) incS3InputBytes(n int64) {
+func (s *connStats) incS3InputBytes(n int64) {
 	atomic.AddUint64(&s.s3InputBytes, uint64(n))
 }
 
 // Increase S3 total output bytes
-func (s *ConnStats) incS3OutputBytes(n int64) {
+func (s *connStats) incS3OutputBytes(n int64) {
 	atomic.AddUint64(&s.s3OutputBytes, uint64(n))
 }
 
 // Return S3 total input bytes
-func (s *ConnStats) getS3InputBytes() uint64 {
+func (s *connStats) getS3InputBytes() uint64 {
 	return atomic.LoadUint64(&s.s3InputBytes)
 }
 
 // Return S3 total output bytes
-func (s *ConnStats) getS3OutputBytes() uint64 {
+func (s *connStats) getS3OutputBytes() uint64 {
 	return atomic.LoadUint64(&s.s3OutputBytes)
 }
 
-// Increase Admin total input bytes
-func (s *ConnStats) incAdminInputBytes(n int64) {
-	atomic.AddUint64(&s.adminInputBytes, uint64(n))
-}
-
-// Increase Admin total output bytes
-func (s *ConnStats) incAdminOutputBytes(n int64) {
-	atomic.AddUint64(&s.adminOutputBytes, uint64(n))
-}
-
-// Return Admin total input bytes
-func (s *ConnStats) getAdminInputBytes() uint64 {
-	return atomic.LoadUint64(&s.adminInputBytes)
-}
-
-// Return Admin total output bytes
-func (s *ConnStats) getAdminOutputBytes() uint64 {
-	return atomic.LoadUint64(&s.adminOutputBytes)
-}
-
 // Return connection stats (total input/output bytes and total s3 input/output bytes)
-func (s *ConnStats) toServerConnStats() ServerConnStats {
-	return ServerConnStats{
-		TotalInputBytes:  s.getTotalInputBytes(),  // Traffic internode received
-		TotalOutputBytes: s.getTotalOutputBytes(), // Traffic internode sent
-		S3InputBytes:     s.getS3InputBytes(),     // Traffic S3 received
-		S3OutputBytes:    s.getS3OutputBytes(),    // Traffic S3 sent
-		AdminInputBytes:  s.getAdminInputBytes(),  // Traffic admin calls received
-		AdminOutputBytes: s.getAdminOutputBytes(), // Traffic admin calls sent
+func (s *connStats) toServerConnStats() serverConnStats {
+	return serverConnStats{
+		internodeInputBytes:  s.getInternodeInputBytes(),  // Traffic internode received
+		internodeOutputBytes: s.getInternodeOutputBytes(), // Traffic internode sent
+		s3InputBytes:         s.getS3InputBytes(),         // Traffic S3 received
+		s3OutputBytes:        s.getS3OutputBytes(),        // Traffic S3 sent
 	}
 }
 
 // Prepare new ConnStats structure
-func newConnStats() *ConnStats {
-	return &ConnStats{}
+func newConnStats() *connStats {
+	return &connStats{}
 }
 
 type bucketS3RXTX struct {
@@ -260,28 +236,28 @@ func (s *bucketConnStats) incS3OutputBytes(bucket string, n int64) {
 	s.stats[bucket] = stats
 }
 
-// Return S3 total input bytes for input bucket
-func (s *bucketConnStats) getS3InputBytes(bucket string) uint64 {
-	s.RLock()
-	defer s.RUnlock()
-
-	stats := s.stats[bucket]
-	if stats == nil {
-		return 0
-	}
-	return stats.s3InputBytes
+type inOutBytes struct {
+	In  uint64
+	Out uint64
 }
 
-// Return S3 total output bytes
-func (s *bucketConnStats) getS3OutputBytes(bucket string) uint64 {
+// Return S3 total input bytes for input bucket
+func (s *bucketConnStats) getS3InOutBytes() map[string]inOutBytes {
 	s.RLock()
 	defer s.RUnlock()
 
-	stats := s.stats[bucket]
-	if stats == nil {
-		return 0
+	if len(s.stats) == 0 {
+		return nil
 	}
-	return stats.s3OutputBytes
+
+	bucketStats := make(map[string]inOutBytes, len(s.stats))
+	for k, v := range s.stats {
+		bucketStats[k] = inOutBytes{
+			In:  v.s3InputBytes,
+			Out: v.s3OutputBytes,
+		}
+	}
+	return bucketStats
 }
 
 // delete metrics once bucket is deleted.

cmd/http-tracer.go

@@ -65,7 +65,7 @@ func getOpName(name string) (op string) {
 
 // If trace is enabled, execute the request if it is traced by other handlers
 // otherwise, generate a trace event with request information but no response.
-func httpTracer(h http.Handler) http.Handler {
+func httpTracerMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		// Setup a http request response recorder - this is needed for
 		// http stats requests and audit if enabled.
@@ -77,7 +77,7 @@ func httpTracer(h http.Handler) http.Handler {
 
 		// Create tracing data structure and associate it to the request context
 		tc := mcontext.TraceCtxt{
-			AmzReqID:         r.Header.Get(xhttp.AmzRequestID),
+			AmzReqID:         w.Header().Get(xhttp.AmzRequestID),
 			RequestRecorder:  reqRecorder,
 			ResponseRecorder: respRecorder,
 		}

cmd/iam-object-store.go

@@ -390,7 +390,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	}
 	listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx)
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to list IAM data: %w", err)
 	}
 
 	// Loads things in the same order as `LoadIAMCache()`
@@ -401,7 +401,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range policiesList {
 		policyName := path.Dir(item)
 		if err := iamOS.loadPolicyDoc(ctx, policyName, cache.iamPolicyDocsMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return err
+			return fmt.Errorf("unable to load the policy doc `%s`: %w", policyName, err)
 		}
 	}
 	setDefaultCannedPolicies(cache.iamPolicyDocsMap)
@@ -412,7 +412,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 		for _, item := range regUsersList {
 			userName := path.Dir(item)
 			if err := iamOS.loadUser(ctx, userName, regUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
-				return err
+				return fmt.Errorf("unable to load the user `%s`: %w", userName, err)
 			}
 		}
 
@@ -421,7 +421,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 		for _, item := range groupsList {
 			group := path.Dir(item)
 			if err := iamOS.loadGroup(ctx, group, cache.iamGroupsMap); err != nil && err != errNoSuchGroup {
-				return err
+				return fmt.Errorf("unable to load the group `%s`: %w", group, err)
 			}
 		}
 	}
@@ -431,7 +431,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range userPolicyMappingsList {
 		userName := strings.TrimSuffix(item, ".json")
 		if err := iamOS.loadMappedPolicy(ctx, userName, regUser, false, cache.iamUserPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return err
+			return fmt.Errorf("unable to load the policy mapping for the user `%s`: %w", userName, err)
 		}
 	}
 
@@ -440,7 +440,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range groupPolicyMappingsList {
 		groupName := strings.TrimSuffix(item, ".json")
 		if err := iamOS.loadMappedPolicy(ctx, groupName, regUser, true, cache.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return err
+			return fmt.Errorf("unable to load the policy mapping for the group `%s`: %w", groupName, err)
 		}
 	}
 
@@ -449,7 +449,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range svcAccList {
 		userName := path.Dir(item)
 		if err := iamOS.loadUser(ctx, userName, svcUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
-			return err
+			return fmt.Errorf("unable to load the service account `%s`: %w", userName, err)
 		}
 	}
 
@@ -458,7 +458,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range stsUsersList {
 		userName := path.Dir(item)
 		if err := iamOS.loadUser(ctx, userName, stsUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
-			return err
+			return fmt.Errorf("unable to load the STS user `%s`: %w", userName, err)
 		}
 	}
 
@@ -467,7 +467,7 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 	for _, item := range stsPolicyMappingsList {
 		stsName := strings.TrimSuffix(item, ".json")
 		if err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, cache.iamUserPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return err
+			return fmt.Errorf("unable to load the policy mapping for the STS user `%s`: %w", stsName, err)
 		}
 	}
 

cmd/iam-store.go

@@ -2177,7 +2177,9 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 		}
 	}
 
-	if !userExists {
+	// If root user has no STS/Service Accounts, userExists would be false here,
+	// so we handle this exception.
+	if !userExists && globalActiveCred.AccessKey != accessKey {
 		return nil, errNoSuchUser
 	}
 

cmd/jwt.go

@@ -46,6 +46,8 @@ var (
 	errAccessKeyDisabled  = errors.New("The access key you provided is disabled")
 	errAuthentication     = errors.New("Authentication failed, check your access credentials")
 	errNoAuthToken        = errors.New("JWT token missing")
+	errSkewedAuthTime     = errors.New("Skewed authenticationdate/time")
+	errMalformedAuth      = errors.New("Malformed authentication input")
 )
 
 // cachedAuthenticateNode will cache authenticateNode results for given values up to ttl.

cmd/last-minute.go

@@ -90,16 +90,6 @@ func (a *AccElem) add(dur time.Duration) {
 	a.N++
 }
 
-// Add a duration to a single element.
-func (a *AccElem) addSize(dur time.Duration, sz int64) {
-	if dur < 0 {
-		dur = 0
-	}
-	a.Total += int64(dur)
-	a.Size += sz
-	a.N++
-}
-
 // Merge b into a.
 func (a *AccElem) merge(b AccElem) {
 	a.N += b.N
@@ -156,11 +146,10 @@ func (l *lastMinuteLatency) add(t time.Duration) {
 }
 
 // Add  a new duration data
-func (l *lastMinuteLatency) addSize(t time.Duration, sz int64) {
-	sec := time.Now().Unix()
+func (l *lastMinuteLatency) addAll(sec int64, a AccElem) {
 	l.forwardTo(sec)
 	winIdx := sec % 60
-	l.Totals[winIdx].addSize(t, sz)
+	l.Totals[winIdx].merge(a)
 	l.LastSec = sec
 }
 

cmd/lock-rest-server.go

@@ -73,7 +73,7 @@ func (l *lockRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool {
 
 func getLockArgs(r *http.Request) (args dsync.LockArgs, err error) {
 	dec := msgpNewReader(io.LimitReader(r.Body, 1000*humanize.KiByte))
-	defer readMsgpReaderPool.Put(dec)
+	defer readMsgpReaderPoolPut(dec)
 	err = args.DecodeMsg(dec)
 	return args, err
 }
@@ -244,18 +244,22 @@ func lockMaintenance(ctx context.Context) {
 
 // registerLockRESTHandlers - register lock rest router.
 func registerLockRESTHandlers(router *mux.Router) {
+	h := func(f http.HandlerFunc) http.HandlerFunc {
+		return collectInternodeStats(httpTraceHdrs(f))
+	}
+
 	lockServer := &lockRESTServer{
 		ll: newLocker(),
 	}
 
 	subrouter := router.PathPrefix(lockRESTPrefix).Subrouter()
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodHealth).HandlerFunc(httpTraceHdrs(lockServer.HealthHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRefresh).HandlerFunc(httpTraceHdrs(lockServer.RefreshHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(httpTraceHdrs(lockServer.LockHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(httpTraceHdrs(lockServer.RLockHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(httpTraceHdrs(lockServer.UnlockHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(httpTraceHdrs(lockServer.RUnlockHandler))
-	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodForceUnlock).HandlerFunc(httpTraceAll(lockServer.ForceUnlockHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodHealth).HandlerFunc(h(lockServer.HealthHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRefresh).HandlerFunc(h(lockServer.RefreshHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodLock).HandlerFunc(h(lockServer.LockHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRLock).HandlerFunc(h(lockServer.RLockHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodUnlock).HandlerFunc(h(lockServer.UnlockHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodRUnlock).HandlerFunc(h(lockServer.RUnlockHandler))
+	subrouter.Methods(http.MethodPost).Path(lockRESTVersionPrefix + lockRESTMethodForceUnlock).HandlerFunc(h(lockServer.ForceUnlockHandler))
 
 	globalLockServer = lockServer.ll
 

cmd/metacache-server-pool.go

@@ -402,7 +402,7 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
 				}
 			}
 
-			queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication)
+			queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication, 0)
 		}
 	}
 }

cmd/metacache-stream.go

@@ -255,12 +255,13 @@ type metacacheReader struct {
 func newMetacacheReader(r io.Reader) *metacacheReader {
 	dec := s2DecPool.Get().(*s2.Reader)
 	dec.Reset(r)
-	mr := msgp.NewReader(dec)
+	mr := msgpNewReader(dec)
 	return &metacacheReader{
 		mr: mr,
 		closer: func() {
 			dec.Reset(nil)
 			s2DecPool.Put(dec)
+			readMsgpReaderPoolPut(mr)
 		},
 		creator: func() error {
 			v, err := mr.ReadByte()

cmd/metrics-realtime.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/disk"
+	"github.com/minio/minio/internal/net"
 )
 
 type collectMetricsOpts struct {
@@ -69,6 +70,18 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 	if types.Contains(madmin.MetricsSiteResync) {
 		m.Aggregated.SiteResync = globalSiteResyncMetrics.report(opts.depID)
 	}
+	if types.Contains(madmin.MetricNet) {
+		m.Aggregated.Net = &madmin.NetMetrics{
+			CollectedAt:   UTCNow(),
+			InterfaceName: globalInternodeInterface,
+		}
+		netStats, err := net.GetInterfaceNetStats(globalInternodeInterface)
+		if err != nil {
+			m.Errors = append(m.Errors, err.Error())
+		} else {
+			m.Aggregated.Net.NetStats = netStats
+		}
+	}
 	// Add types...
 
 	// ByHost is a shallow reference, so careful about sharing.

cmd/metrics-v2.go

@@ -964,7 +964,7 @@ func getS3TTFBDistributionMD() MetricDescription {
 		Subsystem: timeSubsystem,
 		Name:      ttfbDistribution,
 		Help:      "Distribution of the time to first byte across API calls",
-		Type:      histogramMetric,
+		Type:      gaugeMetric,
 	}
 }
 
@@ -1664,8 +1664,9 @@ func getMinioHealingMetrics() *MetricsGroup {
 }
 
 func getFailedItems(seq *healSequence) (m []Metric) {
-	m = make([]Metric, 0, 1)
-	for k, v := range seq.gethealFailedItemsMap() {
+	items := seq.gethealFailedItemsMap()
+	m = make([]Metric, 0, len(items))
+	for k, v := range items {
 		s := strings.Split(k, ",")
 		m = append(m, Metric{
 			Description: getHealObjectsFailTotalMD(),
@@ -1990,6 +1991,67 @@ func getHTTPMetrics() *MetricsGroup {
 				VariableLabels: map[string]string{"api": api},
 			})
 		}
+
+		for bucket, inOut := range globalBucketConnStats.getS3InOutBytes() {
+			recvBytes := inOut.In
+			if recvBytes > 0 {
+				metrics = append(metrics, Metric{
+					Description:    getBucketTrafficReceivedBytes(),
+					Value:          float64(recvBytes),
+					VariableLabels: map[string]string{"bucket": bucket},
+				})
+			}
+			sentBytes := inOut.Out
+			if sentBytes > 0 {
+				metrics = append(metrics, Metric{
+					Description:    getBucketTrafficSentBytes(),
+					Value:          float64(sentBytes),
+					VariableLabels: map[string]string{"bucket": bucket},
+				})
+			}
+
+			httpStats := globalBucketHTTPStats.load(bucket)
+			for k, v := range httpStats.currentS3Requests.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsInFlightMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS3Requests.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsTotalMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS3Canceled.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3RequestsCanceledMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS34xxErrors.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3Requests4xxErrorsMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+
+			for k, v := range httpStats.totalS35xxErrors.Load() {
+				metrics = append(metrics, Metric{
+					Description:    getBucketS3Requests5xxErrorsMD(),
+					Value:          float64(v),
+					VariableLabels: map[string]string{"bucket": bucket, "api": k},
+				})
+			}
+		}
+
 		return
 	})
 	return mg
@@ -2018,20 +2080,20 @@ func getNetworkMetrics() *MetricsGroup {
 			})
 			metrics = append(metrics, Metric{
 				Description: getInterNodeSentBytesMD(),
-				Value:       float64(connStats.TotalOutputBytes),
+				Value:       float64(connStats.internodeOutputBytes),
 			})
 			metrics = append(metrics, Metric{
 				Description: getInterNodeReceivedBytesMD(),
-				Value:       float64(connStats.TotalInputBytes),
+				Value:       float64(connStats.internodeInputBytes),
 			})
 		}
 		metrics = append(metrics, Metric{
 			Description: getS3SentBytesMD(),
-			Value:       float64(connStats.S3OutputBytes),
+			Value:       float64(connStats.s3OutputBytes),
 		})
 		metrics = append(metrics, Metric{
 			Description: getS3ReceivedBytesMD(),
-			Value:       float64(connStats.S3InputBytes),
+			Value:       float64(connStats.s3InputBytes),
 		})
 		return
 	})
@@ -2104,65 +2166,6 @@ func getBucketUsageMetrics() *MetricsGroup {
 				})
 			}
 
-			recvBytes := globalBucketConnStats.getS3InputBytes(bucket)
-			if recvBytes > 0 {
-				metrics = append(metrics, Metric{
-					Description:    getBucketTrafficReceivedBytes(),
-					Value:          float64(recvBytes),
-					VariableLabels: map[string]string{"bucket": bucket},
-				})
-			}
-
-			sentBytes := globalBucketConnStats.getS3OutputBytes(bucket)
-			if sentBytes > 0 {
-				metrics = append(metrics, Metric{
-					Description:    getBucketTrafficSentBytes(),
-					Value:          float64(sentBytes),
-					VariableLabels: map[string]string{"bucket": bucket},
-				})
-			}
-
-			httpStats := globalBucketHTTPStats.load(bucket)
-			for k, v := range httpStats.currentS3Requests.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3RequestsInFlightMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS3Requests.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3RequestsTotalMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS3Canceled.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3RequestsCanceledMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS34xxErrors.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3Requests4xxErrorsMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
-			for k, v := range httpStats.totalS35xxErrors.Load() {
-				metrics = append(metrics, Metric{
-					Description:    getBucketS3Requests5xxErrorsMD(),
-					Value:          float64(v),
-					VariableLabels: map[string]string{"bucket": bucket, "api": k},
-				})
-			}
-
 			if stats.hasReplicationUsage() {
 				for arn, stat := range stats.Stats {
 					metrics = append(metrics, Metric{

cmd/metrics.go

@@ -322,7 +322,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total number of bytes sent to the other peer nodes by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
-		float64(connStats.TotalOutputBytes),
+		float64(connStats.internodeOutputBytes),
 	)
 
 	ch <- prometheus.MustNewConstMetric(
@@ -331,7 +331,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total number of internode bytes received by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
-		float64(connStats.TotalInputBytes),
+		float64(connStats.internodeInputBytes),
 	)
 
 	// Network Sent/Received Bytes (Outbound)
@@ -341,7 +341,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total number of s3 bytes sent by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
-		float64(connStats.S3OutputBytes),
+		float64(connStats.s3OutputBytes),
 	)
 
 	ch <- prometheus.MustNewConstMetric(
@@ -350,7 +350,7 @@ func networkMetricsPrometheus(ch chan<- prometheus.Metric) {
 			"Total number of s3 bytes received by current MinIO server instance",
 			nil, nil),
 		prometheus.CounterValue,
-		float64(connStats.S3InputBytes),
+		float64(connStats.s3InputBytes),
 	)
 }
 

cmd/notification.go

@@ -22,6 +22,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
 	"net/url"
 	"sync"
@@ -87,6 +88,8 @@ func (g *NotificationGroup) Wait() []NotificationPeerErr {
 // The first call to return a non-nil error will be
 // collected in errs slice and returned by Wait().
 func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, addr xnet.Host) {
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
 	g.wg.Add(1)
 
 	go func() {
@@ -103,9 +106,9 @@ func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, a
 					ctx := logger.SetReqInfo(ctx, reqInfo)
 					logger.LogIf(ctx, err)
 				}
-				// Wait for one second and no need wait after last attempt.
+				// Wait for a minimum of 100ms and dynamically increase this based on number of attempts.
 				if i < g.retryCount-1 {
-					time.Sleep(1 * time.Second)
+					time.Sleep(100*time.Millisecond + time.Duration(r.Float64()*float64(time.Second)))
 				}
 				continue
 			}
@@ -804,7 +807,7 @@ func (sys *NotificationSys) addNodeErr(nodeInfo madmin.NodeInfo, peerClient *pee
 	addr := peerClient.host.String()
 	reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr)
 	ctx := logger.SetReqInfo(GlobalContext, reqInfo)
-	logger.LogOnceIf(ctx, err, "add-node-err"+addr)
+	logger.LogOnceIf(ctx, err, "add-node-err-"+addr)
 	nodeInfo.SetAddr(addr)
 	nodeInfo.SetError(err.Error())
 }
@@ -1384,3 +1387,83 @@ func (sys *NotificationSys) GetLastDayTierStats(ctx context.Context) DailyAllTie
 	}
 	return merged
 }
+
+// GetReplicationMRF - Get replication MRF from all peers.
+func (sys *NotificationSys) GetReplicationMRF(ctx context.Context, bucket, node string) (mrfCh chan madmin.ReplicationMRF, err error) {
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	peerChannels := make([]<-chan madmin.ReplicationMRF, len(sys.peerClients))
+	for index, client := range sys.peerClients {
+		if client == nil {
+			continue
+		}
+		host := client.host.String()
+		if host != node && node != "all" {
+			continue
+		}
+		index := index
+		g.Go(func() error {
+			var err error
+			peerChannels[index], err = sys.peerClients[index].GetReplicationMRF(ctx, bucket)
+			return err
+		}, index)
+	}
+	mrfCh = make(chan madmin.ReplicationMRF, 4000)
+	var wg sync.WaitGroup
+
+	for index, err := range g.Wait() {
+		if err != nil {
+			if sys.peerClients[index] != nil {
+				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress",
+					sys.peerClients[index].host.String())
+				logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), err, sys.peerClients[index].host.String())
+			} else {
+				logger.LogOnceIf(ctx, err, "peer-offline")
+			}
+			continue
+		}
+		wg.Add(1)
+		go func(ctx context.Context, peerChannel <-chan madmin.ReplicationMRF, wg *sync.WaitGroup) {
+			defer wg.Done()
+			for {
+				select {
+				case m, ok := <-peerChannel:
+					if !ok {
+						return
+					}
+					select {
+					case <-ctx.Done():
+						return
+					case mrfCh <- m:
+					}
+				case <-ctx.Done():
+					return
+				}
+			}
+		}(ctx, peerChannels[index], &wg)
+	}
+	wg.Add(1)
+	go func(ch chan madmin.ReplicationMRF) error {
+		defer wg.Done()
+		if node != "all" && node != globalLocalNodeName {
+			return nil
+		}
+		mCh, err := globalReplicationPool.getMRF(ctx, bucket)
+		if err != nil {
+			return err
+		}
+		for e := range mCh {
+			select {
+			case <-ctx.Done():
+				return err
+			default:
+				mrfCh <- e
+			}
+		}
+		return nil
+	}(mrfCh)
+	go func(wg *sync.WaitGroup) {
+		wg.Wait()
+		defer close(mrfCh)
+	}(&wg)
+	return mrfCh, nil
+}

cmd/object-api-common.go

@@ -62,7 +62,7 @@ func newStorageAPI(endpoint Endpoint, healthCheck bool) (storage StorageAPI, err
 		if err != nil {
 			return nil, err
 		}
-		return newXLStorageDiskIDCheck(storage), nil
+		return newXLStorageDiskIDCheck(storage, healthCheck), nil
 	}
 
 	return newStorageRESTClient(endpoint, healthCheck), nil

cmd/object-api-datatypes.go

@@ -185,6 +185,7 @@ type ObjectInfo struct {
 	VersionPurgeStatusInternal string
 	VersionPurgeStatus         VersionPurgeStatusType
 
+	replicationDecision string // internal representation of replication decision for use by DeleteObject handler
 	// The total count of all versions of this object
 	NumVersions int
 	//  The modtime of the successor object version if any

cmd/object-api-errors.go

@@ -735,3 +735,15 @@ func isErrInvalidRange(err error) bool {
 	_, ok := err.(InvalidRange)
 	return ok
 }
+
+// ReplicationPermissionCheck - Check if error type is ReplicationPermissionCheck.
+type ReplicationPermissionCheck struct{}
+
+func (e ReplicationPermissionCheck) Error() string {
+	return "Replication permission validation requests cannot be completed"
+}
+
+func isReplicationPermissionCheck(err error) bool {
+	_, ok := err.(ReplicationPermissionCheck)
+	return ok
+}

cmd/object-api-interface.go

@@ -35,8 +35,11 @@ import (
 // CheckPreconditionFn returns true if precondition check failed.
 type CheckPreconditionFn func(o ObjectInfo) bool
 
-// EvalMetadataFn validates input objInfo and returns an updated metadata
-type EvalMetadataFn func(o *ObjectInfo) error
+// EvalMetadataFn validates input objInfo and GetObjectInfo error and returns an updated metadata and replication decision if any
+type EvalMetadataFn func(o *ObjectInfo, gerr error) (ReplicateDecision, error)
+
+// EvalRetentionBypassFn validates input objInfo and GetObjectInfo error and returns an error if retention bypass is not allowed.
+type EvalRetentionBypassFn func(o ObjectInfo, gerr error) error
 
 // GetObjectInfoFn is the signature of GetObjectInfo function.
 type GetObjectInfoFn func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error)
@@ -100,7 +103,8 @@ type ObjectOptions struct {
 
 	InclFreeVersions bool
 
-	MetadataChg bool // is true if it is a metadata update operation.
+	MetadataChg           bool                  // is true if it is a metadata update operation.
+	EvalRetentionBypassFn EvalRetentionBypassFn // only set for enforcing retention bypass on DeleteObject.
 }
 
 // ExpirationOptions represents object options for object expiration at objectLayer.
@@ -182,6 +186,16 @@ func (o *ObjectOptions) PutReplicationState() (r ReplicationState) {
 	return
 }
 
+// SetEvalMetadataFn sets the metadata evaluation function
+func (o *ObjectOptions) SetEvalMetadataFn(f EvalMetadataFn) {
+	o.EvalMetadataFn = f
+}
+
+// SetEvalRetentionBypassFn sets the retention bypass function
+func (o *ObjectOptions) SetEvalRetentionBypassFn(f EvalRetentionBypassFn) {
+	o.EvalRetentionBypassFn = f
+}
+
 // ObjectLayer implements primitives for object API layer.
 type ObjectLayer interface {
 	// Locking operations on object.
@@ -233,7 +247,8 @@ type ObjectLayer interface {
 	AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error
 	CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error)
 
-	SetDriveCounts() []int // list of erasure stripe size for each pool in order.
+	GetDisks(poolIdx, setIdx int) ([]StorageAPI, error) // return the disks belonging to pool and set.
+	SetDriveCounts() []int                              // list of erasure stripe size for each pool in order.
 
 	// Healing operations.
 	HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error)

cmd/object-api-listobjects_test.go

@@ -1887,7 +1887,7 @@ func testListObjectsContinuation(obj ObjectLayer, instanceType string, t1 TestEr
 
 // Initialize FS backend for the benchmark.
 func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) {
-	obj, _, err := initObjectLayer(context.Background(), mustGetPoolEndpoints(disk))
+	obj, _, err := initObjectLayer(context.Background(), mustGetPoolEndpoints(0, disk))
 	if err != nil {
 		t.Fatal(err)
 	}

cmd/object-api-utils.go

@@ -402,7 +402,7 @@ func extractETag(metadata map[string]string) string {
 // to do case insensitive checks.
 func HasPrefix(s string, prefix string) bool {
 	if runtime.GOOS == globalWindowsOSName {
-		return strings.HasPrefix(strings.ToLower(s), strings.ToLower(prefix))
+		return stringsHasPrefixFold(s, prefix)
 	}
 	return strings.HasPrefix(s, prefix)
 }
@@ -1166,7 +1166,7 @@ func hasSpaceFor(di []*DiskInfo, size int64) (bool, error) {
 	var total uint64
 	var nDisks int
 	for _, disk := range di {
-		if disk == nil || disk.Total == 0 || (disk.FreeInodes < diskMinInodes && disk.UsedInodes > 0) {
+		if disk == nil || disk.Total == 0 {
 			// Disk offline, no inodes or something else is wrong.
 			continue
 		}
@@ -1175,16 +1175,20 @@ func hasSpaceFor(di []*DiskInfo, size int64) (bool, error) {
 		available += disk.Total - disk.Used
 	}
 
-	if nDisks < len(di)/2 {
+	if nDisks < len(di)/2 || nDisks <= 0 {
 		return false, fmt.Errorf("not enough online disks to calculate the available space, expected (%d)/(%d)", (len(di)/2)+1, nDisks)
 	}
 
 	// Check we have enough on each disk, ignoring diskFillFraction.
 	perDisk := size / int64(nDisks)
 	for _, disk := range di {
-		if disk == nil || disk.Total == 0 || (disk.FreeInodes < diskMinInodes && disk.UsedInodes > 0) {
+		if disk == nil || disk.Total == 0 {
 			continue
 		}
+		if disk.FreeInodes < diskMinInodes && disk.UsedInodes > 0 {
+			// We have an inode count, but not enough inodes.
+			return false, nil
+		}
 		if int64(disk.Free) <= perDisk {
 			return false, nil
 		}

cmd/object-handlers.go

@@ -454,7 +454,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 					w.Header()[xhttp.AmzVersionID] = []string{gr.ObjInfo.VersionID}
 					w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(gr.ObjInfo.DeleteMarker)}
 				}
-				QueueReplicationHeal(ctx, bucket, gr.ObjInfo)
+				QueueReplicationHeal(ctx, bucket, gr.ObjInfo, 0)
 			}
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -489,7 +489,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			}
 		}
 
-		QueueReplicationHeal(ctx, bucket, gr.ObjInfo)
+		QueueReplicationHeal(ctx, bucket, gr.ObjInfo, 0)
 	}
 
 	// filter object lock metadata if permission does not permit
@@ -715,7 +715,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 				w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 				w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(objInfo.DeleteMarker)}
 			}
-			QueueReplicationHeal(ctx, bucket, objInfo)
+			QueueReplicationHeal(ctx, bucket, objInfo, 0)
 			// do an additional verification whether object exists when object is deletemarker and request
 			// is from replication
 			if opts.CheckDMReplicationReady {
@@ -746,7 +746,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 				}
 			}
 		}
-		QueueReplicationHeal(ctx, bucket, objInfo)
+		QueueReplicationHeal(ctx, bucket, objInfo, 0)
 	}
 
 	// filter object lock metadata if permission does not permit
@@ -1261,7 +1261,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 
 		for k, v := range srcInfo.UserDefined {
-			if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+			if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) {
 				encMetadata[k] = v
 			}
 		}
@@ -1707,6 +1707,12 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 
+	if _, ok := r.Header[xhttp.MinIOSourceReplicationCheck]; ok {
+		// requests to just validate replication settings and permissions are not allowed to write data
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationPermissionCheckError), r.URL)
+		return
+	}
+
 	if err := enforceBucketQuotaHard(ctx, bucket, size); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -2297,6 +2303,11 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
+	if _, ok := r.Header[xhttp.MinIOSourceReplicationCheck]; ok {
+		// requests to just validate replication settings and permissions are not allowed to delete data
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationPermissionCheckError), r.URL)
+		return
+	}
 
 	replica := r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String()
 	if replica {
@@ -2326,33 +2337,29 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		return
 	}
 
-	getObjectInfo := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfo = api.CacheAPI().GetObjectInfo
-	}
-
 	os := newObjSweeper(bucket, object).WithVersion(opts.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended)
-	// Mutations of objects on versioning suspended buckets
-	// affect its null version. Through opts below we select
-	// the null version's remote object to delete if
-	// transitioned.
-	goiOpts := os.GetOpts()
-	goi, gerr := getObjectInfo(ctx, bucket, object, goiOpts)
-	if gerr == nil {
-		os.SetTransitionState(goi.TransitionedObject)
-	}
 
-	dsc := checkReplicateDelete(ctx, bucket, ObjectToDelete{
-		ObjectV: ObjectV{
-			ObjectName: object,
-			VersionID:  opts.VersionID,
-		},
-	}, goi, opts, gerr)
+	opts.SetEvalMetadataFn(func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
+		if replica { // no need to check replication on receiver
+			return dsc, nil
+		}
+		dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{
+			ObjectV: ObjectV{
+				ObjectName: object,
+				VersionID:  opts.VersionID,
+			},
+		}, *oi, opts, gerr)
+		// Mutations of objects on versioning suspended buckets
+		// affect its null version. Through opts below we select
+		// the null version's remote object to delete if
+		// transitioned.
+		if gerr == nil {
+			os.SetTransitionState(oi.TransitionedObject)
+		}
+		return dsc, nil
+	})
 
 	vID := opts.VersionID
-	if dsc.ReplicateAny() {
-		opts.SetDeleteReplicationState(dsc, opts.VersionID)
-	}
 	if replica {
 		opts.SetReplicaStatus(replication.Replica)
 		if opts.VersionPurgeStatus().Empty() {
@@ -2360,25 +2367,21 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			vID = ""
 		}
 	}
-
-	apiErr := ErrNone
-	if vID != "" {
-		apiErr = enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
-			ObjectV: ObjectV{
-				ObjectName: object,
-				VersionID:  vID,
-			},
-		}, goi, gerr)
-		if apiErr != ErrNone && apiErr != ErrNoSuchKey {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL)
-			return
+	opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) (err error) {
+		err = nil
+		if vID != "" {
+			err := enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{
+				ObjectV: ObjectV{
+					ObjectName: object,
+					VersionID:  vID,
+				},
+			}, goi, gerr)
+			if err != nil && !isErrObjectNotFound(err) {
+				return err
+			}
 		}
-	}
-
-	if apiErr == ErrNoSuchKey {
-		writeSuccessNoContent(w)
 		return
-	}
+	})
 
 	deleteObject := objectAPI.DeleteObject
 	if api.CacheAPI() != nil {
@@ -2393,6 +2396,12 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
+		if isErrObjectNotFound(err) {
+			writeSuccessNoContent(w)
+			return
+		}
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
 	}
 
 	if objInfo.Name == "" {
@@ -2419,7 +2428,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 		Host:         handlers.GetSourceIP(r),
 	})
 
-	if dsc.ReplicateAny() {
+	if objInfo.ReplicationStatus == replication.Pending || objInfo.VersionPurgeStatus == Pending {
 		dmVersionID := ""
 		versionID := ""
 		if objInfo.DeleteMarker {
@@ -2434,7 +2443,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 				DeleteMarkerVersionID: dmVersionID,
 				DeleteMarkerMTime:     DeleteMarkerMTime{objInfo.ModTime},
 				DeleteMarker:          objInfo.DeleteMarker,
-				ReplicationState:      objInfo.getReplicationState(dsc.String(), opts.VersionID, false),
+				ReplicationState:      objInfo.getReplicationState(),
 			},
 			Bucket:    bucket,
 			EventType: ReplicateIncomingDelete,
@@ -2505,7 +2514,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 	popts := ObjectOptions{
 		MTime:     opts.MTime,
 		VersionID: opts.VersionID,
-		EvalMetadataFn: func(oi *ObjectInfo) error {
+		EvalMetadataFn: func(oi *ObjectInfo, gerr error) (ReplicateDecision, error) {
 			oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = strings.ToUpper(string(legalHold.Status))
 			oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
 
@@ -2514,7 +2523,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
 			}
-			return nil
+			return dsc, nil
 		},
 	}
 
@@ -2666,9 +2675,9 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 	popts := ObjectOptions{
 		MTime:     opts.MTime,
 		VersionID: opts.VersionID,
-		EvalMetadataFn: func(oi *ObjectInfo) error {
+		EvalMetadataFn: func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
 			if err := enforceRetentionBypassForPut(ctx, r, *oi, objRetention, cred, owner); err != nil {
-				return err
+				return dsc, err
 			}
 			if objRetention.Mode.Valid() {
 				oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(objRetention.Mode)
@@ -2678,12 +2687,12 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 				oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = ""
 			}
 			oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano)
-			dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(*oi, replication.MetadataReplicationType, opts))
+			dsc = mustReplicate(ctx, bucket, object, getMustReplicateOptions(*oi, replication.MetadataReplicationType, opts))
 			if dsc.ReplicateAny() {
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
 			}
-			return nil
+			return dsc, nil
 		},
 	}
 

cmd/object-lambda-handlers.go

@@ -22,7 +22,6 @@ import (
 	"io"
 	"net/http"
 	"net/url"
-	"strings"
 	"time"
 
 	"github.com/klauspost/compress/gzhttp"
@@ -182,7 +181,7 @@ func StatusCode(text string) int {
 func fwdHeadersToS3(h http.Header, w http.ResponseWriter) {
 	const trim = "x-amz-fwd-header-"
 	for k, v := range h {
-		if strings.HasPrefix(strings.ToLower(k), trim) {
+		if stringsHasPrefixFold(k, trim) {
 			w.Header()[k[len(trim):]] = v
 		}
 	}

cmd/peer-rest-client.go

@@ -944,3 +944,34 @@ func (client *peerRESTClient) Netperf(ctx context.Context, duration time.Duratio
 	err = gob.NewDecoder(respBody).Decode(&result)
 	return result, err
 }
+
+// GetReplicationMRF - get replication MRF for bucket
+func (client *peerRESTClient) GetReplicationMRF(ctx context.Context, bucket string) (chan madmin.ReplicationMRF, error) {
+	values := make(url.Values)
+	values.Set(peerRESTBucket, bucket)
+
+	respBody, err := client.callWithContext(ctx, peerRESTMethodGetReplicationMRF, values, nil, -1)
+	if err != nil {
+		return nil, err
+	}
+	dec := gob.NewDecoder(respBody)
+	ch := make(chan madmin.ReplicationMRF)
+	go func(ch chan madmin.ReplicationMRF) {
+		defer func() {
+			xhttp.DrainBody(respBody)
+			close(ch)
+		}()
+		for {
+			var entry madmin.ReplicationMRF
+			if err := dec.Decode(&entry); err != nil {
+				return
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case ch <- entry:
+			}
+		}
+	}(ch)
+	return ch, nil
+}

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion = "v30" // Removed bloom filter
+	peerRESTVersion = "v31" // Add replication MRF
 
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
@@ -76,6 +76,7 @@ const (
 	peerRESTMethodDevNull                     = "/devnull"
 	peerRESTMethodNetperf                     = "/netperf"
 	peerRESTMethodMetrics                     = "/metrics"
+	peerRESTMethodGetReplicationMRF           = "/getreplicationmrf"
 )
 
 const (

cmd/peer-rest-server.go

@@ -1313,6 +1313,31 @@ func (s *peerRESTServer) DriveSpeedTestHandler(w http.ResponseWriter, r *http.Re
 	logger.LogIf(r.Context(), gob.NewEncoder(w).Encode(result))
 }
 
+// GetReplicationMRFHandler - returns replication MRF for bucket
+func (s *peerRESTServer) GetReplicationMRFHandler(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	vars := mux.Vars(r)
+	bucketName := vars[peerRESTBucket]
+	ctx := newContext(r, w, "GetReplicationMRF")
+	re, err := globalReplicationPool.getMRF(ctx, bucketName)
+	if err != nil {
+		s.writeErrorResponse(w, err)
+		return
+	}
+	enc := gob.NewEncoder(w)
+
+	for m := range re {
+		if err := enc.Encode(m); err != nil {
+			s.writeErrorResponse(w, errors.New("Encoding mrf failed: "+err.Error()))
+			return
+		}
+	}
+}
+
 // DevNull - everything goes to io.Discard
 func (s *peerRESTServer) DevNull(w http.ResponseWriter, r *http.Request) {
 	if !s.IsValid(w, r) {
@@ -1359,56 +1384,61 @@ func (s *peerRESTServer) NetSpeedTestHandler(w http.ResponseWriter, r *http.Requ
 
 // registerPeerRESTHandlers - register peer rest router.
 func registerPeerRESTHandlers(router *mux.Router) {
+	h := func(f http.HandlerFunc) http.HandlerFunc {
+		return collectInternodeStats(httpTraceHdrs(f))
+	}
+
 	server := &peerRESTServer{}
 	subrouter := router.PathPrefix(peerRESTPrefix).Subrouter()
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodHealth).HandlerFunc(httpTraceHdrs(server.HealthHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(httpTraceHdrs(server.GetLocksHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(httpTraceHdrs(server.ServerInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLocalStorageInfo).HandlerFunc(httpTraceHdrs(server.LocalStorageInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcInfo).HandlerFunc(httpTraceHdrs(server.GetProcInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemInfo).HandlerFunc(httpTraceHdrs(server.GetMemInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMetrics).HandlerFunc(httpTraceHdrs(server.GetMetricsHandler)).Queries(restQueries(peerRESTMetricsTypes)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysErrors).HandlerFunc(httpTraceHdrs(server.GetSysErrorsHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysServices).HandlerFunc(httpTraceHdrs(server.GetSysServicesHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysConfig).HandlerFunc(httpTraceHdrs(server.GetSysConfigHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfo).HandlerFunc(httpTraceHdrs(server.GetOSInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwInfo).HandlerFunc(httpTraceHdrs(server.GetPartitionsHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCPUInfo).HandlerFunc(httpTraceHdrs(server.GetCPUsHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetAllBucketStats).HandlerFunc(httpTraceHdrs(server.GetAllBucketStatsHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteBucketMetadata).HandlerFunc(httpTraceHdrs(server.DeleteBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadBucketMetadata).HandlerFunc(httpTraceHdrs(server.LoadBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetBucketStats).HandlerFunc(httpTraceHdrs(server.GetBucketStatsHandler)).Queries(restQueries(peerRESTBucket)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSignalService).HandlerFunc(httpTraceHdrs(server.SignalServiceHandler)).Queries(restQueries(peerRESTSignal)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadBinary).HandlerFunc(httpTraceHdrs(server.DownloadBinaryHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCommitBinary).HandlerFunc(httpTraceHdrs(server.CommitBinaryHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeletePolicy).HandlerFunc(httpTraceAll(server.DeletePolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicy).HandlerFunc(httpTraceAll(server.LoadPolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicyMapping).HandlerFunc(httpTraceAll(server.LoadPolicyMappingHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteUser).HandlerFunc(httpTraceAll(server.DeleteUserHandler)).Queries(restQueries(peerRESTUser)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteServiceAccount).HandlerFunc(httpTraceAll(server.DeleteServiceAccountHandler)).Queries(restQueries(peerRESTUser)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadUser).HandlerFunc(httpTraceAll(server.LoadUserHandler)).Queries(restQueries(peerRESTUser, peerRESTUserTemp)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadServiceAccount).HandlerFunc(httpTraceAll(server.LoadServiceAccountHandler)).Queries(restQueries(peerRESTUser)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadGroup).HandlerFunc(httpTraceAll(server.LoadGroupHandler)).Queries(restQueries(peerRESTGroup)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodHealth).HandlerFunc(h(server.HealthHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocks).HandlerFunc(h(server.GetLocksHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodServerInfo).HandlerFunc(h(server.ServerInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLocalStorageInfo).HandlerFunc(h(server.LocalStorageInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcInfo).HandlerFunc(h(server.GetProcInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemInfo).HandlerFunc(h(server.GetMemInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMetrics).HandlerFunc(h(server.GetMetricsHandler)).Queries(restQueries(peerRESTMetricsTypes)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysErrors).HandlerFunc(h(server.GetSysErrorsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysServices).HandlerFunc(h(server.GetSysServicesHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysConfig).HandlerFunc(h(server.GetSysConfigHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodOsInfo).HandlerFunc(h(server.GetOSInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDiskHwInfo).HandlerFunc(h(server.GetPartitionsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCPUInfo).HandlerFunc(h(server.GetCPUsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetAllBucketStats).HandlerFunc(h(server.GetAllBucketStatsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteBucketMetadata).HandlerFunc(h(server.DeleteBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadBucketMetadata).HandlerFunc(h(server.LoadBucketMetadataHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetBucketStats).HandlerFunc(h(server.GetBucketStatsHandler)).Queries(restQueries(peerRESTBucket)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSignalService).HandlerFunc(h(server.SignalServiceHandler)).Queries(restQueries(peerRESTSignal)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadBinary).HandlerFunc(h(server.DownloadBinaryHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCommitBinary).HandlerFunc(h(server.CommitBinaryHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeletePolicy).HandlerFunc(h(server.DeletePolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicy).HandlerFunc(h(server.LoadPolicyHandler)).Queries(restQueries(peerRESTPolicy)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadPolicyMapping).HandlerFunc(h(server.LoadPolicyMappingHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteUser).HandlerFunc(h(server.DeleteUserHandler)).Queries(restQueries(peerRESTUser)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDeleteServiceAccount).HandlerFunc(h(server.DeleteServiceAccountHandler)).Queries(restQueries(peerRESTUser)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadUser).HandlerFunc(h(server.LoadUserHandler)).Queries(restQueries(peerRESTUser, peerRESTUserTemp)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadServiceAccount).HandlerFunc(h(server.LoadServiceAccountHandler)).Queries(restQueries(peerRESTUser)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadGroup).HandlerFunc(h(server.LoadGroupHandler)).Queries(restQueries(peerRESTGroup)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetReplicationMRF).HandlerFunc(httpTraceHdrs(server.GetReplicationMRFHandler)).Queries(restQueries(peerRESTBucket)...)
 
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStartProfiling).HandlerFunc(httpTraceAll(server.StartProfilingHandler)).Queries(restQueries(peerRESTProfiler)...)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadProfilingData).HandlerFunc(httpTraceHdrs(server.DownloadProfilingDataHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStartProfiling).HandlerFunc(h(server.StartProfilingHandler)).Queries(restQueries(peerRESTProfiler)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadProfilingData).HandlerFunc(h(server.DownloadProfilingDataHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodTrace).HandlerFunc(server.TraceHandler)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodListen).HandlerFunc(httpTraceHdrs(server.ListenHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodListen).HandlerFunc(h(server.ListenHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodBackgroundHealStatus).HandlerFunc(server.BackgroundHealStatusHandler)
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLog).HandlerFunc(server.ConsoleLogHandler)
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocalDiskIDs).HandlerFunc(httpTraceHdrs(server.GetLocalDiskIDs))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetBandwidth).HandlerFunc(httpTraceHdrs(server.GetBandwidth))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetMetacacheListing).HandlerFunc(httpTraceHdrs(server.GetMetacacheListingHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodUpdateMetacacheListing).HandlerFunc(httpTraceHdrs(server.UpdateMetacacheListingHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetPeerMetrics).HandlerFunc(httpTraceHdrs(server.GetPeerMetrics))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(httpTraceHdrs(server.LoadTransitionTierConfigHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedTest).HandlerFunc(httpTraceHdrs(server.SpeedTestHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveSpeedTest).HandlerFunc(httpTraceHdrs(server.DriveSpeedTestHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(httpTraceHdrs(server.NetSpeedTestHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDevNull).HandlerFunc(httpTraceHdrs(server.DevNull))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadSiteReplicationConfig).HandlerFunc(httpTraceHdrs(server.ReloadSiteReplicationConfigHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadPoolMeta).HandlerFunc(httpTraceHdrs(server.ReloadPoolMetaHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadRebalanceMeta).HandlerFunc(httpTraceHdrs(server.LoadRebalanceMetaHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStopRebalance).HandlerFunc(httpTraceHdrs(server.StopRebalanceHandler))
-	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLastDayTierStats).HandlerFunc(httpTraceHdrs(server.GetLastDayTierStatsHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLocalDiskIDs).HandlerFunc(h(server.GetLocalDiskIDs))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetBandwidth).HandlerFunc(h(server.GetBandwidth))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetMetacacheListing).HandlerFunc(h(server.GetMetacacheListingHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodUpdateMetacacheListing).HandlerFunc(h(server.UpdateMetacacheListingHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetPeerMetrics).HandlerFunc(h(server.GetPeerMetrics))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadTransitionTierConfig).HandlerFunc(h(server.LoadTransitionTierConfigHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedTest).HandlerFunc(h(server.SpeedTestHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveSpeedTest).HandlerFunc(h(server.DriveSpeedTestHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(h(server.NetSpeedTestHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDevNull).HandlerFunc(h(server.DevNull))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadSiteReplicationConfig).HandlerFunc(h(server.ReloadSiteReplicationConfigHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodReloadPoolMeta).HandlerFunc(h(server.ReloadPoolMetaHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodLoadRebalanceMeta).HandlerFunc(h(server.LoadRebalanceMetaHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStopRebalance).HandlerFunc(h(server.StopRebalanceHandler))
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetLastDayTierStats).HandlerFunc(h(server.GetLastDayTierStatsHandler))
 }

cmd/peer-s3-client.go

@@ -260,7 +260,7 @@ func (sys *S3PeerSys) MakeBucket(ctx context.Context, bucket string, opts MakeBu
 				perPoolErrs = append(perPoolErrs, errs[i])
 			}
 		}
-		if poolErr := reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, len(perPoolErrs)/2+1); poolErr != nil {
+		if poolErr := reduceReadQuorumErrs(ctx, perPoolErrs, bucketOpIgnoredErrs, len(perPoolErrs)/2+1); poolErr != nil {
 			return toObjectErr(poolErr, bucket)
 		}
 	}
@@ -303,7 +303,7 @@ func (sys *S3PeerSys) DeleteBucket(ctx context.Context, bucket string, opts Dele
 				perPoolErrs = append(perPoolErrs, errs[i])
 			}
 		}
-		if poolErr := reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, len(perPoolErrs)/2+1); poolErr != nil {
+		if poolErr := reduceReadQuorumErrs(ctx, perPoolErrs, bucketOpIgnoredErrs, len(perPoolErrs)/2+1); poolErr != nil && poolErr != errVolumeNotFound {
 			// re-create successful deletes, since we are return an error.
 			sys.MakeBucket(ctx, bucket, MakeBucketOptions{})
 			return toObjectErr(poolErr, bucket)

cmd/peer-s3-server.go

@@ -333,9 +333,13 @@ func registerPeerS3Handlers(router *mux.Router) {
 	server := &peerS3Server{}
 	subrouter := router.PathPrefix(peerS3Prefix).Subrouter()
 
-	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodHealth).HandlerFunc(httpTraceHdrs(server.HealthHandler))
-	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodMakeBucket).HandlerFunc(httpTraceHdrs(server.MakeBucketHandler))
-	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodDeleteBucket).HandlerFunc(httpTraceHdrs(server.DeleteBucketHandler))
-	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodGetBucketInfo).HandlerFunc(httpTraceHdrs(server.GetBucketInfoHandler))
-	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodListBuckets).HandlerFunc(httpTraceHdrs(server.ListBucketsHandler))
+	h := func(f http.HandlerFunc) http.HandlerFunc {
+		return collectInternodeStats(httpTraceHdrs(f))
+	}
+
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodHealth).HandlerFunc(h(server.HealthHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodMakeBucket).HandlerFunc(h(server.MakeBucketHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodDeleteBucket).HandlerFunc(h(server.DeleteBucketHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodGetBucketInfo).HandlerFunc(h(server.GetBucketInfoHandler))
+	subrouter.Methods(http.MethodPost).Path(peerS3VersionPrefix + peerS3MethodListBuckets).HandlerFunc(h(server.ListBucketsHandler))
 }

cmd/perf-tests.go

@@ -24,6 +24,7 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"net/url"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -31,6 +32,7 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
+	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/pkg/randreader"
 )
 
@@ -240,7 +242,7 @@ func (n *netPerfRX) Connect() {
 	n.Lock()
 	defer n.Unlock()
 	n.activeConnections++
-	atomic.StoreUint64(&globalNetPerfRX.RX, 0)
+	atomic.StoreUint64(&n.RX, 0)
 	n.lastToConnect = time.Now()
 }
 
@@ -338,3 +340,85 @@ func netperf(ctx context.Context, duration time.Duration) madmin.NetperfNodeResu
 	globalNetPerfRX.Reset()
 	return madmin.NetperfNodeResult{Endpoint: "", TX: r.n / uint64(duration.Seconds()), RX: uint64(rx / delta.Seconds()), Error: errStr}
 }
+
+func siteNetperf(ctx context.Context, duration time.Duration) madmin.SiteNetPerfNodeResult {
+	r := &netperfReader{eof: make(chan struct{})}
+	r.buf = make([]byte, 128*humanize.KiByte)
+	rand.Read(r.buf)
+
+	clusterInfos, err := globalSiteReplicationSys.GetClusterInfo(ctx)
+	if err != nil {
+		return madmin.SiteNetPerfNodeResult{Error: err.Error()}
+	}
+
+	// Scale the number of connections from 32 -> 4 from small to large clusters.
+	connectionsPerPeer := 3 + (29+len(clusterInfos.Sites)-1)/len(clusterInfos.Sites)
+
+	errStr := ""
+	var wg sync.WaitGroup
+
+	for _, info := range clusterInfos.Sites {
+		// skip self
+		if globalDeploymentID == info.DeploymentID {
+			continue
+		}
+		info := info
+		wg.Add(connectionsPerPeer)
+		for i := 0; i < connectionsPerPeer; i++ {
+			go func() {
+				defer wg.Done()
+				cli, err := globalSiteReplicationSys.getAdminClient(ctx, info.DeploymentID)
+				if err != nil {
+					return
+				}
+				rp := cli.GetEndpointURL()
+				reqURL := &url.URL{
+					Scheme: rp.Scheme,
+					Host:   rp.Host,
+					Path:   adminPathPrefix + adminAPIVersionPrefix + adminAPISiteReplicationDevNull,
+				}
+				req, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL.String(), r)
+				if err != nil {
+					return
+				}
+				client := &http.Client{
+					Timeout:   duration + 10*time.Second,
+					Transport: globalRemoteTargetTransport,
+				}
+				resp, err := client.Do(req)
+				if err != nil {
+					return
+				}
+				defer xhttp.DrainBody(resp.Body)
+			}()
+		}
+	}
+
+	time.Sleep(duration)
+	close(r.eof)
+	wg.Wait()
+	for {
+		if globalSiteNetPerfRX.ActiveConnections() == 0 || contextCanceled(ctx) {
+			break
+		}
+		time.Sleep(time.Second)
+	}
+	rx := float64(globalSiteNetPerfRX.RXSample)
+	delta := globalSiteNetPerfRX.firstToDisconnect.Sub(globalSiteNetPerfRX.lastToConnect)
+	// If the first disconnected before the last connected, we likely had a network issue.
+	if delta <= 0 {
+		rx = 0
+		errStr = "detected network disconnections, possibly an unstable network"
+	}
+
+	globalSiteNetPerfRX.Reset()
+	return madmin.SiteNetPerfNodeResult{
+		Endpoint:        "",
+		TX:              r.n,
+		TXTotalDuration: duration,
+		RX:              uint64(rx),
+		RXTotalDuration: delta,
+		Error:           errStr,
+		TotalConn:       uint64(connectionsPerPeer),
+	}
+}

cmd/post-policy-fan-out.go

@@ -36,6 +36,7 @@ type fanOutOptions struct {
 	Key      []byte
 	KmsCtx   kms.Context
 	Checksum *hash.Checksum
+	MD5Hex   string
 }
 
 // fanOutPutObject takes an input source reader and fans out multiple PUT operations
@@ -53,7 +54,14 @@ func fanOutPutObject(ctx context.Context, bucket string, objectAPI ObjectLayer,
 
 			objInfos[idx] = ObjectInfo{Name: req.Key}
 
-			hr, err := hash.NewReader(bytes.NewReader(fanOutBuf), int64(len(fanOutBuf)), "", "", -1)
+			hopts := hash.Options{
+				Size:       int64(len(fanOutBuf)),
+				MD5Hex:     opts.MD5Hex,
+				SHA256Hex:  "",
+				ActualSize: -1,
+				DisableMD5: true,
+			}
+			hr, err := hash.NewReaderWithOpts(bytes.NewReader(fanOutBuf), hopts)
 			if err != nil {
 				errs[idx] = err
 				return

cmd/routers.go

@@ -41,32 +41,34 @@ func registerDistErasureRouters(router *mux.Router, endpointServerPools Endpoint
 	registerLockRESTHandlers(router)
 }
 
-// List of some generic handlers which are applied for all incoming requests.
-var globalHandlers = []mux.MiddlewareFunc{
-	// The generic tracer needs to be the first handler
-	// to catch all requests returned early by any other handler
-	httpTracer,
-	// Auth handler verifies incoming authorization headers and
-	// routes them accordingly. Client receives a HTTP error for
-	// invalid/unsupported signatures.
+// List of some generic middlewares which are applied for all incoming requests.
+var globalMiddlewares = []mux.MiddlewareFunc{
+	// set x-amz-request-id header and others
+	addCustomHeadersMiddleware,
+	// The generic tracer needs to be the first middleware to catch all requests
+	// returned early by any other middleware (but after the middleware that
+	// sets the amz request id).
+	httpTracerMiddleware,
+	// Auth middleware verifies incoming authorization headers and routes them
+	// accordingly. Client receives a HTTP error for invalid/unsupported
+	// signatures.
 	//
 	// Validates all incoming requests to have a valid date header.
-	setAuthHandler,
-	// Redirect some pre-defined browser request paths to a static location prefix.
-	setBrowserRedirectHandler,
-	// Adds 'crossdomain.xml' policy handler to serve legacy flash clients.
-	setCrossDomainPolicy,
+	setAuthMiddleware,
+	// Redirect some pre-defined browser request paths to a static location
+	// prefix.
+	setBrowserRedirectMiddleware,
+	// Adds 'crossdomain.xml' policy middleware to serve legacy flash clients.
+	setCrossDomainPolicyMiddleware,
 	// Limits all body and header sizes to a maximum fixed limit
-	setRequestLimitHandler,
+	setRequestLimitMiddleware,
 	// Validate all the incoming requests.
-	setRequestValidityHandler,
-	// set x-amz-request-id header.
-	addCustomHeaders,
-	// Add upload forwarding handler for site replication
-	setUploadForwardingHandler,
-	// Add bucket forwarding handler
-	setBucketForwardingHandler,
-	// Add new handlers here.
+	setRequestValidityMiddleware,
+	// Add upload forwarding middleware for site replication
+	setUploadForwardingMiddleware,
+	// Add bucket forwarding middleware
+	setBucketForwardingMiddleware,
+	// Add new middlewares here.
 }
 
 // configureServer handler returns final handler for the http server.
@@ -98,7 +100,7 @@ func configureServerHandler(endpointServerPools EndpointServerPools) (http.Handl
 	// Add API router
 	registerAPIRouter(router)
 
-	router.Use(globalHandlers...)
+	router.Use(globalMiddlewares...)
 
 	return router, nil
 }

cmd/s3-zip-handlers.go

@@ -519,10 +519,10 @@ func updateObjectMetadataWithZipInfo(ctx context.Context, objectAPI ObjectLayer,
 	popts := ObjectOptions{
 		MTime:     srcInfo.ModTime,
 		VersionID: srcInfo.VersionID,
-		EvalMetadataFn: func(oi *ObjectInfo) error {
+		EvalMetadataFn: func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
 			oi.UserDefined[archiveTypeMetadataKey] = at
 			oi.UserDefined[archiveInfoMetadataKey] = zipInfoStr
-			return nil
+			return dsc, nil
 		},
 	}
 

cmd/server-main.go

@@ -248,6 +248,9 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	logger.FatalIf(err, "Invalid command line arguments")
 	globalNodes = globalEndpoints.GetNodes()
 
+	// Initialize, see which NIC the service is running on, and save it as global
+	_ = getGlobalInternodeInterface(ctx.String("interface"))
+
 	globalLocalNodeName = GetLocalPeer(globalEndpoints, globalMinioHost, globalMinioPort)
 	nodeNameSum := sha256.Sum256([]byte(globalLocalNodeName))
 	globalLocalNodeNameHex = hex.EncodeToString(nodeNameSum[:])
@@ -256,7 +259,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	globalProxyTransport = NewCustomHTTPProxyTransport()()
 	globalProxyEndpoints = GetProxyEndpoints(globalEndpoints)
 	globalInternodeTransport = NewInternodeHTTPTransport()()
-	globalRemoteTargetTransport = NewRemoteTargetHTTPTransport()()
+	globalRemoteTargetTransport = NewRemoteTargetHTTPTransport(false)()
 
 	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
 		PassHost:     true,
@@ -461,6 +464,33 @@ func initConfigSubsystem(ctx context.Context, newObject ObjectLayer) error {
 	return nil
 }
 
+func getGlobalInternodeInterface(interfs ...string) string {
+	globalInternodeInterfaceOnce.Do(func() {
+		if len(interfs) != 0 && strings.TrimSpace(interfs[0]) != "" {
+			globalInternodeInterface = interfs[0]
+			return
+		}
+		ip := "127.0.0.1"
+		host, _ := mustSplitHostPort(globalMinioAddr)
+		if host != "" {
+			ip = host
+		}
+		globalInternodeInterface = ip
+		ifs, _ := net.Interfaces()
+		for _, interf := range ifs {
+			addrs, err := interf.Addrs()
+			if err == nil {
+				for _, addr := range addrs {
+					if strings.SplitN(addr.String(), "/", 2)[0] == ip {
+						globalInternodeInterface = interf.Name
+					}
+				}
+			}
+		}
+	})
+	return globalInternodeInterface
+}
+
 // Return the list of address that MinIO server needs to listen on:
 //   - Returning 127.0.0.1 is necessary so Console will be able to send
 //     requests to the local S3 API.
@@ -628,6 +658,7 @@ func serverMain(ctx *cli.Context) {
 	if err != nil {
 		logFatalErrs(err, Endpoint{}, true)
 	}
+	bootstrapTrace("newObjectLayer (initialized)")
 
 	xhttp.SetDeploymentID(globalDeploymentID)
 	xhttp.SetMinIOVersion(Version)
@@ -689,6 +720,7 @@ func serverMain(ctx *cli.Context) {
 	go func() {
 		bootstrapTrace("globalIAMSys.Init")
 		globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
+		bootstrapTrace("globalIAMSys.Initialized")
 
 		// Initialize Console UI
 		if globalBrowserEnabled {
@@ -786,23 +818,26 @@ func serverMain(ctx *cli.Context) {
 		// Initialize bucket metadata sub-system.
 		bootstrapTrace("globalBucketMetadataSys.Init")
 		globalBucketMetadataSys.Init(GlobalContext, buckets, newObject)
+		bootstrapTrace("globalBucketMetadataSys.Initialized")
 
 		// initialize replication resync state.
-		bootstrapTrace("go initResync")
-		go globalReplicationPool.initResync(GlobalContext, buckets, newObject)
+		bootstrapTrace("initResync")
+		globalReplicationPool.initResync(GlobalContext, buckets, newObject)
 
 		// Initialize site replication manager after bucket metadata
 		bootstrapTrace("globalSiteReplicationSys.Init")
 		globalSiteReplicationSys.Init(GlobalContext, newObject)
+		bootstrapTrace("globalSiteReplicationSys.Initialized")
 
 		// Initialize quota manager.
 		bootstrapTrace("globalBucketQuotaSys.Init")
 		globalBucketQuotaSys.Init(newObject)
+		bootstrapTrace("globalBucketQuotaSys.Initialized")
 
 		// Populate existing buckets to the etcd backend
 		if globalDNSConfig != nil {
 			// Background this operation.
-			bootstrapTrace("initFederatorBackend")
+			bootstrapTrace("go initFederatorBackend")
 			go initFederatorBackend(buckets, newObject)
 		}
 

cmd/server-main_test.go

@@ -35,7 +35,7 @@ func TestNewObjectLayer(t *testing.T) {
 	}
 	defer removeRoots(disks)
 
-	obj, err := newObjectLayer(ctx, mustGetPoolEndpoints(disks...))
+	obj, err := newObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
 	if err != nil {
 		t.Fatal("Unexpected object layer initialization error", err)
 	}
@@ -55,7 +55,7 @@ func TestNewObjectLayer(t *testing.T) {
 	}
 	defer removeRoots(disks)
 
-	obj, err = newObjectLayer(ctx, mustGetPoolEndpoints(disks...))
+	obj, err = newObjectLayer(ctx, mustGetPoolEndpoints(0, disks...))
 	if err != nil {
 		t.Fatal("Unexpected object layer initialization error", err)
 	}