Send kafka notification messages in batches when queue_dir is enabled (#18164)

Fixes #18124

.dockerignore

@@ -1,10 +1,17 @@
 .git
 .github
-docs
 default.etcd
 *.gz
 *.tar.gz
 *.bzip2
 *.zip
 browser/node_modules
-node_modules
+node_modules
+docs/debugging/s3-verify/s3-verify
+docs/debugging/xl-meta/xl-meta
+docs/debugging/s3-check-md5/s3-check-md5
+docs/debugging/hash-set/hash-set
+docs/debugging/healing-bin/healing-bin
+docs/debugging/inspect/inspect
+docs/debugging/pprofgoparser/pprofgoparser
+docs/debugging/reorder-disks/reorder-disks

.github/workflows/mint.yml

@@ -37,7 +37,11 @@ jobs:
 
       - name: build-minio
         run: |
-          TAG="minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
+          TAG="quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
+
+      - name: multipart uploads test
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/multipart/migrate.sh "${{ steps.vars.outputs.sha_short }}"
 
       - name: compress and encrypt
         run: |
@@ -59,7 +63,14 @@ jobs:
              docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true
              docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true
           done
-          docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }}
+
+          docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site1.yaml rm -s -f || true
+          docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site2.yaml rm -s -f || true
+          for volume in $(docker volume ls -q | grep minio); do
+            docker volume rm ${volume} || true
+          done
+
+          docker rmi -f quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}
           docker system prune -f || true
           docker volume prune -f || true
           docker volume rm $(docker volume ls -q -f dangling=true) || true

.github/workflows/mint/minio-compress-encrypt.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
   expose:
     - "9000"

.github/workflows/mint/minio-erasure.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" edata{1...4}
   expose:
     - "9000"

.github/workflows/mint/minio-pools.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
   command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
   expose:
     - "9000"

.github/workflows/multipart/docker-compose-site1.yaml

@@ -0,0 +1,66 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:${RELEASE}
+  command: server http://site1-minio{1...4}/data{1...2}
+  environment:
+    - MINIO_PROMETHEUS_AUTH_TYPE=public
+    - CI=true
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  site1-minio1:
+    <<: *minio-common
+    hostname: site1-minio1
+    volumes:
+      - site1-data1-1:/data1
+      - site1-data1-2:/data2
+
+  site1-minio2:
+    <<: *minio-common
+    hostname: site1-minio2
+    volumes:
+      - site1-data2-1:/data1
+      - site1-data2-2:/data2
+
+  site1-minio3:
+    <<: *minio-common
+    hostname: site1-minio3
+    volumes:
+      - site1-data3-1:/data1
+      - site1-data3-2:/data2
+
+  site1-minio4:
+    <<: *minio-common
+    hostname: site1-minio4
+    volumes:
+      - site1-data4-1:/data1
+      - site1-data4-2:/data2
+
+  site1-nginx:
+    image: nginx:1.19.2-alpine
+    hostname: site1-nginx
+    volumes:
+      - ./nginx-site1.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9001:9001"
+    depends_on:
+      - site1-minio1
+      - site1-minio2
+      - site1-minio3
+      - site1-minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  site1-data1-1:
+  site1-data1-2:
+  site1-data2-1:
+  site1-data2-2:
+  site1-data3-1:
+  site1-data3-2:
+  site1-data4-1:
+  site1-data4-2:

.github/workflows/multipart/docker-compose-site2.yaml

@@ -0,0 +1,66 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:${RELEASE}
+  command: server http://site2-minio{1...4}/data{1...2}
+  environment:
+    - MINIO_PROMETHEUS_AUTH_TYPE=public
+    - CI=true
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  site2-minio1:
+    <<: *minio-common
+    hostname: site2-minio1
+    volumes:
+      - site2-data1-1:/data1
+      - site2-data1-2:/data2
+
+  site2-minio2:
+    <<: *minio-common
+    hostname: site2-minio2
+    volumes:
+      - site2-data2-1:/data1
+      - site2-data2-2:/data2
+
+  site2-minio3:
+    <<: *minio-common
+    hostname: site2-minio3
+    volumes:
+      - site2-data3-1:/data1
+      - site2-data3-2:/data2
+
+  site2-minio4:
+    <<: *minio-common
+    hostname: site2-minio4
+    volumes:
+      - site2-data4-1:/data1
+      - site2-data4-2:/data2
+
+  site2-nginx:
+    image: nginx:1.19.2-alpine
+    hostname: site2-nginx
+    volumes:
+      - ./nginx-site2.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9002:9002"
+    depends_on:
+      - site2-minio1
+      - site2-minio2
+      - site2-minio3
+      - site2-minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  site2-data1-1:
+  site2-data1-2:
+  site2-data2-1:
+  site2-data2-2:
+  site2-data3-1:
+  site2-data3-2:
+  site2-data4-1:
+  site2-data4-2:

.github/workflows/multipart/migrate.sh

@@ -0,0 +1,115 @@
+#!/bin/bash
+
+set -x
+
+## change working directory
+cd .github/workflows/multipart/
+
+docker-compose -f docker-compose-site1.yaml rm -s -f
+docker-compose -f docker-compose-site2.yaml rm -s -f
+for volume in $(docker volume ls -q | grep minio); do
+	docker volume rm ${volume}
+done
+
+if [ ! -f ./mc ]; then
+	wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+fi
+
+(
+	cd /tmp
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+)
+
+export RELEASE=RELEASE.2023-08-29T23-07-35Z
+
+docker-compose -f docker-compose-site1.yaml up -d
+docker-compose -f docker-compose-site2.yaml up -d
+
+sleep 30s
+
+./mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4
+./mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4
+
+./mc ready site1/
+./mc ready site2/
+
+./mc admin replicate add site1 site2
+./mc mb site1/testbucket/
+./mc cp -r --quiet /usr/bin site1/testbucket/
+
+sleep 5
+
+s3-check-md5 -h
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads"
+	exit 1
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads"
+	exit 1
+fi
+
+./mc cp -r --quiet /usr/bin site1/testbucket/
+
+sleep 5
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+## we do not need to fail here, since we are going to test
+## upgrading to master, healing and being able to recover
+## the last version.
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads ${failed_count_site1}"
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads ${failed_count_site2}"
+fi
+
+export RELEASE=${1}
+
+docker-compose -f docker-compose-site1.yaml up -d
+docker-compose -f docker-compose-site2.yaml up -d
+
+./mc ready site1/
+./mc ready site2/
+
+for i in $(seq 1 10); do
+	# mc admin heal -r --remove when used against a LB endpoint
+	# behaves flaky, let this run 10 times before giving up
+	./mc admin heal -r --remove --json site1/ 2>&1 >/dev/null
+	./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null
+done
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads"
+	exit 1
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads"
+	exit 1
+fi
+
+docker-compose -f docker-compose-site1.yaml rm -s -f
+docker-compose -f docker-compose-site2.yaml rm -s -f
+for volume in $(docker volume ls -q | grep minio); do
+	docker volume rm ${volume}
+done
+
+docker system prune -f || true
+docker volume prune -f || true
+docker volume rm $(docker volume ls -q -f dangling=true) || true
+
+## change working directory
+cd ../../../

.github/workflows/multipart/nginx-site1.conf

@@ -0,0 +1,61 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server site1-minio1:9000;
+        server site1-minio2:9000;
+        server site1-minio3:9000;
+        server site1-minio4:9000;
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}

.github/workflows/multipart/nginx-site2.conf

@@ -0,0 +1,61 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server site2-minio1:9000;
+        server site2-minio2:9000;
+        server site2-minio3:9000;
+        server site2-minio4:9000;
+    }
+
+    server {
+        listen       9002;
+        listen  [::]:9002;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}

.github/workflows/run-mint.sh

@@ -16,7 +16,7 @@ docker volume rm $(docker volume ls -f dangling=true) || true
 cd .github/workflows/mint
 
 docker-compose -f minio-${MODE}.yaml up -d
-sleep 5m
+sleep 30s
 
 docker system prune -f || true
 docker volume prune -f || true

.github/workflows/vulncheck.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.21.0
+          go-version: 1.21.1
           check-latest: true
       - name: Get official govulncheck
         run: go install golang.org/x/vuln/cmd/govulncheck@latest

.gitignore

@@ -32,6 +32,8 @@ minio.RELEASE*
 mc
 nancy
 inspects/*
+.bin/
+*.gz
 docs/debugging/s3-verify/s3-verify
 docs/debugging/xl-meta/xl-meta
 docs/debugging/s3-check-md5/s3-check-md5
@@ -39,5 +41,4 @@ docs/debugging/hash-set/hash-set
 docs/debugging/healing-bin/healing-bin
 docs/debugging/inspect/inspect
 docs/debugging/pprofgoparser/pprofgoparser
-.bin/
-*.gz
+docs/debugging/reorder-disks/reorder-disks

.golangci.yml

@@ -13,7 +13,6 @@ linters:
   enable:
     - durationcheck
     - gocritic
-    - gofmt
     - gofumpt
     - goimports
     - gomodguard

CREDITS

@@ -2392,6 +2392,7 @@ https://github.com/IBM/sarama
 # MIT License
 
 Copyright (c) 2013 Shopify
+
 Copyright (c) 2023 IBM Corporation
 
 Permission is hereby granted, free of charge, to any person obtaining
@@ -2922,33 +2923,6 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 ================================================================
 
-github.com/benbjohnson/clock
-https://github.com/benbjohnson/clock
-----------------------------------------------------------------
-The MIT License (MIT)
-
-Copyright (c) 2014 Ben Johnson
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
-================================================================
-
 github.com/beorn7/perks
 https://github.com/beorn7/perks
 ----------------------------------------------------------------
@@ -3089,7 +3063,7 @@ https://github.com/charmbracelet/lipgloss
 ----------------------------------------------------------------
 MIT License
 
-Copyright (c) 2021 Charmbracelet, Inc
+Copyright (c) 2021-2023 Charmbracelet, Inc
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -11869,6 +11843,8 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
 github.com/hashicorp/golang-lru
 https://github.com/hashicorp/golang-lru
 ----------------------------------------------------------------
+Copyright (c) 2014 HashiCorp, Inc.
+
 Mozilla Public License, version 2.0
 
 1. Definitions
@@ -20017,6 +19993,673 @@ For more information on this, and how to apply and follow the GNU AGPL, see
 
 ================================================================
 
+github.com/minio/pkg/v2
+https://github.com/minio/pkg/v2
+----------------------------------------------------------------
+                    GNU AFFERO GENERAL PUBLIC LICENSE
+                       Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+  A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate.  Many developers of free software are heartened and
+encouraged by the resulting cooperation.  However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+  The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community.  It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server.  Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+  An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals.  This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU Affero General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Remote Network Interaction; Use with the GNU General Public License.
+
+  Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software.  This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time.  Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source.  For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code.  There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+<https://www.gnu.org/licenses/>.
+
+================================================================
+
 github.com/minio/selfupdate
 https://github.com/minio/selfupdate
 ----------------------------------------------------------------

Makefile

@@ -6,7 +6,7 @@ GOARCH := $(shell go env GOARCH)
 GOOS := $(shell go env GOOS)
 
 VERSION ?= $(shell git describe --tags)
-TAG ?= "minio/minio:$(VERSION)"
+TAG ?= "quay.io/minio/minio:$(VERSION)"
 
 GOLANGCI_VERSION = v1.51.2
 GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
@@ -45,22 +45,22 @@ lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
 
 check: test
-test: verifiers build ## builds minio, runs linters, tests
+test: verifiers build build-debugging ## builds minio, runs linters, tests
 	@echo "Running unit tests"
 	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...
 
-test-root-disable: install
+test-root-disable: install-race
 	@echo "Running minio root lockdown tests"
 	@env bash $(PWD)/buildscripts/disable-root.sh
 
-test-decom: install
+test-decom: install-race
 	@echo "Running minio decom tests"
 	@env bash $(PWD)/docs/distributed/decom.sh
 	@env bash $(PWD)/docs/distributed/decom-encrypted.sh
 	@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
 	@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh
 
-test-upgrade: build
+test-upgrade: install-race
 	@echo "Running minio upgrade tests"
 	@(env bash $(PWD)/buildscripts/minio-upgrade.sh)
 
@@ -86,18 +86,18 @@ test-replication-3site:
 test-delete-replication:
 	@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
 
-test-replication: install test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
+test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 
-test-site-replication-ldap: install ## verify automatic site replication
+test-site-replication-ldap: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with LDAP)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh)
 
-test-site-replication-oidc: install ## verify automatic site replication
+test-site-replication-oidc: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with OIDC)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-oidc.sh)
 
-test-site-replication-minio: install ## verify automatic site replication
+test-site-replication-minio: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)
 
@@ -128,6 +128,9 @@ verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/resolve-right-versions.sh)
 
+build-debugging:
+	@(env bash $(PWD)/docs/debugging/build.sh)
+
 build: checks ## builds minio to $(PWD)
 	@echo "Building minio binary to './minio'"
 	@CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
@@ -159,6 +162,12 @@ docker: build ## builds minio docker container
 	@echo "Building minio docker image '$(TAG)'"
 	@docker build -q --no-cache -t $(TAG) . -f Dockerfile
 
+install-race: checks ## builds minio to $(PWD)
+	@echo "Building minio binary to './minio'"
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
+	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
+
 install: build ## builds minio and installs it to $GOPATH/bin.
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio

buildscripts/disable-root.sh

@@ -56,6 +56,8 @@ done
 
 set +e
 
+sleep 10
+
 ./mc ls minioadm/
 if [ $? -ne 0 ]; then
 	echo "listing failed, 'minioadmin' should be enabled"

cmd/admin-bucket-handlers.go

@@ -94,7 +94,7 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		Quota:     data,
 		UpdatedAt: updatedAt,
 	}
-	if quotaConfig.Quota == 0 {
+	if quotaConfig.Size == 0 || quotaConfig.Quota == 0 {
 		bucketMeta.Quota = nil
 	}
 

cmd/admin-handler-utils.go

@@ -27,14 +27,14 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // validateAdminReq will validate request against and return whether it is allowed.
 // If any of the supplied actions are allowed it will be successful.
 // If nil ObjectLayer is returned, the operation is not permitted.
 // When nil ObjectLayer has been returned an error has always been sent to w.
-func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
+func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...policy.AdminAction) (ObjectLayer, auth.Credentials) {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil {
@@ -78,7 +78,7 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 
 	var apiErr APIError
 	switch e := err.(type) {
-	case iampolicy.Error:
+	case policy.Error:
 		apiErr = APIError{
 			Code:           "XMinioMalformedIAMPolicy",
 			Description:    e.Error(),

cmd/admin-handlers-config-kv.go

@@ -38,14 +38,14 @@ import (
 	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -149,7 +149,7 @@ type setConfigResult struct {
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -242,7 +242,7 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -288,7 +288,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -321,7 +321,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -367,7 +367,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -405,7 +405,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -430,7 +430,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -482,7 +482,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}

cmd/admin-handlers-idp-config.go

@@ -34,11 +34,11 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/ldap"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -220,7 +220,7 @@ func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *ht
 func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -273,7 +273,7 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
 func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -332,7 +332,7 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
 func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}

cmd/admin-handlers-idp-ldap.go

@@ -25,7 +25,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
@@ -50,7 +50,7 @@ func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r
 	// Check authorization.
 
 	objectAPI, cred := validateAdminReq(ctx, w, r,
-		iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
+		policy.ListGroupsAdminAction, policy.ListUsersAdminAction, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -96,7 +96,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 
 	// Check authorization.
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.UpdatePolicyAssociationAction)
 	if objectAPI == nil {
 		return
 	}

cmd/admin-handlers-pools.go

@@ -26,7 +26,7 @@ import (
 
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 var (
@@ -37,7 +37,7 @@ var (
 func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -113,7 +113,7 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -159,7 +159,7 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -200,7 +200,7 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -233,7 +233,7 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -304,7 +304,7 @@ func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -344,7 +344,7 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
 	if objectAPI == nil {
 		return
 	}

cmd/admin-handlers-users.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -19,6 +19,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -34,14 +35,14 @@ import (
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // RemoveUser - DELETE /minio/admin/v3/remove-user?accessKey=<access_key>
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.DeleteUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -85,7 +86,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ListUsersAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -119,7 +120,7 @@ func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ListUsersAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -186,10 +187,10 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 		checkDenyOnly = true
 	}
 
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.GetUserAdminAction,
+		Action:          policy.GetUserAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -218,7 +219,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.AddUserToGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -288,7 +289,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -315,7 +316,7 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListGroupsAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -339,7 +340,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.EnableGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -382,7 +383,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
+	objectAPI, creds := validateAdminReq(ctx, w, r, policy.EnableUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -470,10 +471,10 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		checkDenyOnly = true
 	}
 
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.CreateUserAdminAction,
+		Action:          policy.CreateUserAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -544,10 +545,10 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.ListTemporaryAccountsAdminAction,
+		Action:          policy.ListTemporaryAccountsAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -556,16 +557,16 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	stsAccount, policy, err := globalIAMSys.GetTemporaryAccount(ctx, accessKey)
+	stsAccount, sessionPolicy, err := globalIAMSys.GetTemporaryAccount(ctx, accessKey)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	var stsAccountPolicy iampolicy.Policy
+	var stsAccountPolicy policy.Policy
 
-	if policy != nil {
-		stsAccountPolicy = *policy
+	if sessionPolicy != nil {
+		stsAccountPolicy = *sessionPolicy
 	} else {
 		policiesNames, err := globalIAMSys.PolicyDBGet(stsAccount.ParentUser, false)
 		if err != nil {
@@ -584,7 +585,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 	infoResp := madmin.TemporaryAccountInfoResp{
 		ParentUser:    stsAccount.ParentUser,
 		AccountStatus: stsAccount.Status,
-		ImpliedPolicy: policy == nil,
+		ImpliedPolicy: sessionPolicy == nil,
 		Policy:        string(policyJSON),
 		Expiration:    &stsAccount.Expiration,
 	}
@@ -709,10 +710,10 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		//
 		// This allows turning off service accounts for request sender,
 		// if there is no deny statement this call is implicitly enabled.
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
+		if !globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     requestorUser,
 			Groups:          requestorGroups,
-			Action:          iampolicy.CreateServiceAccountAdminAction,
+			Action:          policy.CreateServiceAccountAdminAction,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
 			Claims:          cred.Claims,
@@ -743,10 +744,10 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	} else {
 		// Need permission if we are creating a service account for a
 		// user <> to the request sender
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
+		if !globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     requestorUser,
 			Groups:          requestorGroups,
-			Action:          iampolicy.CreateServiceAccountAdminAction,
+			Action:          policy.CreateServiceAccountAdminAction,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
 			Claims:          cred.Claims,
@@ -773,9 +774,9 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		// latter, a group notion is not supported.
 	}
 
-	var sp *iampolicy.Policy
+	var sp *policy.Policy
 	if len(createReq.Policy) > 0 {
-		sp, err = iampolicy.ParseConfig(bytes.NewReader(createReq.Policy))
+		sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy))
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -864,10 +865,10 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.UpdateServiceAccountAdminAction,
+		Action:          policy.UpdateServiceAccountAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -903,9 +904,9 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	var sp *iampolicy.Policy
+	var sp *policy.Policy
 	if len(updateReq.NewPolicy) > 0 {
-		sp, err = iampolicy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
+		sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -970,16 +971,16 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	svcAccount, policy, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
+	svcAccount, sessionPolicy, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.ListServiceAccountsAdminAction,
+		Action:          policy.ListServiceAccountsAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -995,10 +996,10 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		}
 	}
 
-	var svcAccountPolicy iampolicy.Policy
+	var svcAccountPolicy policy.Policy
 
-	if policy != nil {
-		svcAccountPolicy = *policy
+	if sessionPolicy != nil {
+		svcAccountPolicy = *sessionPolicy
 	} else {
 		policiesNames, err := globalIAMSys.PolicyDBGet(svcAccount.ParentUser, false)
 		if err != nil {
@@ -1024,7 +1025,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		Name:          svcAccount.Name,
 		Description:   svcAccount.Description,
 		AccountStatus: svcAccount.Status,
-		ImpliedPolicy: policy == nil,
+		ImpliedPolicy: sessionPolicy == nil,
 		Policy:        string(policyJSON),
 		Expiration:    expiration,
 	}
@@ -1067,10 +1068,10 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 	// sender), check that the user has permissions.
 	user := r.Form.Get("user")
 	if user != "" && user != cred.AccessKey {
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
+		if !globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.ListServiceAccountsAdminAction,
+			Action:          policy.ListServiceAccountsAdminAction,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
 			Claims:          cred.Claims,
@@ -1148,10 +1149,10 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	// since this is a delete call we shall allow it to be deleted if possible.
 	svcAccount, _, _ := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
 
-	adminPrivilege := globalIAMSys.IsAllowed(iampolicy.Args{
+	adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.RemoveServiceAccountAdminAction,
+		Action:          policy.RemoveServiceAccountAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -1192,7 +1193,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	writeSuccessNoContent(w)
 }
 
-// AccountInfoHandler returns usage
+// AccountInfoHandler returns usage, permissions and other bucket metadata for incoming us
 func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
@@ -1219,10 +1220,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	enablePrefixUsage := r.Form.Get("prefix-usage") == "true"
 
 	isAllowedAccess := func(bucketName string) (rd, wr bool) {
-		if globalIAMSys.IsAllowed(iampolicy.Args{
+		if globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.ListBucketAction,
+			Action:          policy.ListBucketAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
@@ -1232,10 +1233,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			rd = true
 		}
 
-		if globalIAMSys.IsAllowed(iampolicy.Args{
+		if globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.GetBucketLocationAction,
+			Action:          policy.GetBucketLocationAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
@@ -1245,10 +1246,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			rd = true
 		}
 
-		if globalIAMSys.IsAllowed(iampolicy.Args{
+		if globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.PutObjectAction,
+			Action:          policy.PutObjectAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
@@ -1261,12 +1262,30 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	// Load the latest calculated data usage
-	dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
+	bucketStorageCache.Once.Do(func() {
+		// Set this to 10 secs since its enough, as scanner
+		// does not update the bucket usage values frequently.
+		bucketStorageCache.TTL = 10 * time.Second
+
+		// Rely on older value if usage loading fails from disk.
+		bucketStorageCache.Relax = true
+		bucketStorageCache.Update = func() (interface{}, error) {
+			ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
+			defer done()
+
+			return loadDataUsageFromBackend(ctx, objectAPI)
+		}
+	})
+
+	var dataUsageInfo DataUsageInfo
+	v, _ := bucketStorageCache.Get()
+	if v != nil {
+		dataUsageInfo, _ = v.(DataUsageInfo)
+	}
 
 	// If etcd, dns federation configured list buckets from etcd.
-	var buckets []BucketInfo
 	var err error
+	var buckets []BucketInfo
 	if globalDNSConfig != nil && globalBucketFederation {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && !IsErrIgnored(err,
@@ -1285,7 +1304,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			return buckets[i].Name < buckets[j].Name
 		})
 	} else {
-		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{Cached: true})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -1298,14 +1317,14 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		accountName = cred.ParentUser
 	}
 
-	roleArn := iampolicy.Args{Claims: cred.Claims}.GetRoleArn()
-	policySetFromClaims, hasPolicyClaim := iampolicy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
-	var effectivePolicy iampolicy.Policy
+	roleArn := policy.Args{Claims: cred.Claims}.GetRoleArn()
+	policySetFromClaims, hasPolicyClaim := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
+	var effectivePolicy policy.Policy
 
 	var buf []byte
 	switch {
 	case accountName == globalActiveCred.AccessKey:
-		for _, policy := range iampolicy.DefaultPolicies {
+		for _, policy := range policy.DefaultPolicies {
 			if policy.Name == "consoleAdmin" {
 				effectivePolicy = policy.Definition
 				break
@@ -1417,7 +1436,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1463,7 +1482,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1475,7 +1494,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	newPolicies := make(map[string]iampolicy.Policy)
+	newPolicies := make(map[string]policy.Policy)
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
@@ -1494,7 +1513,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1505,7 +1524,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	newPolicies := make(map[string]iampolicy.Policy)
+	newPolicies := make(map[string]policy.Policy)
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
@@ -1524,7 +1543,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DeletePolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1550,7 +1569,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.CreatePolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1582,7 +1601,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	iamPolicy, err := iampolicy.ParseConfig(bytes.NewReader(iamPolicyBytes))
+	iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1614,7 +1633,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1688,7 +1707,7 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht
 
 	// Check authorization.
 	objectAPI, cred := validateAdminReq(ctx, w, r,
-		iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
+		policy.ListGroupsAdminAction, policy.ListUsersAdminAction, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1726,8 +1745,8 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht
 func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction,
-		iampolicy.AttachPolicyAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.UpdatePolicyAssociationAction,
+		policy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1826,7 +1845,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
 	// Get current object layer instance.
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportIAMAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportIAMAction)
 	if objectAPI == nil {
 		return
 	}
@@ -2096,7 +2115,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			return
 		default:
 			defer f.Close()
-			var allPolicies map[string]iampolicy.Policy
+			var allPolicies map[string]policy.Policy
 			data, err = io.ReadAll(f)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL)
@@ -2177,10 +2196,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					checkDenyOnly = true
 				}
 
-				if !globalIAMSys.IsAllowed(iampolicy.Args{
+				if !globalIAMSys.IsAllowed(policy.Args{
 					AccountName:     cred.AccessKey,
 					Groups:          cred.Groups,
-					Action:          iampolicy.CreateUserAdminAction,
+					Action:          policy.CreateUserAdminAction,
 					ConditionValues: getConditionValues(r, "", cred),
 					IsOwner:         owner,
 					Claims:          cred.Claims,
@@ -2257,10 +2276,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			for user, svcAcctReq := range serviceAcctReqs {
-				var sp *iampolicy.Policy
+				var sp *policy.Policy
 				var err error
 				if len(svcAcctReq.SessionPolicy) > 0 {
-					sp, err = iampolicy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy))
+					sp, err = policy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy))
 					if err != nil {
 						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
 						return
@@ -2271,10 +2290,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
 					return
 				}
-				if !globalIAMSys.IsAllowed(iampolicy.Args{
+				if !globalIAMSys.IsAllowed(policy.Args{
 					AccountName:     cred.AccessKey,
 					Groups:          cred.Groups,
-					Action:          iampolicy.CreateServiceAccountAdminAction,
+					Action:          policy.CreateServiceAccountAdminAction,
 					ConditionValues: getConditionValues(r, "", cred),
 					IsOwner:         owner,
 					Claims:          cred.Claims,

cmd/admin-handlers.go

@@ -57,7 +57,7 @@ import (
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/logger/message/log"
 	xnet "github.com/minio/pkg/v2/net"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 	"github.com/secure-io/sio-go"
 )
 
@@ -81,7 +81,7 @@ const (
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -252,11 +252,11 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	var objectAPI ObjectLayer
 	switch serviceSig {
 	case serviceRestart:
-		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction)
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceRestartAdminAction)
 	case serviceStop:
-		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction)
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceStopAdminAction)
 	case serviceFreeze, serviceUnFreeze:
-		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceFreezeAdminAction)
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceFreezeAdminAction)
 	}
 	if objectAPI == nil {
 		return
@@ -331,7 +331,7 @@ type ServerHTTPStats struct {
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.StorageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -370,7 +370,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -479,7 +479,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -562,7 +562,7 @@ type PeerLocks struct {
 func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ForceUnlockAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ForceUnlockAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -597,7 +597,7 @@ func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.TopLocksAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -648,7 +648,7 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -733,7 +733,7 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -828,7 +828,7 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -926,7 +926,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1110,7 +1110,7 @@ func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmi
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1131,7 +1131,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1179,7 +1179,7 @@ func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) ClientDevNullExtraTime(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.BandwidthMonitorAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.BandwidthMonitorAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1196,7 +1196,7 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 	ctx := r.Context()
 
 	timeStart := time.Now()
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.BandwidthMonitorAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.BandwidthMonitorAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1238,7 +1238,7 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1284,7 +1284,7 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1445,7 +1445,7 @@ func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo,
 func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1566,7 +1566,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.TraceAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -1581,7 +1581,9 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	traceCh := make(chan madmin.TraceInfo, 4000)
+
+	// Keep 100k buffered channel, should be sufficient to ensure we do not lose any events.
+	traceCh := make(chan madmin.TraceInfo, 100000)
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 
@@ -1605,7 +1607,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		peer.Trace(traceCh, ctx.Done(), traceOpts)
 	}
 
-	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	keepAliveTicker := time.NewTicker(time.Second)
 	defer keepAliveTicker.Stop()
 
 	enc := json.NewEncoder(w)
@@ -1637,7 +1639,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConsoleLogAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1718,7 +1720,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSCreateKeyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1739,7 +1741,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1776,7 +1778,7 @@ func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -2376,7 +2378,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -2483,7 +2485,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ServerInfoAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -2627,12 +2629,12 @@ func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
 	return loggerInfo, auditloggerInfo
 }
 
-func embedFileInZip(zipWriter *zip.Writer, name string, data []byte) error {
+func embedFileInZip(zipWriter *zip.Writer, name string, data []byte, fileMode os.FileMode) error {
 	// Send profiling data to zip as file
 	header, zerr := zip.FileInfoHeader(dummyFileInfo{
 		name:    name,
 		size:    int64(len(data)),
-		mode:    0o600,
+		mode:    fileMode,
 		modTime: UTCNow(),
 		isDir:   false,
 		sys:     nil,
@@ -2735,7 +2737,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	ctx := r.Context()
 
 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.InspectDataAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.InspectDataAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -2863,7 +2865,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		defer inspectZipW.Close()
 
 		if b := getClusterMetaInfo(ctx); len(b) > 0 {
-			logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b))
+			logger.LogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b, 0o600))
 		}
 	}
 
@@ -2927,11 +2929,16 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 		sb.WriteString(pool.CmdLine)
 	}
 	sb.WriteString("\n")
-	logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes()))
+	logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes(), 0o600))
+
+	scheme := "https"
+	if !globalIsTLS {
+		scheme = "http"
+	}
 
 	// save MinIO start script to inspect command
 	var scrb bytes.Buffer
-	scrb.WriteString(`#!/usr/bin/env bash
+	fmt.Fprintf(&scrb, `#!/usr/bin/env bash
 
 function main() {
 	for file in $(ls -1); do
@@ -2940,10 +2947,10 @@ function main() {
 	done
 
 	# Read content of inspect-input.txt
-	MINIO_OPTS=$(grep "Server command line args" <./inspect-input.txt | sed "s/Server command line args: //g" | sed -r "s#https:\/\/#\.\/#g")
+	MINIO_OPTS=$(grep "Server command line args" <./inspect-input.txt | sed "s/Server command line args: //g" | sed -r "s#%s:\/\/#\.\/#g")
 
 	# Start MinIO instance using the options
-	START_CMD="CI=on MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server ${MINIO_OPTS} &"
+	START_CMD="CI=on _MINIO_AUTO_DRIVE_HEALING=off minio server ${MINIO_OPTS} &"
 	echo
 	echo "Starting MinIO instance: ${START_CMD}"
 	echo
@@ -2955,9 +2962,8 @@ function main() {
 	sleep 10
 }
 
-main "$@"`,
-	)
-	logger.LogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes()))
+main "$@"`, scheme)
+	logger.LogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes(), 0o755))
 }
 
 func getSubnetAdminPublicKey() []byte {

cmd/admin-heal-ops.go

@@ -683,13 +683,6 @@ func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
 	}
 }
 
-func (h *healSequence) logHeal(healType madmin.HealItemType) {
-	h.mutex.Lock()
-	h.scannedItemsMap[healType]++
-	h.lastHealActivity = UTCNow()
-	h.mutex.Unlock()
-}
-
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
 	// Send heal request
 	task := healTask{

cmd/api-errors.go

@@ -2236,8 +2236,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrSlowDownWrite
 	case InsufficientReadQuorum:
 		apiErr = ErrSlowDownRead
-	case InvalidMarkerPrefixCombination:
-		apiErr = ErrNotImplemented
 	case InvalidUploadIDKeyCombination:
 		apiErr = ErrNotImplemented
 	case MalformedUploadID:

cmd/api-errors_test.go

@@ -42,7 +42,6 @@ var toAPIErrorTests = []struct {
 	{err: InvalidPart{}, errCode: ErrInvalidPart},
 	{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
 	{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
-	{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
 	{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
 	{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
 	{err: PartTooSmall{}, errCode: ErrEntityTooSmall},

cmd/auth-handler.go

@@ -567,7 +567,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 
 	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
 	// The verification happens implicit during reading.
-	reader, err := hash.NewReader(r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
+	reader, err := hash.NewReader(ctx, r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}

cmd/auth-handler_test.go

@@ -28,7 +28,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/internal/auth"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 type nullReader struct{}
@@ -443,7 +443,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 	}
 	for i, testCase := range testCases {
-		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
+		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, policy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}

cmd/background-newdisks-heal-ops.go

@@ -347,7 +347,7 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 
 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
 
-	if env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
+	if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn || env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
 		go monitorLocalDisksAndHeal(ctx, z)
 	}
 }

cmd/batch-handlers.go

@@ -28,7 +28,6 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
-	"path"
 	"runtime"
 	"strconv"
 	"strings"
@@ -38,6 +37,7 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
@@ -49,7 +49,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	"github.com/minio/pkg/v2/env"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 	"github.com/minio/pkg/v2/workers"
 	"gopkg.in/yaml.v2"
 )
@@ -114,7 +114,7 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 	srcObject := srcObjInfo.Name
 	tgtObject := srcObjInfo.Name
 	if r.Target.Prefix != "" {
-		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
+		tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name)
 	}
 
 	versionID := srcObjInfo.VersionID
@@ -167,7 +167,7 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 	}
 	defer rd.Close()
 
-	hr, err := hash.NewReader(rd, objInfo.Size, "", "", objInfo.Size)
+	hr, err := hash.NewReader(ctx, rd, objInfo.Size, "", "", objInfo.Size)
 	if err != nil {
 		return err
 	}
@@ -182,7 +182,7 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 	srcObject := srcObjInfo.Name
 	tgtObject := srcObjInfo.Name
 	if r.Target.Prefix != "" {
-		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
+		tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name)
 	}
 	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
 		opts.VersionID = ""
@@ -230,7 +230,7 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 		}
 		defer rd.Close()
 
-		hr, err = hash.NewReader(io.LimitReader(rd, objInfo.Size), objInfo.Size, "", "", objInfo.Size)
+		hr, err = hash.NewReader(ctx, io.LimitReader(rd, objInfo.Size), objInfo.Size, "", "", objInfo.Size)
 		if err != nil {
 			return err
 		}
@@ -270,6 +270,10 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
+	isTags := len(r.Flags.Filter.Tags) != 0
+	isMetadata := len(r.Flags.Filter.Metadata) != 0
+	isStorageClassOnly := len(r.Flags.Filter.Metadata) == 1 && strings.EqualFold(r.Flags.Filter.Metadata[0].Key, xhttp.AmzStorageClass)
+
 	skip := func(oi ObjectInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
@@ -290,7 +294,8 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			// skip all objects that are created after the specified time.
 			return true
 		}
-		if len(r.Flags.Filter.Tags) > 0 {
+
+		if isTags {
 			// Only parse object tags if tags filter is specified.
 			tagMap := map[string]string{}
 			tagStr := oi.UserTags
@@ -313,23 +318,19 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			return false
 		}
 
-		if len(r.Flags.Filter.Metadata) > 0 {
-			for _, kv := range r.Flags.Filter.Metadata {
-				for k, v := range oi.UserDefined {
-					if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
-						continue
-					}
-					// We only need to match x-amz-meta or standardHeaders
-					if kv.Match(BatchJobKV{Key: k, Value: v}) {
-						return true
-					}
+		for _, kv := range r.Flags.Filter.Metadata {
+			for k, v := range oi.UserDefined {
+				if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
+					continue
+				}
+				// We only need to match x-amz-meta or standardHeaders
+				if kv.Match(BatchJobKV{Key: k, Value: v}) {
+					return true
 				}
 			}
-
-			// None of the provided metadata filters match skip the object.
-			return false
 		}
 
+		// None of the provided filters match
 		return false
 	}
 
@@ -384,17 +385,32 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		for obj := range objInfoCh {
 			oi := toObjectInfo(r.Source.Bucket, obj.Key, obj)
 			if !minioSrc {
-				oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, miniogo.StatObjectOptions{})
-				if err == nil {
-					oi = toObjectInfo(r.Source.Bucket, obj.Key, oi2)
-				} else {
-					if isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) ||
-						isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
+				// Check if metadata filter was requested and it is expected to have
+				// all user metadata or just storageClass. If its only storageClass
+				// List() already returns relevant information for filter to be applied.
+				if isMetadata && !isStorageClassOnly {
+					oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, miniogo.StatObjectOptions{})
+					if err == nil {
+						oi = toObjectInfo(r.Source.Bucket, obj.Key, oi2)
+					} else {
+						if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) &&
+							!isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
+							logger.LogIf(ctx, err)
+						}
+						continue
+					}
+				}
+				if isTags {
+					tags, err := c.GetObjectTagging(ctx, r.Source.Bucket, obj.Key, minio.GetObjectTaggingOptions{})
+					if err == nil {
+						oi.UserTags = tags.String()
+					} else {
+						if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) &&
+							!isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
+							logger.LogIf(ctx, err)
+						}
 						continue
 					}
-					logger.LogIf(ctx, err)
-					cancel()
-					return err
 				}
 			}
 			if skip(oi) {
@@ -482,10 +498,12 @@ func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo
 		ReplicationStatusInternal: objInfo.ReplicationStatus,
 		UserTags:                  tags.String(),
 	}
+
 	oi.UserDefined = make(map[string]string, len(objInfo.Metadata))
 	for k, v := range objInfo.Metadata {
 		oi.UserDefined[k] = v[0]
 	}
+
 	ce, ok := oi.UserDefined[xhttp.ContentEncoding]
 	if !ok {
 		ce, ok = oi.UserDefined[strings.ToLower(xhttp.ContentEncoding)]
@@ -493,6 +511,12 @@ func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo
 	if ok {
 		oi.ContentEncoding = ce
 	}
+
+	_, ok = oi.UserDefined[xhttp.AmzStorageClass]
+	if !ok {
+		oi.UserDefined[xhttp.AmzStorageClass] = objInfo.StorageClass
+	}
+
 	return oi
 }
 
@@ -822,7 +846,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 
-	skip := func(info FileInfo) (ok bool) {
+	selectObj := func(info FileInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
 			return false
@@ -932,7 +956,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		results := make(chan ObjectInfo, 100)
 		if err := api.Walk(ctx, r.Source.Bucket, r.Source.Prefix, results, ObjectOptions{
 			WalkMarker: lastObject,
-			WalkFilter: skip,
+			WalkFilter: selectObj,
 		}); err != nil {
 			cancel()
 			// Do not need to retry if we can't list objects on source.
@@ -1258,7 +1282,7 @@ func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (p
 func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListBatchJobsAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListBatchJobsAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1308,7 +1332,7 @@ var errNoSuchJob = errors.New("no such job")
 func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DescribeBatchJobAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DescribeBatchJobAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1343,7 +1367,7 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.StartBatchJobAction)
+	objectAPI, creds := validateAdminReq(ctx, w, r, policy.StartBatchJobAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1397,7 +1421,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CancelBatchJobAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.CancelBatchJobAction)
 	if objectAPI == nil {
 		return
 	}

cmd/bucket-handlers.go

@@ -1139,7 +1139,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	hashReader, err := hash.NewReader(reader, fileSize, "", "", fileSize)
+	hashReader, err := hash.NewReader(ctx, reader, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -1254,7 +1254,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				return
 			}
 			// do not try to verify encrypted content/
-			hashReader, err = hash.NewReader(reader, -1, "", "", -1)
+			hashReader, err = hash.NewReader(ctx, reader, -1, "", "", -1)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return

cmd/bucket-quota.go

@@ -29,14 +29,12 @@ import (
 )
 
 // BucketQuotaSys - map of bucket and quota configuration.
-type BucketQuotaSys struct {
-	bucketStorageCache timedValue
-}
+type BucketQuotaSys struct{}
 
 // Get - Get quota configuration.
 func (sys *BucketQuotaSys) Get(ctx context.Context, bucketName string) (*madmin.BucketQuota, error) {
-	qCfg, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
-	return qCfg, err
+	cfg, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName)
+	return cfg, err
 }
 
 // NewBucketQuotaSys returns initialized BucketQuotaSys
@@ -44,16 +42,18 @@ func NewBucketQuotaSys() *BucketQuotaSys {
 	return &BucketQuotaSys{}
 }
 
+var bucketStorageCache timedValue
+
 // Init initialize bucket quota.
 func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
-	sys.bucketStorageCache.Once.Do(func() {
+	bucketStorageCache.Once.Do(func() {
 		// Set this to 10 secs since its enough, as scanner
 		// does not update the bucket usage values frequently.
-		sys.bucketStorageCache.TTL = 10 * time.Second
+		bucketStorageCache.TTL = 10 * time.Second
 		// Rely on older value if usage loading fails from disk.
-		sys.bucketStorageCache.Relax = true
-		sys.bucketStorageCache.Update = func() (interface{}, error) {
-			ctx, done := context.WithTimeout(context.Background(), 1*time.Second)
+		bucketStorageCache.Relax = true
+		bucketStorageCache.Update = func() (interface{}, error) {
+			ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
 			defer done()
 
 			return loadDataUsageFromBackend(ctx, objAPI)
@@ -63,16 +63,17 @@ func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 
 // GetBucketUsageInfo return bucket usage info for a given bucket
 func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, error) {
-	v, err := sys.bucketStorageCache.Get()
-	if err != nil && v != nil {
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
-	}
-	if v == nil {
-		logger.LogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
+	v, err := bucketStorageCache.Get()
+	timedout := OperationTimedOut{}
+	if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &timedout) {
+		if v != nil {
+			logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
+		} else {
+			logger.LogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
+		}
 	}
 
 	var bui BucketUsageInfo
-
 	dui, ok := v.(DataUsageInfo)
 	if ok {
 		bui = dui.BucketsUsage[bucket]
@@ -107,8 +108,16 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
 		return err
 	}
 
-	if q != nil && q.Type == madmin.HardQuota && q.Quota > 0 {
-		if uint64(size) >= q.Quota { // check if file size already exceeds the quota
+	var quotaSize uint64
+	if q != nil && q.Type == madmin.HardQuota {
+		if q.Size > 0 {
+			quotaSize = q.Size
+		} else if q.Quota > 0 {
+			quotaSize = q.Quota
+		}
+	}
+	if quotaSize > 0 {
+		if uint64(size) >= quotaSize { // check if file size already exceeds the quota
 			return BucketQuotaExceeded{Bucket: bucket}
 		}
 
@@ -117,7 +126,7 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
 			return err
 		}
 
-		if bui.Size > 0 && ((bui.Size + uint64(size)) >= q.Quota) {
+		if bui.Size > 0 && ((bui.Size + uint64(size)) >= quotaSize) {
 			return BucketQuotaExceeded{Bucket: bucket}
 		}
 	}

cmd/bucket-replication-handlers.go

@@ -232,10 +232,10 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseW
 	enc := json.NewEncoder(w)
 	stats := globalReplicationStats.getLatestReplicationStats(bucket)
 	bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
-	bwMap := bwRpt.BucketStats[bucket]
+	bwMap := bwRpt.BucketStats
 	for arn, st := range stats.ReplicationStats.Stats {
-		if bwMap != nil {
-			if bw, ok := bwMap[arn]; ok {
+		for opts, bw := range bwMap {
+			if opts.ReplicationARN != "" && opts.ReplicationARN == arn {
 				st.BandWidthLimitInBytesPerSecond = bw.LimitInBytesPerSecond
 				st.CurrentBandwidthInBytesPerSecond = bw.CurrentBandwidthInBytesPerSecond
 				stats.ReplicationStats.Stats[arn] = st
@@ -288,10 +288,10 @@ func (api objectAPIHandlers) GetBucketReplicationMetricsV2Handler(w http.Respons
 	enc := json.NewEncoder(w)
 	stats := globalReplicationStats.getLatestReplicationStats(bucket)
 	bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket)
-	bwMap := bwRpt.BucketStats[bucket]
+	bwMap := bwRpt.BucketStats
 	for arn, st := range stats.ReplicationStats.Stats {
-		if bwMap != nil {
-			if bw, ok := bwMap[arn]; ok {
+		for opts, bw := range bwMap {
+			if opts.ReplicationARN != "" && opts.ReplicationARN == arn {
 				st.BandWidthLimitInBytesPerSecond = bw.LimitInBytesPerSecond
 				st.CurrentBandwidthInBytesPerSecond = bw.CurrentBandwidthInBytesPerSecond
 				stats.ReplicationStats.Stats[arn] = st
@@ -476,27 +476,17 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	var tgtStats map[string]TargetReplicationResyncStatus
-	globalReplicationPool.resyncer.RLock()
-	brs, ok := globalReplicationPool.resyncer.statusMap[bucket]
-	if ok {
-		tgtStats = brs.cloneTgtStats()
-	}
-	globalReplicationPool.resyncer.RUnlock()
-	if !ok {
-		brs, err = loadBucketResyncMetadata(ctx, bucket, objectAPI)
-		if err != nil {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
-				Bucket: bucket,
-				Err:    fmt.Errorf("No replication resync status available for %s", arn),
-			}), r.URL)
-			return
-		}
-		tgtStats = brs.cloneTgtStats()
+	brs, err := loadBucketResyncMetadata(ctx, bucket, objectAPI)
+	if err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+			Bucket: bucket,
+			Err:    fmt.Errorf("replication resync status not available for %s (%s)", arn, err.Error()),
+		}), r.URL)
+		return
 	}
 
 	var rinfo ResyncTargetsInfo
-	for tarn, st := range tgtStats {
+	for tarn, st := range brs.TargetsMap {
 		if arn != "" && tarn != arn {
 			continue
 		}

cmd/bucket-replication-utils.go

@@ -31,8 +31,8 @@ import (
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 )
 
 //go:generate msgp -file=$GOFILE
@@ -167,7 +167,21 @@ func (ri replicatedInfos) Action() replicationAction {
 var replStatusRegex = regexp.MustCompile(`([^=].*?)=([^,].*?);`)
 
 // TargetReplicationStatus - returns replication status of a target
-func (o *ObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) {
+func (ri ReplicateObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) {
+	repStatMatches := replStatusRegex.FindAllStringSubmatch(ri.ReplicationStatusInternal, -1)
+	for _, repStatMatch := range repStatMatches {
+		if len(repStatMatch) != 3 {
+			return
+		}
+		if repStatMatch[1] == arn {
+			return replication.StatusType(repStatMatch[2])
+		}
+	}
+	return
+}
+
+// TargetReplicationStatus - returns replication status of a target
+func (o ObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) {
 	repStatMatches := replStatusRegex.FindAllStringSubmatch(o.ReplicationStatusInternal, -1)
 	for _, repStatMatch := range repStatMatches {
 		if len(repStatMatch) != 3 {
@@ -185,7 +199,6 @@ type replicateTargetDecision struct {
 	Synchronous bool   // Synchronous replication configured.
 	Arn         string // ARN of replication target
 	ID          string
-	Tgt         *TargetClient
 }
 
 func (t *replicateTargetDecision) String() string {
@@ -207,7 +220,7 @@ type ReplicateDecision struct {
 }
 
 // ReplicateAny returns true if atleast one target qualifies for replication
-func (d *ReplicateDecision) ReplicateAny() bool {
+func (d ReplicateDecision) ReplicateAny() bool {
 	for _, t := range d.targetsMap {
 		if t.Replicate {
 			return true
@@ -217,7 +230,7 @@ func (d *ReplicateDecision) ReplicateAny() bool {
 }
 
 // Synchronous returns true if atleast one target qualifies for synchronous replication
-func (d *ReplicateDecision) Synchronous() bool {
+func (d ReplicateDecision) Synchronous() bool {
 	for _, t := range d.targetsMap {
 		if t.Synchronous {
 			return true
@@ -226,7 +239,7 @@ func (d *ReplicateDecision) Synchronous() bool {
 	return false
 }
 
-func (d *ReplicateDecision) String() string {
+func (d ReplicateDecision) String() string {
 	b := new(bytes.Buffer)
 	for key, value := range d.targetsMap {
 		fmt.Fprintf(b, "%s=%s,", key, value.String())
@@ -243,7 +256,7 @@ func (d *ReplicateDecision) Set(t replicateTargetDecision) {
 }
 
 // PendingStatus returns a stringified representation of internal replication status with all targets marked as `PENDING`
-func (d *ReplicateDecision) PendingStatus() string {
+func (d ReplicateDecision) PendingStatus() string {
 	b := new(bytes.Buffer)
 	for _, k := range d.targetsMap {
 		if k.Replicate {
@@ -259,11 +272,11 @@ type ResyncDecision struct {
 }
 
 // Empty returns true if no targets with resync decision present
-func (r *ResyncDecision) Empty() bool {
+func (r ResyncDecision) Empty() bool {
 	return r.targets == nil
 }
 
-func (r *ResyncDecision) mustResync() bool {
+func (r ResyncDecision) mustResync() bool {
 	for _, v := range r.targets {
 		if v.Replicate {
 			return true
@@ -272,15 +285,12 @@ func (r *ResyncDecision) mustResync() bool {
 	return false
 }
 
-func (r *ResyncDecision) mustResyncTarget(tgtArn string) bool {
+func (r ResyncDecision) mustResyncTarget(tgtArn string) bool {
 	if r.targets == nil {
 		return false
 	}
 	v, ok := r.targets[tgtArn]
-	if ok && v.Replicate {
-		return true
-	}
-	return false
+	return ok && v.Replicate
 }
 
 // ResyncTargetDecision is struct that represents resync decision for this target
@@ -301,35 +311,20 @@ func parseReplicateDecision(ctx context.Context, bucket, s string) (r ReplicateD
 	if len(s) == 0 {
 		return
 	}
-	pairs := strings.Split(s, ",")
-	for _, p := range pairs {
+	for _, p := range strings.Split(s, ",") {
+		if p == "" {
+			continue
+		}
 		slc := strings.Split(p, "=")
 		if len(slc) != 2 {
 			return r, errInvalidReplicateDecisionFormat
 		}
-		tgtStr := strings.TrimPrefix(slc[1], "\"")
-		tgtStr = strings.TrimSuffix(tgtStr, "\"")
+		tgtStr := strings.TrimSuffix(strings.TrimPrefix(slc[1], `"`), `"`)
 		tgt := strings.Split(tgtStr, ";")
 		if len(tgt) != 4 {
 			return r, errInvalidReplicateDecisionFormat
 		}
-		var replicate, sync bool
-		var err error
-		replicate, err = strconv.ParseBool(tgt[0])
-		if err != nil {
-			return r, err
-		}
-		sync, err = strconv.ParseBool(tgt[1])
-		if err != nil {
-			return r, err
-		}
-		tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(slc[0])
-		if tgtClnt == nil {
-			// Skip stale targets if any and log them to be missing atleast once.
-			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, slc[0]), slc[0])
-			// We save the targetDecision even when its not configured or stale.
-		}
-		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: replicate, Synchronous: sync, Arn: tgt[2], ID: tgt[3], Tgt: tgtClnt}
+		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: tgt[0] == "true", Synchronous: tgt[1] == "true", Arn: tgt[2], ID: tgt[3]}
 	}
 	return
 }
@@ -496,8 +491,8 @@ func getCompositeVersionPurgeStatus(m map[string]VersionPurgeStatusType) Version
 }
 
 // getHealReplicateObjectInfo returns info needed by heal replication in ReplicateObjectInfo
-func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) ReplicateObjectInfo {
-	oi := objInfo.Clone()
+func getHealReplicateObjectInfo(oi ObjectInfo, rcfg replicationConfig) ReplicateObjectInfo {
+	userDefined := cloneMSS(oi.UserDefined)
 	if rcfg.Config != nil && rcfg.Config.RoleArn != "" {
 		// For backward compatibility of objects pending/failed replication.
 		// Save replication related statuses in the new internal representation for
@@ -508,17 +503,15 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 		if !oi.VersionPurgeStatus.Empty() {
 			oi.VersionPurgeStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.VersionPurgeStatus)
 		}
-		for k, v := range oi.UserDefined {
+		for k, v := range userDefined {
 			if strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset) {
-				delete(oi.UserDefined, k)
-				oi.UserDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
+				delete(userDefined, k)
+				userDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
 			}
 		}
 	}
-	var dsc ReplicateDecision
-	var tgtStatuses map[string]replication.StatusType
-	var purgeStatuses map[string]VersionPurgeStatusType
 
+	var dsc ReplicateDecision
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{
 			ObjectV: ObjectV{
@@ -530,33 +523,45 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
 		}, nil)
 	} else {
-		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{
-			UserDefined: oi.UserDefined,
-		}, replication.HealReplicationType, ObjectOptions{}))
+		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(userDefined, oi.UserTags, "", replication.HealReplicationType, ObjectOptions{}))
 	}
-	tgtStatuses = replicationStatusesMap(oi.ReplicationStatusInternal)
-	purgeStatuses = versionPurgeStatusesMap(oi.VersionPurgeStatusInternal)
-	existingObjResync := rcfg.Resync(GlobalContext, oi, &dsc, tgtStatuses)
-	tm, _ := time.Parse(time.RFC3339Nano, oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp])
+
+	tgtStatuses := replicationStatusesMap(oi.ReplicationStatusInternal)
+	purgeStatuses := versionPurgeStatusesMap(oi.VersionPurgeStatusInternal)
+	existingObjResync := rcfg.Resync(GlobalContext, oi, dsc, tgtStatuses)
+	tm, _ := time.Parse(time.RFC3339Nano, userDefined[ReservedMetadataPrefixLower+ReplicationTimestamp])
+	rstate := oi.ReplicationState()
+	rstate.ReplicateDecisionStr = dsc.String()
+	asz, _ := oi.GetActualSize()
+
 	return ReplicateObjectInfo{
-		ObjectInfo:           oi,
+		Name:                       oi.Name,
+		Size:                       oi.Size,
+		ActualSize:                 asz,
+		Bucket:                     oi.Bucket,
+		VersionID:                  oi.VersionID,
+		ETag:                       oi.ETag,
+		ModTime:                    oi.ModTime,
+		ReplicationStatus:          oi.ReplicationStatus,
+		ReplicationStatusInternal:  oi.ReplicationStatusInternal,
+		DeleteMarker:               oi.DeleteMarker,
+		VersionPurgeStatusInternal: oi.VersionPurgeStatusInternal,
+		VersionPurgeStatus:         oi.VersionPurgeStatus,
+
+		ReplicationState:     rstate,
 		OpType:               replication.HealReplicationType,
 		Dsc:                  dsc,
 		ExistingObjResync:    existingObjResync,
 		TargetStatuses:       tgtStatuses,
 		TargetPurgeStatuses:  purgeStatuses,
 		ReplicationTimestamp: tm,
+		SSEC:                 crypto.SSEC.IsEncrypted(oi.UserDefined),
+		UserTags:             oi.UserTags,
 	}
 }
 
-func (ri *ReplicateObjectInfo) getReplicationState() ReplicationState {
-	rs := ri.ObjectInfo.getReplicationState()
-	rs.ReplicateDecisionStr = ri.Dsc.String()
-	return rs
-}
-
-// vID here represents the versionID client specified in request - need to distinguish between delete marker and delete marker deletion
-func (o *ObjectInfo) getReplicationState() ReplicationState {
+// ReplicationState - returns replication state using other internal replication metadata in ObjectInfo
+func (o ObjectInfo) ReplicationState() ReplicationState {
 	rs := ReplicationState{
 		ReplicationStatusInternal:  o.ReplicationStatusInternal,
 		VersionPurgeStatusInternal: o.VersionPurgeStatusInternal,
@@ -577,7 +582,7 @@ func (o *ObjectInfo) getReplicationState() ReplicationState {
 }
 
 // ReplicationState returns replication state using other internal replication metadata in ObjectToDelete
-func (o *ObjectToDelete) ReplicationState() ReplicationState {
+func (o ObjectToDelete) ReplicationState() ReplicationState {
 	r := ReplicationState{
 		ReplicationStatusInternal:  o.DeleteMarkerReplicationStatus,
 		VersionPurgeStatusInternal: o.VersionPurgeStatuses,

cmd/bucket-replication.go

@@ -49,6 +49,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/tinylib/msgp/msgp"
 	"github.com/zeebo/xxh3"
 )
 
@@ -223,21 +224,19 @@ func (o mustReplicateOptions) isMetadataReplication() bool {
 	return o.opType == replication.MetadataReplicationType
 }
 
-func getMustReplicateOptions(o ObjectInfo, op replication.Type, opts ObjectOptions) mustReplicateOptions {
-	if !op.Valid() {
-		op = replication.ObjectReplicationType
-		if o.metadataOnly {
-			op = replication.MetadataReplicationType
-		}
-	}
-	meta := cloneMSS(o.UserDefined)
-	if o.UserTags != "" {
-		meta[xhttp.AmzObjectTagging] = o.UserTags
+func (o ObjectInfo) getMustReplicateOptions(op replication.Type, opts ObjectOptions) mustReplicateOptions {
+	return getMustReplicateOptions(o.UserDefined, o.UserTags, o.ReplicationStatus, op, opts)
+}
+
+func getMustReplicateOptions(userDefined map[string]string, userTags string, status replication.StatusType, op replication.Type, opts ObjectOptions) mustReplicateOptions {
+	meta := cloneMSS(userDefined)
+	if userTags != "" {
+		meta[xhttp.AmzObjectTagging] = userTags
 	}
 
 	return mustReplicateOptions{
 		meta:               meta,
-		status:             o.ReplicationStatus,
+		status:             status,
 		opType:             op,
 		replicationRequest: opts.ReplicationRequest,
 	}
@@ -355,40 +354,43 @@ func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelet
 		OpType:       replication.DeleteReplicationType,
 	}
 	tgtArns := rcfg.FilterTargetArns(opts)
-	if len(tgtArns) > 0 {
-		dsc.targetsMap = make(map[string]replicateTargetDecision, len(tgtArns))
-		var sync, replicate bool
-		for _, tgtArn := range tgtArns {
-			opts.TargetArn = tgtArn
-			replicate = rcfg.Replicate(opts)
-			// when incoming delete is removal of a delete marker(a.k.a versioned delete),
-			// GetObjectInfo returns extra information even though it returns errFileNotFound
-			if gerr != nil {
-				validReplStatus := false
-				switch oi.TargetReplicationStatus(tgtArn) {
-				case replication.Pending, replication.Completed, replication.Failed:
-					validReplStatus = true
-				}
-				if oi.DeleteMarker && (validReplStatus || replicate) {
-					dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
-					continue
-				} else {
-					// can be the case that other cluster is down and duplicate `mc rm --vid`
-					// is issued - this still needs to be replicated back to the other target
+	dsc.targetsMap = make(map[string]replicateTargetDecision, len(tgtArns))
+	if len(tgtArns) == 0 {
+		return dsc
+	}
+	var sync, replicate bool
+	for _, tgtArn := range tgtArns {
+		opts.TargetArn = tgtArn
+		replicate = rcfg.Replicate(opts)
+		// when incoming delete is removal of a delete marker(a.k.a versioned delete),
+		// GetObjectInfo returns extra information even though it returns errFileNotFound
+		if gerr != nil {
+			validReplStatus := false
+			switch oi.TargetReplicationStatus(tgtArn) {
+			case replication.Pending, replication.Completed, replication.Failed:
+				validReplStatus = true
+			}
+			if oi.DeleteMarker && (validReplStatus || replicate) {
+				dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
+				continue
+			} else {
+				// can be the case that other cluster is down and duplicate `mc rm --vid`
+				// is issued - this still needs to be replicated back to the other target
+				if !oi.VersionPurgeStatus.Empty() {
 					replicate = oi.VersionPurgeStatus == Pending || oi.VersionPurgeStatus == Failed
 					dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync))
-					continue
 				}
+				continue
 			}
-			tgt := globalBucketTargetSys.GetRemoteTargetClient(tgtArn)
-			// the target online status should not be used here while deciding
-			// whether to replicate deletes as the target could be temporarily down
-			tgtDsc := newReplicateTargetDecision(tgtArn, false, false)
-			if tgt != nil {
-				tgtDsc = newReplicateTargetDecision(tgtArn, replicate, tgt.replicateSync)
-			}
-			dsc.Set(tgtDsc)
 		}
+		tgt := globalBucketTargetSys.GetRemoteTargetClient(tgtArn)
+		// the target online status should not be used here while deciding
+		// whether to replicate deletes as the target could be temporarily down
+		tgtDsc := newReplicateTargetDecision(tgtArn, false, false)
+		if tgt != nil {
+			tgtDsc = newReplicateTargetDecision(tgtArn, replicate, tgt.replicateSync)
+		}
+		dsc.Set(tgtDsc)
 	}
 	return dsc
 }
@@ -482,15 +484,10 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 	ctx = lkctx.Context()
 	defer lk.Unlock(lkctx)
 
+	rinfos := replicatedInfos{Targets: make([]replicatedTargetInfo, 0, len(dsc.targetsMap))}
 	var wg sync.WaitGroup
-	var rinfos replicatedInfos
-	rinfos.Targets = make([]replicatedTargetInfo, len(dsc.targetsMap))
-	idx := -1
+	var mu sync.Mutex
 	for _, tgtEntry := range dsc.targetsMap {
-		idx++
-		if tgtEntry.Tgt == nil {
-			continue
-		}
 		if !tgtEntry.Replicate {
 			continue
 		}
@@ -498,11 +495,33 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		if dobj.TargetArn != "" && dobj.TargetArn != tgtEntry.Arn {
 			continue
 		}
+		tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(tgtEntry.Arn)
+		if tgtClnt == nil {
+			// Skip stale targets if any and log them to be missing atleast once.
+			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtEntry.Arn), tgtEntry.Arn)
+			sendEvent(eventArgs{
+				EventName:  event.ObjectReplicationNotTracked,
+				BucketName: bucket,
+				Object: ObjectInfo{
+					Bucket:       bucket,
+					Name:         dobj.ObjectName,
+					VersionID:    versionID,
+					DeleteMarker: dobj.DeleteMarker,
+				},
+				UserAgent: "Internal: [Replication]",
+				Host:      globalLocalNodeName,
+			})
+			continue
+		}
 		wg.Add(1)
-		go func(index int, tgt *TargetClient) {
+		go func(tgt *TargetClient) {
 			defer wg.Done()
-			rinfos.Targets[index] = replicateDeleteToTarget(ctx, dobj, tgt)
-		}(idx, tgtEntry.Tgt)
+			tgtInfo := replicateDeleteToTarget(ctx, dobj, tgt)
+
+			mu.Lock()
+			rinfos.Targets = append(rinfos.Targets, tgtInfo)
+			mu.Unlock()
+		}(tgtClnt)
 	}
 	wg.Wait()
 
@@ -702,10 +721,8 @@ func getCopyObjMetadata(oi ObjectInfo, sc string) map[string]string {
 		meta[xhttp.ContentType] = oi.ContentType
 	}
 
-	if oi.UserTags != "" {
-		meta[xhttp.AmzObjectTagging] = oi.UserTags
-		meta[xhttp.AmzTagDirective] = "REPLACE"
-	}
+	meta[xhttp.AmzObjectTagging] = oi.UserTags
+	meta[xhttp.AmzTagDirective] = "REPLACE"
 
 	if sc == "" {
 		sc = oi.StorageClass
@@ -887,7 +904,7 @@ func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replicati
 	}
 
 	t, _ := tags.ParseObjectTags(oi1.UserTags)
-	if !reflect.DeepEqual(oi2.UserTags, t.ToMap()) || (oi2.UserTagCount != len(t.ToMap())) {
+	if (oi2.UserTagCount > 0 && !reflect.DeepEqual(oi2.UserTags, t.ToMap())) || (oi2.UserTagCount != len(t.ToMap())) {
 		return replicateMetadata
 	}
 
@@ -964,9 +981,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		})
 	}()
 
-	objInfo := ri.ObjectInfo
-	bucket := objInfo.Bucket
-	object := objInfo.Name
+	bucket := ri.Bucket
+	object := ri.Name
 
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
@@ -974,7 +990,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
-			Object:     objInfo,
+			Object:     ri.ToObjectInfo(),
 			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
@@ -982,8 +998,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	}
 	tgtArns := cfg.FilterTargetArns(replication.ObjectOpts{
 		Name:     object,
-		SSEC:     crypto.SSEC.IsEncrypted(objInfo.UserDefined),
-		UserTags: objInfo.UserTags,
+		SSEC:     ri.SSEC,
+		UserTags: ri.UserTags,
 	})
 	// Lock the object name before starting replication.
 	// Use separate lock that doesn't collide with regular objects.
@@ -993,7 +1009,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
-			Object:     objInfo,
+			Object:     ri.ToObjectInfo(),
 			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
@@ -1003,32 +1019,38 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	ctx = lkctx.Context()
 	defer lk.Unlock(lkctx)
 
+	rinfos := replicatedInfos{Targets: make([]replicatedTargetInfo, 0, len(tgtArns))}
 	var wg sync.WaitGroup
-	var rinfos replicatedInfos
-	rinfos.Targets = make([]replicatedTargetInfo, len(tgtArns))
-	for i, tgtArn := range tgtArns {
+	var mu sync.Mutex
+	for _, tgtArn := range tgtArns {
 		tgt := globalBucketTargetSys.GetRemoteTargetClient(tgtArn)
 		if tgt == nil {
 			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtArn), tgtArn)
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
-				Object:     objInfo,
+				Object:     ri.ToObjectInfo(),
 				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
 			continue
 		}
 		wg.Add(1)
-		go func(index int, tgt *TargetClient) {
+		go func(tgt *TargetClient) {
 			defer wg.Done()
+
+			var tgtInfo replicatedTargetInfo
 			if ri.OpType == replication.ObjectReplicationType {
 				// all incoming calls go through optimized path.
-				rinfos.Targets[index] = ri.replicateObject(ctx, objectAPI, tgt)
+				tgtInfo = ri.replicateObject(ctx, objectAPI, tgt)
 			} else {
-				rinfos.Targets[index] = ri.replicateAll(ctx, objectAPI, tgt)
+				tgtInfo = ri.replicateAll(ctx, objectAPI, tgt)
 			}
-		}(i, tgt)
+
+			mu.Lock()
+			rinfos.Targets = append(rinfos.Targets, tgtInfo)
+			mu.Unlock()
+		}(tgt)
 	}
 	wg.Wait()
 
@@ -1043,10 +1065,11 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	newReplStatusInternal := rinfos.ReplicationStatusInternal()
 	// Note that internal replication status(es) may match for previously replicated objects - in such cases
 	// metadata should be updated with last resync timestamp.
-	if objInfo.ReplicationStatusInternal != newReplStatusInternal || rinfos.ReplicationResynced() {
+	objInfo := ri.ToObjectInfo()
+	if ri.ReplicationStatusInternal != newReplStatusInternal || rinfos.ReplicationResynced() {
 		popts := ObjectOptions{
-			MTime:     objInfo.ModTime,
-			VersionID: objInfo.VersionID,
+			MTime:     ri.ModTime,
+			VersionID: ri.VersionID,
 			EvalMetadataFn: func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) {
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = newReplStatusInternal
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
@@ -1056,14 +1079,18 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 						oi.UserDefined[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp
 					}
 				}
-				if objInfo.UserTags != "" {
-					oi.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
+				if ri.UserTags != "" {
+					oi.UserDefined[xhttp.AmzObjectTagging] = ri.UserTags
 				}
 				return dsc, nil
 			},
 		}
 
-		_, _ = objectAPI.PutObjectMetadata(ctx, bucket, object, popts)
+		uobjInfo, _ := objectAPI.PutObjectMetadata(ctx, bucket, object, popts)
+		if uobjInfo.Name != "" {
+			objInfo = uobjInfo
+		}
+
 		opType := replication.MetadataReplicationType
 		if rinfos.Action() == replicateAll {
 			opType = replication.ObjectReplicationType
@@ -1099,23 +1126,21 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 // The source object is then updated to reflect the replication status.
 func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI ObjectLayer, tgt *TargetClient) (rinfo replicatedTargetInfo) {
 	startTime := time.Now()
-	objInfo := ri.ObjectInfo.Clone()
-	bucket := objInfo.Bucket
-	object := objInfo.Name
-	sz, _ := objInfo.GetActualSize()
+	bucket := ri.Bucket
+	object := ri.Name
 
 	rAction := replicateAll
 	rinfo = replicatedTargetInfo{
-		Size:                  sz,
+		Size:                  ri.ActualSize,
 		Arn:                   tgt.ARN,
-		PrevReplicationStatus: objInfo.TargetReplicationStatus(tgt.ARN),
+		PrevReplicationStatus: ri.TargetReplicationStatus(tgt.ARN),
 		ReplicationStatus:     replication.Failed,
 		OpType:                ri.OpType,
 		ReplicationAction:     rAction,
 		endpoint:              tgt.EndpointURL().Host,
 		secure:                tgt.EndpointURL().Scheme == "https",
 	}
-	if ri.ObjectInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
+	if ri.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
 		rinfo.ReplicationStatus = replication.Completed
 		rinfo.ReplicationResynced = true
 		return
@@ -1126,7 +1151,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
-			Object:     objInfo,
+			Object:     ri.ToObjectInfo(),
 			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
@@ -1137,12 +1162,13 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
 
 	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
-		VersionID:        objInfo.VersionID,
+		VersionID:        ri.VersionID,
 		Versioned:        versioned,
 		VersionSuspended: versionSuspended,
 	})
 	if err != nil {
 		if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+			objInfo := ri.ToObjectInfo()
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
@@ -1156,7 +1182,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 	defer gr.Close()
 
-	objInfo = gr.ObjInfo
+	objInfo := gr.ObjInfo
+
 	// make sure we have the latest metadata for metrics calculation
 	rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN)
 
@@ -1217,8 +1244,10 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 
 	opts := &bandwidth.MonitorReaderOptions{
-		Bucket:     objInfo.Bucket,
-		TargetARN:  tgt.ARN,
+		BucketOptions: bandwidth.BucketOptions{
+			Name:           ri.Bucket,
+			ReplicationARN: tgt.ARN,
+		},
 		HeaderSize: headerSize,
 	}
 	newCtx := ctx
@@ -1255,10 +1284,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 // The source object is then updated to reflect the replication status.
 func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI ObjectLayer, tgt *TargetClient) (rinfo replicatedTargetInfo) {
 	startTime := time.Now()
-	objInfo := ri.ObjectInfo.Clone()
-	bucket := objInfo.Bucket
-	object := objInfo.Name
-	sz, _ := objInfo.GetActualSize()
+	bucket := ri.Bucket
+	object := ri.Name
 
 	// set defaults for replication action based on operation being performed - actual
 	// replication action can only be determined after stat on remote. This default is
@@ -1266,9 +1293,9 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	rAction := replicateMetadata
 
 	rinfo = replicatedTargetInfo{
-		Size:                  sz,
+		Size:                  ri.ActualSize,
 		Arn:                   tgt.ARN,
-		PrevReplicationStatus: objInfo.TargetReplicationStatus(tgt.ARN),
+		PrevReplicationStatus: ri.TargetReplicationStatus(tgt.ARN),
 		ReplicationStatus:     replication.Failed,
 		OpType:                ri.OpType,
 		ReplicationAction:     rAction,
@@ -1281,7 +1308,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
-			Object:     objInfo,
+			Object:     ri.ToObjectInfo(),
 			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
@@ -1292,12 +1319,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
 
 	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
-		VersionID:        objInfo.VersionID,
+		VersionID:        ri.VersionID,
 		Versioned:        versioned,
 		VersionSuspended: versionSuspended,
 	})
 	if err != nil {
 		if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
+			objInfo := ri.ToObjectInfo()
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
@@ -1311,7 +1339,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 	defer gr.Close()
 
-	objInfo = gr.ObjInfo
+	objInfo := gr.ObjInfo
 	// make sure we have the latest metadata for metrics calculation
 	rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN)
 
@@ -1378,7 +1406,9 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			}
 			// object with same VersionID already exists, replication kicked off by
 			// PutObject might have completed
-			if objInfo.TargetReplicationStatus(tgt.ARN) == replication.Pending || objInfo.TargetReplicationStatus(tgt.ARN) == replication.Failed || ri.OpType == replication.ExistingObjectReplicationType {
+			if objInfo.TargetReplicationStatus(tgt.ARN) == replication.Pending ||
+				objInfo.TargetReplicationStatus(tgt.ARN) == replication.Failed ||
+				ri.OpType == replication.ExistingObjectReplicationType {
 				// if metadata is not updated for some reason after replication, such as
 				// 503 encountered while updating metadata - make sure to set ReplicationStatus
 				// as Completed.
@@ -1456,8 +1486,10 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 		}
 
 		opts := &bandwidth.MonitorReaderOptions{
-			Bucket:     objInfo.Bucket,
-			TargetARN:  tgt.ARN,
+			BucketOptions: bandwidth.BucketOptions{
+				Name:           objInfo.Bucket,
+				ReplicationARN: tgt.ARN,
+			},
 			HeaderSize: headerSize,
 		}
 		newCtx := ctx
@@ -1498,9 +1530,18 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 	var uploadedParts []minio.CompletePart
 	// new multipart must not set mtime as it may lead to erroneous cleanups at various intervals.
 	opts.Internal.SourceMTime = time.Time{} // this value is saved properly in CompleteMultipartUpload()
-	nctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
-	defer cancel()
-	uploadID, err := c.NewMultipartUpload(nctx, bucket, object, opts)
+	var uploadID string
+	attempts := 1
+	for attempts <= 3 {
+		nctx, cancel := context.WithTimeout(ctx, time.Minute)
+		uploadID, err = c.NewMultipartUpload(nctx, bucket, object, opts)
+		cancel()
+		if err == nil {
+			break
+		}
+		attempts++
+		time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
+	}
 	if err != nil {
 		return err
 	}
@@ -1521,7 +1562,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 					fmt.Errorf("trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster",
 						humanize.Ordinal(attempts), uploadID, bucket, object, aerr))
 				attempts++
-				time.Sleep(time.Second)
+				time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
 			}
 		}
 	}()
@@ -1532,7 +1573,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 	)
 
 	for _, partInfo := range objInfo.Parts {
-		hr, err = hash.NewReader(io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
+		hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
 		if err != nil {
 			return err
 		}
@@ -1658,9 +1699,8 @@ type ReplicationPool struct {
 	resyncer *replicationResyncer
 
 	// workers:
-	workers         []chan ReplicationWorkerOperation
-	lrgworkers      []chan ReplicationWorkerOperation
-	existingWorkers chan ReplicationWorkerOperation
+	workers    []chan ReplicationWorkerOperation
+	lrgworkers []chan ReplicationWorkerOperation
 
 	// mrf:
 	mrfWorkerKillCh chan struct{}
@@ -1720,8 +1760,6 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 	pool := &ReplicationPool{
 		workers:         make([]chan ReplicationWorkerOperation, 0, workers),
 		lrgworkers:      make([]chan ReplicationWorkerOperation, 0, LargeWorkerCount),
-		existingWorkers: make(chan ReplicationWorkerOperation, 100000),
-
 		mrfReplicaCh:    make(chan ReplicationWorkerOperation, 100000),
 		mrfWorkerKillCh: make(chan struct{}, failedWorkers),
 		resyncer:        newresyncer(),
@@ -1735,7 +1773,6 @@ func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPool
 	pool.AddLargeWorkers()
 	pool.ResizeWorkers(workers, 0)
 	pool.ResizeFailedWorkers(failedWorkers)
-	go pool.AddWorker(pool.existingWorkers, nil)
 	go pool.resyncer.PersistToDisk(ctx, o)
 	go pool.processMRF()
 	go pool.persistMRF()
@@ -1962,9 +1999,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 
 	var ch, healCh chan<- ReplicationWorkerOperation
 	switch ri.OpType {
-	case replication.ExistingObjectReplicationType:
-		ch = p.existingWorkers
-	case replication.HealReplicationType:
+	case replication.HealReplicationType, replication.ExistingObjectReplicationType:
 		ch = p.mrfReplicaCh
 		healCh = p.getWorkerCh(ri.Name, ri.Bucket, ri.Size)
 	default:
@@ -2023,9 +2058,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 	}
 	var ch chan<- ReplicationWorkerOperation
 	switch doi.OpType {
-	case replication.ExistingObjectReplicationType:
-		ch = p.existingWorkers
-	case replication.HealReplicationType:
+	case replication.HealReplicationType, replication.ExistingObjectReplicationType:
 		fallthrough
 	default:
 		ch = p.getWorkerCh(doi.Bucket, doi.ObjectName, 0)
@@ -2229,8 +2262,38 @@ func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, rs
 	return oi, proxy
 }
 
-func scheduleReplication(ctx context.Context, objInfo ObjectInfo, o ObjectLayer, dsc ReplicateDecision, opType replication.Type) {
-	ri := ReplicateObjectInfo{ObjectInfo: objInfo, OpType: opType, Dsc: dsc, EventType: ReplicateIncoming}
+func scheduleReplication(ctx context.Context, oi ObjectInfo, o ObjectLayer, dsc ReplicateDecision, opType replication.Type) {
+	tgtStatuses := replicationStatusesMap(oi.ReplicationStatusInternal)
+	purgeStatuses := versionPurgeStatusesMap(oi.VersionPurgeStatusInternal)
+	tm, _ := time.Parse(time.RFC3339Nano, oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp])
+	rstate := oi.ReplicationState()
+	rstate.ReplicateDecisionStr = dsc.String()
+	asz, _ := oi.GetActualSize()
+
+	ri := ReplicateObjectInfo{
+		Name:                       oi.Name,
+		Size:                       oi.Size,
+		ActualSize:                 asz,
+		Bucket:                     oi.Bucket,
+		VersionID:                  oi.VersionID,
+		ETag:                       oi.ETag,
+		ModTime:                    oi.ModTime,
+		ReplicationStatus:          oi.ReplicationStatus,
+		ReplicationStatusInternal:  oi.ReplicationStatusInternal,
+		DeleteMarker:               oi.DeleteMarker,
+		VersionPurgeStatusInternal: oi.VersionPurgeStatusInternal,
+		VersionPurgeStatus:         oi.VersionPurgeStatus,
+
+		ReplicationState:     rstate,
+		OpType:               opType,
+		Dsc:                  dsc,
+		TargetStatuses:       tgtStatuses,
+		TargetPurgeStatuses:  purgeStatuses,
+		ReplicationTimestamp: tm,
+		SSEC:                 crypto.SSEC.IsEncrypted(oi.UserDefined),
+		UserTags:             oi.UserTags,
+	}
+
 	if dsc.Synchronous() {
 		replicateObject(ctx, ri, o)
 	} else {
@@ -2259,7 +2322,7 @@ func (c replicationConfig) Replicate(opts replication.ObjectOpts) bool {
 }
 
 // Resync returns true if replication reset is requested
-func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo, dsc *ReplicateDecision, tgtStatuses map[string]replication.StatusType) (r ResyncDecision) {
+func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo, dsc ReplicateDecision, tgtStatuses map[string]replication.StatusType) (r ResyncDecision) {
 	if c.Empty() {
 		return
 	}
@@ -2268,8 +2331,6 @@ func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo, dsc *Repli
 	if oi.DeleteMarker {
 		opts := replication.ObjectOpts{
 			Name:           oi.Name,
-			SSEC:           crypto.SSEC.IsEncrypted(oi.UserDefined),
-			UserTags:       oi.UserTags,
 			DeleteMarker:   oi.DeleteMarker,
 			VersionID:      oi.VersionID,
 			OpType:         replication.DeleteReplicationType,
@@ -2290,23 +2351,19 @@ func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo, dsc *Repli
 	}
 
 	// Ignore previous replication status when deciding if object can be re-replicated
-	objInfo := oi.Clone()
-	objInfo.ReplicationStatusInternal = ""
-	objInfo.VersionPurgeStatusInternal = ""
-	objInfo.ReplicationStatus = ""
-	objInfo.VersionPurgeStatus = ""
-	delete(objInfo.UserDefined, xhttp.AmzBucketReplicationStatus)
-	resyncdsc := mustReplicate(ctx, oi.Bucket, oi.Name, getMustReplicateOptions(objInfo, replication.ExistingObjectReplicationType, ObjectOptions{}))
-	dsc = &resyncdsc
-	return c.resync(oi, dsc, tgtStatuses)
+	userDefined := cloneMSS(oi.UserDefined)
+	delete(userDefined, xhttp.AmzBucketReplicationStatus)
+
+	rdsc := mustReplicate(ctx, oi.Bucket, oi.Name, getMustReplicateOptions(userDefined, oi.UserTags, "", replication.ExistingObjectReplicationType, ObjectOptions{}))
+	return c.resync(oi, rdsc, tgtStatuses)
 }
 
 // wrapper function for testability. Returns true if a new reset is requested on
 // already replicated objects OR object qualifies for existing object replication
 // and no reset requested.
-func (c replicationConfig) resync(oi ObjectInfo, dsc *ReplicateDecision, tgtStatuses map[string]replication.StatusType) (r ResyncDecision) {
+func (c replicationConfig) resync(oi ObjectInfo, dsc ReplicateDecision, tgtStatuses map[string]replication.StatusType) (r ResyncDecision) {
 	r = ResyncDecision{
-		targets: make(map[string]ResyncTargetDecision),
+		targets: make(map[string]ResyncTargetDecision, len(dsc.targetsMap)),
 	}
 	if c.remotes == nil {
 		return
@@ -2527,6 +2584,13 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	workers := make([]chan ReplicateObjectInfo, resyncParallelRoutines)
 	resultCh := make(chan TargetReplicationResyncStatus, 1)
 	defer close(resultCh)
+	go func() {
+		for r := range resultCh {
+			s.incStats(r, opts)
+			globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
+		}
+	}()
+
 	var wg sync.WaitGroup
 	for i := 0; i < resyncParallelRoutines; i++ {
 		wg.Add(1)
@@ -2556,7 +2620,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 							ObjectName:            roi.Name,
 							DeleteMarkerVersionID: dmVersionID,
 							VersionID:             versionID,
-							ReplicationState:      roi.getReplicationState(),
+							ReplicationState:      roi.ReplicationState,
 							DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 							DeleteMarker:          roi.DeleteMarker,
 						},
@@ -2631,12 +2695,6 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	for i := 0; i < resyncParallelRoutines; i++ {
 		close(workers[i])
 	}
-	go func() {
-		for r := range resultCh {
-			s.incStats(r, opts)
-			globalSiteResyncMetrics.updateMetric(r, opts.resyncID)
-		}
-	}()
 	wg.Wait()
 	resyncStatus = ResyncCompleted
 }
@@ -3008,7 +3066,7 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 				ObjectName:            roi.Name,
 				DeleteMarkerVersionID: dmVersionID,
 				VersionID:             versionID,
-				ReplicationState:      roi.getReplicationState(),
+				ReplicationState:      roi.ReplicationState,
 				DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
 				DeleteMarker:          roi.DeleteMarker,
 			},
@@ -3048,7 +3106,7 @@ func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcf
 
 const (
 	mrfSaveInterval  = 5 * time.Minute
-	mrfQueueInterval = 6 * time.Minute
+	mrfQueueInterval = mrfSaveInterval + time.Minute // A minute higher than save interval
 
 	mrfRetryLimit = 3 // max number of retries before letting scanner catch up on this object version
 	mrfMaxEntries = 1000000
@@ -3063,46 +3121,37 @@ func (p *ReplicationPool) persistMRF() {
 	mTimer := time.NewTimer(mrfSaveInterval)
 	defer mTimer.Stop()
 
-	saveMRFToDisk := func(drain bool) {
+	saveMRFToDisk := func() {
 		if len(entries) == 0 {
 			return
 		}
-		cctx := p.ctx
-		if drain {
-			cctx = context.Background()
-			// drain all mrf entries and save to disk
-			for e := range p.mrfSaveCh {
-				entries[e.versionID] = e
-			}
-		}
 
 		// queue all entries for healing before overwriting the node mrf file
 		if !contextCanceled(p.ctx) {
 			p.queueMRFHeal()
 		}
 
-		if err := p.saveMRFEntries(cctx, entries); err != nil {
-			logger.LogOnceIf(p.ctx, fmt.Errorf("unable to persist replication failures to disk:%w", err), string(replicationSubsystem))
-		}
+		p.saveMRFEntries(p.ctx, entries)
 
 		entries = make(map[string]MRFReplicateEntry)
 	}
 	for {
 		select {
 		case <-mTimer.C:
-			saveMRFToDisk(false)
+			saveMRFToDisk()
 			mTimer.Reset(mrfSaveInterval)
 		case <-p.ctx.Done():
 			p.mrfStopCh <- struct{}{}
 			close(p.mrfSaveCh)
-			saveMRFToDisk(true)
+			// We try to save if possible, but we don't care beyond that.
+			saveMRFToDisk()
 			return
 		case e, ok := <-p.mrfSaveCh:
 			if !ok {
 				return
 			}
 			if len(entries) >= mrfMaxEntries {
-				saveMRFToDisk(true)
+				saveMRFToDisk()
 			}
 			entries[e.versionID] = e
 		}
@@ -3132,14 +3181,45 @@ func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) {
 	}
 }
 
+func (p *ReplicationPool) persistToDrive(ctx context.Context, v MRFReplicateEntries, data []byte) {
+	newReader := func() io.ReadCloser {
+		r, w := io.Pipe()
+		go func() {
+			mw := msgp.NewWriter(w)
+			n, err := mw.Write(data)
+			if err != nil {
+				w.CloseWithError(err)
+				return
+			}
+			if n != len(data) {
+				w.CloseWithError(io.ErrShortWrite)
+				return
+			}
+			err = v.EncodeMsg(mw)
+			mw.Flush()
+			w.CloseWithError(err)
+		}()
+		return r
+	}
+
+	for _, localDrive := range globalLocalDrives {
+		r := newReader()
+		err := localDrive.CreateFile(ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), -1, r)
+		r.Close()
+		if err == nil {
+			break
+		}
+	}
+}
+
 // save mrf entries to nodenamehex.bin
-func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string]MRFReplicateEntry) error {
+func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string]MRFReplicateEntry) {
 	if !p.initialized() {
-		return nil
+		return
 	}
 	atomic.StoreUint64(&globalReplicationStats.mrfStats.LastFailedCount, uint64(len(entries)))
 	if len(entries) == 0 {
-		return nil
+		return
 	}
 
 	v := MRFReplicateEntries{
@@ -3152,53 +3232,60 @@ func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string
 	binary.LittleEndian.PutUint16(data[0:2], mrfMetaFormat)
 	binary.LittleEndian.PutUint16(data[2:4], mrfMetaVersion)
 
-	buf, err := v.MarshalMsg(data)
-	if err != nil {
-		return err
-	}
-
-	for _, localDrive := range globalLocalDrives {
-		if err := localDrive.WriteAll(ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), buf); err == nil {
-			break
-		}
-	}
-	return nil
+	p.persistToDrive(ctx, v, data)
 }
 
 // load mrf entries from disk
-func (p *ReplicationPool) loadMRF(data []byte) (re MRFReplicateEntries, e error) {
-	if !p.initialized() {
+func (p *ReplicationPool) loadMRF() (mrfRec MRFReplicateEntries, err error) {
+	loadMRF := func(rc io.ReadCloser) (re MRFReplicateEntries, err error) {
+		defer rc.Close()
+
+		if !p.initialized() {
+			return re, nil
+		}
+		data := make([]byte, 4)
+		n, err := rc.Read(data)
+		if err != nil {
+			return re, err
+		}
+		if n != len(data) {
+			return re, errors.New("replication mrf: no data")
+		}
+		// Read resync meta header
+		switch binary.LittleEndian.Uint16(data[0:2]) {
+		case mrfMetaFormat:
+		default:
+			return re, fmt.Errorf("replication mrf: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
+		}
+		switch binary.LittleEndian.Uint16(data[2:4]) {
+		case mrfMetaVersion:
+		default:
+			return re, fmt.Errorf("replication mrf: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
+		}
+
+		// OK, parse data.
+		// ignore any parsing errors, we do not care this file is generated again anyways.
+		re.DecodeMsg(msgp.NewReader(rc))
+
 		return re, nil
 	}
-	if len(data) == 0 {
-		// Seems to be empty.
-		return re, nil
-	}
-	if len(data) <= 4 {
-		return re, fmt.Errorf("replication mrf: no data")
-	}
-	// Read resync meta header
-	switch binary.LittleEndian.Uint16(data[0:2]) {
-	case mrfMetaFormat:
-	default:
-		return re, fmt.Errorf("replication mrf: unknown format: %d", binary.LittleEndian.Uint16(data[0:2]))
-	}
-	switch binary.LittleEndian.Uint16(data[2:4]) {
-	case mrfMetaVersion:
-	default:
-		return re, fmt.Errorf("replication mrf: unknown version: %d", binary.LittleEndian.Uint16(data[2:4]))
-	}
-	// OK, parse data.
-	if _, err := re.UnmarshalMsg(data[4:]); err != nil {
-		return re, err
+	for _, localDrive := range globalLocalDrives {
+		rc, err := localDrive.ReadFileStream(p.ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), 0, -1)
+		if err != nil {
+			continue
+		}
+
+		mrfRec, err = loadMRF(rc)
+		if err != nil {
+			continue
+		}
+
+		// finally delete the file after processing mrf entries
+		localDrive.Delete(p.ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), DeleteOptions{})
+		break
 	}
 
-	switch re.Version {
-	case mrfMetaVersionV1:
-	default:
-		return re, fmt.Errorf("unexpected mrf meta version: %d", re.Version)
-	}
-	return re, nil
+	return mrfRec, nil
 }
 
 func (p *ReplicationPool) processMRF() {
@@ -3241,39 +3328,28 @@ func (p *ReplicationPool) queueMRFHeal() error {
 		return errServerNotInitialized
 	}
 
-	for _, localDrive := range globalLocalDrives {
-		buf, err := localDrive.ReadAll(p.ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"))
-		if err != nil {
-			continue
-		}
-
-		mrfRec, err := p.loadMRF(buf)
-		if err != nil {
-			continue
-		}
-
-		// finally delete the file after processing mrf entries
-		localDrive.Delete(p.ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), DeleteOptions{})
-
-		// queue replication heal in a goroutine to avoid holding up mrf save routine
-		go func(mrfRec MRFReplicateEntries) {
-			for vID, e := range mrfRec.Entries {
-				ctx, cancel := context.WithTimeout(p.ctx, time.Second) // Do not waste more than a second on this.
-
-				oi, err := p.objLayer.GetObjectInfo(ctx, e.Bucket, e.Object, ObjectOptions{
-					VersionID: vID,
-				})
-				cancel()
-				if err != nil {
-					continue
-				}
-
-				QueueReplicationHeal(p.ctx, e.Bucket, oi, e.RetryCount)
-			}
-		}(mrfRec)
-		break
+	mrfRec, err := p.loadMRF()
+	if err != nil {
+		return err
 	}
 
+	// queue replication heal in a goroutine to avoid holding up mrf save routine
+	go func() {
+		for vID, e := range mrfRec.Entries {
+			ctx, cancel := context.WithTimeout(p.ctx, time.Second) // Do not waste more than a second on this.
+
+			oi, err := p.objLayer.GetObjectInfo(ctx, e.Bucket, e.Object, ObjectOptions{
+				VersionID: vID,
+			})
+			cancel()
+			if err != nil {
+				continue
+			}
+
+			QueueReplicationHeal(p.ctx, e.Bucket, oi, e.RetryCount)
+		}
+	}()
+
 	return nil
 }
 
@@ -3283,33 +3359,28 @@ func (p *ReplicationPool) initialized() bool {
 
 // getMRF returns MRF entries for this node.
 func (p *ReplicationPool) getMRF(ctx context.Context, bucket string) (ch chan madmin.ReplicationMRF, err error) {
+	mrfRec, err := p.loadMRF()
+	if err != nil {
+		return nil, err
+	}
+
 	mrfCh := make(chan madmin.ReplicationMRF, 100)
 	go func() {
 		defer close(mrfCh)
-		for _, localDrive := range globalLocalDrives {
-			buf, err := localDrive.ReadAll(ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"))
-			if err != nil {
+		for vID, e := range mrfRec.Entries {
+			if e.Bucket != bucket && bucket != "" {
 				continue
 			}
-			mrfRec, err := p.loadMRF(buf)
-			if err != nil {
-				continue
-			}
-			for vID, e := range mrfRec.Entries {
-				if e.Bucket != bucket && bucket != "" {
-					continue
-				}
-				select {
-				case mrfCh <- madmin.ReplicationMRF{
-					NodeName:   globalLocalNodeName,
-					Object:     e.Object,
-					VersionID:  vID,
-					Bucket:     e.Bucket,
-					RetryCount: e.RetryCount,
-				}:
-				case <-ctx.Done():
-					return
-				}
+			select {
+			case mrfCh <- madmin.ReplicationMRF{
+				NodeName:   globalLocalNodeName,
+				Object:     e.Object,
+				VersionID:  vID,
+				Bucket:     e.Bucket,
+				RetryCount: e.RetryCount,
+			}:
+			case <-ctx.Done():
+				return
 			}
 		}
 	}()

cmd/bucket-replication_test.go

@@ -88,7 +88,7 @@ var replicationConfigTests = []struct {
 func TestReplicationResync(t *testing.T) {
 	ctx := context.Background()
 	for i, test := range replicationConfigTests {
-		if sync := test.rcfg.Resync(ctx, test.info, &test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
+		if sync := test.rcfg.Resync(ctx, test.info, test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
 			t.Errorf("Test%d (%s): Resync  got %t , want %t", i+1, test.name, sync.mustResync(), test.expectedSync)
 		}
 	}
@@ -283,7 +283,7 @@ var (
 
 func TestReplicationResyncwrapper(t *testing.T) {
 	for i, test := range replicationConfigTests2 {
-		if sync := test.rcfg.resync(test.info, &test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
+		if sync := test.rcfg.resync(test.info, test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
 			t.Errorf("%s (%s): Replicationresync  got %t , want %t", fmt.Sprintf("Test%d - %s", i+1, time.Now().Format(http.TimeFormat)), test.name, sync.mustResync(), test.expectedSync)
 		}
 	}

cmd/bucket-targets.go

@@ -489,31 +489,27 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 	defer sys.Unlock()
 
 	// Remove existingtarget and arn association
-	if tgts, ok := sys.targetsMap[bucket]; ok {
-		for _, t := range tgts {
+	if stgts, ok := sys.targetsMap[bucket]; ok {
+		for _, t := range stgts {
 			delete(sys.arnRemotesMap, t.Arn)
 		}
 		delete(sys.targetsMap, bucket)
 	}
 
-	// No need for more if not adding anything
-	if tgts == nil || tgts.Empty() {
-		globalBucketMonitor.DeleteBucket(bucket)
-		return
-	}
-
-	if len(tgts.Targets) > 0 {
-		sys.targetsMap[bucket] = tgts.Targets
-	}
-	for _, tgt := range tgts.Targets {
-		tgtClient, err := sys.getRemoteTargetClient(&tgt)
-		if err != nil {
-			continue
+	if tgts != nil {
+		for _, tgt := range tgts.Targets {
+			tgtClient, err := sys.getRemoteTargetClient(&tgt)
+			if err != nil {
+				continue
+			}
+			sys.arnRemotesMap[tgt.Arn] = tgtClient
+			sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
+		}
+
+		if !tgts.Empty() {
+			sys.targetsMap[bucket] = tgts.Targets
 		}
-		sys.arnRemotesMap[tgt.Arn] = tgtClient
-		sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
 	}
-	sys.targetsMap[bucket] = tgts.Targets
 }
 
 // create minio-go clients for buckets having remote targets
@@ -524,9 +520,6 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
 	}
 	sys.Lock()
 	defer sys.Unlock()
-	if len(cfg.Targets) > 0 {
-		sys.targetsMap[bucket.Name] = cfg.Targets
-	}
 	for _, tgt := range cfg.Targets {
 		tgtClient, err := sys.getRemoteTargetClient(&tgt)
 		if err != nil {

cmd/common-main.go

@@ -29,7 +29,6 @@ import (
 	"fmt"
 	"math/rand"
 	"net"
-	"net/http"
 	"net/url"
 	"os"
 	"path"
@@ -54,7 +53,6 @@ import (
 	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
@@ -71,10 +69,7 @@ import (
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
 
-var (
-	shardDiskTimeDelta     time.Duration
-	defaultAWSCredProvider []credentials.Provider
-)
+var shardDiskTimeDelta time.Duration
 
 func init() {
 	if runtime.GOOS == "windows" {
@@ -112,23 +107,16 @@ func init() {
 	gob.Register(madmin.XFSErrorConfigs{})
 	gob.Register(map[string]interface{}{})
 
-	defaultAWSCredProvider = []credentials.Provider{
-		&credentials.IAM{
-			Client: &http.Client{
-				Transport: NewHTTPTransport(),
-			},
-		},
-	}
-
 	var err error
 	shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m"))
 	if err != nil {
 		shardDiskTimeDelta = 1 * time.Minute
 	}
 
-	// All minio-go API operations shall be performed only once,
+	// All minio-go and madmin-go API operations shall be performed only once,
 	// another way to look at this is we are turning off retries.
 	minio.MaxRetry = 1
+	madmin.MaxRetry = 1
 }
 
 const consolePrefix = "CONSOLE_"
@@ -143,29 +131,33 @@ func minioConfigToConsoleFeatures() {
 		// This will save users from providing a certificate with IP or FQDN SAN that points to the local host.
 		os.Setenv("CONSOLE_MINIO_SERVER", fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(globalIsTLS), globalMinioPort))
 	}
-	if value := env.Get("MINIO_LOG_QUERY_URL", ""); value != "" {
+	if value := env.Get(config.EnvMinIOLogQueryURL, ""); value != "" {
 		os.Setenv("CONSOLE_LOG_QUERY_URL", value)
-		if value := env.Get("MINIO_LOG_QUERY_AUTH_TOKEN", ""); value != "" {
+		if value := env.Get(config.EnvMinIOLogQueryAuthToken, ""); value != "" {
 			os.Setenv("CONSOLE_LOG_QUERY_AUTH_TOKEN", value)
 		}
 	}
 	// pass the console subpath configuration
-	if value := env.Get(config.EnvBrowserRedirectURL, ""); value != "" {
+	if globalBrowserRedirectURL != nil {
 		subPath := path.Clean(pathJoin(strings.TrimSpace(globalBrowserRedirectURL.Path), SlashSeparator))
 		if subPath != SlashSeparator {
 			os.Setenv("CONSOLE_SUBPATH", subPath)
 		}
 	}
 	// Enable if prometheus URL is set.
-	if value := env.Get("MINIO_PROMETHEUS_URL", ""); value != "" {
+	if value := env.Get(config.EnvMinIOPrometheusURL, ""); value != "" {
 		os.Setenv("CONSOLE_PROMETHEUS_URL", value)
-		if value := env.Get("MINIO_PROMETHEUS_JOB_ID", "minio-job"); value != "" {
+		if value := env.Get(config.EnvMinIOPrometheusJobID, "minio-job"); value != "" {
 			os.Setenv("CONSOLE_PROMETHEUS_JOB_ID", value)
 			// Support additional labels for more granular filtering.
-			if value := env.Get("MINIO_PROMETHEUS_EXTRA_LABELS", ""); value != "" {
+			if value := env.Get(config.EnvMinIOPrometheusExtraLabels, ""); value != "" {
 				os.Setenv("CONSOLE_PROMETHEUS_EXTRA_LABELS", value)
 			}
 		}
+		// Support Prometheus Auth Token
+		if value := env.Get(config.EnvMinIOPrometheusAuthToken, ""); value != "" {
+			os.Setenv("CONSOLE_PROMETHEUS_AUTH_TOKEN", value)
+		}
 	}
 	// Enable if LDAP is enabled.
 	if globalIAMSys.LDAPConfig.Enabled() {
@@ -391,7 +383,7 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	if consoleAddr == "" {
 		p, err := xnet.GetFreePort()
 		if err != nil {
-			logger.FatalIf(err, "Unable to get free port for console on the host")
+			logger.FatalIf(err, "Unable to get free port for Console UI on the host")
 		}
 		consoleAddr = net.JoinHostPort("", p.String())
 	}
@@ -405,6 +397,15 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	}
 
 	globalMinioHost, globalMinioPort = mustSplitHostPort(addr)
+	if globalMinioPort == "0" {
+		p, err := xnet.GetFreePort()
+		if err != nil {
+			logger.FatalIf(err, "Unable to get free port for S3 API on the host")
+		}
+		globalMinioPort = p.String()
+		globalDynamicAPIPort = true
+	}
+
 	globalMinioConsoleHost, globalMinioConsolePort = mustSplitHostPort(consoleAddr)
 
 	if globalMinioPort == globalMinioConsolePort {
@@ -640,6 +641,7 @@ func handleCommonEnvVars() {
 			}
 			globalBrowserRedirectURL = u
 		}
+		globalBrowserRedirect = env.Get(config.EnvBrowserRedirect, config.EnableOn) == config.EnableOn
 	}
 
 	if serverURL := env.Get(config.EnvMinIOServerURL, ""); serverURL != "" {
@@ -664,10 +666,14 @@ func handleCommonEnvVars() {
 		logger.Fatal(config.ErrInvalidFSOSyncValue(err), "Invalid MINIO_FS_OSYNC value in environment variable")
 	}
 
-	if rootDiskSize := env.Get(config.EnvRootDiskThresholdSize, ""); rootDiskSize != "" {
+	rootDiskSize := env.Get(config.EnvRootDriveThresholdSize, "")
+	if rootDiskSize == "" {
+		rootDiskSize = env.Get(config.EnvRootDiskThresholdSize, "")
+	}
+	if rootDiskSize != "" {
 		size, err := humanize.ParseBytes(rootDiskSize)
 		if err != nil {
-			logger.Fatal(err, fmt.Sprintf("Invalid %s value in environment variable", config.EnvRootDiskThresholdSize))
+			logger.Fatal(err, fmt.Sprintf("Invalid %s value in root drive threshold environment variable", rootDiskSize))
 		}
 		globalRootDiskThreshold = size
 	}
@@ -802,6 +808,9 @@ func handleKMSConfig() {
 				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientCert))
 			}
 		}
+		if !env.IsSet(kms.EnvKESKeyName) {
+			logger.Fatal(errors.New("Invalid KES configuration"), fmt.Sprintf("The mandatory environment variable %q not set", kms.EnvKESKeyName))
+		}
 
 		var endpoints []string
 		for _, endpoint := range strings.Split(env.Get(kms.EnvKESEndpoint, ""), ",") {

cmd/config-common.go

@@ -71,7 +71,7 @@ func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string)
 }
 
 func saveConfigWithOpts(ctx context.Context, store objectIO, configFile string, data []byte, opts ObjectOptions) error {
-	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
+	hashReader, err := hash.NewReader(ctx, bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
 	if err != nil {
 		return err
 	}

cmd/data-scanner-metric.go

@@ -66,10 +66,12 @@ const (
 	scannerMetricYield
 	scannerMetricCleanAbandoned
 	scannerMetricApplyNonCurrent
+	scannerMetricHealAbandonedVersion
 
 	// START Trace metrics:
 	scannerMetricStartTrace
 	scannerMetricScanObject // Scan object. All operations included.
+	scannerMetricHealAbandonedObject
 
 	// END realtime metrics:
 	scannerMetricLastRealtime

cmd/data-scanner.go

@@ -54,11 +54,10 @@ const (
 	dataScannerForceCompactAtFolders = 1_000_000                        // Compact when this many subfolders in a single folder (even top level).
 	dataScannerStartDelay            = 1 * time.Minute                  // Time to wait on startup and between cycles.
 
-	healDeleteDangling    = true
-	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
-	healObjectSelectProb  = 512 // Overall probability of a file being scanned; one in n.
+	healDeleteDangling   = true
+	healObjectSelectProb = 1024 // Overall probability of a file being scanned; one in n.
 
-	dataScannerExcessiveVersionsThreshold = 1000  // Issue a warning when a single object has more versions than this
+	dataScannerExcessiveVersionsThreshold = 100   // Issue a warning when a single object has more versions than this
 	dataScannerExcessiveFoldersThreshold  = 50000 // Issue a warning when a folder has more subfolders than this in a *set*
 )
 
@@ -66,7 +65,7 @@ var (
 	globalHealConfig heal.Config
 
 	// Sleeper values are updated when config is loaded.
-	scannerSleeper = newDynamicSleeper(10, 10*time.Second, true)
+	scannerSleeper = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
 	scannerCycle   = uatomic.NewDuration(dataScannerStartDelay)
 )
 
@@ -274,7 +273,6 @@ type folderScanner struct {
 // rarer if the bloom filter for the path is clean and no lifecycles are applied.
 // Skipped leaves have their totals transferred from the previous cycle.
 //
-// A clean leaf will be included once every healFolderIncludeProb for partial heal scans.
 // When selected there is a one in healObjectSelectProb that any object will be chosen for heal scan.
 //
 // Compaction happens when either:
@@ -409,9 +407,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 
 		var existingFolders, newFolders []cachedFolder
 		var foundObjects bool
-		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
+		err := readDirFn(pathJoin(f.root, folder.name), func(entName string, typ os.FileMode) error {
 			// Parse
-			entName = pathClean(path.Join(folder.name, entName))
+			entName = pathClean(pathJoin(folder.name, entName))
 			if entName == "" || entName == folder.name {
 				if f.dataUsageScannerDebug {
 					console.Debugf(scannerLogPrefix+" no entity (%s,%s)\n", f.root, entName)
@@ -461,7 +459,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 
 			// Get file size, ignore errors.
 			item := scannerItem{
-				Path:        path.Join(f.root, entName),
+				Path:        pathJoin(f.root, entName),
 				Typ:         typ,
 				bucket:      bucket,
 				prefix:      path.Dir(prefix),
@@ -496,7 +494,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// Object already accounted for, remove from heal map,
 			// simply because getSize() function already heals the
 			// object.
-			delete(abandonedChildren, path.Join(item.bucket, item.objectPath()))
+			delete(abandonedChildren, pathJoin(item.bucket, item.objectPath()))
 
 			into.addSizes(sz)
 			into.Objects++
@@ -691,6 +689,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				reportNotFound: true,
 				minDisks:       f.disksQuorum,
 				agreed: func(entry metaCacheEntry) {
+					f.updateCurrentPath(entry.name)
 					if f.dataUsageScannerDebug {
 						console.Debugf(healObjectsPrefix+" got agreement: %v\n", entry.name)
 					}
@@ -704,6 +703,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						// this object might be dangling.
 						entry, _ = entries.firstFound()
 					}
+					// wait timer per object.
+					wait := scannerSleeper.Timer(ctx)
+					defer wait()
+					f.updateCurrentPath(entry.name)
+					stopFn := globalScannerMetrics.log(scannerMetricHealAbandonedObject, f.root, entry.name)
+					custom := make(map[string]string)
+					defer stopFn(custom)
 
 					if f.dataUsageScannerDebug {
 						console.Debugf(healObjectsPrefix+" resolved to: %v, dir: %v\n", entry.name, entry.isDir())
@@ -713,13 +719,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						return
 					}
 
-					// wait on timer per object.
-					wait := scannerSleeper.Timer(ctx)
-
 					// We got an entry which we should be able to heal.
 					fiv, err := entry.fileInfoVersions(bucket)
 					if err != nil {
-						wait()
 						err := bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
 							object:    entry.name,
@@ -732,21 +734,28 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						return
 					}
 
+					custom["versions"] = fmt.Sprint(len(fiv.Versions))
+					var successVersions, failVersions int
 					for _, ver := range fiv.Versions {
-						// Sleep and reset.
-						wait()
-						wait = scannerSleeper.Timer(ctx)
-
+						stopFn := globalScannerMetrics.timeSize(scannerMetricHealAbandonedVersion)
 						err := bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
 							object:    fiv.Name,
 							versionID: ver.VersionID,
 						}, madmin.HealItemObject)
+						stopFn(int(ver.Size))
 						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 							logger.LogIf(ctx, err)
 						}
+						if err == nil {
+							successVersions++
+						} else {
+							failVersions++
+						}
 						foundObjs = foundObjs || err == nil
 					}
+					custom["success_versions"] = fmt.Sprint(successVersions)
+					custom["failed_versions"] = fmt.Sprint(failVersions)
 				},
 				// Too many disks failed.
 				finished: func(errs []error) {
@@ -873,7 +882,7 @@ type getSizeFn func(item scannerItem) (sizeSummary, error)
 func (i *scannerItem) transformMetaDir() {
 	split := strings.Split(i.prefix, SlashSeparator)
 	if len(split) > 1 {
-		i.prefix = path.Join(split[:len(split)-1]...)
+		i.prefix = pathJoin(split[:len(split)-1]...)
 	} else {
 		i.prefix = ""
 	}
@@ -920,7 +929,8 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje
 
 	versionID := oi.VersionID
 	rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
-	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi)
+	replcfg, _ := getReplicationConfig(ctx, i.bucket)
+	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, replcfg, oi)
 	if i.debug {
 		if versionID != "" {
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action)
@@ -1061,17 +1071,6 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 // applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
 // after applying lifecycle checks configured.
 func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
-	if i.heal.enabled {
-		if healDeleteDangling {
-			done := globalScannerMetrics.time(scannerMetricCleanAbandoned)
-			err := o.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
-			done()
-			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err))
-			}
-		}
-	}
-
 	objInfos, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
 	if err != nil {
 		return nil, err
@@ -1112,7 +1111,17 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 			done := globalScannerMetrics.time(scannerMetricHealCheck)
 			size = i.applyHealing(ctx, o, oi)
 			done()
+
+			if healDeleteDangling {
+				done := globalScannerMetrics.time(scannerMetricCleanAbandoned)
+				err := o.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
+				done()
+				if err != nil {
+					logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err))
+				}
+			}
 		}
+
 		// replicate only if lifecycle rules are not applied.
 		done := globalScannerMetrics.time(scannerMetricCheckReplication)
 		i.healReplication(ctx, o, oi.Clone(), sizeS)
@@ -1121,7 +1130,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 	return size
 }
 
-func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) lifecycle.Event {
+func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event {
 	event := lc.Eval(obj.ToLifecycleOpts())
 	if serverDebugLog {
 		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action)
@@ -1153,6 +1162,9 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr loc
 			}
 			return lifecycle.Event{Action: lifecycle.NoneAction}
 		}
+		if rcfg != nil && !obj.VersionPurgeStatus.Empty() && rcfg.HasActiveRules(obj.Name, true) {
+			return lifecycle.Event{Action: lifecycle.NoneAction}
+		}
 	}
 
 	return event
@@ -1247,7 +1259,7 @@ func applyLifecycleAction(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)
 
 // objectPath returns the prefix and object name.
 func (i *scannerItem) objectPath() string {
-	return path.Join(i.prefix, i.objectName)
+	return pathJoin(i.prefix, i.objectName)
 }
 
 // healReplication will heal a scanned item that has failed replication.

cmd/data-usage-cache.go

@@ -38,6 +38,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
+	"github.com/valyala/bytebufferpool"
 )
 
 //go:generate msgp -file $GOFILE -unexported
@@ -346,20 +347,18 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.ReplicationStats.ReplicaSize += uint64(summary.replicaSize)
 	e.ReplicationStats.ReplicaCount += uint64(summary.replicaCount)
 
-	if summary.replTargetStats != nil {
-		for arn, st := range summary.replTargetStats {
-			tgtStat, ok := e.ReplicationStats.Targets[arn]
-			if !ok {
-				tgtStat = replicationStats{}
-			}
-			tgtStat.PendingSize += uint64(st.pendingSize)
-			tgtStat.FailedSize += uint64(st.failedSize)
-			tgtStat.ReplicatedSize += uint64(st.replicatedSize)
-			tgtStat.ReplicatedCount += uint64(st.replicatedCount)
-			tgtStat.FailedCount += st.failedCount
-			tgtStat.PendingCount += st.pendingCount
-			e.ReplicationStats.Targets[arn] = tgtStat
+	for arn, st := range summary.replTargetStats {
+		tgtStat, ok := e.ReplicationStats.Targets[arn]
+		if !ok {
+			tgtStat = replicationStats{}
 		}
+		tgtStat.PendingSize += uint64(st.pendingSize)
+		tgtStat.FailedSize += uint64(st.failedSize)
+		tgtStat.ReplicatedSize += uint64(st.replicatedSize)
+		tgtStat.ReplicatedCount += uint64(st.replicatedCount)
+		tgtStat.FailedCount += st.failedCount
+		tgtStat.PendingCount += st.pendingCount
+		e.ReplicationStats.Targets[arn] = tgtStat
 	}
 	if summary.tiers != nil {
 		if e.AllTierStats == nil {
@@ -925,39 +924,55 @@ type objectIO interface {
 // load the cache content with name from minioMetaBackgroundOpsBucket.
 // Only backend errors are returned as errors.
 // The loader is optimistic and has no locking, but tries 5 times before giving up.
-// If the object is not found or unable to deserialize d is cleared and nil error is returned.
+// If the object is not found, a nil error with empty data usage cache is returned.
 func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
-	// Abandon if more than 5 minutes, so we don't hold up scanner.
-	ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
-	defer cancel()
+	// By defaut, empty data usage cache
+	*d = dataUsageCache{}
+
+	load := func(name string, timeout time.Duration) (bool, error) {
+		// Abandon if more than time.Minute, so we don't hold up scanner.
+		// drive timeout by default is 2 minutes, we do not need to wait longer.
+		ctx, cancel := context.WithTimeout(ctx, timeout)
+		defer cancel()
 
-	// Caches are read+written without locks,
-	retries := 0
-	for retries < 5 {
 		r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, ObjectOptions{NoLock: true})
 		if err != nil {
 			switch err.(type) {
 			case ObjectNotFound, BucketNotFound:
+				return false, nil
 			case InsufficientReadQuorum, StorageErr:
-				retries++
-				time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
-				continue
-			default:
-				return toObjectErr(err, dataUsageBucket, name)
+				return true, nil
 			}
-			*d = dataUsageCache{}
-			return nil
-		}
-		if err := d.deserialize(r); err != nil {
-			r.Close()
-			retries++
-			time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
-			continue
+			return false, err
 		}
+		err = d.deserialize(r)
 		r.Close()
-		return nil
+		return err != nil, nil
 	}
-	*d = dataUsageCache{}
+
+	// Caches are read+written without locks,
+	retries := 0
+	for retries < 5 {
+		retry, err := load(name, time.Minute)
+		if err != nil {
+			return toObjectErr(err, dataUsageBucket, name)
+		}
+		if !retry {
+			break
+		}
+		retry, err = load(name+".bkp", 30*time.Second)
+		if err == nil && !retry {
+			// Only return when we have valid data from the backup
+			break
+		}
+		retries++
+		time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
+	}
+
+	if retries == 5 {
+		logger.LogOnceIf(ctx, fmt.Errorf("maximum retry reached to load the data usage cache `%s`", name), "retry-loading-data-usage-cache")
+	}
+
 	return nil
 }
 
@@ -967,47 +982,52 @@ var maxConcurrentScannerSaves = make(chan struct{}, 4)
 // save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
 // Note that no locking is done when saving.
 func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
-	var r io.Reader
-	maxConcurrentScannerSaves <- struct{}{}
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case maxConcurrentScannerSaves <- struct{}{}:
+	}
 	defer func() {
-		<-maxConcurrentScannerSaves
-	}()
-	// If big, do streaming...
-	size := int64(-1)
-	if len(d.Cache) > 10000 {
-		pr, pw := io.Pipe()
-		go func() {
-			pw.CloseWithError(d.serializeTo(pw))
-		}()
-		defer pr.Close()
-		r = pr
-	} else {
-		var buf bytes.Buffer
-		err := d.serializeTo(&buf)
-		if err != nil {
-			return err
+		select {
+		case <-ctx.Done():
+		case <-maxConcurrentScannerSaves:
 		}
-		r = &buf
-		size = int64(buf.Len())
+	}()
+
+	buf := bytebufferpool.Get()
+	defer func() {
+		buf.Reset()
+		bytebufferpool.Put(buf)
+	}()
+
+	if err := d.serializeTo(buf); err != nil {
+		return err
 	}
 
-	hr, err := hash.NewReader(r, size, "", "", size)
+	hr, err := hash.NewReader(ctx, bytes.NewReader(buf.Bytes()), int64(buf.Len()), "", "", int64(buf.Len()))
 	if err != nil {
 		return err
 	}
 
-	// Abandon if more than 5 minutes, so we don't hold up scanner.
-	ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
-	defer cancel()
-	_, err = store.PutObject(ctx,
-		dataUsageBucket,
-		name,
-		NewPutObjReader(hr),
-		ObjectOptions{NoLock: true})
-	if isErrBucketNotFound(err) {
-		return nil
+	save := func(name string, timeout time.Duration) error {
+		// Abandon if more than a minute, so we don't hold up scanner.
+		ctx, cancel := context.WithTimeout(ctx, timeout)
+		defer cancel()
+
+		_, err = store.PutObject(ctx,
+			dataUsageBucket,
+			name,
+			NewPutObjReader(hr),
+			ObjectOptions{NoLock: true})
+		if isErrBucketNotFound(err) {
+			return nil
+		}
+		return err
 	}
-	return err
+	defer save(name+".bkp", 30*time.Second) // Keep a backup as well
+
+	// drive timeout by default is 2 minutes, we do not need to wait longer.
+	return save(name, time.Minute)
 }
 
 // dataUsageCacheVer indicates the cache version.

cmd/data-usage.go

@@ -21,6 +21,7 @@ import (
 	"context"
 	"errors"
 	"strings"
+	"time"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/internal/logger"
@@ -42,6 +43,7 @@ const (
 
 // storeDataUsageInBackend will store all objects sent on the gui channel until closed.
 func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan DataUsageInfo) {
+	attempts := 1
 	for dataUsageInfo := range dui {
 		json := jsoniter.ConfigCompatibleWithStandardLibrary
 		dataUsageJSON, err := json.Marshal(dataUsageInfo)
@@ -49,12 +51,19 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 			logger.LogIf(ctx, err)
 			continue
 		}
+		if attempts > 10 {
+			saveConfig(ctx, objAPI, dataUsageObjNamePath+".bkp", dataUsageJSON) // Save a backup every 10th update.
+			attempts = 1
+		}
 		if err = saveConfig(ctx, objAPI, dataUsageObjNamePath, dataUsageJSON); err != nil {
 			logger.LogIf(ctx, err)
 		}
+		attempts++
 	}
 }
 
+var prefixUsageCache timedValue
+
 // loadPrefixUsageFromBackend returns prefix usages found in passed buckets
 //
 //	e.g.:  /testbucket/prefix => 355601334
@@ -67,37 +76,57 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket
 
 	cache := dataUsageCache{}
 
-	m := make(map[string]uint64)
-	for _, pool := range z.serverPools {
-		for _, er := range pool.sets {
-			// Load bucket usage prefixes
-			if err := cache.load(ctx, er, bucket+slashSeparator+dataUsageCacheName); err == nil {
-				root := cache.find(bucket)
-				if root == nil {
-					// We dont have usage information for this bucket in this
-					// set, go to the next set
-					continue
-				}
+	prefixUsageCache.Once.Do(func() {
+		prefixUsageCache.TTL = 30 * time.Second
 
-				for id, usageInfo := range cache.flattenChildrens(*root) {
-					prefix := decodeDirObject(strings.TrimPrefix(id, bucket+slashSeparator))
-					// decodeDirObject to avoid any __XLDIR__ objects
-					m[prefix] += uint64(usageInfo.Size)
+		// No need to fail upon Update() error, fallback to old value.
+		prefixUsageCache.Relax = true
+		prefixUsageCache.Update = func() (interface{}, error) {
+			m := make(map[string]uint64)
+			for _, pool := range z.serverPools {
+				for _, er := range pool.sets {
+					// Load bucket usage prefixes
+					ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
+					ok := cache.load(ctx, er, bucket+slashSeparator+dataUsageCacheName) == nil
+					done()
+					if ok {
+						root := cache.find(bucket)
+						if root == nil {
+							// We dont have usage information for this bucket in this
+							// set, go to the next set
+							continue
+						}
+
+						for id, usageInfo := range cache.flattenChildrens(*root) {
+							prefix := decodeDirObject(strings.TrimPrefix(id, bucket+slashSeparator))
+							// decodeDirObject to avoid any __XLDIR__ objects
+							m[prefix] += uint64(usageInfo.Size)
+						}
+					}
 				}
 			}
+			return m, nil
 		}
+	})
+
+	v, _ := prefixUsageCache.Get()
+	if v != nil {
+		return v.(map[string]uint64), nil
 	}
 
-	return m, nil
+	return map[string]uint64{}, nil
 }
 
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
 	buf, err := readConfig(ctx, objAPI, dataUsageObjNamePath)
 	if err != nil {
-		if errors.Is(err, errConfigNotFound) {
-			return DataUsageInfo{}, nil
+		buf, err = readConfig(ctx, objAPI, dataUsageObjNamePath+".bkp")
+		if err != nil {
+			if errors.Is(err, errConfigNotFound) {
+				return DataUsageInfo{}, nil
+			}
+			return DataUsageInfo{}, toObjectErr(err, minioMetaBucket, dataUsageObjNamePath)
 		}
-		return DataUsageInfo{}, toObjectErr(err, minioMetaBucket, dataUsageObjNamePath)
 	}
 
 	var dataUsageInfo DataUsageInfo

cmd/disk-cache-backend.go

@@ -604,7 +604,7 @@ func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, met
 	// move part saved in writeback directory and cache.json atomically
 	if finalizeWB {
 		wbdir := getCacheWriteBackSHADir(c.dir, bucket, object)
-		if err = renameAll(pathJoin(wbdir, cacheDataFile), pathJoin(cachedPath, cacheDataFile)); err != nil {
+		if err = renameAll(pathJoin(wbdir, cacheDataFile), pathJoin(cachedPath, cacheDataFile), c.dir); err != nil {
 			return err
 		}
 		removeAll(wbdir) // cleanup writeback/shadir
@@ -1273,7 +1273,7 @@ func (c *diskCache) NewMultipartUpload(ctx context.Context, bucket, object, uID
 
 	cachePath := getMultipartCacheSHADir(c.dir, bucket, object)
 	uploadIDDir := path.Join(cachePath, uploadID)
-	if err := mkdirAll(uploadIDDir, 0o777); err != nil {
+	if err := mkdirAll(uploadIDDir, 0o777, c.dir); err != nil {
 		return uploadID, err
 	}
 	metaPath := pathJoin(uploadIDDir, cacheMetaJSONFile)
@@ -1441,7 +1441,7 @@ func newCachePartEncryptReader(ctx context.Context, bucket, object string, partI
 		info := ObjectInfo{Size: size}
 		wantSize = info.EncryptedSize()
 	}
-	hReader, err := hash.NewReader(content, wantSize, "", "", size)
+	hReader, err := hash.NewReader(ctx, content, wantSize, "", "", size)
 	if err != nil {
 		return nil, err
 	}
@@ -1570,9 +1570,9 @@ func (c *diskCache) CompleteMultipartUpload(ctx context.Context, bucket, object,
 	jsonSave(f, uploadMeta)
 	for _, pi := range uploadedParts {
 		part := fmt.Sprintf("part.%d", pi.PartNumber)
-		renameAll(pathJoin(uploadIDDir, part), pathJoin(cachePath, part))
+		renameAll(pathJoin(uploadIDDir, part), pathJoin(cachePath, part), c.dir)
 	}
-	renameAll(pathJoin(uploadIDDir, cacheMetaJSONFile), pathJoin(cachePath, cacheMetaJSONFile))
+	renameAll(pathJoin(uploadIDDir, cacheMetaJSONFile), pathJoin(cachePath, cacheMetaJSONFile), c.dir)
 	removeAll(uploadIDDir) // clean up any unused parts in the uploadIDDir
 	return uploadMeta.ToObjectInfo(), nil
 }

cmd/disk-cache.go

@@ -734,7 +734,7 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 	defer cLock.Unlock(lkctx)
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, "", "", r.ActualSize())
+	hashReader, err := hash.NewReader(ctx, pipeReader, size, "", "", r.ActualSize())
 	if err != nil {
 		return
 	}
@@ -795,7 +795,7 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 	if st == CommitComplete || st.String() == "" {
 		return
 	}
-	hashReader, err := hash.NewReader(cReader, oi.Size, "", "", oi.Size)
+	hashReader, err := hash.NewReader(ctx, cReader, oi.Size, "", "", oi.Size)
 	if err != nil {
 		return
 	}
@@ -1059,7 +1059,7 @@ func (c *cacheObjects) PutObjectPart(ctx context.Context, bucket, object, upload
 	info = PartInfo{}
 	// Initialize pipe to stream data to backend
 	pipeReader, pipeWriter := io.Pipe()
-	hashReader, err := hash.NewReader(pipeReader, size, "", "", data.ActualSize())
+	hashReader, err := hash.NewReader(ctx, pipeReader, size, "", "", data.ActualSize())
 	if err != nil {
 		return
 	}

cmd/encryption-v1.go

@@ -728,7 +728,7 @@ func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
 // DecryptedSize returns the size of the object after decryption in bytes.
 // It returns an error if the object is not encrypted or marked as encrypted
 // but has an invalid size.
-func (o *ObjectInfo) DecryptedSize() (int64, error) {
+func (o ObjectInfo) DecryptedSize() (int64, error) {
 	if _, ok := crypto.IsEncrypted(o.UserDefined); !ok {
 		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}

cmd/erasure-common.go

@@ -74,62 +74,6 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	return newDisks
 }
 
-// getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
-// ensures to skip disks if they are not healing and online.
-func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
-	disks := er.getDisks()
-
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	if !optimized {
-		var newDisks []StorageAPI
-		for _, i := range r.Perm(len(disks)) {
-			newDisks = append(newDisks, disks[i])
-		}
-		return newDisks
-	}
-
-	var wg sync.WaitGroup
-	var mu sync.Mutex
-	newDisks := map[uint64][]StorageAPI{}
-	// Based on the random shuffling return back randomized disks.
-	for _, i := range r.Perm(len(disks)) {
-		i := i
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			if disks[i] == nil {
-				return
-			}
-			di, err := disks[i].DiskInfo(context.Background(), false)
-			if err != nil || di.Healing {
-				// - Do not consume disks which are not reachable
-				//   unformatted or simply not accessible for some reason.
-				//
-				// - Do not consume disks which are being healed
-				//
-				// - Future: skip busy disks
-				return
-			}
-
-			mu.Lock()
-			// Capture disks usage wise upto resolution of MiB
-			newDisks[di.Used/1024/1024] = append(newDisks[di.Used/1024/1024], disks[i])
-			mu.Unlock()
-		}()
-	}
-	wg.Wait()
-
-	var max uint64
-	for k := range newDisks {
-		if k > max {
-			max = k
-		}
-	}
-
-	// Return disks which have maximum disk usage common.
-	return newDisks[max]
-}
-
 // readMultipleFiles Reads raw data from all specified files from all disks.
 func readMultipleFiles(ctx context.Context, disks []StorageAPI, req ReadMultipleReq, readQuorum int) ([]ReadMultipleResp, error) {
 	resps := make([]chan ReadMultipleResp, len(disks))

cmd/erasure-healing.go

@@ -563,11 +563,26 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	}
 
 	if !latestMeta.XLV1 && !latestMeta.Deleted && !recreate && disksToHealCount > latestMeta.Erasure.ParityBlocks {
-		// When disk to heal count is greater than parity blocks we should simply error out.
-		err := fmt.Errorf("(%d > %d) more drives are expected to heal than parity, returned errors: %v (dataErrs %v) -> %s/%s(%s)", disksToHealCount, latestMeta.Erasure.ParityBlocks, errs, dataErrs, bucket, object, versionID)
-		logger.LogOnceIf(ctx, err, "heal-object-count-gt-parity")
-		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
-			bucket, object, versionID), err
+		// Allow for dangling deletes, on versions that have DataDir missing etc.
+		// this would end up restoring the correct readable versions.
+		m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrs, ObjectOptions{
+			VersionID: versionID,
+		})
+		errs = make([]error, len(errs))
+		for i := range errs {
+			errs[i] = err
+		}
+		if err == nil {
+			// Dangling object successfully purged, size is '0'
+			m.Size = 0
+		}
+		// Generate file/version not found with default heal result
+		err = errFileNotFound
+		if versionID != "" {
+			err = errFileVersionNotFound
+		}
+		return er.defaultHealResult(m, storageDisks, storageEndpoints,
+			errs, bucket, object, versionID), err
 	}
 
 	cleanFileInfo := func(fi FileInfo) FileInfo {
@@ -1077,6 +1092,22 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	}
 
 	if !validMeta.IsValid() {
+		// validMeta is invalid because notFoundPartsErrs is
+		// greater than parity blocks, thus invalidating the FileInfo{}
+		// every dataErrs[i], metaArr[i] is an empty FileInfo{}
+		dataBlocks := (len(ndataErrs) + 1) / 2
+		if notFoundPartsErrs > dataBlocks {
+			// Not using parity to ensure that we do not delete
+			// any valid content, if any is recoverable. But if
+			// notFoundDataDirs are already greater than the data
+			// blocks all bets are off and it is safe to purge.
+			//
+			// This is purely a defensive code, ideally parityBlocks
+			// is sufficient, however we can't know that since we
+			// do have the FileInfo{}.
+			return validMeta, true
+		}
+
 		// We have no idea what this file is, leave it as is.
 		return validMeta, false
 	}

cmd/erasure-multipart.go

@@ -269,7 +269,13 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 
 	var uploadIDs []string
 	var disk StorageAPI
-	for _, disk = range er.getLoadBalancedDisks(true) {
+	disks := er.getLoadBalancedLocalDisks()
+	if len(disks) == 0 {
+		// using er.getLoadBalancedLocalDisks() has one side-affect where
+		// on a pooled setup all disks are remote, add a fallback
+		disks = er.getOnlineDisks()
+	}
+	for _, disk = range disks {
 		uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
 		if err != nil {
 			if errors.Is(err, errDiskNotFound) {
@@ -1242,7 +1248,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 
-	defer er.deleteAll(context.Background(), minioMetaMultipartBucket, uploadIDPath)
+	defer func() {
+		if err == nil {
+			er.deleteAll(context.Background(), minioMetaMultipartBucket, uploadIDPath)
+		}
+	}()
 
 	// Rename the multipart object to final location.
 	onlineDisks, versionsDisparity, err := renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,

cmd/erasure-object.go

@@ -46,6 +46,7 @@ import (
 	"github.com/minio/pkg/v2/mimedb"
 	"github.com/minio/pkg/v2/sync/errgroup"
 	"github.com/minio/pkg/v2/wildcard"
+	"github.com/tinylib/msgp/msgp"
 	uatomic "go.uber.org/atomic"
 )
 
@@ -497,7 +498,14 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		tags := make(map[string]interface{}, 4)
 		tags["set"] = er.setIndex
 		tags["pool"] = er.poolIndex
-		tags["parity"] = m.Erasure.ParityBlocks
+		tags["merrs"] = errors.Join(errs...)
+		tags["derrs"] = errors.Join(dataErrs...)
+		if m.IsValid() {
+			tags["size"] = m.Size
+			tags["mtime"] = m.ModTime.Format(http.TimeFormat)
+			tags["parity"] = m.Erasure.ParityBlocks
+		}
+
 		if cok {
 			tags["caller"] = fmt.Sprintf("%s:%d", file, line)
 		}
@@ -578,6 +586,9 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		errFileVersionNotFound,
 		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
 		io.EOF,              // some times we would read without locks, ignore these errors
+		msgp.ErrShortBytes,
+		context.DeadlineExceeded,
+		context.Canceled,
 	}
 	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)
 
@@ -1617,10 +1628,12 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 
 	var lc *lifecycle.Lifecycle
 	var rcfg lock.Retention
+	var replcfg *replication.Config
 	if opts.Expiration.Expire {
 		// Check if the current bucket has a configured lifecycle policy
 		lc, _ = globalLifecycleSys.Get(bucket)
 		rcfg, _ = globalBucketObjectLockSys.Get(bucket)
+		replcfg, _ = getReplicationConfig(ctx, bucket)
 	}
 
 	// expiration attempted on a bucket with no lifecycle
@@ -1673,7 +1686,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 
 	if opts.Expiration.Expire {
 		if gerr == nil {
-			evt := evalActionFromLifecycle(ctx, *lc, rcfg, goi)
+			evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
 			var isErr bool
 			switch evt.Action {
 			case lifecycle.NoneAction:
@@ -2151,7 +2164,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
 		}
 		defer gr.Close()
-		hashReader, err := hash.NewReader(gr, gr.ObjInfo.Size, "", "", gr.ObjInfo.Size)
+		hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", gr.ObjInfo.Size)
 		if err != nil {
 			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
 		}
@@ -2176,7 +2189,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 
 	// rehydrate the parts back on disk as per the original xl.meta prior to transition
 	for _, partInfo := range oi.Parts {
-		hr, err := hash.NewReader(io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size)
+		hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size)
 		if err != nil {
 			return setRestoreHeaderFn(oi, err)
 		}

cmd/erasure-server-pool-decom.go

@@ -68,6 +68,34 @@ type PoolDecommissionInfo struct {
 	BytesFailed             int64 `json:"bytesDecommissionedFailed" msg:"bf"`
 }
 
+// Clone make a copy of PoolDecommissionInfo
+func (pd *PoolDecommissionInfo) Clone() *PoolDecommissionInfo {
+	if pd == nil {
+		return nil
+	}
+	if pd.StartTime.IsZero() {
+		return nil
+	}
+	return &PoolDecommissionInfo{
+		StartTime:               pd.StartTime,
+		StartSize:               pd.StartSize,
+		TotalSize:               pd.TotalSize,
+		CurrentSize:             pd.CurrentSize,
+		Complete:                pd.Complete,
+		Failed:                  pd.Failed,
+		Canceled:                pd.Canceled,
+		QueuedBuckets:           pd.QueuedBuckets,
+		DecommissionedBuckets:   pd.DecommissionedBuckets,
+		Bucket:                  pd.Bucket,
+		Prefix:                  pd.Prefix,
+		Object:                  pd.Object,
+		ItemsDecommissioned:     pd.ItemsDecommissioned,
+		ItemsDecommissionFailed: pd.ItemsDecommissionFailed,
+		BytesDone:               pd.BytesDone,
+		BytesFailed:             pd.BytesFailed,
+	}
+}
+
 // bucketPop should be called when a bucket is done decommissioning.
 // Adds the bucket to the list of decommissioned buckets and updates resume numbers.
 func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
@@ -118,6 +146,16 @@ type PoolStatus struct {
 	Decommission *PoolDecommissionInfo `json:"decommissionInfo,omitempty" msg:"dec"`
 }
 
+// Clone returns a copy of PoolStatus
+func (ps PoolStatus) Clone() PoolStatus {
+	return PoolStatus{
+		ID:           ps.ID,
+		CmdLine:      ps.CmdLine,
+		LastUpdate:   ps.LastUpdate,
+		Decommission: ps.Decommission.Clone(),
+	}
+}
+
 //go:generate msgp -file $GOFILE -unexported
 type poolMeta struct {
 	Version int          `msg:"v"`
@@ -375,16 +413,17 @@ func (p *poolMeta) load(ctx context.Context, pool *erasureSets, pools []*erasure
 
 func (p *poolMeta) CountItem(idx int, size int64, failed bool) {
 	pd := p.Pools[idx].Decommission
-	if pd != nil {
-		if failed {
-			pd.ItemsDecommissionFailed++
-			pd.BytesFailed += size
-		} else {
-			pd.ItemsDecommissioned++
-			pd.BytesDone += size
-		}
-		p.Pools[idx].Decommission = pd
+	if pd == nil {
+		return
 	}
+	if failed {
+		pd.ItemsDecommissionFailed++
+		pd.BytesFailed += size
+	} else {
+		pd.ItemsDecommissioned++
+		pd.BytesDone += size
+	}
+	p.Pools[idx].Decommission = pd
 }
 
 func (p *poolMeta) updateAfter(ctx context.Context, idx int, pools []*erasureSets, duration time.Duration) (bool, error) {
@@ -569,7 +608,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, ObjectOptions{})
 		parts := make([]CompletePart, len(objInfo.Parts))
 		for i, part := range objInfo.Parts {
-			hr, err := hash.NewReader(io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
+			hr, err := hash.NewReader(ctx, io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
 			if err != nil {
 				return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 			}
@@ -603,7 +642,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		return err
 	}
 
-	hr, err := hash.NewReader(io.LimitReader(gr, objInfo.Size), objInfo.Size, "", "", actualSize)
+	hr, err := hash.NewReader(ctx, io.LimitReader(gr, objInfo.Size), objInfo.Size, "", "", actualSize)
 	if err != nil {
 		return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 	}
@@ -695,8 +734,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 
 	// Check if bucket is object locked.
 	lr, _ := globalBucketObjectLockSys.Get(bi.Name)
-
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	rcfg, _ := getReplicationConfig(ctx, bi.Name)
 
 	for setIdx, set := range pool.sets {
 		set := set
@@ -708,7 +746,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
 
-			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 			switch {
 			case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
 				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)
@@ -908,11 +946,11 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						go decommissionEntry(entry)
 					},
 				)
-				if err == nil {
+				if err == nil || errors.Is(err, context.Canceled) {
 					break
 				}
 				setN := humanize.Ordinal(setIdx + 1)
-				retryDur := time.Duration(r.Float64() * float64(5*time.Second))
+				retryDur := time.Duration(rand.Float64() * float64(5*time.Second))
 				logger.LogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v, retrying in %v", setN, err, retryDur), "decom-listing-failed"+setN)
 				time.Sleep(retryDur)
 			}
@@ -1011,6 +1049,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 
 			// Check if bucket is object locked.
 			lr, _ := globalBucketObjectLockSys.Get(bi.Name)
+			rcfg, _ := getReplicationConfig(ctx, bi.Name)
 
 			filterLifecycle := func(bucket, object string, fi FileInfo) bool {
 				if lc == nil {
@@ -1019,7 +1058,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 				versioned := vc != nil && vc.Versioned(object)
 				objInfo := fi.ToObjectInfo(bucket, object, versioned)
 
-				evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+				evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 				switch {
 				case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
 					applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)
@@ -1185,15 +1224,15 @@ func (z *erasureServerPools) Status(ctx context.Context, idx int) (PoolStatus, e
 		return PoolStatus{}, errInvalidArgument
 	}
 
-	z.poolMetaMutex.RLock()
-	defer z.poolMetaMutex.RUnlock()
-
 	pi, err := z.getDecommissionPoolSpaceInfo(idx)
 	if err != nil {
 		return PoolStatus{}, err
 	}
 
-	poolInfo := z.poolMeta.Pools[idx]
+	z.poolMetaMutex.RLock()
+	defer z.poolMetaMutex.RUnlock()
+
+	poolInfo := z.poolMeta.Pools[idx].Clone()
 	if poolInfo.Decommission != nil {
 		poolInfo.Decommission.TotalSize = pi.Total
 		if poolInfo.Decommission.Failed || poolInfo.Decommission.Canceled {

cmd/erasure-server-pool-rebalance.go

@@ -440,6 +440,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 	lc, _ := globalLifecycleSys.Get(bucket)
 	// Check if bucket is object locked.
 	lr, _ := globalBucketObjectLockSys.Get(bucket)
+	rcfg, _ := getReplicationConfig(ctx, bucket)
 
 	pool := z.serverPools[poolIdx]
 	const envRebalanceWorkers = "_MINIO_REBALANCE_WORKERS"
@@ -467,7 +468,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)
 
-			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 			if evt.Action.Delete() {
 				globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Rebal)
 				return true
@@ -736,7 +737,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 
 		parts := make([]CompletePart, len(oi.Parts))
 		for i, part := range oi.Parts {
-			hr, err := hash.NewReader(io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
+			hr, err := hash.NewReader(ctx, io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
 			if err != nil {
 				return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err)
 			}
@@ -766,7 +767,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 		return err
 	}
 
-	hr, err := hash.NewReader(gr, oi.Size, "", "", actualSize)
+	hr, err := hash.NewReader(ctx, gr, oi.Size, "", "", actualSize)
 	if err != nil {
 		return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err)
 	}

cmd/erasure-server-pool.go

@@ -25,6 +25,7 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
@@ -1323,6 +1324,85 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	}
 	opts.setBucketMeta(ctx)
 
+	ri := logger.GetReqInfo(ctx)
+	hadoop := ri != nil && strings.Contains(ri.UserAgent, `Hadoop `) && strings.Contains(ri.UserAgent, "scala/")
+	matches := func() bool {
+		if prefix == "" {
+			return false
+		}
+		// List of standard files supported by s3a
+		// that involves a List() on a directory
+		// where directory is actually an object on
+		// namespace.
+		for _, k := range []string{
+			"_SUCCESS/",
+			".parquet/",
+			".csv/",
+			".json/",
+			".avro/",
+			".orc/",
+			".txt/",
+			// Add any other files in future
+		} {
+			if strings.HasSuffix(prefix, k) {
+				return true
+			}
+		}
+		return false
+	}
+	if hadoop && matches() && delimiter == SlashSeparator && maxKeys == 2 && marker == "" {
+		// Optimization for Spark/Hadoop workload where spark sends a garbage
+		// request of this kind
+		//
+		// GET /testbucket/?list-type=2&delimiter=%2F&max-keys=2&prefix=parquet%2F_SUCCESS%2F&fetch-owner=false
+		//
+		// Here spark is expecting that the List() return empty instead, so from MinIO's point
+		// of view if we simply do a GetObjectInfo() on this prefix by treating it as an object
+		// We save a lot of calls over the network.
+		//
+		// This happens repeatedly for all objects that are created concurrently() avoiding this
+		// as a List() call is an important performance improvement.
+		//
+		// Spark based s3a committers are a  big enough use-case to have this optimization.
+		//
+		// A sample code to see the improvements is as follows, this sample code is
+		// simply a read on JSON from MinIO and write it back as "parquet".
+		//
+		// import org.apache.spark.sql.SparkSession
+		// import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
+		// object SparkJSONRead {
+		//   def main(args: Array[String]): Unit = {
+		//     val spark:SparkSession = SparkSession.builder()
+		//       .appName("SparkByExample")
+		//       .master("local[1]").getOrCreate()
+		//
+		//     spark.sparkContext.setLogLevel("ERROR")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.endpoint", "http://minio-lb:9000")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.path.style.access", "true")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.access.key", "minioadmin")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.secret.key", "minioadmin")
+		//
+		//     val df = spark.read.json("s3a://testbucket/s3.json")
+		//
+		//     df.write.parquet("s3a://testbucket/parquet/")
+		//   }
+		// }
+		objInfo, err := z.GetObjectInfo(ctx, bucket, path.Dir(prefix), ObjectOptions{NoLock: true})
+		if err == nil {
+			if opts.Lifecycle != nil {
+				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo)
+				if evt.Action.Delete() {
+					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
+					if !evt.Action.DeleteRestored() {
+						// Skip entry if ILM action was DeleteVersionAction or DeleteAction
+						return loi, nil
+					}
+				}
+			}
+			return loi, nil
+		}
+	}
+
 	if len(prefix) > 0 && maxKeys == 1 && marker == "" {
 		// Optimization for certain applications like
 		// - Cohesity
@@ -1334,7 +1414,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		objInfo, err := z.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
 		if err == nil {
 			if opts.Lifecycle != nil {
-				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
+				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo)
 				if evt.Action.Delete() {
 					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
 					if !evt.Action.DeleteRestored() {
@@ -1716,10 +1796,42 @@ func (z *erasureServerPools) deleteAll(ctx context.Context, bucket, prefix strin
 	}
 }
 
+var listBucketsCache timedValue
+
 // List all buckets from one of the serverPools, we are not doing merge
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all serverPools.
 func (z *erasureServerPools) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
+	if opts.Cached {
+		listBucketsCache.Once.Do(func() {
+			listBucketsCache.TTL = time.Second
+
+			listBucketsCache.Relax = true
+			listBucketsCache.Update = func() (interface{}, error) {
+				ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+				buckets, err = z.s3Peer.ListBuckets(ctx, opts)
+				cancel()
+				if err != nil {
+					return nil, err
+				}
+				for i := range buckets {
+					createdAt, err := globalBucketMetadataSys.CreatedAt(buckets[i].Name)
+					if err == nil {
+						buckets[i].Created = createdAt
+					}
+				}
+				return buckets, nil
+			}
+		})
+
+		v, _ := listBucketsCache.Get()
+		if v != nil {
+			return v.([]BucketInfo), nil
+		}
+
+		return buckets, nil
+	}
+
 	buckets, err = z.s3Peer.ListBuckets(ctx, opts)
 	if err != nil {
 		return nil, err

cmd/erasure.go

@@ -182,7 +182,6 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				DiskIndex: endpoints[index].DiskIdx,
 			}
 			if disks[index] == OfflineDisk {
-				logger.LogOnceIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]), "get-disks-info-offline-"+di.Endpoint)
 				di.State = diskErrToDriveState(errDiskNotFound)
 				disksInfo[index] = di
 				return nil

cmd/format-erasure.go

@@ -275,7 +275,7 @@ func formatErasureMigrateV2ToV3(data []byte, export, version string) ([]byte, er
 
 	tmpOld := pathJoin(export, minioMetaTmpDeletedBucket, mustGetUUID())
 	if err := renameAll(pathJoin(export, minioMetaMultipartBucket),
-		tmpOld); err != nil && err != errFileNotFound {
+		tmpOld, export); err != nil && err != errFileNotFound {
 		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
 			pathJoin(export, minioMetaMultipartBucket),
 			tmpOld,

cmd/ftp-server.go

@@ -21,11 +21,11 @@ import (
 	"context"
 	"crypto/subtle"
 	"fmt"
-	"io"
 	"net"
 	"os"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/minio/cli"
 	"github.com/minio/minio/internal/logger"
@@ -75,6 +75,69 @@ func (log *minioLogger) PrintResponse(sessionID string, code int, message string
 	}
 }
 
+func acceptSFTPConnection(conn net.Conn, config *ssh.ServerConfig) {
+	// For SSH handshake keep a 2 minute deadline, as OpensSSH default.
+	conn.SetDeadline(time.Now().Add(2 * time.Minute))
+
+	// Before use, a handshake must be performed on the incoming net.Conn.
+	sconn, chans, reqs, err := ssh.NewServerConn(conn, config)
+	if err != nil {
+		return
+	}
+
+	// Once we are done with SSH handshake, remove deadline.
+	conn.SetDeadline(time.Time{})
+
+	// The incoming Request channel must be serviced.
+	go ssh.DiscardRequests(reqs)
+
+	// Service the incoming Channel channel.
+	for newChannel := range chans {
+		// Channels have a type, depending on the application level
+		// protocol intended. In the case of an SFTP session, this is "subsystem"
+		// with a payload string of "<length=4>sftp"
+		if newChannel.ChannelType() != "session" {
+			newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
+			continue
+		}
+
+		channel, requests, err := newChannel.Accept()
+		if err != nil {
+			logger.LogOnceIf(context.Background(), fmt.Errorf("unable to accept the connection requests channel: %w", err), "accept-channel-sftp")
+			continue
+		}
+
+		// Sessions have out-of-band requests such as "shell",
+		// "pty-req" and "env".  Here we handle only the
+		// "subsystem" request.
+		go func(in <-chan *ssh.Request) {
+			for req := range in {
+				ok := false
+				if req.Type == "subsystem" {
+					if len(req.Payload) > 4 && string(req.Payload[4:]) == "sftp" {
+						ok = true
+						go handleSFTPConnection(channel, sconn)
+					}
+				}
+
+				if req.WantReply {
+					// We only reply to SSH packets that have `sftp` payload, all other
+					// packets are rejected
+					req.Reply(ok, nil)
+				}
+			}
+		}(requests)
+	}
+}
+
+func handleSFTPConnection(channel ssh.Channel, sconn *ssh.ServerConn) {
+	// Create the server instance for the channel using the handler we created above.
+	server := sftp.NewRequestServer(channel, NewSFTPDriver(sconn.Permissions), sftp.WithRSAllocator())
+	defer server.Close()
+
+	server.Serve()
+}
+
 func startSFTPServer(c *cli.Context) {
 	args := c.StringSlice("sftp")
 
@@ -176,54 +239,29 @@ func startSFTPServer(c *cli.Context) {
 
 	logger.Info(fmt.Sprintf("MinIO SFTP Server listening on %s", net.JoinHostPort(publicIP, strconv.Itoa(port))))
 
+	var tempDelay time.Duration // how long to sleep on accept failure
+
 	for {
-		nConn, err := listener.Accept()
+		conn, err := listener.Accept()
 		if err != nil {
-			logger.LogIf(context.Background(), err)
-			continue
-		}
-
-		// Before use, a handshake must be performed on the incoming net.Conn.
-		sconn, chans, reqs, err := ssh.NewServerConn(nConn, config)
-		if err != nil {
-			logger.LogIf(context.Background(), err)
-			continue
-		}
-
-		// The incoming Request channel must be serviced.
-		go ssh.DiscardRequests(reqs)
-
-		// Service the incoming Channel channel.
-		for newChannel := range chans {
-			// Channels have a type, depending on the application level
-			// protocol intended. In the case of an SFTP session, this is "subsystem"
-			// with a payload string of "<length=4>sftp"
-			if newChannel.ChannelType() != "session" {
-				newChannel.Reject(ssh.UnknownChannelType, "unknown channel type")
+			// From https://github.com/golang/go/blob/4aa1efed4853ea067d665a952eee77c52faac774/src/net/http/server.go#L3046
+			if ne, ok := err.(net.Error); ok && ne.Temporary() {
+				if tempDelay == 0 {
+					tempDelay = 5 * time.Millisecond
+				} else {
+					tempDelay *= 2
+				}
+				if max := 1 * time.Second; tempDelay > max {
+					tempDelay = max
+				}
+				logger.LogOnceIf(context.Background(), fmt.Errorf("error while accepting connections: %w, retrying in %s", err, tempDelay), "accept-limit-sftp")
+				time.Sleep(tempDelay)
 				continue
 			}
-			channel, requests, err := newChannel.Accept()
-			if err != nil {
-				logger.Fatal(err, "unable to accept the connection requests channel")
-			}
-
-			// Sessions have out-of-band requests such as "shell",
-			// "pty-req" and "env".  Here we handle only the
-			// "subsystem" request.
-			go func(in <-chan *ssh.Request) {
-				for req := range in {
-					// We only reply to SSH packets that have `sftp` payload.
-					req.Reply(req.Type == "subsystem" && string(req.Payload[4:]) == "sftp", nil)
-				}
-			}(requests)
-
-			server := sftp.NewRequestServer(channel, NewSFTPDriver(sconn.Permissions))
-			if err := server.Serve(); err == io.EOF {
-				server.Close()
-			} else if err != nil {
-				logger.Fatal(err, "unable to start SFTP server")
-			}
+			logger.Fatal(err, "unrecoverable error while accepting new connections")
 		}
+
+		go acceptSFTPConnection(conn, config)
 	}
 }
 

cmd/generic-handlers.go

@@ -152,7 +152,7 @@ func setBrowserRedirectMiddleware(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		read := r.Method == http.MethodGet || r.Method == http.MethodHead
 		// Re-direction is handled specifically for browser requests.
-		if guessIsBrowserReq(r) && read {
+		if guessIsBrowserReq(r) && read && globalBrowserRedirect {
 			// Fetch the redirect location if any.
 			if u := getRedirectLocation(r); u != nil {
 				// Employ a temporary re-direct.
@@ -231,7 +231,8 @@ func guessIsMetricsReq(req *http.Request) bool {
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsPathLegacy ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ClusterPath ||
 		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2NodePath ||
-		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2BucketPath
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2BucketPath ||
+		req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ResourcePath
 }
 
 // guessIsRPCReq - returns true if the request is for an RPC endpoint.

cmd/global-heal.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"fmt"
+	"runtime"
 	"sort"
 	"time"
 
@@ -30,6 +31,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	"github.com/minio/pkg/v2/wildcard"
+	"github.com/minio/pkg/v2/workers"
 )
 
 const (
@@ -132,30 +134,8 @@ func getLocalBackgroundHealStatus(ctx context.Context, o ObjectLayer) (madmin.Bg
 	return status, true
 }
 
-func mustGetHealSequence(ctx context.Context) *healSequence {
-	// Get background heal sequence to send elements to heal
-	for {
-		globalHealStateLK.RLock()
-		hstate := globalBackgroundHealState
-		globalHealStateLK.RUnlock()
-
-		if hstate == nil {
-			time.Sleep(time.Second)
-			continue
-		}
-
-		bgSeq, ok := hstate.getHealSequenceByToken(bgHealingUUID)
-		if !ok {
-			time.Sleep(time.Second)
-			continue
-		}
-		return bgSeq
-	}
-}
-
 // healErasureSet lists and heals all objects in a specific erasure set
 func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, tracker *healingTracker) error {
-	bgSeq := mustGetHealSequence(ctx)
 	scanMode := madmin.HealNormalScan
 
 	// Make sure to copy since `buckets slice`
@@ -173,6 +153,30 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		}
 	}
 
+	info, err := tracker.disk.DiskInfo(ctx, false)
+	if err != nil {
+		return fmt.Errorf("unable to get disk information before healing it: %w", err)
+	}
+
+	var numHealers uint64
+
+	if numCores := uint64(runtime.GOMAXPROCS(0)); info.NRRequests > numCores {
+		numHealers = numCores / 4
+	} else {
+		numHealers = info.NRRequests / 4
+	}
+	if numHealers < 4 {
+		numHealers = 4
+	}
+	// allow overriding this value as well..
+	if v := globalHealConfig.GetWorkers(); v > 0 {
+		numHealers = uint64(v)
+	}
+
+	logger.Info(fmt.Sprintf("Healing drive '%s' - use %d parallel workers.", tracker.disk.String(), numHealers))
+
+	jt, _ := workers.New(int(numHealers))
+
 	var retErr error
 	// Heal all buckets with all objects
 	for _, bucket := range healBuckets {
@@ -267,6 +271,8 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 
 		// Note: updates from healEntry to tracker must be sent on results channel.
 		healEntry := func(bucket string, entry metaCacheEntry) {
+			defer jt.Give()
+
 			if entry.name == "" && len(entry.metadata) == 0 {
 				// ignore entries that don't have metadata.
 				return
@@ -291,14 +297,17 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				}
 			}
 
+			// erasureObjects layer needs object names to be encoded
+			encodedEntryName := encodeDirObject(entry.name)
+
 			var result healEntryResult
 			fivs, err := entry.fileInfoVersions(bucket)
 			if err != nil {
-				err := bgSeq.queueHealTask(healSource{
-					bucket:    bucket,
-					object:    entry.name,
-					versionID: "",
-				}, madmin.HealItemObject)
+				_, err := er.HealObject(ctx, bucket, encodedEntryName, "",
+					madmin.HealOpts{
+						ScanMode: scanMode,
+						Remove:   healDeleteDangling,
+					})
 				if err != nil {
 					if isErrObjectNotFound(err) {
 						// queueing happens across namespace, ignore
@@ -321,11 +330,11 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				if version.ModTime.After(tracker.Started) {
 					continue
 				}
-				if err := bgSeq.queueHealTask(healSource{
-					bucket:    bucket,
-					object:    version.Name,
-					versionID: version.VersionID,
-				}, madmin.HealItemObject); err != nil {
+				if _, err := er.HealObject(ctx, bucket, encodedEntryName,
+					version.VersionID, madmin.HealOpts{
+						ScanMode: scanMode,
+						Remove:   healDeleteDangling,
+					}); err != nil {
 					if isErrObjectNotFound(err) {
 						// queueing happens across namespace, ignore
 						// objects that are not found.
@@ -344,7 +353,6 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 				} else {
 					result = healEntrySuccess(uint64(version.Size))
 				}
-				bgSeq.logHeal(madmin.HealItemObject)
 
 				if !send(result) {
 					return
@@ -382,7 +390,8 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 			minDisks:       1,
 			reportNotFound: false,
 			agreed: func(entry metaCacheEntry) {
-				healEntry(actualBucket, entry)
+				jt.Take()
+				go healEntry(actualBucket, entry)
 			},
 			partial: func(entries metaCacheEntries, _ []error) {
 				entry, ok := entries.resolve(&resolver)
@@ -391,10 +400,12 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 					// proceed to heal nonetheless.
 					entry, _ = entries.firstFound()
 				}
-				healEntry(actualBucket, *entry)
+				jt.Take()
+				go healEntry(actualBucket, *entry)
 			},
 			finished: nil,
 		})
+		jt.Wait() // synchronize all the concurrent heal jobs
 		close(results)
 		if err != nil {
 			// Set this such that when we return this function

cmd/globals.go

@@ -152,6 +152,9 @@ var (
 	// and it is automatically deduced.
 	globalBrowserRedirectURL *xnet.URL
 
+	// Disable redirect, default is enabled.
+	globalBrowserRedirect bool
+
 	// This flag is set to 'true' when MINIO_UPDATE env is set to 'off'. Default is false.
 	globalInplaceUpdateDisabled = false
 
@@ -402,6 +405,9 @@ var (
 
 	// Captures all batch jobs metrics globally
 	globalBatchJobsMetrics batchJobMetrics
+
+	// Indicates if server was started as `--address ":0"`
+	globalDynamicAPIPort bool
 	// Add new variable global values here.
 )
 

cmd/handler-utils.go

@@ -300,7 +300,8 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 		globalHTTPStats.currentS3Requests.Inc(api)
 		defer globalHTTPStats.currentS3Requests.Dec(api)
 
-		if bucket != "" && bucket != minioReservedBucket {
+		_, err = globalBucketMetadataSys.Get(bucket) // check if this bucket exists.
+		if bucket != "" && bucket != minioReservedBucket && err == nil {
 			globalBucketHTTPStats.updateHTTPStats(bucket, api, nil)
 		}
 
@@ -316,7 +317,7 @@ func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc {
 			globalConnStats.incS3InputBytes(int64(tc.RequestRecorder.Size()))
 			globalConnStats.incS3OutputBytes(int64(tc.ResponseRecorder.Size()))
 
-			if bucket != "" && bucket != minioReservedBucket {
+			if bucket != "" && bucket != minioReservedBucket && err == nil {
 				globalBucketConnStats.incS3InputBytes(bucket, int64(tc.RequestRecorder.Size()))
 				globalBucketConnStats.incS3OutputBytes(bucket, int64(tc.ResponseRecorder.Size()))
 				globalBucketHTTPStats.updateHTTPStats(bucket, api, tc.ResponseRecorder)

cmd/iam-object-store.go

@@ -29,6 +29,7 @@ import (
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
@@ -343,6 +344,7 @@ var (
 	policyDBServiceAccountsListKey = "policydb/service-accounts/"
 	policyDBGroupsListKey          = "policydb/groups/"
 
+	// List of directories from which to read iam data into memory.
 	allListKeys = []string{
 		usersListKey,
 		svcAccListKey,
@@ -354,34 +356,60 @@ var (
 		policyDBServiceAccountsListKey,
 		policyDBGroupsListKey,
 	}
+
+	// List of directories to skip: we do not read STS directories for better
+	// performance. STS credentials would be stored in memory when they are
+	// first used.
+	iamLoadSkipListKeySet = set.CreateStringSet(
+		stsListKey,
+		policyDBSTSUsersListKey,
+	)
 )
 
 func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (map[string][]string, error) {
 	res := make(map[string][]string)
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator) {
-		if item.Err != nil {
-			return nil, item.Err
+	for _, listKey := range allListKeys {
+		if iamLoadSkipListKeySet.Contains(listKey) {
+			continue
 		}
-
-		found := false
-		for _, listKey := range allListKeys {
-			if strings.HasPrefix(item.Item, listKey) {
-				found = true
-				name := strings.TrimPrefix(item.Item, listKey)
-				res[listKey] = append(res[listKey], name)
-				break
+		for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator+listKey) {
+			if item.Err != nil {
+				return nil, item.Err
 			}
-		}
-
-		if !found && (item.Item != "format.json") {
-			logger.LogIf(ctx, fmt.Errorf("unknown type of IAM file listed: %v", item.Item))
+			res[listKey] = append(res[listKey], item.Item)
 		}
 	}
 	return res, nil
 }
 
+// PurgeExpiredSTS - purge expired STS credentials from object store.
+func (iamOS *IAMObjectStore) PurgeExpiredSTS(ctx context.Context) error {
+	if iamOS.objAPI == nil {
+		return errServerNotInitialized
+	}
+
+	bootstrapTraceMsg("purging expired STS credentials")
+	// Scan STS users on disk and purge expired ones. We do not need to hold a
+	// lock with store.lock() here.
+	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator+stsListKey) {
+		if item.Err != nil {
+			return item.Err
+		}
+		userName := path.Dir(item.Item)
+		// loadUser() will delete expired user during the load - we do not need
+		// to keep the loaded user around in memory, so we reinitialize the map
+		// each time.
+		m := map[string]UserIdentity{}
+		if err := iamOS.loadUser(ctx, userName, stsUser, m); err != nil && err != errNoSuchUser {
+			logger.LogIf(GlobalContext, fmt.Errorf("unable to load user during STS purge: %w (%s)", err, item.Item))
+		}
+
+	}
+	return nil
+}
+
 // Assumes cache is locked by caller.
 func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache) error {
 	if iamOS.objAPI == nil {
@@ -448,29 +476,37 @@ func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iam
 
 	bootstrapTraceMsg("loading service accounts")
 	svcAccList := listedConfigItems[svcAccListKey]
+	svcUsersMap := make(map[string]UserIdentity, len(svcAccList))
 	for _, item := range svcAccList {
 		userName := path.Dir(item)
-		if err := iamOS.loadUser(ctx, userName, svcUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
+		if err := iamOS.loadUser(ctx, userName, svcUser, svcUsersMap); err != nil && err != errNoSuchUser {
 			return fmt.Errorf("unable to load the service account `%s`: %w", userName, err)
 		}
 	}
-
-	bootstrapTraceMsg("loading STS users")
-	stsUsersList := listedConfigItems[stsListKey]
-	for _, item := range stsUsersList {
-		userName := path.Dir(item)
-		if err := iamOS.loadUser(ctx, userName, stsUser, cache.iamUsersMap); err != nil && err != errNoSuchUser {
-			return fmt.Errorf("unable to load the STS user `%s`: %w", userName, err)
+	for _, svcAcc := range svcUsersMap {
+		svcParent := svcAcc.Credentials.ParentUser
+		if _, ok := cache.iamUsersMap[svcParent]; !ok {
+			// If a service account's parent user is not in iamUsersMap, the
+			// parent is an STS account. Such accounts may have a policy mapped
+			// on the parent user, so we load them. This is not needed for the
+			// initial server startup, however, it is needed for the case where
+			// the STS account's policy mapping (for example in LDAP mode) may
+			// be changed and the user's policy mapping in memory is stale
+			// (because the policy change notification was missed by the current
+			// server).
+			//
+			// The "policy not found" error is ignored because the STS account may
+			// not have a policy mapped via its parent (for e.g. in
+			// OIDC/AssumeRoleWithCustomToken/AssumeRoleWithCertificate).
+			err := iamOS.loadMappedPolicy(ctx, svcParent, stsUser, false, cache.iamSTSPolicyMap)
+			if err != nil && !errors.Is(err, errNoSuchPolicy) {
+				return fmt.Errorf("unable to load the policy mapping for the STS user `%s`: %w", svcParent, err)
+			}
 		}
 	}
-
-	bootstrapTraceMsg("loading STS policy mapping")
-	stsPolicyMappingsList := listedConfigItems[policyDBSTSUsersListKey]
-	for _, item := range stsPolicyMappingsList {
-		stsName := strings.TrimSuffix(item, ".json")
-		if err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, cache.iamUserPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return fmt.Errorf("unable to load the policy mapping for the STS user `%s`: %w", stsName, err)
-		}
+	// Copy svcUsersMap to cache.iamUsersMap
+	for k, v := range svcUsersMap {
+		cache.iamUsersMap[k] = v
 	}
 
 	cache.buildUserGroupMemberships()

cmd/iam-store.go

@@ -34,7 +34,7 @@ import (
 	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 const (
@@ -212,12 +212,12 @@ func newMappedPolicy(policy string) MappedPolicy {
 // PolicyDoc represents an IAM policy with some metadata.
 type PolicyDoc struct {
 	Version    int `json:",omitempty"`
-	Policy     iampolicy.Policy
+	Policy     policy.Policy
 	CreateDate time.Time `json:",omitempty"`
 	UpdateDate time.Time `json:",omitempty"`
 }
 
-func newPolicyDoc(p iampolicy.Policy) PolicyDoc {
+func newPolicyDoc(p policy.Policy) PolicyDoc {
 	now := UTCNow().Round(time.Millisecond)
 	return PolicyDoc{
 		Version:    1,
@@ -228,14 +228,14 @@ func newPolicyDoc(p iampolicy.Policy) PolicyDoc {
 }
 
 // defaultPolicyDoc - used to wrap a default policy as PolicyDoc.
-func defaultPolicyDoc(p iampolicy.Policy) PolicyDoc {
+func defaultPolicyDoc(p policy.Policy) PolicyDoc {
 	return PolicyDoc{
 		Version: 1,
 		Policy:  p,
 	}
 }
 
-func (d *PolicyDoc) update(p iampolicy.Policy) {
+func (d *PolicyDoc) update(p policy.Policy) {
 	now := UTCNow().Round(time.Millisecond)
 	d.UpdateDate = now
 	if d.CreateDate.IsZero() {
@@ -248,7 +248,7 @@ func (d *PolicyDoc) update(p iampolicy.Policy) {
 // definitions.
 //
 // The on-disk format of policy definitions has changed (around early 12/2021)
-// from iampolicy.Policy to PolicyDoc. To avoid a migration, loading supports
+// from policy.Policy to PolicyDoc. To avoid a migration, loading supports
 // both the old and the new formats.
 func (d *PolicyDoc) parseJSON(data []byte) error {
 	json := jsoniter.ConfigCompatibleWithStandardLibrary
@@ -283,14 +283,22 @@ type iamCache struct {
 
 	// map of policy names to policy definitions
 	iamPolicyDocsMap map[string]PolicyDoc
-	// map of usernames to credentials
+
+	// map of regular username to credentials
 	iamUsersMap map[string]UserIdentity
+	// map of regular username to policy names
+	iamUserPolicyMap map[string]MappedPolicy
+
+	// STS accounts are loaded on demand and not via the periodic IAM reload.
+	// map of STS access key to credentials
+	iamSTSAccountsMap map[string]UserIdentity
+	// map of STS access key to policy names
+	iamSTSPolicyMap map[string]MappedPolicy
+
 	// map of group names to group info
 	iamGroupsMap map[string]GroupInfo
 	// map of user names to groups they are a member of
 	iamUserGroupMemberships map[string]set.StringSet
-	// map of usernames/temporary access keys to policy names
-	iamUserPolicyMap map[string]MappedPolicy
 	// map of group names to policy names
 	iamGroupPolicyMap map[string]MappedPolicy
 }
@@ -299,9 +307,11 @@ func newIamCache() *iamCache {
 	return &iamCache{
 		iamPolicyDocsMap:        map[string]PolicyDoc{},
 		iamUsersMap:             map[string]UserIdentity{},
+		iamUserPolicyMap:        map[string]MappedPolicy{},
+		iamSTSAccountsMap:       map[string]UserIdentity{},
+		iamSTSPolicyMap:         map[string]MappedPolicy{},
 		iamGroupsMap:            map[string]GroupInfo{},
 		iamUserGroupMemberships: map[string]set.StringSet{},
-		iamUserPolicyMap:        map[string]MappedPolicy{},
 		iamGroupPolicyMap:       map[string]MappedPolicy{},
 	}
 }
@@ -354,9 +364,9 @@ func (c *iamCache) removeGroupFromMembershipsMap(group string) {
 // information in IAM (i.e sys.iam*Map) - this info is stored only in the STS
 // generated credentials. Thus we skip looking up group memberships, user map,
 // and group map and check the appropriate policy maps directly.
-func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]string, time.Time, error) {
+func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool) ([]string, time.Time, error) {
 	if isGroup {
-		if mode == MinIOUsersSysType {
+		if store.getUsersSysType() == MinIOUsersSysType {
 			g, ok := c.iamGroupsMap[name]
 			if !ok {
 				return nil, time.Time{}, errNoSuchGroup
@@ -381,7 +391,20 @@ func (c *iamCache) policyDBGet(mode UsersSysType, name string, isGroup bool) ([]
 		}
 	}
 
-	mp := c.iamUserPolicyMap[name]
+	// For internal IDP regular/service account user accounts, the policy
+	// mapping is iamUserPolicyMap. For STS accounts, the parent user would be
+	// passed here and we lookup the mapping in iamSTSPolicyMap.
+	mp, ok := c.iamUserPolicyMap[name]
+	if !ok {
+		// Since user "name" could be a parent user of an STS account, we lookup
+		// mappings for those too.
+		mp, ok = c.iamSTSPolicyMap[name]
+		if !ok {
+			// Attempt to load parent user mapping for STS accounts
+			store.loadMappedPolicy(context.TODO(), name, stsUser, false, c.iamSTSPolicyMap)
+			mp = c.iamSTSPolicyMap[name]
+		}
+	}
 
 	// returned policy could be empty
 	policies := mp.toSlice()
@@ -407,7 +430,11 @@ func (c *iamCache) updateUserWithClaims(key string, u UserIdentity) error {
 		}
 		u.Credentials.Claims = jwtClaims.Map()
 	}
-	c.iamUsersMap[key] = u
+	if !u.Credentials.IsTemp() {
+		c.iamUsersMap[key] = u
+	} else {
+		c.iamSTSAccountsMap[key] = u
+	}
 	c.updatedAt = time.Now()
 	return nil
 }
@@ -452,13 +479,23 @@ type iamStorageWatcher interface {
 
 // Set default canned policies only if not already overridden by users.
 func setDefaultCannedPolicies(policies map[string]PolicyDoc) {
-	for _, v := range iampolicy.DefaultPolicies {
+	for _, v := range policy.DefaultPolicies {
 		if _, ok := policies[v.Name]; !ok {
 			policies[v.Name] = defaultPolicyDoc(v.Definition)
 		}
 	}
 }
 
+// PurgeExpiredSTS - purges expired STS credentials.
+func (store *IAMStoreSys) PurgeExpiredSTS(ctx context.Context) error {
+	iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore)
+	if !ok {
+		// No purging is done for non-object storage.
+		return nil
+	}
+	return iamOS.PurgeExpiredSTS(ctx)
+}
+
 // LoadIAMCache reads all IAM items and populates a new iamCache object and
 // replaces the in-memory cache object.
 func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
@@ -512,18 +549,6 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
 			return err
 		}
 
-		bootstrapTraceMsg("loading STS users")
-		// load STS temp users
-		if err := store.loadUsers(ctx, stsUser, newCache.iamUsersMap); err != nil {
-			return err
-		}
-
-		bootstrapTraceMsg("loading STS policy mapping")
-		// load STS policy mappings
-		if err := store.loadMappedPolicies(ctx, stsUser, false, newCache.iamUserPolicyMap); err != nil {
-			return err
-		}
-
 		newCache.buildUserGroupMemberships()
 	}
 
@@ -535,9 +560,9 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
 	// were changes to the in-memory cache we should wait for the next
 	// cycle until we can safely update the in-memory cache.
 	//
-	// An in-memory cache must be replaced only if we know for sure that
-	// the values loaded from disk are not stale. They might be stale
-	// if the cached.updatedAt is recent than the refresh cycle began.
+	// An in-memory cache must be replaced only if we know for sure that the
+	// values loaded from disk are not stale. They might be stale if the
+	// cached.updatedAt is more recent than the refresh cycle began.
 	if cache.updatedAt.Before(loadedAt) {
 		// No one has updated anything since the config was loaded,
 		// so we just replace whatever is on the disk into memory.
@@ -547,6 +572,15 @@ func (store *IAMStoreSys) LoadIAMCache(ctx context.Context) error {
 		cache.iamUserGroupMemberships = newCache.iamUserGroupMemberships
 		cache.iamUserPolicyMap = newCache.iamUserPolicyMap
 		cache.iamUsersMap = newCache.iamUsersMap
+		// For STS policy map, we need to merge the new cache with the existing
+		// cache because the periodic IAM reload is partial. The periodic load
+		// here is to account for STS policy mapping changes that should apply
+		// for service accounts derived from such STS accounts (i.e. LDAP STS
+		// accounts).
+		for k, v := range newCache.iamSTSPolicyMap {
+			cache.iamSTSPolicyMap[k] = v
+		}
+
 		cache.updatedAt = time.Now()
 	}
 
@@ -571,6 +605,10 @@ func (store *IAMStoreSys) GetUser(user string) (UserIdentity, bool) {
 	defer store.runlock()
 
 	u, ok := cache.iamUsersMap[user]
+	if !ok {
+		// Check the sts map
+		u, ok = cache.iamSTSAccountsMap[user]
+	}
 	return u, ok
 }
 
@@ -635,14 +673,14 @@ func (store *IAMStoreSys) PolicyDBGet(name string, isGroup bool, groups ...strin
 	cache := store.rlock()
 	defer store.runlock()
 
-	policies, _, err := cache.policyDBGet(store.getUsersSysType(), name, isGroup)
+	policies, _, err := cache.policyDBGet(store, name, isGroup)
 	if err != nil {
 		return nil, err
 	}
 
 	if !isGroup {
 		for _, group := range groups {
-			ps, _, err := cache.policyDBGet(store.getUsersSysType(), group, true)
+			ps, _, err := cache.policyDBGet(store, group, true)
 			if err != nil {
 				return nil, err
 			}
@@ -828,7 +866,7 @@ func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc
 	cache := store.rlock()
 	defer store.runlock()
 
-	ps, updatedAt, err := cache.policyDBGet(store.getUsersSysType(), group, true)
+	ps, updatedAt, err := cache.policyDBGet(store, group, true)
 	if err != nil {
 		return gd, err
 	}
@@ -928,7 +966,16 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 	// Load existing policy mapping
 	var mp MappedPolicy
 	if !isGroup {
-		mp = cache.iamUserPolicyMap[name]
+		if userType == stsUser {
+			stsMap := map[string]MappedPolicy{}
+
+			// Attempt to load parent user mapping for STS accounts
+			store.loadMappedPolicy(context.TODO(), name, stsUser, false, stsMap)
+
+			mp = stsMap[name]
+		} else {
+			mp = cache.iamUserPolicyMap[name]
+		}
 	} else {
 		if store.getUsersSysType() == MinIOUsersSysType {
 			g, ok := cache.iamGroupsMap[name]
@@ -981,7 +1028,11 @@ func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGro
 		return
 	}
 	if !isGroup {
-		cache.iamUserPolicyMap[name] = newPolicyMapping
+		if userType == stsUser {
+			cache.iamSTSPolicyMap[name] = newPolicyMapping
+		} else {
+			cache.iamUserPolicyMap[name] = newPolicyMapping
+		}
 	} else {
 		cache.iamGroupPolicyMap[name] = newPolicyMapping
 	}
@@ -1015,7 +1066,11 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 			return updatedAt, err
 		}
 		if !isGroup {
-			delete(cache.iamUserPolicyMap, name)
+			if userType == stsUser {
+				delete(cache.iamSTSPolicyMap, name)
+			} else {
+				delete(cache.iamUserPolicyMap, name)
+			}
 		} else {
 			delete(cache.iamGroupPolicyMap, name)
 		}
@@ -1035,7 +1090,11 @@ func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string,
 		return updatedAt, err
 	}
 	if !isGroup {
-		cache.iamUserPolicyMap[name] = mp
+		if userType == stsUser {
+			cache.iamSTSPolicyMap[name] = mp
+		} else {
+			cache.iamUserPolicyMap[name] = mp
+		}
 	} else {
 		cache.iamGroupPolicyMap[name] = mp
 	}
@@ -1104,7 +1163,9 @@ func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string) error
 	cache := store.lock()
 	defer store.unlock()
 
-	// Check if policy is mapped to any existing user or group.
+	// Check if policy is mapped to any existing user or group. If so, we do not
+	// allow deletion of the policy. If the policy is mapped to an STS account,
+	// we do allow deletion.
 	users := []string{}
 	groups := []string{}
 	for u, mp := range cache.iamUserPolicyMap {
@@ -1148,16 +1209,16 @@ func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string) error
 
 // GetPolicy - gets the policy definition. Allows specifying multiple comma
 // separated policies - returns a combined policy.
-func (store *IAMStoreSys) GetPolicy(name string) (iampolicy.Policy, error) {
+func (store *IAMStoreSys) GetPolicy(name string) (policy.Policy, error) {
 	if name == "" {
-		return iampolicy.Policy{}, errInvalidArgument
+		return policy.Policy{}, errInvalidArgument
 	}
 
 	cache := store.rlock()
 	defer store.runlock()
 
 	policies := newMappedPolicy(name).toSlice()
-	var toMerge []iampolicy.Policy
+	var toMerge []policy.Policy
 	for _, policy := range policies {
 		if policy == "" {
 			continue
@@ -1168,7 +1229,7 @@ func (store *IAMStoreSys) GetPolicy(name string) (iampolicy.Policy, error) {
 		}
 		toMerge = append(toMerge, v.Policy)
 	}
-	return iampolicy.MergePolicies(toMerge...), nil
+	return policy.MergePolicies(toMerge...), nil
 }
 
 // GetPolicyDoc - gets the policy doc which has the policy and some metadata.
@@ -1190,7 +1251,7 @@ func (store *IAMStoreSys) GetPolicyDoc(name string) (r PolicyDoc, err error) {
 }
 
 // SetPolicy - creates a policy with name.
-func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iampolicy.Policy) (time.Time, error) {
+func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy policy.Policy) (time.Time, error) {
 	if policy.IsEmpty() || name == "" {
 		return time.Time{}, errInvalidArgument
 	}
@@ -1220,7 +1281,7 @@ func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iam
 
 // ListPolicies - fetches all policies from storage and updates cache as well.
 // If bucketName is non-empty, returns policies matching the bucket.
-func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (map[string]iampolicy.Policy, error) {
+func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (map[string]policy.Policy, error) {
 	cache := store.lock()
 	defer store.unlock()
 
@@ -1236,7 +1297,7 @@ func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (
 	cache.iamPolicyDocsMap = m
 	cache.updatedAt = time.Now()
 
-	ret := map[string]iampolicy.Policy{}
+	ret := map[string]policy.Policy{}
 	for k, v := range m {
 		if bucketName == "" || v.Policy.MatchResource(bucketName) {
 			ret[k] = v.Policy
@@ -1289,10 +1350,10 @@ func (store *IAMStoreSys) listPolicyDocs(ctx context.Context, bucketName string)
 }
 
 // helper function - does not take locks.
-func filterPolicies(cache *iamCache, policyName string, bucketName string) (string, iampolicy.Policy) {
+func filterPolicies(cache *iamCache, policyName string, bucketName string) (string, policy.Policy) {
 	var policies []string
 	mp := newMappedPolicy(policyName)
-	var toMerge []iampolicy.Policy
+	var toMerge []policy.Policy
 	for _, policy := range mp.toSlice() {
 		if policy == "" {
 			continue
@@ -1306,7 +1367,7 @@ func filterPolicies(cache *iamCache, policyName string, bucketName string) (stri
 			toMerge = append(toMerge, p.Policy)
 		}
 	}
-	return strings.Join(policies, ","), iampolicy.MergePolicies(toMerge...)
+	return strings.Join(policies, ","), policy.MergePolicies(toMerge...)
 }
 
 // FilterPolicies - accepts a comma separated list of policy names as a string
@@ -1314,7 +1375,7 @@ func filterPolicies(cache *iamCache, policyName string, bucketName string) (stri
 // bucketName is non-empty, additionally filters policies matching the bucket.
 // The first returned value is the list of currently existing policies, and the
 // second is their combined policy definition.
-func (store *IAMStoreSys) FilterPolicies(policyName string, bucketName string) (string, iampolicy.Policy) {
+func (store *IAMStoreSys) FilterPolicies(policyName string, bucketName string) (string, policy.Policy) {
 	cache := store.rlock()
 	defer store.runlock()
 
@@ -1403,6 +1464,9 @@ func (store *IAMStoreSys) GetUsersWithMappedPolicies() map[string]string {
 	for k, v := range cache.iamUserPolicyMap {
 		result[k] = v.Policies
 	}
+	for k, v := range cache.iamSTSPolicyMap {
+		result[k] = v.Policies
+	}
 	return result
 }
 
@@ -1425,10 +1489,25 @@ func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error
 				break
 			}
 		}
+		for _, v := range cache.iamSTSAccountsMap {
+			if v.Credentials.ParentUser == name {
+				groups = v.Credentials.Groups
+				break
+			}
+		}
 		mappedPolicy, ok := cache.iamUserPolicyMap[name]
 		if !ok {
-			return u, errNoSuchUser
+			mappedPolicy, ok = cache.iamSTSPolicyMap[name]
 		}
+		if !ok {
+			// Attempt to load parent user mapping for STS accounts
+			store.loadMappedPolicy(context.TODO(), name, stsUser, false, cache.iamSTSPolicyMap)
+			mappedPolicy, ok = cache.iamSTSPolicyMap[name]
+			if !ok {
+				return u, errNoSuchUser
+			}
+		}
+
 		return madmin.UserInfo{
 			PolicyName: mappedPolicy.Policies,
 			MemberOf:   groups,
@@ -1467,8 +1546,11 @@ func (store *IAMStoreSys) PolicyMappingNotificationHandler(ctx context.Context,
 	cache := store.lock()
 	defer store.unlock()
 
-	m := cache.iamGroupPolicyMap
-	if !isGroup {
+	var m map[string]MappedPolicy
+	switch {
+	case isGroup:
+		m = cache.iamGroupPolicyMap
+	default:
 		m = cache.iamUserPolicyMap
 	}
 	err := store.loadMappedPolicy(ctx, userOrGroup, userType, isGroup, m)
@@ -1493,10 +1575,23 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 	cache := store.lock()
 	defer store.unlock()
 
-	err := store.loadUser(ctx, accessKey, userType, cache.iamUsersMap)
+	var m map[string]UserIdentity
+	switch userType {
+	case stsUser:
+		m = cache.iamSTSAccountsMap
+	default:
+		m = cache.iamUsersMap
+	}
+	err := store.loadUser(ctx, accessKey, userType, m)
+
 	if err == errNoSuchUser {
 		// User was deleted - we update the cache.
-		delete(cache.iamUsersMap, accessKey)
+		delete(m, accessKey)
+
+		// Since cache was updated, we update the timestamp.
+		defer func() {
+			cache.updatedAt = time.Now()
+		}()
 
 		// 1. Start with updating user-group memberships
 		if store.getUsersSysType() == MinIOUsersSysType {
@@ -1514,12 +1609,14 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 
 		// 2. Remove any derived credentials from memory
 		if userType == regUser {
-			for _, u := range cache.iamUsersMap {
+			for k, u := range cache.iamUsersMap {
 				if u.Credentials.IsServiceAccount() && u.Credentials.ParentUser == accessKey {
-					delete(cache.iamUsersMap, u.Credentials.AccessKey)
+					delete(cache.iamUsersMap, k)
 				}
-				if u.Credentials.IsTemp() && u.Credentials.ParentUser == accessKey {
-					delete(cache.iamUsersMap, u.Credentials.AccessKey)
+			}
+			for k, u := range cache.iamSTSAccountsMap {
+				if u.Credentials.ParentUser == accessKey {
+					delete(cache.iamSTSAccountsMap, k)
 				}
 			}
 		}
@@ -1527,7 +1624,6 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 		// 3. Delete any mapped policy
 		delete(cache.iamUserPolicyMap, accessKey)
 
-		cache.updatedAt = time.Now()
 		return nil
 	}
 
@@ -1535,31 +1631,43 @@ func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey
 		return err
 	}
 
-	if userType != svcUser {
-		err = store.loadMappedPolicy(ctx, accessKey, userType, false, cache.iamUserPolicyMap)
-		// Ignore policy not mapped error
-		if err != nil && !errors.Is(err, errNoSuchPolicy) {
-			return err
-		}
-	}
+	// Since cache was updated, we update the timestamp.
+	defer func() {
+		cache.updatedAt = time.Now()
+	}()
 
-	// We are on purpose not persisting the policy map for parent
-	// user, although this is a hack, it is a good enough hack
-	// at this point in time - we need to overhaul our OIDC
-	// usage with service accounts with a more cleaner implementation
-	//
-	// This mapping is necessary to ensure that valid credentials
-	// have necessary ParentUser present - this is mainly for only
-	// webIdentity based STS tokens.
-	u, ok := cache.iamUsersMap[accessKey]
-	if ok {
-		cred := u.Credentials
-		if cred.IsTemp() && cred.ParentUser != "" && cred.ParentUser != globalActiveCred.AccessKey {
-			if _, ok := cache.iamUserPolicyMap[cred.ParentUser]; !ok {
-				cache.iamUserPolicyMap[cred.ParentUser] = cache.iamUserPolicyMap[accessKey]
-				cache.updatedAt = time.Now()
-			}
+	cred := m[accessKey].Credentials
+	switch userType {
+	case stsUser:
+		// For STS accounts a policy is mapped to the parent user (if a mapping exists).
+		err = store.loadMappedPolicy(ctx, cred.ParentUser, userType, false, cache.iamSTSPolicyMap)
+	case svcUser:
+		// For service accounts, the parent may be a regular (internal) IDP
+		// user or a "virtual" user (parent of an STS account).
+		//
+		// If parent is a regular user => policy mapping is done on that parent itself.
+		//
+		// If parent is "virtual" => policy mapping is done on the virtual
+		// parent and that virtual parent is an stsUser.
+		//
+		// To load the appropriate mapping, we check the parent user type.
+		_, parentIsRegularUser := cache.iamUsersMap[cred.ParentUser]
+		if parentIsRegularUser {
+			err = store.loadMappedPolicy(ctx, cred.ParentUser, regUser, false, cache.iamUserPolicyMap)
+		} else {
+			err = store.loadMappedPolicy(ctx, cred.ParentUser, stsUser, false, cache.iamSTSPolicyMap)
 		}
+	case regUser:
+		// For regular users, we load the mapped policy.
+		err = store.loadMappedPolicy(ctx, accessKey, userType, false, cache.iamUserPolicyMap)
+	default:
+		// This is just to ensure that we have covered all cases for new
+		// code in future.
+		panic("unknown user type")
+	}
+	// Ignore policy not mapped error
+	if err != nil && !errors.Is(err, errNoSuchPolicy) {
+		return err
 	}
 
 	return nil
@@ -1648,7 +1756,7 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
 			return time.Time{}, err
 		}
 
-		cache.iamUserPolicyMap[cred.ParentUser] = mp
+		cache.iamSTSPolicyMap[cred.ParentUser] = mp
 	}
 
 	u := newUserIdentity(cred)
@@ -1657,8 +1765,7 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
 		return time.Time{}, err
 	}
 
-	cache.iamUsersMap[accessKey] = u
-
+	cache.iamSTSAccountsMap[accessKey] = u
 	cache.updatedAt = time.Now()
 
 	return u.UpdatedAt, nil
@@ -1800,6 +1907,25 @@ func (store *IAMStoreSys) listUserPolicyMappings(cache *iamCache, users []string
 		})
 	}
 
+	stsMap := map[string]MappedPolicy{}
+	for _, user := range users {
+		// Attempt to load parent user mapping for STS accounts
+		store.loadMappedPolicy(context.TODO(), user, stsUser, false, stsMap)
+	}
+
+	for user, mappedPolicy := range stsMap {
+		if userPredicate != nil && !userPredicate(user) {
+			continue
+		}
+
+		ps := mappedPolicy.toSlice()
+		sort.Strings(ps)
+		r = append(r, madmin.UserPolicyEntities{
+			User:     user,
+			Policies: ps,
+		})
+	}
+
 	sort.Slice(r, func(i, j int) bool {
 		return r[i].User < r[j].User
 	})
@@ -1864,6 +1990,32 @@ func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, policies []string,
 		}
 	}
 
+	if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok {
+		for item := range listIAMConfigItems(context.Background(), iamOS.objAPI, iamConfigPrefix+SlashSeparator+policyDBSTSUsersListKey) {
+			user := strings.TrimSuffix(item.Item, ".json")
+			if userPredicate != nil && !userPredicate(user) {
+				continue
+			}
+
+			var mappedPolicy MappedPolicy
+			store.loadIAMConfig(context.Background(), &mappedPolicy, getMappedPolicyPath(user, stsUser, false))
+
+			commonPolicySet := mappedPolicy.policySet()
+			if !queryPolSet.IsEmpty() {
+				commonPolicySet = commonPolicySet.Intersection(queryPolSet)
+			}
+			for _, policy := range commonPolicySet.ToSlice() {
+				s, ok := policyToUsersMap[policy]
+				if !ok {
+					policyToUsersMap[policy] = set.CreateStringSet(user)
+				} else {
+					s.Add(user)
+					policyToUsersMap[policy] = s
+				}
+			}
+		}
+	}
+
 	policyToGroupsMap := make(map[string]set.StringSet)
 	for group, mappedPolicy := range cache.iamGroupPolicyMap {
 		if groupPredicate != nil && !groupPredicate(group) {
@@ -2076,8 +2228,8 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 
 	// sessionPolicy is nil and there is embedded policy attached we remove
 	// embedded policy at that point.
-	if _, ok := m[iampolicy.SessionPolicyName]; ok && opts.sessionPolicy == nil {
-		delete(m, iampolicy.SessionPolicyName)
+	if _, ok := m[policy.SessionPolicyName]; ok && opts.sessionPolicy == nil {
+		delete(m, policy.SessionPolicyName)
 		m[iamPolicyClaimNameSA()] = inheritedPolicyType
 	}
 
@@ -2096,7 +2248,7 @@ func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey st
 		}
 
 		// Overwrite session policy claims.
-		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
+		m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
 		m[iamPolicyClaimNameSA()] = embeddedPolicyType
 	}
 
@@ -2155,19 +2307,10 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 	cache := store.rlock()
 	defer store.runlock()
 
-	userExists := false
 	var serviceAccounts []auth.Credentials
 	for _, u := range cache.iamUsersMap {
-		isDerived := false
 		v := u.Credentials
-		if v.IsServiceAccount() || v.IsTemp() {
-			isDerived = true
-		}
-
-		if !isDerived && v.AccessKey == accessKey {
-			userExists = true
-		} else if isDerived && v.ParentUser == accessKey {
-			userExists = true
+		if accessKey != "" && v.ParentUser == accessKey {
 			if v.IsServiceAccount() {
 				// Hide secret key & session key here
 				v.SecretKey = ""
@@ -2177,12 +2320,6 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 		}
 	}
 
-	// If root user has no STS/Service Accounts, userExists would be false here,
-	// so we handle this exception.
-	if !userExists && globalActiveCred.AccessKey != accessKey {
-		return nil, errNoSuchUser
-	}
-
 	return serviceAccounts, nil
 }
 
@@ -2251,10 +2388,20 @@ func (store *IAMStoreSys) GetSTSAndServiceAccounts() []auth.Credentials {
 	var res []auth.Credentials
 	for _, u := range cache.iamUsersMap {
 		cred := u.Credentials
-		if cred.IsTemp() || cred.IsServiceAccount() {
+		if cred.IsTemp() {
+			panic("unexpected STS credential found in iamUsersMap")
+		}
+		if cred.IsServiceAccount() {
 			res = append(res, cred)
 		}
 	}
+	for _, u := range cache.iamSTSAccountsMap {
+		if !u.Credentials.IsTemp() {
+			panic("unexpected non STS credential found in iamSTSAccountsMap")
+		}
+		res = append(res, u.Credentials)
+	}
+
 	return res
 }
 
@@ -2289,35 +2436,56 @@ func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) {
 	cache.updatedAt = time.Now()
 
 	_, found := cache.iamUsersMap[accessKey]
+
+	// Check for regular user access key
 	if !found {
 		store.loadUser(ctx, accessKey, regUser, cache.iamUsersMap)
 		if _, found = cache.iamUsersMap[accessKey]; found {
 			// load mapped policies
 			store.loadMappedPolicy(ctx, accessKey, regUser, false, cache.iamUserPolicyMap)
-		} else {
-			// check for service account
-			store.loadUser(ctx, accessKey, svcUser, cache.iamUsersMap)
-			if svc, found := cache.iamUsersMap[accessKey]; found {
-				// Load parent user and mapped policies.
-				if store.getUsersSysType() == MinIOUsersSysType {
-					store.loadUser(ctx, svc.Credentials.ParentUser, regUser, cache.iamUsersMap)
-				}
+		}
+	}
+
+	// Check for service account
+	if !found {
+		store.loadUser(ctx, accessKey, svcUser, cache.iamUsersMap)
+		if svc, found := cache.iamUsersMap[accessKey]; found {
+			// Load parent user and mapped policies.
+			if store.getUsersSysType() == MinIOUsersSysType {
+				store.loadUser(ctx, svc.Credentials.ParentUser, regUser, cache.iamUsersMap)
 				store.loadMappedPolicy(ctx, svc.Credentials.ParentUser, regUser, false, cache.iamUserPolicyMap)
 			} else {
-				// check for STS account
-				store.loadUser(ctx, accessKey, stsUser, cache.iamUsersMap)
-				if _, found = cache.iamUsersMap[accessKey]; found {
-					// Load mapped policy
-					store.loadMappedPolicy(ctx, accessKey, stsUser, false, cache.iamUserPolicyMap)
-				}
+				// In case of LDAP the parent user's policy mapping needs to be
+				// loaded into sts map
+				store.loadMappedPolicy(ctx, svc.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap)
 			}
 		}
 	}
 
+	// Check for STS account
+	stsAccountFound := false
+	var stsUserCred UserIdentity
+	if !found {
+		store.loadUser(ctx, accessKey, stsUser, cache.iamSTSAccountsMap)
+		if stsUserCred, found = cache.iamSTSAccountsMap[accessKey]; found {
+			// Load mapped policy
+			store.loadMappedPolicy(ctx, stsUserCred.Credentials.ParentUser, stsUser, false, cache.iamSTSPolicyMap)
+			stsAccountFound = true
+		}
+	}
+
 	// Load any associated policy definitions
-	for _, policy := range cache.iamUserPolicyMap[accessKey].toSlice() {
-		if _, found = cache.iamPolicyDocsMap[policy]; !found {
-			store.loadPolicyDoc(ctx, policy, cache.iamPolicyDocsMap)
+	if !stsAccountFound {
+		for _, policy := range cache.iamUserPolicyMap[accessKey].toSlice() {
+			if _, found = cache.iamPolicyDocsMap[policy]; !found {
+				store.loadPolicyDoc(ctx, policy, cache.iamPolicyDocsMap)
+			}
+		}
+	} else {
+		for _, policy := range cache.iamSTSPolicyMap[stsUserCred.Credentials.AccessKey].toSlice() {
+			if _, found = cache.iamPolicyDocsMap[policy]; !found {
+				store.loadPolicyDoc(ctx, policy, cache.iamPolicyDocsMap)
+			}
 		}
 	}
 }

cmd/iam.go

@@ -47,7 +47,7 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 	etcd "go.etcd.io/etcd/client/v3"
 )
 
@@ -319,44 +319,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 
 	refreshInterval := sys.iamRefreshInterval
 
-	// Set up polling for expired accounts and credentials purging.
-	switch {
-	case sys.OpenIDConfig.ProviderEnabled():
-		go func() {
-			timer := time.NewTimer(refreshInterval)
-			defer timer.Stop()
-			for {
-				select {
-				case <-timer.C:
-					sys.purgeExpiredCredentialsForExternalSSO(ctx)
-
-					timer.Reset(refreshInterval)
-				case <-ctx.Done():
-					return
-				}
-			}
-		}()
-	case sys.LDAPConfig.Enabled():
-		go func() {
-			timer := time.NewTimer(refreshInterval)
-			defer timer.Stop()
-
-			for {
-				select {
-				case <-timer.C:
-					sys.purgeExpiredCredentialsForLDAP(ctx)
-					sys.updateGroupMembershipsForLDAP(ctx)
-
-					timer.Reset(refreshInterval)
-				case <-ctx.Done():
-					return
-				}
-			}
-		}()
-	}
-
-	// Start watching changes to storage.
-	go sys.watch(ctx)
+	go sys.periodicRoutines(ctx, refreshInterval)
 
 	// Load RoleARNs
 	sys.rolesMap = make(map[arn.ARN]string)
@@ -377,6 +340,79 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 	bootstrapTraceMsg("finishing IAM loading")
 }
 
+func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Duration) {
+	// Watch for IAM config changes for iamStorageWatcher.
+	watcher, isWatcher := sys.store.IAMStorageAPI.(iamStorageWatcher)
+	if isWatcher {
+		go func() {
+			ch := watcher.watch(ctx, iamConfigPrefix)
+			for event := range ch {
+				if err := sys.loadWatchedEvent(ctx, event); err != nil {
+					// we simply log errors
+					logger.LogIf(ctx, fmt.Errorf("Failure in loading watch event: %v", err))
+				}
+			}
+		}()
+	}
+
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	// Add a random interval of up to 20% of the base interval.
+	randInterval := func() time.Duration {
+		return time.Duration(r.Float64() * float64(baseInterval) * 0.2)
+	}
+
+	var maxDurationSecondsForLog float64 = 5
+	timer := time.NewTimer(baseInterval + randInterval())
+	defer timer.Stop()
+
+	for {
+		select {
+		case <-timer.C:
+			// Load all IAM items (except STS creds) periodically.
+			refreshStart := time.Now()
+			if err := sys.Load(ctx, false); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (took %.2fs): %v", time.Since(refreshStart).Seconds(), err))
+			} else {
+				took := time.Since(refreshStart).Seconds()
+				if took > maxDurationSecondsForLog {
+					// Log if we took a lot of time to load.
+					logger.Info("IAM refresh took %.2fs", took)
+				}
+			}
+
+			// The following actions are performed about once in 4 times that
+			// IAM is refreshed:
+			if r.Intn(4) == 0 {
+				// Purge expired STS credentials.
+				purgeStart := time.Now()
+				if err := sys.store.PurgeExpiredSTS(ctx); err != nil {
+					logger.LogIf(ctx, fmt.Errorf("Failure in periodic STS purge for IAM (took %.2fs): %v", time.Since(purgeStart).Seconds(), err))
+				} else {
+					took := time.Since(purgeStart).Seconds()
+					if took > maxDurationSecondsForLog {
+						// Log if we took a lot of time to load.
+						logger.Info("IAM expired STS purge took %.2fs", took)
+					}
+				}
+
+				// Poll and remove accounts for those users who were removed
+				// from LDAP/OpenID.
+				if sys.LDAPConfig.Enabled() {
+					sys.purgeExpiredCredentialsForLDAP(ctx)
+					sys.updateGroupMembershipsForLDAP(ctx)
+				}
+				if sys.OpenIDConfig.ProviderEnabled() {
+					sys.purgeExpiredCredentialsForExternalSSO(ctx)
+				}
+			}
+
+			timer.Reset(baseInterval + randInterval())
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
 func (sys *IAMSys) validateAndAddRolePolicyMappings(ctx context.Context, m map[arn.ARN]string) {
 	// Validate that policies associated with roles are defined. If
 	// authZ plugin is set, role policies are just claims sent to
@@ -428,45 +464,6 @@ func (sys *IAMSys) HasWatcher() bool {
 	return sys.store.HasWatcher()
 }
 
-func (sys *IAMSys) watch(ctx context.Context) {
-	watcher, ok := sys.store.IAMStorageAPI.(iamStorageWatcher)
-	if ok {
-		ch := watcher.watch(ctx, iamConfigPrefix)
-		for event := range ch {
-			if err := sys.loadWatchedEvent(ctx, event); err != nil {
-				// we simply log errors
-				logger.LogIf(ctx, fmt.Errorf("Failure in loading watch event: %v", err))
-			}
-		}
-		return
-	}
-
-	var maxRefreshDurationSecondsForLog float64 = 10
-
-	// Load all items periodically
-	timer := time.NewTimer(sys.iamRefreshInterval)
-	defer timer.Stop()
-	for {
-		select {
-		case <-timer.C:
-			refreshStart := time.Now()
-			if err := sys.Load(ctx, false); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (took %.2fs): %v", time.Since(refreshStart).Seconds(), err))
-			} else {
-				took := time.Since(refreshStart).Seconds()
-				if took > maxRefreshDurationSecondsForLog {
-					// Log if we took a lot of time to load.
-					logger.Info("IAM refresh took %.2fs", took)
-				}
-			}
-
-			timer.Reset(sys.iamRefreshInterval)
-		case <-ctx.Done():
-			return
-		}
-	}
-}
-
 func (sys *IAMSys) loadWatchedEvent(ctx context.Context, event iamWatchEvent) (err error) {
 	usersPrefix := strings.HasPrefix(event.keyPath, iamConfigUsersPrefix)
 	groupsPrefix := strings.HasPrefix(event.keyPath, iamConfigGroupsPrefix)
@@ -536,7 +533,7 @@ func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPe
 		return errServerNotInitialized
 	}
 
-	for _, v := range iampolicy.DefaultPolicies {
+	for _, v := range policy.DefaultPolicies {
 		if v.Name == policyName {
 			if err := checkConfig(ctx, globalObjectAPI, getPolicyDocPath(policyName)); err != nil && err == errConfigNotFound {
 				return fmt.Errorf("inbuilt policy `%s` not allowed to be deleted", policyName)
@@ -589,7 +586,7 @@ func (sys *IAMSys) InfoPolicy(policyName string) (*madmin.PolicyInfo, error) {
 }
 
 // ListPolicies - lists all canned policies.
-func (sys *IAMSys) ListPolicies(ctx context.Context, bucketName string) (map[string]iampolicy.Policy, error) {
+func (sys *IAMSys) ListPolicies(ctx context.Context, bucketName string) (map[string]policy.Policy, error) {
 	if !sys.Initialized() {
 		return nil, errServerNotInitialized
 	}
@@ -607,7 +604,7 @@ func (sys *IAMSys) ListPolicyDocs(ctx context.Context, bucketName string) (map[s
 }
 
 // SetPolicy - sets a new named policy.
-func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy.Policy) (time.Time, error) {
+func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p policy.Policy) (time.Time, error) {
 	if !sys.Initialized() {
 		return time.Time{}, errServerNotInitialized
 	}
@@ -849,13 +846,20 @@ func (sys *IAMSys) GetUserInfo(ctx context.Context, name string) (u madmin.UserI
 		return u, errServerNotInitialized
 	}
 
+	loadUserCalled := false
 	select {
 	case <-sys.configLoaded:
 	default:
 		sys.store.LoadUser(ctx, name)
+		loadUserCalled = true
 	}
 
-	return sys.store.GetUserInfo(name)
+	userInfo, err := sys.store.GetUserInfo(name)
+	if err == errNoSuchUser && !loadUserCalled {
+		sys.store.LoadUser(ctx, name)
+		userInfo, err = sys.store.GetUserInfo(name)
+	}
+	return userInfo, err
 }
 
 // QueryPolicyEntities - queries policy associations for builtin users/groups/policies.
@@ -915,7 +919,7 @@ func (sys *IAMSys) notifyForServiceAccount(ctx context.Context, accessKey string
 }
 
 type newServiceAccountOpts struct {
-	sessionPolicy              *iampolicy.Policy
+	sessionPolicy              *policy.Policy
 	accessKey                  string
 	secretKey                  string
 	name, description          string
@@ -962,7 +966,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 	m[parentClaim] = parentUser
 
 	if len(policyBuf) > 0 {
-		m[iampolicy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
+		m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf)
 		m[iamPolicyClaimNameSA()] = embeddedPolicyType
 	} else {
 		m[iamPolicyClaimNameSA()] = inheritedPolicyType
@@ -1014,7 +1018,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 }
 
 type updateServiceAccountOpts struct {
-	sessionPolicy     *iampolicy.Policy
+	sessionPolicy     *policy.Policy
 	secretKey         string
 	status            string
 	name, description string
@@ -1065,7 +1069,7 @@ func (sys *IAMSys) ListTempAccounts(ctx context.Context, accessKey string) ([]Us
 }
 
 // GetServiceAccount - wrapper method to get information about a service account
-func (sys *IAMSys) GetServiceAccount(ctx context.Context, accessKey string) (auth.Credentials, *iampolicy.Policy, error) {
+func (sys *IAMSys) GetServiceAccount(ctx context.Context, accessKey string) (auth.Credentials, *policy.Policy, error) {
 	sa, embeddedPolicy, err := sys.getServiceAccount(ctx, accessKey)
 	if err != nil {
 		return auth.Credentials{}, nil, err
@@ -1076,7 +1080,7 @@ func (sys *IAMSys) GetServiceAccount(ctx context.Context, accessKey string) (aut
 	return sa.Credentials, embeddedPolicy, nil
 }
 
-func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (UserIdentity, *iampolicy.Policy, error) {
+func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (UserIdentity, *policy.Policy, error) {
 	sa, jwtClaims, err := sys.getAccountWithClaims(ctx, accessKey)
 	if err != nil {
 		if err == errNoSuchAccount {
@@ -1088,16 +1092,16 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (Use
 		return UserIdentity{}, nil, errNoSuchServiceAccount
 	}
 
-	var embeddedPolicy *iampolicy.Policy
+	var embeddedPolicy *policy.Policy
 
 	pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA())
-	sp, spok := jwtClaims.Lookup(iampolicy.SessionPolicyName)
+	sp, spok := jwtClaims.Lookup(policy.SessionPolicyName)
 	if ptok && spok && pt == embeddedPolicyType {
 		policyBytes, err := base64.StdEncoding.DecodeString(sp)
 		if err != nil {
 			return UserIdentity{}, nil, err
 		}
-		embeddedPolicy, err = iampolicy.ParseConfig(bytes.NewReader(policyBytes))
+		embeddedPolicy, err = policy.ParseConfig(bytes.NewReader(policyBytes))
 		if err != nil {
 			return UserIdentity{}, nil, err
 		}
@@ -1107,7 +1111,7 @@ func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (Use
 }
 
 // GetTemporaryAccount - wrapper method to get information about a temporary account
-func (sys *IAMSys) GetTemporaryAccount(ctx context.Context, accessKey string) (auth.Credentials, *iampolicy.Policy, error) {
+func (sys *IAMSys) GetTemporaryAccount(ctx context.Context, accessKey string) (auth.Credentials, *policy.Policy, error) {
 	tmpAcc, embeddedPolicy, err := sys.getTempAccount(ctx, accessKey)
 	if err != nil {
 		return auth.Credentials{}, nil, err
@@ -1118,7 +1122,7 @@ func (sys *IAMSys) GetTemporaryAccount(ctx context.Context, accessKey string) (a
 	return tmpAcc.Credentials, embeddedPolicy, nil
 }
 
-func (sys *IAMSys) getTempAccount(ctx context.Context, accessKey string) (UserIdentity, *iampolicy.Policy, error) {
+func (sys *IAMSys) getTempAccount(ctx context.Context, accessKey string) (UserIdentity, *policy.Policy, error) {
 	tmpAcc, claims, err := sys.getAccountWithClaims(ctx, accessKey)
 	if err != nil {
 		if err == errNoSuchAccount {
@@ -1130,15 +1134,15 @@ func (sys *IAMSys) getTempAccount(ctx context.Context, accessKey string) (UserId
 		return UserIdentity{}, nil, errNoSuchTempAccount
 	}
 
-	var embeddedPolicy *iampolicy.Policy
+	var embeddedPolicy *policy.Policy
 
-	sp, spok := claims.Lookup(iampolicy.SessionPolicyName)
+	sp, spok := claims.Lookup(policy.SessionPolicyName)
 	if spok {
 		policyBytes, err := base64.StdEncoding.DecodeString(sp)
 		if err != nil {
 			return UserIdentity{}, nil, err
 		}
-		embeddedPolicy, err = iampolicy.ParseConfig(bytes.NewReader(policyBytes))
+		embeddedPolicy, err = policy.ParseConfig(bytes.NewReader(policyBytes))
 		if err != nil {
 			return UserIdentity{}, nil, err
 		}
@@ -1421,31 +1425,24 @@ func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (u UserIdentit
 		return u, false
 	}
 
-	fallback := false
+	if accessKey == globalActiveCred.AccessKey {
+		return newUserIdentity(globalActiveCred), true
+	}
+
+	loadUserCalled := false
 	select {
 	case <-sys.configLoaded:
 	default:
 		sys.store.LoadUser(ctx, accessKey)
-		fallback = true
+		loadUserCalled = true
 	}
 
 	u, ok = sys.store.GetUser(accessKey)
-	if !ok && !fallback {
-		// accessKey not found, also
-		// IAM store is not in fallback mode
-		// we can try to reload again from
-		// the IAM store and see if credential
-		// exists now. If it doesn't proceed to
-		// fail.
+	if !ok && !loadUserCalled {
 		sys.store.LoadUser(ctx, accessKey)
 		u, ok = sys.store.GetUser(accessKey)
 	}
 
-	if !ok {
-		if accessKey == globalActiveCred.AccessKey {
-			return newUserIdentity(globalActiveCred), true
-		}
-	}
 	return u, ok && u.Credentials.IsValid()
 }
 
@@ -1723,11 +1720,11 @@ func (sys *IAMSys) PolicyDBGet(name string, isGroup bool, groups ...string) ([]s
 	return sys.store.PolicyDBGet(name, isGroup, groups...)
 }
 
-const sessionPolicyNameExtracted = iampolicy.SessionPolicyName + "-extracted"
+const sessionPolicyNameExtracted = policy.SessionPolicyName + "-extracted"
 
 // IsAllowedServiceAccount - checks if the given service account is allowed to perform
 // actions. The permission of the parent user is checked first
-func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser string) bool {
+func (sys *IAMSys) IsAllowedServiceAccount(args policy.Args, parentUser string) bool {
 	// Verify if the parent claim matches the parentUser.
 	p, ok := args.Claims[parentClaim]
 	if ok {
@@ -1778,7 +1775,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 		// Finally, if there is no parent policy, check if a policy claim is
 		// present.
 		if len(svcPolicies) == 0 {
-			policySet, _ := iampolicy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID())
+			policySet, _ := policy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID())
 			svcPolicies = policySet.ToSlice()
 		}
 	}
@@ -1788,7 +1785,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 		return false
 	}
 
-	var combinedPolicy iampolicy.Policy
+	var combinedPolicy policy.Policy
 	// Policies were found, evaluate all of them.
 	if !isOwnerDerived {
 		availablePoliciesStr, c := sys.store.FilterPolicies(strings.Join(svcPolicies, ","), "")
@@ -1831,7 +1828,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 	}
 
 	// Check if policy is parseable.
-	subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
+	subPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
 	if err != nil {
 		// Log any error in input session policy config.
 		logger.LogIf(GlobalContext, err)
@@ -1853,7 +1850,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 // IsAllowedSTS is meant for STS based temporary credentials,
 // which implements claims validation and verification other than
 // applying policies.
-func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
+func (sys *IAMSys) IsAllowedSTS(args policy.Args, parentUser string) bool {
 	// 1. Determine mapped policies
 
 	isOwnerDerived := parentUser == globalActiveCred.AccessKey
@@ -1905,7 +1902,7 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 
 	// 2. Combine the mapped policies into a single combined policy.
 
-	var combinedPolicy iampolicy.Policy
+	var combinedPolicy policy.Policy
 	if !isOwnerDerived {
 		var err error
 		combinedPolicy, err = sys.store.GetPolicy(strings.Join(policies, ","))
@@ -1937,7 +1934,7 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 	return isOwnerDerived || combinedPolicy.IsAllowed(args)
 }
 
-func isAllowedBySessionPolicy(args iampolicy.Args) (hasSessionPolicy bool, isAllowed bool) {
+func isAllowedBySessionPolicy(args policy.Args) (hasSessionPolicy bool, isAllowed bool) {
 	hasSessionPolicy = false
 	isAllowed = false
 
@@ -1957,7 +1954,7 @@ func isAllowedBySessionPolicy(args iampolicy.Args) (hasSessionPolicy bool, isAll
 	}
 
 	// Check if policy is parseable.
-	subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
+	subPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
 	if err != nil {
 		// Log any error in input session policy config.
 		logger.LogIf(GlobalContext, err)
@@ -1974,13 +1971,13 @@ func isAllowedBySessionPolicy(args iampolicy.Args) (hasSessionPolicy bool, isAll
 }
 
 // GetCombinedPolicy returns a combined policy combining all policies
-func (sys *IAMSys) GetCombinedPolicy(policies ...string) iampolicy.Policy {
+func (sys *IAMSys) GetCombinedPolicy(policies ...string) policy.Policy {
 	_, policy := sys.store.FilterPolicies(strings.Join(policies, ","), "")
 	return policy
 }
 
 // IsAllowed - checks given policy args is allowed to continue the Rest API.
-func (sys *IAMSys) IsAllowed(args iampolicy.Args) bool {
+func (sys *IAMSys) IsAllowed(args policy.Args) bool {
 	// If opa is configured, use OPA always.
 	if authz := newGlobalAuthZPluginFn(); authz != nil {
 		ok, err := authz.IsAllowed(args)

cmd/jwt.go

@@ -28,7 +28,7 @@ import (
 	"github.com/minio/minio/internal/auth"
 	xjwt "github.com/minio/minio/internal/jwt"
 	"github.com/minio/minio/internal/logger"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 const (
@@ -148,7 +148,7 @@ func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, b
 		}
 
 		// Now check if we have a sessionPolicy.
-		if _, ok = eclaims[iampolicy.SessionPolicyName]; ok {
+		if _, ok = eclaims[policy.SessionPolicyName]; ok {
 			owner = false
 		} else {
 			owner = globalActiveCred.AccessKey == ucred.ParentUser

cmd/kms-handlers.go

@@ -29,14 +29,14 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )
 
 // KMSStatusHandler - GET /minio/kms/v1/status
 func (a kmsAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSStatus")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSStatusAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSStatusAction)
 	if objectAPI == nil {
 		return
 	}
@@ -74,7 +74,7 @@ func (a kmsAPIHandlers) KMSMetricsHandler(w http.ResponseWriter, r *http.Request
 	ctx := newContext(r, w, "KMSMetrics")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSMetricsAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSMetricsAction)
 	if objectAPI == nil {
 		return
 	}
@@ -106,7 +106,7 @@ func (a kmsAPIHandlers) KMSAPIsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "KMSAPIs")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSAPIAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSAPIAction)
 	if objectAPI == nil {
 		return
 	}
@@ -143,7 +143,7 @@ func (a kmsAPIHandlers) KMSVersionHandler(w http.ResponseWriter, r *http.Request
 	ctx := newContext(r, w, "KMSVersion")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSVersionAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSVersionAction)
 	if objectAPI == nil {
 		return
 	}
@@ -183,7 +183,7 @@ func (a kmsAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Reque
 	}
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSCreateKeyAction)
 	if objectAPI == nil {
 		return
 	}
@@ -211,7 +211,7 @@ func (a kmsAPIHandlers) KMSDeleteKeyHandler(w http.ResponseWriter, r *http.Reque
 	ctx := newContext(r, w, "KMSDeleteKey")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSDeleteKeyAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSDeleteKeyAction)
 	if objectAPI == nil {
 		return
 	}
@@ -246,7 +246,7 @@ func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Reques
 	}
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSListKeysAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSListKeysAction)
 	if objectAPI == nil {
 		return
 	}
@@ -297,7 +297,7 @@ func (a kmsAPIHandlers) KMSImportKeyHandler(w http.ResponseWriter, r *http.Reque
 	ctx := newContext(r, w, "KMSImportKey")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSImportKeyAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSImportKeyAction)
 	if objectAPI == nil {
 		return
 	}
@@ -328,7 +328,7 @@ func (a kmsAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Reque
 	ctx := newContext(r, w, "KMSKeyStatus")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAction)
 	if objectAPI == nil {
 		return
 	}
@@ -404,7 +404,7 @@ func (a kmsAPIHandlers) KMSDescribePolicyHandler(w http.ResponseWriter, r *http.
 	ctx := newContext(r, w, "KMSDescribePolicy")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSDescribePolicyAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSDescribePolicyAction)
 	if objectAPI == nil {
 		return
 	}
@@ -440,7 +440,7 @@ func (a kmsAPIHandlers) KMSAssignPolicyHandler(w http.ResponseWriter, r *http.Re
 	ctx := newContext(r, w, "KMSAssignPolicy")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSAssignPolicyAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSAssignPolicyAction)
 	if objectAPI == nil {
 		return
 	}
@@ -472,7 +472,7 @@ func (a kmsAPIHandlers) KMSDeletePolicyHandler(w http.ResponseWriter, r *http.Re
 	ctx := newContext(r, w, "KMSDeletePolicy")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSDeletePolicyAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSDeletePolicyAction)
 	if objectAPI == nil {
 		return
 	}
@@ -499,7 +499,7 @@ func (a kmsAPIHandlers) KMSListPoliciesHandler(w http.ResponseWriter, r *http.Re
 	ctx := newContext(r, w, "KMSListPolicies")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSListPoliciesAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSListPoliciesAction)
 	if objectAPI == nil {
 		return
 	}
@@ -546,7 +546,7 @@ func (a kmsAPIHandlers) KMSGetPolicyHandler(w http.ResponseWriter, r *http.Reque
 	ctx := newContext(r, w, "KMSGetPolicy")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSGetPolicyAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSGetPolicyAction)
 	if objectAPI == nil {
 		return
 	}
@@ -578,7 +578,7 @@ func (a kmsAPIHandlers) KMSDescribeIdentityHandler(w http.ResponseWriter, r *htt
 	ctx := newContext(r, w, "KMSDescribeIdentity")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSDescribeIdentityAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSDescribeIdentityAction)
 	if objectAPI == nil {
 		return
 	}
@@ -619,7 +619,7 @@ func (a kmsAPIHandlers) KMSDescribeSelfIdentityHandler(w http.ResponseWriter, r
 	ctx := newContext(r, w, "KMSDescribeSelfIdentity")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSDescribeSelfIdentityAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSDescribeSelfIdentityAction)
 	if objectAPI == nil {
 		return
 	}
@@ -659,7 +659,7 @@ func (a kmsAPIHandlers) KMSDeleteIdentityHandler(w http.ResponseWriter, r *http.
 	ctx := newContext(r, w, "KMSDeleteIdentity")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSDeleteIdentityAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSDeleteIdentityAction)
 	if objectAPI == nil {
 		return
 	}
@@ -686,7 +686,7 @@ func (a kmsAPIHandlers) KMSListIdentitiesHandler(w http.ResponseWriter, r *http.
 	ctx := newContext(r, w, "KMSListIdentities")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSListIdentitiesAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSListIdentitiesAction)
 	if objectAPI == nil {
 		return
 	}

cmd/listen-notification-handlers.go

@@ -116,7 +116,7 @@ func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r
 
 	// Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	listenCh := make(chan event.Event, 4000)
+	listenCh := make(chan event.Event, globalAPIConfig.getRequestsPoolCapacity()*len(globalEndpoints.Hostnames()))
 
 	peers, _ := newPeerRestClients(globalEndpoints)
 

cmd/metacache-server-pool.go

@@ -38,7 +38,7 @@ func renameAllBucketMetacache(epPath string) error {
 		if typ == os.ModeDir {
 			tmpMetacacheOld := pathutil.Join(epPath, minioMetaTmpDeletedBucket, mustGetUUID())
 			if err := renameAll(pathJoin(epPath, minioMetaBucket, metacachePrefixForID(name, slashSeparator)),
-				tmpMetacacheOld); err != nil && err != errFileNotFound {
+				tmpMetacacheOld, epPath); err != nil && err != errFileNotFound {
 				return fmt.Errorf("unable to rename (%s -> %s) %w",
 					pathJoin(epPath, minioMetaBucket+metacachePrefixForID(minioMetaBucket, slashSeparator)),
 					tmpMetacacheOld,
@@ -60,6 +60,10 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 	if err := checkListObjsArgs(ctx, o.Bucket, o.Prefix, o.Marker, z); err != nil {
 		return entries, err
 	}
+	// Marker points to before the prefix, just ignore it.
+	if o.Marker < o.Prefix {
+		o.Marker = ""
+	}
 
 	// Marker is set validate pre-condition.
 	if o.Marker != "" && o.Prefix != "" {
@@ -366,7 +370,7 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
 
 			objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned)
 			if o.Lifecycle != nil {
-				act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo).Action
+				act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo).Action
 				skip = act.Delete()
 				if act.DeleteRestored() {
 					// do not skip DeleteRestored* actions
@@ -394,7 +398,7 @@ func applyBucketActions(ctx context.Context, o listPathOptions, in <-chan metaCa
 			objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned)
 
 			if o.Lifecycle != nil {
-				evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, objInfo)
+				evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo)
 				if evt.Action.Delete() {
 					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
 					if !evt.Action.DeleteRestored() {

cmd/metacache-set.go

@@ -761,7 +761,7 @@ func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCache
 			return nil
 		}
 		o.debugln(color.Green("saveMetaCacheStream:")+" saving block", b.n, "to", o.objectPath(b.n))
-		r, err := hash.NewReader(bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)))
+		r, err := hash.NewReader(ctx, bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data)))
 		logger.LogIf(ctx, err)
 		custom := b.headerKV()
 		_, err = er.putMetacacheObject(ctx, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{

cmd/metrics-realtime.go

@@ -19,12 +19,15 @@ package cmd
 
 import (
 	"context"
+	"fmt"
 	"net/http"
 	"time"
 
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/disk"
 	"github.com/minio/minio/internal/net"
+	c "github.com/shirou/gopsutil/v3/cpu"
+	"github.com/shirou/gopsutil/v3/load"
 )
 
 type collectMetricsOpts struct {
@@ -89,6 +92,34 @@ func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m ma
 			m.Aggregated.Net.NetStats = netStats
 		}
 	}
+	if types.Contains(madmin.MetricsMem) {
+		m.Aggregated.Mem = &madmin.MemMetrics{
+			CollectedAt: UTCNow(),
+		}
+		m.Aggregated.Mem.Info = madmin.GetMemInfo(GlobalContext, globalMinioAddr)
+	}
+	if types.Contains(madmin.MetricsCPU) {
+		m.Aggregated.CPU = &madmin.CPUMetrics{
+			CollectedAt: UTCNow(),
+		}
+		cm, err := c.Times(false)
+		if err != nil {
+			m.Errors = append(m.Errors, err.Error())
+		} else {
+			// not collecting per-cpu stats, so there will be only one element
+			if len(cm) == 1 {
+				m.Aggregated.CPU.TimesStat = &cm[0]
+			} else {
+				m.Errors = append(m.Errors, fmt.Sprintf("Expected one CPU stat, got %d", len(cm)))
+			}
+		}
+		loadStat, err := load.Avg()
+		if err != nil {
+			m.Errors = append(m.Errors, err.Error())
+		} else {
+			m.Aggregated.CPU.LoadStat = loadStat
+		}
+	}
 	// Add types...
 
 	// ByHost is a shallow reference, so careful about sharing.

cmd/metrics-resource.go

@@ -0,0 +1,453 @@
+// Copyright (c) 2015-2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"net/http"
+	"sync"
+	"time"
+
+	"github.com/minio/madmin-go/v3"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/shirou/gopsutil/v3/host"
+)
+
+const (
+	resourceMetricsCollectionInterval = time.Minute
+	resourceMetricsCacheInterval      = time.Minute
+
+	// drive stats
+	totalInodes    MetricName = "total_inodes"
+	readsPerSec    MetricName = "reads_per_sec"
+	writesPerSec   MetricName = "writes_per_sec"
+	readsKBPerSec  MetricName = "reads_kb_per_sec"
+	writesKBPerSec MetricName = "writes_kb_per_sec"
+	readsAwait     MetricName = "reads_await"
+	writesAwait    MetricName = "writes_await"
+	percUtil       MetricName = "perc_util"
+	usedInodes     MetricName = "used_inodes"
+
+	// network stats
+	interfaceRxBytes  MetricName = "rx_bytes"
+	interfaceRxErrors MetricName = "rx_errors"
+	interfaceTxBytes  MetricName = "tx_bytes"
+	interfaceTxErrors MetricName = "tx_errors"
+
+	// memory stats
+	memUsed      MetricName = "used"
+	memFree      MetricName = "free"
+	memShared    MetricName = "shared"
+	memBuffers   MetricName = "buffers"
+	memCache     MetricName = "cache"
+	memAvailable MetricName = "available"
+
+	// cpu stats
+	cpuUser   MetricName = "user"
+	cpuSystem MetricName = "system"
+	cpuIOWait MetricName = "iowait"
+	cpuIdle   MetricName = "idle"
+	cpuNice   MetricName = "nice"
+	cpuSteal  MetricName = "steal"
+	cpuLoad1  MetricName = "load1"
+	cpuLoad5  MetricName = "load5"
+	cpuLoad15 MetricName = "load15"
+)
+
+var (
+	resourceCollector *minioResourceCollector
+	// resourceMetricsMap is a map of subsystem to its metrics
+	resourceMetricsMap map[MetricSubsystem]ResourceMetrics
+	// resourceMetricsHelpMap maps metric name to its help string
+	resourceMetricsHelpMap map[MetricName]string
+	resourceMetricsGroups  []*MetricsGroup
+)
+
+// PeerResourceMetrics represents the resource metrics
+// retrieved from a peer, along with errors if any
+type PeerResourceMetrics struct {
+	Metrics map[MetricSubsystem]ResourceMetrics
+	Errors  []string
+}
+
+// ResourceMetrics is a map of unique key identifying
+// a resource metric (e.g. reads_per_sec_{node}_{drive})
+// to its data
+type ResourceMetrics map[string]ResourceMetric
+
+// ResourceMetric represents a single resource metric
+// The metrics are collected from all servers periodically
+// and stored in the resource metrics map.
+// It also maintains the count of number of times this metric
+// was collected since the server started, and the sum,
+// average and max values across the same.
+type ResourceMetric struct {
+	Name   MetricName
+	Labels map[string]string
+
+	// value captured in current cycle
+	Current float64
+
+	// Used when system provides cumulative (since uptime) values
+	// helps in calculating the current value by comparing the new
+	// cumulative value with previous one
+	Cumulative float64
+
+	Max   float64
+	Avg   float64
+	Sum   float64
+	Count uint64
+}
+
+func init() {
+	interval := fmt.Sprintf("%ds", int(resourceMetricsCollectionInterval.Seconds()))
+	resourceMetricsHelpMap = map[MetricName]string{
+		interfaceRxBytes:  "Bytes received on the interface in " + interval,
+		interfaceRxErrors: "Receive errors in " + interval,
+		interfaceTxBytes:  "Bytes transmitted in " + interval,
+		interfaceTxErrors: "Transmit errors in " + interval,
+		total:             "Total memory on the node",
+		memUsed:           "Used memory on the node",
+		memFree:           "Free memory on the node",
+		memShared:         "Shared memory on the node",
+		memBuffers:        "Buffers memory on the node",
+		memCache:          "Cache memory on the node",
+		memAvailable:      "Available memory on the node",
+		readsPerSec:       "Reads per second on a drive",
+		writesPerSec:      "Writes per second on a drive",
+		readsKBPerSec:     "Kilobytes read per second on a drive",
+		writesKBPerSec:    "Kilobytes written per second on a drive",
+		readsAwait:        "Average time for read requests to be served on a drive",
+		writesAwait:       "Average time for write requests to be served on a drive",
+		percUtil:          "Percentage of time the disk was busy since uptime",
+		usedBytes:         "Used bytes on a drive",
+		totalBytes:        "Total bytes on a drive",
+		usedInodes:        "Total inodes used on a drive",
+		totalInodes:       "Total inodes on a drive",
+		cpuUser:           "CPU user time",
+		cpuSystem:         "CPU system time",
+		cpuIdle:           "CPU idle time",
+		cpuIOWait:         "CPU ioWait time",
+		cpuSteal:          "CPU steal time",
+		cpuNice:           "CPU nice time",
+		cpuLoad1:          "CPU load average 1min",
+		cpuLoad5:          "CPU load average 5min",
+		cpuLoad15:         "CPU load average 15min",
+	}
+	resourceMetricsGroups = []*MetricsGroup{
+		getResourceMetrics(),
+	}
+
+	resourceCollector = newMinioResourceCollector(resourceMetricsGroups)
+}
+
+func updateResourceMetrics(subSys MetricSubsystem, name MetricName, val float64, labels map[string]string, isCumulative bool) {
+	subsysMetrics, found := resourceMetricsMap[subSys]
+	if !found {
+		subsysMetrics = ResourceMetrics{}
+	}
+
+	// labels are used to uniquely identify a metric
+	// e.g. reads_per_sec_{drive} inside the map
+	sfx := ""
+	for _, v := range labels {
+		if len(sfx) > 0 {
+			sfx += "_"
+		}
+		sfx += v
+	}
+
+	key := string(name) + "_" + sfx
+	metric, found := subsysMetrics[key]
+	if !found {
+		metric = ResourceMetric{
+			Name:   name,
+			Labels: labels,
+		}
+	}
+
+	if isCumulative {
+		metric.Current = val - metric.Cumulative
+		metric.Cumulative = val
+	} else {
+		metric.Current = val
+	}
+
+	if metric.Current > metric.Max {
+		metric.Max = val
+	}
+
+	metric.Sum += metric.Current
+	metric.Count++
+
+	metric.Avg = metric.Sum / float64(metric.Count)
+	subsysMetrics[key] = metric
+
+	resourceMetricsMap[subSys] = subsysMetrics
+}
+
+func collectDriveMetrics(m madmin.RealtimeMetrics) {
+	upt, _ := host.Uptime()
+	kib := 1 << 10
+	sectorSize := uint64(512)
+
+	for d, dm := range m.ByDisk {
+		stats := dm.IOStats
+		labels := map[string]string{"drive": d}
+		updateResourceMetrics(driveSubsystem, readsPerSec, float64(stats.ReadIOs)/float64(upt), labels, false)
+
+		readBytes := stats.ReadSectors * sectorSize
+		readKib := float64(readBytes) / float64(kib)
+		readKibPerSec := readKib / float64(upt)
+		updateResourceMetrics(driveSubsystem, readsKBPerSec, readKibPerSec, labels, false)
+
+		updateResourceMetrics(driveSubsystem, writesPerSec, float64(stats.WriteIOs)/float64(upt), labels, false)
+
+		writeBytes := stats.WriteSectors * sectorSize
+		writeKib := float64(writeBytes) / float64(kib)
+		writeKibPerSec := writeKib / float64(upt)
+		updateResourceMetrics(driveSubsystem, writesKBPerSec, writeKibPerSec, labels, false)
+
+		rdAwait := 0.0
+		if stats.ReadIOs > 0 {
+			rdAwait = float64(stats.ReadTicks) / float64(stats.ReadIOs)
+		}
+		updateResourceMetrics(driveSubsystem, readsAwait, rdAwait, labels, false)
+
+		wrAwait := 0.0
+		if stats.WriteIOs > 0 {
+			wrAwait = float64(stats.WriteTicks) / float64(stats.WriteIOs)
+		}
+		updateResourceMetrics(driveSubsystem, writesAwait, wrAwait, labels, false)
+
+		updateResourceMetrics(driveSubsystem, percUtil, float64(stats.TotalTicks)/float64(upt*10), labels, false)
+	}
+
+	globalLocalDrivesMu.RLock()
+	gld := globalLocalDrives
+	globalLocalDrivesMu.RUnlock()
+
+	for _, d := range gld {
+		labels := map[string]string{"drive": d.Endpoint().RawPath}
+		di, err := d.DiskInfo(GlobalContext, false)
+		if err == nil {
+			updateResourceMetrics(driveSubsystem, usedBytes, float64(di.Used), labels, false)
+			updateResourceMetrics(driveSubsystem, totalBytes, float64(di.Total), labels, false)
+			updateResourceMetrics(driveSubsystem, usedInodes, float64(di.UsedInodes), labels, false)
+			updateResourceMetrics(driveSubsystem, totalInodes, float64(di.FreeInodes+di.UsedInodes), labels, false)
+		}
+	}
+}
+
+func collectLocalResourceMetrics() {
+	var types madmin.MetricType = madmin.MetricsDisk | madmin.MetricNet | madmin.MetricsMem | madmin.MetricsCPU
+
+	m := collectLocalMetrics(types, collectMetricsOpts{
+		hosts: map[string]struct{}{
+			globalLocalNodeName: {},
+		},
+	})
+
+	for host, hm := range m.ByHost {
+		if len(host) > 0 {
+			if hm.Net != nil && len(hm.Net.NetStats.Name) > 0 {
+				stats := hm.Net.NetStats
+				labels := map[string]string{"interface": stats.Name}
+				updateResourceMetrics(interfaceSubsystem, interfaceRxBytes, float64(stats.RxBytes), labels, true)
+				updateResourceMetrics(interfaceSubsystem, interfaceRxErrors, float64(stats.RxErrors), labels, true)
+				updateResourceMetrics(interfaceSubsystem, interfaceTxBytes, float64(stats.TxBytes), labels, true)
+				updateResourceMetrics(interfaceSubsystem, interfaceTxErrors, float64(stats.TxErrors), labels, true)
+			}
+			if hm.Mem != nil && len(hm.Mem.Info.Addr) > 0 {
+				labels := map[string]string{}
+				stats := hm.Mem.Info
+				updateResourceMetrics(memSubsystem, total, float64(stats.Total), labels, false)
+				updateResourceMetrics(memSubsystem, memUsed, float64(stats.Used), labels, false)
+				updateResourceMetrics(memSubsystem, memFree, float64(stats.Free), labels, false)
+				updateResourceMetrics(memSubsystem, memShared, float64(stats.Shared), labels, false)
+				updateResourceMetrics(memSubsystem, memBuffers, float64(stats.Buffers), labels, false)
+				updateResourceMetrics(memSubsystem, memAvailable, float64(stats.Available), labels, false)
+				updateResourceMetrics(memSubsystem, memCache, float64(stats.Cache), labels, false)
+			}
+			if hm.CPU != nil {
+				labels := map[string]string{}
+				ts := hm.CPU.TimesStat
+				if ts != nil {
+					tot := ts.User + ts.System + ts.Idle + ts.Iowait + ts.Nice + ts.Steal
+					cpuUserVal := math.Round(ts.User/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuUser, cpuUserVal, labels, false)
+					cpuSystemVal := math.Round(ts.System/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuSystem, cpuSystemVal, labels, false)
+					cpuIdleVal := math.Round(ts.Idle/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuIdle, cpuIdleVal, labels, false)
+					cpuIOWaitVal := math.Round(ts.Iowait/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuIOWait, cpuIOWaitVal, labels, false)
+					cpuNiceVal := math.Round(ts.Nice/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuNice, cpuNiceVal, labels, false)
+					cpuStealVal := math.Round(ts.Steal/tot*100*100) / 100
+					updateResourceMetrics(cpuSubsystem, cpuSteal, cpuStealVal, labels, false)
+				}
+				ls := hm.CPU.LoadStat
+				if ls != nil {
+					updateResourceMetrics(cpuSubsystem, cpuLoad1, ls.Load1, labels, false)
+					updateResourceMetrics(cpuSubsystem, cpuLoad5, ls.Load5, labels, false)
+					updateResourceMetrics(cpuSubsystem, cpuLoad15, ls.Load15, labels, false)
+				}
+			}
+			break // only one host expected
+		}
+	}
+
+	collectDriveMetrics(m)
+}
+
+// startResourceMetricsCollection - starts the job for collecting resource metrics
+func startResourceMetricsCollection() {
+	resourceMetricsMap = map[MetricSubsystem]ResourceMetrics{}
+	metricsTimer := time.NewTimer(resourceMetricsCollectionInterval)
+	defer metricsTimer.Stop()
+
+	collectLocalResourceMetrics()
+
+	for {
+		select {
+		case <-GlobalContext.Done():
+			return
+		case <-metricsTimer.C:
+			collectLocalResourceMetrics()
+
+			// Reset the timer for next cycle.
+			metricsTimer.Reset(resourceMetricsCollectionInterval)
+		}
+	}
+}
+
+// minioResourceCollector is the Collector for resource metrics
+type minioResourceCollector struct {
+	metricsGroups []*MetricsGroup
+	desc          *prometheus.Desc
+}
+
+// Describe sends the super-set of all possible descriptors of metrics
+func (c *minioResourceCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- c.desc
+}
+
+// Collect is called by the Prometheus registry when collecting metrics.
+func (c *minioResourceCollector) Collect(out chan<- prometheus.Metric) {
+	var wg sync.WaitGroup
+	publish := func(in <-chan Metric) {
+		defer wg.Done()
+		for metric := range in {
+			labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
+			collectMetric(metric, labels, values, "resource", out)
+		}
+	}
+
+	// Call peer api to fetch metrics
+	wg.Add(2)
+	go publish(ReportMetrics(GlobalContext, c.metricsGroups))
+	go publish(globalNotificationSys.GetResourceMetrics(GlobalContext))
+	wg.Wait()
+}
+
+// newMinioResourceCollector describes the collector
+// and returns reference of minio resource Collector
+// It creates the Prometheus Description which is used
+// to define Metric and  help string
+func newMinioResourceCollector(metricsGroups []*MetricsGroup) *minioResourceCollector {
+	return &minioResourceCollector{
+		metricsGroups: metricsGroups,
+		desc:          prometheus.NewDesc("minio_resource_stats", "Resource statistics exposed by MinIO server", nil, nil),
+	}
+}
+
+func prepareResourceMetrics(rm ResourceMetric, subSys MetricSubsystem, requireAvgMax bool) []Metric {
+	help := resourceMetricsHelpMap[rm.Name]
+	name := rm.Name
+
+	metrics := []Metric{}
+	metrics = append(metrics, Metric{
+		Description:    getResourceMetricDescription(subSys, name, help),
+		Value:          rm.Current,
+		VariableLabels: rm.Labels,
+	})
+
+	if requireAvgMax {
+		avgName := MetricName(fmt.Sprintf("%s_avg", name))
+		avgHelp := fmt.Sprintf("%s (avg)", help)
+		metrics = append(metrics, Metric{
+			Description:    getResourceMetricDescription(subSys, avgName, avgHelp),
+			Value:          math.Round(rm.Avg*100) / 100,
+			VariableLabels: rm.Labels,
+		})
+
+		maxName := MetricName(fmt.Sprintf("%s_max", name))
+		maxHelp := fmt.Sprintf("%s (max)", help)
+		metrics = append(metrics, Metric{
+			Description:    getResourceMetricDescription(subSys, maxName, maxHelp),
+			Value:          rm.Max,
+			VariableLabels: rm.Labels,
+		})
+	}
+
+	return metrics
+}
+
+func getResourceMetricDescription(subSys MetricSubsystem, name MetricName, help string) MetricDescription {
+	return MetricDescription{
+		Namespace: nodeMetricNamespace,
+		Subsystem: subSys,
+		Name:      name,
+		Help:      help,
+		Type:      gaugeMetric,
+	}
+}
+
+func getResourceMetrics() *MetricsGroup {
+	mg := &MetricsGroup{
+		cacheInterval: resourceMetricsCacheInterval,
+	}
+	mg.RegisterRead(func(ctx context.Context) []Metric {
+		metrics := []Metric{}
+
+		subSystems := []MetricSubsystem{interfaceSubsystem, memSubsystem, driveSubsystem, cpuSubsystem}
+		for _, subSys := range subSystems {
+			stats, found := resourceMetricsMap[subSys]
+			if found {
+				requireAvgMax := true
+				if subSys == driveSubsystem {
+					requireAvgMax = false
+				}
+				for _, m := range stats {
+					metrics = append(metrics, prepareResourceMetrics(m, subSys, requireAvgMax)...)
+				}
+			}
+		}
+
+		return metrics
+	})
+	return mg
+}
+
+// metricsResourceHandler is the prometheus handler for resource metrics
+func metricsResourceHandler() http.Handler {
+	return metricsHTTPHandler(resourceCollector, "handler.MetricsResource")
+}

cmd/metrics-router.go

@@ -25,10 +25,11 @@ import (
 )
 
 const (
-	prometheusMetricsPathLegacy    = "/prometheus/metrics"
-	prometheusMetricsV2ClusterPath = "/v2/metrics/cluster"
-	prometheusMetricsV2BucketPath  = "/v2/metrics/bucket"
-	prometheusMetricsV2NodePath    = "/v2/metrics/node"
+	prometheusMetricsPathLegacy     = "/prometheus/metrics"
+	prometheusMetricsV2ClusterPath  = "/v2/metrics/cluster"
+	prometheusMetricsV2BucketPath   = "/v2/metrics/bucket"
+	prometheusMetricsV2NodePath     = "/v2/metrics/node"
+	prometheusMetricsV2ResourcePath = "/v2/metrics/resource"
 )
 
 // Standard env prometheus auth type
@@ -57,4 +58,5 @@ func registerMetricsRouter(router *mux.Router) {
 	metricsRouter.Handle(prometheusMetricsV2ClusterPath, auth(metricsServerHandler()))
 	metricsRouter.Handle(prometheusMetricsV2BucketPath, auth(metricsBucketHandler()))
 	metricsRouter.Handle(prometheusMetricsV2NodePath, auth(metricsNodeHandler()))
+	metricsRouter.Handle(prometheusMetricsV2ResourcePath, auth(metricsResourceHandler()))
 }

cmd/metrics-v2.go

@@ -132,6 +132,9 @@ const (
 	capacityRawSubsystem      MetricSubsystem = "capacity_raw"
 	capacityUsableSubsystem   MetricSubsystem = "capacity_usable"
 	driveSubsystem            MetricSubsystem = "drive"
+	interfaceSubsystem        MetricSubsystem = "if"
+	memSubsystem              MetricSubsystem = "mem"
+	cpuSubsystem              MetricSubsystem = "cpu_avg"
 	storageClassSubsystem     MetricSubsystem = "storage_class"
 	fileDescriptorSubsystem   MetricSubsystem = "file_descriptor"
 	goRoutines                MetricSubsystem = "go_routine"
@@ -539,12 +542,12 @@ func getNodeRRSParityMD() MetricDescription {
 	}
 }
 
-func getNodeDrivesFreeInodes() MetricDescription {
+func getNodeDrivesFreeInodesMD() MetricDescription {
 	return MetricDescription{
 		Namespace: nodeMetricNamespace,
 		Subsystem: driveSubsystem,
 		Name:      freeInodes,
-		Help:      "Total free inodes",
+		Help:      "Free inodes on a drive",
 		Type:      gaugeMetric,
 	}
 }
@@ -2085,99 +2088,101 @@ func getReplicationClusterMetrics() *MetricsGroup {
 	)
 
 	mg.RegisterRead(func(_ context.Context) []Metric {
+		var ml []Metric
 		// common operational metrics for bucket replication and site replication - published
 		// at cluster level
-		qs := globalReplicationStats.getNodeQueueStatsSummary()
-		activeWorkersCount := Metric{
-			Description:    getClusterReplActiveWorkersCountMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
-		avgActiveWorkersCount := Metric{
-			Description:    getClusterReplAvgActiveWorkersCountMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
-		maxActiveWorkersCount := Metric{
-			Description:    getClusterReplMaxActiveWorkersCountMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
-		currInQueueCount := Metric{
-			Description:    getClusterReplCurrQueuedOperationsMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
-		currInQueueBytes := Metric{
-			Description:    getClusterReplCurrQueuedBytesMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
+		if globalReplicationStats != nil {
+			qs := globalReplicationStats.getNodeQueueStatsSummary()
+			activeWorkersCount := Metric{
+				Description:    getClusterReplActiveWorkersCountMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
+			avgActiveWorkersCount := Metric{
+				Description:    getClusterReplAvgActiveWorkersCountMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
+			maxActiveWorkersCount := Metric{
+				Description:    getClusterReplMaxActiveWorkersCountMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
+			currInQueueCount := Metric{
+				Description:    getClusterReplCurrQueuedOperationsMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
+			currInQueueBytes := Metric{
+				Description:    getClusterReplCurrQueuedBytesMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
 
-		currTransferRate := Metric{
-			Description:    getClusterReplCurrentTransferRateMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
-		avgQueueCount := Metric{
-			Description:    getClusterReplAvgQueuedOperationsMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
-		avgQueueBytes := Metric{
-			Description:    getClusterReplAvgQueuedBytesMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
-		maxQueueCount := Metric{
-			Description:    getClusterReplMaxQueuedOperationsMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
-		maxQueueBytes := Metric{
-			Description:    getClusterReplMaxQueuedBytesMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
-		avgTransferRate := Metric{
-			Description:    getClusterReplAvgTransferRateMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
-		maxTransferRate := Metric{
-			Description:    getClusterReplMaxTransferRateMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-		}
-		mrfCount := Metric{
-			Description:    getClusterReplMRFFailedOperationsMD(),
-			VariableLabels: map[string]string{serverName: qs.NodeName},
-			Value:          float64(qs.MRFStats.LastFailedCount),
-		}
+			currTransferRate := Metric{
+				Description:    getClusterReplCurrentTransferRateMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
+			avgQueueCount := Metric{
+				Description:    getClusterReplAvgQueuedOperationsMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
+			avgQueueBytes := Metric{
+				Description:    getClusterReplAvgQueuedBytesMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
+			maxQueueCount := Metric{
+				Description:    getClusterReplMaxQueuedOperationsMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
+			maxQueueBytes := Metric{
+				Description:    getClusterReplMaxQueuedBytesMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
+			avgTransferRate := Metric{
+				Description:    getClusterReplAvgTransferRateMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
+			maxTransferRate := Metric{
+				Description:    getClusterReplMaxTransferRateMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+			}
+			mrfCount := Metric{
+				Description:    getClusterReplMRFFailedOperationsMD(),
+				VariableLabels: map[string]string{serverName: qs.NodeName},
+				Value:          float64(qs.MRFStats.LastFailedCount),
+			}
 
-		if qs.QStats.Avg.Count > 0 || qs.QStats.Curr.Count > 0 {
-			qt := qs.QStats
-			currInQueueBytes.Value = qt.Curr.Bytes
-			currInQueueCount.Value = qt.Curr.Count
-			avgQueueBytes.Value = qt.Avg.Bytes
-			avgQueueCount.Value = qt.Avg.Count
-			maxQueueBytes.Value = qt.Max.Bytes
-			maxQueueCount.Value = qt.Max.Count
-		}
-		activeWorkersCount.Value = float64(qs.ActiveWorkers.Curr)
-		avgActiveWorkersCount.Value = float64(qs.ActiveWorkers.Avg)
-		maxActiveWorkersCount.Value = float64(qs.ActiveWorkers.Max)
+			if qs.QStats.Avg.Count > 0 || qs.QStats.Curr.Count > 0 {
+				qt := qs.QStats
+				currInQueueBytes.Value = qt.Curr.Bytes
+				currInQueueCount.Value = qt.Curr.Count
+				avgQueueBytes.Value = qt.Avg.Bytes
+				avgQueueCount.Value = qt.Avg.Count
+				maxQueueBytes.Value = qt.Max.Bytes
+				maxQueueCount.Value = qt.Max.Count
+			}
+			activeWorkersCount.Value = float64(qs.ActiveWorkers.Curr)
+			avgActiveWorkersCount.Value = float64(qs.ActiveWorkers.Avg)
+			maxActiveWorkersCount.Value = float64(qs.ActiveWorkers.Max)
 
-		if len(qs.XferStats) > 0 {
-			tots := qs.XferStats[Total]
-			currTransferRate.Value = tots.Curr
-			avgTransferRate.Value = tots.Avg
-			maxTransferRate.Value = tots.Peak
+			if len(qs.XferStats) > 0 {
+				tots := qs.XferStats[Total]
+				currTransferRate.Value = tots.Curr
+				avgTransferRate.Value = tots.Avg
+				maxTransferRate.Value = tots.Peak
+			}
+			ml = []Metric{
+				activeWorkersCount,
+				avgActiveWorkersCount,
+				maxActiveWorkersCount,
+				currInQueueCount,
+				currInQueueBytes,
+				avgQueueCount,
+				avgQueueBytes,
+				maxQueueCount,
+				maxQueueBytes,
+				currTransferRate,
+				avgTransferRate,
+				maxTransferRate,
+				mrfCount,
+			}
 		}
-		ml := []Metric{
-			activeWorkersCount,
-			avgActiveWorkersCount,
-			maxActiveWorkersCount,
-			currInQueueCount,
-			currInQueueBytes,
-			avgQueueCount,
-			avgQueueBytes,
-			maxQueueCount,
-			maxQueueBytes,
-			currTransferRate,
-			avgTransferRate,
-			maxTransferRate,
-			mrfCount,
-		}
-
 		for ep, health := range globalBucketTargetSys.healthStats() {
 			// link latency current
 			m := Metric{
@@ -3019,17 +3024,21 @@ func getBucketUsageMetrics() *MetricsGroup {
 				})
 			}
 			if !globalSiteReplicationSys.isEnabled() {
-				stats := bucketReplStats[bucket].ReplicationStats
-				metrics = append(metrics, Metric{
-					Description:    getRepReceivedBytesMD(bucketMetricNamespace),
-					Value:          float64(stats.ReplicaSize),
-					VariableLabels: map[string]string{"bucket": bucket},
-				})
-				metrics = append(metrics, Metric{
-					Description:    getRepReceivedOperationsMD(bucketMetricNamespace),
-					Value:          float64(stats.ReplicaCount),
-					VariableLabels: map[string]string{"bucket": bucket},
-				})
+				var stats BucketReplicationStats
+				s, ok := bucketReplStats[bucket]
+				if ok {
+					stats = s.ReplicationStats
+					metrics = append(metrics, Metric{
+						Description:    getRepReceivedBytesMD(bucketMetricNamespace),
+						Value:          float64(stats.ReplicaSize),
+						VariableLabels: map[string]string{"bucket": bucket},
+					})
+					metrics = append(metrics, Metric{
+						Description:    getRepReceivedOperationsMD(bucketMetricNamespace),
+						Value:          float64(stats.ReplicaCount),
+						VariableLabels: map[string]string{"bucket": bucket},
+					})
+				}
 				if stats.hasReplicationUsage() {
 					for arn, stat := range stats.Stats {
 						metrics = append(metrics, Metric{
@@ -3201,7 +3210,7 @@ func getLocalStorageMetrics() *MetricsGroup {
 			})
 
 			metrics = append(metrics, Metric{
-				Description:    getNodeDrivesFreeInodes(),
+				Description:    getNodeDrivesFreeInodesMD(),
 				Value:          float64(disk.FreeInodes),
 				VariableLabels: map[string]string{"drive": disk.DrivePath},
 			})
@@ -3540,6 +3549,61 @@ func getKMSMetrics() *MetricsGroup {
 	return mg
 }
 
+func collectMetric(metric Metric, labels []string, values []string, metricName string, out chan<- prometheus.Metric) {
+	if metric.Description.Type == histogramMetric {
+		if metric.Histogram == nil {
+			return
+		}
+		for k, v := range metric.Histogram {
+			pmetric, err := prometheus.NewConstMetric(
+				prometheus.NewDesc(
+					prometheus.BuildFQName(string(metric.Description.Namespace),
+						string(metric.Description.Subsystem),
+						string(metric.Description.Name)),
+					metric.Description.Help,
+					append(labels, metric.HistogramBucketLabel),
+					metric.StaticLabels,
+				),
+				prometheus.GaugeValue,
+				float64(v),
+				append(values, k)...)
+			if err != nil {
+				// Enable for debugging
+				if serverDebugLog {
+					logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v+%v", err, values, metric.Histogram), metricName+"-metrics-histogram")
+				}
+			} else {
+				out <- pmetric
+			}
+		}
+		return
+	}
+	metricType := prometheus.GaugeValue
+	if metric.Description.Type == counterMetric {
+		metricType = prometheus.CounterValue
+	}
+	pmetric, err := prometheus.NewConstMetric(
+		prometheus.NewDesc(
+			prometheus.BuildFQName(string(metric.Description.Namespace),
+				string(metric.Description.Subsystem),
+				string(metric.Description.Name)),
+			metric.Description.Help,
+			labels,
+			metric.StaticLabels,
+		),
+		metricType,
+		metric.Value,
+		values...)
+	if err != nil {
+		// Enable for debugging
+		if serverDebugLog {
+			logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), metricName+"-metrics")
+		}
+	} else {
+		out <- pmetric
+	}
+}
+
 type minioBucketCollector struct {
 	metricsGroups []*MetricsGroup
 	desc          *prometheus.Desc
@@ -3564,52 +3628,7 @@ func (c *minioBucketCollector) Collect(out chan<- prometheus.Metric) {
 		defer wg.Done()
 		for metric := range in {
 			labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
-			if metric.Description.Type == histogramMetric {
-				if metric.Histogram == nil {
-					continue
-				}
-				for k, v := range metric.Histogram {
-					pmetric, err := prometheus.NewConstMetric(
-						prometheus.NewDesc(
-							prometheus.BuildFQName(string(metric.Description.Namespace),
-								string(metric.Description.Subsystem),
-								string(metric.Description.Name)),
-							metric.Description.Help,
-							append(labels, metric.HistogramBucketLabel),
-							metric.StaticLabels,
-						),
-						prometheus.GaugeValue,
-						float64(v),
-						append(values, k)...)
-					if err != nil {
-						logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v+%v", err, values, metric.Histogram), "bucket-metrics-histogram")
-					} else {
-						out <- pmetric
-					}
-				}
-				continue
-			}
-			metricType := prometheus.GaugeValue
-			if metric.Description.Type == counterMetric {
-				metricType = prometheus.CounterValue
-			}
-			pmetric, err := prometheus.NewConstMetric(
-				prometheus.NewDesc(
-					prometheus.BuildFQName(string(metric.Description.Namespace),
-						string(metric.Description.Subsystem),
-						string(metric.Description.Name)),
-					metric.Description.Help,
-					labels,
-					metric.StaticLabels,
-				),
-				metricType,
-				metric.Value,
-				values...)
-			if err != nil {
-				logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), "bucket-metrics")
-			} else {
-				out <- pmetric
-			}
+			collectMetric(metric, labels, values, "bucket", out)
 		}
 	}
 
@@ -3644,52 +3663,7 @@ func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) {
 		defer wg.Done()
 		for metric := range in {
 			labels, values := getOrderedLabelValueArrays(metric.VariableLabels)
-			if metric.Description.Type == histogramMetric {
-				if metric.Histogram == nil {
-					continue
-				}
-				for k, v := range metric.Histogram {
-					pmetric, err := prometheus.NewConstMetric(
-						prometheus.NewDesc(
-							prometheus.BuildFQName(string(metric.Description.Namespace),
-								string(metric.Description.Subsystem),
-								string(metric.Description.Name)),
-							metric.Description.Help,
-							append(labels, metric.HistogramBucketLabel),
-							metric.StaticLabels,
-						),
-						prometheus.GaugeValue,
-						float64(v),
-						append(values, k)...)
-					if err != nil {
-						logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v:%v", err, values, metric.Histogram), "cluster-metrics-histogram")
-					} else {
-						out <- pmetric
-					}
-				}
-				continue
-			}
-			metricType := prometheus.GaugeValue
-			if metric.Description.Type == counterMetric {
-				metricType = prometheus.CounterValue
-			}
-			pmetric, err := prometheus.NewConstMetric(
-				prometheus.NewDesc(
-					prometheus.BuildFQName(string(metric.Description.Namespace),
-						string(metric.Description.Subsystem),
-						string(metric.Description.Name)),
-					metric.Description.Help,
-					labels,
-					metric.StaticLabels,
-				),
-				metricType,
-				metric.Value,
-				values...)
-			if err != nil {
-				logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values), "cluster-metrics")
-			} else {
-				out <- pmetric
-			}
+			collectMetric(metric, labels, values, "cluster", out)
 		}
 	}
 
@@ -3822,11 +3796,11 @@ func newMinioCollectorNode(metricsGroups []*MetricsGroup) *minioNodeCollector {
 	}
 }
 
-func metricsBucketHandler() http.Handler {
+func metricsHTTPHandler(c prometheus.Collector, funcName string) http.Handler {
 	registry := prometheus.NewRegistry()
 
 	// Report all other metrics
-	logger.CriticalIf(GlobalContext, registry.Register(bucketCollector))
+	logger.CriticalIf(GlobalContext, registry.Register(c))
 
 	// DefaultGatherers include golang metrics and process metrics.
 	gatherers := prometheus.Gatherers{
@@ -3836,16 +3810,14 @@ func metricsBucketHandler() http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
 		if ok {
-			tc.FuncName = "handler.MetricsBucket"
+			tc.FuncName = funcName
 			tc.ResponseRecorder.LogErrBody = true
 		}
 
 		mfs, err := gatherers.Gather()
-		if err != nil {
-			if len(mfs) == 0 {
-				writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
-				return
-			}
+		if err != nil && len(mfs) == 0 {
+			writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
+			return
 		}
 
 		contentType := expfmt.Negotiate(r.Header)
@@ -3865,6 +3837,10 @@ func metricsBucketHandler() http.Handler {
 	})
 }
 
+func metricsBucketHandler() http.Handler {
+	return metricsHTTPHandler(bucketCollector, "handler.MetricsBucket")
+}
+
 func metricsServerHandler() http.Handler {
 	registry := prometheus.NewRegistry()
 
@@ -3884,11 +3860,9 @@ func metricsServerHandler() http.Handler {
 		}
 
 		mfs, err := gatherers.Gather()
-		if err != nil {
-			if len(mfs) == 0 {
-				writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
-				return
-			}
+		if err != nil && len(mfs) == 0 {
+			writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL)
+			return
 		}
 
 		contentType := expfmt.Negotiate(r.Header)

cmd/metrics.go

@@ -25,7 +25,7 @@ import (
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/mcontext"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/expfmt"
 )
@@ -616,7 +616,7 @@ func AuthMiddleware(h http.Handler) http.Handler {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
 
 		claims, groups, owner, authErr := metricsRequestAuthenticate(r)
-		if authErr != nil || !claims.VerifyIssuer("prometheus", true) {
+		if authErr != nil || (claims != nil && !claims.VerifyIssuer("prometheus", true)) {
 			if ok {
 				tc.FuncName = "handler.MetricsAuth"
 				tc.ResponseRecorder.LogErrBody = true
@@ -633,10 +633,10 @@ func AuthMiddleware(h http.Handler) http.Handler {
 		}
 
 		// For authenticated users apply IAM policy.
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
+		if !globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.PrometheusAdminAction,
+			Action:          policy.PrometheusAdminAction,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
 			Claims:          cred.Claims,

cmd/notification.go

@@ -297,7 +297,7 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 		profilingDataFound = true
 
 		for typ, data := range data {
-			err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", client.host.String(), typ), data)
+			err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", client.host.String(), typ), data, 0o600)
 			if err != nil {
 				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String())
 				ctx := logger.SetReqInfo(ctx, reqInfo)
@@ -325,11 +325,11 @@ func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io
 
 	// Send profiling data to zip as file
 	for typ, data := range data {
-		err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", thisAddr, typ), data)
+		err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", thisAddr, typ), data, 0o600)
 		logger.LogIf(ctx, err)
 	}
 	if b := getClusterMetaInfo(ctx); len(b) > 0 {
-		logger.LogIf(ctx, embedFileInZip(zipWriter, "cluster.info", b))
+		logger.LogIf(ctx, embedFileInZip(zipWriter, "cluster.info", b, 0o600))
 	}
 
 	return
@@ -783,6 +783,27 @@ func (sys *NotificationSys) GetMetrics(ctx context.Context, t madmin.MetricType,
 	return reply
 }
 
+// GetResourceMetrics - gets the resource metrics from all nodes excluding self.
+func (sys *NotificationSys) GetResourceMetrics(ctx context.Context) <-chan Metric {
+	if sys == nil {
+		return nil
+	}
+	g := errgroup.WithNErrs(len(sys.peerClients))
+	peerChannels := make([]<-chan Metric, len(sys.peerClients))
+	for index := range sys.peerClients {
+		index := index
+		g.Go(func() error {
+			if sys.peerClients[index] == nil {
+				return errPeerNotReachable
+			}
+			var err error
+			peerChannels[index], err = sys.peerClients[index].GetResourceMetrics(ctx)
+			return err
+		}, index)
+	}
+	return sys.collectPeerMetrics(ctx, peerChannels, g)
+}
+
 // GetSysConfig - Get information about system config
 // (only the config that are of concern to minio)
 func (sys *NotificationSys) GetSysConfig(ctx context.Context) []madmin.SysConfig {
@@ -1099,40 +1120,70 @@ func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...
 	}
 	reports = append(reports, globalBucketMonitor.GetReport(bandwidth.SelectBuckets(buckets...)))
 	consolidatedReport := bandwidth.BucketBandwidthReport{
-		BucketStats: make(map[string]map[string]bandwidth.Details),
+		BucketStats: make(map[bandwidth.BucketOptions]bandwidth.Details),
 	}
 	for _, report := range reports {
 		if report == nil || report.BucketStats == nil {
 			continue
 		}
-		for bucket := range report.BucketStats {
-			d, ok := consolidatedReport.BucketStats[bucket]
+		for opts := range report.BucketStats {
+			d, ok := consolidatedReport.BucketStats[opts]
 			if !ok {
-				consolidatedReport.BucketStats[bucket] = make(map[string]bandwidth.Details)
-				d = consolidatedReport.BucketStats[bucket]
-				for arn := range d {
-					d[arn] = bandwidth.Details{
-						LimitInBytesPerSecond: report.BucketStats[bucket][arn].LimitInBytesPerSecond,
-					}
+				d = bandwidth.Details{
+					LimitInBytesPerSecond: report.BucketStats[opts].LimitInBytesPerSecond,
 				}
 			}
-			for arn, st := range report.BucketStats[bucket] {
-				bwDet := bandwidth.Details{}
-				if bw, ok := d[arn]; ok {
-					bwDet = bw
-				}
-				if bwDet.LimitInBytesPerSecond < st.LimitInBytesPerSecond {
-					bwDet.LimitInBytesPerSecond = st.LimitInBytesPerSecond
-				}
-				bwDet.CurrentBandwidthInBytesPerSecond += st.CurrentBandwidthInBytesPerSecond
-				d[arn] = bwDet
-				consolidatedReport.BucketStats[bucket] = d
+			dt, ok := report.BucketStats[opts]
+			if ok {
+				d.CurrentBandwidthInBytesPerSecond += dt.CurrentBandwidthInBytesPerSecond
 			}
+			consolidatedReport.BucketStats[opts] = d
 		}
 	}
 	return consolidatedReport
 }
 
+func (sys *NotificationSys) collectPeerMetrics(ctx context.Context, peerChannels []<-chan Metric, g *errgroup.Group) <-chan Metric {
+	ch := make(chan Metric)
+	var wg sync.WaitGroup
+	for index, err := range g.Wait() {
+		if err != nil {
+			if sys.peerClients[index] != nil {
+				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress",
+					sys.peerClients[index].host.String())
+				logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), err, sys.peerClients[index].host.String())
+			} else {
+				logger.LogOnceIf(ctx, err, "peer-offline")
+			}
+			continue
+		}
+		wg.Add(1)
+		go func(ctx context.Context, peerChannel <-chan Metric, wg *sync.WaitGroup) {
+			defer wg.Done()
+			for {
+				select {
+				case m, ok := <-peerChannel:
+					if !ok {
+						return
+					}
+					select {
+					case ch <- m:
+					case <-ctx.Done():
+						return
+					}
+				case <-ctx.Done():
+					return
+				}
+			}
+		}(ctx, peerChannels[index], &wg)
+	}
+	go func(wg *sync.WaitGroup, ch chan Metric) {
+		wg.Wait()
+		close(ch)
+	}(&wg, ch)
+	return ch
+}
+
 // GetBucketMetrics - gets the cluster level bucket metrics from all nodes excluding self.
 func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan Metric {
 	if sys == nil {
@@ -1151,45 +1202,7 @@ func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan Metric
 			return err
 		}, index)
 	}
-
-	ch := make(chan Metric)
-	var wg sync.WaitGroup
-	for index, err := range g.Wait() {
-		if err != nil {
-			if sys.peerClients[index] != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress",
-					sys.peerClients[index].host.String())
-				logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), err, sys.peerClients[index].host.String())
-			} else {
-				logger.LogOnceIf(ctx, err, "peer-offline")
-			}
-			continue
-		}
-		wg.Add(1)
-		go func(ctx context.Context, peerChannel <-chan Metric, wg *sync.WaitGroup) {
-			defer wg.Done()
-			for {
-				select {
-				case m, ok := <-peerChannel:
-					if !ok {
-						return
-					}
-					select {
-					case ch <- m:
-					case <-ctx.Done():
-						return
-					}
-				case <-ctx.Done():
-					return
-				}
-			}
-		}(ctx, peerChannels[index], &wg)
-	}
-	go func(wg *sync.WaitGroup, ch chan Metric) {
-		wg.Wait()
-		close(ch)
-	}(&wg, ch)
-	return ch
+	return sys.collectPeerMetrics(ctx, peerChannels, g)
 }
 
 // GetClusterMetrics - gets the cluster metrics from all nodes excluding self.
@@ -1210,45 +1223,7 @@ func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) <-chan Metric
 			return err
 		}, index)
 	}
-
-	ch := make(chan Metric)
-	var wg sync.WaitGroup
-	for index, err := range g.Wait() {
-		if err != nil {
-			if sys.peerClients[index] != nil {
-				reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress",
-					sys.peerClients[index].host.String())
-				logger.LogOnceIf(logger.SetReqInfo(ctx, reqInfo), err, sys.peerClients[index].host.String())
-			} else {
-				logger.LogOnceIf(ctx, err, "peer-offline")
-			}
-			continue
-		}
-		wg.Add(1)
-		go func(ctx context.Context, peerChannel <-chan Metric, wg *sync.WaitGroup) {
-			defer wg.Done()
-			for {
-				select {
-				case m, ok := <-peerChannel:
-					if !ok {
-						return
-					}
-					select {
-					case ch <- m:
-					case <-ctx.Done():
-						return
-					}
-				case <-ctx.Done():
-					return
-				}
-			}
-		}(ctx, peerChannels[index], &wg)
-	}
-	go func(wg *sync.WaitGroup, ch chan Metric) {
-		wg.Wait()
-		close(ch)
-	}(&wg, ch)
-	return ch
+	return sys.collectPeerMetrics(ctx, peerChannels, g)
 }
 
 // ServiceFreeze freezes all S3 API calls when 'freeze' is true,

cmd/object-api-datatypes.go

@@ -109,6 +109,9 @@ type ObjectInfo struct {
 	// Total object size.
 	Size int64
 
+	// Actual size is the real size of the object uploaded by client.
+	ActualSize *int64
+
 	// IsDir indicates if the object is prefix.
 	IsDir bool
 
@@ -282,9 +285,45 @@ func (o ObjectInfo) tierStats() tierStats {
 	return ts
 }
 
+// ToObjectInfo converts a replication object info to a partial ObjectInfo
+// do not rely on this function to give you correct ObjectInfo, this
+// function is merely and optimization.
+func (ri ReplicateObjectInfo) ToObjectInfo() ObjectInfo {
+	return ObjectInfo{
+		Name:                       ri.Name,
+		Bucket:                     ri.Bucket,
+		VersionID:                  ri.VersionID,
+		ModTime:                    ri.ModTime,
+		UserTags:                   ri.UserTags,
+		Size:                       ri.Size,
+		ActualSize:                 &ri.ActualSize,
+		ReplicationStatus:          ri.ReplicationStatus,
+		ReplicationStatusInternal:  ri.ReplicationStatusInternal,
+		VersionPurgeStatus:         ri.VersionPurgeStatus,
+		VersionPurgeStatusInternal: ri.VersionPurgeStatusInternal,
+		DeleteMarker:               true,
+		UserDefined:                map[string]string{},
+	}
+}
+
 // ReplicateObjectInfo represents object info to be replicated
 type ReplicateObjectInfo struct {
-	ObjectInfo
+	Name                       string
+	Bucket                     string
+	VersionID                  string
+	ETag                       string
+	Size                       int64
+	ActualSize                 int64
+	ModTime                    time.Time
+	UserTags                   string
+	SSEC                       bool
+	ReplicationStatus          replication.StatusType
+	ReplicationStatusInternal  string
+	VersionPurgeStatusInternal string
+	VersionPurgeStatus         VersionPurgeStatusType
+	ReplicationState           ReplicationState
+	DeleteMarker               bool
+
 	OpType               replication.Type
 	EventType            string
 	RetryCount           uint32
@@ -529,7 +568,7 @@ type PartInfo struct {
 	// Size in bytes of the part.
 	Size int64
 
-	// Decompressed Size.
+	// Real size of the object uploaded by client.
 	ActualSize int64
 
 	// Checksum values

cmd/object-api-errors.go

@@ -348,15 +348,6 @@ func (e InvalidUploadIDKeyCombination) Error() string {
 	return fmt.Sprintf("Invalid combination of uploadID marker '%s' and marker '%s'", e.UploadIDMarker, e.KeyMarker)
 }
 
-// InvalidMarkerPrefixCombination - invalid marker and prefix combination.
-type InvalidMarkerPrefixCombination struct {
-	Marker, Prefix string
-}
-
-func (e InvalidMarkerPrefixCombination) Error() string {
-	return fmt.Sprintf("Invalid combination of marker '%s' and prefix '%s'", e.Marker, e.Prefix)
-}
-
 // BucketPolicyNotFound - no bucket policy found.
 type BucketPolicyNotFound GenericError
 

cmd/object-api-input-checks.go

@@ -78,17 +78,6 @@ func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string, obj g
 			Object: prefix,
 		}
 	}
-	// Verify if marker has prefix.
-	if marker != "" && !HasPrefix(marker, prefix) {
-		logger.LogIf(ctx, InvalidMarkerPrefixCombination{
-			Marker: marker,
-			Prefix: prefix,
-		})
-		return InvalidMarkerPrefixCombination{
-			Marker: marker,
-			Prefix: prefix,
-		}
-	}
 	return nil
 }
 

cmd/object-api-interface.go

@@ -146,6 +146,7 @@ type DeleteBucketOptions struct {
 // BucketOptions provides options for ListBuckets and GetBucketInfo call.
 type BucketOptions struct {
 	Deleted bool // true only when site replication is enabled
+	Cached  bool // true only when we are requesting a cached response instead of hitting the disk for example ListBuckets() call.
 }
 
 // SetReplicaStatus sets replica status and timestamp for delete operations in ObjectOptions

cmd/object-api-listobjects_test.go

@@ -393,7 +393,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 	resultCases := []ListObjectsInfo{
 		// ListObjectsResult-0.
 		// Testing for listing all objects in the bucket, (testCase 20,21,22).
-		{
+		0: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -409,7 +409,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-1.
 		// Used for asserting the truncated case, (testCase 23).
-		{
+		1: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -421,7 +421,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-2.
 		// (TestCase 24).
-		{
+		2: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -432,7 +432,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-3.
 		// (TestCase 25).
-		{
+		3: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -443,7 +443,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-4.
 		// Again used for truncated case.
 		// (TestCase 26).
-		{
+		4: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -452,7 +452,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-5.
 		// Used for Asserting prefixes.
 		// Used for test case with prefix "new", (testCase 27-29).
-		{
+		5: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -463,7 +463,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-6.
 		// Used for Asserting prefixes.
 		// Used for test case with prefix = "obj", (testCase 30).
-		{
+		6: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj0"},
@@ -474,7 +474,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-7.
 		// Used for Asserting prefixes and truncation.
 		// Used for test case with prefix = "new" and maxKeys = 1, (testCase 31).
-		{
+		7: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -483,7 +483,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-8.
 		// Used for Asserting prefixes.
 		// Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32).
-		{
+		8: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "obj0"},
@@ -493,7 +493,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-9.
 		// Used for asserting the case with marker, but without prefix.
 		// marker is set to "newPrefix0" in the testCase, (testCase 33).
-		{
+		9: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix1"},
@@ -505,7 +505,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-10.
 		// marker is set to "newPrefix1" in the testCase, (testCase 34).
-		{
+		10: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newzen/zen/recurse/again/again/again/pics"},
@@ -516,7 +516,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-11.
 		// marker is set to "obj0" in the testCase, (testCase 35).
-		{
+		11: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
@@ -525,7 +525,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-12.
 		// Marker is set to "obj1" in the testCase, (testCase 36).
-		{
+		12: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj2"},
@@ -533,7 +533,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-13.
 		// Marker is set to "man" in the testCase, (testCase37).
-		{
+		13: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -546,7 +546,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-14.
 		// Marker is set to "Abc" in the testCase, (testCase 39).
-		{
+		14: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -562,7 +562,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-15.
 		// Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40).
-		{
+		15: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"},
@@ -576,7 +576,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-16.
 		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41).
-		{
+		16: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -591,7 +591,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// Used for asserting the case with marker, without prefix but with truncation.
 		// Marker =  "newPrefix0" & maxKeys = 3 in the testCase, (testCase42).
 		// Output truncated to 3 values.
-		{
+		17: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix1"},
@@ -602,7 +602,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-18.
 		// Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43).
 		// Output truncated to 1 value.
-		{
+		18: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "newzen/zen/recurse/again/again/again/pics"},
@@ -611,7 +611,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-19.
 		// Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44).
 		// Output truncated to 1 value.
-		{
+		19: {
 			IsTruncated: true,
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
@@ -619,7 +619,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-20.
 		// Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45).
-		{
+		20: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
@@ -628,7 +628,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-21.
 		// Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46).
-		{
+		21: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj2"},
@@ -636,7 +636,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-22.
 		// Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47).
-		{
+		22: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix1"},
@@ -645,7 +645,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-23.
 		// Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55).
-		{
+		23: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia/India/India-summer-photos-1"},
@@ -655,7 +655,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 
 		// ListObjectsResult-24.
 		// Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56).
-		{
+		24: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -666,7 +666,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 
 		// ListObjectsResult-25.
 		// Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57).
-		{
+		25: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia-maps.png"},
@@ -675,7 +675,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-26.
 		// prefix = "new" and delimiter is set in the testCase.(testCase 58).
-		{
+		26: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -685,7 +685,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-27.
 		// Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59).
-		{
+		27: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "Asia/India/India-summer-photos-1"},
@@ -694,7 +694,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-28.
 		// Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60).
-		{
+		28: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -707,7 +707,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-29.
 		// Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimiter set, (testCase 61).
-		{
+		29: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "newPrefix0"},
@@ -720,12 +720,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		},
 		// ListObjectsResult-30.
 		// Prefix and Delimiter is set to '/', (testCase 62).
-		{
+		30: {
 			IsTruncated: false,
 			Objects:     []ObjectInfo{},
 		},
 		// ListObjectsResult-31 Empty directory, recursive listing
-		{
+		31: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
@@ -734,7 +734,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 			},
 		},
 		// ListObjectsResult-32 Empty directory, non recursive listing
-		{
+		32: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "obj1"},
@@ -743,7 +743,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 			Prefixes: []string{"temporary/"},
 		},
 		// ListObjectsResult-33 Listing empty directory only
-		{
+		33: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "temporary/0/"},
@@ -752,12 +752,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		// ListObjectsResult-34:
 		//    * Listing with marker > last object should return empty
 		//    * Listing an object with a trailing slash and '/' delimiter
-		{
+		34: {
 			IsTruncated: false,
 			Objects:     []ObjectInfo{},
 		},
 		// ListObjectsResult-35 list with custom uncommon delimiter
-		{
+		35: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "file1/receipt.json"},
@@ -765,12 +765,12 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 			Prefixes: []string{"file1/guidSplunk"},
 		},
 		// ListObjectsResult-36 list with nextmarker prefix and maxKeys set to 1.
-		{
+		36: {
 			IsTruncated: true,
 			Prefixes:    []string{"dir/day_id=2017-10-10/"},
 		},
 		// ListObjectsResult-37 list with prefix match 2 levels deep
-		{
+		37: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "foo/201910/1112"},
@@ -778,7 +778,7 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 			},
 		},
 		// ListObjectsResult-38 list with prefix match 1 level deep
-		{
+		38: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "foo/201910/1112"},
@@ -788,14 +788,14 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 			},
 		},
 		// ListObjectsResult-39 list with prefix match 1 level deep
-		{
+		39: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "201910/foo/bar/xl.meta/1.txt"},
 			},
 		},
 		// ListObjectsResult-40
-		{
+		40: {
 			IsTruncated: false,
 			Objects: []ObjectInfo{
 				{Name: "aaa"},
@@ -829,9 +829,10 @@ func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, v
 		{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
 		{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
 		{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
-		// Testing for failure cases with both perfix and marker (11).
-		// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
-		{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},
+		// If marker is *after* the last possible object from the prefix it should return an empty list.
+		{"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true},
+		// If the marker is *before* the first possible object from the prefix it should return the first object.
+		{"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true},
 		// Setting a non-existing directory to be prefix (12-13).
 		{"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true},
 		{"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true},
@@ -1573,9 +1574,8 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
 		{"volatile-bucket-1", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false},
 		{"volatile-bucket-2", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false},
 		{"volatile-bucket-3", "", "", "", 0, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false},
-		// Testing for failure cases with both perfix and marker (9).
-		// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
-		{"test-bucket-list-object", "asia", "europe-object", "", 0, ListObjectsInfo{}, fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false},
+		// If marker is *after* the last possible object from the prefix it should return an empty list.
+		{"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true},
 		// Setting a non-existing directory to be prefix (10-11).
 		{"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true},
 		{"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true},
@@ -1671,6 +1671,7 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
 		{testBuckets[4], "file1/", "", "guidSplunk", 1000, resultCases[35], nil, true},
 		// Test listing at prefix with expected prefix markers
 		{testBuckets[5], "dir/", "", SlashSeparator, 1, resultCases[36], nil, true},
+		{"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true},
 	}
 
 	for i, testCase := range testCases {

cmd/object-api-multipart_test.go

@@ -1051,12 +1051,11 @@ func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHan
 		// Valid, existing bucket, delimiter not supported, returns empty values (Test number 8-9).
 		{bucketNames[0], "", "", "", "*", 0, ListMultipartsInfo{Delimiter: "*"}, nil, true},
 		{bucketNames[0], "", "", "", "-", 0, ListMultipartsInfo{Delimiter: "-"}, nil, true},
-		// Testing for failure cases with both perfix and marker (Test number 10).
-		// The prefix and marker combination to be valid it should satisfy strings.HasPrefix(marker, prefix).
+		// If marker is *after* the last possible object from the prefix it should return an empty list.
 		{
-			bucketNames[0], "asia", "europe-object", "", "", 0,
-			ListMultipartsInfo{},
-			fmt.Errorf("Invalid combination of marker '%s' and prefix '%s'", "europe-object", "asia"), false,
+			bucketNames[0], "Asia", "europe-object", "", "", 0,
+			ListMultipartsInfo{KeyMarker: "europe-object", Prefix: "Asia", IsTruncated: false},
+			nil, true,
 		},
 		// Setting an invalid combination of uploadIDMarker and Marker (Test number 11-12).
 		{

cmd/object-api-utils.go

@@ -342,6 +342,15 @@ func mustGetUUID() string {
 	return u.String()
 }
 
+// mustGetUUIDBytes - get a random UUID as 16 bytes unencoded.
+func mustGetUUIDBytes() []byte {
+	u, err := uuid.NewRandom()
+	if err != nil {
+		logger.CriticalIf(GlobalContext, err)
+	}
+	return u[:]
+}
+
 // Create an s3 compatible MD5sum for complete multipart transaction.
 func getCompleteMultipartMD5(parts []CompletePart) string {
 	var finalMD5Bytes []byte
@@ -507,7 +516,10 @@ func (o *ObjectInfo) IsCompressedOK() (bool, error) {
 }
 
 // GetActualSize - returns the actual size of the stored object
-func (o *ObjectInfo) GetActualSize() (int64, error) {
+func (o ObjectInfo) GetActualSize() (int64, error) {
+	if o.ActualSize != nil {
+		return *o.ActualSize, nil
+	}
 	if o.IsCompressed() {
 		sizeStr, ok := o.UserDefined[ReservedMetadataPrefix+"actual-size"]
 		if !ok {

cmd/object-handlers.go

@@ -474,7 +474,8 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 		// Automatically remove the object/version if an expiry lifecycle rule can be applied
 		if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 			rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-			event := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
+			replcfg, _ := getReplicationConfig(ctx, bucket)
+			event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo)
 			if event.Action.Delete() {
 				// apply whatever the expiry rule is.
 				applyExpiryRule(event, lcEventSrc_s3GetObject, objInfo)
@@ -732,7 +733,8 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 		// Automatically remove the object/version if an expiry lifecycle rule can be applied
 		if lc, err := globalLifecycleSys.Get(bucket); err == nil {
 			rcfg, _ := globalBucketObjectLockSys.Get(bucket)
-			event := evalActionFromLifecycle(ctx, *lc, rcfg, objInfo)
+			replcfg, _ := getReplicationConfig(ctx, bucket)
+			event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo)
 			if event.Action.Delete() {
 				// apply whatever the expiry rule is.
 				applyExpiryRule(event, lcEventSrc_s3HeadObject, objInfo)
@@ -1182,7 +1184,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
 		compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10)
 
-		reader = etag.NewReader(reader, nil)
+		reader = etag.NewReader(ctx, reader, nil, nil)
 		wantEncryption := crypto.Requested(r.Header)
 		s2c, cb := newS2CompressReader(reader, actualSize, wantEncryption)
 		dstOpts.IndexCB = cb
@@ -1195,7 +1197,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		reader = gr
 	}
 
-	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualSize)
+	srcInfo.Reader, err = hash.NewReader(ctx, reader, length, "", "", actualSize)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1316,7 +1318,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		}
 
 		// do not try to verify encrypted content
-		srcInfo.Reader, err = hash.NewReader(reader, targetSize, "", "", actualSize)
+		srcInfo.Reader, err = hash.NewReader(ctx, reader, targetSize, "", "", actualSize)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -1427,7 +1429,12 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
 		srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = rs
 	}
-	if dsc := mustReplicate(ctx, dstBucket, dstObject, getMustReplicateOptions(srcInfo, replication.UnsetReplicationType, dstOpts)); dsc.ReplicateAny() {
+
+	op := replication.ObjectReplicationType
+	if srcInfo.metadataOnly {
+		op = replication.MetadataReplicationType
+	}
+	if dsc := mustReplicate(ctx, dstBucket, dstObject, srcInfo.getMustReplicateOptions(op, dstOpts)); dsc.ReplicateAny() {
 		srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
 		srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 	}
@@ -1500,6 +1507,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 			writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL)
 			return
 		}
+		objInfo.UserDefined = cloneMSS(opts.UserMetadata)
 		objInfo.ETag = remoteObjInfo.ETag
 		objInfo.ModTime = remoteObjInfo.LastModified
 	} else {
@@ -1533,8 +1541,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime)
 	encodedSuccessResponse := encodeResponse(response)
 
-	if dsc := mustReplicate(ctx, dstBucket, dstObject, getMustReplicateOptions(objInfo, replication.UnsetReplicationType, dstOpts)); dsc.ReplicateAny() {
-		scheduleReplication(ctx, objInfo.Clone(), objectAPI, dsc, replication.ObjectReplicationType)
+	if dsc := mustReplicate(ctx, dstBucket, dstObject, objInfo.getMustReplicateOptions(replication.ObjectReplicationType, dstOpts)); dsc.ReplicateAny() {
+		scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType)
 	}
 
 	setPutObjHeaders(w, objInfo, false)
@@ -1737,7 +1745,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
 		metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10)
 
-		actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
+		actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -1757,8 +1765,20 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		md5hex = "" // Do not try to verify the content.
 		sha256hex = ""
 	}
-
-	hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
+	var hashReader *hash.Reader
+	// Optimization: If SSE-KMS and SSE-C did not request Content-Md5. Use uuid as etag
+	if !etag.ContentMD5Requested(r.Header) && (crypto.S3KMS.IsRequested(r.Header) || crypto.SSEC.IsRequested(r.Header)) {
+		hashReader, err = hash.NewReaderWithOpts(ctx, reader, hash.Options{
+			Size:       size,
+			MD5Hex:     md5hex,
+			SHA256Hex:  sha256hex,
+			ActualSize: actualSize,
+			DisableMD5: false,
+			ForceMD5:   mustGetUUIDBytes(),
+		})
+	} else {
+		hashReader, err = hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
+	}
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -1815,9 +1835,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
 	}
-	if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(ObjectInfo{
-		UserDefined: metadata,
-	}, replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
+	if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
 		metadata[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 		metadata[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
 	}
@@ -1856,7 +1874,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 
 		// do not try to verify encrypted content
-		hashReader, err = hash.NewReader(etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
+		hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -1923,10 +1941,8 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			}
 		}
 	}
-	if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(ObjectInfo{
-		UserDefined: metadata,
-	}, replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
-		scheduleReplication(ctx, objInfo.Clone(), objectAPI, dsc, replication.ObjectReplicationType)
+	if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
+		scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType)
 	}
 
 	setPutObjHeaders(w, objInfo, false)
@@ -2075,7 +2091,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		}
 	}
 
-	hreader, err := hash.NewReader(reader, size, md5hex, sha256hex, size)
+	hreader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, size)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -2126,7 +2142,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2
 			metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10)
 
-			actualReader, err := hash.NewReader(reader, size, "", "", actualSize)
+			actualReader, err := hash.NewReader(ctx, reader, size, "", "", actualSize)
 			if err != nil {
 				return err
 			}
@@ -2140,7 +2156,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			size = -1 // Since compressed size is un-predictable.
 		}
 
-		hashReader, err := hash.NewReader(reader, size, "", "", actualSize)
+		hashReader, err := hash.NewReader(ctx, reader, size, "", "", actualSize)
 		if err != nil {
 			return err
 		}
@@ -2182,9 +2198,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			return ObjectLocked{}
 		}
 
-		if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(ObjectInfo{
-			UserDefined: metadata,
-		}, replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
+		if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
 			metadata[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 			metadata[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
 
@@ -2212,7 +2226,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			}
 
 			// do not try to verify encrypted content
-			hashReader, err = hash.NewReader(etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
+			hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
 			if err != nil {
 				return err
 			}
@@ -2235,10 +2249,8 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 			return err
 		}
 
-		if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(ObjectInfo{
-			UserDefined: metadata,
-		}, replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
-			scheduleReplication(ctx, objInfo.Clone(), objectAPI, dsc, replication.ObjectReplicationType)
+		if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
+			scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType)
 		}
 
 		// Notify object created event.
@@ -2406,7 +2418,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-		if isErrObjectNotFound(err) {
+		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			writeSuccessNoContent(w)
 			return
 		}
@@ -2453,7 +2465,7 @@ func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.
 				DeleteMarkerVersionID: dmVersionID,
 				DeleteMarkerMTime:     DeleteMarkerMTime{objInfo.ModTime},
 				DeleteMarker:          objInfo.DeleteMarker,
-				ReplicationState:      objInfo.getReplicationState(),
+				ReplicationState:      objInfo.ReplicationState(),
 			},
 			Bucket:    bucket,
 			EventType: ReplicateIncomingDelete,
@@ -2528,7 +2540,7 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 			oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = strings.ToUpper(string(legalHold.Status))
 			oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano)
 
-			dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(*oi, replication.MetadataReplicationType, opts))
+			dsc := mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts))
 			if dsc.ReplicateAny() {
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
@@ -2543,9 +2555,9 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 		return
 	}
 
-	dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(objInfo, replication.MetadataReplicationType, opts))
+	dsc := mustReplicate(ctx, bucket, object, objInfo.getMustReplicateOptions(replication.MetadataReplicationType, opts))
 	if dsc.ReplicateAny() {
-		scheduleReplication(ctx, objInfo.Clone(), objectAPI, dsc, replication.MetadataReplicationType)
+		scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.MetadataReplicationType)
 	}
 
 	writeSuccessResponseHeadersOnly(w)
@@ -2697,7 +2709,7 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 				oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = ""
 			}
 			oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano)
-			dsc = mustReplicate(ctx, bucket, object, getMustReplicateOptions(*oi, replication.MetadataReplicationType, opts))
+			dsc = mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts))
 			if dsc.ReplicateAny() {
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 				oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
@@ -2712,9 +2724,9 @@ func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r
 		return
 	}
 
-	dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(objInfo, replication.MetadataReplicationType, opts))
+	dsc := mustReplicate(ctx, bucket, object, objInfo.getMustReplicateOptions(replication.MetadataReplicationType, opts))
 	if dsc.ReplicateAny() {
-		scheduleReplication(ctx, objInfo.Clone(), objectAPI, dsc, replication.MetadataReplicationType)
+		scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.MetadataReplicationType)
 	}
 
 	writeSuccessResponseHeadersOnly(w)
@@ -2923,9 +2935,7 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
 	}
 	tagsStr := tags.String()
 
-	oi := objInfo.Clone()
-	oi.UserTags = tagsStr
-	dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(oi, replication.MetadataReplicationType, opts))
+	dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(objInfo.UserDefined, tagsStr, objInfo.ReplicationStatus, replication.MetadataReplicationType, opts))
 	if dsc.ReplicateAny() {
 		opts.UserDefined = make(map[string]string)
 		opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
@@ -2941,7 +2951,7 @@ func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *h
 	}
 
 	if dsc.ReplicateAny() {
-		scheduleReplication(ctx, objInfo.Clone(), objAPI, dsc, replication.MetadataReplicationType)
+		scheduleReplication(ctx, objInfo, objAPI, dsc, replication.MetadataReplicationType)
 	}
 
 	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
@@ -3003,7 +3013,7 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 		return
 	}
 
-	dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(oi, replication.MetadataReplicationType, opts))
+	dsc := mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts))
 	if dsc.ReplicateAny() {
 		opts.UserDefined = make(map[string]string)
 		opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
@@ -3017,7 +3027,7 @@ func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r
 	}
 
 	if dsc.ReplicateAny() {
-		scheduleReplication(ctx, oi.Clone(), objAPI, dsc, replication.MetadataReplicationType)
+		scheduleReplication(ctx, oi, objAPI, dsc, replication.MetadataReplicationType)
 	}
 
 	if oi.VersionID != "" && oi.VersionID != nullVersionID {

cmd/object-lambda-handlers.go

@@ -70,11 +70,8 @@ func getLambdaEventData(bucket, object string, cred auth.Credentials, r *http.Re
 			reqParams.Set(k, v)
 		}
 	}
-	extraHeaders := http.Header{}
-	if rng := r.Header.Get(xhttp.Range); rng != "" {
-		extraHeaders.Set(xhttp.Range, r.Header.Get(xhttp.Range))
-	}
 
+	extraHeaders := http.Header{}
 	u, err := clnt.PresignHeader(r.Context(), http.MethodGet, bucket, object, duration, reqParams, extraHeaders)
 	if err != nil {
 		return levent.Event{}, err

cmd/object-multipart-handlers.go

@@ -164,9 +164,7 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
 	}
-	if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(ObjectInfo{
-		UserDefined: metadata,
-	}, replication.ObjectReplicationType, ObjectOptions{})); dsc.ReplicateAny() {
+	if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, ObjectOptions{})); dsc.ReplicateAny() {
 		metadata[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano)
 		metadata[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus()
 	}
@@ -450,7 +448,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 	}
 
 	actualPartSize = length
-	var reader io.Reader = etag.NewReader(gr, nil)
+	var reader io.Reader = etag.NewReader(ctx, gr, nil, nil)
 
 	mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts)
 	if err != nil {
@@ -473,7 +471,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		length = -1
 	}
 
-	srcInfo.Reader, err = hash.NewReader(reader, length, "", "", actualPartSize)
+	srcInfo.Reader, err = hash.NewReader(ctx, reader, length, "", "", actualPartSize)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -528,7 +526,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			wantSize = info.EncryptedSize()
 		}
 
-		srcInfo.Reader, err = hash.NewReader(reader, wantSize, "", "", actualPartSize)
+		srcInfo.Reader, err = hash.NewReader(ctx, reader, wantSize, "", "", actualPartSize)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -717,7 +715,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 	_, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"]
 	var idxCb func() []byte
 	if isCompressed {
-		actualReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
+		actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -738,7 +736,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		sha256hex = ""
 	}
 
-	hashReader, err := hash.NewReader(reader, size, md5hex, sha256hex, actualSize)
+	hashReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -800,7 +798,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			wantSize = info.EncryptedSize()
 		}
 		// do not try to verify encrypted content
-		hashReader, err = hash.NewReader(etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
+		hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
@@ -957,6 +955,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	opts.Versioned = versioned
+	opts.VersionSuspended = suspended
 
 	// First, we compute the ETag of the multipart object.
 	// The ETag of a multi-part object is always:
@@ -997,8 +997,8 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	}
 
 	setPutObjHeaders(w, objInfo, false)
-	if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(objInfo, replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
-		scheduleReplication(ctx, objInfo.Clone(), objectAPI, dsc, replication.ObjectReplicationType)
+	if dsc := mustReplicate(ctx, bucket, object, objInfo.getMustReplicateOptions(replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
+		scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType)
 	}
 	if _, ok := r.Header[xhttp.MinIOSourceReplicationRequest]; ok {
 		actualSize, _ := objInfo.GetActualSize()
@@ -1075,8 +1075,13 @@ func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter,
 	}
 	opts := ObjectOptions{}
 	if err := abortMultipartUpload(ctx, bucket, object, uploadID, opts); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
+		switch err.(type) {
+		case InvalidUploadID:
+			// Do not have return an error for non-existent upload-id
+		default:
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
 	}
 
 	writeSuccessNoContent(w)

cmd/os-instrumented.go

@@ -129,15 +129,15 @@ func Mkdir(dirPath string, mode os.FileMode) (err error) {
 }
 
 // MkdirAll captures time taken to call os.MkdirAll
-func MkdirAll(dirPath string, mode os.FileMode) (err error) {
+func MkdirAll(dirPath string, mode os.FileMode, baseDir string) (err error) {
 	defer updateOSMetrics(osMetricMkdirAll, dirPath)(err)
-	return osMkdirAll(dirPath, mode)
+	return osMkdirAll(dirPath, mode, baseDir)
 }
 
 // Rename captures time taken to call os.Rename
 func Rename(src, dst string) (err error) {
 	defer updateOSMetrics(osMetricRename, src, dst)(err)
-	return os.Rename(src, dst)
+	return RenameSys(src, dst)
 }
 
 // OpenFile captures time taken to call os.OpenFile

cmd/os-reliable.go

@@ -73,7 +73,7 @@ func reliableRemoveAll(dirPath string) (err error) {
 // Wrapper functions to os.MkdirAll, which calls reliableMkdirAll
 // this is to ensure that if there is a racy parent directory
 // delete in between we can simply retry the operation.
-func mkdirAll(dirPath string, mode os.FileMode) (err error) {
+func mkdirAll(dirPath string, mode os.FileMode, baseDir string) (err error) {
 	if dirPath == "" {
 		return errInvalidArgument
 	}
@@ -82,7 +82,7 @@ func mkdirAll(dirPath string, mode os.FileMode) (err error) {
 		return err
 	}
 
-	if err = reliableMkdirAll(dirPath, mode); err != nil {
+	if err = reliableMkdirAll(dirPath, mode, baseDir); err != nil {
 		// File path cannot be verified since one of the parents is a file.
 		if isSysErrNotDir(err) {
 			return errFileAccessDenied
@@ -100,11 +100,11 @@ func mkdirAll(dirPath string, mode os.FileMode) (err error) {
 
 // Reliably retries os.MkdirAll if for some reason os.MkdirAll returns
 // syscall.ENOENT (parent does not exist).
-func reliableMkdirAll(dirPath string, mode os.FileMode) (err error) {
+func reliableMkdirAll(dirPath string, mode os.FileMode, baseDir string) (err error) {
 	i := 0
 	for {
 		// Creates all the parent directories, with mode 0777 mkdir honors system umask.
-		if err = osMkdirAll(dirPath, mode); err != nil {
+		if err = osMkdirAll(dirPath, mode, baseDir); err != nil {
 			// Retry only for the first retryable error.
 			if osIsNotExist(err) && i == 0 {
 				i++
@@ -120,7 +120,7 @@ func reliableMkdirAll(dirPath string, mode os.FileMode) (err error) {
 // and reliableRenameAll. This is to ensure that if there is a
 // racy parent directory delete in between we can simply retry
 // the operation.
-func renameAll(srcFilePath, dstFilePath string) (err error) {
+func renameAll(srcFilePath, dstFilePath, baseDir string) (err error) {
 	if srcFilePath == "" || dstFilePath == "" {
 		return errInvalidArgument
 	}
@@ -132,7 +132,7 @@ func renameAll(srcFilePath, dstFilePath string) (err error) {
 		return err
 	}
 
-	if err = reliableRename(srcFilePath, dstFilePath); err != nil {
+	if err = reliableRename(srcFilePath, dstFilePath, baseDir); err != nil {
 		switch {
 		case isSysErrNotDir(err) && !osIsNotExist(err):
 			// Windows can have both isSysErrNotDir(err) and osIsNotExist(err) returning
@@ -162,8 +162,8 @@ func renameAll(srcFilePath, dstFilePath string) (err error) {
 
 // Reliably retries os.RenameAll if for some reason os.RenameAll returns
 // syscall.ENOENT (parent does not exist).
-func reliableRename(srcFilePath, dstFilePath string) (err error) {
-	if err = reliableMkdirAll(path.Dir(dstFilePath), 0o777); err != nil {
+func reliableRename(srcFilePath, dstFilePath, baseDir string) (err error) {
+	if err = reliableMkdirAll(path.Dir(dstFilePath), 0o777, baseDir); err != nil {
 		return err
 	}
 

cmd/os-reliable_test.go

@@ -29,15 +29,15 @@ func TestOSMkdirAll(t *testing.T) {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
 
-	if err = mkdirAll("", 0o777); err != errInvalidArgument {
+	if err = mkdirAll("", 0o777, ""); err != errInvalidArgument {
 		t.Fatal("Unexpected error", err)
 	}
 
-	if err = mkdirAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0o777); err != errFileNameTooLong {
+	if err = mkdirAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0o777, ""); err != errFileNameTooLong {
 		t.Fatal("Unexpected error", err)
 	}
 
-	if err = mkdirAll(pathJoin(path, "success-vol", "success-object"), 0o777); err != nil {
+	if err = mkdirAll(pathJoin(path, "success-vol", "success-object"), 0o777, ""); err != nil {
 		t.Fatal("Unexpected error", err)
 	}
 }
@@ -50,25 +50,25 @@ func TestOSRenameAll(t *testing.T) {
 		t.Fatalf("Unable to create xlStorage test setup, %s", err)
 	}
 
-	if err = mkdirAll(pathJoin(path, "testvolume1"), 0o777); err != nil {
+	if err = mkdirAll(pathJoin(path, "testvolume1"), 0o777, ""); err != nil {
 		t.Fatal(err)
 	}
-	if err = renameAll("", "foo"); err != errInvalidArgument {
+	if err = renameAll("", "foo", ""); err != errInvalidArgument {
 		t.Fatal(err)
 	}
-	if err = renameAll("foo", ""); err != errInvalidArgument {
+	if err = renameAll("foo", "", ""); err != errInvalidArgument {
 		t.Fatal(err)
 	}
-	if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != nil {
+	if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2"), ""); err != nil {
 		t.Fatal(err)
 	}
-	if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2")); err != errFileNotFound {
+	if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2"), ""); err != errFileNotFound {
 		t.Fatal(err)
 	}
-	if err = renameAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2")); err != errFileNameTooLong {
+	if err = renameAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2"), ""); err != errFileNameTooLong {
 		t.Fatal("Unexpected error", err)
 	}
-	if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001")); err != errFileNameTooLong {
+	if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), ""); err != errFileNameTooLong {
 		t.Fatal("Unexpected error", err)
 	}
 }

cmd/os-rename_linux.go

@@ -21,6 +21,8 @@
 package cmd
 
 import (
+	"syscall"
+
 	"golang.org/x/sys/unix"
 )
 
@@ -29,3 +31,8 @@ func Rename2(src, dst string) (err error) {
 	defer updateOSMetrics(osMetricRename2, src, dst)(err)
 	return unix.Renameat2(unix.AT_FDCWD, src, unix.AT_FDCWD, dst, uint(2)) // RENAME_EXCHANGE from 'man renameat2'
 }
+
+// RenameSys is low level call in case of Linux this uses syscall.Rename() directly.
+func RenameSys(src, dst string) (err error) {
+	return syscall.Rename(src, dst)
+}

cmd/os-rename_nolinux.go

@@ -20,10 +20,18 @@
 
 package cmd
 
-import "errors"
+import (
+	"errors"
+	"os"
+)
 
 // Rename2 is not implemented in a non linux environment
 func Rename2(src, dst string) (err error) {
 	defer updateOSMetrics(osMetricRename2, src, dst)(errors.New("not implemented, skipping"))
 	return errSkipFile
 }
+
+// RenameSys is low level call in case of non-Linux this just uses os.Rename()
+func RenameSys(src, dst string) (err error) {
+	return os.Rename(src, dst)
+}

cmd/os_other.go

@@ -31,7 +31,8 @@ func access(name string) error {
 	return err
 }
 
-func osMkdirAll(dirPath string, perm os.FileMode) error {
+func osMkdirAll(dirPath string, perm os.FileMode, _ string) error {
+	// baseDir is not honored in plan9 and solaris platforms.
 	return os.MkdirAll(dirPath, perm)
 }
 

cmd/os_unix.go

@@ -24,6 +24,7 @@ import (
 	"bytes"
 	"fmt"
 	"os"
+	"strings"
 	"sync"
 	"syscall"
 	"unsafe"
@@ -47,7 +48,13 @@ func access(name string) error {
 // directories that MkdirAll creates.
 // If path is already a directory, MkdirAll does nothing
 // and returns nil.
-func osMkdirAll(dirPath string, perm os.FileMode) error {
+func osMkdirAll(dirPath string, perm os.FileMode, baseDir string) error {
+	if baseDir != "" {
+		if strings.HasPrefix(baseDir, dirPath) {
+			return nil
+		}
+	}
+
 	// Slow path: make sure parent exists and then call Mkdir for path.
 	i := len(dirPath)
 	for i > 0 && os.IsPathSeparator(dirPath[i-1]) { // Skip trailing path separator.
@@ -61,7 +68,7 @@ func osMkdirAll(dirPath string, perm os.FileMode) error {
 
 	if j > 1 {
 		// Create parent.
-		if err := osMkdirAll(dirPath[:j-1], perm); err != nil {
+		if err := osMkdirAll(dirPath[:j-1], perm, baseDir); err != nil {
 			return err
 		}
 	}

cmd/os_windows.go

@@ -31,7 +31,8 @@ func access(name string) error {
 	return err
 }
 
-func osMkdirAll(dirPath string, perm os.FileMode) error {
+func osMkdirAll(dirPath string, perm os.FileMode, _ string) error {
+	// baseDir is not honored in windows platform
 	return os.MkdirAll(dirPath, perm)
 }
 

cmd/peer-rest-client.go

@@ -229,6 +229,33 @@ func (client *peerRESTClient) GetMetrics(ctx context.Context, t madmin.MetricTyp
 	return info, err
 }
 
+func (client *peerRESTClient) GetResourceMetrics(ctx context.Context) (<-chan Metric, error) {
+	respBody, err := client.callWithContext(ctx, peerRESTMethodResourceMetrics, nil, nil, -1)
+	if err != nil {
+		return nil, err
+	}
+	dec := gob.NewDecoder(respBody)
+	ch := make(chan Metric)
+	go func(ch chan<- Metric) {
+		defer func() {
+			xhttp.DrainBody(respBody)
+			close(ch)
+		}()
+		for {
+			var metric Metric
+			if err := dec.Decode(&metric); err != nil {
+				return
+			}
+			select {
+			case <-ctx.Done():
+				return
+			case ch <- metric:
+			}
+		}
+	}(ch)
+	return ch, nil
+}
+
 // GetProcInfo - fetch MinIO process information for a remote node.
 func (client *peerRESTClient) GetProcInfo(ctx context.Context) (info madmin.ProcInfo, err error) {
 	respBody, err := client.callWithContext(ctx, peerRESTMethodProcInfo, nil, nil, -1)

cmd/peer-rest-common.go

@@ -77,6 +77,7 @@ const (
 	peerRESTMethodDevNull                     = "/devnull"
 	peerRESTMethodNetperf                     = "/netperf"
 	peerRESTMethodMetrics                     = "/metrics"
+	peerRESTMethodResourceMetrics             = "/resourcemetrics"
 	peerRESTMethodGetReplicationMRF           = "/getreplicationmrf"
 	peerRESTMethodGetSRMetrics                = "/getsrmetrics"
 )

cmd/peer-rest-server.go

@@ -474,6 +474,22 @@ func (s *peerRESTServer) GetMetricsHandler(w http.ResponseWriter, r *http.Reques
 	logger.LogIf(ctx, gob.NewEncoder(w).Encode(info))
 }
 
+func (s *peerRESTServer) GetResourceMetrics(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		s.writeErrorResponse(w, errors.New("invalid request"))
+		return
+	}
+
+	enc := gob.NewEncoder(w)
+
+	for m := range ReportMetrics(r.Context(), resourceMetricsGroups) {
+		if err := enc.Encode(m); err != nil {
+			s.writeErrorResponse(w, errors.New("Encoding metric failed: "+err.Error()))
+			return
+		}
+	}
+}
+
 // GetSysConfigHandler - returns system config information.
 // (only the config that are of concern to minio)
 func (s *peerRESTServer) GetSysConfigHandler(w http.ResponseWriter, r *http.Request) {
@@ -939,7 +955,7 @@ func (s *peerRESTServer) ListenHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Listen Publisher uses nonblocking publish and hence does not wait for slow subscribers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	ch := make(chan event.Event, 2000)
+	ch := make(chan event.Event, globalAPIConfig.getRequestsPoolCapacity())
 
 	err := globalHTTPListen.Subscribe(mask, ch, doneCh, func(ev event.Event) bool {
 		if ev.S3.Bucket.Name != "" && values.Get(peerRESTListenBucket) != "" {
@@ -994,7 +1010,7 @@ func (s *peerRESTServer) TraceHandler(w http.ResponseWriter, r *http.Request) {
 
 	// Trace Publisher uses nonblocking publish and hence does not wait for slow subscribers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	ch := make(chan madmin.TraceInfo, 2000)
+	ch := make(chan madmin.TraceInfo, 100000)
 	err = globalTrace.Subscribe(traceOpts.TraceTypes(), ch, r.Context().Done(), func(entry madmin.TraceInfo) bool {
 		return shouldTrace(entry, traceOpts)
 	})
@@ -1143,13 +1159,13 @@ func (s *peerRESTServer) ConsoleLogHandler(w http.ResponseWriter, r *http.Reques
 	doneCh := make(chan struct{})
 	defer close(doneCh)
 
-	ch := make(chan log.Info, 2000)
+	ch := make(chan log.Info, 100000)
 	err := globalConsoleSys.Subscribe(ch, doneCh, "", 0, madmin.LogMaskAll, nil)
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
-	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	keepAliveTicker := time.NewTicker(time.Second)
 	defer keepAliveTicker.Stop()
 
 	enc := gob.NewEncoder(w)
@@ -1438,6 +1454,7 @@ func registerPeerRESTHandlers(router *mux.Router) {
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodProcInfo).HandlerFunc(h(server.GetProcInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMemInfo).HandlerFunc(h(server.GetMemInfoHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodMetrics).HandlerFunc(h(server.GetMetricsHandler)).Queries(restQueries(peerRESTMetricsTypes)...)
+	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodResourceMetrics).HandlerFunc(h(server.GetResourceMetrics))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysErrors).HandlerFunc(h(server.GetSysErrorsHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysServices).HandlerFunc(h(server.GetSysServicesHandler))
 	subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSysConfig).HandlerFunc(h(server.GetSysConfigHandler))

cmd/peer-s3-client.go

@@ -23,6 +23,7 @@ import (
 	"errors"
 	"io"
 	"net/url"
+	"sort"
 	"strconv"
 
 	xhttp "github.com/minio/minio/internal/http"
@@ -123,8 +124,10 @@ func NewS3PeerSys(endpoints EndpointServerPools) *S3PeerSys {
 	}
 }
 
-// ListBuckets lists buckets across all servers and returns a possible consistent view
-func (sys *S3PeerSys) ListBuckets(ctx context.Context, opts BucketOptions) (result []BucketInfo, err error) {
+// ListBuckets lists buckets across all nodes and returns a consistent view:
+//   - Return an error when a pool cannot return N/2+1 valid bucket information
+//   - For each pool, check if the bucket exists in N/2+1 nodes before including it in the final result
+func (sys *S3PeerSys) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) {
 	g := errgroup.WithNErrs(len(sys.peerClients))
 
 	nodeBuckets := make([][]BucketInfo, len(sys.peerClients))
@@ -148,25 +151,52 @@ func (sys *S3PeerSys) ListBuckets(ctx context.Context, opts BucketOptions) (resu
 
 	errs = append(errs, g.Wait()...)
 
-	quorum := len(sys.peerClients)/2 + 1
-	if err = reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, quorum); err != nil {
-		return nil, err
-	}
+	// The list of buckets in a map to avoid duplication
+	resultMap := make(map[string]BucketInfo)
 
-	bucketsMap := make(map[string]struct{})
-	for idx, buckets := range nodeBuckets {
-		if errs[idx] != nil {
-			continue
+	for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ {
+		perPoolErrs := make([]error, 0, len(sys.peerClients))
+		for i, client := range sys.peerClients {
+			if slices.Contains(client.GetPools(), poolIdx) {
+				perPoolErrs = append(perPoolErrs, errs[i])
+			}
 		}
-		for _, bi := range buckets {
-			_, ok := bucketsMap[bi.Name]
-			if !ok {
-				bucketsMap[bi.Name] = struct{}{}
-				result = append(result, bi)
+		quorum := len(perPoolErrs)/2 + 1
+		if poolErr := reduceWriteQuorumErrs(ctx, perPoolErrs, bucketOpIgnoredErrs, quorum); poolErr != nil {
+			return nil, poolErr
+		}
+
+		bucketsMap := make(map[string]int)
+		for idx, buckets := range nodeBuckets {
+			if buckets == nil {
+				continue
+			}
+			if !slices.Contains(sys.peerClients[idx].GetPools(), poolIdx) {
+				continue
+			}
+			for _, bi := range buckets {
+				_, ok := resultMap[bi.Name]
+				if ok {
+					// Skip it, this bucket is found in another pool
+					continue
+				}
+				bucketsMap[bi.Name]++
+				if bucketsMap[bi.Name] == quorum {
+					resultMap[bi.Name] = bi
+				}
 			}
 		}
 	}
 
+	result := make([]BucketInfo, 0, len(resultMap))
+	for _, bi := range resultMap {
+		result = append(result, bi)
+	}
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Name < result[j].Name
+	})
+
 	return result, nil
 }
 

cmd/peer-s3-server.go

@@ -22,7 +22,6 @@ import (
 	"encoding/gob"
 	"errors"
 	"net/http"
-	"sort"
 
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
@@ -124,10 +123,6 @@ func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []Bucket
 		}
 	}
 
-	sort.Slice(buckets, func(i, j int) bool {
-		return buckets[i].Name < buckets[j].Name
-	})
-
 	return buckets, nil
 }
 

cmd/perf-tests.go

@@ -306,7 +306,7 @@ func netperf(ctx context.Context, duration time.Duration) madmin.NetperfNodeResu
 					defer wg.Done()
 					err := globalNotificationSys.peerClients[index].DevNull(ctx, r)
 					if err != nil {
-						errStr = err.Error()
+						errStr = fmt.Sprintf("error with %s: %s", globalNotificationSys.peerClients[index].String(), err.Error())
 					}
 				}()
 			}