fix: file consistency issue on SFTP upload (#18422 )

* creating a byte buffer for SFTP file segments * Adding an error condition for when there are remaining segments in the queue * Simplification of the queue using a map
Avoid allocs for MRF on-disk header (#18425 )
2026-08-14 10:43:15 +03:00 · 2023-11-11 00:14:41 -08:00 · 2023-11-10 19:54:46 -08:00 · 2023-11-10 16:12:35 -08:00 · 2023-11-10 10:10:14 -08:00 · 2023-11-10 02:15:13 -08:00
270 changed files with 8547 additions and 12931 deletions
@@ -1,10 +1,17 @@
 .git
 .github
-docs
 default.etcd
 *.gz
 *.tar.gz
 *.bzip2
 *.zip
 browser/node_modules
-node_modules
+node_modules
+docs/debugging/s3-verify/s3-verify
+docs/debugging/xl-meta/xl-meta
+docs/debugging/s3-check-md5/s3-check-md5
+docs/debugging/hash-set/hash-set
+docs/debugging/healing-bin/healing-bin
+docs/debugging/inspect/inspect
+docs/debugging/pprofgoparser/pprofgoparser
+docs/debugging/reorder-disks/reorder-disks
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
    - master
+    - next

 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
    - master
+    - next

 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
    - master
+    - next

 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
    - master
+    - next

 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
    - master
+    - next

 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
    - master
+    - next

 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
    - master
+    - next

 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
      - master
+      - next

 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -37,7 +38,11 @@ jobs:

      - name: build-minio
        run: |
-          TAG="minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
+          TAG="quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}" make docker
+
+      - name: multipart uploads test
+        run: |
+          ${GITHUB_WORKSPACE}/.github/workflows/multipart/migrate.sh "${{ steps.vars.outputs.sha_short }}"

      - name: compress and encrypt
        run: |
@@ -59,7 +64,14 @@ jobs:
             docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true
             docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true
          done
-          docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }}
+
+          docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site1.yaml rm -s -f || true
+          docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site2.yaml rm -s -f || true
+          for volume in $(docker volume ls -q | grep minio); do
+            docker volume rm ${volume} || true
+          done
+
+          docker rmi -f quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}
          docker system prune -f || true
          docker volume prune -f || true
          docker volume rm $(docker volume ls -q -f dangling=true) || true
@@ -2,7 +2,7 @@ version: '3.7'

 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
  command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
  expose:
    - "9000"
@@ -16,10 +16,10 @@ x-minio-common: &minio-common
    MINIO_COMPRESSION_ALLOW_ENCRYPTION: "on"
    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
  healthcheck:
-    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-    interval: 30s
-    timeout: 20s
-    retries: 3
+    test: ["CMD", "mc", "ready", "local"]
+    interval: 5s
+    timeout: 5s
+    retries: 5

 # starts 4 docker containers running minio server instances.
 # using nginx reverse proxy, load balancing, you can access
@@ -2,7 +2,7 @@ version: '3.7'

 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
  command: server --console-address ":9001" edata{1...4}
  expose:
    - "9000"
@@ -13,10 +13,10 @@ x-minio-common: &minio-common
    MINIO_ROOT_PASSWORD: "minio123"
    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
  healthcheck:
-    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-    interval: 30s
-    timeout: 20s
-    retries: 3
+    test: ["CMD", "mc", "ready", "local"]
+    interval: 5s
+    timeout: 5s
+    retries: 5

 # starts 4 docker containers running minio server instances.
 # using nginx reverse proxy, load balancing, you can access
@@ -2,7 +2,7 @@ version: '3.7'

 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: minio/minio:${JOB_NAME}
+  image: quay.io/minio/minio:${JOB_NAME}
  command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
  expose:
    - "9000"
@@ -13,10 +13,10 @@ x-minio-common: &minio-common
    MINIO_ROOT_PASSWORD: "minio123"
    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
  healthcheck:
-    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-    interval: 30s
-    timeout: 20s
-    retries: 3
+    test: ["CMD", "mc", "ready", "local"]
+    interval: 5s
+    timeout: 5s
+    retries: 5

 # starts 4 docker containers running minio server instances.
 # using nginx reverse proxy, load balancing, you can access
@@ -0,0 +1,66 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:${RELEASE}
+  command: server http://site1-minio{1...4}/data{1...2}
+  environment:
+    - MINIO_PROMETHEUS_AUTH_TYPE=public
+    - CI=true
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  site1-minio1:
+    <<: *minio-common
+    hostname: site1-minio1
+    volumes:
+      - site1-data1-1:/data1
+      - site1-data1-2:/data2
+
+  site1-minio2:
+    <<: *minio-common
+    hostname: site1-minio2
+    volumes:
+      - site1-data2-1:/data1
+      - site1-data2-2:/data2
+
+  site1-minio3:
+    <<: *minio-common
+    hostname: site1-minio3
+    volumes:
+      - site1-data3-1:/data1
+      - site1-data3-2:/data2
+
+  site1-minio4:
+    <<: *minio-common
+    hostname: site1-minio4
+    volumes:
+      - site1-data4-1:/data1
+      - site1-data4-2:/data2
+
+  site1-nginx:
+    image: nginx:1.19.2-alpine
+    hostname: site1-nginx
+    volumes:
+      - ./nginx-site1.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9001:9001"
+    depends_on:
+      - site1-minio1
+      - site1-minio2
+      - site1-minio3
+      - site1-minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  site1-data1-1:
+  site1-data1-2:
+  site1-data2-1:
+  site1-data2-2:
+  site1-data3-1:
+  site1-data3-2:
+  site1-data4-1:
+  site1-data4-2:
@@ -0,0 +1,66 @@
+version: '3.7'
+
+# Settings and configurations that are common for all containers
+x-minio-common: &minio-common
+  image: quay.io/minio/minio:${RELEASE}
+  command: server http://site2-minio{1...4}/data{1...2}
+  environment:
+    - MINIO_PROMETHEUS_AUTH_TYPE=public
+    - CI=true
+
+# starts 4 docker containers running minio server instances.
+# using nginx reverse proxy, load balancing, you can access
+# it through port 9000.
+services:
+  site2-minio1:
+    <<: *minio-common
+    hostname: site2-minio1
+    volumes:
+      - site2-data1-1:/data1
+      - site2-data1-2:/data2
+
+  site2-minio2:
+    <<: *minio-common
+    hostname: site2-minio2
+    volumes:
+      - site2-data2-1:/data1
+      - site2-data2-2:/data2
+
+  site2-minio3:
+    <<: *minio-common
+    hostname: site2-minio3
+    volumes:
+      - site2-data3-1:/data1
+      - site2-data3-2:/data2
+
+  site2-minio4:
+    <<: *minio-common
+    hostname: site2-minio4
+    volumes:
+      - site2-data4-1:/data1
+      - site2-data4-2:/data2
+
+  site2-nginx:
+    image: nginx:1.19.2-alpine
+    hostname: site2-nginx
+    volumes:
+      - ./nginx-site2.conf:/etc/nginx/nginx.conf:ro
+    ports:
+      - "9002:9002"
+    depends_on:
+      - site2-minio1
+      - site2-minio2
+      - site2-minio3
+      - site2-minio4
+
+## By default this config uses default local driver,
+## For custom volumes replace with volume driver configuration.
+volumes:
+  site2-data1-1:
+  site2-data1-2:
+  site2-data2-1:
+  site2-data2-2:
+  site2-data3-1:
+  site2-data3-2:
+  site2-data4-1:
+  site2-data4-2:
@@ -0,0 +1,115 @@
+#!/bin/bash
+
+set -x
+
+## change working directory
+cd .github/workflows/multipart/
+
+docker-compose -f docker-compose-site1.yaml rm -s -f
+docker-compose -f docker-compose-site2.yaml rm -s -f
+for volume in $(docker volume ls -q | grep minio); do
+	docker volume rm ${volume}
+done
+
+if [ ! -f ./mc ]; then
+	wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc &&
+		chmod +x mc
+fi
+
+(
+	cd /tmp
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+)
+
+export RELEASE=RELEASE.2023-08-29T23-07-35Z
+
+docker-compose -f docker-compose-site1.yaml up -d
+docker-compose -f docker-compose-site2.yaml up -d
+
+sleep 30s
+
+./mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4
+./mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4
+
+./mc ready site1/
+./mc ready site2/
+
+./mc admin replicate add site1 site2
+./mc mb site1/testbucket/
+./mc cp -r --quiet /usr/bin site1/testbucket/
+
+sleep 5
+
+s3-check-md5 -h
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads"
+	exit 1
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads"
+	exit 1
+fi
+
+./mc cp -r --quiet /usr/bin site1/testbucket/
+
+sleep 5
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+## we do not need to fail here, since we are going to test
+## upgrading to master, healing and being able to recover
+## the last version.
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads ${failed_count_site1}"
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads ${failed_count_site2}"
+fi
+
+export RELEASE=${1}
+
+docker-compose -f docker-compose-site1.yaml up -d
+docker-compose -f docker-compose-site2.yaml up -d
+
+./mc ready site1/
+./mc ready site2/
+
+for i in $(seq 1 10); do
+	# mc admin heal -r --remove when used against a LB endpoint
+	# behaves flaky, let this run 10 times before giving up
+	./mc admin heal -r --remove --json site1/ 2>&1 >/dev/null
+	./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null
+done
+
+failed_count_site1=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+failed_count_site2=$(s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l)
+
+if [ $failed_count_site1 -ne 0 ]; then
+	echo "failed with multipart on site1 uploads"
+	exit 1
+fi
+
+if [ $failed_count_site2 -ne 0 ]; then
+	echo "failed with multipart on site2 uploads"
+	exit 1
+fi
+
+docker-compose -f docker-compose-site1.yaml rm -s -f
+docker-compose -f docker-compose-site2.yaml rm -s -f
+for volume in $(docker volume ls -q | grep minio); do
+	docker volume rm ${volume}
+done
+
+docker system prune -f || true
+docker volume prune -f || true
+docker volume rm $(docker volume ls -q -f dangling=true) || true
+
+## change working directory
+cd ../../../
@@ -0,0 +1,61 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server site1-minio1:9000;
+        server site1-minio2:9000;
+        server site1-minio3:9000;
+        server site1-minio4:9000;
+    }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}
@@ -0,0 +1,61 @@
+user  nginx;
+worker_processes  auto;
+
+error_log  /var/log/nginx/error.log warn;
+pid        /var/run/nginx.pid;
+
+events {
+    worker_connections  4096;
+}
+
+http {
+    include       /etc/nginx/mime.types;
+    default_type  application/octet-stream;
+
+    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
+                      '$status $body_bytes_sent "$http_referer" '
+                      '"$http_user_agent" "$http_x_forwarded_for"';
+
+    access_log  /var/log/nginx/access.log  main;
+    sendfile        on;
+    keepalive_timeout  65;
+
+    # include /etc/nginx/conf.d/*.conf;
+
+    upstream minio {
+        server site2-minio1:9000;
+        server site2-minio2:9000;
+        server site2-minio3:9000;
+        server site2-minio4:9000;
+    }
+
+    server {
+        listen       9002;
+        listen  [::]:9002;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+        proxy_request_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://minio;
+        }
+    }
+}
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
    - master
+    - next

 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
    - master
+    - next

 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -16,7 +16,7 @@ docker volume rm $(docker volume ls -f dangling=true) || true
 cd .github/workflows/mint

 docker-compose -f minio-${MODE}.yaml up -d
-sleep 5m
+sleep 30s

 docker system prune -f || true
 docker volume prune -f || true
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
    - master
+    - next

 permissions:
  contents: read
@@ -4,6 +4,7 @@ on:
  pull_request:
    branches:
    - master
+    - next

 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
@@ -3,9 +3,11 @@ on:
  pull_request:
    branches:
      - master
+      - next
  push:
    branches:
      - master
+      - next

 permissions:
  contents: read # to fetch code (actions/checkout)
@@ -20,7 +22,7 @@ jobs:
      - name: Set up Go
        uses: actions/setup-go@v3
        with:
-          go-version: 1.21.0
+          go-version: 1.21.4
          check-latest: true
      - name: Get official govulncheck
        run: go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -32,6 +32,8 @@ minio.RELEASE*
 mc
 nancy
 inspects/*
+.bin/
+*.gz
 docs/debugging/s3-verify/s3-verify
 docs/debugging/xl-meta/xl-meta
 docs/debugging/s3-check-md5/s3-check-md5
@@ -39,5 +41,5 @@ docs/debugging/hash-set/hash-set
 docs/debugging/healing-bin/healing-bin
 docs/debugging/inspect/inspect
 docs/debugging/pprofgoparser/pprofgoparser
-.bin/
-*.gz
+docs/debugging/reorder-disks/reorder-disks
+docs/debugging/populate-hard-links/populate-hardlinks
@@ -13,7 +13,6 @@ linters:
  enable:
    - durationcheck
    - gocritic
-    - gofmt
    - gofumpt
    - goimports
    - gomodguard
@@ -1,8 +1,6 @@
 FROM minio/minio:latest

-ENV PATH=/opt/bin:$PATH
-
-COPY ./minio /opt/bin/minio
+COPY ./minio /usr/bin/minio
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh

 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
@@ -1,4 +1,31 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
+FROM golang:1.21-alpine as build
+
+ARG TARGETARCH
+ARG RELEASE
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+# Install curl and minisign
+RUN apk add -U --no-cache ca-certificates && \
+    apk add -U --no-cache curl && \
+    go install aead.dev/minisign/cmd/minisign@v0.2.0
+
+# Download minio binary and signature file
+RUN curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
+    curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    chmod +x /go/bin/minio
+
+# Download mc binary and signature file
+RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    chmod +x /go/bin/mc
+
+# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
+RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
+    minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
+
+FROM registry.access.redhat.com/ubi9/ubi-micro:9.2

 ARG RELEASE

@@ -17,33 +44,18 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
    MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
    MINIO_CONFIG_ENV_FILE=config.env \
-    PATH=/opt/bin:$PATH
+    MC_CONFIG_DIR=/tmp/.mc
+
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/mc /usr/bin/mc

-COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE
-
-RUN \
-     microdnf clean all && \
-     microdnf update --nodocs && \
-     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
-     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
-     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE} -o /opt/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
-     microdnf clean all && \
-     chmod +x /opt/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     /usr/bin/verify-minio.sh && \
-     microdnf clean all
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh

 EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
 VOLUME ["/data"]

+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 CMD ["minio"]
@@ -1,6 +1,31 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
+FROM golang:1.21-alpine as build

 ARG TARGETARCH
+ARG RELEASE
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+# Install curl and minisign
+RUN apk add -U --no-cache ca-certificates && \
+    apk add -U --no-cache curl && \
+    go install aead.dev/minisign/cmd/minisign@v0.2.0
+
+# Download minio binary and signature file
+RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    chmod +x /go/bin/minio
+
+# Download mc binary and signature file
+RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    chmod +x /go/bin/mc
+
+# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
+RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
+    minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
+
+FROM registry.access.redhat.com/ubi9/ubi-micro:latest

 ARG RELEASE

@@ -19,35 +44,18 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
    MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
    MINIO_CONFIG_ENV_FILE=config.env \
-    PATH=/opt/bin:$PATH
+    MC_CONFIG_DIR=/tmp/.mc
+
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/mc /usr/bin/mc

-COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE
-
-RUN \
-     microdnf clean all && \
-     microdnf update --nodocs && \
-     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
-     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /opt/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
-     curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /opt/bin/mc && \
-     microdnf clean all && \
-     chmod +x /opt/bin/minio && \
-     chmod +x /opt/bin/mc && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     /usr/bin/verify-minio.sh && \
-     microdnf clean all
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh

 EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
 VOLUME ["/data"]

+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 CMD ["minio"]
@@ -1,6 +1,25 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
+FROM golang:1.21-alpine as build

 ARG TARGETARCH
+ARG RELEASE
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+# Install curl and minisign
+RUN apk add -U --no-cache ca-certificates && \
+    apk add -U --no-cache curl && \
+    go install aead.dev/minisign/cmd/minisign@v0.2.0
+
+# Download minio binary and signature file
+RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /go/bin/minio && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /go/bin/minio.minisig && \
+    chmod +x /go/bin/minio
+
+# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
+RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
+
+FROM registry.access.redhat.com/ubi9/ubi-micro:9.2

 ARG RELEASE

@@ -18,34 +37,17 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
    MINIO_ROOT_PASSWORD_FILE=secret_key \
    MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
-    MINIO_CONFIG_ENV_FILE=config.env \
-    PATH=/opt/bin:$PATH
+    MINIO_CONFIG_ENV_FILE=config.env
+
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/minio /usr/bin/minio

-COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 COPY CREDITS /licenses/CREDITS
 COPY LICENSE /licenses/LICENSE
-
-RUN \
-     microdnf clean all && \
-     microdnf update --nodocs && \
-     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
-     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /opt/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /opt/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /opt/bin/minio.minisig && \
-     microdnf clean all && \
-     chmod +x /opt/bin/minio && \
-     chmod +x /usr/bin/docker-entrypoint.sh && \
-     chmod +x /usr/bin/verify-minio.sh && \
-     /usr/bin/verify-minio.sh && \
-     microdnf clean all
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh

 EXPOSE 9000
-
-ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
-
 VOLUME ["/data"]

+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 CMD ["minio"]
@@ -0,0 +1,61 @@
+FROM golang:1.21-alpine as build
+
+ARG TARGETARCH
+ARG RELEASE
+
+ENV GOPATH /go
+ENV CGO_ENABLED 0
+
+# Install curl and minisign
+RUN apk add -U --no-cache ca-certificates && \
+    apk add -U --no-cache curl && \
+    go install aead.dev/minisign/cmd/minisign@v0.2.0
+
+# Download minio binary and signature file
+RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \
+    curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \
+    chmod +x /go/bin/minio
+
+# Download mc binary and signature file
+RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \
+    curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \
+    chmod +x /go/bin/mc
+
+# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN"
+RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \
+    minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
+
+FROM registry.access.redhat.com/ubi8/ubi-micro:latest
+
+ARG RELEASE
+
+LABEL name="MinIO" \
+      vendor="MinIO Inc <dev@min.io>" \
+      maintainer="MinIO Inc <dev@min.io>" \
+      version="${RELEASE}" \
+      release="${RELEASE}" \
+      summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
+      description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
+
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
+    MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
+    MINIO_CONFIG_ENV_FILE=config.env \
+    MC_CONFIG_DIR=/tmp/.mc
+
+COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY --from=build /go/bin/minio /usr/bin/minio
+COPY --from=build /go/bin/mc /usr/bin/mc
+
+COPY CREDITS /licenses/CREDITS
+COPY LICENSE /licenses/LICENSE
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
+
+EXPOSE 9000
+VOLUME ["/data"]
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+CMD ["minio"]
@@ -6,7 +6,7 @@ GOARCH := $(shell go env GOARCH)
 GOOS := $(shell go env GOOS)

 VERSION ?= $(shell git describe --tags)
-TAG ?= "minio/minio:$(VERSION)"
+TAG ?= "quay.io/minio/minio:$(VERSION)"

 GOLANGCI_VERSION = v1.51.2
 GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION)
@@ -45,22 +45,22 @@ lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix

 check: test
-test: verifiers build ## builds minio, runs linters, tests
+test: verifiers build build-debugging ## builds minio, runs linters, tests
 	@echo "Running unit tests"
 	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...

-test-root-disable: install
+test-root-disable: install-race
 	@echo "Running minio root lockdown tests"
 	@env bash $(PWD)/buildscripts/disable-root.sh

-test-decom: install
+test-decom: install-race
 	@echo "Running minio decom tests"
 	@env bash $(PWD)/docs/distributed/decom.sh
 	@env bash $(PWD)/docs/distributed/decom-encrypted.sh
 	@env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh
 	@env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh

-test-upgrade: build
+test-upgrade: install-race
 	@echo "Running minio upgrade tests"
 	@(env bash $(PWD)/buildscripts/minio-upgrade.sh)

@@ -86,18 +86,18 @@ test-replication-3site:
 test-delete-replication:
 	@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)

-test-replication: install test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
+test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
 	@echo "Running tests for replicating three sites"

-test-site-replication-ldap: install ## verify automatic site replication
+test-site-replication-ldap: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with LDAP)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh)

-test-site-replication-oidc: install ## verify automatic site replication
+test-site-replication-oidc: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with OIDC)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-oidc.sh)

-test-site-replication-minio: install ## verify automatic site replication
+test-site-replication-minio: install-race ## verify automatic site replication
 	@echo "Running tests for automatic site replication of IAM (with MinIO IDP)"
 	@(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh)

@@ -128,6 +128,9 @@ verify-healing-inconsistent-versions: ## verify resolving inconsistent versions
 	@GORACE=history_size=7 CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/resolve-right-versions.sh)

+build-debugging:
+	@(env bash $(PWD)/docs/debugging/build.sh)
+
 build: checks ## builds minio to $(PWD)
 	@echo "Building minio binary to './minio'"
 	@CGO_ENABLED=0 go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
@@ -159,6 +162,12 @@ docker: build ## builds minio docker container
 	@echo "Building minio docker image '$(TAG)'"
 	@docker build -q --no-cache -t $(TAG) . -f Dockerfile

+install-race: checks ## builds minio to $(PWD)
+	@echo "Building minio binary to './minio'"
+	@GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
+	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
+	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
+
 install: build ## builds minio and installs it to $GOPATH/bin.
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"
 	@mkdir -p $(GOPATH)/bin && cp -f $(PWD)/minio $(GOPATH)/bin/minio
@@ -56,6 +56,8 @@ done

 set +e

+sleep 10
+
 ./mc ls minioadm/
 if [ $? -ne 0 ]; then
 	echo "listing failed, 'minioadmin' should be enabled"
@@ -87,8 +87,8 @@ function verify_rewrite() {
 		exit 1
 	fi

-	go build ./docs/debugging/s3-check-md5/
-	if ! ./s3-check-md5 \
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+	if ! s3-check-md5 \
 		-debug \
 		-versions \
 		-access-key minio \
@@ -113,7 +113,7 @@ function verify_rewrite() {
 	go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
 	sleep 1

-	if ! ./s3-check-md5 \
+	if ! s3-check-md5 \
 		-debug \
 		-versions \
 		-access-key minio \
@@ -82,8 +82,8 @@ function start_minio_16drive() {
 	rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
 	sleep 10

-	go build ./docs/debugging/s3-check-md5/
-	if ! ./s3-check-md5 \
+	go install github.com/minio/minio/docs/debugging/s3-check-md5@latest
+	if ! s3-check-md5 \
 		-debug \
 		-access-key minio \
 		-secret-key minio123 \
@@ -111,7 +111,7 @@ function start_minio_16drive() {
 		exit 1
 	fi

-	if ! ./s3-check-md5 \
+	if ! s3-check-md5 \
 		-debug \
 		-access-key minio \
 		-secret-key minio123 \
@@ -134,7 +134,7 @@ function start_minio_16drive() {

 	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket

-	if ! ./s3-check-md5 \
+	if ! s3-check-md5 \
 		-debug \
 		-access-key minio \
 		-secret-key minio123 \
@@ -9,11 +9,6 @@ x-minio-common: &minio-common
  expose:
    - "9000"
    - "9001"
-  healthcheck:
-    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
-    interval: 30s
-    timeout: 20s
-    retries: 3

 # starts 4 docker containers running minio server instances.
 # using nginx reverse proxy, load balancing, you can access
@@ -392,7 +392,8 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 	// of bucket metadata
 	zipWriter := zip.NewWriter(w)
 	defer zipWriter.Close()
-	rawDataFn := func(r io.Reader, filename string, sz int) error {
+
+	rawDataFn := func(r io.Reader, filename string, sz int) {
 		header, zerr := zip.FileInfoHeader(dummyFileInfo{
 			name:    filename,
 			size:    int64(sz),
@@ -401,20 +402,13 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 			isDir:   false,
 			sys:     nil,
 		})
-		if zerr != nil {
-			logger.LogIf(ctx, zerr)
-			return nil
+		if zerr == nil {
+			header.Method = zip.Deflate
+			zwriter, zerr := zipWriter.CreateHeader(header)
+			if zerr == nil {
+				io.Copy(zwriter, r)
+			}
 		}
-		header.Method = zip.Deflate
-		zwriter, zerr := zipWriter.CreateHeader(header)
-		if zerr != nil {
-			logger.LogIf(ctx, zerr)
-			return nil
-		}
-		if _, err := io.Copy(zwriter, r); err != nil {
-			logger.LogIf(ctx, err)
-		}
-		return nil
 	}

 	cfgFiles := []string{
@@ -446,10 +440,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketLifecycleConfig:
 				config, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
 				if err != nil {
@@ -465,10 +456,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketQuotaConfigFile:
 				config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket)
 				if err != nil {
@@ -483,10 +471,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketSSEConfig:
 				config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket)
 				if err != nil {
@@ -501,10 +486,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketTaggingConfig:
 				config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket)
 				if err != nil {
@@ -519,10 +501,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case objectLockConfig:
 				config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket)
 				if err != nil {
@@ -538,10 +517,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketVersioningConfig:
 				config, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket)
 				if err != nil {
@@ -557,10 +533,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketReplicationConfig:
 				config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket)
 				if err != nil {
@@ -575,11 +548,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			case bucketTargetsFile:
 				config, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
 				if err != nil {
@@ -595,10 +564,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
 					return
 				}
-				if err = rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)); err != nil {
-					writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL)
-					return
-				}
+				rawDataFn(bytes.NewReader(configData), cfgPath, len(configData))
 			}
 		}
 	}
@@ -27,14 +27,14 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )

 // validateAdminReq will validate request against and return whether it is allowed.
 // If any of the supplied actions are allowed it will be successful.
 // If nil ObjectLayer is returned, the operation is not permitted.
 // When nil ObjectLayer has been returned an error has always been sent to w.
-func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
+func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...policy.AdminAction) (ObjectLayer, auth.Credentials) {
 	// Get current object layer instance.
 	objectAPI := newObjectLayerFn()
 	if objectAPI == nil || globalNotificationSys == nil {
@@ -78,7 +78,7 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {

 	var apiErr APIError
 	switch e := err.(type) {
-	case iampolicy.Error:
+	case policy.Error:
 		apiErr = APIError{
 			Code:           "XMinioMalformedIAMPolicy",
 			Description:    e.Error(),
@@ -28,7 +28,6 @@ import (

 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/etcd"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
@@ -38,14 +37,14 @@ import (
 	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )

 // DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv
 func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -149,7 +148,7 @@ type setConfigResult struct {
 func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -242,7 +241,7 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -288,7 +287,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -321,7 +320,7 @@ func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -367,7 +366,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -405,7 +404,7 @@ func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *h
 func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -430,7 +429,7 @@ func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -482,7 +481,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -500,8 +499,6 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 			switch hkv.Key {
 			case config.EtcdSubSys:
 				off = !etcd.Enabled(item.Config)
-			case config.CacheSubSys:
-				off = !cache.Enabled(item.Config)
 			case config.StorageClassSubSys:
 				off = !storageclass.Enabled(item.Config)
 			case config.PolicyPluginSubSys:
@@ -34,11 +34,11 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/ldap"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )

 func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) {
-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -121,7 +121,7 @@ func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.R

 	// IDP config is not dynamic. Sanity check.
 	if dynamic {
-		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL)
+		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), "", r.URL)
 		return
 	}

@@ -220,7 +220,7 @@ func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *ht
 func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -273,7 +273,7 @@ func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http
 func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -332,7 +332,7 @@ func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.
 func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConfigUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -25,7 +25,7 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )

 // ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies.
@@ -50,7 +50,7 @@ func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r
 	// Check authorization.

 	objectAPI, cred := validateAdminReq(ctx, w, r,
-		iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
+		policy.ListGroupsAdminAction, policy.ListUsersAdminAction, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -96,7 +96,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.

 	// Check authorization.

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.UpdatePolicyAssociationAction)
 	if objectAPI == nil {
 		return
 	}
@@ -26,7 +26,7 @@ import (

 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )

 var (
@@ -37,7 +37,7 @@ var (
 func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -113,7 +113,7 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -159,7 +159,7 @@ func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -200,7 +200,7 @@ func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction, iampolicy.DecommissionAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -233,7 +233,7 @@ func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -304,7 +304,7 @@ func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -344,7 +344,7 @@ func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.RebalanceAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -31,6 +31,7 @@ import (

 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v3"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/policy"
@@ -537,7 +538,7 @@ func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.

 	connectTime := time.Now()
 	for {
-		n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
+		n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte)
 		atomic.AddUint64(&globalSiteNetPerfRX.RX, uint64(n))
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			// If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec)
@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -19,6 +19,7 @@ package cmd

 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -34,14 +35,14 @@ import (
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )

 // RemoveUser - DELETE /minio/admin/v3/remove-user?accessKey=<access_key>
 func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.DeleteUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -85,7 +86,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ListUsersAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -119,7 +120,7 @@ func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.ListUsersAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.ListUsersAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -186,10 +187,10 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 		checkDenyOnly = true
 	}

-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.GetUserAdminAction,
+		Action:          policy.GetUserAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -218,7 +219,7 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AddUserToGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.AddUserToGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -288,7 +289,7 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -315,7 +316,7 @@ func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListGroupsAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListGroupsAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -339,7 +340,7 @@ func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.EnableGroupAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.EnableGroupAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -382,7 +383,7 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
+	objectAPI, creds := validateAdminReq(ctx, w, r, policy.EnableUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -470,10 +471,10 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		checkDenyOnly = true
 	}

-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.CreateUserAdminAction,
+		Action:          policy.CreateUserAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -544,10 +545,10 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 		return
 	}

-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.ListTemporaryAccountsAdminAction,
+		Action:          policy.ListTemporaryAccountsAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -556,16 +557,16 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 		return
 	}

-	stsAccount, policy, err := globalIAMSys.GetTemporaryAccount(ctx, accessKey)
+	stsAccount, sessionPolicy, err := globalIAMSys.GetTemporaryAccount(ctx, accessKey)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

-	var stsAccountPolicy iampolicy.Policy
+	var stsAccountPolicy policy.Policy

-	if policy != nil {
-		stsAccountPolicy = *policy
+	if sessionPolicy != nil {
+		stsAccountPolicy = *sessionPolicy
 	} else {
 		policiesNames, err := globalIAMSys.PolicyDBGet(stsAccount.ParentUser, false)
 		if err != nil {
@@ -584,7 +585,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 	infoResp := madmin.TemporaryAccountInfoResp{
 		ParentUser:    stsAccount.ParentUser,
 		AccountStatus: stsAccount.Status,
-		ImpliedPolicy: policy == nil,
+		ImpliedPolicy: sessionPolicy == nil,
 		Policy:        string(policyJSON),
 		Expiration:    &stsAccount.Expiration,
 	}
@@ -709,10 +710,10 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		//
 		// This allows turning off service accounts for request sender,
 		// if there is no deny statement this call is implicitly enabled.
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
+		if !globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     requestorUser,
 			Groups:          requestorGroups,
-			Action:          iampolicy.CreateServiceAccountAdminAction,
+			Action:          policy.CreateServiceAccountAdminAction,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
 			Claims:          cred.Claims,
@@ -743,10 +744,10 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 	} else {
 		// Need permission if we are creating a service account for a
 		// user <> to the request sender
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
+		if !globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     requestorUser,
 			Groups:          requestorGroups,
-			Action:          iampolicy.CreateServiceAccountAdminAction,
+			Action:          policy.CreateServiceAccountAdminAction,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
 			Claims:          cred.Claims,
@@ -773,13 +774,16 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		// latter, a group notion is not supported.
 	}

-	var sp *iampolicy.Policy
+	var sp *policy.Policy
 	if len(createReq.Policy) > 0 {
-		sp, err = iampolicy.ParseConfig(bytes.NewReader(createReq.Policy))
+		sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy))
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
+		if sp.Version == "" && len(sp.Statements) == 0 {
+			sp = nil
+		}
 	}

 	opts.sessionPolicy = sp
@@ -864,10 +868,10 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}

-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.UpdateServiceAccountAdminAction,
+		Action:          policy.UpdateServiceAccountAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -903,13 +907,16 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}

-	var sp *iampolicy.Policy
+	var sp *policy.Policy
 	if len(updateReq.NewPolicy) > 0 {
-		sp, err = iampolicy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
+		sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
+		if sp.Version == "" && len(sp.Statements) == 0 {
+			sp = nil
+		}
 	}
 	opts := updateServiceAccountOpts{
 		secretKey:     updateReq.NewSecretKey,
@@ -970,16 +977,16 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		return
 	}

-	svcAccount, policy, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
+	svcAccount, sessionPolicy, err := globalIAMSys.GetServiceAccount(ctx, accessKey)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+	if !globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.ListServiceAccountsAdminAction,
+		Action:          policy.ListServiceAccountsAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -995,10 +1002,13 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		}
 	}

-	var svcAccountPolicy iampolicy.Policy
+	// if session policy is nil or empty, then it is implied policy
+	impliedPolicy := sessionPolicy == nil || (sessionPolicy.Version == "" && len(sessionPolicy.Statements) == 0)

-	if policy != nil {
-		svcAccountPolicy = *policy
+	var svcAccountPolicy policy.Policy
+
+	if !impliedPolicy {
+		svcAccountPolicy = *sessionPolicy
 	} else {
 		policiesNames, err := globalIAMSys.PolicyDBGet(svcAccount.ParentUser, false)
 		if err != nil {
@@ -1024,7 +1034,7 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 		Name:          svcAccount.Name,
 		Description:   svcAccount.Description,
 		AccountStatus: svcAccount.Status,
-		ImpliedPolicy: policy == nil,
+		ImpliedPolicy: impliedPolicy,
 		Policy:        string(policyJSON),
 		Expiration:    expiration,
 	}
@@ -1067,10 +1077,10 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 	// sender), check that the user has permissions.
 	user := r.Form.Get("user")
 	if user != "" && user != cred.AccessKey {
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
+		if !globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.ListServiceAccountsAdminAction,
+			Action:          policy.ListServiceAccountsAdminAction,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
 			Claims:          cred.Claims,
@@ -1146,12 +1156,16 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re

 	// We do not care if service account is readable or not at this point,
 	// since this is a delete call we shall allow it to be deleted if possible.
-	svcAccount, _, _ := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
+	svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, serviceAccount)
+	if errors.Is(err, errNoSuchServiceAccount) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL)
+		return
+	}

-	adminPrivilege := globalIAMSys.IsAllowed(iampolicy.Args{
+	adminPrivilege := globalIAMSys.IsAllowed(policy.Args{
 		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
-		Action:          iampolicy.RemoveServiceAccountAdminAction,
+		Action:          policy.RemoveServiceAccountAdminAction,
 		ConditionValues: getConditionValues(r, "", cred),
 		IsOwner:         owner,
 		Claims:          cred.Claims,
@@ -1192,7 +1206,7 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 	writeSuccessNoContent(w)
 }

-// AccountInfoHandler returns usage
+// AccountInfoHandler returns usage, permissions and other bucket metadata for incoming us
 func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

@@ -1219,10 +1233,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	enablePrefixUsage := r.Form.Get("prefix-usage") == "true"

 	isAllowedAccess := func(bucketName string) (rd, wr bool) {
-		if globalIAMSys.IsAllowed(iampolicy.Args{
+		if globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.ListBucketAction,
+			Action:          policy.ListBucketAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
@@ -1232,10 +1246,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			rd = true
 		}

-		if globalIAMSys.IsAllowed(iampolicy.Args{
+		if globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.GetBucketLocationAction,
+			Action:          policy.GetBucketLocationAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
@@ -1245,10 +1259,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			rd = true
 		}

-		if globalIAMSys.IsAllowed(iampolicy.Args{
+		if globalIAMSys.IsAllowed(policy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
-			Action:          iampolicy.PutObjectAction,
+			Action:          policy.PutObjectAction,
 			BucketName:      bucketName,
 			ConditionValues: getConditionValues(r, "", cred),
 			IsOwner:         owner,
@@ -1261,12 +1275,30 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}

-	// Load the latest calculated data usage
-	dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI)
+	bucketStorageCache.Once.Do(func() {
+		// Set this to 10 secs since its enough, as scanner
+		// does not update the bucket usage values frequently.
+		bucketStorageCache.TTL = 10 * time.Second
+
+		// Rely on older value if usage loading fails from disk.
+		bucketStorageCache.Relax = true
+		bucketStorageCache.Update = func() (interface{}, error) {
+			ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
+			defer done()
+
+			return loadDataUsageFromBackend(ctx, objectAPI)
+		}
+	})
+
+	var dataUsageInfo DataUsageInfo
+	v, _ := bucketStorageCache.Get()
+	if v != nil {
+		dataUsageInfo, _ = v.(DataUsageInfo)
+	}

 	// If etcd, dns federation configured list buckets from etcd.
-	var buckets []BucketInfo
 	var err error
+	var buckets []BucketInfo
 	if globalDNSConfig != nil && globalBucketFederation {
 		dnsBuckets, err := globalDNSConfig.List()
 		if err != nil && !IsErrIgnored(err,
@@ -1285,7 +1317,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 			return buckets[i].Name < buckets[j].Name
 		})
 	} else {
-		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{})
+		buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{Cached: true})
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
@@ -1298,14 +1330,14 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		accountName = cred.ParentUser
 	}

-	roleArn := iampolicy.Args{Claims: cred.Claims}.GetRoleArn()
-	policySetFromClaims, hasPolicyClaim := iampolicy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
-	var effectivePolicy iampolicy.Policy
+	roleArn := policy.Args{Claims: cred.Claims}.GetRoleArn()
+	policySetFromClaims, hasPolicyClaim := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID())
+	var effectivePolicy policy.Policy

 	var buf []byte
 	switch {
 	case accountName == globalActiveCred.AccessKey:
-		for _, policy := range iampolicy.DefaultPolicies {
+		for _, policy := range policy.DefaultPolicies {
 			if policy.Name == "consoleAdmin" {
 				effectivePolicy = policy.Definition
 				break
@@ -1417,7 +1449,7 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.GetPolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1463,7 +1495,7 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1475,7 +1507,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}

-	newPolicies := make(map[string]iampolicy.Policy)
+	newPolicies := make(map[string]policy.Policy)
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
@@ -1494,7 +1526,7 @@ func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListUserPoliciesAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1505,7 +1537,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 		return
 	}

-	newPolicies := make(map[string]iampolicy.Policy)
+	newPolicies := make(map[string]policy.Policy)
 	for name, p := range policies {
 		_, err = json.Marshal(p)
 		if err != nil {
@@ -1524,7 +1556,7 @@ func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DeletePolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DeletePolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1550,7 +1582,7 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CreatePolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.CreatePolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1582,7 +1614,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}

-	iamPolicy, err := iampolicy.ParseConfig(bytes.NewReader(iamPolicyBytes))
+	iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1614,7 +1646,7 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1688,7 +1720,7 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht

 	// Check authorization.
 	objectAPI, cred := validateAdminReq(ctx, w, r,
-		iampolicy.ListGroupsAdminAction, iampolicy.ListUsersAdminAction, iampolicy.ListUserPoliciesAdminAction)
+		policy.ListGroupsAdminAction, policy.ListUsersAdminAction, policy.ListUserPoliciesAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1726,8 +1758,8 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht
 func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction,
-		iampolicy.AttachPolicyAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, policy.UpdatePolicyAssociationAction,
+		policy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1826,7 +1858,7 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

 	// Get current object layer instance.
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ExportIAMAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportIAMAction)
 	if objectAPI == nil {
 		return
 	}
@@ -2096,7 +2128,7 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 			return
 		default:
 			defer f.Close()
-			var allPolicies map[string]iampolicy.Policy
+			var allPolicies map[string]policy.Policy
 			data, err = io.ReadAll(f)
 			if err != nil {
 				writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL)
@@ -2177,10 +2209,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					checkDenyOnly = true
 				}

-				if !globalIAMSys.IsAllowed(iampolicy.Args{
+				if !globalIAMSys.IsAllowed(policy.Args{
 					AccountName:     cred.AccessKey,
 					Groups:          cred.Groups,
-					Action:          iampolicy.CreateUserAdminAction,
+					Action:          policy.CreateUserAdminAction,
 					ConditionValues: getConditionValues(r, "", cred),
 					IsOwner:         owner,
 					Claims:          cred.Claims,
@@ -2257,10 +2289,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			for user, svcAcctReq := range serviceAcctReqs {
-				var sp *iampolicy.Policy
+				var sp *policy.Policy
 				var err error
 				if len(svcAcctReq.SessionPolicy) > 0 {
-					sp, err = iampolicy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy))
+					sp, err = policy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy))
 					if err != nil {
 						writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL)
 						return
@@ -2271,10 +2303,10 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
 					return
 				}
-				if !globalIAMSys.IsAllowed(iampolicy.Args{
+				if !globalIAMSys.IsAllowed(policy.Args{
 					AccountName:     cred.AccessKey,
 					Groups:          cred.Groups,
-					Action:          iampolicy.CreateServiceAccountAdminAction,
+					Action:          policy.CreateServiceAccountAdminAction,
 					ConditionValues: getConditionValues(r, "", cred),
 					IsOwner:         owner,
 					Claims:          cred.Claims,
@@ -52,18 +52,21 @@ import (
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/handlers"
 	xhttp "github.com/minio/minio/internal/http"
+	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/v2/logger/message/log"
 	xnet "github.com/minio/pkg/v2/net"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 	"github.com/secure-io/sio-go"
 )

 const (
 	maxEConfigJSONSize        = 262272
 	kubernetesVersionEndpoint = "https://kubernetes.default.svc/version"
+	anonymizeParam            = "anonymize"
+	anonymizeStrict           = "strict"
 )

 // Only valid query params for mgmt admin APIs.
@@ -81,7 +84,7 @@ const (
 func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerUpdateAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerUpdateAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -252,11 +255,11 @@ func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request)
 	var objectAPI ObjectLayer
 	switch serviceSig {
 	case serviceRestart:
-		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceRestartAdminAction)
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceRestartAdminAction)
 	case serviceStop:
-		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceStopAdminAction)
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceStopAdminAction)
 	case serviceFreeze, serviceUnFreeze:
-		objectAPI, _ = validateAdminReq(ctx, w, r, iampolicy.ServiceFreezeAdminAction)
+		objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceFreezeAdminAction)
 	}
 	if objectAPI == nil {
 		return
@@ -331,7 +334,7 @@ type ServerHTTPStats struct {
 func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.StorageInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.StorageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -370,7 +373,7 @@ func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ServerInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -479,7 +482,7 @@ func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DataUsageInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DataUsageInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -562,7 +565,7 @@ type PeerLocks struct {
 func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ForceUnlockAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ForceUnlockAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -597,7 +600,7 @@ func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Requ
 func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.TopLocksAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.TopLocksAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -648,7 +651,7 @@ func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.R
 	ctx := r.Context()

 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -733,7 +736,7 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 	ctx := r.Context()

 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -754,11 +757,8 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 			return
 		}
 	}
-	// read request body
-	io.CopyN(io.Discard, r.Body, 1)

 	globalProfilerMu.Lock()
-
 	if globalProfiler == nil {
 		globalProfiler = make(map[string]minioProfiler, 10)
 	}
@@ -828,7 +828,7 @@ func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *htt
 	ctx := r.Context()

 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ProfilingAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -926,7 +926,7 @@ func extractHealInitParams(vars map[string]string, qParms url.Values, r io.Reade
 func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1001,7 +1001,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 							Message:   hr.errBody,
 							Resource:  r.URL.Path,
 							RequestID: w.Header().Get(xhttp.AmzRequestID),
-							HostID:    globalDeploymentID,
+							HostID:    globalDeploymentID(),
 						})
 					}
 					if !started {
@@ -1056,7 +1056,7 @@ func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}

-	respCh := make(chan healResp)
+	respCh := make(chan healResp, 1)
 	switch {
 	case hip.forceStop:
 		go func() {
@@ -1110,7 +1110,7 @@ func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmi
 func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1131,7 +1131,7 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1179,7 +1179,7 @@ func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request
 func (a adminAPIHandlers) ClientDevNullExtraTime(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.BandwidthMonitorAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.BandwidthMonitorAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1196,7 +1196,7 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 	ctx := r.Context()

 	timeStart := time.Now()
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.BandwidthMonitorAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.BandwidthMonitorAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1218,7 +1218,7 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 	totalRx := int64(0)
 	connectTime := time.Now()
 	for {
-		n, err := io.CopyN(io.Discard, r.Body, 128*humanize.KiByte)
+		n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte)
 		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
 			// would mean the network is not stable. Logging here will help in debugging network issues.
 			if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) {
@@ -1238,7 +1238,7 @@ func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1284,7 +1284,7 @@ func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1400,13 +1400,21 @@ func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.
 }

 func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) {
-	if err = objectAPI.MakeBucket(ctx, bucketName, MakeBucketOptions{}); err != nil {
+	if err = objectAPI.MakeBucket(ctx, bucketName, MakeBucketOptions{VersioningEnabled: globalSiteReplicationSys.isEnabled()}); err != nil {
 		if _, ok := err.(BucketExists); !ok {
 			// Only BucketExists error can be ignored.
 			return false, err
 		}
 		bucketExists = true
 	}
+
+	if globalSiteReplicationSys.isEnabled() {
+		configData := []byte(`<VersioningConfiguration><Status>Enabled</Status><ExcludedPrefixes><Prefix>speedtest/*</Prefix></ExcludedPrefixes></VersioningConfiguration>`)
+		if _, err = globalBucketMetadataSys.Update(ctx, bucketName, bucketVersioningConfig, configData); err != nil {
+			return false, err
+		}
+	}
+
 	return bucketExists, nil
 }

@@ -1445,7 +1453,7 @@ func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo,
 func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1566,7 +1574,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.TraceAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.TraceAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -1581,7 +1589,9 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {

 	// Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers.
 	// Use buffered channel to take care of burst sends or slow w.Write()
-	traceCh := make(chan madmin.TraceInfo, 4000)
+
+	// Keep 100k buffered channel, should be sufficient to ensure we do not lose any events.
+	traceCh := make(chan madmin.TraceInfo, 100000)

 	peers, _ := newPeerRestClients(globalEndpoints)

@@ -1605,7 +1615,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		peer.Trace(traceCh, ctx.Done(), traceOpts)
 	}

-	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	keepAliveTicker := time.NewTicker(time.Second)
 	defer keepAliveTicker.Stop()

 	enc := json.NewEncoder(w)
@@ -1637,7 +1647,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ConsoleLogAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConsoleLogAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1718,7 +1728,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSCreateKeyAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSCreateKeyAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1739,7 +1749,7 @@ func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Req
 func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1776,7 +1786,7 @@ func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.KMSKeyStatusAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1980,7 +1990,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 		Domain:        domain,
 		Region:        globalSite.Region,
 		SQSARN:        globalEventNotifier.GetARNList(false),
-		DeploymentID:  globalDeploymentID,
+		DeploymentID:  globalDeploymentID(),
 		Buckets:       buckets,
 		Objects:       objects,
 		Versions:      versions,
@@ -2025,8 +2035,14 @@ func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo {

 func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *url.Values, healthInfoCh chan madmin.HealthInfo, healthInfo madmin.HealthInfo) {
 	hostAnonymizer := createHostAnonymizer()
-	// anonAddr - Anonymizes hosts in given input string.
+
+	anonParam := query.Get(anonymizeParam)
+	// anonAddr - Anonymizes hosts in given input string
+	// (only if the anonymize param is set to srict).
 	anonAddr := func(addr string) string {
+		if anonParam != anonymizeStrict {
+			return addr
+		}
 		newAddr, found := hostAnonymizer[addr]
 		if found {
 			return newAddr
@@ -2087,6 +2103,20 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 		}
 	}

+	getAndWriteNetInfo := func() {
+		if query.Get(string(madmin.HealthDataTypeSysNet)) == "true" {
+			localNetInfo := madmin.GetNetInfo(globalLocalNodeName, globalInternodeInterface)
+			healthInfo.Sys.NetInfo = append(healthInfo.Sys.NetInfo, localNetInfo)
+
+			peerNetInfos := globalNotificationSys.GetNetInfo(healthCtx)
+			for _, n := range peerNetInfos {
+				anonymizeAddr(&n)
+				healthInfo.Sys.NetInfo = append(healthInfo.Sys.NetInfo, n)
+			}
+			partialWrite(healthInfo)
+		}
+	}
+
 	getAndWriteOSInfo := func() {
 		if query.Get("sysosinfo") == "true" {
 			localOSInfo := madmin.GetOSInfo(healthCtx, globalLocalNodeName)
@@ -2187,6 +2217,10 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 	}

 	anonymizeCmdLine := func(cmdLine string) string {
+		if anonParam != anonymizeStrict {
+			return cmdLine
+		}
+
 		if !globalIsDistErasure {
 			// FS mode - single server - hard code to `server1`
 			anonCmdLine := strings.ReplaceAll(cmdLine, globalLocalNodeName, "server1")
@@ -2308,6 +2342,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 		getAndWritePlatformInfo()
 		getAndWriteCPUs()
 		getAndWritePartitions()
+		getAndWriteNetInfo()
 		getAndWriteOSInfo()
 		getAndWriteMemInfo()
 		getAndWriteProcInfo()
@@ -2376,7 +2411,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.HealthInfoAdminAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -2390,7 +2425,7 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 		Version:   madmin.HealthInfoVersion,
 		Minio: madmin.MinioHealthInfo{
 			Info: madmin.MinioInfo{
-				DeploymentID: globalDeploymentID,
+				DeploymentID: globalDeploymentID(),
 			},
 		},
 	}
@@ -2483,7 +2518,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	ctx := r.Context()

 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.ServerInfoAdminAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ServerInfoAdminAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -2688,7 +2723,7 @@ func getClusterMetaInfo(ctx context.Context) []byte {
 		ci.Info.NoOfBuckets = dataUsageInfo.BucketsCount
 		ci.Info.NoOfObjects = dataUsageInfo.ObjectsTotalCount

-		ci.DeploymentID = globalDeploymentID
+		ci.DeploymentID = globalDeploymentID()
 		ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.NoOfServers, ci.Info.NoOfDrives, ci.Info.MinioVersion)

 		select {
@@ -2735,7 +2770,7 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	ctx := r.Context()

 	// Validate request signature.
-	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.InspectDataAction, "")
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.InspectDataAction, "")
 	if adminAPIErr != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
 		return
@@ -2929,9 +2964,14 @@ func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Requ
 	sb.WriteString("\n")
 	logger.LogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes(), 0o600))

+	scheme := "https"
+	if !globalIsTLS {
+		scheme = "http"
+	}
+
 	// save MinIO start script to inspect command
 	var scrb bytes.Buffer
-	scrb.WriteString(`#!/usr/bin/env bash
+	fmt.Fprintf(&scrb, `#!/usr/bin/env bash

 function main() {
 	for file in $(ls -1); do
@@ -2940,10 +2980,10 @@ function main() {
 	done

 	# Read content of inspect-input.txt
-	MINIO_OPTS=$(grep "Server command line args" <./inspect-input.txt | sed "s/Server command line args: //g" | sed -r "s#https:\/\/#\.\/#g")
+	MINIO_OPTS=$(grep "Server command line args" <./inspect-input.txt | sed "s/Server command line args: //g" | sed -r "s#%s:\/\/#\.\/#g")

 	# Start MinIO instance using the options
-	START_CMD="CI=on _MINIO_AUTO_DISK_HEALING=off minio server ${MINIO_OPTS} &"
+	START_CMD="CI=on _MINIO_AUTO_DRIVE_HEALING=off minio server ${MINIO_OPTS} &"
 	echo
 	echo "Starting MinIO instance: ${START_CMD}"
 	echo
@@ -2955,8 +2995,7 @@ function main() {
 	sleep 10
 }

-main "$@"`,
-	)
+main "$@"`, scheme)
 	logger.LogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes(), 0o755))
 }

@@ -683,13 +683,6 @@ func (h *healSequence) healSequenceStart(objAPI ObjectLayer) {
 	}
 }

-func (h *healSequence) logHeal(healType madmin.HealItemType) {
-	h.mutex.Lock()
-	h.scannedItemsMap[healType]++
-	h.lastHealActivity = UTCNow()
-	h.mutex.Unlock()
-}
-
 func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error {
 	// Send heal request
 	task := healTask{
@@ -310,6 +310,7 @@ const (
 	ErrSiteReplicationBucketMetaError
 	ErrSiteReplicationIAMError
 	ErrSiteReplicationConfigMissing
+	ErrSiteReplicationIAMConfigMismatch

 	// Pool rebalance errors
 	ErrAdminRebalanceAlreadyStarted
@@ -1512,6 +1513,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Site not found in site replication configuration",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrSiteReplicationIAMConfigMismatch: {
+		Code:           "XMinioSiteReplicationIAMConfigMismatch",
+		Description:    "IAM configuration mismatch between sites",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminRebalanceAlreadyStarted: {
 		Code:           "XMinioAdminRebalanceAlreadyStarted",
 		Description:    "Pool rebalance is already started",
@@ -2236,8 +2242,6 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrSlowDownWrite
 	case InsufficientReadQuorum:
 		apiErr = ErrSlowDownRead
-	case InvalidMarkerPrefixCombination:
-		apiErr = ErrNotImplemented
 	case InvalidUploadIDKeyCombination:
 		apiErr = ErrNotImplemented
 	case MalformedUploadID:
@@ -2339,6 +2343,12 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case dns.ErrBucketConflict:
 		apiErr = ErrBucketAlreadyExists
 	default:
+		if _, ok := err.(tags.Error); ok {
+			// tag errors are not exported, so we check their custom interface to avoid logging.
+			// The correct type is inserted by toAPIError.
+			apiErr = ErrInternalError
+			break
+		}
 		var ie, iw int
 		// This work-around is to handle the issue golang/go#30648
 		//nolint:gocritic
@@ -42,7 +42,6 @@ var toAPIErrorTests = []struct {
 	{err: InvalidPart{}, errCode: ErrInvalidPart},
 	{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
 	{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
-	{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
 	{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
 	{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
 	{err: PartTooSmall{}, errCode: ErrEntityTooSmall},
@@ -133,11 +133,6 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 		w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat))
 	}

-	if globalCacheConfig.Enabled {
-		w.Header().Set(xhttp.XCache, objInfo.CacheStatus.String())
-		w.Header().Set(xhttp.XCacheLookup, objInfo.CacheLookupStatus.String())
-	}
-
 	// Set tag count if object has tags
 	if len(objInfo.UserTags) > 0 {
 		tags, _ := tags.ParseObjectTags(objInfo.UserTags)
@@ -978,7 +978,7 @@ func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, er
 		BucketName: reqInfo.BucketName,
 		Key:        reqInfo.ObjectName,
 		RequestID:  w.Header().Get(xhttp.AmzRequestID),
-		HostID:     globalDeploymentID,
+		HostID:     globalDeploymentID(),
 	}
 	encodedErrorResponse := encodeResponseJSON(errorResponse)
 	writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON)
@@ -61,18 +61,6 @@ func newObjectLayerFn() ObjectLayer {
 	return globalObjectAPI
 }

-func newCachedObjectLayerFn() CacheObjectLayer {
-	globalObjLayerMutex.RLock()
-	defer globalObjLayerMutex.RUnlock()
-	return globalCacheObjectAPI
-}
-
-func setCacheObjectLayer(c CacheObjectLayer) {
-	globalObjLayerMutex.Lock()
-	globalCacheObjectAPI = c
-	globalObjLayerMutex.Unlock()
-}
-
 func setObjectLayer(o ObjectLayer) {
 	globalObjLayerMutex.Lock()
 	globalObjectAPI = o
@@ -82,7 +70,6 @@ func setObjectLayer(o ObjectLayer) {
 // objectAPIHandler implements and provides http handlers for S3 API.
 type objectAPIHandlers struct {
 	ObjectAPI func() ObjectLayer
-	CacheAPI  func() CacheObjectLayer
 }

 // getHost tries its best to return the request host.
@@ -189,7 +176,6 @@ func registerAPIRouter(router *mux.Router) {
 	// Initialize API.
 	api := objectAPIHandlers{
 		ObjectAPI: newObjectLayerFn,
-		CacheAPI:  newCachedObjectLayerFn,
 	}

 	// API Router
@@ -567,7 +567,7 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty

 	// Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present.
 	// The verification happens implicit during reading.
-	reader, err := hash.NewReader(r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
+	reader, err := hash.NewReader(ctx, r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}
@@ -28,7 +28,7 @@ import (
 	"time"

 	"github.com/minio/minio/internal/auth"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 )

 type nullReader struct{}
@@ -443,7 +443,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 		{Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied},
 	}
 	for i, testCase := range testCases {
-		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
+		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, policy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}
@@ -347,7 +347,7 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {

 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)

-	if env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
+	if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn || env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
 		go monitorLocalDisksAndHeal(ctx, z)
 	}
 }
@@ -28,7 +28,6 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
-	"path"
 	"runtime"
 	"strconv"
 	"strings"
@@ -38,6 +37,7 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/lithammer/shortuuid/v4"
 	"github.com/minio/madmin-go/v3"
+	"github.com/minio/minio-go/v7"
 	miniogo "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
@@ -49,7 +49,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/console"
 	"github.com/minio/pkg/v2/env"
-	iampolicy "github.com/minio/pkg/v2/policy"
+	"github.com/minio/pkg/v2/policy"
 	"github.com/minio/pkg/v2/workers"
 	"gopkg.in/yaml.v2"
 )
@@ -114,7 +114,7 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 	srcObject := srcObjInfo.Name
 	tgtObject := srcObjInfo.Name
 	if r.Target.Prefix != "" {
-		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
+		tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name)
 	}

 	versionID := srcObjInfo.VersionID
@@ -163,11 +163,11 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 	}
 	rd, objInfo, _, err := core.GetObject(ctx, srcBucket, srcObject, gopts)
 	if err != nil {
-		return err
+		return ErrorRespToObjectError(err, srcBucket, srcObject, srcObjInfo.VersionID)
 	}
 	defer rd.Close()

-	hr, err := hash.NewReader(rd, objInfo.Size, "", "", objInfo.Size)
+	hr, err := hash.NewReader(ctx, rd, objInfo.Size, "", "", objInfo.Size)
 	if err != nil {
 		return err
 	}
@@ -182,7 +182,7 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 	srcObject := srcObjInfo.Name
 	tgtObject := srcObjInfo.Name
 	if r.Target.Prefix != "" {
-		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
+		tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name)
 	}
 	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
 		opts.VersionID = ""
@@ -226,11 +226,11 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 		}
 		rd, objInfo, _, err := c.GetObject(ctx, srcBucket, srcObject, gopts)
 		if err != nil {
-			return err
+			return ErrorRespToObjectError(err, srcBucket, srcObject, srcObjInfo.VersionID)
 		}
 		defer rd.Close()

-		hr, err = hash.NewReader(io.LimitReader(rd, objInfo.Size), objInfo.Size, "", "", objInfo.Size)
+		hr, err = hash.NewReader(ctx, io.LimitReader(rd, objInfo.Size), objInfo.Size, "", "", objInfo.Size)
 		if err != nil {
 			return err
 		}
@@ -270,6 +270,10 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))

+	isTags := len(r.Flags.Filter.Tags) != 0
+	isMetadata := len(r.Flags.Filter.Metadata) != 0
+	isStorageClassOnly := len(r.Flags.Filter.Metadata) == 1 && strings.EqualFold(r.Flags.Filter.Metadata[0].Key, xhttp.AmzStorageClass)
+
 	skip := func(oi ObjectInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
@@ -290,7 +294,8 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			// skip all objects that are created after the specified time.
 			return true
 		}
-		if len(r.Flags.Filter.Tags) > 0 {
+
+		if isTags {
 			// Only parse object tags if tags filter is specified.
 			tagMap := map[string]string{}
 			tagStr := oi.UserTags
@@ -313,23 +318,19 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 			return false
 		}

-		if len(r.Flags.Filter.Metadata) > 0 {
-			for _, kv := range r.Flags.Filter.Metadata {
-				for k, v := range oi.UserDefined {
-					if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
-						continue
-					}
-					// We only need to match x-amz-meta or standardHeaders
-					if kv.Match(BatchJobKV{Key: k, Value: v}) {
-						return true
-					}
+		for _, kv := range r.Flags.Filter.Metadata {
+			for k, v := range oi.UserDefined {
+				if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) {
+					continue
+				}
+				// We only need to match x-amz-meta or standardHeaders
+				if kv.Match(BatchJobKV{Key: k, Value: v}) {
+					return true
 				}
 			}
-
-			// None of the provided metadata filters match skip the object.
-			return false
 		}

+		// None of the provided filters match
 		return false
 	}

@@ -384,17 +385,32 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 		for obj := range objInfoCh {
 			oi := toObjectInfo(r.Source.Bucket, obj.Key, obj)
 			if !minioSrc {
-				oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, miniogo.StatObjectOptions{})
-				if err == nil {
-					oi = toObjectInfo(r.Source.Bucket, obj.Key, oi2)
-				} else {
-					if isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) ||
-						isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
+				// Check if metadata filter was requested and it is expected to have
+				// all user metadata or just storageClass. If its only storageClass
+				// List() already returns relevant information for filter to be applied.
+				if isMetadata && !isStorageClassOnly {
+					oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, miniogo.StatObjectOptions{})
+					if err == nil {
+						oi = toObjectInfo(r.Source.Bucket, obj.Key, oi2)
+					} else {
+						if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) &&
+							!isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
+							logger.LogIf(ctx, err)
+						}
+						continue
+					}
+				}
+				if isTags {
+					tags, err := c.GetObjectTagging(ctx, r.Source.Bucket, obj.Key, minio.GetObjectTaggingOptions{})
+					if err == nil {
+						oi.UserTags = tags.String()
+					} else {
+						if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) &&
+							!isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
+							logger.LogIf(ctx, err)
+						}
 						continue
 					}
-					logger.LogIf(ctx, err)
-					cancel()
-					return err
 				}
 			}
 			if skip(oi) {
@@ -482,10 +498,12 @@ func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo
 		ReplicationStatusInternal: objInfo.ReplicationStatus,
 		UserTags:                  tags.String(),
 	}
+
 	oi.UserDefined = make(map[string]string, len(objInfo.Metadata))
 	for k, v := range objInfo.Metadata {
 		oi.UserDefined[k] = v[0]
 	}
+
 	ce, ok := oi.UserDefined[xhttp.ContentEncoding]
 	if !ok {
 		ce, ok = oi.UserDefined[strings.ToLower(xhttp.ContentEncoding)]
@@ -493,6 +511,12 @@ func toObjectInfo(bucket, object string, objInfo miniogo.ObjectInfo) ObjectInfo
 	if ok {
 		oi.ContentEncoding = ce
 	}
+
+	_, ok = oi.UserDefined[xhttp.AmzStorageClass]
+	if !ok {
+		oi.UserDefined[xhttp.AmzStorageClass] = objInfo.StorageClass
+	}
+
 	return oi
 }

@@ -822,7 +846,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))

-	skip := func(info FileInfo) (ok bool) {
+	selectObj := func(info FileInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
 			return false
@@ -932,7 +956,7 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		results := make(chan ObjectInfo, 100)
 		if err := api.Walk(ctx, r.Source.Bucket, r.Source.Prefix, results, ObjectOptions{
 			WalkMarker: lastObject,
-			WalkFilter: skip,
+			WalkFilter: selectObj,
 		}); err != nil {
 			cancel()
 			// Do not need to retry if we can't list objects on source.
@@ -1258,7 +1282,7 @@ func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (p
 func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.ListBatchJobsAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListBatchJobsAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1308,7 +1332,7 @@ var errNoSuchJob = errors.New("no such job")
 func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DescribeBatchJobAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.DescribeBatchJobAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1343,7 +1367,7 @@ func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Reques
 func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.StartBatchJobAction)
+	objectAPI, creds := validateAdminReq(ctx, w, r, policy.StartBatchJobAction)
 	if objectAPI == nil {
 		return
 	}
@@ -1397,7 +1421,7 @@ func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request)
 func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.CancelBatchJobAction)
+	objectAPI, _ := validateAdminReq(ctx, w, r, policy.CancelBatchJobAction)
 	if objectAPI == nil {
 		return
 	}
@@ -474,9 +474,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	}

 	deleteObjectsFn := objectAPI.DeleteObjects
-	if api.CacheAPI() != nil {
-		deleteObjectsFn = api.CacheAPI().DeleteObjects
-	}

 	// Return Malformed XML as S3 spec if the number of objects is empty
 	if len(deleteObjectsReq.Objects) == 0 || len(deleteObjectsReq.Objects) > maxDeleteList {
@@ -486,9 +483,6 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,

 	objectsToDelete := map[ObjectToDelete]int{}
 	getObjectInfoFn := objectAPI.GetObjectInfo
-	if api.CacheAPI() != nil {
-		getObjectInfoFn = api.CacheAPI().GetObjectInfo
-	}

 	var (
 		hasLockEnabled bool
@@ -716,7 +710,7 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		if os == nil { // skip objects that weren't deleted due to invalid versionID etc.
 			continue
 		}
-		logger.LogIf(ctx, os.Sweep())
+		os.Sweep()
 	}
 }

@@ -794,7 +788,7 @@ func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Req

 	// check if client is attempting to create more buckets, complain about it.
 	if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets {
-		logger.LogIf(ctx, fmt.Errorf("An attempt to create %d buckets beyond recommended %d", currBuckets+1, maxBuckets))
+		logger.LogIf(ctx, fmt.Errorf("Please avoid creating more buckets %d beyond recommended %d", currBuckets+1, maxBuckets))
 	}

 	opts := MakeBucketOptions{
@@ -989,7 +983,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		}

 		var b bytes.Buffer
-		if fileName == "" {
+		if name != "file" {
 			if http.CanonicalHeaderKey(name) == http.CanonicalHeaderKey("x-minio-fanout-list") {
 				dec := json.NewDecoder(part)

@@ -1045,11 +1039,14 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		break
 	}

-	if _, ok := formValues["Key"]; !ok {
+	if keyName, ok := formValues["Key"]; !ok {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The name of the uploaded key is missing"))
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
+	} else if fileName == "" && len(keyName) >= 1 {
+		// if we can't get fileName. We use keyName[0] to fileName
+		fileName = keyName[0]
 	}

 	if fileName == "" {
@@ -1139,9 +1136,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}

-	hashReader, err := hash.NewReader(reader, fileSize, "", "", fileSize)
+	hashReader, err := hash.NewReader(ctx, reader, fileSize, "", "", fileSize)
 	if err != nil {
-		logger.LogIf(ctx, err)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
@@ -1254,7 +1250,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				return
 			}
 			// do not try to verify encrypted content/
-			hashReader, err = hash.NewReader(reader, -1, "", "", -1)
+			hashReader, err = hash.NewReader(ctx, reader, -1, "", "", -1)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
@@ -43,6 +43,7 @@ import (
 	"github.com/minio/minio/internal/s3select"
 	"github.com/minio/pkg/v2/env"
 	"github.com/minio/pkg/v2/workers"
+	"github.com/zeebo/xxh3"
 )

 const (
@@ -444,7 +445,8 @@ func genTransitionObjName(bucket string) (string, error) {
 		return "", err
 	}
 	us := u.String()
-	obj := fmt.Sprintf("%s/%s/%s/%s/%s", globalDeploymentID, bucket, us[0:2], us[2:4], us)
+	hash := xxh3.HashString(pathJoin(globalDeploymentID(), bucket))
+	obj := fmt.Sprintf("%s/%s/%s/%s", strconv.FormatUint(hash, 16), us[0:2], us[2:4], us)
 	return obj, nil
 }

@@ -29,9 +29,7 @@ import (
 )

 // BucketQuotaSys - map of bucket and quota configuration.
-type BucketQuotaSys struct {
-	bucketStorageCache timedValue
-}
+type BucketQuotaSys struct{}

 // Get - Get quota configuration.
 func (sys *BucketQuotaSys) Get(ctx context.Context, bucketName string) (*madmin.BucketQuota, error) {
@@ -44,16 +42,18 @@ func NewBucketQuotaSys() *BucketQuotaSys {
 	return &BucketQuotaSys{}
 }

+var bucketStorageCache timedValue
+
 // Init initialize bucket quota.
 func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
-	sys.bucketStorageCache.Once.Do(func() {
+	bucketStorageCache.Once.Do(func() {
 		// Set this to 10 secs since its enough, as scanner
 		// does not update the bucket usage values frequently.
-		sys.bucketStorageCache.TTL = 10 * time.Second
+		bucketStorageCache.TTL = 10 * time.Second
 		// Rely on older value if usage loading fails from disk.
-		sys.bucketStorageCache.Relax = true
-		sys.bucketStorageCache.Update = func() (interface{}, error) {
-			ctx, done := context.WithTimeout(context.Background(), 1*time.Second)
+		bucketStorageCache.Relax = true
+		bucketStorageCache.Update = func() (interface{}, error) {
+			ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
 			defer done()

 			return loadDataUsageFromBackend(ctx, objAPI)
@@ -63,16 +63,17 @@ func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {

 // GetBucketUsageInfo return bucket usage info for a given bucket
 func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, error) {
-	v, err := sys.bucketStorageCache.Get()
-	if err != nil && v != nil {
-		logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
-	}
-	if v == nil {
-		logger.LogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
+	v, err := bucketStorageCache.Get()
+	timedout := OperationTimedOut{}
+	if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &timedout) {
+		if v != nil {
+			logger.LogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket)
+		} else {
+			logger.LogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket)
+		}
 	}

 	var bui BucketUsageInfo
-
 	dui, ok := v.(DataUsageInfo)
 	if ok {
 		bui = dui.BucketsUsage[bucket]
@@ -476,27 +476,17 @@ func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.Response
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	var tgtStats map[string]TargetReplicationResyncStatus
-	globalReplicationPool.resyncer.RLock()
-	brs, ok := globalReplicationPool.resyncer.statusMap[bucket]
-	if ok {
-		tgtStats = brs.cloneTgtStats()
-	}
-	globalReplicationPool.resyncer.RUnlock()
-	if !ok {
-		brs, err = loadBucketResyncMetadata(ctx, bucket, objectAPI)
-		if err != nil {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
-				Bucket: bucket,
-				Err:    fmt.Errorf("No replication resync status available for %s", arn),
-			}), r.URL)
-			return
-		}
-		tgtStats = brs.cloneTgtStats()
+	brs, err := loadBucketResyncMetadata(ctx, bucket, objectAPI)
+	if err != nil {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{
+			Bucket: bucket,
+			Err:    fmt.Errorf("replication resync status not available for %s (%s)", arn, err.Error()),
+		}), r.URL)
+		return
 	}

 	var rinfo ResyncTargetsInfo
-	for tarn, st := range tgtStats {
+	for tarn, st := range brs.TargetsMap {
 		if arn != "" && tarn != arn {
 			continue
 		}
@@ -585,7 +575,7 @@ func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.Respon
 		if rule.Status == replication.Disabled {
 			continue
 		}
-		clnt := globalBucketTargetSys.GetRemoteTargetClient(rule.Destination.Bucket)
+		clnt := globalBucketTargetSys.GetRemoteTargetClient(bucket, rule.Destination.Bucket)
 		if clnt == nil {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetNotFoundError, fmt.Errorf("replication config with rule ID %s has a stale target", rule.ID)), r.URL)
 			return
@@ -31,8 +31,8 @@ import (

 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 )

 //go:generate msgp -file=$GOFILE
@@ -167,7 +167,21 @@ func (ri replicatedInfos) Action() replicationAction {
 var replStatusRegex = regexp.MustCompile(`([^=].*?)=([^,].*?);`)

 // TargetReplicationStatus - returns replication status of a target
-func (o *ObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) {
+func (ri ReplicateObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) {
+	repStatMatches := replStatusRegex.FindAllStringSubmatch(ri.ReplicationStatusInternal, -1)
+	for _, repStatMatch := range repStatMatches {
+		if len(repStatMatch) != 3 {
+			return
+		}
+		if repStatMatch[1] == arn {
+			return replication.StatusType(repStatMatch[2])
+		}
+	}
+	return
+}
+
+// TargetReplicationStatus - returns replication status of a target
+func (o ObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) {
 	repStatMatches := replStatusRegex.FindAllStringSubmatch(o.ReplicationStatusInternal, -1)
 	for _, repStatMatch := range repStatMatches {
 		if len(repStatMatch) != 3 {
@@ -185,7 +199,6 @@ type replicateTargetDecision struct {
 	Synchronous bool   // Synchronous replication configured.
 	Arn         string // ARN of replication target
 	ID          string
-	Tgt         *TargetClient
 }

 func (t *replicateTargetDecision) String() string {
@@ -207,7 +220,7 @@ type ReplicateDecision struct {
 }

 // ReplicateAny returns true if atleast one target qualifies for replication
-func (d *ReplicateDecision) ReplicateAny() bool {
+func (d ReplicateDecision) ReplicateAny() bool {
 	for _, t := range d.targetsMap {
 		if t.Replicate {
 			return true
@@ -217,7 +230,7 @@ func (d *ReplicateDecision) ReplicateAny() bool {
 }

 // Synchronous returns true if atleast one target qualifies for synchronous replication
-func (d *ReplicateDecision) Synchronous() bool {
+func (d ReplicateDecision) Synchronous() bool {
 	for _, t := range d.targetsMap {
 		if t.Synchronous {
 			return true
@@ -226,7 +239,7 @@ func (d *ReplicateDecision) Synchronous() bool {
 	return false
 }

-func (d *ReplicateDecision) String() string {
+func (d ReplicateDecision) String() string {
 	b := new(bytes.Buffer)
 	for key, value := range d.targetsMap {
 		fmt.Fprintf(b, "%s=%s,", key, value.String())
@@ -243,7 +256,7 @@ func (d *ReplicateDecision) Set(t replicateTargetDecision) {
 }

 // PendingStatus returns a stringified representation of internal replication status with all targets marked as `PENDING`
-func (d *ReplicateDecision) PendingStatus() string {
+func (d ReplicateDecision) PendingStatus() string {
 	b := new(bytes.Buffer)
 	for _, k := range d.targetsMap {
 		if k.Replicate {
@@ -259,11 +272,11 @@ type ResyncDecision struct {
 }

 // Empty returns true if no targets with resync decision present
-func (r *ResyncDecision) Empty() bool {
+func (r ResyncDecision) Empty() bool {
 	return r.targets == nil
 }

-func (r *ResyncDecision) mustResync() bool {
+func (r ResyncDecision) mustResync() bool {
 	for _, v := range r.targets {
 		if v.Replicate {
 			return true
@@ -272,15 +285,12 @@ func (r *ResyncDecision) mustResync() bool {
 	return false
 }

-func (r *ResyncDecision) mustResyncTarget(tgtArn string) bool {
+func (r ResyncDecision) mustResyncTarget(tgtArn string) bool {
 	if r.targets == nil {
 		return false
 	}
 	v, ok := r.targets[tgtArn]
-	if ok && v.Replicate {
-		return true
-	}
-	return false
+	return ok && v.Replicate
 }

 // ResyncTargetDecision is struct that represents resync decision for this target
@@ -301,35 +311,20 @@ func parseReplicateDecision(ctx context.Context, bucket, s string) (r ReplicateD
 	if len(s) == 0 {
 		return
 	}
-	pairs := strings.Split(s, ",")
-	for _, p := range pairs {
+	for _, p := range strings.Split(s, ",") {
+		if p == "" {
+			continue
+		}
 		slc := strings.Split(p, "=")
 		if len(slc) != 2 {
 			return r, errInvalidReplicateDecisionFormat
 		}
-		tgtStr := strings.TrimPrefix(slc[1], "\"")
-		tgtStr = strings.TrimSuffix(tgtStr, "\"")
+		tgtStr := strings.TrimSuffix(strings.TrimPrefix(slc[1], `"`), `"`)
 		tgt := strings.Split(tgtStr, ";")
 		if len(tgt) != 4 {
 			return r, errInvalidReplicateDecisionFormat
 		}
-		var replicate, sync bool
-		var err error
-		replicate, err = strconv.ParseBool(tgt[0])
-		if err != nil {
-			return r, err
-		}
-		sync, err = strconv.ParseBool(tgt[1])
-		if err != nil {
-			return r, err
-		}
-		tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(slc[0])
-		if tgtClnt == nil {
-			// Skip stale targets if any and log them to be missing atleast once.
-			logger.LogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, slc[0]), slc[0])
-			// We save the targetDecision even when its not configured or stale.
-		}
-		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: replicate, Synchronous: sync, Arn: tgt[2], ID: tgt[3], Tgt: tgtClnt}
+		r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: tgt[0] == "true", Synchronous: tgt[1] == "true", Arn: tgt[2], ID: tgt[3]}
 	}
 	return
 }
@@ -496,8 +491,8 @@ func getCompositeVersionPurgeStatus(m map[string]VersionPurgeStatusType) Version
 }

 // getHealReplicateObjectInfo returns info needed by heal replication in ReplicateObjectInfo
-func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) ReplicateObjectInfo {
-	oi := objInfo.Clone()
+func getHealReplicateObjectInfo(oi ObjectInfo, rcfg replicationConfig) ReplicateObjectInfo {
+	userDefined := cloneMSS(oi.UserDefined)
 	if rcfg.Config != nil && rcfg.Config.RoleArn != "" {
 		// For backward compatibility of objects pending/failed replication.
 		// Save replication related statuses in the new internal representation for
@@ -508,17 +503,15 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 		if !oi.VersionPurgeStatus.Empty() {
 			oi.VersionPurgeStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.VersionPurgeStatus)
 		}
-		for k, v := range oi.UserDefined {
+		for k, v := range userDefined {
 			if strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset) {
-				delete(oi.UserDefined, k)
-				oi.UserDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
+				delete(userDefined, k)
+				userDefined[targetResetHeader(rcfg.Config.RoleArn)] = v
 			}
 		}
 	}
-	var dsc ReplicateDecision
-	var tgtStatuses map[string]replication.StatusType
-	var purgeStatuses map[string]VersionPurgeStatusType

+	var dsc ReplicateDecision
 	if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() {
 		dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{
 			ObjectV: ObjectV{
@@ -530,33 +523,45 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
 		}, nil)
 	} else {
-		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{
-			UserDefined: oi.UserDefined,
-		}, replication.HealReplicationType, ObjectOptions{}))
+		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(userDefined, oi.UserTags, "", replication.HealReplicationType, ObjectOptions{}))
 	}
-	tgtStatuses = replicationStatusesMap(oi.ReplicationStatusInternal)
-	purgeStatuses = versionPurgeStatusesMap(oi.VersionPurgeStatusInternal)
-	existingObjResync := rcfg.Resync(GlobalContext, oi, &dsc, tgtStatuses)
-	tm, _ := time.Parse(time.RFC3339Nano, oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp])
+
+	tgtStatuses := replicationStatusesMap(oi.ReplicationStatusInternal)
+	purgeStatuses := versionPurgeStatusesMap(oi.VersionPurgeStatusInternal)
+	existingObjResync := rcfg.Resync(GlobalContext, oi, dsc, tgtStatuses)
+	tm, _ := time.Parse(time.RFC3339Nano, userDefined[ReservedMetadataPrefixLower+ReplicationTimestamp])
+	rstate := oi.ReplicationState()
+	rstate.ReplicateDecisionStr = dsc.String()
+	asz, _ := oi.GetActualSize()
+
 	return ReplicateObjectInfo{
-		ObjectInfo:           oi,
+		Name:                       oi.Name,
+		Size:                       oi.Size,
+		ActualSize:                 asz,
+		Bucket:                     oi.Bucket,
+		VersionID:                  oi.VersionID,
+		ETag:                       oi.ETag,
+		ModTime:                    oi.ModTime,
+		ReplicationStatus:          oi.ReplicationStatus,
+		ReplicationStatusInternal:  oi.ReplicationStatusInternal,
+		DeleteMarker:               oi.DeleteMarker,
+		VersionPurgeStatusInternal: oi.VersionPurgeStatusInternal,
+		VersionPurgeStatus:         oi.VersionPurgeStatus,
+
+		ReplicationState:     rstate,
 		OpType:               replication.HealReplicationType,
 		Dsc:                  dsc,
 		ExistingObjResync:    existingObjResync,
 		TargetStatuses:       tgtStatuses,
 		TargetPurgeStatuses:  purgeStatuses,
 		ReplicationTimestamp: tm,
+		SSEC:                 crypto.SSEC.IsEncrypted(oi.UserDefined),
+		UserTags:             oi.UserTags,
 	}
 }

-func (ri *ReplicateObjectInfo) getReplicationState() ReplicationState {
-	rs := ri.ObjectInfo.getReplicationState()
-	rs.ReplicateDecisionStr = ri.Dsc.String()
-	return rs
-}
-
-// vID here represents the versionID client specified in request - need to distinguish between delete marker and delete marker deletion
-func (o *ObjectInfo) getReplicationState() ReplicationState {
+// ReplicationState - returns replication state using other internal replication metadata in ObjectInfo
+func (o ObjectInfo) ReplicationState() ReplicationState {
 	rs := ReplicationState{
 		ReplicationStatusInternal:  o.ReplicationStatusInternal,
 		VersionPurgeStatusInternal: o.VersionPurgeStatusInternal,
@@ -577,7 +582,7 @@ func (o *ObjectInfo) getReplicationState() ReplicationState {
 }

 // ReplicationState returns replication state using other internal replication metadata in ObjectToDelete
-func (o *ObjectToDelete) ReplicationState() ReplicationState {
+func (o ObjectToDelete) ReplicationState() ReplicationState {
 	r := ReplicationState{
 		ReplicationStatusInternal:  o.DeleteMarkerReplicationStatus,
 		VersionPurgeStatusInternal: o.VersionPurgeStatuses,
@@ -88,7 +88,7 @@ var replicationConfigTests = []struct {
 func TestReplicationResync(t *testing.T) {
 	ctx := context.Background()
 	for i, test := range replicationConfigTests {
-		if sync := test.rcfg.Resync(ctx, test.info, &test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
+		if sync := test.rcfg.Resync(ctx, test.info, test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
 			t.Errorf("Test%d (%s): Resync  got %t , want %t", i+1, test.name, sync.mustResync(), test.expectedSync)
 		}
 	}
@@ -283,7 +283,7 @@ var (

 func TestReplicationResyncwrapper(t *testing.T) {
 	for i, test := range replicationConfigTests2 {
-		if sync := test.rcfg.resync(test.info, &test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
+		if sync := test.rcfg.resync(test.info, test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync {
 			t.Errorf("%s (%s): Replicationresync  got %t , want %t", fmt.Sprintf("Test%d - %s", i+1, time.Now().Format(http.TimeFormat)), test.name, sync.mustResync(), test.expectedSync)
 		}
 	}
@@ -40,14 +40,29 @@ const (
 	defaultHealthCheckReloadDuration = 30 * time.Minute
 )

+type arnTarget struct {
+	Client      *TargetClient
+	lastRefresh time.Time
+}
+
+// arnErrs represents number of errors seen for a ARN and if update is in progress
+// to refresh remote targets from bucket metadata.
+type arnErrs struct {
+	count            int64
+	updateInProgress bool
+	bucket           string
+}
+
 // BucketTargetSys represents bucket targets subsystem
 type BucketTargetSys struct {
 	sync.RWMutex
-	arnRemotesMap map[string]*TargetClient
+	arnRemotesMap map[string]arnTarget
 	targetsMap    map[string][]madmin.BucketTarget
 	hMutex        sync.RWMutex
 	hc            map[string]epHealth
 	hcClient      *madmin.AnonymousClient
+	aMutex        sync.RWMutex
+	arnErrsMap    map[string]arnErrs // map of ARN to error count of failures to get target
 }

 type latencyStat struct {
@@ -364,7 +379,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	}

 	sys.targetsMap[bucket] = newtgts
-	sys.arnRemotesMap[tgt.Arn] = clnt
+	sys.arnRemotesMap[tgt.Arn] = arnTarget{Client: clnt}
 	sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
 	return nil
 }
@@ -432,11 +447,71 @@ func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr str
 	return nil
 }

+func (sys *BucketTargetSys) markRefreshInProgress(bucket, arn string) {
+	sys.aMutex.Lock()
+	defer sys.aMutex.Unlock()
+	if v, ok := sys.arnErrsMap[arn]; !ok {
+		sys.arnErrsMap[arn] = arnErrs{
+			updateInProgress: true,
+			count:            v.count + 1,
+			bucket:           bucket,
+		}
+	}
+}
+
+func (sys *BucketTargetSys) markRefreshDone(bucket, arn string) {
+	sys.aMutex.Lock()
+	defer sys.aMutex.Unlock()
+	if v, ok := sys.arnErrsMap[arn]; ok {
+		sys.arnErrsMap[arn] = arnErrs{
+			updateInProgress: false,
+			count:            v.count,
+			bucket:           bucket,
+		}
+	}
+}
+
+func (sys *BucketTargetSys) isReloadingTarget(bucket, arn string) bool {
+	sys.aMutex.RLock()
+	defer sys.aMutex.RUnlock()
+	if v, ok := sys.arnErrsMap[arn]; ok {
+		return v.updateInProgress
+	}
+	return false
+}
+
+func (sys *BucketTargetSys) incTargetErr(bucket, arn string) {
+	sys.aMutex.Lock()
+	defer sys.aMutex.Unlock()
+	if v, ok := sys.arnErrsMap[arn]; ok {
+		sys.arnErrsMap[arn] = arnErrs{
+			updateInProgress: v.updateInProgress,
+			count:            v.count + 1,
+		}
+	}
+}
+
 // GetRemoteTargetClient returns minio-go client for replication target instance
-func (sys *BucketTargetSys) GetRemoteTargetClient(arn string) *TargetClient {
+func (sys *BucketTargetSys) GetRemoteTargetClient(bucket, arn string) *TargetClient {
 	sys.RLock()
-	defer sys.RUnlock()
-	return sys.arnRemotesMap[arn]
+	tgt := sys.arnRemotesMap[arn]
+	sys.RUnlock()
+
+	if tgt.Client != nil {
+		return tgt.Client
+	}
+	defer func() { // lazy refresh remote targets
+		if tgt.Client == nil && !sys.isReloadingTarget(bucket, arn) && (tgt.lastRefresh.Equal(timeSentinel) || tgt.lastRefresh.Before(UTCNow().Add(-5*time.Minute))) {
+			tgts, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket)
+			if err == nil {
+				sys.markRefreshInProgress(bucket, arn)
+				sys.UpdateAllTargets(bucket, tgts)
+				sys.markRefreshDone(bucket, arn)
+			}
+		}
+		sys.incTargetErr(bucket, arn)
+	}()
+	return nil
 }

 // GetRemoteBucketTargetByArn returns BucketTarget for a ARN
@@ -457,8 +532,9 @@ func (sys *BucketTargetSys) GetRemoteBucketTargetByArn(ctx context.Context, buck
 // NewBucketTargetSys - creates new replication system.
 func NewBucketTargetSys(ctx context.Context) *BucketTargetSys {
 	sys := &BucketTargetSys{
-		arnRemotesMap: make(map[string]*TargetClient),
+		arnRemotesMap: make(map[string]arnTarget),
 		targetsMap:    make(map[string][]madmin.BucketTarget),
+		arnErrsMap:    make(map[string]arnErrs),
 		hc:            make(map[string]epHealth),
 		hcClient:      newHCClient(),
 	}
@@ -502,7 +578,10 @@ func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketT
 			if err != nil {
 				continue
 			}
-			sys.arnRemotesMap[tgt.Arn] = tgtClient
+			sys.arnRemotesMap[tgt.Arn] = arnTarget{
+				Client:      tgtClient,
+				lastRefresh: UTCNow(),
+			}
 			sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit)
 		}

@@ -526,7 +605,7 @@ func (sys *BucketTargetSys) set(bucket BucketInfo, meta BucketMetadata) {
 			logger.LogIf(GlobalContext, err)
 			continue
 		}
-		sys.arnRemotesMap[tgt.Arn] = tgtClient
+		sys.arnRemotesMap[tgt.Arn] = arnTarget{Client: tgtClient}
 		sys.updateBandwidthLimit(bucket.Name, tgt.Arn, tgt.BandwidthLimit)
 	}
 	sys.targetsMap[bucket.Name] = cfg.Targets
@@ -131,7 +131,7 @@ func performCallhome(ctx context.Context) {
 		Version:   madmin.HealthInfoVersion,
 		Minio: madmin.MinioHealthInfo{
 			Info: madmin.MinioInfo{
-				DeploymentID: globalDeploymentID,
+				DeploymentID: globalDeploymentID(),
 			},
 		},
 	}
@@ -29,7 +29,6 @@ import (
 	"fmt"
 	"math/rand"
 	"net"
-	"net/http"
 	"net/url"
 	"os"
 	"path"
@@ -54,7 +53,6 @@ import (
 	"github.com/minio/kes-go"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
-	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
@@ -71,10 +69,7 @@ import (
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn

-var (
-	shardDiskTimeDelta     time.Duration
-	defaultAWSCredProvider []credentials.Provider
-)
+var shardDiskTimeDelta time.Duration

 func init() {
 	if runtime.GOOS == "windows" {
@@ -112,30 +107,23 @@ func init() {
 	gob.Register(madmin.XFSErrorConfigs{})
 	gob.Register(map[string]interface{}{})

-	defaultAWSCredProvider = []credentials.Provider{
-		&credentials.IAM{
-			Client: &http.Client{
-				Transport: NewHTTPTransport(),
-			},
-		},
-	}
-
 	var err error
 	shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m"))
 	if err != nil {
 		shardDiskTimeDelta = 1 * time.Minute
 	}

-	// All minio-go API operations shall be performed only once,
+	// All minio-go and madmin-go API operations shall be performed only once,
 	// another way to look at this is we are turning off retries.
 	minio.MaxRetry = 1
+	madmin.MaxRetry = 1
 }

 const consolePrefix = "CONSOLE_"

 func minioConfigToConsoleFeatures() {
-	os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID)
-	os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID)
+	os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID())
+	os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID())
 	if globalMinioEndpoint != "" {
 		os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint)
 	} else {
@@ -143,29 +131,33 @@ func minioConfigToConsoleFeatures() {
 		// This will save users from providing a certificate with IP or FQDN SAN that points to the local host.
 		os.Setenv("CONSOLE_MINIO_SERVER", fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(globalIsTLS), globalMinioPort))
 	}
-	if value := env.Get("MINIO_LOG_QUERY_URL", ""); value != "" {
+	if value := env.Get(config.EnvMinIOLogQueryURL, ""); value != "" {
 		os.Setenv("CONSOLE_LOG_QUERY_URL", value)
-		if value := env.Get("MINIO_LOG_QUERY_AUTH_TOKEN", ""); value != "" {
+		if value := env.Get(config.EnvMinIOLogQueryAuthToken, ""); value != "" {
 			os.Setenv("CONSOLE_LOG_QUERY_AUTH_TOKEN", value)
 		}
 	}
 	// pass the console subpath configuration
-	if value := env.Get(config.EnvBrowserRedirectURL, ""); value != "" {
+	if globalBrowserRedirectURL != nil {
 		subPath := path.Clean(pathJoin(strings.TrimSpace(globalBrowserRedirectURL.Path), SlashSeparator))
 		if subPath != SlashSeparator {
 			os.Setenv("CONSOLE_SUBPATH", subPath)
 		}
 	}
 	// Enable if prometheus URL is set.
-	if value := env.Get("MINIO_PROMETHEUS_URL", ""); value != "" {
+	if value := env.Get(config.EnvMinIOPrometheusURL, ""); value != "" {
 		os.Setenv("CONSOLE_PROMETHEUS_URL", value)
-		if value := env.Get("MINIO_PROMETHEUS_JOB_ID", "minio-job"); value != "" {
+		if value := env.Get(config.EnvMinIOPrometheusJobID, "minio-job"); value != "" {
 			os.Setenv("CONSOLE_PROMETHEUS_JOB_ID", value)
 			// Support additional labels for more granular filtering.
-			if value := env.Get("MINIO_PROMETHEUS_EXTRA_LABELS", ""); value != "" {
+			if value := env.Get(config.EnvMinIOPrometheusExtraLabels, ""); value != "" {
 				os.Setenv("CONSOLE_PROMETHEUS_EXTRA_LABELS", value)
 			}
 		}
+		// Support Prometheus Auth Token
+		if value := env.Get(config.EnvMinIOPrometheusAuthToken, ""); value != "" {
+			os.Setenv("CONSOLE_PROMETHEUS_AUTH_TOKEN", value)
+		}
 	}
 	// Enable if LDAP is enabled.
 	if globalIAMSys.LDAPConfig.Enabled() {
@@ -175,9 +167,12 @@ func minioConfigToConsoleFeatures() {
 	if value := env.Get(config.EnvBrowserLoginAnimation, "on"); value != "" {
 		os.Setenv("CONSOLE_ANIMATED_LOGIN", value)
 	}
+
 	// Pass on the session duration environment variable, else we will default to 12 hours
-	if value := env.Get(config.EnvBrowserSessionDuration, ""); value != "" {
-		os.Setenv("CONSOLE_STS_DURATION", value)
+	if valueSts := env.Get(config.EnvMinioStsDuration, ""); valueSts != "" {
+		os.Setenv("CONSOLE_STS_DURATION", valueSts)
+	} else if valueSession := env.Get(config.EnvBrowserSessionDuration, ""); valueSession != "" {
+		os.Setenv("CONSOLE_STS_DURATION", valueSession)
 	}

 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
@@ -199,7 +194,7 @@ func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg {
 			DisplayName:             cfg.DisplayName,
 			ClientID:                cfg.ClientID,
 			ClientSecret:            cfg.ClientSecret,
-			HMACSalt:                globalDeploymentID,
+			HMACSalt:                globalDeploymentID(),
 			HMACPassphrase:          cfg.ClientID,
 			Scopes:                  strings.Join(cfg.DiscoveryDoc.ScopesSupported, ","),
 			Userinfo:                cfg.ClaimUserinfo,
@@ -391,7 +386,7 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	if consoleAddr == "" {
 		p, err := xnet.GetFreePort()
 		if err != nil {
-			logger.FatalIf(err, "Unable to get free port for console on the host")
+			logger.FatalIf(err, "Unable to get free port for Console UI on the host")
 		}
 		consoleAddr = net.JoinHostPort("", p.String())
 	}
@@ -405,6 +400,15 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	}

 	globalMinioHost, globalMinioPort = mustSplitHostPort(addr)
+	if globalMinioPort == "0" {
+		p, err := xnet.GetFreePort()
+		if err != nil {
+			logger.FatalIf(err, "Unable to get free port for S3 API on the host")
+		}
+		globalMinioPort = p.String()
+		globalDynamicAPIPort = true
+	}
+
 	globalMinioConsoleHost, globalMinioConsolePort = mustSplitHostPort(consoleAddr)

 	if globalMinioPort == globalMinioConsolePort {
@@ -618,8 +622,6 @@ func loadEnvVarsFromFiles() {
 }

 func handleCommonEnvVars() {
-	loadEnvVarsFromFiles()
-
 	var err error
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {
@@ -640,6 +642,7 @@ func handleCommonEnvVars() {
 			}
 			globalBrowserRedirectURL = u
 		}
+		globalBrowserRedirect = env.Get(config.EnvBrowserRedirect, config.EnableOn) == config.EnableOn
 	}

 	if serverURL := env.Get(config.EnvMinIOServerURL, ""); serverURL != "" {
@@ -664,10 +667,14 @@ func handleCommonEnvVars() {
 		logger.Fatal(config.ErrInvalidFSOSyncValue(err), "Invalid MINIO_FS_OSYNC value in environment variable")
 	}

-	if rootDiskSize := env.Get(config.EnvRootDiskThresholdSize, ""); rootDiskSize != "" {
+	rootDiskSize := env.Get(config.EnvRootDriveThresholdSize, "")
+	if rootDiskSize == "" {
+		rootDiskSize = env.Get(config.EnvRootDiskThresholdSize, "")
+	}
+	if rootDiskSize != "" {
 		size, err := humanize.ParseBytes(rootDiskSize)
 		if err != nil {
-			logger.Fatal(err, fmt.Sprintf("Invalid %s value in environment variable", config.EnvRootDiskThresholdSize))
+			logger.Fatal(err, fmt.Sprintf("Invalid %s value in root drive threshold environment variable", rootDiskSize))
 		}
 		globalRootDiskThreshold = size
 	}
@@ -802,6 +809,9 @@ func handleKMSConfig() {
 				logger.Fatal(errors.New("ambigious KMS configuration"), fmt.Sprintf("The environment contains %q as well as %q", kms.EnvKESAPIKey, kms.EnvKESClientCert))
 			}
 		}
+		if !env.IsSet(kms.EnvKESKeyName) {
+			logger.Fatal(errors.New("Invalid KES configuration"), fmt.Sprintf("The mandatory environment variable %q not set", kms.EnvKESKeyName))
+		}

 		var endpoints []string
 		for _, endpoint := range strings.Split(env.Get(kms.EnvKESEndpoint, ""), ",") {
@@ -71,7 +71,7 @@ func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string)
 }

 func saveConfigWithOpts(ctx context.Context, store objectIO, configFile string, data []byte, opts ObjectOptions) error {
-	hashReader, err := hash.NewReader(bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
+	hashReader, err := hash.NewReader(ctx, bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data)))
 	if err != nil {
 		return err
 	}
@@ -27,7 +27,6 @@ import (
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/api"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/callhome"
 	"github.com/minio/minio/internal/config/compress"
 	"github.com/minio/minio/internal/config/dns"
@@ -46,7 +45,6 @@ import (
 	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/env"
 )
@@ -54,7 +52,6 @@ import (
 func initHelp() {
 	kvs := map[string]config.KVS{
 		config.EtcdSubSys:           etcd.DefaultKVS,
-		config.CacheSubSys:          cache.DefaultKVS,
 		config.CompressionSubSys:    compress.DefaultKVS,
 		config.IdentityLDAPSubSys:   xldap.DefaultKVS,
 		config.IdentityOpenIDSubSys: openid.DefaultKVS,
@@ -209,10 +206,6 @@ func initHelp() {
 			Key:         config.EtcdSubSys,
 			Description: "persist IAM assets externally to etcd",
 		},
-		config.HelpKV{
-			Key:         config.CacheSubSys,
-			Description: "[DEPRECATED] add caching storage tier",
-		},
 	}

 	if globalIsErasure {
@@ -232,7 +225,6 @@ func initHelp() {
 		config.APISubSys:            api.Help,
 		config.StorageClassSubSys:   storageclass.Help,
 		config.EtcdSubSys:           etcd.Help,
-		config.CacheSubSys:          cache.Help,
 		config.CompressionSubSys:    compress.Help,
 		config.HealSubSys:           heal.Help,
 		config.ScannerSubSys:        scanner.Help,
@@ -302,10 +294,6 @@ func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, o
 				return err
 			}
 		}
-	case config.CacheSubSys:
-		if _, err := cache.LookupConfig(s[config.CacheSubSys][config.Default]); err != nil {
-			return err
-		}
 	case config.CompressionSubSys:
 		if _, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default]); err != nil {
 			return err
@@ -493,20 +481,6 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
 	}

-	globalCacheConfig, err = cache.LookupConfig(s[config.CacheSubSys][config.Default])
-	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to setup cache: %w", err))
-	}
-
-	if globalCacheConfig.Enabled {
-		if cacheEncKey := env.Get(cache.EnvCacheEncryptionKey, ""); cacheEncKey != "" {
-			globalCacheKMS, err = kms.Parse(cacheEncKey)
-			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to setup encryption cache: %w", err))
-			}
-		}
-	}
-
 	globalAutoEncryption = crypto.LookupAutoEncryption() // Enable auto-encryption if enabled
 	if globalAutoEncryption && GlobalKMS == nil {
 		logger.Fatal(errors.New("no KMS configured"), "MINIO_KMS_AUTO_ENCRYPTION requires a valid KMS configuration")
@@ -1,336 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"os"
-	"testing"
-)
-
-// Test if config v1 is purged
-func TestServerConfigMigrateV1(t *testing.T) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	objLayer, fsDir, err := prepareFS(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(fsDir)
-	err = newTestConfig(globalMinioDefaultRegion, objLayer)
-	if err != nil {
-		t.Fatalf("Init Test config failed")
-	}
-	rootPath := t.TempDir()
-	globalConfigDir = &ConfigDir{path: rootPath}
-
-	globalObjLayerMutex.Lock()
-	globalObjectAPI = objLayer
-	globalObjLayerMutex.Unlock()
-
-	// Create a V1 config json file and store it
-	configJSON := "{ \"version\":\"1\", \"accessKeyId\":\"abcde\", \"secretAccessKey\":\"abcdefgh\"}"
-	configPath := rootPath + "/fsUsers.json"
-	if err := os.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
-		t.Fatal("Unexpected error: ", err)
-	}
-
-	// Fire a migrateConfig()
-	if err := migrateConfig(); err != nil {
-		t.Fatal("Unexpected error: ", err)
-	}
-
-	// Check if config v1 is removed from filesystem
-	if _, err := os.Stat(configPath); err == nil || !osIsNotExist(err) {
-		t.Fatal("Config V1 file is not purged")
-	}
-
-	// Initialize server config and check again if everything is fine
-	if err := loadConfig(objLayer, nil); err != nil {
-		t.Fatalf("Unable to initialize from updated config file %s", err)
-	}
-}
-
-// Test if all migrate code returns nil when config file does not
-// exist
-func TestServerConfigMigrateInexistentConfig(t *testing.T) {
-	globalConfigDir = &ConfigDir{path: t.TempDir()}
-
-	if err := migrateV2ToV3(); err != nil {
-		t.Fatal("migrate v2 to v3 should succeed when no config file is found")
-	}
-	if err := migrateV3ToV4(); err != nil {
-		t.Fatal("migrate v3 to v4 should succeed when no config file is found")
-	}
-	if err := migrateV4ToV5(); err != nil {
-		t.Fatal("migrate v4 to v5 should succeed when no config file is found")
-	}
-	if err := migrateV5ToV6(); err != nil {
-		t.Fatal("migrate v5 to v6 should succeed when no config file is found")
-	}
-	if err := migrateV6ToV7(); err != nil {
-		t.Fatal("migrate v6 to v7 should succeed when no config file is found")
-	}
-	if err := migrateV7ToV8(); err != nil {
-		t.Fatal("migrate v7 to v8 should succeed when no config file is found")
-	}
-	if err := migrateV8ToV9(); err != nil {
-		t.Fatal("migrate v8 to v9 should succeed when no config file is found")
-	}
-	if err := migrateV9ToV10(); err != nil {
-		t.Fatal("migrate v9 to v10 should succeed when no config file is found")
-	}
-	if err := migrateV10ToV11(); err != nil {
-		t.Fatal("migrate v10 to v11 should succeed when no config file is found")
-	}
-	if err := migrateV11ToV12(); err != nil {
-		t.Fatal("migrate v11 to v12 should succeed when no config file is found")
-	}
-	if err := migrateV12ToV13(); err != nil {
-		t.Fatal("migrate v12 to v13 should succeed when no config file is found")
-	}
-	if err := migrateV13ToV14(); err != nil {
-		t.Fatal("migrate v13 to v14 should succeed when no config file is found")
-	}
-	if err := migrateV14ToV15(); err != nil {
-		t.Fatal("migrate v14 to v15 should succeed when no config file is found")
-	}
-	if err := migrateV15ToV16(); err != nil {
-		t.Fatal("migrate v15 to v16 should succeed when no config file is found")
-	}
-	if err := migrateV16ToV17(); err != nil {
-		t.Fatal("migrate v16 to v17 should succeed when no config file is found")
-	}
-	if err := migrateV17ToV18(); err != nil {
-		t.Fatal("migrate v17 to v18 should succeed when no config file is found")
-	}
-	if err := migrateV18ToV19(); err != nil {
-		t.Fatal("migrate v18 to v19 should succeed when no config file is found")
-	}
-	if err := migrateV19ToV20(); err != nil {
-		t.Fatal("migrate v19 to v20 should succeed when no config file is found")
-	}
-	if err := migrateV20ToV21(); err != nil {
-		t.Fatal("migrate v20 to v21 should succeed when no config file is found")
-	}
-	if err := migrateV21ToV22(); err != nil {
-		t.Fatal("migrate v21 to v22 should succeed when no config file is found")
-	}
-	if err := migrateV22ToV23(); err != nil {
-		t.Fatal("migrate v22 to v23 should succeed when no config file is found")
-	}
-	if err := migrateV23ToV24(); err != nil {
-		t.Fatal("migrate v23 to v24 should succeed when no config file is found")
-	}
-	if err := migrateV24ToV25(); err != nil {
-		t.Fatal("migrate v24 to v25 should succeed when no config file is found")
-	}
-	if err := migrateV25ToV26(); err != nil {
-		t.Fatal("migrate v25 to v26 should succeed when no config file is found")
-	}
-	if err := migrateV26ToV27(); err != nil {
-		t.Fatal("migrate v26 to v27 should succeed when no config file is found")
-	}
-	if err := migrateV27ToV28(); err != nil {
-		t.Fatal("migrate v27 to v28 should succeed when no config file is found")
-	}
-}
-
-// Test if a config migration from v2 to v33 is successfully done
-func TestServerConfigMigrateV2toV33(t *testing.T) {
-	rootPath := t.TempDir()
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	globalConfigDir = &ConfigDir{path: rootPath}
-
-	objLayer, fsDir, err := prepareFS(ctx)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(fsDir)
-
-	configPath := rootPath + SlashSeparator + minioConfigFile
-
-	// Create a corrupted config file
-	if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\","), 0o644); err != nil {
-		t.Fatal("Unexpected error: ", err)
-	}
-	// Fire a migrateConfig()
-	if err := migrateConfig(); err == nil {
-		t.Fatal("migration should fail with corrupted config file")
-	}
-
-	accessKey := "accessfoo"
-	secretKey := "secretfoo"
-
-	// Create a V2 config json file and store it
-	configJSON := "{ \"version\":\"2\", \"credentials\": {\"accessKeyId\":\"" + accessKey + "\", \"secretAccessKey\":\"" + secretKey + "\", \"region\":\"us-east-1\"}, \"mongoLogger\":{\"addr\":\"127.0.0.1:3543\", \"db\":\"foodb\", \"collection\":\"foo\"}, \"syslogLogger\":{\"network\":\"127.0.0.1:543\", \"addr\":\"addr\"}, \"fileLogger\":{\"filename\":\"log.out\"}}"
-	if err := os.WriteFile(configPath, []byte(configJSON), 0o644); err != nil {
-		t.Fatal("Unexpected error: ", err)
-	}
-
-	// Fire a migrateConfig()
-	if err := migrateConfig(); err != nil {
-		t.Fatal("Unexpected error: ", err)
-	}
-
-	if err := migrateConfigToMinioSys(objLayer); err != nil {
-		t.Fatal("Unexpected error: ", err)
-	}
-
-	srvCfg, err := readConfigWithoutMigrate(context.Background(), objLayer)
-	if err != nil {
-		t.Fatal("Unexpected error: ", err)
-	}
-
-	if err = saveServerConfig(GlobalContext, objLayer, srvCfg); err != nil {
-		t.Fatal("Unexpected error: ", err)
-	}
-
-	// Initialize server config and check again if everything is fine
-	if err := loadConfig(objLayer, nil); err != nil {
-		t.Fatalf("Unable to initialize from updated config file %s", err)
-	}
-}
-
-// Test if all migrate code returns error with corrupted config files
-func TestServerConfigMigrateFaultyConfig(t *testing.T) {
-	rootPath := t.TempDir()
-
-	globalConfigDir = &ConfigDir{path: rootPath}
-	configPath := rootPath + SlashSeparator + minioConfigFile
-
-	// Create a corrupted config file
-	if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\", \"test\":"), 0o644); err != nil {
-		t.Fatal("Unexpected error: ", err)
-	}
-
-	// Test different migrate versions and be sure they are returning an error
-	if err := migrateV2ToV3(); err == nil {
-		t.Fatal("migrateConfigV2ToV3() should fail with a corrupted json")
-	}
-	if err := migrateV3ToV4(); err == nil {
-		t.Fatal("migrateConfigV3ToV4() should fail with a corrupted json")
-	}
-	if err := migrateV4ToV5(); err == nil {
-		t.Fatal("migrateConfigV4ToV5() should fail with a corrupted json")
-	}
-	if err := migrateV5ToV6(); err == nil {
-		t.Fatal("migrateConfigV5ToV6() should fail with a corrupted json")
-	}
-	if err := migrateV6ToV7(); err == nil {
-		t.Fatal("migrateConfigV6ToV7() should fail with a corrupted json")
-	}
-	if err := migrateV7ToV8(); err == nil {
-		t.Fatal("migrateConfigV7ToV8() should fail with a corrupted json")
-	}
-	if err := migrateV8ToV9(); err == nil {
-		t.Fatal("migrateConfigV8ToV9() should fail with a corrupted json")
-	}
-	if err := migrateV9ToV10(); err == nil {
-		t.Fatal("migrateConfigV9ToV10() should fail with a corrupted json")
-	}
-	if err := migrateV10ToV11(); err == nil {
-		t.Fatal("migrateConfigV10ToV11() should fail with a corrupted json")
-	}
-	if err := migrateV11ToV12(); err == nil {
-		t.Fatal("migrateConfigV11ToV12() should fail with a corrupted json")
-	}
-	if err := migrateV12ToV13(); err == nil {
-		t.Fatal("migrateConfigV12ToV13() should fail with a corrupted json")
-	}
-	if err := migrateV13ToV14(); err == nil {
-		t.Fatal("migrateConfigV13ToV14() should fail with a corrupted json")
-	}
-	if err := migrateV14ToV15(); err == nil {
-		t.Fatal("migrateConfigV14ToV15() should fail with a corrupted json")
-	}
-	if err := migrateV15ToV16(); err == nil {
-		t.Fatal("migrateConfigV15ToV16() should fail with a corrupted json")
-	}
-	if err := migrateV16ToV17(); err == nil {
-		t.Fatal("migrateConfigV16ToV17() should fail with a corrupted json")
-	}
-	if err := migrateV17ToV18(); err == nil {
-		t.Fatal("migrateConfigV17ToV18() should fail with a corrupted json")
-	}
-	if err := migrateV18ToV19(); err == nil {
-		t.Fatal("migrateConfigV18ToV19() should fail with a corrupted json")
-	}
-	if err := migrateV19ToV20(); err == nil {
-		t.Fatal("migrateConfigV19ToV20() should fail with a corrupted json")
-	}
-	if err := migrateV20ToV21(); err == nil {
-		t.Fatal("migrateConfigV20ToV21() should fail with a corrupted json")
-	}
-	if err := migrateV21ToV22(); err == nil {
-		t.Fatal("migrateConfigV21ToV22() should fail with a corrupted json")
-	}
-	if err := migrateV22ToV23(); err == nil {
-		t.Fatal("migrateConfigV22ToV23() should fail with a corrupted json")
-	}
-	if err := migrateV23ToV24(); err == nil {
-		t.Fatal("migrateConfigV23ToV24() should fail with a corrupted json")
-	}
-	if err := migrateV24ToV25(); err == nil {
-		t.Fatal("migrateConfigV24ToV25() should fail with a corrupted json")
-	}
-	if err := migrateV25ToV26(); err == nil {
-		t.Fatal("migrateConfigV25ToV26() should fail with a corrupted json")
-	}
-	if err := migrateV26ToV27(); err == nil {
-		t.Fatal("migrateConfigV26ToV27() should fail with a corrupted json")
-	}
-	if err := migrateV27ToV28(); err == nil {
-		t.Fatal("migrateConfigV27ToV28() should fail with a corrupted json")
-	}
-}
-
-// Test if all migrate code returns error with corrupted config files
-func TestServerConfigMigrateCorruptedConfig(t *testing.T) {
-	rootPath := t.TempDir()
-
-	globalConfigDir = &ConfigDir{path: rootPath}
-	configPath := rootPath + SlashSeparator + minioConfigFile
-
-	for i := 3; i <= 17; i++ {
-		// Create a corrupted config file
-		if err := os.WriteFile(configPath, []byte(fmt.Sprintf("{ \"version\":\"%d\", \"credential\": { \"accessKey\": 1 } }", i)),
-			0o644); err != nil {
-			t.Fatal("Unexpected error: ", err)
-		}
-
-		// Test different migrate versions and be sure they are returning an error
-		if err := migrateConfig(); err == nil {
-			t.Fatal("migrateConfig() should fail with a corrupted json")
-		}
-	}
-
-	// Create a corrupted config file for version '2'.
-	if err := os.WriteFile(configPath, []byte("{ \"version\":\"2\", \"credentials\": { \"accessKeyId\": 1 } }"), 0o644); err != nil {
-		t.Fatal("Unexpected error: ", err)
-	}
-
-	// Test different migrate versions and be sure they are returning an error
-	if err := migrateConfig(); err == nil {
-		t.Fatal("migrateConfig() should fail with a corrupted json")
-	}
-}
@@ -18,418 +18,18 @@
 package cmd

 import (
-	"sync"
-
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/compress"
 	xldap "github.com/minio/minio/internal/config/identity/ldap"
 	"github.com/minio/minio/internal/config/identity/openid"
 	"github.com/minio/minio/internal/config/notify"
 	"github.com/minio/minio/internal/config/policy/opa"
 	"github.com/minio/minio/internal/config/storageclass"
-	"github.com/minio/minio/internal/event/target"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/quick"
 )

-type configV1 struct {
-	Version   string `json:"version"`
-	AccessKey string `json:"accessKeyId"`
-	SecretKey string `json:"secretAccessKey"`
-}
-
-type configV2 struct {
-	Version     string `json:"version"`
-	Credentials struct {
-		AccessKey string `json:"accessKeyId"`
-		SecretKey string `json:"secretAccessKey"`
-		Region    string `json:"region"`
-	} `json:"credentials"`
-	MongoLogger struct {
-		Addr       string `json:"addr"`
-		DB         string `json:"db"`
-		Collection string `json:"collection"`
-	} `json:"mongoLogger"`
-	SyslogLogger struct {
-		Network string `json:"network"`
-		Addr    string `json:"addr"`
-	} `json:"syslogLogger"`
-	FileLogger struct {
-		Filename string `json:"filename"`
-	} `json:"fileLogger"`
-}
-
-// backendV3 type.
-type backendV3 struct {
-	Type  string   `json:"type"`
-	Disk  string   `json:"disk,omitempty"`
-	Disks []string `json:"disks,omitempty"`
-}
-
-// syslogLogger v3
-type syslogLoggerV3 struct {
-	Enable bool   `json:"enable"`
-	Addr   string `json:"address"`
-	Level  string `json:"level"`
-}
-
-// loggerV3 type.
-type loggerV3 struct {
-	Console struct {
-		Enable bool   `json:"enable"`
-		Level  string `json:"level"`
-	}
-	File struct {
-		Enable   bool   `json:"enable"`
-		Filename string `json:"fileName"`
-		Level    string `json:"level"`
-	}
-	Syslog struct {
-		Enable bool   `json:"enable"`
-		Addr   string `json:"address"`
-		Level  string `json:"level"`
-	} `json:"syslog"`
-	// Add new loggers here.
-}
-
-// configV3 server configuration version '3'.
-type configV3 struct {
-	Version string `json:"version"`
-
-	// Backend configuration.
-	Backend backendV3 `json:"backend"`
-
-	// http Server configuration.
-	Addr string `json:"address"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-
-	// Additional error logging configuration.
-	Logger loggerV3 `json:"logger"`
-}
-
-// logger type representing version '4' logger config.
-type loggerV4 struct {
-	Console struct {
-		Enable bool   `json:"enable"`
-		Level  string `json:"level"`
-	} `json:"console"`
-	File struct {
-		Enable   bool   `json:"enable"`
-		Filename string `json:"fileName"`
-		Level    string `json:"level"`
-	} `json:"file"`
-	Syslog struct {
-		Enable bool   `json:"enable"`
-		Addr   string `json:"address"`
-		Level  string `json:"level"`
-	} `json:"syslog"`
-}
-
-// configV4 server configuration version '4'.
-type configV4 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-
-	// Additional error logging configuration.
-	Logger loggerV4 `json:"logger"`
-}
-
-// logger type representing version '5' logger config.
-type loggerV5 struct {
-	Console struct {
-		Enable bool   `json:"enable"`
-		Level  string `json:"level"`
-	} `json:"console"`
-	File struct {
-		Enable   bool   `json:"enable"`
-		Filename string `json:"fileName"`
-		Level    string `json:"level"`
-	} `json:"file"`
-	Syslog struct {
-		Enable bool   `json:"enable"`
-		Addr   string `json:"address"`
-		Level  string `json:"level"`
-	} `json:"syslog"`
-	AMQP struct {
-		Enable       bool   `json:"enable"`
-		Level        string `json:"level"`
-		URL          string `json:"url"`
-		Exchange     string `json:"exchange"`
-		RoutingKey   string `json:"routingKey"`
-		ExchangeType string `json:"exchangeType"`
-		Mandatory    bool   `json:"mandatory"`
-		Immediate    bool   `json:"immediate"`
-		Durable      bool   `json:"durable"`
-		Internal     bool   `json:"internal"`
-		NoWait       bool   `json:"noWait"`
-		AutoDeleted  bool   `json:"autoDeleted"`
-	} `json:"amqp"`
-	ElasticSearch struct {
-		Enable bool   `json:"enable"`
-		Level  string `json:"level"`
-		URL    string `json:"url"`
-		Index  string `json:"index"`
-	} `json:"elasticsearch"`
-	Redis struct {
-		Enable   bool   `json:"enable"`
-		Level    string `json:"level"`
-		Addr     string `json:"address"`
-		Password string `json:"password"`
-		Key      string `json:"key"`
-	} `json:"redis"`
-}
-
-// configV5 server configuration version '5'.
-type configV5 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-
-	// Additional error logging configuration.
-	Logger loggerV5 `json:"logger"`
-}
-
-// consoleLogger - default logger if not other logging is enabled.
-type consoleLoggerV1 struct {
-	Enable bool   `json:"enable"`
-	Level  string `json:"level"`
-}
-
-type fileLoggerV1 struct {
-	Enable   bool   `json:"enable"`
-	Filename string `json:"fileName"`
-	Level    string `json:"level"`
-}
-
-type loggerV6 struct {
-	Console consoleLoggerV1 `json:"console"`
-	File    fileLoggerV1    `json:"file"`
-	Syslog  syslogLoggerV3  `json:"syslog"`
-}
-
-// configV6 server configuration version '6'.
-type configV6 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-
-	// Additional error logging configuration.
-	Logger loggerV6 `json:"logger"`
-
-	// Notification queue configuration.
-	Notify notifierV1 `json:"notify"`
-}
-
-// Notifier represents collection of supported notification queues in version
-// 1 without NATS streaming.
-type notifierV1 struct {
-	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
-	NATS          map[string]natsNotifyV1             `json:"nats"`
-	ElasticSearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
-	Redis         map[string]target.RedisArgs         `json:"redis"`
-	PostgreSQL    map[string]target.PostgreSQLArgs    `json:"postgresql"`
-	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
-}
-
-// Notifier represents collection of supported notification queues in version 2
-// with NATS streaming but without webhook.
-type notifierV2 struct {
-	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
-	NATS          map[string]target.NATSArgs          `json:"nats"`
-	ElasticSearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
-	Redis         map[string]target.RedisArgs         `json:"redis"`
-	PostgreSQL    map[string]target.PostgreSQLArgs    `json:"postgresql"`
-	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
-}
-
-// configV7 server configuration version '7'.
-type serverConfigV7 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-
-	// Additional error logging configuration.
-	Logger loggerV6 `json:"logger"`
-
-	// Notification queue configuration.
-	Notify notifierV1 `json:"notify"`
-}
-
-// serverConfigV8 server configuration version '8'. Adds NATS notify.Config
-// configuration.
-type serverConfigV8 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-
-	// Additional error logging configuration.
-	Logger loggerV6 `json:"logger"`
-
-	// Notification queue configuration.
-	Notify notifierV1 `json:"notify"`
-}
-
-// serverConfigV9 server configuration version '9'. Adds PostgreSQL
-// notify.Config configuration.
-type serverConfigV9 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-
-	// Additional error logging configuration.
-	Logger loggerV6 `json:"logger"`
-
-	// Notification queue configuration.
-	Notify notifierV1 `json:"notify"`
-}
-
-type loggerV7 struct {
-	sync.RWMutex
-	Console consoleLoggerV1 `json:"console"`
-	File    fileLoggerV1    `json:"file"`
-}
-
-// serverConfigV10 server configuration version '10' which is like
-// version '9' except it drops support of syslog config, and makes the
-// RWMutex global (so it does not exist in this struct).
-type serverConfigV10 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-
-	// Additional error logging configuration.
-	Logger loggerV7 `json:"logger"`
-
-	// Notification queue configuration.
-	Notify notifierV1 `json:"notify"`
-}
-
-// natsNotifyV1 - structure was valid until config V 11
-type natsNotifyV1 struct {
-	Enable       bool   `json:"enable"`
-	Address      string `json:"address"`
-	Subject      string `json:"subject"`
-	Username     string `json:"username"`
-	Password     string `json:"password"`
-	Token        string `json:"token"`
-	Secure       bool   `json:"secure"`
-	PingInterval int64  `json:"pingInterval"`
-}
-
-// serverConfigV11 server configuration version '11' which is like
-// version '10' except it adds support for Kafka notifications.
-type serverConfigV11 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-
-	// Additional error logging configuration.
-	Logger loggerV7 `json:"logger"`
-
-	// Notification queue configuration.
-	Notify notifierV1 `json:"notify"`
-}
-
-// serverConfigV12 server configuration version '12' which is like
-// version '11' except it adds support for NATS streaming notifications.
-type serverConfigV12 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-
-	// Additional error logging configuration.
-	Logger loggerV7 `json:"logger"`
-
-	// Notification queue configuration.
-	Notify notifierV2 `json:"notify"`
-}
-
-type notifierV3 struct {
-	AMQP          map[string]target.AMQPArgs          `json:"amqp"`
-	Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"`
-	Kafka         map[string]target.KafkaArgs         `json:"kafka"`
-	MQTT          map[string]target.MQTTArgs          `json:"mqtt"`
-	MySQL         map[string]target.MySQLArgs         `json:"mysql"`
-	NATS          map[string]target.NATSArgs          `json:"nats"`
-	PostgreSQL    map[string]target.PostgreSQLArgs    `json:"postgresql"`
-	Redis         map[string]target.RedisArgs         `json:"redis"`
-	Webhook       map[string]target.WebhookArgs       `json:"webhook"`
-}
-
-// serverConfigV13 server configuration version '13' which is like
-// version '12' except it adds support for webhook notification.
-type serverConfigV13 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-
-	// Additional error logging configuration.
-	Logger *loggerV7 `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifierV3 `json:"notify"`
-}
-
-// serverConfigV14 server configuration version '14' which is like
-// version '13' except it adds support of browser param.
-type serverConfigV14 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-
-	// Additional error logging configuration.
-	Logger *loggerV7 `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifierV3 `json:"notify"`
-}
-
-// serverConfigV15 server configuration version '15' which is like
-// version '14' except it adds mysql support
-type serverConfigV15 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-
-	// Additional error logging configuration.
-	Logger *loggerV7 `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifierV3 `json:"notify"`
-}
-
 // FileLogger is introduced to workaround the dependency about logrus
 type FileLogger struct {
 	Enable   bool   `json:"enable"`
@@ -441,282 +41,6 @@ type ConsoleLogger struct {
 	Enable bool `json:"enable"`
 }

-// Loggers struct is defined with FileLogger and ConsoleLogger
-// although they are removed from logging logic. They are
-// kept here just to workaround the dependency migration
-// code/logic has on them.
-type loggers struct {
-	sync.RWMutex
-	Console ConsoleLogger `json:"console"`
-	File    FileLogger    `json:"file"`
-}
-
-// serverConfigV16 server configuration version '16' which is like
-// version '15' except it makes a change to logging configuration.
-type serverConfigV16 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-
-	// Additional error logging configuration.
-	Logger *loggers `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifierV3 `json:"notify"`
-}
-
-// serverConfigV17 server configuration version '17' which is like
-// version '16' except it adds support for "format" parameter in
-// database event notification targets: PostgreSQL, MySQL, Redis and
-// Elasticsearch.
-type serverConfigV17 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-
-	// Additional error logging configuration.
-	Logger *loggers `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifierV3 `json:"notify"`
-}
-
-// serverConfigV18 server configuration version '18' which is like
-// version '17' except it adds support for "deliveryMode" parameter in
-// the AMQP notification target.
-type serverConfigV18 struct {
-	sync.RWMutex
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-
-	// Additional error logging configuration.
-	Logger *loggers `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifierV3 `json:"notify"`
-}
-
-// serverConfigV19 server configuration version '19' which is like
-// version '18' except it adds support for MQTT notifications.
-type serverConfigV19 struct {
-	sync.RWMutex
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-
-	// Additional error logging configuration.
-	Logger *loggers `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifierV3 `json:"notify"`
-}
-
-// serverConfigV20 server configuration version '20' which is like
-// version '19' except it adds support for VirtualHostDomain
-type serverConfigV20 struct {
-	sync.RWMutex
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-	Domain     string           `json:"domain"`
-
-	// Additional error logging configuration.
-	Logger *loggers `json:"logger"`
-
-	// Notification queue configuration.
-	Notify *notifierV3 `json:"notify"`
-}
-
-// serverConfigV21 is just like version '20' without logger field
-type serverConfigV21 struct {
-	sync.RWMutex
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-	Domain     string           `json:"domain"`
-
-	// Notification queue configuration.
-	Notify *notifierV3 `json:"notify"`
-}
-
-// serverConfigV22 is just like version '21' with added support
-// for StorageClass.
-type serverConfigV22 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-	Domain     string           `json:"domain"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Notification queue configuration.
-	Notify notifierV3 `json:"notify"`
-}
-
-// serverConfigV23 is just like version '22' with addition of cache field.
-type serverConfigV23 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-	Domain     string           `json:"domain"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notifierV3 `json:"notify"`
-}
-
-// serverConfigV24 is just like version '23', we had to revert
-// the changes which were made in 6fb06045028b7a57c37c60a612c8e50735279ab4
-type serverConfigV24 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-	Domain     string           `json:"domain"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notifierV3 `json:"notify"`
-}
-
-// serverConfigV25 is just like version '24', stores additionally
-// worm variable.
-type serverConfigV25 struct {
-	quick.Config `json:"-"` // ignore interfaces
-
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-	Worm       config.BoolFlag  `json:"worm"`
-	Domain     string           `json:"domain"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notifierV3 `json:"notify"`
-}
-
-// serverConfigV26 is just like version '25', stores additionally
-// cache max use value in 'cache.Config'.
-type serverConfigV26 struct {
-	quick.Config `json:"-"` // ignore interfaces
-
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-	Worm       config.BoolFlag  `json:"worm"`
-	Domain     string           `json:"domain"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notifierV3 `json:"notify"`
-}
-
-// serverConfigV27 is just like version '26', stores additionally
-// the logger field
-type serverConfigV27 struct {
-	quick.Config `json:"-"` // ignore interfaces
-
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Browser    config.BoolFlag  `json:"browser"`
-	Worm       config.BoolFlag  `json:"worm"`
-	Domain     string           `json:"domain"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notifierV3 `json:"notify"`
-
-	// Logger configuration
-	Logger logger.Config `json:"logger"`
-}
-
-// serverConfigV28 is just like version '27', additionally
-// storing KMS config
-type serverConfigV28 struct {
-	quick.Config `json:"-"` // ignore interfaces
-
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Worm       config.BoolFlag  `json:"worm"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notifierV3 `json:"notify"`
-
-	// Logger configuration
-	Logger logger.Config `json:"logger"`
-}
-
 // serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in all notification targets.
 type serverConfigV33 struct {
 	quick.Config `json:"-"` // ignore interfaces
@@ -731,9 +55,6 @@ type serverConfigV33 struct {
 	// Storage class configuration
 	StorageClass storageclass.Config `json:"storageclass"`

-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
 	// Notification queue configuration.
 	Notify notify.Config `json:"notify"`

@@ -66,10 +66,12 @@ const (
 	scannerMetricYield
 	scannerMetricCleanAbandoned
 	scannerMetricApplyNonCurrent
+	scannerMetricHealAbandonedVersion

 	// START Trace metrics:
 	scannerMetricStartTrace
 	scannerMetricScanObject // Scan object. All operations included.
+	scannerMetricHealAbandonedObject

 	// END realtime metrics:
 	scannerMetricLastRealtime
@@ -54,11 +54,10 @@ const (
 	dataScannerForceCompactAtFolders = 1_000_000                        // Compact when this many subfolders in a single folder (even top level).
 	dataScannerStartDelay            = 1 * time.Minute                  // Time to wait on startup and between cycles.

-	healDeleteDangling    = true
-	healFolderIncludeProb = 32  // Include a clean folder one in n cycles.
-	healObjectSelectProb  = 512 // Overall probability of a file being scanned; one in n.
+	healDeleteDangling   = true
+	healObjectSelectProb = 1024 // Overall probability of a file being scanned; one in n.

-	dataScannerExcessiveVersionsThreshold = 1000  // Issue a warning when a single object has more versions than this
+	dataScannerExcessiveVersionsThreshold = 100   // Issue a warning when a single object has more versions than this
 	dataScannerExcessiveFoldersThreshold  = 50000 // Issue a warning when a folder has more subfolders than this in a *set*
 )

@@ -66,7 +65,7 @@ var (
 	globalHealConfig heal.Config

 	// Sleeper values are updated when config is loaded.
-	scannerSleeper = newDynamicSleeper(10, 10*time.Second, true)
+	scannerSleeper = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults
 	scannerCycle   = uatomic.NewDuration(dataScannerStartDelay)
 )

@@ -274,7 +273,6 @@ type folderScanner struct {
 // rarer if the bloom filter for the path is clean and no lifecycles are applied.
 // Skipped leaves have their totals transferred from the previous cycle.
 //
-// A clean leaf will be included once every healFolderIncludeProb for partial heal scans.
 // When selected there is a one in healObjectSelectProb that any object will be chosen for heal scan.
 //
 // Compaction happens when either:
@@ -345,7 +343,6 @@ func scanDataFolder(ctx context.Context, disks []StorageAPI, basePath string, ca
 		// No useful information...
 		return cache, err
 	}
-
 	s.newCache.Info.LastUpdate = UTCNow()
 	s.newCache.Info.NextCycle = cache.Info.NextCycle
 	return s.newCache, nil
@@ -409,9 +406,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int

 		var existingFolders, newFolders []cachedFolder
 		var foundObjects bool
-		err := readDirFn(path.Join(f.root, folder.name), func(entName string, typ os.FileMode) error {
+		err := readDirFn(pathJoin(f.root, folder.name), func(entName string, typ os.FileMode) error {
 			// Parse
-			entName = pathClean(path.Join(folder.name, entName))
+			entName = pathClean(pathJoin(folder.name, entName))
 			if entName == "" || entName == folder.name {
 				if f.dataUsageScannerDebug {
 					console.Debugf(scannerLogPrefix+" no entity (%s,%s)\n", f.root, entName)
@@ -461,7 +458,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int

 			// Get file size, ignore errors.
 			item := scannerItem{
-				Path:        path.Join(f.root, entName),
+				Path:        pathJoin(f.root, entName),
 				Typ:         typ,
 				bucket:      bucket,
 				prefix:      path.Dir(prefix),
@@ -496,7 +493,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			// Object already accounted for, remove from heal map,
 			// simply because getSize() function already heals the
 			// object.
-			delete(abandonedChildren, path.Join(item.bucket, item.objectPath()))
+			delete(abandonedChildren, pathJoin(item.bucket, item.objectPath()))

 			into.addSizes(sz)
 			into.Objects++
@@ -691,6 +688,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 				reportNotFound: true,
 				minDisks:       f.disksQuorum,
 				agreed: func(entry metaCacheEntry) {
+					f.updateCurrentPath(entry.name)
 					if f.dataUsageScannerDebug {
 						console.Debugf(healObjectsPrefix+" got agreement: %v\n", entry.name)
 					}
@@ -704,6 +702,13 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						// this object might be dangling.
 						entry, _ = entries.firstFound()
 					}
+					// wait timer per object.
+					wait := scannerSleeper.Timer(ctx)
+					defer wait()
+					f.updateCurrentPath(entry.name)
+					stopFn := globalScannerMetrics.log(scannerMetricHealAbandonedObject, f.root, entry.name)
+					custom := make(map[string]string)
+					defer stopFn(custom)

 					if f.dataUsageScannerDebug {
 						console.Debugf(healObjectsPrefix+" resolved to: %v, dir: %v\n", entry.name, entry.isDir())
@@ -713,13 +718,9 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						return
 					}

-					// wait on timer per object.
-					wait := scannerSleeper.Timer(ctx)
-
 					// We got an entry which we should be able to heal.
 					fiv, err := entry.fileInfoVersions(bucket)
 					if err != nil {
-						wait()
 						err := bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
 							object:    entry.name,
@@ -732,21 +733,28 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 						return
 					}

+					custom["versions"] = fmt.Sprint(len(fiv.Versions))
+					var successVersions, failVersions int
 					for _, ver := range fiv.Versions {
-						// Sleep and reset.
-						wait()
-						wait = scannerSleeper.Timer(ctx)
-
+						stopFn := globalScannerMetrics.timeSize(scannerMetricHealAbandonedVersion)
 						err := bgSeq.queueHealTask(healSource{
 							bucket:    bucket,
 							object:    fiv.Name,
 							versionID: ver.VersionID,
 						}, madmin.HealItemObject)
+						stopFn(int(ver.Size))
 						if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) {
 							logger.LogIf(ctx, err)
 						}
+						if err == nil {
+							successVersions++
+						} else {
+							failVersions++
+						}
 						foundObjs = foundObjs || err == nil
 					}
+					custom["success_versions"] = fmt.Sprint(successVersions)
+					custom["failed_versions"] = fmt.Sprint(failVersions)
 				},
 				// Too many disks failed.
 				finished: func(errs []error) {
@@ -873,7 +881,7 @@ type getSizeFn func(item scannerItem) (sizeSummary, error)
 func (i *scannerItem) transformMetaDir() {
 	split := strings.Split(i.prefix, SlashSeparator)
 	if len(split) > 1 {
-		i.prefix = path.Join(split[:len(split)-1]...)
+		i.prefix = pathJoin(split[:len(split)-1]...)
 	} else {
 		i.prefix = ""
 	}
@@ -920,7 +928,8 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje

 	versionID := oi.VersionID
 	rCfg, _ := globalBucketObjectLockSys.Get(i.bucket)
-	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, oi)
+	replcfg, _ := getReplicationConfig(ctx, i.bucket)
+	lcEvt := evalActionFromLifecycle(ctx, *i.lifeCycle, rCfg, replcfg, oi)
 	if i.debug {
 		if versionID != "" {
 			console.Debugf(applyActionsLogPrefix+" lifecycle: %q (version-id=%s), Initial scan: %v\n", i.objectPath(), versionID, lcEvt.Action)
@@ -1061,17 +1070,6 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 // applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
 // after applying lifecycle checks configured.
 func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]ObjectInfo, error) {
-	if i.heal.enabled {
-		if healDeleteDangling {
-			done := globalScannerMetrics.time(scannerMetricCleanAbandoned)
-			err := o.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
-			done()
-			if err != nil {
-				logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err))
-			}
-		}
-	}
-
 	objInfos, err := i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
 	if err != nil {
 		return nil, err
@@ -1112,7 +1110,17 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 			done := globalScannerMetrics.time(scannerMetricHealCheck)
 			size = i.applyHealing(ctx, o, oi)
 			done()
+
+			if healDeleteDangling {
+				done := globalScannerMetrics.time(scannerMetricCleanAbandoned)
+				err := o.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling})
+				done()
+				if err != nil {
+					logger.LogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err))
+				}
+			}
 		}
+
 		// replicate only if lifecycle rules are not applied.
 		done := globalScannerMetrics.time(scannerMetricCheckReplication)
 		i.healReplication(ctx, o, oi.Clone(), sizeS)
@@ -1121,7 +1129,7 @@ func (i *scannerItem) applyActions(ctx context.Context, o ObjectLayer, oi Object
 	return size
 }

-func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, obj ObjectInfo) lifecycle.Event {
+func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event {
 	event := lc.Eval(obj.ToLifecycleOpts())
 	if serverDebugLog {
 		console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action)
@@ -1153,6 +1161,9 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr loc
 			}
 			return lifecycle.Event{Action: lifecycle.NoneAction}
 		}
+		if rcfg != nil && !obj.VersionPurgeStatus.Empty() && rcfg.HasActiveRules(obj.Name, true) {
+			return lifecycle.Event{Action: lifecycle.NoneAction}
+		}
 	}

 	return event
@@ -1247,7 +1258,7 @@ func applyLifecycleAction(event lifecycle.Event, src lcEventSrc, obj ObjectInfo)

 // objectPath returns the prefix and object name.
 func (i *scannerItem) objectPath() string {
-	return path.Join(i.prefix, i.objectName)
+	return pathJoin(i.prefix, i.objectName)
 }

 // healReplication will heal a scanned item that has failed replication.
@@ -38,6 +38,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
 	"github.com/tinylib/msgp/msgp"
+	"github.com/valyala/bytebufferpool"
 )

 //go:generate msgp -file $GOFILE -unexported
@@ -77,8 +78,8 @@ func newAllTierStats() *allTierStats {
 	}
 }

-func (ats *allTierStats) addSizes(sz sizeSummary) {
-	for tier, st := range sz.tiers {
+func (ats *allTierStats) addSizes(tiers map[string]tierStats) {
+	for tier, st := range tiers {
 		ats.Tiers[tier] = ats.Tiers[tier].add(st)
 	}
 }
@@ -94,18 +95,16 @@ func (ats *allTierStats) clone() *allTierStats {
 		return nil
 	}
 	dst := *ats
-	if dst.Tiers != nil {
-		dst.Tiers = make(map[string]tierStats, len(dst.Tiers))
-		for tier, st := range dst.Tiers {
-			dst.Tiers[tier] = st
-		}
+	dst.Tiers = make(map[string]tierStats, len(ats.Tiers))
+	for tier, st := range ats.Tiers {
+		dst.Tiers[tier] = st
 	}
 	return &dst
 }

-func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[string]madmin.TierStats {
+func (ats *allTierStats) populateStats(stats map[string]madmin.TierStats) {
 	if ats == nil {
-		return stats
+		return
 	}

 	// Update stats for tiers as they become available.
@@ -116,7 +115,7 @@ func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[strin
 			NumObjects:  st.NumObjects,
 		}
 	}
-	return stats
+	return
 }

 // tierStats holds per-tier stats of a remote tier.
@@ -127,10 +126,11 @@ type tierStats struct {
 }

 func (ts tierStats) add(u tierStats) tierStats {
-	ts.TotalSize += u.TotalSize
-	ts.NumVersions += u.NumVersions
-	ts.NumObjects += u.NumObjects
-	return ts
+	return tierStats{
+		TotalSize:   ts.TotalSize + u.TotalSize,
+		NumVersions: ts.NumVersions + u.NumVersions,
+		NumObjects:  ts.NumObjects + u.NumObjects,
+	}
 }

 //msgp:tuple replicationStatsV1
@@ -196,12 +196,11 @@ func (r *replicationAllStats) clone() *replicationAllStats {
 	dst := *r

 	// Copy individual targets.
-	if dst.Targets != nil {
-		dst.Targets = make(map[string]replicationStats, len(dst.Targets))
-		for k, v := range r.Targets {
-			dst.Targets[k] = v
-		}
+	dst.Targets = make(map[string]replicationStats, len(r.Targets))
+	for k, v := range r.Targets {
+		dst.Targets[k] = v
 	}
+
 	return &dst
 }

@@ -346,26 +345,24 @@ func (e *dataUsageEntry) addSizes(summary sizeSummary) {
 	e.ReplicationStats.ReplicaSize += uint64(summary.replicaSize)
 	e.ReplicationStats.ReplicaCount += uint64(summary.replicaCount)

-	if summary.replTargetStats != nil {
-		for arn, st := range summary.replTargetStats {
-			tgtStat, ok := e.ReplicationStats.Targets[arn]
-			if !ok {
-				tgtStat = replicationStats{}
-			}
-			tgtStat.PendingSize += uint64(st.pendingSize)
-			tgtStat.FailedSize += uint64(st.failedSize)
-			tgtStat.ReplicatedSize += uint64(st.replicatedSize)
-			tgtStat.ReplicatedCount += uint64(st.replicatedCount)
-			tgtStat.FailedCount += st.failedCount
-			tgtStat.PendingCount += st.pendingCount
-			e.ReplicationStats.Targets[arn] = tgtStat
+	for arn, st := range summary.replTargetStats {
+		tgtStat, ok := e.ReplicationStats.Targets[arn]
+		if !ok {
+			tgtStat = replicationStats{}
 		}
+		tgtStat.PendingSize += uint64(st.pendingSize)
+		tgtStat.FailedSize += uint64(st.failedSize)
+		tgtStat.ReplicatedSize += uint64(st.replicatedSize)
+		tgtStat.ReplicatedCount += uint64(st.replicatedCount)
+		tgtStat.FailedCount += st.failedCount
+		tgtStat.PendingCount += st.pendingCount
+		e.ReplicationStats.Targets[arn] = tgtStat
 	}
-	if summary.tiers != nil {
+	if len(summary.tiers) != 0 {
 		if e.AllTierStats == nil {
 			e.AllTierStats = newAllTierStats()
 		}
-		e.AllTierStats.addSizes(summary)
+		e.AllTierStats.addSizes(summary.tiers)
 	}
 }

@@ -404,7 +401,7 @@ func (e *dataUsageEntry) merge(other dataUsageEntry) {
 		e.ObjVersions[i] += v
 	}

-	if other.AllTierStats != nil {
+	if other.AllTierStats != nil && len(other.AllTierStats.Tiers) != 0 {
 		if e.AllTierStats == nil {
 			e.AllTierStats = newAllTierStats()
 		}
@@ -925,39 +922,55 @@ type objectIO interface {
 // load the cache content with name from minioMetaBackgroundOpsBucket.
 // Only backend errors are returned as errors.
 // The loader is optimistic and has no locking, but tries 5 times before giving up.
-// If the object is not found or unable to deserialize d is cleared and nil error is returned.
+// If the object is not found, a nil error with empty data usage cache is returned.
 func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error {
-	// Abandon if more than 5 minutes, so we don't hold up scanner.
-	ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
-	defer cancel()
+	// By defaut, empty data usage cache
+	*d = dataUsageCache{}
+
+	load := func(name string, timeout time.Duration) (bool, error) {
+		// Abandon if more than time.Minute, so we don't hold up scanner.
+		// drive timeout by default is 2 minutes, we do not need to wait longer.
+		ctx, cancel := context.WithTimeout(ctx, timeout)
+		defer cancel()

-	// Caches are read+written without locks,
-	retries := 0
-	for retries < 5 {
 		r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, ObjectOptions{NoLock: true})
 		if err != nil {
 			switch err.(type) {
 			case ObjectNotFound, BucketNotFound:
+				return false, nil
 			case InsufficientReadQuorum, StorageErr:
-				retries++
-				time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
-				continue
-			default:
-				return toObjectErr(err, dataUsageBucket, name)
+				return true, nil
 			}
-			*d = dataUsageCache{}
-			return nil
-		}
-		if err := d.deserialize(r); err != nil {
-			r.Close()
-			retries++
-			time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
-			continue
+			return false, err
 		}
+		err = d.deserialize(r)
 		r.Close()
-		return nil
+		return err != nil, nil
 	}
-	*d = dataUsageCache{}
+
+	// Caches are read+written without locks,
+	retries := 0
+	for retries < 5 {
+		retry, err := load(name, time.Minute)
+		if err != nil {
+			return toObjectErr(err, dataUsageBucket, name)
+		}
+		if !retry {
+			break
+		}
+		retry, err = load(name+".bkp", 30*time.Second)
+		if err == nil && !retry {
+			// Only return when we have valid data from the backup
+			break
+		}
+		retries++
+		time.Sleep(time.Duration(rand.Int63n(int64(time.Second))))
+	}
+
+	if retries == 5 {
+		logger.LogOnceIf(ctx, fmt.Errorf("maximum retry reached to load the data usage cache `%s`", name), "retry-loading-data-usage-cache")
+	}
+
 	return nil
 }

@@ -967,47 +980,47 @@ var maxConcurrentScannerSaves = make(chan struct{}, 4)
 // save the content of the cache to minioMetaBackgroundOpsBucket with the provided name.
 // Note that no locking is done when saving.
 func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error {
-	var r io.Reader
-	maxConcurrentScannerSaves <- struct{}{}
-	defer func() {
-		<-maxConcurrentScannerSaves
-	}()
-	// If big, do streaming...
-	size := int64(-1)
-	if len(d.Cache) > 10000 {
-		pr, pw := io.Pipe()
-		go func() {
-			pw.CloseWithError(d.serializeTo(pw))
-		}()
-		defer pr.Close()
-		r = pr
-	} else {
-		var buf bytes.Buffer
-		err := d.serializeTo(&buf)
-		if err != nil {
-			return err
-		}
-		r = &buf
-		size = int64(buf.Len())
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case maxConcurrentScannerSaves <- struct{}{}:
 	}

-	hr, err := hash.NewReader(r, size, "", "", size)
-	if err != nil {
+	buf := bytebufferpool.Get()
+	defer func() {
+		<-maxConcurrentScannerSaves
+		buf.Reset()
+		bytebufferpool.Put(buf)
+	}()
+
+	if err := d.serializeTo(buf); err != nil {
 		return err
 	}

-	// Abandon if more than 5 minutes, so we don't hold up scanner.
-	ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
-	defer cancel()
-	_, err = store.PutObject(ctx,
-		dataUsageBucket,
-		name,
-		NewPutObjReader(hr),
-		ObjectOptions{NoLock: true})
-	if isErrBucketNotFound(err) {
-		return nil
+	save := func(name string, timeout time.Duration) error {
+		hr, err := hash.NewReader(ctx, bytes.NewReader(buf.Bytes()), int64(buf.Len()), "", "", int64(buf.Len()))
+		if err != nil {
+			return err
+		}
+
+		// Abandon if more than a minute, so we don't hold up scanner.
+		ctx, cancel := context.WithTimeout(ctx, timeout)
+		defer cancel()
+
+		_, err = store.PutObject(ctx,
+			dataUsageBucket,
+			name,
+			NewPutObjReader(hr),
+			ObjectOptions{NoLock: true})
+		if isErrBucketNotFound(err) {
+			return nil
+		}
+		return err
 	}
-	return err
+	defer save(name+".bkp", 5*time.Second) // Keep a backup as well
+
+	// drive timeout by default is 2 minutes, we do not need to wait longer.
+	return save(name, time.Minute)
 }

 // dataUsageCacheVer indicates the cache version.
@@ -107,35 +107,22 @@ func (dui DataUsageInfo) tierStats() []madmin.TierInfo {
 		return nil
 	}

-	cfgs := globalTierConfigMgr.ListTiers()
-	if len(cfgs) == 0 {
+	if globalTierConfigMgr.Empty() {
 		return nil
 	}

-	ts := make(map[string]madmin.TierStats, len(cfgs)+1)
+	ts := make(map[string]madmin.TierStats)
+	dui.TierStats.populateStats(ts)
+
 	infos := make([]madmin.TierInfo, 0, len(ts))
-
-	// Add STANDARD (hot-tier)
-	ts[minioHotTier] = madmin.TierStats{}
-	infos = append(infos, madmin.TierInfo{
-		Name: minioHotTier,
-		Type: "internal",
-	})
-	// Add configured remote tiers
-	for _, cfg := range cfgs {
-		ts[cfg.Name] = madmin.TierStats{}
+	for tier, stats := range ts {
 		infos = append(infos, madmin.TierInfo{
-			Name: cfg.Name,
-			Type: cfg.Type.String(),
+			Name:  tier,
+			Type:  globalTierConfigMgr.TierType(tier),
+			Stats: stats,
 		})
 	}

-	ts = dui.TierStats.adminStats(ts)
-	for i := range infos {
-		info := infos[i]
-		infos[i].Stats = ts[info.Name]
-	}
-
 	sort.Slice(infos, func(i, j int) bool {
 		if infos[i].Type == "internal" {
 			return true
@@ -21,6 +21,7 @@ import (
 	"context"
 	"errors"
 	"strings"
+	"time"

 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio/internal/logger"
@@ -42,6 +43,7 @@ const (

 // storeDataUsageInBackend will store all objects sent on the gui channel until closed.
 func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan DataUsageInfo) {
+	attempts := 1
 	for dataUsageInfo := range dui {
 		json := jsoniter.ConfigCompatibleWithStandardLibrary
 		dataUsageJSON, err := json.Marshal(dataUsageInfo)
@@ -49,12 +51,19 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 			logger.LogIf(ctx, err)
 			continue
 		}
+		if attempts > 10 {
+			saveConfig(ctx, objAPI, dataUsageObjNamePath+".bkp", dataUsageJSON) // Save a backup every 10th update.
+			attempts = 1
+		}
 		if err = saveConfig(ctx, objAPI, dataUsageObjNamePath, dataUsageJSON); err != nil {
 			logger.LogIf(ctx, err)
 		}
+		attempts++
 	}
 }

+var prefixUsageCache timedValue
+
 // loadPrefixUsageFromBackend returns prefix usages found in passed buckets
 //
 //	e.g.:  /testbucket/prefix => 355601334
@@ -67,37 +76,57 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket

 	cache := dataUsageCache{}

-	m := make(map[string]uint64)
-	for _, pool := range z.serverPools {
-		for _, er := range pool.sets {
-			// Load bucket usage prefixes
-			if err := cache.load(ctx, er, bucket+slashSeparator+dataUsageCacheName); err == nil {
-				root := cache.find(bucket)
-				if root == nil {
-					// We dont have usage information for this bucket in this
-					// set, go to the next set
-					continue
-				}
+	prefixUsageCache.Once.Do(func() {
+		prefixUsageCache.TTL = 30 * time.Second

-				for id, usageInfo := range cache.flattenChildrens(*root) {
-					prefix := decodeDirObject(strings.TrimPrefix(id, bucket+slashSeparator))
-					// decodeDirObject to avoid any __XLDIR__ objects
-					m[prefix] += uint64(usageInfo.Size)
+		// No need to fail upon Update() error, fallback to old value.
+		prefixUsageCache.Relax = true
+		prefixUsageCache.Update = func() (interface{}, error) {
+			m := make(map[string]uint64)
+			for _, pool := range z.serverPools {
+				for _, er := range pool.sets {
+					// Load bucket usage prefixes
+					ctx, done := context.WithTimeout(context.Background(), 2*time.Second)
+					ok := cache.load(ctx, er, bucket+slashSeparator+dataUsageCacheName) == nil
+					done()
+					if ok {
+						root := cache.find(bucket)
+						if root == nil {
+							// We dont have usage information for this bucket in this
+							// set, go to the next set
+							continue
+						}
+
+						for id, usageInfo := range cache.flattenChildrens(*root) {
+							prefix := decodeDirObject(strings.TrimPrefix(id, bucket+slashSeparator))
+							// decodeDirObject to avoid any __XLDIR__ objects
+							m[prefix] += uint64(usageInfo.Size)
+						}
+					}
 				}
 			}
+			return m, nil
 		}
+	})
+
+	v, _ := prefixUsageCache.Get()
+	if v != nil {
+		return v.(map[string]uint64), nil
 	}

-	return m, nil
+	return map[string]uint64{}, nil
 }

 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) {
 	buf, err := readConfig(ctx, objAPI, dataUsageObjNamePath)
 	if err != nil {
-		if errors.Is(err, errConfigNotFound) {
-			return DataUsageInfo{}, nil
+		buf, err = readConfig(ctx, objAPI, dataUsageObjNamePath+".bkp")
+		if err != nil {
+			if errors.Is(err, errConfigNotFound) {
+				return DataUsageInfo{}, nil
+			}
+			return DataUsageInfo{}, toObjectErr(err, minioMetaBucket, dataUsageObjNamePath)
 		}
-		return DataUsageInfo{}, toObjectErr(err, minioMetaBucket, dataUsageObjNamePath)
 	}

 	var dataUsageInfo DataUsageInfo
@@ -1,60 +0,0 @@
-//go:build windows
-// +build windows
-
-/*
- * MinIO Object Storage (c) 2021 MinIO, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package cmd
-
-import (
-	"errors"
-	"os"
-
-	"github.com/djherbis/atime"
-	"golang.org/x/sys/windows/registry"
-)
-
-// Return error if Atime is disabled on the O/S
-func checkAtimeSupport(dir string) (err error) {
-	file, err := os.CreateTemp(dir, "prefix")
-	if err != nil {
-		return
-	}
-	defer os.Remove(file.Name())
-	defer file.Close()
-	finfo1, err := os.Stat(file.Name())
-	if err != nil {
-		return
-	}
-	atime.Get(finfo1)
-
-	k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SYSTEM\CurrentControlSet\Control\FileSystem`, registry.QUERY_VALUE)
-	if err != nil {
-		return
-	}
-	defer k.Close()
-
-	setting, _, err := k.GetIntegerValue("NtfsDisableLastAccessUpdate")
-	if err != nil {
-		return
-	}
-
-	lowSetting := setting & 0xFFFF
-	if lowSetting != uint64(0x0000) && lowSetting != uint64(0x0002) {
-		return errors.New("Atime not supported")
-	}
-	return
-}
@@ -1,88 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"sync/atomic"
-)
-
-// CacheDiskStats represents cache disk statistics
-// such as current disk usage and available.
-type CacheDiskStats struct {
-	// used cache size
-	UsageSize uint64
-	// total cache disk capacity
-	TotalCapacity uint64
-	// indicates if usage is high or low, if high value is '1', if low its '0'
-	UsageState int32
-	// indicates the current usage percentage of this cache disk
-	UsagePercent uint64
-	Dir          string
-}
-
-// GetUsageLevelString gets the string representation for the usage level.
-func (c *CacheDiskStats) GetUsageLevelString() (u string) {
-	if atomic.LoadInt32(&c.UsageState) == 0 {
-		return "low"
-	}
-	return "high"
-}
-
-// CacheStats - represents bytes served from cache,
-// cache hits and cache misses.
-type CacheStats struct {
-	BytesServed  uint64
-	Hits         uint64
-	Misses       uint64
-	GetDiskStats func() []CacheDiskStats
-}
-
-// Increase total bytes served from cache
-func (s *CacheStats) incBytesServed(n int64) {
-	atomic.AddUint64(&s.BytesServed, uint64(n))
-}
-
-// Increase cache hit by 1
-func (s *CacheStats) incHit() {
-	atomic.AddUint64(&s.Hits, 1)
-}
-
-// Increase cache miss by 1
-func (s *CacheStats) incMiss() {
-	atomic.AddUint64(&s.Misses, 1)
-}
-
-// Get total bytes served
-func (s *CacheStats) getBytesServed() uint64 {
-	return atomic.LoadUint64(&s.BytesServed)
-}
-
-// Get total cache hits
-func (s *CacheStats) getHits() uint64 {
-	return atomic.LoadUint64(&s.Hits)
-}
-
-// Get total cache misses
-func (s *CacheStats) getMisses() uint64 {
-	return atomic.LoadUint64(&s.Misses)
-}
-
-// Prepare new CacheStats structure
-func newCacheStats() *CacheStats {
-	return &CacheStats{}
-}
@@ -1,587 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"container/list"
-	"errors"
-	"fmt"
-	"io"
-	"math"
-	"os"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/minio/minio/internal/crypto"
-	"github.com/minio/minio/internal/etag"
-)
-
-// CacheStatusType - whether the request was served from cache.
-type CacheStatusType string
-
-const (
-	// CacheHit - whether object was served from cache.
-	CacheHit CacheStatusType = "HIT"
-
-	// CacheMiss - object served from backend.
-	CacheMiss CacheStatusType = "MISS"
-)
-
-func (c CacheStatusType) String() string {
-	if c != "" {
-		return string(c)
-	}
-	return string(CacheMiss)
-}
-
-type cacheControl struct {
-	expiry       time.Time
-	maxAge       int
-	sMaxAge      int
-	minFresh     int
-	maxStale     int
-	noStore      bool
-	onlyIfCached bool
-	noCache      bool
-}
-
-func (c *cacheControl) isStale(modTime time.Time) bool {
-	if c == nil {
-		return false
-	}
-	// response will never be stale if only-if-cached is set
-	if c.onlyIfCached {
-		return false
-	}
-	// Cache-Control value no-store indicates never cache
-	if c.noStore {
-		return true
-	}
-	// Cache-Control value no-cache indicates cache entry needs to be revalidated before
-	// serving from cache
-	if c.noCache {
-		return true
-	}
-	now := time.Now()
-
-	if c.sMaxAge > 0 && c.sMaxAge < int(now.Sub(modTime).Seconds()) {
-		return true
-	}
-	if c.maxAge > 0 && c.maxAge < int(now.Sub(modTime).Seconds()) {
-		return true
-	}
-
-	if !c.expiry.Equal(time.Time{}) && c.expiry.Before(time.Now().Add(time.Duration(c.maxStale))) {
-		return true
-	}
-
-	if c.minFresh > 0 && c.minFresh <= int(now.Sub(modTime).Seconds()) {
-		return true
-	}
-
-	return false
-}
-
-// returns struct with cache-control settings from user metadata.
-func cacheControlOpts(o ObjectInfo) *cacheControl {
-	c := cacheControl{}
-	m := o.UserDefined
-	if !o.Expires.Equal(timeSentinel) {
-		c.expiry = o.Expires
-	}
-
-	var headerVal string
-	for k, v := range m {
-		if strings.EqualFold(k, "cache-control") {
-			headerVal = v
-		}
-	}
-	if headerVal == "" {
-		return nil
-	}
-	headerVal = strings.ToLower(headerVal)
-	headerVal = strings.TrimSpace(headerVal)
-
-	vals := strings.Split(headerVal, ",")
-	for _, val := range vals {
-		val = strings.TrimSpace(val)
-
-		if val == "no-store" {
-			c.noStore = true
-			continue
-		}
-		if val == "only-if-cached" {
-			c.onlyIfCached = true
-			continue
-		}
-		if val == "no-cache" {
-			c.noCache = true
-			continue
-		}
-		p := strings.Split(val, "=")
-
-		if len(p) != 2 {
-			continue
-		}
-		if p[0] == "max-age" ||
-			p[0] == "s-maxage" ||
-			p[0] == "min-fresh" ||
-			p[0] == "max-stale" {
-			i, err := strconv.Atoi(p[1])
-			if err != nil {
-				return nil
-			}
-			if p[0] == "max-age" {
-				c.maxAge = i
-			}
-			if p[0] == "s-maxage" {
-				c.sMaxAge = i
-			}
-			if p[0] == "min-fresh" {
-				c.minFresh = i
-			}
-			if p[0] == "max-stale" {
-				c.maxStale = i
-			}
-		}
-	}
-	return &c
-}
-
-// backendDownError returns true if err is due to backend failure or faulty disk if in server mode
-func backendDownError(err error) bool {
-	_, backendDown := err.(BackendDown)
-	return backendDown || IsErr(err, baseErrs...)
-}
-
-// IsCacheable returns if the object should be saved in the cache.
-func (o ObjectInfo) IsCacheable() bool {
-	if globalCacheKMS != nil {
-		return true
-	}
-	_, ok := crypto.IsEncrypted(o.UserDefined)
-	return !ok
-}
-
-// reads file cached on disk from offset upto length
-func readCacheFileStream(filePath string, offset, length int64) (io.ReadCloser, error) {
-	if filePath == "" || offset < 0 {
-		return nil, errInvalidArgument
-	}
-	if err := checkPathLength(filePath); err != nil {
-		return nil, err
-	}
-
-	fr, err := os.Open(filePath)
-	if err != nil {
-		return nil, osErrToFileErr(err)
-	}
-	// Stat to get the size of the file at path.
-	st, err := fr.Stat()
-	if err != nil {
-		err = osErrToFileErr(err)
-		return nil, err
-	}
-
-	if err = os.Chtimes(filePath, time.Now(), st.ModTime()); err != nil {
-		return nil, err
-	}
-
-	// Verify if its not a regular file, since subsequent Seek is undefined.
-	if !st.Mode().IsRegular() {
-		return nil, errIsNotRegular
-	}
-
-	if err = os.Chtimes(filePath, time.Now(), st.ModTime()); err != nil {
-		return nil, err
-	}
-
-	// Seek to the requested offset.
-	if offset > 0 {
-		_, err = fr.Seek(offset, io.SeekStart)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return struct {
-		io.Reader
-		io.Closer
-	}{Reader: io.LimitReader(fr, length), Closer: fr}, nil
-}
-
-func isCacheEncrypted(meta map[string]string) bool {
-	_, ok := meta[SSECacheEncrypted]
-	return ok
-}
-
-// decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
-func decryptCacheObjectETag(info *ObjectInfo) error {
-	if info.IsDir {
-		return nil // Directories are never encrypted.
-	}
-
-	// Depending on the SSE type we handle ETags slightly
-	// differently. ETags encrypted with SSE-S3 must be
-	// decrypted first, since the client expects that
-	// a single-part SSE-S3 ETag is equal to the content MD5.
-	//
-	// For all other SSE types, the ETag is not the content MD5.
-	// Therefore, we don't decrypt but only format it.
-	switch kind, ok := crypto.IsEncrypted(info.UserDefined); {
-	case ok && kind == crypto.S3 && isCacheEncrypted(info.UserDefined):
-		ETag, err := etag.Parse(info.ETag)
-		if err != nil {
-			return err
-		}
-		if !ETag.IsEncrypted() {
-			info.ETag = ETag.Format().String()
-			return nil
-		}
-
-		key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, info.UserDefined, info.Bucket, info.Name)
-		if err != nil {
-			return err
-		}
-		ETag, err = etag.Decrypt(key[:], ETag)
-		if err != nil {
-			return err
-		}
-		info.ETag = ETag.Format().String()
-	case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(info.UserDefined):
-		ETag, err := etag.Parse(info.ETag)
-		if err != nil {
-			return err
-		}
-		info.ETag = ETag.Format().String()
-	}
-	return nil
-}
-
-// decryptCacheObjectETag tries to decrypt the ETag saved in encrypted format using the cache KMS
-func decryptCachePartETags(c *cacheMeta) ([]string, error) {
-	// Depending on the SSE type we handle ETags slightly
-	// differently. ETags encrypted with SSE-S3 must be
-	// decrypted first, since the client expects that
-	// a single-part SSE-S3 ETag is equal to the content MD5.
-	//
-	// For all other SSE types, the ETag is not the content MD5.
-	// Therefore, we don't decrypt but only format it.
-	switch kind, ok := crypto.IsEncrypted(c.Meta); {
-	case ok && kind == crypto.S3 && isCacheEncrypted(c.Meta):
-		key, err := crypto.S3.UnsealObjectKey(globalCacheKMS, c.Meta, c.Bucket, c.Object)
-		if err != nil {
-			return nil, err
-		}
-		etags := make([]string, 0, len(c.PartETags))
-		for i := range c.PartETags {
-			ETag, err := etag.Parse(c.PartETags[i])
-			if err != nil {
-				return nil, err
-			}
-			ETag, err = etag.Decrypt(key[:], ETag)
-			if err != nil {
-				return nil, err
-			}
-			etags = append(etags, ETag.Format().String())
-		}
-		return etags, nil
-	case ok && (kind == crypto.S3KMS || kind == crypto.SSEC) && isCacheEncrypted(c.Meta):
-		etags := make([]string, 0, len(c.PartETags))
-		for i := range c.PartETags {
-			ETag, err := etag.Parse(c.PartETags[i])
-			if err != nil {
-				return nil, err
-			}
-			etags = append(etags, ETag.Format().String())
-		}
-		return etags, nil
-	default:
-		return c.PartETags, nil
-	}
-}
-
-func isMetadataSame(m1, m2 map[string]string) bool {
-	if m1 == nil && m2 == nil {
-		return true
-	}
-	if (m1 == nil && m2 != nil) || (m2 == nil && m1 != nil) {
-		return false
-	}
-	if len(m1) != len(m2) {
-		return false
-	}
-	for k1, v1 := range m1 {
-		if v2, ok := m2[k1]; !ok || (v1 != v2) {
-			return false
-		}
-	}
-	return true
-}
-
-type fileScorer struct {
-	saveBytes uint64
-	now       int64
-	maxHits   int
-	// 1/size for consistent score.
-	sizeMult float64
-
-	// queue is a linked list of files we want to delete.
-	// The list is kept sorted according to score, highest at top, lowest at bottom.
-	queue       list.List
-	queuedBytes uint64
-	seenBytes   uint64
-}
-
-type queuedFile struct {
-	name      string
-	versionID string
-	size      uint64
-	score     float64
-}
-
-// newFileScorer allows to collect files to save a specific number of bytes.
-// Each file is assigned a score based on its age, size and number of hits.
-// A list of files is maintained
-func newFileScorer(saveBytes uint64, now int64, maxHits int) (*fileScorer, error) {
-	if saveBytes == 0 {
-		return nil, errors.New("newFileScorer: saveBytes = 0")
-	}
-	if now < 0 {
-		return nil, errors.New("newFileScorer: now < 0")
-	}
-	if maxHits <= 0 {
-		return nil, errors.New("newFileScorer: maxHits <= 0")
-	}
-	f := fileScorer{saveBytes: saveBytes, maxHits: maxHits, now: now, sizeMult: 1 / float64(saveBytes)}
-	f.queue.Init()
-	return &f, nil
-}
-
-func (f *fileScorer) addFile(name string, accTime time.Time, size int64, hits int) {
-	f.addFileWithObjInfo(ObjectInfo{
-		Name:    name,
-		AccTime: accTime,
-		Size:    size,
-	}, hits)
-}
-
-func (f *fileScorer) addFileWithObjInfo(objInfo ObjectInfo, hits int) {
-	// Calculate how much we want to delete this object.
-	file := queuedFile{
-		name:      objInfo.Name,
-		versionID: objInfo.VersionID,
-		size:      uint64(objInfo.Size),
-	}
-	f.seenBytes += uint64(objInfo.Size)
-
-	var score float64
-	if objInfo.ModTime.IsZero() {
-		// Mod time is not available with disk cache use atime.
-		score = float64(f.now - objInfo.AccTime.Unix())
-	} else {
-		// if not used mod time when mod time is available.
-		score = float64(f.now - objInfo.ModTime.Unix())
-	}
-
-	// Size as fraction of how much we want to save, 0->1.
-	szWeight := math.Max(0, (math.Min(1, float64(file.size)*f.sizeMult)))
-	// 0 at f.maxHits, 1 at 0.
-	hitsWeight := (1.0 - math.Max(0, math.Min(1.0, float64(hits)/float64(f.maxHits))))
-	file.score = score * (1 + 0.25*szWeight + 0.25*hitsWeight)
-	// If we still haven't saved enough, just add the file
-	if f.queuedBytes < f.saveBytes {
-		f.insertFile(file)
-		f.trimQueue()
-		return
-	}
-	// If we score less than the worst, don't insert.
-	worstE := f.queue.Back()
-	if worstE != nil && file.score < worstE.Value.(queuedFile).score {
-		return
-	}
-	f.insertFile(file)
-	f.trimQueue()
-}
-
-// adjustSaveBytes allows to adjust the number of bytes to save.
-// This can be used to adjust the count on the fly.
-// Returns true if there still is a need to delete files (n+saveBytes >0),
-// false if no more bytes needs to be saved.
-func (f *fileScorer) adjustSaveBytes(n int64) bool {
-	if f == nil {
-		return false
-	}
-	if int64(f.saveBytes)+n <= 0 {
-		f.saveBytes = 0
-		f.trimQueue()
-		return false
-	}
-	if n < 0 {
-		f.saveBytes -= ^uint64(n - 1)
-	} else {
-		f.saveBytes += uint64(n)
-	}
-	if f.saveBytes == 0 {
-		f.queue.Init()
-		f.saveBytes = 0
-		return false
-	}
-	if n < 0 {
-		f.trimQueue()
-	}
-	return true
-}
-
-// insertFile will insert a file into the list, sorted by its score.
-func (f *fileScorer) insertFile(file queuedFile) {
-	e := f.queue.Front()
-	for e != nil {
-		v := e.Value.(queuedFile)
-		if v.score < file.score {
-			break
-		}
-		e = e.Next()
-	}
-	f.queuedBytes += file.size
-	// We reached the end.
-	if e == nil {
-		f.queue.PushBack(file)
-		return
-	}
-	f.queue.InsertBefore(file, e)
-}
-
-// trimQueue will trim the back of queue and still keep below wantSave.
-func (f *fileScorer) trimQueue() {
-	for {
-		e := f.queue.Back()
-		if e == nil {
-			return
-		}
-		v := e.Value.(queuedFile)
-		if f.queuedBytes-v.size < f.saveBytes {
-			return
-		}
-		f.queue.Remove(e)
-		f.queuedBytes -= v.size
-	}
-}
-
-func (f *fileScorer) purgeFunc(p func(qfile queuedFile)) {
-	e := f.queue.Front()
-	for e != nil {
-		p(e.Value.(queuedFile))
-		e = e.Next()
-	}
-}
-
-// fileNames returns all queued file names.
-func (f *fileScorer) fileNames() []string {
-	res := make([]string, 0, f.queue.Len())
-	e := f.queue.Front()
-	for e != nil {
-		res = append(res, e.Value.(queuedFile).name)
-		e = e.Next()
-	}
-	return res
-}
-
-func (f *fileScorer) reset() {
-	f.queue.Init()
-	f.queuedBytes = 0
-}
-
-func (f *fileScorer) queueString() string {
-	var res strings.Builder
-	e := f.queue.Front()
-	i := 0
-	for e != nil {
-		v := e.Value.(queuedFile)
-		if i > 0 {
-			res.WriteByte('\n')
-		}
-		res.WriteString(fmt.Sprintf("%03d: %s (score: %.3f, bytes: %d)", i, v.name, v.score, v.size))
-		i++
-		e = e.Next()
-	}
-	return res.String()
-}
-
-// bytesToClear() returns the number of bytes to clear to reach low watermark
-// w.r.t quota given disk total and free space, quota in % allocated to cache
-// and low watermark % w.r.t allowed quota.
-// If the high watermark hasn't been reached 0 will be returned.
-func bytesToClear(total, free int64, quotaPct, lowWatermark, highWatermark uint64) uint64 {
-	used := total - free
-	quotaAllowed := total * (int64)(quotaPct) / 100
-	highWMUsage := total * (int64)(highWatermark*quotaPct) / (100 * 100)
-	if used < highWMUsage {
-		return 0
-	}
-	// Return bytes needed to reach low watermark.
-	lowWMUsage := total * (int64)(lowWatermark*quotaPct) / (100 * 100)
-	return (uint64)(math.Min(float64(quotaAllowed), math.Max(0.0, float64(used-lowWMUsage))))
-}
-
-type multiWriter struct {
-	backendWriter io.Writer
-	cacheWriter   *io.PipeWriter
-	pipeClosed    bool
-}
-
-// multiWriter writes to backend and cache - if cache write
-// fails close the pipe, but continue writing to the backend
-func (t *multiWriter) Write(p []byte) (n int, err error) {
-	n, err = t.backendWriter.Write(p)
-	if err == nil && n != len(p) {
-		err = io.ErrShortWrite
-		return
-	}
-	if err != nil {
-		if !t.pipeClosed {
-			t.cacheWriter.CloseWithError(err)
-		}
-		return
-	}
-
-	// ignore errors writing to cache
-	if !t.pipeClosed {
-		_, cerr := t.cacheWriter.Write(p)
-		if cerr != nil {
-			t.pipeClosed = true
-			t.cacheWriter.CloseWithError(cerr)
-		}
-	}
-	return len(p), nil
-}
-
-func cacheMultiWriter(w1 io.Writer, w2 *io.PipeWriter) io.Writer {
-	return &multiWriter{backendWriter: w1, cacheWriter: w2}
-}
-
-// writebackInProgress returns true if writeback commit is not complete
-func writebackInProgress(m map[string]string) bool {
-	if v, ok := m[writeBackStatusHeader]; ok {
-		switch cacheCommitStatus(v) {
-		case CommitPending, CommitFailed:
-			return true
-		}
-	}
-	return false
-}
@@ -1,177 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"net/http"
-	"reflect"
-	"testing"
-	"time"
-)
-
-func TestGetCacheControlOpts(t *testing.T) {
-	expiry, _ := time.Parse(http.TimeFormat, "Wed, 21 Oct 2015 07:28:00 GMT")
-
-	testCases := []struct {
-		cacheControlHeaderVal string
-		expiryHeaderVal       time.Time
-		expectedCacheControl  *cacheControl
-		expectedErr           bool
-	}{
-		{"", timeSentinel, nil, false},
-		{"max-age=2592000, public", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
-		{"max-age=2592000, no-store", timeSentinel, &cacheControl{maxAge: 2592000, sMaxAge: 0, noStore: true, minFresh: 0, expiry: time.Time{}}, false},
-		{"must-revalidate, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 0, minFresh: 0, expiry: time.Time{}}, false},
-		{"s-maxAge=2500, max-age=600", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, false},
-		{"s-maxAge=2500, max-age=600", expiry, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Date(2015, time.October, 21, 0o7, 28, 0o0, 0o0, time.UTC)}, false},
-		{"s-maxAge=2500, max-age=600s", timeSentinel, &cacheControl{maxAge: 600, sMaxAge: 2500, minFresh: 0, expiry: time.Time{}}, true},
-	}
-
-	for _, testCase := range testCases {
-		t.Run("", func(t *testing.T) {
-			m := make(map[string]string)
-			m["cache-control"] = testCase.cacheControlHeaderVal
-			if !testCase.expiryHeaderVal.Equal(timeSentinel) {
-				m["expires"] = testCase.expiryHeaderVal.String()
-			}
-			c := cacheControlOpts(ObjectInfo{UserDefined: m, Expires: testCase.expiryHeaderVal})
-			if testCase.expectedErr && (c != nil) {
-				t.Errorf("expected err, got <nil>")
-			}
-			if !testCase.expectedErr && !reflect.DeepEqual(c, testCase.expectedCacheControl) {
-				t.Errorf("expected %v, got %v", testCase.expectedCacheControl, c)
-			}
-		})
-	}
-}
-
-func TestIsMetadataSame(t *testing.T) {
-	testCases := []struct {
-		m1       map[string]string
-		m2       map[string]string
-		expected bool
-	}{
-		{nil, nil, true},
-		{nil, map[string]string{}, false},
-		{map[string]string{"k": "v"}, map[string]string{"k": "v"}, true},
-		{map[string]string{"k": "v"}, map[string]string{"a": "b"}, false},
-		{map[string]string{"k1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v1"}, false},
-		{map[string]string{"k1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v2"}, true},
-		{map[string]string{"K1": "v1", "k2": "v2"}, map[string]string{"k1": "v1", "k2": "v2"}, false},
-		{map[string]string{"k1": "v1", "k2": "v2", "k3": "v3"}, map[string]string{"k1": "v1", "k2": "v2"}, false},
-	}
-
-	for i, testCase := range testCases {
-		actual := isMetadataSame(testCase.m1, testCase.m2)
-		if testCase.expected != actual {
-			t.Errorf("test %d expected %v, got %v", i, testCase.expected, actual)
-		}
-	}
-}
-
-func TestNewFileScorer(t *testing.T) {
-	fs, err := newFileScorer(1000, time.Now().Unix(), 10)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if len(fs.fileNames()) != 0 {
-		t.Fatal("non zero files??")
-	}
-	now := time.Now()
-	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
-	fs.addFile("older", now.Add(-time.Hour), 1000, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"older"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-	fs.reset()
-	fs.addFile("bigger", now.Add(-time.Minute), 2000, 10)
-	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"bigger"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-	fs.reset()
-	fs.addFile("less", now.Add(-time.Minute), 1000, 5)
-	fs.addFile("recent", now.Add(-time.Minute), 1000, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-	fs.reset()
-	fs.addFile("small", now.Add(-time.Minute), 200, 10)
-	fs.addFile("medium", now.Add(-time.Minute), 300, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"medium", "small"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-	fs.addFile("large", now.Add(-time.Minute), 700, 10)
-	fs.addFile("xsmol", now.Add(-time.Minute), 7, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"large", "medium"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-
-	fs.reset()
-	fs.addFile("less", now.Add(-time.Minute), 500, 5)
-	fs.addFile("recent", now.Add(-time.Minute), 500, 10)
-	if !fs.adjustSaveBytes(-500) {
-		t.Fatal("we should still need more bytes, got false")
-	}
-	// We should only need 500 bytes now.
-	if !reflect.DeepEqual(fs.fileNames(), []string{"less"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-	if fs.adjustSaveBytes(-500) {
-		t.Fatal("we shouldn't need any more bytes, got true")
-	}
-	fs, err = newFileScorer(1000, time.Now().Unix(), 10)
-	if err != nil {
-		t.Fatal(err)
-	}
-	fs.addFile("bigger", now.Add(-time.Minute), 50, 10)
-	// sorting should be consistent after adjusting savebytes.
-	fs.adjustSaveBytes(-800)
-	fs.addFile("smaller", now.Add(-time.Minute), 40, 10)
-	if !reflect.DeepEqual(fs.fileNames(), []string{"bigger", "smaller"}) {
-		t.Fatal("unexpected file list", fs.queueString())
-	}
-}
-
-func TestBytesToClear(t *testing.T) {
-	testCases := []struct {
-		total         int64
-		free          int64
-		quotaPct      uint64
-		watermarkLow  uint64
-		watermarkHigh uint64
-		expected      uint64
-	}{
-		{total: 1000, free: 800, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 0},
-		{total: 1000, free: 200, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 400},
-		{total: 1000, free: 400, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 240},
-		{total: 1000, free: 600, quotaPct: 40, watermarkLow: 90, watermarkHigh: 90, expected: 40},
-		{total: 1000, free: 600, quotaPct: 40, watermarkLow: 70, watermarkHigh: 70, expected: 120},
-		{total: 1000, free: 1000, quotaPct: 90, watermarkLow: 70, watermarkHigh: 70, expected: 0},
-
-		// High not yet reached..
-		{total: 1000, free: 250, quotaPct: 100, watermarkLow: 50, watermarkHigh: 90, expected: 0},
-		{total: 1000, free: 250, quotaPct: 100, watermarkLow: 50, watermarkHigh: 90, expected: 0},
-	}
-	for i, tc := range testCases {
-		toClear := bytesToClear(tc.total, tc.free, tc.quotaPct, tc.watermarkLow, tc.watermarkHigh)
-		if tc.expected != toClear {
-			t.Errorf("test %d expected %v, got %v", i, tc.expected, toClear)
-		}
-	}
-}
@@ -1,74 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"testing"
-	"time"
-)
-
-// Tests ToObjectInfo function.
-func TestCacheMetadataObjInfo(t *testing.T) {
-	m := cacheMeta{Meta: nil}
-	objInfo := m.ToObjectInfo()
-	if objInfo.Size != 0 {
-		t.Fatal("Unexpected object info value for Size", objInfo.Size)
-	}
-	if !objInfo.ModTime.Equal(time.Time{}) {
-		t.Fatal("Unexpected object info value for ModTime ", objInfo.ModTime)
-	}
-	if objInfo.IsDir {
-		t.Fatal("Unexpected object info value for IsDir", objInfo.IsDir)
-	}
-	if !objInfo.Expires.IsZero() {
-		t.Fatal("Unexpected object info value for Expires ", objInfo.Expires)
-	}
-}
-
-// test wildcard patterns for excluding entries from cache
-func TestCacheExclusion(t *testing.T) {
-	cobjects := &cacheObjects{
-		cache: nil,
-	}
-
-	testCases := []struct {
-		bucketName     string
-		objectName     string
-		excludePattern string
-		expectedResult bool
-	}{
-		{"testbucket", "testobjectmatch", "testbucket/testobj*", true},
-		{"testbucket", "testobjectnomatch", "testbucet/testobject*", false},
-		{"testbucket", "testobject/pref1/obj1", "*/*", true},
-		{"testbucket", "testobject/pref1/obj1", "*/pref1/*", true},
-		{"testbucket", "testobject/pref1/obj1", "testobject/*", false},
-		{"photos", "image1.jpg", "*.jpg", true},
-		{"photos", "europe/paris/seine.jpg", "seine.jpg", false},
-		{"photos", "europe/paris/seine.jpg", "*/seine.jpg", true},
-		{"phil", "z/likes/coffee", "*/likes/*", true},
-		{"failbucket", "no/slash/prefixes", "/failbucket/no/", false},
-		{"failbucket", "no/slash/prefixes", "/failbucket/no/*", false},
-	}
-
-	for i, testCase := range testCases {
-		cobjects.exclude = []string{testCase.excludePattern}
-		if cobjects.isCacheExclude(testCase.bucketName, testCase.objectName) != testCase.expectedResult {
-			t.Fatal("Cache exclusion test failed for case ", i)
-		}
-	}
-}
@@ -478,10 +478,7 @@ func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, m
 func decryptObjectMeta(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) {
 	switch kind, _ := crypto.IsEncrypted(metadata); kind {
 	case crypto.S3:
-		var KMS kms.KMS = GlobalKMS
-		if isCacheEncrypted(metadata) {
-			KMS = globalCacheKMS
-		}
+		KMS := GlobalKMS
 		if KMS == nil {
 			return nil, errKMSNotConfigured
 		}
@@ -728,7 +725,7 @@ func (d *DecryptBlocksReader) Read(p []byte) (int, error) {
 // DecryptedSize returns the size of the object after decryption in bytes.
 // It returns an error if the object is not encrypted or marked as encrypted
 // but has an invalid size.
-func (o *ObjectInfo) DecryptedSize() (int64, error) {
+func (o ObjectInfo) DecryptedSize() (int64, error) {
 	if _, ok := crypto.IsEncrypted(o.UserDefined); !ok {
 		return 0, errors.New("Cannot compute decrypted size of an unencrypted object")
 	}
@@ -915,33 +915,35 @@ func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints
 		}

 		orchestrated := IsKubernetes() || IsDocker()
-		if !orchestrated {
+		reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != ""))
+		// If not orchestrated
+		if !orchestrated &&
+			// and not setup in reverse proxy
+			!reverseProxy {
 			// Check whether same path is not used in endpoints of a host on different port.
 			// Only verify this on baremetal setups, DNS is not available in orchestrated
 			// environments so we can't do much here.
-			{
-				pathIPMap := make(map[string]set.StringSet)
-				hostIPCache := make(map[string]set.StringSet)
-				for _, endpoint := range endpoints {
-					host := endpoint.Hostname()
-					hostIPSet, ok := hostIPCache[host]
-					if !ok {
-						var err error
-						hostIPSet, err = getHostIP(host)
-						if err != nil {
-							return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
-						}
-						hostIPCache[host] = hostIPSet
+			pathIPMap := make(map[string]set.StringSet)
+			hostIPCache := make(map[string]set.StringSet)
+			for _, endpoint := range endpoints {
+				host := endpoint.Hostname()
+				hostIPSet, ok := hostIPCache[host]
+				if !ok {
+					var err error
+					hostIPSet, err = getHostIP(host)
+					if err != nil {
+						return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
 					}
-					if IPSet, ok := pathIPMap[endpoint.Path]; ok {
-						if !IPSet.Intersection(hostIPSet).IsEmpty() {
-							return nil, setupType,
-								config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
-						}
-						pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
-					} else {
-						pathIPMap[endpoint.Path] = hostIPSet
+					hostIPCache[host] = hostIPSet
+				}
+				if IPSet, ok := pathIPMap[endpoint.Path]; ok {
+					if !IPSet.Intersection(hostIPSet).IsEmpty() {
+						return nil, setupType,
+							config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
 					}
+					pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
+				} else {
+					pathIPMap[endpoint.Path] = hostIPSet
 				}
 			}
 		}
@@ -1113,32 +1115,34 @@ func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType,
 	}

 	orchestrated := IsKubernetes() || IsDocker()
-	if !orchestrated {
+	reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != ""))
+	// If not orchestrated
+	if !orchestrated &&
+		// and not setup in reverse proxy
+		!reverseProxy {
 		// Check whether same path is not used in endpoints of a host on different port.
 		// Only verify this on baremetal setups, DNS is not available in orchestrated
 		// environments so we can't do much here.
-		{
-			pathIPMap := make(map[string]set.StringSet)
-			hostIPCache := make(map[string]set.StringSet)
-			for _, endpoint := range endpoints {
-				host := endpoint.Hostname()
-				hostIPSet, ok := hostIPCache[host]
-				if !ok {
-					hostIPSet, err = getHostIP(host)
-					if err != nil {
-						return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
-					}
-					hostIPCache[host] = hostIPSet
+		pathIPMap := make(map[string]set.StringSet)
+		hostIPCache := make(map[string]set.StringSet)
+		for _, endpoint := range endpoints {
+			host := endpoint.Hostname()
+			hostIPSet, ok := hostIPCache[host]
+			if !ok {
+				hostIPSet, err = getHostIP(host)
+				if err != nil {
+					return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
 				}
-				if IPSet, ok := pathIPMap[endpoint.Path]; ok {
-					if !IPSet.Intersection(hostIPSet).IsEmpty() {
-						return endpoints, setupType,
-							config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
-					}
-					pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
-				} else {
-					pathIPMap[endpoint.Path] = hostIPSet
+				hostIPCache[host] = hostIPSet
+			}
+			if IPSet, ok := pathIPMap[endpoint.Path]; ok {
+				if !IPSet.Intersection(hostIPSet).IsEmpty() {
+					return endpoints, setupType,
+						config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
 				}
+				pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
+			} else {
+				pathIPMap[endpoint.Path] = hostIPSet
 			}
 		}
 	}
@@ -62,7 +62,22 @@ func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
 	return newDisks
 }

-func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
+func (er erasureObjects) getOnlineLocalDisks() (newDisks []StorageAPI) {
+	disks := er.getOnlineDisks()
+
+	// Based on the random shuffling return back randomized disks.
+	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+
+	for _, i := range r.Perm(len(disks)) {
+		if disks[i] != nil && disks[i].IsLocal() {
+			newDisks = append(newDisks, disks[i])
+		}
+	}
+
+	return newDisks
+}
+
+func (er erasureObjects) getLocalDisks() (newDisks []StorageAPI) {
 	disks := er.getDisks()
 	// Based on the random shuffling return back randomized disks.
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
@@ -74,62 +89,6 @@ func (er erasureObjects) getLoadBalancedLocalDisks() (newDisks []StorageAPI) {
 	return newDisks
 }

-// getLoadBalancedDisks - fetches load balanced (sufficiently randomized) disk slice.
-// ensures to skip disks if they are not healing and online.
-func (er erasureObjects) getLoadBalancedDisks(optimized bool) []StorageAPI {
-	disks := er.getDisks()
-
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
-	if !optimized {
-		var newDisks []StorageAPI
-		for _, i := range r.Perm(len(disks)) {
-			newDisks = append(newDisks, disks[i])
-		}
-		return newDisks
-	}
-
-	var wg sync.WaitGroup
-	var mu sync.Mutex
-	newDisks := map[uint64][]StorageAPI{}
-	// Based on the random shuffling return back randomized disks.
-	for _, i := range r.Perm(len(disks)) {
-		i := i
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			if disks[i] == nil {
-				return
-			}
-			di, err := disks[i].DiskInfo(context.Background(), false)
-			if err != nil || di.Healing {
-				// - Do not consume disks which are not reachable
-				//   unformatted or simply not accessible for some reason.
-				//
-				// - Do not consume disks which are being healed
-				//
-				// - Future: skip busy disks
-				return
-			}
-
-			mu.Lock()
-			// Capture disks usage wise upto resolution of MiB
-			newDisks[di.Used/1024/1024] = append(newDisks[di.Used/1024/1024], disks[i])
-			mu.Unlock()
-		}()
-	}
-	wg.Wait()
-
-	var max uint64
-	for k := range newDisks {
-		if k > max {
-			max = k
-		}
-	}
-
-	// Return disks which have maximum disk usage common.
-	return newDisks[max]
-}
-
 // readMultipleFiles Reads raw data from all specified files from all disks.
 func readMultipleFiles(ctx context.Context, disks []StorageAPI, req ReadMultipleReq, readQuorum int) ([]ReadMultipleResp, error) {
 	resps := make([]chan ReadMultipleResp, len(disks))
@@ -563,11 +563,26 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object
 	}

 	if !latestMeta.XLV1 && !latestMeta.Deleted && !recreate && disksToHealCount > latestMeta.Erasure.ParityBlocks {
-		// When disk to heal count is greater than parity blocks we should simply error out.
-		err := fmt.Errorf("(%d > %d) more drives are expected to heal than parity, returned errors: %v (dataErrs %v) -> %s/%s(%s)", disksToHealCount, latestMeta.Erasure.ParityBlocks, errs, dataErrs, bucket, object, versionID)
-		logger.LogOnceIf(ctx, err, "heal-object-count-gt-parity")
-		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
-			bucket, object, versionID), err
+		// Allow for dangling deletes, on versions that have DataDir missing etc.
+		// this would end up restoring the correct readable versions.
+		m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrs, ObjectOptions{
+			VersionID: versionID,
+		})
+		errs = make([]error, len(errs))
+		for i := range errs {
+			errs[i] = err
+		}
+		if err == nil {
+			// Dangling object successfully purged, size is '0'
+			m.Size = 0
+		}
+		// Generate file/version not found with default heal result
+		err = errFileNotFound
+		if versionID != "" {
+			err = errFileVersionNotFound
+		}
+		return er.defaultHealResult(m, storageDisks, storageEndpoints,
+			errs, bucket, object, versionID), err
 	}

 	cleanFileInfo := func(fi FileInfo) FileInfo {
@@ -1077,6 +1092,22 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	}

 	if !validMeta.IsValid() {
+		// validMeta is invalid because notFoundPartsErrs is
+		// greater than parity blocks, thus invalidating the FileInfo{}
+		// every dataErrs[i], metaArr[i] is an empty FileInfo{}
+		dataBlocks := (len(ndataErrs) + 1) / 2
+		if notFoundPartsErrs > dataBlocks {
+			// Not using parity to ensure that we do not delete
+			// any valid content, if any is recoverable. But if
+			// notFoundDataDirs are already greater than the data
+			// blocks all bets are off and it is safe to purge.
+			//
+			// This is purely a defensive code, ideally parityBlocks
+			// is sufficient, however we can't know that since we
+			// do have the FileInfo{}.
+			return validMeta, true
+		}
+
 		// We have no idea what this file is, leave it as is.
 		return validMeta, false
 	}
@@ -1111,7 +1142,7 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	if reqInfo != nil {
 		newReqInfo = logger.NewReqInfo(reqInfo.RemoteHost, reqInfo.UserAgent, reqInfo.DeploymentID, reqInfo.RequestID, reqInfo.API, bucket, object)
 	} else {
-		newReqInfo = logger.NewReqInfo("", "", globalDeploymentID, "", "Heal", bucket, object)
+		newReqInfo = logger.NewReqInfo("", "", globalDeploymentID(), "", "Heal", bucket, object)
 	}
 	healCtx := logger.SetReqInfo(GlobalContext, newReqInfo)

@@ -180,6 +180,11 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 	objInfo.ReplicationStatusInternal = fi.ReplicationState.ReplicationStatusInternal
 	objInfo.VersionPurgeStatusInternal = fi.ReplicationState.VersionPurgeStatusInternal
 	objInfo.ReplicationStatus = fi.ReplicationStatus()
+	if objInfo.ReplicationStatus.Empty() { // overlay x-amx-replication-status if present for replicas
+		if st, ok := fi.Metadata[xhttp.AmzBucketReplicationStatus]; ok && st == string(replication.Replica) {
+			objInfo.ReplicationStatus = replication.StatusType(st)
+		}
+	}
 	objInfo.VersionPurgeStatus = fi.VersionPurgeStatus()

 	objInfo.TransitionedObject = TransitionedObject{
@@ -40,7 +40,6 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/v2/mimedb"
 	"github.com/minio/pkg/v2/sync/errgroup"
-	uatomic "go.uber.org/atomic"
 )

 func (er erasureObjects) getUploadIDDir(bucket, object, uploadID string) string {
@@ -163,7 +162,7 @@ func (er erasureObjects) removeObjectPart(bucket, object, uploadID, dataDir stri
 func (er erasureObjects) cleanupStaleUploads(ctx context.Context, expiry time.Duration) {
 	// run multiple cleanup's local to this server.
 	var wg sync.WaitGroup
-	for _, disk := range er.getLoadBalancedLocalDisks() {
+	for _, disk := range er.getLocalDisks() {
 		if disk != nil {
 			wg.Add(1)
 			go func(disk StorageAPI) {
@@ -269,7 +268,19 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec

 	var uploadIDs []string
 	var disk StorageAPI
-	for _, disk = range er.getLoadBalancedDisks(true) {
+	disks := er.getOnlineLocalDisks()
+	if len(disks) == 0 {
+		// using er.getOnlineLocalDisks() has one side-affect where
+		// on a pooled setup all disks are remote, add a fallback
+		disks = er.getOnlineDisks()
+	}
+	for _, disk = range disks {
+		if disk == nil {
+			continue
+		}
+		if !disk.IsOnline() {
+			continue
+		}
 		uploadIDs, err = disk.ListDir(ctx, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1)
 		if err != nil {
 			if errors.Is(err, errDiskNotFound) {
@@ -298,20 +309,21 @@ func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, objec
 		if populatedUploadIds.Contains(uploadID) {
 			continue
 		}
-		fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, pathJoin(er.getUploadIDDir(bucket, object, uploadID)), "", false)
-		if err != nil {
-			if !IsErrIgnored(err, errFileNotFound, errDiskNotFound) {
-				logger.LogIf(ctx, err)
+		// If present, use time stored in ID.
+		startTime := time.Now()
+		if split := strings.Split(uploadID, "x"); len(split) == 2 {
+			t, err := strconv.ParseInt(split[1], 10, 64)
+			if err == nil {
+				startTime = time.Unix(0, t)
 			}
-			// Ignore this invalid upload-id since we are listing here
-			continue
 		}
-		populatedUploadIds.Add(uploadID)
 		uploads = append(uploads, MultipartInfo{
+			Bucket:    bucket,
 			Object:    object,
-			UploadID:  base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID, uploadID))),
-			Initiated: fi.ModTime,
+			UploadID:  base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID(), uploadID))),
+			Initiated: startTime,
 		})
+		populatedUploadIds.Add(uploadID)
 	}

 	sort.Slice(uploads, func(i int, j int) bool {
@@ -393,47 +405,31 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 	// If we have offline disks upgrade the number of erasure codes for this object.
 	parityOrig := parityDrives

-	atomicParityDrives := uatomic.NewInt64(0)
-	atomicOfflineDrives := uatomic.NewInt64(0)
-
-	// Start with current parityDrives
-	atomicParityDrives.Store(int64(parityDrives))
-
-	var wg sync.WaitGroup
+	var offlineDrives int
 	for _, disk := range onlineDisks {
 		if disk == nil {
-			atomicParityDrives.Inc()
-			atomicOfflineDrives.Inc()
+			parityDrives++
+			offlineDrives++
 			continue
 		}
 		if !disk.IsOnline() {
-			atomicParityDrives.Inc()
-			atomicOfflineDrives.Inc()
+			parityDrives++
+			offlineDrives++
 			continue
 		}
-		wg.Add(1)
-		go func(disk StorageAPI) {
-			defer wg.Done()
-			di, err := disk.DiskInfo(ctx, false)
-			if err != nil || di.ID == "" {
-				atomicOfflineDrives.Inc()
-				atomicParityDrives.Inc()
-			}
-		}(disk)
 	}
-	wg.Wait()

-	if int(atomicOfflineDrives.Load()) >= (len(onlineDisks)+1)/2 {
+	if offlineDrives >= (len(onlineDisks)+1)/2 {
 		// if offline drives are more than 50% of the drives
 		// we have no quorum, we shouldn't proceed just
 		// fail at that point.
 		return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
 	}

-	parityDrives = int(atomicParityDrives.Load())
 	if parityDrives >= len(onlineDisks)/2 {
 		parityDrives = len(onlineDisks) / 2
 	}
+
 	if parityOrig != parityDrives {
 		userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
 	}
@@ -486,7 +482,7 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		partsMetadata[index].Metadata = userDefined
 	}
 	uploadUUID := mustGetUUID()
-	uploadID := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID, uploadUUID)))
+	uploadID := base64.RawURLEncoding.EncodeToString([]byte(fmt.Sprintf("%s.%s", globalDeploymentID(), uploadUUID)))
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadUUID)

 	// Write updated `xl.meta` to all disks.
@@ -640,7 +636,8 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 	// accommodate concurrent PutObjectPart requests

 	partSuffix := fmt.Sprintf("part.%d", partID)
-	tmpPart := mustGetUUID()
+	// Random UUID and timestamp for temporary part file.
+	tmpPart := fmt.Sprintf("%sx%d", mustGetUUID(), time.Now().UnixNano())
 	tmpPartPath := pathJoin(tmpPart, partSuffix)

 	// Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete.
@@ -827,46 +824,8 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 		return result, toObjectErr(err, bucket, object, uploadID)
 	}

-	onlineDisks := er.getDisks()
 	uploadIDPath := er.getUploadIDDir(bucket, object, uploadID)

-	if maxParts == 0 {
-		return result, nil
-	}
-
-	if partNumberMarker < 0 {
-		partNumberMarker = 0
-	}
-
-	// Limit output to maxPartsList.
-	if maxParts > maxPartsList-partNumberMarker {
-		maxParts = maxPartsList - partNumberMarker
-	}
-
-	// Read Part info for all parts
-	partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
-	req := ReadMultipleReq{
-		Bucket:     minioMetaMultipartBucket,
-		Prefix:     partPath,
-		MaxSize:    1 << 20, // Each part should realistically not be > 1MiB.
-		MaxResults: maxParts + 1,
-	}
-
-	start := partNumberMarker + 1
-	end := start + maxParts
-
-	// Parts are 1 based, so index 0 is part one, etc.
-	for i := start; i <= end; i++ {
-		req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", i))
-	}
-
-	writeQuorum := fi.WriteQuorum(er.defaultWQuorum())
-
-	partInfoFiles, err := readMultipleFiles(ctx, onlineDisks, req, writeQuorum)
-	if err != nil {
-		return result, err
-	}
-
 	// Populate the result stub.
 	result.Bucket = bucket
 	result.Object = object
@@ -876,33 +835,96 @@ func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, up
 	result.UserDefined = cloneMSS(fi.Metadata)
 	result.ChecksumAlgorithm = fi.Metadata[hash.MinIOMultipartChecksum]

-	// For empty number of parts or maxParts as zero, return right here.
-	if len(partInfoFiles) == 0 || maxParts == 0 {
+	if partNumberMarker < 0 {
+		partNumberMarker = 0
+	}
+
+	// Limit output to maxPartsList.
+	if maxParts > maxPartsList-partNumberMarker {
+		maxParts = maxPartsList - partNumberMarker
+	}
+
+	if maxParts == 0 {
 		return result, nil
 	}

-	for i, part := range partInfoFiles {
-		partN := i + partNumberMarker + 1
-		if part.Error != "" || !part.Exists {
+	// Read Part info for all parts
+	partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
+	req := ReadMultipleReq{
+		Bucket:       minioMetaMultipartBucket,
+		Prefix:       partPath,
+		MaxSize:      1 << 20, // Each part should realistically not be > 1MiB.
+		MaxResults:   maxParts + 1,
+		MetadataOnly: true,
+	}
+
+	start := partNumberMarker + 1
+	end := start + maxParts
+
+	// Parts are 1 based, so index 0 is part one, etc.
+	for i := start; i <= end; i++ {
+		req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", i))
+	}
+
+	var disk StorageAPI
+	disks := er.getOnlineLocalDisks()
+	if len(disks) == 0 {
+		// using er.getOnlineLocalDisks() has one side-affect where
+		// on a pooled setup all disks are remote, add a fallback
+		disks = er.getOnlineDisks()
+	}
+
+	for _, disk = range disks {
+		if disk == nil {
 			continue
 		}

-		var pfi FileInfo
-		_, err := pfi.UnmarshalMsg(part.Data)
-		if err != nil {
-			// Maybe crash or similar.
-			logger.LogIf(ctx, err)
+		if !disk.IsOnline() {
 			continue
 		}

-		partI := pfi.Parts[0]
-		if partN != partI.Number {
-			logger.LogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", i+1, i+1, partI.Number))
-			continue
-		}
+		break
+	}

-		// Add the current part.
-		fi.AddObjectPart(partI.Number, partI.ETag, partI.Size, partI.ActualSize, partI.ModTime, partI.Index, partI.Checksums)
+	g := errgroup.WithNErrs(len(req.Files)).WithConcurrency(32)
+
+	partsInfo := make([]ObjectPartInfo, len(req.Files))
+	for i, file := range req.Files {
+		file := file
+		partN := i + start
+		i := i
+
+		g.Go(func() error {
+			buf, err := disk.ReadAll(ctx, minioMetaMultipartBucket, pathJoin(partPath, file))
+			if err != nil {
+				return err
+			}
+
+			var pfi FileInfo
+			_, err = pfi.UnmarshalMsg(buf)
+			if err != nil {
+				return err
+			}
+
+			if len(pfi.Parts) != 1 {
+				return errors.New("invalid number of parts expected 1, got 0")
+			}
+
+			if partN != pfi.Parts[0].Number {
+				return fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", partN, partN, pfi.Parts[0].Number)
+			}
+
+			partsInfo[i] = pfi.Parts[0]
+			return nil
+		}, i)
+	}
+
+	g.Wait()
+
+	for _, part := range partsInfo {
+		if part.Number != 0 && !part.ModTime.IsZero() {
+			fi.AddObjectPart(part.Number, part.ETag, part.Size, part.ActualSize, part.ModTime, part.Index, part.Checksums)
+		}
 	}

 	// Only parts with higher part numbers will be listed.
@@ -967,11 +989,12 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 	// Read Part info for all parts
 	partPath := pathJoin(uploadIDPath, fi.DataDir) + "/"
 	req := ReadMultipleReq{
-		Bucket:     minioMetaMultipartBucket,
-		Prefix:     partPath,
-		MaxSize:    1 << 20, // Each part should realistically not be > 1MiB.
-		Files:      make([]string, 0, len(parts)),
-		AbortOn404: true,
+		Bucket:       minioMetaMultipartBucket,
+		Prefix:       partPath,
+		MaxSize:      1 << 20, // Each part should realistically not be > 1MiB.
+		Files:        make([]string, 0, len(parts)),
+		AbortOn404:   true,
+		MetadataOnly: true,
 	}
 	for _, part := range parts {
 		req.Files = append(req.Files, fmt.Sprintf("part.%d.meta", part.PartNumber))
@@ -1174,6 +1197,16 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 			return oi, err
 		}
 	}
+
+	// Hold namespace to complete the transaction
+	lk := er.NewNSLock(bucket, object)
+	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+	if err != nil {
+		return oi, err
+	}
+	ctx = lkctx.Context()
+	defer lk.Unlock(lkctx)
+
 	if checksumType.IsSet() {
 		checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
 		cs := hash.NewChecksumFromData(checksumType, checksumCombined)
@@ -1217,15 +1250,6 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}

-	// Hold namespace to complete the transaction
-	lk := er.NewNSLock(bucket, object)
-	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return oi, err
-	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx)
-
 	// Remove parts that weren't present in CompleteMultipartUpload request.
 	for _, curpart := range currentFI.Parts {
 		// Remove part.meta which is not needed anymore.
@@ -1242,7 +1266,11 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}

-	defer er.deleteAll(context.Background(), minioMetaMultipartBucket, uploadIDPath)
+	defer func() {
+		if err == nil {
+			er.deleteAll(context.Background(), minioMetaMultipartBucket, uploadIDPath)
+		}
+	}()

 	// Rename the multipart object to final location.
 	onlineDisks, versionsDisparity, err := renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath,
@@ -46,7 +46,7 @@ import (
 	"github.com/minio/pkg/v2/mimedb"
 	"github.com/minio/pkg/v2/sync/errgroup"
 	"github.com/minio/pkg/v2/wildcard"
-	uatomic "go.uber.org/atomic"
+	"github.com/tinylib/msgp/msgp"
 )

 // list all errors which can be ignored in object operations.
@@ -132,15 +132,18 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 			versionID = mustGetUUID()
 			fi.IsLatest = true // we are creating a new version so this is latest.
 		}
-		modTime = UTCNow()
 	}

+	modTime = UTCNow() // We only preserve modTime if dstOpts.MTime is true.
+	// in all other cases mtime is latest.
+
 	fi.VersionID = versionID // set any new versionID we might have created
 	fi.ModTime = modTime     // set modTime for the new versionID
 	if !dstOpts.MTime.IsZero() {
 		modTime = dstOpts.MTime
 		fi.ModTime = dstOpts.MTime
 	}
+
 	fi.Metadata = srcInfo.UserDefined
 	srcInfo.UserDefined["etag"] = srcInfo.ETag

@@ -188,14 +191,6 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	auditObjectErasureSet(ctx, object, &er)

-	// This is a special call attempted first to check for SOS-API calls.
-	gr, err = veeamSOSAPIGetObject(ctx, bucket, object, rs, opts)
-	if err == nil {
-		return gr, nil
-	}
-	// reset any error to 'nil'
-	err = nil
-
 	var unlockOnDefer bool
 	nsUnlocker := func() {}
 	defer func() {
@@ -451,14 +446,6 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
 	auditObjectErasureSet(ctx, object, &er)

-	// This is a special call attempted first to check for SOS-API calls.
-	info, err = veeamSOSAPIHeadObject(ctx, bucket, object, opts)
-	if err == nil {
-		return info, nil
-	}
-	// reset any error to 'nil'
-	err = nil
-
 	if !opts.NoLock {
 		// Lock the object before reading.
 		lk := er.NewNSLock(bucket, object)
@@ -489,6 +476,24 @@ func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID
 	auditLogInternal(ctx, opts)
 }

+func joinErrors(errs ...error) error {
+	s := make([]string, 0, len(errs))
+	nonNilErrs := make([]any, 0, len(errs))
+	for _, err := range errs {
+		if err == nil {
+			continue
+		}
+		s = append(s, "[%w]")
+		nonNilErrs = append(nonNilErrs, err)
+	}
+	// If all the errors were nil, return nil.
+	if len(nonNilErrs) == 0 {
+		return nil
+	}
+	allErrs := strings.Join(s, "\n")
+	return fmt.Errorf(allErrs, nonNilErrs...)
+}
+
 func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrs []error, opts ObjectOptions) (FileInfo, error) {
 	_, file, line, cok := runtime.Caller(1)
 	var err error
@@ -497,7 +502,14 @@ func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object st
 		tags := make(map[string]interface{}, 4)
 		tags["set"] = er.setIndex
 		tags["pool"] = er.poolIndex
-		tags["parity"] = m.Erasure.ParityBlocks
+		tags["merrs"] = joinErrors(errs...)     // errors.Join(errs...)
+		tags["derrs"] = joinErrors(dataErrs...) // errors.Join(dataErrs...)
+		if m.IsValid() {
+			tags["size"] = m.Size
+			tags["mtime"] = m.ModTime.Format(http.TimeFormat)
+			tags["parity"] = m.Erasure.ParityBlocks
+		}
+
 		if cok {
 			tags["caller"] = fmt.Sprintf("%s:%d", file, line)
 		}
@@ -578,6 +590,9 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		errFileVersionNotFound,
 		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
 		io.EOF,              // some times we would read without locks, ignore these errors
+		msgp.ErrShortBytes,
+		context.DeadlineExceeded,
+		context.Canceled,
 	}
 	ignoredErrs = append(ignoredErrs, objectOpIgnoredErrs...)

@@ -1086,47 +1101,31 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		// If we have offline disks upgrade the number of erasure codes for this object.
 		parityOrig := parityDrives

-		atomicParityDrives := uatomic.NewInt64(0)
-		atomicOfflineDrives := uatomic.NewInt64(0)
-
-		// Start with current parityDrives
-		atomicParityDrives.Store(int64(parityDrives))
-
-		var wg sync.WaitGroup
+		var offlineDrives int
 		for _, disk := range storageDisks {
 			if disk == nil {
-				atomicParityDrives.Inc()
-				atomicOfflineDrives.Inc()
+				parityDrives++
+				offlineDrives++
 				continue
 			}
 			if !disk.IsOnline() {
-				atomicParityDrives.Inc()
-				atomicOfflineDrives.Inc()
+				parityDrives++
+				offlineDrives++
 				continue
 			}
-			wg.Add(1)
-			go func(disk StorageAPI) {
-				defer wg.Done()
-				di, err := disk.DiskInfo(ctx, false)
-				if err != nil || di.ID == "" {
-					atomicOfflineDrives.Inc()
-					atomicParityDrives.Inc()
-				}
-			}(disk)
 		}
-		wg.Wait()

-		if int(atomicOfflineDrives.Load()) >= (len(storageDisks)+1)/2 {
+		if offlineDrives >= (len(storageDisks)+1)/2 {
 			// if offline drives are more than 50% of the drives
 			// we have no quorum, we shouldn't proceed just
 			// fail at that point.
 			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
 		}

-		parityDrives = int(atomicParityDrives.Load())
 		if parityDrives >= len(storageDisks)/2 {
 			parityDrives = len(storageDisks) / 2
 		}
+
 		if parityOrig != parityDrives {
 			userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
 		}
@@ -1617,10 +1616,12 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string

 	var lc *lifecycle.Lifecycle
 	var rcfg lock.Retention
+	var replcfg *replication.Config
 	if opts.Expiration.Expire {
 		// Check if the current bucket has a configured lifecycle policy
 		lc, _ = globalLifecycleSys.Get(bucket)
 		rcfg, _ = globalBucketObjectLockSys.Get(bucket)
+		replcfg, _ = getReplicationConfig(ctx, bucket)
 	}

 	// expiration attempted on a bucket with no lifecycle
@@ -1673,7 +1674,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string

 	if opts.Expiration.Expire {
 		if gerr == nil {
-			evt := evalActionFromLifecycle(ctx, *lc, rcfg, goi)
+			evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi)
 			var isErr bool
 			switch evt.Action {
 			case lifecycle.NoneAction:
@@ -2151,7 +2152,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s
 			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
 		}
 		defer gr.Close()
-		hashReader, err := hash.NewReader(gr, gr.ObjInfo.Size, "", "", gr.ObjInfo.Size)
+		hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", gr.ObjInfo.Size)
 		if err != nil {
 			return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object))
 		}
@@ -2176,7 +2177,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s

 	// rehydrate the parts back on disk as per the original xl.meta prior to transition
 	for _, partInfo := range oi.Parts {
-		hr, err := hash.NewReader(io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size)
+		hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size)
 		if err != nil {
 			return setRestoreHeaderFn(oi, err)
 		}
@@ -68,6 +68,34 @@ type PoolDecommissionInfo struct {
 	BytesFailed             int64 `json:"bytesDecommissionedFailed" msg:"bf"`
 }

+// Clone make a copy of PoolDecommissionInfo
+func (pd *PoolDecommissionInfo) Clone() *PoolDecommissionInfo {
+	if pd == nil {
+		return nil
+	}
+	if pd.StartTime.IsZero() {
+		return nil
+	}
+	return &PoolDecommissionInfo{
+		StartTime:               pd.StartTime,
+		StartSize:               pd.StartSize,
+		TotalSize:               pd.TotalSize,
+		CurrentSize:             pd.CurrentSize,
+		Complete:                pd.Complete,
+		Failed:                  pd.Failed,
+		Canceled:                pd.Canceled,
+		QueuedBuckets:           pd.QueuedBuckets,
+		DecommissionedBuckets:   pd.DecommissionedBuckets,
+		Bucket:                  pd.Bucket,
+		Prefix:                  pd.Prefix,
+		Object:                  pd.Object,
+		ItemsDecommissioned:     pd.ItemsDecommissioned,
+		ItemsDecommissionFailed: pd.ItemsDecommissionFailed,
+		BytesDone:               pd.BytesDone,
+		BytesFailed:             pd.BytesFailed,
+	}
+}
+
 // bucketPop should be called when a bucket is done decommissioning.
 // Adds the bucket to the list of decommissioned buckets and updates resume numbers.
 func (pd *PoolDecommissionInfo) bucketPop(bucket string) {
@@ -118,10 +146,23 @@ type PoolStatus struct {
 	Decommission *PoolDecommissionInfo `json:"decommissionInfo,omitempty" msg:"dec"`
 }

+// Clone returns a copy of PoolStatus
+func (ps PoolStatus) Clone() PoolStatus {
+	return PoolStatus{
+		ID:           ps.ID,
+		CmdLine:      ps.CmdLine,
+		LastUpdate:   ps.LastUpdate,
+		Decommission: ps.Decommission.Clone(),
+	}
+}
+
 //go:generate msgp -file $GOFILE -unexported
 type poolMeta struct {
 	Version int          `msg:"v"`
 	Pools   []PoolStatus `msg:"pls"`
+
+	// Value should not be saved when we have not loaded anything yet.
+	dontSave bool `msg:"-"`
 }

 // A decommission resumable tells us if decommission is worth
@@ -270,6 +311,10 @@ func (p *poolMeta) Decommission(idx int, pi poolSpaceInfo) error {
 }

 func (p poolMeta) IsSuspended(idx int) bool {
+	if idx >= len(p.Pools) {
+		// We don't really know if the pool is suspended or not, since it doesn't exist.
+		return false
+	}
 	return p.Pools[idx].Decommission != nil
 }

@@ -375,16 +420,17 @@ func (p *poolMeta) load(ctx context.Context, pool *erasureSets, pools []*erasure

 func (p *poolMeta) CountItem(idx int, size int64, failed bool) {
 	pd := p.Pools[idx].Decommission
-	if pd != nil {
-		if failed {
-			pd.ItemsDecommissionFailed++
-			pd.BytesFailed += size
-		} else {
-			pd.ItemsDecommissioned++
-			pd.BytesDone += size
-		}
-		p.Pools[idx].Decommission = pd
+	if pd == nil {
+		return
 	}
+	if failed {
+		pd.ItemsDecommissionFailed++
+		pd.BytesFailed += size
+	} else {
+		pd.ItemsDecommissioned++
+		pd.BytesDone += size
+	}
+	p.Pools[idx].Decommission = pd
 }

 func (p *poolMeta) updateAfter(ctx context.Context, idx int, pools []*erasureSets, duration time.Duration) (bool, error) {
@@ -406,6 +452,9 @@ func (p *poolMeta) updateAfter(ctx context.Context, idx int, pools []*erasureSet
 }

 func (p poolMeta) save(ctx context.Context, pools []*erasureSets) error {
+	if p.dontSave {
+		return nil
+	}
 	data := make([]byte, 4, p.Msgsize()+4)

 	// Initialize the header.
@@ -460,36 +509,17 @@ func (z *erasureServerPools) Init(ctx context.Context) error {

 	// if no update is needed return right away.
 	if !update {
+		z.poolMetaMutex.Lock()
 		z.poolMeta = meta
+		z.poolMetaMutex.Unlock()
 	} else {
-		newMeta := poolMeta{} // to update write poolMeta fresh.
-		// looks like new pool was added we need to update,
-		// or this is a fresh installation (or an existing
-		// installation with pool removed)
-		newMeta.Version = poolMetaVersion
-		for idx, pool := range z.serverPools {
-			var skip bool
-			for _, currentPool := range meta.Pools {
-				// Preserve any current pool status.
-				if currentPool.CmdLine == pool.endpoints.CmdLine {
-					newMeta.Pools = append(newMeta.Pools, currentPool)
-					skip = true
-					break
-				}
-			}
-			if skip {
-				continue
-			}
-			newMeta.Pools = append(newMeta.Pools, PoolStatus{
-				CmdLine:    pool.endpoints.CmdLine,
-				ID:         idx,
-				LastUpdate: UTCNow(),
-			})
-		}
+		newMeta := newPoolMeta(z, meta)
 		if err = newMeta.save(ctx, z.serverPools); err != nil {
 			return err
 		}
+		z.poolMetaMutex.Lock()
 		z.poolMeta = newMeta
+		z.poolMetaMutex.Unlock()
 	}

 	pools := meta.returnResumablePools()
@@ -529,6 +559,34 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 	return nil
 }

+func newPoolMeta(z *erasureServerPools, prevMeta poolMeta) poolMeta {
+	newMeta := poolMeta{} // to update write poolMeta fresh.
+	// looks like new pool was added we need to update,
+	// or this is a fresh installation (or an existing
+	// installation with pool removed)
+	newMeta.Version = poolMetaVersion
+	for idx, pool := range z.serverPools {
+		var skip bool
+		for _, currentPool := range prevMeta.Pools {
+			// Preserve any current pool status.
+			if currentPool.CmdLine == pool.endpoints.CmdLine {
+				newMeta.Pools = append(newMeta.Pools, currentPool)
+				skip = true
+				break
+			}
+		}
+		if skip {
+			continue
+		}
+		newMeta.Pools = append(newMeta.Pools, PoolStatus{
+			CmdLine:    pool.endpoints.CmdLine,
+			ID:         idx,
+			LastUpdate: UTCNow(),
+		})
+	}
+	return newMeta
+}
+
 func (z *erasureServerPools) IsDecommissionRunning() bool {
 	z.poolMetaMutex.RLock()
 	defer z.poolMetaMutex.RUnlock()
@@ -569,7 +627,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, ObjectOptions{})
 		parts := make([]CompletePart, len(objInfo.Parts))
 		for i, part := range objInfo.Parts {
-			hr, err := hash.NewReader(io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
+			hr, err := hash.NewReader(ctx, io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
 			if err != nil {
 				return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 			}
@@ -603,7 +661,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		return err
 	}

-	hr, err := hash.NewReader(io.LimitReader(gr, objInfo.Size), objInfo.Size, "", "", actualSize)
+	hr, err := hash.NewReader(ctx, io.LimitReader(gr, objInfo.Size), objInfo.Size, "", "", actualSize)
 	if err != nil {
 		return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 	}
@@ -695,8 +753,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool

 	// Check if bucket is object locked.
 	lr, _ := globalBucketObjectLockSys.Get(bi.Name)
-
-	r := rand.New(rand.NewSource(time.Now().UnixNano()))
+	rcfg, _ := getReplicationConfig(ctx, bi.Name)

 	for setIdx, set := range pool.sets {
 		set := set
@@ -708,7 +765,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)

-			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 			switch {
 			case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
 				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)
@@ -908,11 +965,11 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						go decommissionEntry(entry)
 					},
 				)
-				if err == nil {
+				if err == nil || errors.Is(err, context.Canceled) {
 					break
 				}
 				setN := humanize.Ordinal(setIdx + 1)
-				retryDur := time.Duration(r.Float64() * float64(5*time.Second))
+				retryDur := time.Duration(rand.Float64() * float64(5*time.Second))
 				logger.LogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v, retrying in %v", setN, err, retryDur), "decom-listing-failed"+setN)
 				time.Sleep(retryDur)
 			}
@@ -965,8 +1022,15 @@ func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(err

 func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error {
 	pool := z.serverPools[idx]
-	for _, bucket := range z.poolMeta.PendingBuckets(idx) {
-		if z.poolMeta.isBucketDecommissioned(idx, bucket.String()) {
+	z.poolMetaMutex.RLock()
+	pending := z.poolMeta.PendingBuckets(idx)
+	z.poolMetaMutex.RUnlock()
+
+	for _, bucket := range pending {
+		z.poolMetaMutex.RLock()
+		isDecommissioned := z.poolMeta.isBucketDecommissioned(idx, bucket.String())
+		z.poolMetaMutex.RUnlock()
+		if isDecommissioned {
 			if serverDebugLog {
 				console.Debugln("decommission: already done, moving on", bucket)
 			}
@@ -1011,6 +1075,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error

 			// Check if bucket is object locked.
 			lr, _ := globalBucketObjectLockSys.Get(bi.Name)
+			rcfg, _ := getReplicationConfig(ctx, bi.Name)

 			filterLifecycle := func(bucket, object string, fi FileInfo) bool {
 				if lc == nil {
@@ -1019,7 +1084,7 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 				versioned := vc != nil && vc.Versioned(object)
 				objInfo := fi.ToObjectInfo(bucket, object, versioned)

-				evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+				evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 				switch {
 				case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
 					applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)
@@ -1185,15 +1250,15 @@ func (z *erasureServerPools) Status(ctx context.Context, idx int) (PoolStatus, e
 		return PoolStatus{}, errInvalidArgument
 	}

-	z.poolMetaMutex.RLock()
-	defer z.poolMetaMutex.RUnlock()
-
 	pi, err := z.getDecommissionPoolSpaceInfo(idx)
 	if err != nil {
 		return PoolStatus{}, err
 	}

-	poolInfo := z.poolMeta.Pools[idx]
+	z.poolMetaMutex.RLock()
+	defer z.poolMetaMutex.RUnlock()
+
+	poolInfo := z.poolMeta.Pools[idx].Clone()
 	if poolInfo.Decommission != nil {
 		poolInfo.Decommission.TotalSize = pi.Total
 		if poolInfo.Decommission.Failed || poolInfo.Decommission.Canceled {
@@ -440,6 +440,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 	lc, _ := globalLifecycleSys.Get(bucket)
 	// Check if bucket is object locked.
 	lr, _ := globalBucketObjectLockSys.Get(bucket)
+	rcfg, _ := getReplicationConfig(ctx, bucket)

 	pool := z.serverPools[poolIdx]
 	const envRebalanceWorkers = "_MINIO_REBALANCE_WORKERS"
@@ -467,7 +468,7 @@ func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string,
 			versioned := vc != nil && vc.Versioned(object)
 			objInfo := fi.ToObjectInfo(bucket, object, versioned)

-			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+			evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo)
 			if evt.Action.Delete() {
 				globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Rebal)
 				return true
@@ -736,7 +737,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,

 		parts := make([]CompletePart, len(oi.Parts))
 		for i, part := range oi.Parts {
-			hr, err := hash.NewReader(io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
+			hr, err := hash.NewReader(ctx, io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
 			if err != nil {
 				return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err)
 			}
@@ -766,7 +767,7 @@ func (z *erasureServerPools) rebalanceObject(ctx context.Context, bucket string,
 		return err
 	}

-	hr, err := hash.NewReader(gr, oi.Size, "", "", actualSize)
+	hr, err := hash.NewReader(ctx, gr, oi.Size, "", "", actualSize)
 	if err != nil {
 		return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err)
 	}
@@ -25,6 +25,7 @@ import (
 	"io"
 	"math/rand"
 	"net/http"
+	"path"
 	"sort"
 	"strconv"
 	"strings"
@@ -142,6 +143,12 @@ func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServ

 	z.decommissionCancelers = make([]context.CancelFunc, len(z.serverPools))

+	// Initialize the pool meta, but set it to not save.
+	// When z.Init below has loaded the poolmeta will be initialized,
+	// and allowed to save.
+	z.poolMeta = newPoolMeta(z, poolMeta{})
+	z.poolMeta.dontSave = true
+
 	// initialize the object layer.
 	setObjectLayer(z)

@@ -788,6 +795,16 @@ func (z *erasureServerPools) GetObjectNInfo(ctx context.Context, bucket, object
 		return nil, err
 	}

+	// This is a special call attempted first to check for SOS-API calls.
+	gr, err = veeamSOSAPIGetObject(ctx, bucket, object, rs, opts)
+	if err == nil {
+		return gr, nil
+	}
+
+	// reset any error to 'nil' and any reader to be 'nil'
+	gr = nil
+	err = nil
+
 	object = encodeDirObject(object)

 	if z.SinglePool() {
@@ -917,6 +934,16 @@ func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object s
 		return objInfo, err
 	}

+	// This is a special call attempted first to check for SOS-API calls.
+	objInfo, err = veeamSOSAPIHeadObject(ctx, bucket, object, opts)
+	if err == nil {
+		return objInfo, nil
+	}
+
+	// reset any error to 'nil', and object info to be empty.
+	err = nil
+	objInfo = ObjectInfo{}
+
 	object = encodeDirObject(object)

 	if z.SinglePool() {
@@ -1323,6 +1350,85 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	}
 	opts.setBucketMeta(ctx)

+	ri := logger.GetReqInfo(ctx)
+	hadoop := ri != nil && strings.Contains(ri.UserAgent, `Hadoop `) && strings.Contains(ri.UserAgent, "scala/")
+	matches := func() bool {
+		if prefix == "" {
+			return false
+		}
+		// List of standard files supported by s3a
+		// that involves a List() on a directory
+		// where directory is actually an object on
+		// namespace.
+		for _, k := range []string{
+			"_SUCCESS/",
+			".parquet/",
+			".csv/",
+			".json/",
+			".avro/",
+			".orc/",
+			".txt/",
+			// Add any other files in future
+		} {
+			if strings.HasSuffix(prefix, k) {
+				return true
+			}
+		}
+		return false
+	}
+	if hadoop && matches() && delimiter == SlashSeparator && maxKeys == 2 && marker == "" {
+		// Optimization for Spark/Hadoop workload where spark sends a garbage
+		// request of this kind
+		//
+		// GET /testbucket/?list-type=2&delimiter=%2F&max-keys=2&prefix=parquet%2F_SUCCESS%2F&fetch-owner=false
+		//
+		// Here spark is expecting that the List() return empty instead, so from MinIO's point
+		// of view if we simply do a GetObjectInfo() on this prefix by treating it as an object
+		// We save a lot of calls over the network.
+		//
+		// This happens repeatedly for all objects that are created concurrently() avoiding this
+		// as a List() call is an important performance improvement.
+		//
+		// Spark based s3a committers are a  big enough use-case to have this optimization.
+		//
+		// A sample code to see the improvements is as follows, this sample code is
+		// simply a read on JSON from MinIO and write it back as "parquet".
+		//
+		// import org.apache.spark.sql.SparkSession
+		// import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType}
+		// object SparkJSONRead {
+		//   def main(args: Array[String]): Unit = {
+		//     val spark:SparkSession = SparkSession.builder()
+		//       .appName("SparkByExample")
+		//       .master("local[1]").getOrCreate()
+		//
+		//     spark.sparkContext.setLogLevel("ERROR")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.endpoint", "http://minio-lb:9000")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.path.style.access", "true")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.access.key", "minioadmin")
+		//     spark.sparkContext.hadoopConfiguration.set("fs.s3a.secret.key", "minioadmin")
+		//
+		//     val df = spark.read.json("s3a://testbucket/s3.json")
+		//
+		//     df.write.parquet("s3a://testbucket/parquet/")
+		//   }
+		// }
+		objInfo, err := z.GetObjectInfo(ctx, bucket, path.Dir(prefix), ObjectOptions{NoLock: true})
+		if err == nil {
+			if opts.Lifecycle != nil {
+				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo)
+				if evt.Action.Delete() {
+					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
+					if !evt.Action.DeleteRestored() {
+						// Skip entry if ILM action was DeleteVersionAction or DeleteAction
+						return loi, nil
+					}
+				}
+			}
+			return loi, nil
+		}
+	}
+
 	if len(prefix) > 0 && maxKeys == 1 && marker == "" {
 		// Optimization for certain applications like
 		// - Cohesity
@@ -1334,7 +1440,7 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 		objInfo, err := z.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
 		if err == nil {
 			if opts.Lifecycle != nil {
-				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, objInfo)
+				evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo)
 				if evt.Action.Delete() {
 					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects)
 					if !evt.Action.DeleteRestored() {
@@ -1716,10 +1822,42 @@ func (z *erasureServerPools) deleteAll(ctx context.Context, bucket, prefix strin
 	}
 }

+var listBucketsCache timedValue
+
 // List all buckets from one of the serverPools, we are not doing merge
 // sort here just for simplification. As per design it is assumed
 // that all buckets are present on all serverPools.
 func (z *erasureServerPools) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) {
+	if opts.Cached {
+		listBucketsCache.Once.Do(func() {
+			listBucketsCache.TTL = time.Second
+
+			listBucketsCache.Relax = true
+			listBucketsCache.Update = func() (interface{}, error) {
+				ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+				buckets, err = z.s3Peer.ListBuckets(ctx, opts)
+				cancel()
+				if err != nil {
+					return nil, err
+				}
+				for i := range buckets {
+					createdAt, err := globalBucketMetadataSys.CreatedAt(buckets[i].Name)
+					if err == nil {
+						buckets[i].Created = createdAt
+					}
+				}
+				return buckets, nil
+			}
+		})
+
+		v, _ := listBucketsCache.Get()
+		if v != nil {
+			return v.([]BucketInfo), nil
+		}
+
+		return buckets, nil
+	}
+
 	buckets, err = z.s3Peer.ListBuckets(ctx, opts)
 	if err != nil {
 		return nil, err
@@ -2339,12 +2477,12 @@ func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object s
 		return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts)
 	}

-	idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts)
+	oi, _, err := z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts)
 	if err != nil {
 		return nil, err
 	}

-	return z.serverPools[idx].GetObjectTags(ctx, bucket, object, opts)
+	return tags.ParseObjectTags(oi.UserTags)
 }

 // TransitionObject - transition object content to target tier.
@@ -217,6 +217,10 @@ func (s *erasureSets) connectDisks() {
 				continue
 			}
 		}
+		if cdisk != nil {
+			// Close previous offline disk.
+			cdisk.Close()
+		}

 		wg.Add(1)
 		go func(endpoint Endpoint) {
@@ -18,7 +18,6 @@
 package cmd

 import (
-	"bytes"
 	"context"
 	"encoding/base64"
 	"fmt"
@@ -26,7 +25,6 @@ import (
 	"strings"

 	"github.com/klauspost/reedsolomon"
-	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
 )

@@ -85,7 +83,7 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data

 		// We have written all the blocks, write the last remaining block.
 		if write < int64(len(block)) {
-			n, err := xioutil.Copy(dst, bytes.NewReader(block[:write]))
+			n, err := dst.Write(block[:write])
 			if err != nil {
 				// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
 				// The reader pipe might be closed at ListObjects io.EOF ignore it.
@@ -94,12 +92,12 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 				}
 				return 0, err
 			}
-			totalWritten += n
+			totalWritten += int64(n)
 			break
 		}

 		// Copy the block.
-		n, err := xioutil.Copy(dst, bytes.NewReader(block))
+		n, err := dst.Write(block)
 		if err != nil {
 			// The writer will be closed incase of range queries, which will emit ErrClosedPipe.
 			// The reader pipe might be closed at ListObjects io.EOF ignore it.
@@ -110,10 +108,10 @@ func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, data
 		}

 		// Decrement output size.
-		write -= n
+		write -= int64(n)

 		// Increment written.
-		totalWritten += n
+		totalWritten += int64(n)
 	}

 	// Success.
@@ -182,7 +182,6 @@ func getDisksInfo(disks []StorageAPI, endpoints []Endpoint) (disksInfo []madmin.
 				DiskIndex: endpoints[index].DiskIdx,
 			}
 			if disks[index] == OfflineDisk {
-				logger.LogOnceIf(GlobalContext, fmt.Errorf("%s: %s", errDiskNotFound, endpoints[index]), "get-disks-info-offline-"+di.Endpoint)
 				di.State = diskErrToDriveState(errDiskNotFound)
 				disksInfo[index] = di
 				return nil
@@ -336,7 +335,7 @@ func (er erasureObjects) getOnlineDisksWithHealing() (newDisks []StorageAPI, hea
 func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) {
 	// run multiple cleanup's local to this server.
 	var wg sync.WaitGroup
-	for _, disk := range er.getLoadBalancedLocalDisks() {
+	for _, disk := range er.getLocalDisks() {
 		if disk != nil {
 			wg.Add(1)
 			go func(disk StorageAPI) {
@@ -434,6 +433,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 				}
 				logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-cache-update")
 				updates <- cache.clone()
+
 				lastSave = cache.Info.LastUpdate
 			case v, ok := <-bucketResults:
 				if !ok {
@@ -441,7 +441,7 @@ func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wa
 					cache.Info.NextCycle = wantCycle
 					cache.Info.LastUpdate = time.Now()
 					logger.LogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-channel-closed")
-					updates <- cache
+					updates <- cache.clone()
 					return
 				}
 				cache.replace(v.Name, v.Parent, v.Entry)
@@ -104,7 +104,7 @@ func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI Object
 		for res := range evnot.targetResCh {
 			if res.Err != nil {
 				reqInfo := &logger.ReqInfo{}
-				reqInfo.AppendTags("targetID", res.ID.Name)
+				reqInfo.AppendTags("targetID", res.ID.String())
 				logger.LogOnceIf(logger.SetReqInfo(GlobalContext, reqInfo), res.Err, res.ID.String())
 			}
 		}
@@ -210,6 +210,9 @@ type eventArgs struct {
 func (args eventArgs) ToEvent(escape bool) event.Event {
 	eventTime := UTCNow()
 	uniqueID := fmt.Sprintf("%X", eventTime.UnixNano())
+	if !args.Object.ModTime.IsZero() {
+		uniqueID = fmt.Sprintf("%X", args.Object.ModTime.UnixNano())
+	}

 	respElements := map[string]string{
 		"x-amz-request-id": args.RespElements["requestId"],
@@ -223,7 +226,7 @@ func (args eventArgs) ToEvent(escape bool) event.Event {
 	}

 	// Add deployment as part of response elements.
-	respElements["x-minio-deployment-id"] = globalDeploymentID
+	respElements["x-minio-deployment-id"] = globalDeploymentID()
 	if args.RespElements["content-length"] != "" {
 		respElements["content-length"] = args.RespElements["content-length"]
 	}
@@ -1,498 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"path"
-	"path/filepath"
-	"reflect"
-	"strings"
-
-	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/minio/internal/logger"
-	"github.com/minio/sio"
-)
-
-const (
-	// Represents Cache format json holding details on all other cache drives in use.
-	formatCache = "cache"
-
-	// formatCacheV1.Cache.Version
-	formatCacheVersionV1 = "1"
-	formatCacheVersionV2 = "2"
-
-	formatMetaVersion1 = "1"
-
-	formatCacheV1DistributionAlgo = "CRCMOD"
-)
-
-// Represents the current cache structure with list of
-// disks comprising the disk cache
-// formatCacheV1 - structure holds format config version '1'.
-type formatCacheV1 struct {
-	formatMetaV1
-	Cache struct {
-		Version string `json:"version"` // Version of 'cache' format.
-		This    string `json:"this"`    // This field carries assigned disk uuid.
-		// Disks field carries the input disk order generated the first
-		// time when fresh disks were supplied.
-		Disks []string `json:"disks"`
-		// Distribution algorithm represents the hashing algorithm
-		// to pick the right set index for an object.
-		DistributionAlgo string `json:"distributionAlgo"`
-	} `json:"cache"` // Cache field holds cache format.
-}
-
-// formatCacheV2 is same as formatCacheV1
-type formatCacheV2 = formatCacheV1
-
-// Used to detect the version of "cache" format.
-type formatCacheVersionDetect struct {
-	Cache struct {
-		Version string `json:"version"`
-	} `json:"cache"`
-}
-
-// Return a slice of format, to be used to format uninitialized disks.
-func newFormatCacheV2(drives []string) []*formatCacheV2 {
-	diskCount := len(drives)
-	disks := make([]string, diskCount)
-
-	formats := make([]*formatCacheV2, diskCount)
-
-	for i := 0; i < diskCount; i++ {
-		format := &formatCacheV2{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.DistributionAlgo = formatCacheV1DistributionAlgo
-		format.Cache.This = mustGetUUID()
-		formats[i] = format
-		disks[i] = formats[i].Cache.This
-	}
-	for i := 0; i < diskCount; i++ {
-		format := formats[i]
-		format.Cache.Disks = disks
-	}
-	return formats
-}
-
-// Returns formatCache.Cache.Version
-func formatCacheGetVersion(r io.ReadSeeker) (string, error) {
-	format := &formatCacheVersionDetect{}
-	if err := jsonLoad(r, format); err != nil {
-		return "", err
-	}
-	return format.Cache.Version, nil
-}
-
-// Creates a new cache format.json if unformatted.
-func createFormatCache(fsFormatPath string, format *formatCacheV1) error {
-	// open file using READ & WRITE permission
-	file, err := os.OpenFile(fsFormatPath, os.O_RDWR|os.O_CREATE, 0o666)
-	if err != nil {
-		return err
-	}
-	// Close the locked file upon return.
-	defer file.Close()
-
-	fi, err := file.Stat()
-	if err != nil {
-		return err
-	}
-	if fi.Size() != 0 {
-		// format.json already got created because of another minio process's createFormatCache()
-		return nil
-	}
-	return jsonSave(file, format)
-}
-
-// This function creates a cache format file on disk and returns a slice
-// of format cache config
-func initFormatCache(ctx context.Context, drives []string) (formats []*formatCacheV2, err error) {
-	nformats := newFormatCacheV2(drives)
-	for i, drive := range drives {
-		if err = os.MkdirAll(pathJoin(drive, minioMetaBucket), 0o777); err != nil {
-			logger.GetReqInfo(ctx).AppendTags("drive", drive)
-			logger.LogIf(ctx, err)
-			return nil, err
-		}
-		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
-		// Fresh disk - create format.json for this cfs
-		if err = createFormatCache(cacheFormatPath, nformats[i]); err != nil {
-			logger.GetReqInfo(ctx).AppendTags("drive", drive)
-			logger.LogIf(ctx, err)
-			return nil, err
-		}
-	}
-	return nformats, nil
-}
-
-func loadFormatCache(ctx context.Context, drives []string) ([]*formatCacheV2, bool, error) {
-	formats := make([]*formatCacheV2, len(drives))
-	var formatV2 *formatCacheV2
-	migrating := false
-	for i, drive := range drives {
-		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
-		f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0o666)
-		if err != nil {
-			if osIsNotExist(err) {
-				continue
-			}
-			logger.LogIf(ctx, err)
-			return nil, migrating, err
-		}
-		defer f.Close()
-		format, err := formatMetaCacheV1(f)
-		if err != nil {
-			continue
-		}
-		formatV2 = format
-		if format.Cache.Version != formatCacheVersionV2 {
-			migrating = true
-		}
-		formats[i] = formatV2
-	}
-	return formats, migrating, nil
-}
-
-// unmarshalls the cache format.json into formatCacheV1
-func formatMetaCacheV1(r io.ReadSeeker) (*formatCacheV1, error) {
-	format := &formatCacheV1{}
-	if err := jsonLoad(r, format); err != nil {
-		return nil, err
-	}
-	return format, nil
-}
-
-func checkFormatCacheValue(format *formatCacheV2, migrating bool) error {
-	if format.Format != formatCache {
-		return fmt.Errorf("Unsupported cache format [%s] found", format.Format)
-	}
-
-	// during migration one or more cache drive(s) formats can be out of sync
-	if migrating {
-		// Validate format version and format type.
-		if format.Version != formatMetaVersion1 {
-			return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
-		}
-		if format.Cache.Version != formatCacheVersionV2 && format.Cache.Version != formatCacheVersionV1 {
-			return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
-		}
-		return nil
-	}
-	// Validate format version and format type.
-	if format.Version != formatMetaVersion1 {
-		return fmt.Errorf("Unsupported version of cache format [%s] found", format.Version)
-	}
-	if format.Cache.Version != formatCacheVersionV2 {
-		return fmt.Errorf("Unsupported Cache backend format found [%s]", format.Cache.Version)
-	}
-	return nil
-}
-
-func checkFormatCacheValues(migrating bool, formats []*formatCacheV2) (int, error) {
-	for i, formatCache := range formats {
-		if formatCache == nil {
-			continue
-		}
-		if err := checkFormatCacheValue(formatCache, migrating); err != nil {
-			return i, err
-		}
-		if len(formats) != len(formatCache.Cache.Disks) {
-			return i, fmt.Errorf("Expected number of cache drives %d , got  %d",
-				len(formatCache.Cache.Disks), len(formats))
-		}
-	}
-	return -1, nil
-}
-
-// checkCacheDisksConsistency - checks if "This" disk uuid on each disk is consistent with all "Disks" slices
-// across disks.
-func checkCacheDiskConsistency(formats []*formatCacheV2) error {
-	disks := make([]string, len(formats))
-	// Collect currently available disk uuids.
-	for index, format := range formats {
-		if format == nil {
-			disks[index] = ""
-			continue
-		}
-		disks[index] = format.Cache.This
-	}
-	for i, format := range formats {
-		if format == nil {
-			continue
-		}
-		j := findCacheDiskIndex(disks[i], format.Cache.Disks)
-		if j == -1 {
-			return fmt.Errorf("UUID on positions %d:%d do not match with , expected %s", i, j, disks[i])
-		}
-		if i != j {
-			return fmt.Errorf("UUID on positions %d:%d do not match with , expected %s got %s", i, j, disks[i], format.Cache.Disks[j])
-		}
-	}
-	return nil
-}
-
-// checkCacheDisksSliceConsistency - validate cache Disks order if they are consistent.
-func checkCacheDisksSliceConsistency(formats []*formatCacheV2) error {
-	var sentinelDisks []string
-	// Extract first valid Disks slice.
-	for _, format := range formats {
-		if format == nil {
-			continue
-		}
-		sentinelDisks = format.Cache.Disks
-		break
-	}
-	for _, format := range formats {
-		if format == nil {
-			continue
-		}
-		currentDisks := format.Cache.Disks
-		if !reflect.DeepEqual(sentinelDisks, currentDisks) {
-			return errors.New("inconsistent cache drives found")
-		}
-	}
-	return nil
-}
-
-// findCacheDiskIndex returns position of cache disk in JBOD.
-func findCacheDiskIndex(disk string, disks []string) int {
-	for index, uuid := range disks {
-		if uuid == disk {
-			return index
-		}
-	}
-	return -1
-}
-
-// validate whether cache drives order has changed
-func validateCacheFormats(ctx context.Context, migrating bool, formats []*formatCacheV2) error {
-	count := 0
-	for _, format := range formats {
-		if format == nil {
-			count++
-		}
-	}
-	if count == len(formats) {
-		return errors.New("Cache format files missing on all drives")
-	}
-	if _, err := checkFormatCacheValues(migrating, formats); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-	if err := checkCacheDisksSliceConsistency(formats); err != nil {
-		logger.LogIf(ctx, err)
-		return err
-	}
-	err := checkCacheDiskConsistency(formats)
-	logger.LogIf(ctx, err)
-	return err
-}
-
-// return true if all of the list of cache drives are
-// fresh disks
-func cacheDrivesUnformatted(drives []string) bool {
-	count := 0
-	for _, drive := range drives {
-		cacheFormatPath := pathJoin(drive, minioMetaBucket, formatConfigFile)
-		if _, err := os.Stat(cacheFormatPath); osIsNotExist(err) {
-			count++
-		}
-	}
-	return count == len(drives)
-}
-
-// create format.json for each cache drive if fresh disk or load format from disk
-// Then validate the format for all drives in the cache to ensure order
-// of cache drives has not changed.
-func loadAndValidateCacheFormat(ctx context.Context, drives []string) (formats []*formatCacheV2, migrating bool, err error) {
-	if cacheDrivesUnformatted(drives) {
-		formats, err = initFormatCache(ctx, drives)
-	} else {
-		formats, migrating, err = loadFormatCache(ctx, drives)
-	}
-	if err != nil {
-		return nil, false, err
-	}
-	if err = validateCacheFormats(ctx, migrating, formats); err != nil {
-		return nil, false, err
-	}
-	return formats, migrating, nil
-}
-
-// reads cached object on disk and writes it back after adding bitrot
-// hashsum per block as per the new disk cache format.
-func migrateCacheData(ctx context.Context, c *diskCache, bucket, object, oldfile, destDir string, metadata map[string]string) error {
-	st, err := os.Stat(oldfile)
-	if err != nil {
-		err = osErrToFileErr(err)
-		return err
-	}
-	readCloser, err := readCacheFileStream(oldfile, 0, st.Size())
-	if err != nil {
-		return err
-	}
-	defer readCloser.Close()
-	var reader io.Reader = readCloser
-
-	actualSize := uint64(st.Size())
-	if globalCacheKMS != nil {
-		reader, err = newCacheEncryptReader(ctx, readCloser, bucket, object, metadata)
-		if err != nil {
-			return err
-		}
-		actualSize, _ = sio.EncryptedSize(uint64(st.Size()))
-	}
-	_, _, err = c.bitrotWriteToCache(destDir, cacheDataFile, reader, actualSize)
-	return err
-}
-
-// migrate cache contents from old cacheFS format to new backend format
-// new format is flat
-//
-//	sha(bucket,object)/  <== dir name
-//	    - part.1         <== data
-//	    - cache.json     <== metadata
-func migrateOldCache(ctx context.Context, c *diskCache) error {
-	oldCacheBucketsPath := path.Join(c.dir, minioMetaBucket, "buckets")
-	cacheFormatPath := pathJoin(c.dir, minioMetaBucket, formatConfigFile)
-
-	if _, err := os.Stat(oldCacheBucketsPath); err != nil {
-		// remove .minio.sys sub directories
-		removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
-		removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
-		removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
-		removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
-		// just migrate cache format
-		return migrateCacheFormatJSON(cacheFormatPath)
-	}
-
-	buckets, err := readDir(oldCacheBucketsPath)
-	if err != nil {
-		return err
-	}
-
-	for _, bucket := range buckets {
-		bucket = strings.TrimSuffix(bucket, SlashSeparator)
-		var objMetaPaths []string
-		root := path.Join(oldCacheBucketsPath, bucket)
-		err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
-			if strings.HasSuffix(path, cacheMetaJSONFile) {
-				objMetaPaths = append(objMetaPaths, path)
-			}
-			return nil
-		})
-		if err != nil {
-			return err
-		}
-		for _, oMeta := range objMetaPaths {
-			objSlice := strings.SplitN(oMeta, cacheMetaJSONFile, 2)
-			object := strings.TrimPrefix(objSlice[0], path.Join(oldCacheBucketsPath, bucket))
-			object = strings.TrimSuffix(object, "/")
-
-			destdir := getCacheSHADir(c.dir, bucket, object)
-			if err := os.MkdirAll(destdir, 0o777); err != nil {
-				return err
-			}
-			prevCachedPath := path.Join(c.dir, bucket, object)
-
-			// get old cached metadata
-			oldMetaPath := pathJoin(oldCacheBucketsPath, bucket, object, cacheMetaJSONFile)
-			metaPath := pathJoin(destdir, cacheMetaJSONFile)
-			metaBytes, err := os.ReadFile(oldMetaPath)
-			if err != nil {
-				return err
-			}
-			// marshal cache metadata after adding version and stat info
-			meta := &cacheMeta{}
-			json := jsoniter.ConfigCompatibleWithStandardLibrary
-			if err = json.Unmarshal(metaBytes, &meta); err != nil {
-				return err
-			}
-			// move cached object to new cache directory path
-			// migrate cache data and add bit-rot protection hash sum
-			// at the start of each block
-			if err := migrateCacheData(ctx, c, bucket, object, prevCachedPath, destdir, meta.Meta); err != nil {
-				continue
-			}
-			stat, err := os.Stat(prevCachedPath)
-			if err != nil {
-				if err == errFileNotFound {
-					continue
-				}
-				logger.LogIf(ctx, err)
-				return err
-			}
-			// old cached file can now be removed
-			if err := os.Remove(prevCachedPath); err != nil {
-				return err
-			}
-			// move cached metadata after changing cache metadata version
-			meta.Checksum = CacheChecksumInfoV1{Algorithm: HighwayHash256S.String(), Blocksize: cacheBlkSize}
-			meta.Version = cacheMetaVersion
-			meta.Stat.Size = stat.Size()
-			meta.Stat.ModTime = stat.ModTime()
-			jsonData, err := json.Marshal(meta)
-			if err != nil {
-				return err
-			}
-
-			if err = os.WriteFile(metaPath, jsonData, 0o644); err != nil {
-				return err
-			}
-		}
-
-		// delete old bucket from cache, now that all contents are cleared
-		removeAll(path.Join(c.dir, bucket))
-	}
-
-	// remove .minio.sys sub directories
-	removeAll(path.Join(c.dir, minioMetaBucket, "multipart"))
-	removeAll(path.Join(c.dir, minioMetaBucket, "tmp"))
-	removeAll(path.Join(c.dir, minioMetaBucket, "trash"))
-	removeAll(path.Join(c.dir, minioMetaBucket, "buckets"))
-
-	return migrateCacheFormatJSON(cacheFormatPath)
-}
-
-func migrateCacheFormatJSON(cacheFormatPath string) error {
-	// now migrate format.json
-	f, err := os.OpenFile(cacheFormatPath, os.O_RDWR, 0o666)
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	formatV1 := formatCacheV1{}
-	if err := jsonLoad(f, &formatV1); err != nil {
-		return err
-	}
-
-	formatV2 := &formatCacheV2{}
-	formatV2.formatMetaV1 = formatV1.formatMetaV1
-	formatV2.Version = formatMetaVersion1
-	formatV2.Cache = formatV1.Cache
-	formatV2.Cache.Version = formatCacheVersionV2
-	return jsonSave(f, formatV2)
-}
@@ -1,321 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"os"
-	"testing"
-)
-
-// TestDiskCacheFormat - tests initFormatCache, formatMetaGetFormatBackendCache, formatCacheGetVersion.
-func TestDiskCacheFormat(t *testing.T) {
-	ctx := context.Background()
-	fsDirs, err := getRandomDisks(1)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	_, err = initFormatCache(ctx, fsDirs)
-	if err != nil {
-		t.Fatal(err)
-	}
-	// Do the basic sanity checks to check if initFormatCache() did its job.
-	cacheFormatPath := pathJoin(fsDirs[0], minioMetaBucket, formatConfigFile)
-	f, err := os.OpenFile(cacheFormatPath, os.O_RDWR|os.O_SYNC, 0)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer f.Close()
-	version, err := formatCacheGetVersion(f)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if version != formatCacheVersionV2 {
-		t.Fatalf(`expected: %s, got: %s`, formatCacheVersionV2, version)
-	}
-
-	// Corrupt the format.json file and test the functions.
-	// formatMetaGetFormatBackendFS, formatFSGetVersion, initFormatFS should return errors.
-	if err = f.Truncate(0); err != nil {
-		t.Fatal(err)
-	}
-	if _, err = f.WriteString("b"); err != nil {
-		t.Fatal(err)
-	}
-
-	if _, _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil {
-		t.Fatal("expected to fail")
-	}
-
-	// With unknown formatMetaV1.Version formatMetaGetFormatCache, initFormatCache should return error.
-	if err = f.Truncate(0); err != nil {
-		t.Fatal(err)
-	}
-	// Here we set formatMetaV1.Version to "2"
-	if _, err = f.WriteString(`{"version":"2","format":"cache","cache":{"version":"1"}}`); err != nil {
-		t.Fatal(err)
-	}
-
-	if _, _, err = loadAndValidateCacheFormat(context.Background(), fsDirs); err == nil {
-		t.Fatal("expected to fail")
-	}
-}
-
-// generates a valid format.json for Cache backend.
-func genFormatCacheValid() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV1{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	return formatConfigs
-}
-
-// generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidVersion() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV1{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	// Corrupt version numbers.
-	formatConfigs[0].Version = "2"
-	formatConfigs[3].Version = "-1"
-	return formatConfigs
-}
-
-// generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidFormat() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV2{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	// Corrupt format.
-	formatConfigs[0].Format = "cach"
-	formatConfigs[3].Format = "cach"
-	return formatConfigs
-}
-
-// generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidCacheVersion() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV2{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV1
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	// Corrupt version numbers.
-	formatConfigs[0].Cache.Version = "10"
-	formatConfigs[3].Cache.Version = "-1"
-	return formatConfigs
-}
-
-// generates a invalid format.json version for Cache backend.
-func genFormatCacheInvalidDisksCount() []*formatCacheV2 {
-	disks := make([]string, 7)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV2{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	return formatConfigs
-}
-
-// generates a invalid format.json Disks for Cache backend.
-func genFormatCacheInvalidDisks() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV1{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	// Corrupt Disks entries on disk 6 and disk 8.
-	formatConfigs[5].Cache.Disks = disks
-	formatConfigs[7].Cache.Disks = disks
-	return formatConfigs
-}
-
-// generates a invalid format.json This disk UUID for Cache backend.
-func genFormatCacheInvalidThis() []*formatCacheV1 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV1, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV1{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	// Make disk 5 and disk 8 have inconsistent disk uuid's.
-	formatConfigs[4].Cache.This = mustGetUUID()
-	formatConfigs[7].Cache.This = mustGetUUID()
-	return formatConfigs
-}
-
-// generates a invalid format.json Disk UUID in wrong order for Cache backend.
-func genFormatCacheInvalidDisksOrder() []*formatCacheV2 {
-	disks := make([]string, 8)
-	formatConfigs := make([]*formatCacheV2, 8)
-	for index := range disks {
-		disks[index] = mustGetUUID()
-	}
-	for index := range disks {
-		format := &formatCacheV1{}
-		format.Version = formatMetaVersion1
-		format.Format = formatCache
-		format.Cache.Version = formatCacheVersionV2
-		format.Cache.This = disks[index]
-		format.Cache.Disks = disks
-		formatConfigs[index] = format
-	}
-	// Re order disks for failure case.
-	disks1 := make([]string, 8)
-	copy(disks1, disks)
-	disks1[1], disks1[2] = disks[2], disks[1]
-	formatConfigs[2].Cache.Disks = disks1
-	return formatConfigs
-}
-
-// Wrapper for calling FormatCache tests - validates
-//   - valid format
-//   - unrecognized version number
-//   - unrecognized format tag
-//   - unrecognized cache version
-//   - wrong number of Disks entries
-//   - invalid This uuid
-//   - invalid Disks order
-func TestFormatCache(t *testing.T) {
-	formatInputCases := [][]*formatCacheV1{
-		genFormatCacheValid(),
-		genFormatCacheInvalidVersion(),
-		genFormatCacheInvalidFormat(),
-		genFormatCacheInvalidCacheVersion(),
-		genFormatCacheInvalidDisksCount(),
-		genFormatCacheInvalidDisks(),
-		genFormatCacheInvalidThis(),
-		genFormatCacheInvalidDisksOrder(),
-	}
-	testCases := []struct {
-		formatConfigs []*formatCacheV1
-		shouldPass    bool
-	}{
-		{
-			formatConfigs: formatInputCases[0],
-			shouldPass:    true,
-		},
-		{
-			formatConfigs: formatInputCases[1],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[2],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[3],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[4],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[5],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[6],
-			shouldPass:    false,
-		},
-		{
-			formatConfigs: formatInputCases[7],
-			shouldPass:    false,
-		},
-	}
-
-	for i, testCase := range testCases {
-		err := validateCacheFormats(context.Background(), false, testCase.formatConfigs)
-		if err != nil && testCase.shouldPass {
-			t.Errorf("Test %d: Expected to pass but failed with %s", i+1, err)
-		}
-		if err == nil && !testCase.shouldPass {
-			t.Errorf("Test %d: Expected to fail but passed instead", i+1)
-		}
-	}
-}
@@ -275,7 +275,7 @@ func formatErasureMigrateV2ToV3(data []byte, export, version string) ([]byte, er

 	tmpOld := pathJoin(export, minioMetaTmpDeletedBucket, mustGetUUID())
 	if err := renameAll(pathJoin(export, minioMetaMultipartBucket),
-		tmpOld); err != nil && err != errFileNotFound {
+		tmpOld, export); err != nil && err != errFileNotFound {
 		logger.LogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate",
 			pathJoin(export, minioMetaMultipartBucket),
 			tmpOld,
--- a/Show More
+++ b/Show More