feat: allow expiration of all versions via ILM Expiration action (#17521 )

Following extension allows users to specify immediate purge of all versions as soon as the latest version of this object has expired. ``` <LifecycleConfiguration> <Rule> <ID>ClassADocRule</ID> <Filter> <Prefix>classA/</Prefix> </Filter> <Status>Enabled</Status> <Expiration> <Days>3650</Days> <ExpiredObjectAllVersions>true</ExpiredObjectAllVersions> </Expiration> </Rule> ... ```
fix: support LDAP settings properly in ftp/sftp (#17536 )
2026-11-12 10:35:33 +03:00 · 2023-06-28 22:12:28 -07:00 · 2023-06-28 13:15:21 -07:00 · 2023-06-28 09:43:50 -07:00 · 2023-06-27 11:59:40 -07:00 · 2023-06-27 09:45:50 -07:00
399 changed files with 20344 additions and 10201 deletions
@@ -1,5 +0,0 @@
 # Config file for markdownlint-cli
 MD033:
  allowed_elements:
    - details
    - summary
@@ -20,13 +20,8 @@ jobs:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
-        go-version: [1.20.x, 1.19.x]
+        go-version: [1.20.x]
        os: [ubuntu-latest, windows-latest]
        exclude:
          - os: ubuntu-latest
            go-version: 1.19.x
          - os: windows-latest
            go-version: 1.20.x
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v3
@@ -1,30 +0,0 @@
 name: Markdown Linter
 on:
  pull_request:
    branches:
    - master
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
 concurrency: 
  group: ${{ github.workflow }}-${{ github.head_ref }}
  cancel-in-progress: true
 permissions:
  contents: read
 jobs:
  lint:
    name: Lint all docs
    runs-on: ubuntu-latest
    steps:
    - name: Check out code
      uses: actions/checkout@v3
    - name: Lint all docs
      run: |
        npm install -g markdownlint-cli
        markdownlint --fix '**/*.md' \
          --config /home/runner/work/minio/minio/.github/markdown-lint-cfg.yaml \
          --disable MD013 MD040 MD051
@@ -0,0 +1,57 @@
 name: Mint Tests
 on:
  pull_request:
    branches:
    - master
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
 concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref }}
  cancel-in-progress: true
 permissions:
  contents: read
 jobs:
  mint-test:
    runs-on: mint
    timeout-minutes: 120
    steps:
      - name: cleanup #https://github.com/actions/checkout/issues/273
        run: |
          sudo -S rm -rf ${GITHUB_WORKSPACE}
          mkdir ${GITHUB_WORKSPACE}
      - name: checkout-step
        uses: actions/checkout@v3
      - name: setup-go-step
        uses: actions/setup-go@v2
        with:
          go-version: 1.20.x
      - name: github sha short
        id: vars
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT
      - name: build-minio
        run: |
          make install
          docker build . -t "minio/minio:${{ steps.vars.outputs.sha_short }}"
      - name: compress and encrypt
        run: |
          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "compress-encrypt" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
      - name: multiple pools
        run: |
          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "pools" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
      - name: standalone erasure
        run: |
          ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}"
          docker rmi -f minio/minio:${{ steps.vars.outputs.sha_short }}
@@ -0,0 +1,80 @@
 version: '3.7'
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
  image: minio/minio:${JOB_NAME}
  command: server --console-address ":9001" http://minio{1...4}/cdata{1...2}
  expose:
    - "9000"
    - "9001"
  environment:
    MINIO_CI_CD: "on"
    MINIO_ROOT_USER: "minio"
    MINIO_ROOT_PASSWORD: "minio123"
    MINIO_COMPRESSION_ENABLE: "on"
    MINIO_COMPRESSION_MIME_TYPES: "*"
    MINIO_COMPRESSION_ALLOW_ENCRYPTION: "on"
    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
    interval: 30s
    timeout: 20s
    retries: 3
 # starts 4 docker containers running minio server instances.
 # using nginx reverse proxy, load balancing, you can access
 # it through port 9000.
 services:
  minio1:
    <<: *minio-common
    hostname: minio1
    volumes:
      - cdata1-1:/cdata1
      - cdata1-2:/cdata2
  minio2:
    <<: *minio-common
    hostname: minio2
    volumes:
      - cdata2-1:/cdata1
      - cdata2-2:/cdata2
  minio3:
    <<: *minio-common
    hostname: minio3
    volumes:
      - cdata3-1:/cdata1
      - cdata3-2:/cdata2
  minio4:
    <<: *minio-common
    hostname: minio4
    volumes:
      - cdata4-1:/cdata1
      - cdata4-2:/cdata2
  nginx:
    image: nginx:1.19.2-alpine
    hostname: nginx
    volumes:
      - ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "9000:9000"
      - "9001:9001"
    depends_on:
      - minio1
      - minio2
      - minio3
      - minio4
 ## By default this config uses default local driver,
 ## For custom volumes replace with volume driver configuration.
 volumes:
  cdata1-1:
  cdata1-2:
  cdata2-1:
  cdata2-2:
  cdata3-1:
  cdata3-2:
  cdata4-1:
  cdata4-2:
@@ -0,0 +1,51 @@
 version: '3.7'
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
  image: minio/minio:${JOB_NAME}
  command: server --console-address ":9001" edata{1...4}
  expose:
    - "9000"
    - "9001"
  environment:
    MINIO_CI_CD: "on"
    MINIO_ROOT_USER: "minio"
    MINIO_ROOT_PASSWORD: "minio123"
    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
    interval: 30s
    timeout: 20s
    retries: 3
 # starts 4 docker containers running minio server instances.
 # using nginx reverse proxy, load balancing, you can access
 # it through port 9000.
 services:
  minio1:
    <<: *minio-common
    hostname: minio1
    volumes:
      - edata1-1:/edata1
      - edata1-2:/edata2
      - edata1-3:/edata3
      - edata1-4:/edata4
  nginx:
    image: nginx:1.19.2-alpine
    hostname: nginx
    volumes:
      - ./nginx-1-node.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "9000:9000"
      - "9001:9001"
    depends_on:
      - minio1
 ## By default this config uses default local driver,
 ## For custom volumes replace with volume driver configuration.
 volumes:
  edata1-1:
  edata1-2:
  edata1-3:
  edata1-4:
@@ -0,0 +1,117 @@
 version: '3.7'
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
  image: minio/minio:${JOB_NAME}
  command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2}
  expose:
    - "9000"
    - "9001"
  environment:
    MINIO_CI_CD: "on"
    MINIO_ROOT_USER: "minio"
    MINIO_ROOT_PASSWORD: "minio123"
    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
    interval: 30s
    timeout: 20s
    retries: 3
 # starts 4 docker containers running minio server instances.
 # using nginx reverse proxy, load balancing, you can access
 # it through port 9000.
 services:
  minio1:
    <<: *minio-common
    hostname: minio1
    volumes:
      - pdata1-1:/pdata1
      - pdata1-2:/pdata2
  minio2:
    <<: *minio-common
    hostname: minio2
    volumes:
      - pdata2-1:/pdata1
      - pdata2-2:/pdata2
  minio3:
    <<: *minio-common
    hostname: minio3
    volumes:
      - pdata3-1:/pdata1
      - pdata3-2:/pdata2
  minio4:
    <<: *minio-common
    hostname: minio4
    volumes:
      - pdata4-1:/pdata1
      - pdata4-2:/pdata2
  minio5:
    <<: *minio-common
    hostname: minio5
    volumes:
      - pdata5-1:/pdata1
      - pdata5-2:/pdata2
  minio6:
    <<: *minio-common
    hostname: minio6
    volumes:
      - pdata6-1:/pdata1
      - pdata6-2:/pdata2
  minio7:
    <<: *minio-common
    hostname: minio7
    volumes:
      - pdata7-1:/pdata1
      - pdata7-2:/pdata2
  minio8:
    <<: *minio-common
    hostname: minio8
    volumes:
      - pdata8-1:/pdata1
      - pdata8-2:/pdata2
  nginx:
    image: nginx:1.19.2-alpine
    hostname: nginx
    volumes:
      - ./nginx-8-node.conf:/etc/nginx/nginx.conf:ro
    ports:
      - "9000:9000"
      - "9001:9001"
    depends_on:
      - minio1
      - minio2
      - minio3
      - minio4
      - minio5
      - minio6
      - minio7
      - minio8
 ## By default this config uses default local driver,
 ## For custom volumes replace with volume driver configuration.
 volumes:
  pdata1-1:
  pdata1-2:
  pdata2-1:
  pdata2-2:
  pdata3-1:
  pdata3-2:
  pdata4-1:
  pdata4-2:
  pdata5-1:
  pdata5-2:
  pdata6-1:
  pdata6-2:
  pdata7-1:
  pdata7-2:
  pdata8-1:
  pdata8-2:
@@ -0,0 +1,100 @@
 user  nginx;
 worker_processes  auto;
 error_log  /var/log/nginx/error.log warn;
 pid        /var/run/nginx.pid;
 events {
    worker_connections  4096;
 }
 http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;
    # include /etc/nginx/conf.d/*.conf;
    upstream minio {
        server minio1:9000;
    }
    upstream console {
        ip_hash;
        server minio1:9001;
    }
    server {
        listen       9000;
        listen  [::]:9000;
        server_name  localhost;
        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;
        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;
            proxy_pass http://minio;
        }
    }
    server {
        listen       9001;
        listen  [::]:9001;
        server_name  localhost;
        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;
        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-NginX-Proxy true;
            # This is necessary to pass the correct IP to be hashed
            real_ip_header X-Real-IP;
            proxy_connect_timeout 300;
            # To support websocket
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            chunked_transfer_encoding off;
            proxy_pass http://console;
        }
    }
 }
@@ -0,0 +1,106 @@
 user  nginx;
 worker_processes  auto;
 error_log  /var/log/nginx/error.log warn;
 pid        /var/run/nginx.pid;
 events {
    worker_connections  4096;
 }
 http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;
    # include /etc/nginx/conf.d/*.conf;
    upstream minio {
        server minio1:9000;
        server minio2:9000;
        server minio3:9000;
        server minio4:9000;
    }
    upstream console {
        ip_hash;
        server minio1:9001;
        server minio2:9001;
        server minio3:9001;
        server minio4:9001;
    }
    server {
        listen       9000;
        listen  [::]:9000;
        server_name  localhost;
        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;
        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;
            proxy_pass http://minio;
        }
    }
    server {
        listen       9001;
        listen  [::]:9001;
        server_name  localhost;
        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;
        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-NginX-Proxy true;
            # This is necessary to pass the correct IP to be hashed
            real_ip_header X-Real-IP;
            proxy_connect_timeout 300;
            # To support websocket
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            chunked_transfer_encoding off;
            proxy_pass http://console;
        }
    }
 }
@@ -0,0 +1,114 @@
 user  nginx;
 worker_processes  auto;
 error_log  /var/log/nginx/error.log warn;
 pid        /var/run/nginx.pid;
 events {
    worker_connections  4096;
 }
 http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;
    # include /etc/nginx/conf.d/*.conf;
    upstream minio {
        server minio1:9000;
        server minio2:9000;
        server minio3:9000;
        server minio4:9000;
        server minio5:9000;
        server minio6:9000;
        server minio7:9000;
        server minio8:9000;
    }
    upstream console {
        ip_hash;
        server minio1:9001;
        server minio2:9001;
        server minio3:9001;
        server minio4:9001;
        server minio5:9001;
        server minio6:9001;
        server minio7:9001;
        server minio8:9001;
    }
    server {
        listen       9000;
        listen  [::]:9000;
        server_name  localhost;
        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;
        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;
            proxy_pass http://minio;
        }
    }
    server {
        listen       9001;
        listen  [::]:9001;
        server_name  localhost;
        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;
        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-NginX-Proxy true;
            # This is necessary to pass the correct IP to be hashed
            real_ip_header X-Real-IP;
            proxy_connect_timeout 300;
            # To support websocket
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            chunked_transfer_encoding off;
            proxy_pass http://console;
        }
    }
 }
@@ -0,0 +1,106 @@
 user  nginx;
 worker_processes  auto;
 error_log  /var/log/nginx/error.log warn;
 pid        /var/run/nginx.pid;
 events {
    worker_connections  4096;
 }
 http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;
    # include /etc/nginx/conf.d/*.conf;
    upstream minio {
        server minio1:9000;
        server minio2:9000;
        server minio3:9000;
        server minio4:9000;
    }
    upstream console {
        ip_hash;
        server minio1:9001;
        server minio2:9001;
        server minio3:9001;
        server minio4:9001;
    }
    server {
        listen       9000;
        listen  [::]:9000;
        server_name  localhost;
        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;
        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_connect_timeout 300;
            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
            proxy_http_version 1.1;
            proxy_set_header Connection "";
            chunked_transfer_encoding off;
            proxy_pass http://minio;
        }
    }
    server {
        listen       9001;
        listen  [::]:9001;
        server_name  localhost;
        # To allow special characters in headers
        ignore_invalid_headers off;
        # Allow any size file to be uploaded.
        # Set to a value such as 1000m; to restrict file size to a specific value
        client_max_body_size 0;
        # To disable buffering
        proxy_buffering off;
        proxy_request_buffering off;
        location / {
            proxy_set_header Host $http_host;
            proxy_set_header X-Real-IP $remote_addr;
            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
            proxy_set_header X-Forwarded-Proto $scheme;
            proxy_set_header X-NginX-Proxy true;
            # This is necessary to pass the correct IP to be hashed
            real_ip_header X-Real-IP;
            proxy_connect_timeout 300;
            # To support websocket
            proxy_http_version 1.1;
            proxy_set_header Upgrade $http_upgrade;
            proxy_set_header Connection "upgrade";
            chunked_transfer_encoding off;
            proxy_pass http://console;
        }
    }
 }
@@ -0,0 +1,34 @@
 name: Root lockdown tests
 on:
  pull_request:
    branches:
    - master
 # This ensures that previous jobs for the PR are canceled when the PR is
 # updated.
 concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref }}
  cancel-in-progress: true
 permissions:
  contents: read
 jobs:
  build:
    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        go-version: [1.20.x]
        os: [ubuntu-latest]
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-go@v3
        with:
          go-version: ${{ matrix.go-version }}
          check-latest: true
      - name: Start root lockdown tests
        run: |
          make test-root-disable
@@ -0,0 +1,38 @@
 #!/bin/bash
 set -ex
 export MODE="$1"
 export ACCESS_KEY="$2"
 export SECRET_KEY="$3"
 export JOB_NAME="$4"
 export MINT_MODE="full"
 docker system prune -f || true
 docker volume prune -f || true
 docker volume rm $(docker volume ls -f dangling=true) || true
 ## change working directory
 cd .github/workflows/mint
 docker-compose -f minio-${MODE}.yaml up -d
 sleep 5m
 docker run --rm --net=host \
 	--name="mint-${MODE}-${JOB_NAME}" \
 	-e SERVER_ENDPOINT="127.0.0.1:9000" \
 	-e ACCESS_KEY="${ACCESS_KEY}" \
 	-e SECRET_KEY="${SECRET_KEY}" \
 	-e ENABLE_HTTPS=0 \
 	-e MINT_MODE="${MINT_MODE}" \
 	docker.io/minio/mint:edge
 docker-compose -f minio-${MODE}.yaml down || true
 sleep 10s
 docker system prune -f || true
 docker volume prune -f || true
 docker volume rm $(docker volume ls -f dangling=true) || true
 ## change working directory
 cd ../../../
@@ -0,0 +1,22 @@
 name: Shell formatting checks
 on:
  pull_request:
    branches:
    - master
 permissions:
  contents: read
 jobs:
  build:
    name: runner / shfmt
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: luizm/action-sh-checker@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SHFMT_OPTS: "-s"
        with:
          sh_checker_shellcheck_disable: true # disable for now
@@ -20,7 +20,7 @@ jobs:
      - name: Set up Go
        uses: actions/setup-go@v3
        with:
-          go-version: 1.20.x
+          go-version: 1.19.10
          check-latest: true
      - name: Get official govulncheck
        run: go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
 ARG RELEASE
@@ -27,9 +27,8 @@ COPY LICENSE /licenses/LICENSE
 RUN \
     microdnf clean all && \
     microdnf update --nodocs && \
     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install minisign --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE} -o /opt/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
 ARG TARGETARCH
@@ -29,9 +29,8 @@ COPY LICENSE /licenses/LICENSE
 RUN \
     microdnf clean all && \
     microdnf update --nodocs && \
     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools --nodocs && \
     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install minisign --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /opt/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8
 ARG TARGETARCH
@@ -29,9 +29,8 @@ COPY LICENSE /licenses/LICENSE
 RUN \
     microdnf clean all && \
     microdnf update --nodocs && \
     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install minisign --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /opt/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /opt/bin/minio.sha256sum && \
@@ -37,6 +37,10 @@ check-gen: ## check for updated autogenerated files
 	@(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false)
 lint: getdeps ## runs golangci-lint suite of linters
 	@echo "Running $@ check"
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml
 lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes
 	@echo "Running $@ check"
 	@$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix
@@ -45,6 +49,10 @@ test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
 	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...
 test-root-disable: install
 	@echo "Running minio root lockdown tests"
 	@env bash $(PWD)/buildscripts/disable-root.sh
 test-decom: install
 	@echo "Running minio decom tests"
 	@env bash $(PWD)/docs/distributed/decom.sh
@@ -66,6 +74,9 @@ test-iam: build ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
 	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 test-sio-error:
 	@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
 test-replication-2site:
 	@(env bash $(PWD)/docs/bucket/replication/setup_2site_existing_replication.sh)
@@ -75,7 +86,7 @@ test-replication-3site:
 test-delete-replication:
 	@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)
-test-replication: install test-replication-2site test-replication-3site test-delete-replication ## verify multi site replication
+test-replication: install test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
 	@echo "Running tests for replicating three sites"
 test-site-replication-ldap: install ## verify automatic site replication
@@ -86,8 +86,6 @@ chmod +x minio
 ./minio server /data
 ```
 Replace ``/data`` with the path to the drive or directory in which you want MinIO to store data.
 The following table lists supported architectures. Replace the `wget` URL with the architecture for your Linux host.
 | Architecture                   | URL                                                        |
@@ -3,19 +3,19 @@
 _init() {
-    shopt -s extglob
+	shopt -s extglob
-    ## Minimum required versions for build dependencies
+	## Minimum required versions for build dependencies
-    GIT_VERSION="1.0"
+	GIT_VERSION="1.0"
-    GO_VERSION="1.16"
+	GO_VERSION="1.16"
-    OSX_VERSION="10.8"
+	OSX_VERSION="10.8"
-    KNAME=$(uname -s)
+	KNAME=$(uname -s)
-    ARCH=$(uname -m)
+	ARCH=$(uname -m)
-    case "${KNAME}" in
+	case "${KNAME}" in
-        SunOS )
+	SunOS)
-            ARCH=$(isainfo -k)
+		ARCH=$(isainfo -k)
-            ;;
+		;;
-    esac
+	esac
 }
 ## FIXME:
@@ -28,24 +28,23 @@ _init() {
 ## }
 ##
 readlink() {
-    TARGET_FILE=$1
+	TARGET_FILE=$1
-    cd `dirname $TARGET_FILE`
+	cd $(dirname $TARGET_FILE)
-    TARGET_FILE=`basename $TARGET_FILE`
+	TARGET_FILE=$(basename $TARGET_FILE)
-    # Iterate down a (possible) chain of symlinks
+	# Iterate down a (possible) chain of symlinks
-    while [ -L "$TARGET_FILE" ]
+	while [ -L "$TARGET_FILE" ]; do
-    do
+		TARGET_FILE=$(env readlink $TARGET_FILE)
-        TARGET_FILE=$(env readlink $TARGET_FILE)
+		cd $(dirname $TARGET_FILE)
-        cd `dirname $TARGET_FILE`
+		TARGET_FILE=$(basename $TARGET_FILE)
-        TARGET_FILE=`basename $TARGET_FILE`
+	done
    done
-    # Compute the canonicalized name by finding the physical path
+	# Compute the canonicalized name by finding the physical path
-    # for the directory we're in and appending the target file.
+	# for the directory we're in and appending the target file.
-    PHYS_DIR=`pwd -P`
+	PHYS_DIR=$(pwd -P)
-    RESULT=$PHYS_DIR/$TARGET_FILE
+	RESULT=$PHYS_DIR/$TARGET_FILE
-    echo $RESULT
+	echo $RESULT
 }
 ## FIXME:
@@ -59,84 +58,86 @@ readlink() {
 ## }
 ##
 check_minimum_version() {
-    IFS='.' read -r -a varray1 <<< "$1"
+	IFS='.' read -r -a varray1 <<<"$1"
-    IFS='.' read -r -a varray2 <<< "$2"
+	IFS='.' read -r -a varray2 <<<"$2"
-    for i in "${!varray1[@]}"; do
+	for i in "${!varray1[@]}"; do
-        if [[ ${varray1[i]} -lt ${varray2[i]} ]]; then
+		if [[ ${varray1[i]} -lt ${varray2[i]} ]]; then
-            return 0
+			return 0
-        elif [[ ${varray1[i]} -gt ${varray2[i]} ]]; then
+		elif [[ ${varray1[i]} -gt ${varray2[i]} ]]; then
-            return 1
+			return 1
-        fi
+		fi
-    done
+	done
-    return 0
+	return 0
 }
 assert_is_supported_arch() {
-    case "${ARCH}" in
+	case "${ARCH}" in
-        x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x )
+	x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x | loong64 | loongarch64)
-            return
+		return
-            ;;
+		;;
-        *)
+	*)
-            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x]"
+		echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x, loong64, loongarch64]"
-            exit 1
+		exit 1
-    esac
+		;;
 	esac
 }
 assert_is_supported_os() {
-    case "${KNAME}" in
+	case "${KNAME}" in
-        Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS )
+	Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS)
-            return
+		return
-            ;;
+		;;
-        Darwin )
+	Darwin)
-            osx_host_version=$(env sw_vers -productVersion)
+		osx_host_version=$(env sw_vers -productVersion)
-            if ! check_minimum_version "${OSX_VERSION}" "${osx_host_version}"; then
+		if ! check_minimum_version "${OSX_VERSION}" "${osx_host_version}"; then
-                echo "OSX version '${osx_host_version}' is not supported. Minimum supported version: ${OSX_VERSION}"
+			echo "OSX version '${osx_host_version}' is not supported. Minimum supported version: ${OSX_VERSION}"
-                exit 1
+			exit 1
-            fi
+		fi
-            return
+		return
-            ;;
+		;;
-        *)
+	*)
-            echo "OS '${KNAME}' is not supported. Supported OS: [Linux, FreeBSD, OpenBSD, NetBSD, Darwin, DragonFly]"
+		echo "OS '${KNAME}' is not supported. Supported OS: [Linux, FreeBSD, OpenBSD, NetBSD, Darwin, DragonFly]"
-            exit 1
+		exit 1
-    esac
+		;;
 	esac
 }
 assert_check_golang_env() {
-    if ! which go >/dev/null 2>&1; then
+	if ! which go >/dev/null 2>&1; then
-        echo "Cannot find go binary in your PATH configuration, please refer to Go installation document at https://golang.org/doc/install"
+		echo "Cannot find go binary in your PATH configuration, please refer to Go installation document at https://golang.org/doc/install"
-        exit 1
+		exit 1
-    fi
+	fi
-    installed_go_version=$(go version | sed 's/^.* go\([0-9.]*\).*$/\1/')
+	installed_go_version=$(go version | sed 's/^.* go\([0-9.]*\).*$/\1/')
-    if ! check_minimum_version "${GO_VERSION}" "${installed_go_version}"; then
+	if ! check_minimum_version "${GO_VERSION}" "${installed_go_version}"; then
-        echo "Go runtime version '${installed_go_version}' is unsupported. Minimum supported version: ${GO_VERSION} to compile."
+		echo "Go runtime version '${installed_go_version}' is unsupported. Minimum supported version: ${GO_VERSION} to compile."
-        exit 1
+		exit 1
-    fi
+	fi
 }
 assert_check_deps() {
-    # support unusual Git versions such as: 2.7.4 (Apple Git-66)
+	# support unusual Git versions such as: 2.7.4 (Apple Git-66)
-    installed_git_version=$(git version | perl -ne '$_ =~ m/git version (.*?)( |$)/; print "$1\n";')
+	installed_git_version=$(git version | perl -ne '$_ =~ m/git version (.*?)( |$)/; print "$1\n";')
-    if ! check_minimum_version "${GIT_VERSION}" "${installed_git_version}"; then
+	if ! check_minimum_version "${GIT_VERSION}" "${installed_git_version}"; then
-        echo "Git version '${installed_git_version}' is not supported. Minimum supported version: ${GIT_VERSION}"
+		echo "Git version '${installed_git_version}' is not supported. Minimum supported version: ${GIT_VERSION}"
-        exit 1
+		exit 1
-    fi
+	fi
 }
 main() {
-    ## Check for supported arch
+	## Check for supported arch
-    assert_is_supported_arch
+	assert_is_supported_arch
-    ## Check for supported os
+	## Check for supported os
-    assert_is_supported_os
+	assert_is_supported_os
-    ## Check for Go environment
+	## Check for Go environment
-    assert_check_golang_env
+	assert_check_golang_env
-    ## Check for dependencies
+	## Check for dependencies
-    assert_check_deps
+	assert_check_deps
 }
 _init && main "$@"
@@ -5,33 +5,33 @@ set -e
 [ -n "$BASH_XTRACEFD" ] && set -x
 function _init() {
-    ## All binaries are static make sure to disable CGO.
+	## All binaries are static make sure to disable CGO.
-    export CGO_ENABLED=0
+	export CGO_ENABLED=0
-    ## List of architectures and OS to test coss compilation.
+	## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
+	SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips"
 }
 function _build() {
-    local osarch=$1
+	local osarch=$1
-    IFS=/ read -r -a arr <<<"$osarch"
+	IFS=/ read -r -a arr <<<"$osarch"
-    os="${arr[0]}"
+	os="${arr[0]}"
-    arch="${arr[1]}"
+	arch="${arr[1]}"
-    package=$(go list -f '{{.ImportPath}}')
+	package=$(go list -f '{{.ImportPath}}')
-    printf -- "--> %15s:%s\n" "${osarch}" "${package}"
+	printf -- "--> %15s:%s\n" "${osarch}" "${package}"
-    # go build -trimpath to build the binary.
+	# go build -trimpath to build the binary.
-    export GOOS=$os
+	export GOOS=$os
-    export GOARCH=$arch
+	export GOARCH=$arch
-    export GO111MODULE=on
+	export GO111MODULE=on
-    go build -trimpath -tags kqueue -o /dev/null
+	go build -trimpath -tags kqueue -o /dev/null
 }
 function main() {
-    echo "Testing builds for OS/Arch: ${SUPPORTED_OSARCH}"
+	echo "Testing builds for OS/Arch: ${SUPPORTED_OSARCH}"
-    for each_osarch in ${SUPPORTED_OSARCH}; do
+	for each_osarch in ${SUPPORTED_OSARCH}; do
-        _build "${each_osarch}"
+		_build "${each_osarch}"
-    done
+	done
 }
 _init && main "$@"
@@ -0,0 +1,119 @@
 #!/bin/bash
 set -x
 export MINIO_CI_CD=1
 killall -9 minio
 rm -rf ${HOME}/tmp/dist
 scheme="http"
 nr_servers=4
 addr="localhost"
 args=""
 for ((i = 0; i < $((nr_servers)); i++)); do
 	args="$args $scheme://$addr:$((9100 + i))/${HOME}/tmp/dist/path1/$i"
 done
 echo $args
 for ((i = 0; i < $((nr_servers)); i++)); do
 	(minio server --address ":$((9100 + i))" $args 2>&1 >/tmp/log$i.txt) &
 done
 sleep 10s
 if [ ! -f ./mc ]; then
 	wget --quiet -O ./mc https://dl.minio.io/client/mc/release/linux-amd64/./mc &&
 		chmod +x mc
 fi
 set +e
 export MC_HOST_minioadm=http://minioadmin:minioadmin@localhost:9100/
 ./mc ls minioadm/
 ./mc admin config set minioadm/ api root_access=off
 sleep 3s # let things settle a little
 ./mc ls minioadm/
 if [ $? -eq 0 ]; then
 	echo "listing succeeded, 'minioadmin' was not disabled"
 	exit 1
 fi
 set -e
 killall -9 minio
 export MINIO_API_ROOT_ACCESS=on
 for ((i = 0; i < $((nr_servers)); i++)); do
 	(minio server --address ":$((9100 + i))" $args 2>&1 >/tmp/log$i.txt) &
 done
 set +e
 ./mc ls minioadm/
 if [ $? -ne 0 ]; then
 	echo "listing failed, 'minioadmin' should be enabled"
 	exit 1
 fi
 killall -9 minio
 rm -rf /tmp/multisitea/
 rm -rf /tmp/multisiteb/
 echo "Setup site-replication and then disable root credentials"
 minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
 	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
 minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
 	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
 minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
 	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
 minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
 	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
 sleep 20s
 export MC_HOST_sitea=http://minioadmin:minioadmin@127.0.0.1:9001
 export MC_HOST_siteb=http://minioadmin:minioadmin@127.0.0.1:9004
 ./mc admin replicate add sitea siteb
 ./mc admin user add sitea foobar foo12345
 ./mc admin policy attach sitea/ consoleAdmin --user=foobar
 ./mc admin user info siteb foobar
 killall -9 minio
 echo "turning off root access, however site replication must continue"
 export MINIO_API_ROOT_ACCESS=off
 minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
 	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
 minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
 	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
 minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
 	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
 minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
 	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
 sleep 20s
 export MC_HOST_sitea=http://foobar:foo12345@127.0.0.1:9001
 export MC_HOST_siteb=http://foobar:foo12345@127.0.0.1:9004
 ./mc admin user add sitea foobar-admin foo12345
 sleep 2s
 ./mc admin user info siteb foobar-admin
@@ -6,88 +6,87 @@ set -x
 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
+	echo "minio executable binary not found in current directory"
-    exit 1
+	exit 1
 fi
 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
+	echo "minio executable binary not found in current directory"
-    exit 1
+	exit 1
 fi
 function start_minio_4drive() {
-    start_port=$1
+	start_port=$1
-    export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_USER=minio
-    export MINIO_ROOT_PASSWORD=minio123
+	export MINIO_ROOT_PASSWORD=minio123
-    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
-    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+	unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
-    export MINIO_CI_CD=1
+	export MINIO_CI_CD=1
-    mkdir ${WORK_DIR}
+	mkdir ${WORK_DIR}
-    C_PWD=${PWD}
+	C_PWD=${PWD}
-    if [ ! -x "$PWD/mc" ]; then
+	if [ ! -x "$PWD/mc" ]; then
-	MC_BUILD_DIR="mc-$RANDOM"
+		MC_BUILD_DIR="mc-$RANDOM"
-	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+		if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-	    echo "failed to download https://github.com/minio/mc"
+			echo "failed to download https://github.com/minio/mc"
-	    purge "${MC_BUILD_DIR}"
+			purge "${MC_BUILD_DIR}"
-	    exit 1
+			exit 1
 		fi
 		(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
 		# remove mc source.
 		purge "${MC_BUILD_DIR}"
 	fi
-	(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
+	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" >"${WORK_DIR}/server1.log" 2>&1 &
 	pid=$!
 	disown $pid
 	sleep 5
-	# remove mc source.
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
-	purge "${MC_BUILD_DIR}"
+		echo "server1 log:"
-    fi
+		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
 		purge "$WORK_DIR"
 		exit 1
 	fi
-    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" > "${WORK_DIR}/server1.log" 2>&1 &
+	"${PWD}/mc" mb --with-versioning minio/bucket
    pid=$!
    disown $pid
    sleep 5
-    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	for i in $(seq 1 4); do
-	echo "server1 log:"
+		"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	purge "$WORK_DIR"
 	exit 1
    fi
-    "${PWD}/mc" mb --with-versioning minio/bucket
+		sudo chown -R root. "${WORK_DIR}/disk${i}"
-    for i in $(seq 1 4); do
+		"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
 	"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
-	sudo chown -R root. "${WORK_DIR}/disk${i}"
+		sudo chown -R ${USER}. "${WORK_DIR}/disk${i}"
 	done
-	"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
+	for vid in $("${PWD}/mc" ls --json --versions minio/bucket/testobj | jq -r .versionId); do
 		"${PWD}/mc" cat --vid "${vid}" minio/bucket/testobj | md5sum
 	done
-	sudo chown -R ${USER}. "${WORK_DIR}/disk${i}"
+	pkill minio
-    done
+	sleep 3
    for vid in $("${PWD}/mc" ls --json --versions minio/bucket/testobj | jq -r .versionId); do
 	"${PWD}/mc" cat --vid "${vid}" minio/bucket/testobj | md5sum
    done
    pkill minio
    sleep 3
 }
 function main() {
-    start_port=$(shuf -i 10000-65000 -n 1)
+	start_port=$(shuf -i 10000-65000 -n 1)
-    start_minio_4drive ${start_port}
+	start_minio_4drive ${start_port}
 }
-function purge()
+function purge() {
-{
+	rm -rf "$1"
    rm -rf "$1"
 }
-( main "$@" )
+(main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -27,7 +27,7 @@ import (
 	"os"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 )
 func main() {
@@ -4,89 +4,89 @@ trap 'cleanup $LINENO' ERR
 # shellcheck disable=SC2120
 cleanup() {
-    MINIO_VERSION=dev docker-compose \
+	MINIO_VERSION=dev docker-compose \
-                 -f "buildscripts/upgrade-tests/compose.yml" \
+		-f "buildscripts/upgrade-tests/compose.yml" \
-                 rm -s -f
+		rm -s -f
-    docker volume prune -f
+	docker volume prune -f
 }
 verify_checksum_after_heal() {
-    local sum1
+	local sum1
-    sum1=$(curl -s "$2" | sha256sum);
+	sum1=$(curl -s "$2" | sha256sum)
-    mc admin heal --json -r "$1" >/dev/null; # test after healing
+	mc admin heal --json -r "$1" >/dev/null # test after healing
-    local sum1_heal
+	local sum1_heal
-    sum1_heal=$(curl -s "$2" | sha256sum);
+	sum1_heal=$(curl -s "$2" | sha256sum)
-    if [ "${sum1_heal}" != "${sum1}" ]; then
+	if [ "${sum1_heal}" != "${sum1}" ]; then
-        echo "mismatch expected ${sum1_heal}, got ${sum1}"
+		echo "mismatch expected ${sum1_heal}, got ${sum1}"
-        exit 1;
+		exit 1
-    fi
+	fi
 }
 verify_checksum_mc() {
-    local expected
+	local expected
-    expected=$(mc cat "$1" | sha256sum)
+	expected=$(mc cat "$1" | sha256sum)
-    local got
+	local got
-    got=$(mc cat "$2" | sha256sum)
+	got=$(mc cat "$2" | sha256sum)
-    if [ "${expected}" != "${got}" ]; then
+	if [ "${expected}" != "${got}" ]; then
-        echo "mismatch - expected ${expected}, got ${got}"
+		echo "mismatch - expected ${expected}, got ${got}"
-        exit 1;
+		exit 1
-    fi
+	fi
-    echo "matches - ${expected}, got ${got}"
+	echo "matches - ${expected}, got ${got}"
 }
 add_alias() {
-    for i in $(seq 1 4); do
+	for i in $(seq 1 4); do
-        echo "... attempting to add alias $i"
+		echo "... attempting to add alias $i"
-        until (mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin); do
+		until (mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin); do
-            echo "...waiting... for 5secs" && sleep 5
+			echo "...waiting... for 5secs" && sleep 5
-        done
+		done
-    done
+	done
-    echo "Sleeping for nginx"
+	echo "Sleeping for nginx"
-    sleep 20
+	sleep 20
 }
 __init__() {
-    sudo apt install curl -y
+	sudo apt install curl -y
-    export GOPATH=/tmp/gopath
+	export GOPATH=/tmp/gopath
-    export PATH=${PATH}:${GOPATH}/bin
+	export PATH=${PATH}:${GOPATH}/bin
-    go install github.com/minio/mc@latest
+	go install github.com/minio/mc@latest
-    TAG=minio/minio:dev make docker
+	TAG=minio/minio:dev make docker
-    MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \
+	MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \
-                 -f "buildscripts/upgrade-tests/compose.yml" \
+		-f "buildscripts/upgrade-tests/compose.yml" \
-                 up -d --build
+		up -d --build
-    add_alias
+	add_alias
-    mc mb minio/minio-test/
+	mc mb minio/minio-test/
-    mc cp ./minio minio/minio-test/to-read/
+	mc cp ./minio minio/minio-test/to-read/
-    mc cp /etc/hosts minio/minio-test/to-read/hosts
+	mc cp /etc/hosts minio/minio-test/to-read/hosts
-    mc anonymous set download minio/minio-test
+	mc anonymous set download minio/minio-test
-    verify_checksum_mc ./minio minio/minio-test/to-read/minio
+	verify_checksum_mc ./minio minio/minio-test/to-read/minio
-    curl -s http://127.0.0.1:9000/minio-test/to-read/hosts | sha256sum
+	curl -s http://127.0.0.1:9000/minio-test/to-read/hosts | sha256sum
-    MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop
+	MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop
 }
 main() {
-    MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build
+	MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build
-    add_alias
+	add_alias
-    verify_checksum_after_heal minio/minio-test http://127.0.0.1:9000/minio-test/to-read/hosts
+	verify_checksum_after_heal minio/minio-test http://127.0.0.1:9000/minio-test/to-read/hosts
-    verify_checksum_mc ./minio minio/minio-test/to-read/minio
+	verify_checksum_mc ./minio minio/minio-test/to-read/minio
-    verify_checksum_mc /etc/hosts minio/minio-test/to-read/hosts
+	verify_checksum_mc /etc/hosts minio/minio-test/to-read/hosts
-    cleanup
+	cleanup
 }
-( __init__ "$@" && main "$@" )
+(__init__ "$@" && main "$@")
@@ -6,5 +6,5 @@ export GORACE="history_size=7"
 export MINIO_API_REQUESTS_MAX=10000
 for d in $(go list ./...); do
-    CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
+	CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
 done
@@ -6,67 +6,66 @@ set -x
 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
+	echo "minio executable binary not found in current directory"
-    exit 1
+	exit 1
 fi
 function start_minio_5drive() {
-    start_port=$1
+	start_port=$1
-    export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_USER=minio
-    export MINIO_ROOT_PASSWORD=minio123
+	export MINIO_ROOT_PASSWORD=minio123
-    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
-    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+	unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
-    export MINIO_CI_CD=1
+	export MINIO_CI_CD=1
-    MC_BUILD_DIR="mc-$RANDOM"
+	MC_BUILD_DIR="mc-$RANDOM"
-    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-	echo "failed to download https://github.com/minio/mc"
+		echo "failed to download https://github.com/minio/mc"
 		purge "${MC_BUILD_DIR}"
 		exit 1
 	fi
 	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
 	# remove mc source.
 	purge "${MC_BUILD_DIR}"
 	exit 1
    fi
-    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+	"${WORK_DIR}/mc" cp --quiet -r "buildscripts/cicd-corpus/" "${WORK_DIR}/cicd-corpus/"
-    # remove mc source.
+	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" >"${WORK_DIR}/server1.log" 2>&1 &
-    purge "${MC_BUILD_DIR}"
+	pid=$!
 	disown $pid
 	sleep 5
-    "${WORK_DIR}/mc" cp --quiet -r "buildscripts/cicd-corpus/" "${WORK_DIR}/cicd-corpus/"
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
 		echo "server1 log:"
 		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
 		purge "$WORK_DIR"
 		exit 1
 	fi
-    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" > "${WORK_DIR}/server1.log" 2>&1 &
+	"${WORK_DIR}/mc" stat minio/bucket/testobj
    pid=$!
    disown $pid
    sleep 5
-    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	pkill minio
-	echo "server1 log:"
+	sleep 3
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	purge "$WORK_DIR"
 	exit 1
    fi
    "${WORK_DIR}/mc" stat minio/bucket/testobj
    pkill minio
    sleep 3
 }
 function main() {
-    start_port=$(shuf -i 10000-65000 -n 1)
+	start_port=$(shuf -i 10000-65000 -n 1)
-    start_minio_5drive ${start_port}
+	start_minio_5drive ${start_port}
 }
-function purge()
+function purge() {
-{
+	rm -rf "$1"
    rm -rf "$1"
 }
-( main "$@" )
+(main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -6,146 +6,151 @@ set -x
 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO_OLD=( "$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO_OLD=("$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server)
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
+	echo "minio executable binary not found in current directory"
-    exit 1
+	exit 1
 fi
 function download_old_release() {
-    if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then
+	if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then
-	curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z
+		curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z
-	chmod a+x minio.RELEASE.2020-10-28T08-16-50Z
+		chmod a+x minio.RELEASE.2020-10-28T08-16-50Z
-    fi
+	fi
 }
 function verify_rewrite() {
-    start_port=$1
+	start_port=$1
-    export MINIO_ACCESS_KEY=minio
+	export MINIO_ACCESS_KEY=minio
-    export MINIO_SECRET_KEY=minio123
+	export MINIO_SECRET_KEY=minio123
-    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
-    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+	unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
-    export MINIO_CI_CD=1
+	export MINIO_CI_CD=1
-    MC_BUILD_DIR="mc-$RANDOM"
+	MC_BUILD_DIR="mc-$RANDOM"
-    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-	echo "failed to download https://github.com/minio/mc"
+		echo "failed to download https://github.com/minio/mc"
 		purge "${MC_BUILD_DIR}"
 		exit 1
 	fi
 	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
 	# remove mc source.
 	purge "${MC_BUILD_DIR}"
 	exit 1
    fi
-    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+	"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
 	pid=$!
 	disown $pid
 	sleep 10
-    # remove mc source.
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
-    purge "${MC_BUILD_DIR}"
+		echo "server1 log:"
 		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
 		purge "$WORK_DIR"
 		exit 1
 	fi
-    "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+	"${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock
-    pid=$!
+	"${WORK_DIR}/mc" cp \
-    disown $pid
+		buildscripts/verify-build.sh \
-    sleep 10
+		minio/healing-rewrite-bucket/ \
 		--disable-multipart --quiet
-    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	"${WORK_DIR}/mc" cp \
-	echo "server1 log:"
+		buildscripts/verify-build.sh \
-	cat "${WORK_DIR}/server1.log"
+		minio/healing-rewrite-bucket/ \
-	echo "FAILED"
+		--disable-multipart --quiet
 	purge "$WORK_DIR"
 	exit 1
    fi
-    "${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock
+	"${WORK_DIR}/mc" cp \
-    "${WORK_DIR}/mc" cp \
+		buildscripts/verify-build.sh \
-		     buildscripts/verify-build.sh \
+		minio/healing-rewrite-bucket/ \
-		     minio/healing-rewrite-bucket/ \
+		--disable-multipart --quiet
 		     --disable-multipart --quiet
-    "${WORK_DIR}/mc" cp \
+	kill ${pid}
-		     buildscripts/verify-build.sh \
+	sleep 3
 		     minio/healing-rewrite-bucket/ \
 		     --disable-multipart --quiet
-    "${WORK_DIR}/mc" cp \
+	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
-		     buildscripts/verify-build.sh \
+	pid=$!
-		     minio/healing-rewrite-bucket/ \
+	disown $pid
-		     --disable-multipart --quiet
+	sleep 10
-    kill ${pid}
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
-    sleep 3
+		echo "server1 log:"
 		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
 		purge "$WORK_DIR"
 		exit 1
 	fi
-    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+	go build ./docs/debugging/s3-check-md5/
-    pid=$!
+	if ! ./s3-check-md5 \
-    disown $pid
+		-debug \
-    sleep 10
+		-versions \
 		-access-key minio \
 		-secret-key minio123 \
 		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
 		echo "server1 log:"
 		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
 		mkdir -p inspects
 		(
 			cd inspects
 			"${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**
 		)
-    if ! ps -p ${pid} 1>&2 >/dev/null; then
+		"${WORK_DIR}/mc" mb play/inspects
-	echo "server1 log:"
+		"${WORK_DIR}/mc" mirror inspects play/inspects
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	purge "$WORK_DIR"
 	exit 1
    fi
-    go build ./docs/debugging/s3-check-md5/
+		purge "$WORK_DIR"
-    if ! ./s3-check-md5 \
+		exit 1
-	 -debug \
+	fi
 	 -versions \
 	 -access-key minio \
 	 -secret-key minio123 \
 	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
 	echo "server1 log:"
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	mkdir -p inspects
 	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
-	"${WORK_DIR}/mc" mb play/inspects
+	go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
-	"${WORK_DIR}/mc" mirror inspects play/inspects
+	sleep 1
-	purge "$WORK_DIR"
+	if ! ./s3-check-md5 \
-	exit 1
+		-debug \
-    fi
+		-versions \
 		-access-key minio \
 		-secret-key minio123 \
 		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
 		echo "server1 log:"
 		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
 		mkdir -p inspects
 		(
 			cd inspects
 			"${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**
 		)
-    go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
+		"${WORK_DIR}/mc" mb play/inspects
-    sleep 1
+		"${WORK_DIR}/mc" mirror inspects play/inspects
-    if ! ./s3-check-md5 \
+		purge "$WORK_DIR"
-	 -debug \
+		exit 1
-	 -versions \
+	fi
 	 -access-key minio \
 	 -secret-key minio123 \
 	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
 	echo "server1 log:"
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	mkdir -p inspects
 	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
-	"${WORK_DIR}/mc" mb play/inspects
+	kill ${pid}
 	"${WORK_DIR}/mc" mirror inspects play/inspects
 	purge "$WORK_DIR"
 	exit 1
    fi
    kill ${pid}
 }
 function main() {
-    download_old_release
+	download_old_release
-    start_port=$(shuf -i 10000-65000 -n 1)
+	start_port=$(shuf -i 10000-65000 -n 1)
-    verify_rewrite ${start_port}
+	verify_rewrite ${start_port}
 }
-function purge()
+function purge() {
-{
+	rm -rf "$1"
    rm -rf "$1"
 }
-( main "$@" )
+(main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -6,167 +6,172 @@ set -o pipefail
 set -x
 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
+	echo "minio executable binary not found in current directory"
-    exit 1
+	exit 1
 fi
 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO_OLD=( "$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO_OLD=("$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server)
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
 function download_old_release() {
-    if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
+	if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
-	curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
+		curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
-	chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
+		chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
-    fi
+	fi
 }
 function start_minio_16drive() {
-    start_port=$1
+	start_port=$1
-    export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_USER=minio
-    export MINIO_ROOT_PASSWORD=minio123
+	export MINIO_ROOT_PASSWORD=minio123
-    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
-    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+	unset MINIO_KMS_AUTO_ENCRYPTION         # do not auto-encrypt objects
-    export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
+	export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
-    export MINIO_CI_CD=1
+	export MINIO_CI_CD=1
-    MC_BUILD_DIR="mc-$RANDOM"
+	MC_BUILD_DIR="mc-$RANDOM"
-    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-	echo "failed to download https://github.com/minio/mc"
+		echo "failed to download https://github.com/minio/mc"
 		purge "${MC_BUILD_DIR}"
 		exit 1
 	fi
 	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
 	# remove mc source.
 	purge "${MC_BUILD_DIR}"
 	exit 1
    fi
-    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+	"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
 	pid=$!
 	disown $pid
 	sleep 30
-    # remove mc source.
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
-    purge "${MC_BUILD_DIR}"
+		echo "server1 log:"
 		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
 		purge "$WORK_DIR"
 		exit 1
 	fi
-    "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+	shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
-    pid=$!
+	"${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
-    disown $pid
+	"${WORK_DIR}/mc" cp \
-    sleep 30
+		"${WORK_DIR}/unaligned" \
 		minio/healing-shard-bucket/unaligned \
 		--disable-multipart --quiet
-    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	## "unaligned" object name gets consistently distributed
-	echo "server1 log:"
+	## to disks in following distribution order
-	cat "${WORK_DIR}/server1.log"
+	##
-	echo "FAILED"
+	## NOTE: if you change the name make sure to change the
-	purge "$WORK_DIR"
+	## distribution order present here
-	exit 1
+	##
-    fi
+	## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
-    shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
+	## make sure to remove the "last" data shard
-    "${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
+	rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
-    "${WORK_DIR}/mc" cp \
+	sleep 10
-		     "${WORK_DIR}/unaligned" \
+	## Heal the shard
-		     minio/healing-shard-bucket/unaligned \
+	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
-		     --disable-multipart --quiet
+	## then remove any other data shard let's pick first disk
 	## - 1st data shard.
 	rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
 	sleep 10
-    ## "unaligned" object name gets consistently distributed
+	go build ./docs/debugging/s3-check-md5/
-    ## to disks in following distribution order
+	if ! ./s3-check-md5 \
-    ##
+		-debug \
-    ## NOTE: if you change the name make sure to change the
+		-access-key minio \
-    ## distribution order present here
+		-secret-key minio123 \
-    ##
+		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
-    ## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+		echo "server1 log:"
 		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
 		purge "$WORK_DIR"
 		exit 1
 	fi
-    ## make sure to remove the "last" data shard
+	pkill minio
-    rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
+	sleep 3
    sleep 10
    ## Heal the shard
    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
    ## then remove any other data shard let's pick first disk
    ## - 1st data shard.
    rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
    sleep 10
-    go build ./docs/debugging/s3-check-md5/
+	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
-    if ! ./s3-check-md5 \
+	pid=$!
-	 -debug \
+	disown $pid
-	 -access-key minio \
+	sleep 30
 	 -secret-key minio123 \
 	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
 	echo "server1 log:"
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	purge "$WORK_DIR"
 	exit 1
    fi
-    pkill minio
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
-    sleep 3
+		echo "server1 log:"
 		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
 		purge "$WORK_DIR"
 		exit 1
 	fi
-    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+	if ! ./s3-check-md5 \
-    pid=$!
+		-debug \
-    disown $pid
+		-access-key minio \
-    sleep 30
+		-secret-key minio123 \
 		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
 		echo "server1 log:"
 		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
 		mkdir -p inspects
 		(
 			cd inspects
 			"${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**
 		)
-    if ! ps -p ${pid} 1>&2 >/dev/null; then
+		"${WORK_DIR}/mc" mb play/inspects
-	echo "server1 log:"
+		"${WORK_DIR}/mc" mirror inspects play/inspects
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	purge "$WORK_DIR"
 	exit 1
    fi
-    if ! ./s3-check-md5 \
+		purge "$WORK_DIR"
-	 -debug \
+		exit 1
-	 -access-key minio \
+	fi
 	 -secret-key minio123 \
 	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
 	echo "server1 log:"
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	mkdir -p inspects
 	(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
-	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
 	"${WORK_DIR}/mc" mirror inspects play/inspects
-	purge "$WORK_DIR"
+	if ! ./s3-check-md5 \
-	exit 1
+		-debug \
-    fi
+		-access-key minio \
 		-secret-key minio123 \
 		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
 		echo "server1 log:"
 		cat "${WORK_DIR}/server1.log"
 		echo "FAILED"
 		mkdir -p inspects
 		(
 			cd inspects
 			"${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**
 		)
-    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
+		"${WORK_DIR}/mc" mb play/inspects
 		"${WORK_DIR}/mc" mirror inspects play/inspects
-    if ! ./s3-check-md5 \
+		purge "$WORK_DIR"
-	 -debug \
+		exit 1
-	 -access-key minio \
+	fi
 	 -secret-key minio123 \
 	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
 	echo "server1 log:"
 	cat "${WORK_DIR}/server1.log"
 	echo "FAILED"
 	mkdir -p inspects
 	(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
-	"${WORK_DIR}/mc" mb play/inspects
+	pkill minio
-	"${WORK_DIR}/mc" mirror inspects play/inspects
+	sleep 3
 	purge "$WORK_DIR"
 	exit 1
    fi
    pkill minio
    sleep 3
 }
 function main() {
-    download_old_release
+	download_old_release
-    start_port=$(shuf -i 10000-65000 -n 1)
+	start_port=$(shuf -i 10000-65000 -n 1)
-    start_minio_16drive ${start_port}
+	start_minio_16drive ${start_port}
 }
-function purge()
+function purge() {
-{
+	rm -rf "$1"
    rm -rf "$1"
 }
-( main "$@" )
+(main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -6,8 +6,8 @@ set -E
 set -o pipefail
 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
+	echo "minio executable binary not found in current directory"
-    exit 1
+	exit 1
 fi
 WORK_DIR="$PWD/.verify-$RANDOM"
@@ -25,285 +25,270 @@ export ENABLE_ADMIN=1
 export MINIO_CI_CD=1
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR")
 FILE_1_MB="$MINT_DATA_DIR/datafile-1-MB"
 FILE_65_MB="$MINT_DATA_DIR/datafile-65-MB"
 FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh"
-function start_minio_fs()
+function start_minio_fs() {
-{
+	export MINIO_ROOT_USER=$ACCESS_KEY
-    export MINIO_ROOT_USER=$ACCESS_KEY
+	export MINIO_ROOT_PASSWORD=$SECRET_KEY
-    export MINIO_ROOT_PASSWORD=$SECRET_KEY
+	"${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
-    "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
+	sleep 10
    sleep 10
 }
-function start_minio_erasure()
+function start_minio_erasure() {
-{
+	"${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 &
-    "${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 &
+	sleep 15
    sleep 15
 }
-function start_minio_erasure_sets()
+function start_minio_erasure_sets() {
-{
+	export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}"
-    export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}"
+	"${MINIO[@]}" server >"$WORK_DIR/erasure-minio-sets.log" 2>&1 &
-    "${MINIO[@]}" server > "$WORK_DIR/erasure-minio-sets.log" 2>&1 &
+	sleep 15
    sleep 15
 }
-function start_minio_pool_erasure_sets()
+function start_minio_pool_erasure_sets() {
-{
+	export MINIO_ROOT_USER=$ACCESS_KEY
-    export MINIO_ROOT_USER=$ACCESS_KEY
+	export MINIO_ROOT_PASSWORD=$SECRET_KEY
-    export MINIO_ROOT_PASSWORD=$SECRET_KEY
+	export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}"
-    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}"
+	"${MINIO[@]}" server --address ":9000" >"$WORK_DIR/pool-minio-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address ":9000" > "$WORK_DIR/pool-minio-9000.log" 2>&1 &
+	"${MINIO[@]}" server --address ":9001" >"$WORK_DIR/pool-minio-9001.log" 2>&1 &
    "${MINIO[@]}" server --address ":9001" > "$WORK_DIR/pool-minio-9001.log" 2>&1 &
-    sleep 40
+	sleep 40
 }
-function start_minio_pool_erasure_sets_ipv6()
+function start_minio_pool_erasure_sets_ipv6() {
-{
+	export MINIO_ROOT_USER=$ACCESS_KEY
-    export MINIO_ROOT_USER=$ACCESS_KEY
+	export MINIO_ROOT_PASSWORD=$SECRET_KEY
-    export MINIO_ROOT_PASSWORD=$SECRET_KEY
+	export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}"
-    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}"
+	"${MINIO[@]}" server --address="[::1]:9000" >"$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
+	"${MINIO[@]}" server --address="[::1]:9001" >"$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
    "${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
-    sleep 40
+	sleep 40
 }
-function start_minio_dist_erasure()
+function start_minio_dist_erasure() {
-{
+	export MINIO_ROOT_USER=$ACCESS_KEY
-    export MINIO_ROOT_USER=$ACCESS_KEY
+	export MINIO_ROOT_PASSWORD=$SECRET_KEY
-    export MINIO_ROOT_PASSWORD=$SECRET_KEY
+	export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4"
-    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4"
+	for i in $(seq 0 3); do
-    for i in $(seq 0 3); do
+		"${MINIO[@]}" server --address ":900${i}" >"$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
-        "${MINIO[@]}" server --address ":900${i}" > "$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
+	done
    done
-    sleep 40
+	sleep 40
 }
-function run_test_fs()
+function run_test_fs() {
-{
+	start_minio_fs
    start_minio_fs
-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	rv=$?
-    pkill minio
+	pkill minio
-    sleep 3
+	sleep 3
-    if [ "$rv" -ne 0 ]; then
+	if [ "$rv" -ne 0 ]; then
-        cat "$WORK_DIR/fs-minio.log"
+		cat "$WORK_DIR/fs-minio.log"
-    fi
+	fi
-    rm -f "$WORK_DIR/fs-minio.log"
+	rm -f "$WORK_DIR/fs-minio.log"
-    return "$rv"
+	return "$rv"
 }
-function run_test_erasure_sets()
+function run_test_erasure_sets() {
-{
+	start_minio_erasure_sets
    start_minio_erasure_sets
-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	rv=$?
-    pkill minio
+	pkill minio
-    sleep 3
+	sleep 3
-    if [ "$rv" -ne 0 ]; then
+	if [ "$rv" -ne 0 ]; then
-        cat "$WORK_DIR/erasure-minio-sets.log"
+		cat "$WORK_DIR/erasure-minio-sets.log"
-    fi
+	fi
-    rm -f "$WORK_DIR/erasure-minio-sets.log"
+	rm -f "$WORK_DIR/erasure-minio-sets.log"
-    return "$rv"
+	return "$rv"
 }
-function run_test_pool_erasure_sets()
+function run_test_pool_erasure_sets() {
-{
+	start_minio_pool_erasure_sets
    start_minio_pool_erasure_sets
-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	rv=$?
-    pkill minio
+	pkill minio
-    sleep 3
+	sleep 3
-    if [ "$rv" -ne 0 ]; then
+	if [ "$rv" -ne 0 ]; then
-        for i in $(seq 0 1); do
+		for i in $(seq 0 1); do
-            echo "server$i log:"
+			echo "server$i log:"
-            cat "$WORK_DIR/pool-minio-900$i.log"
+			cat "$WORK_DIR/pool-minio-900$i.log"
-        done
+		done
-    fi
+	fi
-    for i in $(seq 0 1); do
+	for i in $(seq 0 1); do
-        rm -f "$WORK_DIR/pool-minio-900$i.log"
+		rm -f "$WORK_DIR/pool-minio-900$i.log"
-    done
+	done
-    return "$rv"
+	return "$rv"
 }
-function run_test_pool_erasure_sets_ipv6()
+function run_test_pool_erasure_sets_ipv6() {
-{
+	start_minio_pool_erasure_sets_ipv6
    start_minio_pool_erasure_sets_ipv6
-    export SERVER_ENDPOINT="[::1]:9000"
+	export SERVER_ENDPOINT="[::1]:9000"
-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	rv=$?
-    pkill minio
+	pkill minio
-    sleep 3
+	sleep 3
-    if [ "$rv" -ne 0 ]; then
+	if [ "$rv" -ne 0 ]; then
-        for i in $(seq 0 1); do
+		for i in $(seq 0 1); do
-            echo "server$i log:"
+			echo "server$i log:"
-            cat "$WORK_DIR/pool-minio-ipv6-900$i.log"
+			cat "$WORK_DIR/pool-minio-ipv6-900$i.log"
-        done
+		done
-    fi
+	fi
-    for i in $(seq 0 1); do
+	for i in $(seq 0 1); do
-        rm -f "$WORK_DIR/pool-minio-ipv6-900$i.log"
+		rm -f "$WORK_DIR/pool-minio-ipv6-900$i.log"
-    done
+	done
-    return "$rv"
+	return "$rv"
 }
-function run_test_erasure()
+function run_test_erasure() {
-{
+	start_minio_erasure
    start_minio_erasure
-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	rv=$?
-    pkill minio
+	pkill minio
-    sleep 3
+	sleep 3
-    if [ "$rv" -ne 0 ]; then
+	if [ "$rv" -ne 0 ]; then
-        cat "$WORK_DIR/erasure-minio.log"
+		cat "$WORK_DIR/erasure-minio.log"
-    fi
+	fi
-    rm -f "$WORK_DIR/erasure-minio.log"
+	rm -f "$WORK_DIR/erasure-minio.log"
-    return "$rv"
+	return "$rv"
 }
-function run_test_dist_erasure()
+function run_test_dist_erasure() {
-{
+	start_minio_dist_erasure
    start_minio_dist_erasure
-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	rv=$?
-    pkill minio
+	pkill minio
-    sleep 3
+	sleep 3
-    if [ "$rv" -ne 0 ]; then
+	if [ "$rv" -ne 0 ]; then
-        echo "server1 log:"
+		echo "server1 log:"
-        cat "$WORK_DIR/dist-minio-9000.log"
+		cat "$WORK_DIR/dist-minio-9000.log"
-        echo "server2 log:"
+		echo "server2 log:"
-        cat "$WORK_DIR/dist-minio-9001.log"
+		cat "$WORK_DIR/dist-minio-9001.log"
-        echo "server3 log:"
+		echo "server3 log:"
-        cat "$WORK_DIR/dist-minio-9002.log"
+		cat "$WORK_DIR/dist-minio-9002.log"
-        echo "server4 log:"
+		echo "server4 log:"
-        cat "$WORK_DIR/dist-minio-9003.log"
+		cat "$WORK_DIR/dist-minio-9003.log"
-    fi
+	fi
-    rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log"
+	rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log"
-    return "$rv"
+	return "$rv"
 }
-function purge()
+function purge() {
-{
+	rm -rf "$1"
    rm -rf "$1"
 }
-function __init__()
+function __init__() {
-{
+	echo "Initializing environment"
-    echo "Initializing environment"
+	mkdir -p "$WORK_DIR"
-    mkdir -p "$WORK_DIR"
+	mkdir -p "$MINIO_CONFIG_DIR"
-    mkdir -p "$MINIO_CONFIG_DIR"
+	mkdir -p "$MINT_DATA_DIR"
    mkdir -p "$MINT_DATA_DIR"
-    MC_BUILD_DIR="mc-$RANDOM"
+	MC_BUILD_DIR="mc-$RANDOM"
-    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-        echo "failed to download https://github.com/minio/mc"
+		echo "failed to download https://github.com/minio/mc"
-        purge "${MC_BUILD_DIR}"
+		purge "${MC_BUILD_DIR}"
-        exit 1
+		exit 1
-    fi
+	fi
-    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
-    # remove mc source.
+	# remove mc source.
-    purge "${MC_BUILD_DIR}"
+	purge "${MC_BUILD_DIR}"
-    shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null
+	shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null
-    shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
+	shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
-    ## version is purposefully set to '3' for minio to migrate configuration file
+	## version is purposefully set to '3' for minio to migrate configuration file
-    echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
+	echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json"
-    if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then
+	if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then
-        echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh"
+		echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh"
-        exit 1
+		exit 1
-    fi
+	fi
-    sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS"
+	sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS"
-    chmod a+x "$FUNCTIONAL_TESTS"
+	chmod a+x "$FUNCTIONAL_TESTS"
 }
-function main()
+function main() {
-{
+	echo "Testing in FS setup"
-    echo "Testing in FS setup"
+	if ! run_test_fs; then
-    if ! run_test_fs; then
+		echo "FAILED"
-        echo "FAILED"
+		purge "$WORK_DIR"
-        purge "$WORK_DIR"
+		exit 1
-        exit 1
+	fi
    fi
-    echo "Testing in Erasure setup"
+	echo "Testing in Erasure setup"
-    if ! run_test_erasure; then
+	if ! run_test_erasure; then
-        echo "FAILED"
+		echo "FAILED"
-        purge "$WORK_DIR"
+		purge "$WORK_DIR"
-        exit 1
+		exit 1
-    fi
+	fi
-    echo "Testing in Distributed Erasure setup"
+	echo "Testing in Distributed Erasure setup"
-    if ! run_test_dist_erasure; then
+	if ! run_test_dist_erasure; then
-        echo "FAILED"
+		echo "FAILED"
-        purge "$WORK_DIR"
+		purge "$WORK_DIR"
-        exit 1
+		exit 1
-    fi
+	fi
-    echo "Testing in Erasure setup as sets"
+	echo "Testing in Erasure setup as sets"
-    if ! run_test_erasure_sets; then
+	if ! run_test_erasure_sets; then
-        echo "FAILED"
+		echo "FAILED"
-        purge "$WORK_DIR"
+		purge "$WORK_DIR"
-        exit 1
+		exit 1
-    fi
+	fi
-    echo "Testing in Distributed Eraure expanded setup"
+	echo "Testing in Distributed Eraure expanded setup"
-    if ! run_test_pool_erasure_sets; then
+	if ! run_test_pool_erasure_sets; then
-        echo "FAILED"
+		echo "FAILED"
-        purge "$WORK_DIR"
+		purge "$WORK_DIR"
-        exit 1
+		exit 1
-    fi
+	fi
-    echo "Testing in Distributed Erasure expanded setup with ipv6"
+	echo "Testing in Distributed Erasure expanded setup with ipv6"
-    if ! run_test_pool_erasure_sets_ipv6; then
+	if ! run_test_pool_erasure_sets_ipv6; then
-        echo "FAILED"
+		echo "FAILED"
-        purge "$WORK_DIR"
+		purge "$WORK_DIR"
-        exit 1
+		exit 1
-    fi
+	fi
-    purge "$WORK_DIR"
+	purge "$WORK_DIR"
 }
-( __init__ "$@" && main "$@" )
+(__init__ "$@" && main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -4,94 +4,93 @@ set -E
 set -o pipefail
 set -x
 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
+	echo "minio executable binary not found in current directory"
-    exit 1
+	exit 1
 fi
 WORK_DIR="$(mktemp -d)"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
 function start_minio() {
-    start_port=$1
+	start_port=$1
-    export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_USER=minio
-    export MINIO_ROOT_PASSWORD=minio123
+	export MINIO_ROOT_PASSWORD=minio123
-    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+	unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
-    unset MINIO_CI_CD
+	unset MINIO_CI_CD
-    unset CI
+	unset CI
-    args=()
+	args=()
-    for i in $(seq 1 4); do
+	for i in $(seq 1 4); do
-            args+=("http://localhost:$[${start_port}+$i]${WORK_DIR}/mnt/disk$i/ ")
+		args+=("http://localhost:$((start_port + i))${WORK_DIR}/mnt/disk$i/ ")
-    done
+	done
-    for i in $(seq 1 4); do
+	for i in $(seq 1 4); do
-            "${MINIO[@]}" --address ":$[$start_port+$i]" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" &
+		"${MINIO[@]}" --address ":$((start_port + i))" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" &
-    done
+	done
-    # Wait until all nodes return 403
+	# Wait until all nodes return 403
-    for i in $(seq 1 4); do
+	for i in $(seq 1 4); do
-	    while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$[$start_port+$i])" -ne "403" ]; do
+		while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$((start_port + i)))" -ne "403" ]; do
-		    echo -n ".";
+			echo -n "."
-                    sleep 1;
+			sleep 1
-            done
+		done
-    done
+	done
- }
+}
 # Prepare fake disks with losetup
 function prepare_block_devices() {
-        mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
+	set -e
-        for i in 1 2 3 4; do
+	mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
-                dd if=/dev/zero of=${WORK_DIR}/disks/img.$i bs=1M count=2048
+	sudo modprobe loop
-                mkfs.ext4 -F ${WORK_DIR}/disks/img.$i
+	for i in 1 2 3 4; do
-                sudo mknod /dev/minio-loopdisk$i b 7 $[256-$i]
+		dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000
-                sudo losetup /dev/minio-loopdisk$i ${WORK_DIR}/disks/img.$i
+		device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i})
-                mkdir -p ${WORK_DIR}/mnt/disk$i/
+		sudo mkfs.ext4 -F ${device}
-                sudo mount /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
+		mkdir -p ${WORK_DIR}/mnt/disk${i}/
-                sudo chown "$(id -u):$(id -g)" /dev/minio-loopdisk$i ${WORK_DIR}/mnt/disk$i/
+		sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/
-        done
+		sudo chown "$(id -u):$(id -g)" ${device} ${WORK_DIR}/mnt/disk${i}/
 	done
 	set +e
 }
 # Start a distributed MinIO setup, unmount one disk and check if it is formatted
 function main() {
-    start_port=$(shuf -i 10000-65000 -n 1)
+	start_port=$(shuf -i 10000-65000 -n 1)
-    start_minio ${start_port}
+	start_minio ${start_port}
-    # Unmount the disk, after the unmount the device id
+	# Unmount the disk, after the unmount the device id
-    # /tmp/xxx/mnt/disk4 will be the same as '/' and it
+	# /tmp/xxx/mnt/disk4 will be the same as '/' and it
-    # will be detected as root disk
+	# will be detected as root disk
-    while [ "$u" != "0" ]; do
+	while [ "$u" != "0" ]; do
-	    sudo umount ${WORK_DIR}/mnt/disk4/
+		sudo umount ${WORK_DIR}/mnt/disk4/
-	    u=$?
+		u=$?
-	    sleep 1
+		sleep 1
-    done
+	done
-    # Wait until MinIO self heal kicks in
+	# Wait until MinIO self heal kicks in
-    sleep 60
+	sleep 60
-    if [ -f ${WORK_DIR}/mnt/disk4/.minio.sys/format.json ]; then
+	if [ -f ${WORK_DIR}/mnt/disk4/.minio.sys/format.json ]; then
-	    echo "A root disk is formatted unexpectedely"
+		echo "A root disk is formatted unexpectedely"
-            cat "${WORK_DIR}/server4.log"
+		cat "${WORK_DIR}/server4.log"
-            exit -1
+		exit -1
-    fi
+	fi
 }
 function cleanup() {
-    pkill minio
+	pkill minio
-    sudo umount ${WORK_DIR}/mnt/disk{1..3}/
+	sudo umount ${WORK_DIR}/mnt/disk{1..3}/
-    sudo rm /dev/minio-loopdisk*
+	sudo rm /dev/minio-loopdisk*
-    rm -rf "$WORK_DIR"
+	rm -rf "$WORK_DIR"
 }
-( prepare_block_devices )
+(prepare_block_devices)
-( main "$@" )
+(main "$@")
 rv=$?
 cleanup
 exit "$rv"
@@ -5,139 +5,135 @@ set -E
 set -o pipefail
 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
+	echo "minio executable binary not found in current directory"
-    exit 1
+	exit 1
 fi
 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)
 function start_minio_3_node() {
-    export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_USER=minio
-    export MINIO_ROOT_PASSWORD=minio123
+	export MINIO_ROOT_PASSWORD=minio123
-    export MINIO_ERASURE_SET_DRIVE_COUNT=6
+	export MINIO_ERASURE_SET_DRIVE_COUNT=6
-    export MINIO_CI_CD=1
+	export MINIO_CI_CD=1
-    start_port=$2
+	start_port=$2
-    args=""
+	args=""
    for i in $(seq 1 3); do
 	args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
    done
    "${MINIO[@]}" --address ":$[$start_port+1]" $args > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
    pid1=$!
    disown ${pid1}
    "${MINIO[@]}" --address ":$[$start_port+2]" $args > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
    pid2=$!
    disown $pid2
    "${MINIO[@]}" --address ":$[$start_port+3]" $args > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
    pid3=$!
    disown $pid3
    sleep "$1"
    if ! ps -p $pid1 1>&2 > /dev/null; then
 	echo "server1 log:"
 	cat "${WORK_DIR}/dist-minio-server1.log"
 	echo "FAILED"
 	purge "$WORK_DIR"
 	exit 1
    fi
    if ! ps -p $pid2 1>&2 > /dev/null; then
 	echo "server2 log:"
 	cat "${WORK_DIR}/dist-minio-server2.log"
 	echo "FAILED"
 	purge "$WORK_DIR"
 	exit 1
    fi
    if ! ps -p $pid3 1>&2 > /dev/null; then
 	echo "server3 log:"
 	cat "${WORK_DIR}/dist-minio-server3.log"
 	echo "FAILED"
 	purge "$WORK_DIR"
 	exit 1
    fi
    if ! pkill minio; then
 	for i in $(seq 1 3); do
-	    echo "server$i log:"
+		args="$args http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/1/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/2/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/3/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/4/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/5/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/6/"
 	    cat "${WORK_DIR}/dist-minio-server$i.log"
 	done
 	echo "FAILED"
 	purge "$WORK_DIR"
 	exit 1
    fi
-    sleep 1;
+	"${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 &
-    if pgrep minio; then
+	pid1=$!
-	# forcibly killing, to proceed further properly.
+	disown ${pid1}
-	if ! pkill -9 minio; then
+
-	    echo "no minio process running anymore, proceed."
+	"${MINIO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-minio-server2.log" 2>&1 &
 	pid2=$!
 	disown $pid2
 	"${MINIO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-minio-server3.log" 2>&1 &
 	pid3=$!
 	disown $pid3
 	sleep "$1"
 	if ! ps -p $pid1 1>&2 >/dev/null; then
 		echo "server1 log:"
 		cat "${WORK_DIR}/dist-minio-server1.log"
 		echo "FAILED"
 		purge "$WORK_DIR"
 		exit 1
 	fi
    fi
 }
 	if ! ps -p $pid2 1>&2 >/dev/null; then
 		echo "server2 log:"
 		cat "${WORK_DIR}/dist-minio-server2.log"
 		echo "FAILED"
 		purge "$WORK_DIR"
 		exit 1
 	fi
 	if ! ps -p $pid3 1>&2 >/dev/null; then
 		echo "server3 log:"
 		cat "${WORK_DIR}/dist-minio-server3.log"
 		echo "FAILED"
 		purge "$WORK_DIR"
 		exit 1
 	fi
 	if ! pkill minio; then
 		for i in $(seq 1 3); do
 			echo "server$i log:"
 			cat "${WORK_DIR}/dist-minio-server$i.log"
 		done
 		echo "FAILED"
 		purge "$WORK_DIR"
 		exit 1
 	fi
 	sleep 1
 	if pgrep minio; then
 		# forcibly killing, to proceed further properly.
 		if ! pkill -9 minio; then
 			echo "no minio process running anymore, proceed."
 		fi
 	fi
 }
 function check_online() {
-    if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
+	if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
-	echo "1"
+		echo "1"
-    fi
+	fi
 }
-function purge()
+function purge() {
-{
+	rm -rf "$1"
    rm -rf "$1"
 }
-function __init__()
+function __init__() {
-{
+	echo "Initializing environment"
-    echo "Initializing environment"
+	mkdir -p "$WORK_DIR"
-    mkdir -p "$WORK_DIR"
+	mkdir -p "$MINIO_CONFIG_DIR"
    mkdir -p "$MINIO_CONFIG_DIR"
-    ## version is purposefully set to '3' for minio to migrate configuration file
+	## version is purposefully set to '3' for minio to migrate configuration file
-    echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
+	echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json"
 }
 function perform_test() {
-    start_minio_3_node 120 $2
+	start_minio_3_node 120 $2
-    echo "Testing Distributed Erasure setup healing of drives"
+	echo "Testing Distributed Erasure setup healing of drives"
-    echo "Remove the contents of the disks belonging to '${1}' erasure set"
+	echo "Remove the contents of the disks belonging to '${1}' erasure set"
-    rm -rf ${WORK_DIR}/${1}/*/
+	rm -rf ${WORK_DIR}/${1}/*/
-    start_minio_3_node 120 $2
+	start_minio_3_node 120 $2
-    rv=$(check_online)
+	rv=$(check_online)
-    if [ "$rv" == "1" ]; then
+	if [ "$rv" == "1" ]; then
-	for i in $(seq 1 3); do
+		for i in $(seq 1 3); do
-	    echo "server$i log:"
+			echo "server$i log:"
-	    cat "${WORK_DIR}/dist-minio-server$i.log"
+			cat "${WORK_DIR}/dist-minio-server$i.log"
-	done
+		done
-	pkill -9 minio
+		pkill -9 minio
-	echo "FAILED"
+		echo "FAILED"
-	purge "$WORK_DIR"
+		purge "$WORK_DIR"
-	exit 1
+		exit 1
-    fi
+	fi
 }
-function main()
+function main() {
-{
+	# use same ports for all tests
-    # use same ports for all tests
+	start_port=$(shuf -i 10000-65000 -n 1)
    start_port=$(shuf -i 10000-65000 -n 1)
-    perform_test "2" ${start_port}
+	perform_test "2" ${start_port}
-    perform_test "1" ${start_port}
+	perform_test "1" ${start_port}
-    perform_test "3" ${start_port}
+	perform_test "3" ${start_port}
 }
-( __init__ "$@" && main "$@" )
+(__init__ "$@" && main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -91,7 +91,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 		acl := &accessControlPolicy{}
 		if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
 			if terr, ok := err.(*xml.SyntaxError); ok && terr.Msg == io.EOF.Error() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingSecurityHeader),
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML),
 					r.URL)
 				return
 			}
@@ -32,7 +32,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/klauspost/compress/zip"
 	"github.com/minio/kes-go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	objectlock "github.com/minio/minio/internal/bucket/object/lock"
@@ -85,11 +85,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}
 	if quotaConfig.Type == "fifo" {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
 	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -215,6 +210,11 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
 	if globalSiteReplicationSys.isEnabled() && !update {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetDenyAddError, err), r.URL)
 		return
 	}
 	if update {
 		// overlay the updates on existing target
 		tgt := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, target.Arn)
@@ -225,14 +225,14 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		for _, op := range ops {
 			switch op {
 			case madmin.CredentialsUpdateType:
-				if globalSiteReplicationSys.isEnabled() {
+				if !globalSiteReplicationSys.isEnabled() {
-					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetDenyEditError, err), r.URL)
+					// credentials update is possible only in bucket replication. User will never
-					return
+					// know the site replicator creds.
 					tgt.Credentials = target.Credentials
 					tgt.TargetBucket = target.TargetBucket
 					tgt.Secure = target.Secure
 					tgt.Endpoint = target.Endpoint
 				}
 				tgt.Credentials = target.Credentials
 				tgt.TargetBucket = target.TargetBucket
 				tgt.Secure = target.Secure
 				tgt.Endpoint = target.Endpoint
 			case madmin.SyncUpdateType:
 				tgt.ReplicationSync = target.ReplicationSync
 			case madmin.ProxyUpdateType:
@@ -460,7 +460,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					return
 				}
 			case bucketLifecycleConfig:
-				config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
+				config, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
 				if err != nil {
 					if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
 						continue
@@ -705,7 +705,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			}
 			if _, ok := bucketMap[bucket]; !ok {
 				opts := MakeBucketOptions{
-					LockEnabled: config.ObjectLockEnabled == "Enabled",
+					LockEnabled: config.Enabled(),
 				}
 				err = objectAPI.MakeBucket(ctx, bucket, opts)
 				if err != nil {
@@ -1032,11 +1032,6 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}
 			if quotaConfig.Type == "fifo" {
 				rpt.SetStatus(bucket, fileName, fmt.Errorf("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore"))
 				continue
 			}
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, err)
@@ -24,7 +24,7 @@ import (
 	"net/http"
 	"github.com/minio/kes-go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -168,10 +168,16 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			}
 		case errors.Is(err, errPolicyInUse):
 			apiErr = APIError{
-				Code:           "XMinioAdminPolicyInUse",
+				Code:           "XMinioIAMPolicyInUse",
 				Description:    "The policy cannot be removed, as it is in use",
 				HTTPStatusCode: http.StatusBadRequest,
 			}
 		case errors.Is(err, errSessionPolicyTooLarge):
 			apiErr = APIError{
 				Code:           "XMinioIAMServiceAccountSessionPolicyTooLarge",
 				Description:    err.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
 		case errors.Is(err, kes.ErrKeyExists):
 			apiErr = APIError{
 				Code:           "XMinioKMSKeyExists",
@@ -220,12 +226,10 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 // toAdminAPIErrCode - converts errErasureWriteQuorum error to admin API
 // specific error.
 func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
-	switch err {
+	if errors.Is(err, errErasureWriteQuorum) {
 	case errErasureWriteQuorum:
 		return ErrAdminConfigNoQuorum
 	default:
 		return toAPIErrorCode(ctx, err)
 	}
 	return toAPIErrorCode(ctx, err)
 }
 // wraps export error for more context
@@ -26,7 +26,7 @@ import (
 	"strconv"
 	"strings"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/cache"
 	"github.com/minio/minio/internal/config/etcd"
@@ -83,7 +83,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
-	if err = validateConfig(cfg, subSys); err != nil {
+	if err = validateConfig(ctx, cfg, subSys); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -211,7 +211,12 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
 		return
 	}
-	if verr := validateConfig(result.Cfg, result.SubSys); verr != nil {
+	tgts, err := config.ParseConfigTargetID(bytes.NewReader(kvBytes))
 	if err != nil {
 		return
 	}
 	ctx = context.WithValue(ctx, config.ContextKeyForTargetFromConfig, tgts)
 	if verr := validateConfig(ctx, result.Cfg, result.SubSys); verr != nil {
 		err = badConfigErr{Err: verr}
 		return
 	}
@@ -236,6 +241,8 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
 // 1. `subsys:target` -> request for config of a single subsystem and target pair.
 // 2. `subsys:` -> request for config of a single subsystem and the default target.
 // 3. `subsys` -> request for config of all targets for the given subsystem.
 //
 // This is a reporting API and config secrets are redacted in the response.
 func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfigKV")
@@ -263,7 +270,7 @@ func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		}
 	}
-	subSysConfigs, err := cfg.GetSubsysInfo(subSys, target)
+	subSysConfigs, err := cfg.GetSubsysInfo(subSys, target, true)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -353,7 +360,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}
-	if err = validateConfig(cfg, ""); err != nil {
+	if err = validateConfig(ctx, cfg, ""); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -464,7 +471,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
-	if err = validateConfig(cfg, ""); err != nil {
+	if err = validateConfig(ctx, cfg, ""); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -485,7 +492,9 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 }
 // GetConfigHandler - GET /minio/admin/v3/config
-// Get config.json of this minio setup.
+//
 // This endpoint is mainly for exporting and backing up the configuration.
 // Secrets are not redacted.
 func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "GetConfig")
@@ -502,7 +511,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 	hkvs := config.HelpSubSysMap[""]
 	for _, hkv := range hkvs {
 		// We ignore the error below, as we cannot get one.
-		cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "")
+		cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "", false)
 		for _, item := range cfgSubsysItems {
 			off := item.Config.Get(config.Enable) == config.EnableOff
@@ -541,7 +550,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 // setLoggerWebhookSubnetProxy - Sets the logger webhook's subnet proxy value to
 // one being set for subnet proxy
 func setLoggerWebhookSubnetProxy(subSys string, cfg config.Config) bool {
-	if subSys == config.SubnetSubSys {
+	if subSys == config.SubnetSubSys || subSys == config.LoggerWebhookSubSys {
 		subnetWebhookCfg := cfg[config.LoggerWebhookSubSys][subnet.LoggerWebhookName]
 		loggerWebhookSubnetProxy := subnetWebhookCfg.Get(logger.Proxy)
 		subnetProxy := cfg[config.SubnetSubSys][config.Default].Get(logger.Proxy)
@@ -26,7 +26,7 @@ import (
 	"net/http"
 	"strings"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	cfgldap "github.com/minio/minio/internal/config/identity/ldap"
@@ -125,7 +125,7 @@ func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.Resp
 		return
 	}
-	if err = validateConfig(cfg, subSys); err != nil {
+	if err = validateConfig(ctx, cfg, subSys); err != nil {
 		var validationErr ldap.Validation
 		if errors.As(err, &validationErr) {
@@ -422,7 +422,17 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	if err = validateConfig(cfg, subSys); err != nil {
+	if err = validateConfig(ctx, cfg, subSys); err != nil {
 		var validationErr ldap.Validation
 		if errors.As(err, &validationErr) {
 			// If we got an LDAP validation error, we need to send appropriate
 			// error message back to client (likely mc).
 			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPValidation),
 				validationErr.FormatError(), r.URL)
 			return
 		}
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -22,7 +22,7 @@ import (
 	"io"
 	"net/http"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -150,7 +150,7 @@ func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.
 	}
 	// Call IAM subsystem
-	updatedAt, addedOrRemoved, err := globalIAMSys.PolicyDBUpdateLDAP(ctx, isAttach, par)
+	updatedAt, addedOrRemoved, _, err := globalIAMSys.PolicyDBUpdateLDAP(ctx, isAttach, par)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -95,7 +95,7 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 		poolIndices = append(poolIndices, idx)
 	}
-	if len(poolIndices) > 0 && globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
+	if len(poolIndices) > 0 && !globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
 		ep := globalEndpoints[poolIndices[0]].Endpoints[0]
 		for nodeIdx, proxyEp := range globalProxyEndpoints {
 			if proxyEp.Endpoint.Host == ep.Host {
@@ -26,7 +26,7 @@ import (
 	"strings"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/mux"
 	"github.com/minio/minio/internal/logger"
@@ -30,9 +30,9 @@ import (
 	"testing"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	minio "github.com/minio/minio-go/v7"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 )
 func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
@@ -26,11 +26,10 @@ import (
 	"net/http"
 	"os"
 	"sort"
 	"strings"
 	"time"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config/dns"
 	"github.com/minio/minio/internal/logger"
@@ -44,7 +43,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -62,6 +61,13 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	// When the user is root credential you are not allowed to
 	// remove the root user. Also you cannot delete yourself.
 	if accessKey == globalActiveCred.AccessKey || accessKey == cred.AccessKey {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
 		return
 	}
 	if err := globalIAMSys.DeleteUser(ctx, accessKey, true); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -239,6 +245,26 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
 	// Reject if the group add and remove are temporary credentials, or root credential.
 	for _, member := range updReq.Members {
 		ok, _, err := globalIAMSys.IsTempUser(member)
 		if err != nil && err != errNoSuchUser {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
 		if ok {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
 			return
 		}
 		// When the user is root credential you are not allowed to
 		// add policies for root user.
 		if member == globalActiveCred.AccessKey {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
 			return
 		}
 	}
 	var updatedAt time.Time
 	if updReq.IsRemove {
 		updatedAt, err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
@@ -374,7 +400,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
+	objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -383,9 +409,9 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	accessKey := vars["accessKey"]
 	status := vars["status"]
-	// This API is not allowed to lookup master access key user status
+	// you cannot enable or disable yourself.
-	if accessKey == globalActiveCred.AccessKey {
+	if accessKey == creds.AccessKey {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
 		return
 	}
@@ -582,6 +608,7 @@ func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Re
 		AccountStatus: stsAccount.Status,
 		ImpliedPolicy: policy == nil,
 		Policy:        string(policyJSON),
 		Expiration:    &stsAccount.Expiration,
 	}
 	data, err := json.Marshal(infoResp)
@@ -637,6 +664,13 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 	if err := createReq.Validate(); err != nil {
 		// Since this validation would happen client side as well, we only send
 		// a generic error message here.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
 		return
 	}
 	var (
 		targetUser   string
 		targetGroups []string
@@ -649,12 +683,17 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		targetUser = cred.AccessKey
 	}
 	description := createReq.Description
 	if description == "" {
 		description = createReq.Comment
 	}
 	opts := newServiceAccountOpts{
-		accessKey:  createReq.AccessKey,
+		accessKey:   createReq.AccessKey,
-		secretKey:  createReq.SecretKey,
+		secretKey:   createReq.SecretKey,
-		comment:    createReq.Comment,
+		name:        createReq.Name,
-		expiration: createReq.Expiration,
+		description: description,
-		claims:     make(map[string]interface{}),
+		expiration:  createReq.Expiration,
 		claims:      make(map[string]interface{}),
 	}
 	// Find the user for the request sender (as it may be sent via a service
@@ -807,7 +846,8 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 					AccessKey:     newCred.AccessKey,
 					SecretKey:     newCred.SecretKey,
 					Groups:        newCred.Groups,
-					Comment:       newCred.Comment,
+					Name:          newCred.Name,
 					Description:   newCred.Description,
 					Claims:        opts.claims,
 					SessionPolicy: createReq.Policy,
 					Status:        auth.AccountOn,
@@ -882,6 +922,13 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 	if err := updateReq.Validate(); err != nil {
 		// Since this validation would happen client side as well, we only send
 		// a generic error message here.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
 		return
 	}
 	var sp *iampolicy.Policy
 	if len(updateReq.NewPolicy) > 0 {
 		sp, err = iampolicy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
@@ -893,7 +940,8 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 	opts := updateServiceAccountOpts{
 		secretKey:     updateReq.NewSecretKey,
 		status:        updateReq.NewStatus,
-		comment:       updateReq.NewComment,
+		name:          updateReq.NewName,
 		description:   updateReq.NewDescription,
 		expiration:    updateReq.NewExpiration,
 		sessionPolicy: sp,
 	}
@@ -912,7 +960,8 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 					AccessKey:     accessKey,
 					SecretKey:     opts.secretKey,
 					Status:        opts.status,
-					Comment:       opts.comment,
+					Name:          opts.name,
 					Description:   opts.description,
 					SessionPolicy: updateReq.NewPolicy,
 					Expiration:    updateReq.NewExpiration,
 				},
@@ -1000,7 +1049,8 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ
 	infoResp := madmin.InfoServiceAccountResp{
 		ParentUser:    svcAccount.ParentUser,
-		Comment:       svcAccount.Comment,
+		Name:          svcAccount.Name,
 		Description:   svcAccount.Description,
 		AccountStatus: svcAccount.Status,
 		ImpliedPolicy: policy == nil,
 		Policy:        string(policyJSON),
@@ -1072,14 +1122,18 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 		return
 	}
-	var serviceAccountsNames []string
+	var serviceAccountList []madmin.ServiceAccountInfo
 	for _, svc := range serviceAccounts {
-		serviceAccountsNames = append(serviceAccountsNames, svc.AccessKey)
+		expiryTime := svc.Expiration
 		serviceAccountList = append(serviceAccountList, madmin.ServiceAccountInfo{
 			AccessKey:  svc.AccessKey,
 			Expiration: &expiryTime,
 		})
 	}
 	listResp := madmin.ListServiceAccountsResp{
-		Accounts: serviceAccountsNames,
+		Accounts: serviceAccountList,
 	}
 	data, err := json.Marshal(listResp)
@@ -1626,6 +1680,12 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
 			return
 		}
 		// When the user is root credential you are not allowed to
 		// add policies for root user.
 		if entityName == globalActiveCred.AccessKey {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
 			return
 		}
 	}
 	// Validate that user or group exists.
@@ -1710,24 +1770,40 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht
 	writeSuccessResponseJSON(w, econfigData)
 }
-// AttachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/attach
+// AttachDetachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/policy/{operation}
-func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
+func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AttachPolicyBuiltin")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.UpdatePolicyAssociationAction,
 		iampolicy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
-	cred, _, s3Err := validateAdminSignature(ctx, r, "")
+	if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 {
-	if s3Err != ErrNone {
+		// More than maxConfigSize bytes were available
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL)
 		return
 	}
 	password := cred.SecretKey
 	// Ensure body content type is opaque to ensure that request body has not
 	// been interpreted as form data.
 	contentType := r.Header.Get("Content-Type")
 	if contentType != "application/octet-stream" {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL)
 		return
 	}
 	operation := mux.Vars(r)["operation"]
 	if operation != "attach" && operation != "detach" {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
 		return
 	}
 	isAttach := operation == "attach"
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1745,224 +1821,42 @@ func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 		return
 	}
-	userOrGroup := par.User
+	updatedAt, addedOrRemoved, _, err := globalIAMSys.PolicyDBUpdateBuiltin(ctx, isAttach, par)
-	isGroup := false
+	if err != nil {
-
+		if err == errNoSuchUser || err == errNoSuchGroup {
-	if userOrGroup == "" {
+			if globalIAMSys.LDAPConfig.Enabled() {
-		userOrGroup = par.Group
+				// When LDAP is enabled, warn user that they are using the wrong
-		isGroup = true
+				// API. FIXME: error can be no such group as well - fix errNoSuchUserLDAPWarn
-	}
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUserLDAPWarn), r.URL)
 	if isGroup {
 		_, err := globalIAMSys.GetGroupDescription(userOrGroup)
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
 	} else {
 		ok, _, err := globalIAMSys.IsTempUser(userOrGroup)
 		if err != nil && err != errNoSuchUser {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
 		if ok {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
 			return
 		}
 		// Validate that user exists.
 		if globalIAMSys.GetUsersSysType() == MinIOUsersSysType {
 			_, ok := globalIAMSys.GetUser(ctx, userOrGroup)
 			if !ok {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
 				return
 			}
 		}
 	}
 	userType := regUser
 	if globalIAMSys.GetUsersSysType() == LDAPUsersSysType {
 		userType = stsUser
 	}
 	var existingPolicies []string
 	if isGroup {
 		existingPolicies, err = globalIAMSys.PolicyDBGet(userOrGroup, true)
 	} else {
 		existingPolicies, err = globalIAMSys.GetUserPolicies(userOrGroup)
 	}
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	policyMap := make(map[string]bool)
+	respBody := madmin.PolicyAssociationResp{
 	for _, p := range existingPolicies {
 		policyMap[p] = true
 	}
 	policiesToAttach := par.Policies
 	// Check if policy is already attached to user.
 	for _, p := range policiesToAttach {
 		if _, ok := policyMap[p]; ok {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyAlreadyAttached), r.URL)
 			return
 		}
 	}
 	existingPolicies = append(existingPolicies, policiesToAttach...)
 	newPolicies := strings.Join(existingPolicies, ",")
 	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, userOrGroup, newPolicies, userType, isGroup)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{
 			UserOrGroup: userOrGroup,
 			UserType:    int(userType),
 			IsGroup:     isGroup,
 			Policy:      strings.Join(policiesToAttach, ","),
 		},
 		UpdatedAt: updatedAt,
 	}))
 	writeResponse(w, http.StatusCreated, nil, mimeNone)
 }
 // DetachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/detach
 func (a adminAPIHandlers) DetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DetachPolicyBuiltin")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.AttachPolicyAdminAction)
 	if objectAPI == nil {
 		return
 	}
-
+	if isAttach {
-	cred, _, s3Err := validateAdminSignature(ctx, r, "")
+		respBody.PoliciesAttached = addedOrRemoved
 	if s3Err != ErrNone {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 		return
 	}
 	password := cred.SecretKey
 	reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 	var par madmin.PolicyAssociationReq
 	if err = json.Unmarshal(reqBytes, &par); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 	if err = par.IsValid(); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 	userOrGroup := par.User
 	isGroup := false
 	if userOrGroup == "" {
 		userOrGroup = par.Group
 		isGroup = true
 	}
 	if isGroup {
 		_, err := globalIAMSys.GetGroupDescription(userOrGroup)
 		if err != nil {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
 	} else {
-		ok, _, err := globalIAMSys.IsTempUser(userOrGroup)
+		respBody.PoliciesDetached = addedOrRemoved
 		if err != nil && err != errNoSuchUser {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 			return
 		}
 		if ok {
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
 			return
 		}
 		// Validate that user exists.
 		if globalIAMSys.GetUsersSysType() == MinIOUsersSysType {
 			_, ok := globalIAMSys.GetUser(ctx, userOrGroup)
 			if !ok {
 				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
 				return
 			}
 		}
 	}
-	userType := regUser
+	data, err := json.Marshal(respBody)
 	if globalIAMSys.GetUsersSysType() == LDAPUsersSysType {
 		userType = stsUser
 	}
 	var existingPolicies []string
 	if isGroup {
 		existingPolicies, err = globalIAMSys.PolicyDBGet(userOrGroup, true)
 	} else {
 		existingPolicies, err = globalIAMSys.GetUserPolicies(userOrGroup)
 	}
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	policyMap := make(map[string]bool)
+	encryptedData, err := madmin.EncryptData(password, data)
 	for _, p := range existingPolicies {
 		policyMap[p] = true
 	}
 	policiesToDetach := par.Policies
 	// Check if policy is already attached to user.
 	for _, p := range policiesToDetach {
 		if _, ok := policyMap[p]; ok {
 			delete(policyMap, p)
 		} else {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyNotAttached), r.URL)
 			return
 		}
 	}
 	newPoliciesSl := []string{}
 	for p := range policyMap {
 		newPoliciesSl = append(newPoliciesSl, p)
 	}
 	newPolicies := strings.Join(newPoliciesSl, ",")
 	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, userOrGroup, newPolicies, userType, isGroup)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	logger.LogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
+	writeSuccessResponseJSON(w, encryptedData)
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{
 			UserOrGroup: userOrGroup,
 			UserType:    int(userType),
 			IsGroup:     isGroup,
 			Policy:      newPolicies,
 		},
 		UpdatedAt: updatedAt,
 	}))
 	// Return successful JSON response
 	writeSuccessNoContent(w)
 }
 const (
@@ -2055,13 +1949,15 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 			userAccounts := make(map[string]madmin.AddOrUpdateUserReq)
 			for u, uid := range userIdentities {
 				status := madmin.AccountDisabled
 				if uid.Credentials.IsValid() {
 					status = madmin.AccountEnabled
 				}
 				userAccounts[u] = madmin.AddOrUpdateUserReq{
 					SecretKey: uid.Credentials.SecretKey,
-					Status:    status,
+					Status: func() madmin.AccountStatus {
 						// Export current credential status
 						if uid.Credentials.Status == auth.AccountOff {
 							return madmin.AccountDisabled
 						}
 						return madmin.AccountEnabled
 					}(),
 				}
 			}
 			userData, err := json.Marshal(userAccounts)
@@ -2100,6 +1996,10 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 			svcAccts := make(map[string]madmin.SRSvcAccCreate)
 			for user, acc := range serviceAccounts {
 				if user == siteReplicatorSvcAcc {
 					// skip site-replication service account.
 					continue
 				}
 				claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.Credentials.AccessKey)
 				if err != nil {
 					writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
@@ -2448,7 +2348,8 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					opts := updateServiceAccountOpts{
 						secretKey:     svcAcctReq.SecretKey,
 						status:        svcAcctReq.Status,
-						comment:       svcAcctReq.Comment,
+						name:          svcAcctReq.Name,
 						description:   svcAcctReq.Description,
 						expiration:    svcAcctReq.Expiration,
 						sessionPolicy: sp,
 					}
@@ -2460,12 +2361,14 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					continue
 				}
 				opts := newServiceAccountOpts{
-					accessKey:     user,
+					accessKey:                  user,
-					secretKey:     svcAcctReq.SecretKey,
+					secretKey:                  svcAcctReq.SecretKey,
-					sessionPolicy: sp,
+					sessionPolicy:              sp,
-					claims:        svcAcctReq.Claims,
+					claims:                     svcAcctReq.Claims,
-					comment:       svcAcctReq.Comment,
+					name:                       svcAcctReq.Name,
-					expiration:    svcAcctReq.Expiration,
+					description:                svcAcctReq.Description,
 					expiration:                 svcAcctReq.Expiration,
 					allowSiteReplicatorAccount: false,
 				}
 				// In case of LDAP we need to resolve the targetUser to a DN and
@@ -33,10 +33,9 @@ import (
 	"testing"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/signer"
@@ -877,7 +876,7 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) {
 	// Create an madmin client with user creds
 	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
-		Creds:  cr.NewStaticV4(accessKey, secretKey, ""),
+		Creds:  credentials.NewStaticV4(accessKey, secretKey, ""),
 		Secure: s.secure,
 	})
 	if err != nil {
@@ -1072,7 +1071,7 @@ func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) {
 	// Create an madmin client with user creds
 	userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{
-		Creds:  cr.NewStaticV4(accessKey, secretKey, ""),
+		Creds:  credentials.NewStaticV4(accessKey, secretKey, ""),
 		Secure: s.secure,
 	})
 	if err != nil {
@@ -1329,7 +1328,11 @@ func (c *check) assertSvcAccAppearsInListing(ctx context.Context, madmClient *ma
 	if err != nil {
 		c.Fatalf("unable to list svc accounts: %v", err)
 	}
-	if !set.CreateStringSet(listResp.Accounts...).Contains(svcAK) {
+	var accessKeys []string
 	for _, item := range listResp.Accounts {
 		accessKeys = append(accessKeys, item.AccessKey)
 	}
 	if !set.CreateStringSet(accessKeys...).Contains(svcAK) {
 		c.Fatalf("service account did not appear in listing!")
 	}
 }
@@ -45,8 +45,8 @@ import (
 	"github.com/dustin/go-humanize"
 	"github.com/klauspost/compress/zip"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
-	"github.com/minio/madmin-go/v2/estream"
+	"github.com/minio/madmin-go/v3/estream"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/dsync"
 	"github.com/minio/minio/internal/handlers"
@@ -770,7 +770,7 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 		var err error
 		duration, err = time.ParseDuration(dstr)
 		if err != nil {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 			return
 		}
 	}
@@ -1525,7 +1525,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		return shouldTrace(entry, traceOpts)
 	})
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -1605,7 +1605,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 	err = globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
@@ -1855,7 +1855,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 	assignPoolNumbers(servers)
 	var poolsInfo map[int]map[int]madmin.ErasureSetInfo
-	var backend interface{}
+	var backend madmin.ErasureBackend
 	mode := madmin.ItemInitializing
@@ -1896,6 +1896,8 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 			OfflineDisks:     offlineDisks.Sum(),
 			StandardSCParity: backendInfo.StandardSCParity,
 			RRSCParity:       backendInfo.RRSCParity,
 			TotalSets:        backendInfo.TotalSets,
 			DrivesPerSet:     backendInfo.DrivesPerSet,
 		}
 		if poolsInfoEnabled {
@@ -2101,6 +2103,27 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 		}
 	}
 	// collect all realtime metrics except disk
 	// disk metrics are already included under drive info of each server
 	getRealtimeMetrics := func() *madmin.RealtimeMetrics {
 		var m madmin.RealtimeMetrics
 		var types madmin.MetricType = madmin.MetricsAll &^ madmin.MetricsDisk
 		mLocal := collectLocalMetrics(types, collectMetricsOpts{})
 		m.Merge(&mLocal)
 		cctx, cancel := context.WithTimeout(healthCtx, time.Second/2)
 		mRemote := collectRemoteMetrics(cctx, types, collectMetricsOpts{})
 		cancel()
 		m.Merge(&mRemote)
 		for idx, host := range m.Hosts {
 			m.Hosts[idx] = anonAddr(host)
 		}
 		for host, metrics := range m.ByHost {
 			m.ByHost[anonAddr(host)] = metrics
 			delete(m.ByHost, host)
 		}
 		return &m
 	}
 	anonymizeCmdLine := func(cmdLine string) string {
 		if !globalIsDistErasure {
 			// FS mode - single server - hard code to `server1`
@@ -2278,6 +2301,7 @@ func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *ur
 				TLS:          &tls,
 				IsKubernetes: &isK8s,
 				IsDocker:     &isDocker,
 				Metrics:      getRealtimeMetrics(),
 			}
 			partialWrite(healthInfo)
 		}
@@ -2509,13 +2533,13 @@ func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
 	var auditloggerInfo []madmin.Audit
 	for _, tgt := range logger.SystemTargets() {
 		if tgt.Endpoint() != "" {
-			loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): logger.TargetStatus(tgt)})
+			loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): logger.TargetStatus(GlobalContext, tgt)})
 		}
 	}
 	for _, tgt := range logger.AuditTargets() {
 		if tgt.Endpoint() != "" {
-			auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): logger.TargetStatus(tgt)})
+			auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): logger.TargetStatus(GlobalContext, tgt)})
 		}
 	}
@@ -31,7 +31,7 @@ import (
 	"testing"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/mux"
 )
@@ -479,8 +479,8 @@ func TestTopLockEntries(t *testing.T) {
 		if len(exp) != len(got) {
 			return 0, false
 		}
-		sort.Sort(byResourceUID{exp})
+		sort.Slice(exp, byResourceUID{exp}.Less)
-		sort.Sort(byResourceUID{got})
+		sort.Slice(got, byResourceUID{got}.Less)
 		// printEntries(exp)
 		// printEntries(got)
 		for i, e := range exp {
@@ -27,7 +27,7 @@ import (
 	"sync"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 )
@@ -784,6 +784,11 @@ func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
 }
 func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
 	if h.clientToken == bgHealingUUID {
 		// For background heal do nothing.
 		return nil
 	}
 	if err := h.healDiskMeta(objAPI); err != nil {
 		return err
 	}
@@ -869,13 +874,8 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly
 	if !h.settings.Recursive {
 		if h.object != "" {
-			// Check if an object named as the objPrefix exists,
+			if err := h.healObject(bucket, h.object, ""); err != nil {
-			// and if so heal it.
+				return err
 			oi, err := objAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
 			if err == nil {
 				if err = h.healObject(bucket, h.object, oi.VersionID); err != nil {
 					return err
 				}
 			}
 		}
@@ -22,7 +22,7 @@ import (
 	"github.com/klauspost/compress/gzhttp"
 	"github.com/klauspost/compress/gzip"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 )
@@ -162,11 +162,8 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			HandlerFunc(gz(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup))).
 			Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
-		// Attach policies to user or group
+		// Attach/Detach policies to/from user or group
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/attach").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachPolicyBuiltin)))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/{operation}").HandlerFunc(gz(httpTraceHdrs(adminAPI.AttachDetachPolicyBuiltin)))
 		// Detach policies from user or group
 		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/detach").HandlerFunc(gz(httpTraceHdrs(adminAPI.DetachPolicyBuiltin)))
 		// Remove user IAM
 		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveUser))).Queries("accessKey", "{accessKey:.*}")
@@ -26,7 +26,7 @@ import (
 	"strings"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
@@ -87,7 +87,6 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 	}
 	props := madmin.ServerProperties{
 		State:    string(madmin.ItemInitializing),
 		Endpoint: addr,
 		Uptime:   UTCNow().Unix() - globalBootTime.Unix(),
 		Version:  Version,
@@ -146,7 +145,8 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		props.State = string(madmin.ItemOnline)
 		props.Disks = storageInfo.Disks
 	} else {
-		props.State = string(madmin.ItemOffline)
+		props.State = string(madmin.ItemInitializing)
 		props.Disks = getOfflineDisks("", globalEndpoints)
 	}
 	return props
@@ -28,9 +28,10 @@ import (
 	"strings"
 	"github.com/Azure/azure-storage-blob-go/azblob"
 	"github.com/minio/minio/internal/ioutil"
 	"google.golang.org/api/googleapi"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/auth"
@@ -86,6 +87,7 @@ const (
 	ErrInternalError
 	ErrInvalidAccessKeyID
 	ErrAccessKeyDisabled
 	ErrInvalidArgument
 	ErrInvalidBucketName
 	ErrInvalidDigest
 	ErrInvalidRange
@@ -134,7 +136,7 @@ const (
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
 	ErrReplicationDenyEditError
-	ErrRemoteTargetDenyEditError
+	ErrRemoteTargetDenyAddError
 	ErrReplicationNoExistingObjects
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
@@ -148,6 +150,7 @@ const (
 	ErrMethodNotAllowed
 	ErrInvalidPart
 	ErrInvalidPartOrder
 	ErrMissingPart
 	ErrAuthorizationHeaderMalformed
 	ErrMalformedPOSTRequest
 	ErrPOSTFileRequired
@@ -181,8 +184,10 @@ const (
 	ErrBucketAlreadyExists
 	ErrMetadataTooLarge
 	ErrUnsupportedMetadata
 	ErrUnsupportedHostHeader
 	ErrMaximumExpires
-	ErrSlowDown
+	ErrSlowDownRead
 	ErrSlowDownWrite
 	ErrInvalidPrefixMarker
 	ErrBadRequest
 	ErrKeyTooLongError
@@ -199,6 +204,7 @@ const (
 	ErrInvalidTagDirective
 	ErrPolicyAlreadyAttached
 	ErrPolicyNotAttached
 	ErrExcessData
 	// Add new error codes here.
 	// SSE-S3/SSE-KMS related API errors
@@ -251,10 +257,11 @@ const (
 	ErrInvalidObjectName
 	ErrInvalidObjectNamePrefixSlash
 	ErrInvalidResourceName
 	ErrInvalidLifecycleQueryParameter
 	ErrServerNotInitialized
-	ErrOperationTimedOut
+	ErrRequestTimedout
 	ErrClientDisconnected
-	ErrOperationMaxedOut
+	ErrTooManyRequests
 	ErrInvalidRequest
 	ErrTransitionStorageClassNotFoundError
 	// MinIO storage class error codes
@@ -266,6 +273,7 @@ const (
 	ErrMalformedJSON
 	ErrAdminNoSuchUser
 	ErrAdminNoSuchUserLDAPWarn
 	ErrAdminNoSuchGroup
 	ErrAdminGroupNotEmpty
 	ErrAdminGroupDisabled
@@ -527,6 +535,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your proposed upload exceeds the maximum allowed object size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrExcessData: {
 		Code:           "ExcessData",
 		Description:    "More data provided than indicated content length",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrPolicyTooLarge: {
 		Code:           "PolicyTooLarge",
 		Description:    "Policy exceeds the maximum allowed document size.",
@@ -552,6 +565,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your account is disabled; please contact your administrator.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrInvalidArgument: {
 		Code:           "InvalidArgument",
 		Description:    "Invalid argument",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidBucketName: {
 		Code:           "InvalidBucketName",
 		Description:    "The specified bucket is not valid.",
@@ -677,6 +695,11 @@ var errorCodes = errorCodeMap{
 		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrMissingPart: {
 		Code:           "InvalidRequest",
 		Description:    "You must specify at least one part",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidPartOrder: {
 		Code:           "InvalidPartOrder",
 		Description:    "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
@@ -822,11 +845,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Request is not valid yet",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-	ErrSlowDown: {
+	ErrSlowDownRead: {
-		Code:           "SlowDown",
+		Code:           "SlowDownRead",
 		Description:    "Resource requested is unreadable, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrSlowDownWrite: {
 		Code:           "SlowDownWrite",
 		Description:    "Resource requested is unwritable, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrInvalidPrefixMarker: {
 		Code:           "InvalidPrefixMarker",
 		Description:    "Invalid marker prefix combination",
@@ -927,9 +955,9 @@ var errorCodes = errorCodeMap{
 		Description:    "No matching ExistingsObjects rule enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrRemoteTargetDenyEditError: {
+	ErrRemoteTargetDenyAddError: {
-		Code:           "XMinioAdminRemoteTargetDenyEdit",
+		Code:           "XMinioAdminRemoteTargetDenyAdd",
-		Description:    "Cannot alter remote target endpoint since this server is in a cluster replication setup. use `mc admin replicate update`",
+		Description:    "Cannot add remote target endpoint since this server is in a cluster replication setup",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrReplicationDenyEditError: {
@@ -1100,8 +1128,13 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidEncryptionMethod: {
-		Code:           "InvalidRequest",
+		Code:           "InvalidArgument",
-		Description:    "The encryption method specified is not supported",
+		Description:    "Server Side Encryption with AWS KMS managed key requires HTTP header x-amz-server-side-encryption : aws:kms",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrIncompatibleEncryptionMethod: {
 		Code:           "InvalidArgument",
 		Description:    "Server Side Encryption with Customer provided key is incompatible with the encryption method specified",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidEncryptionKeyID: {
@@ -1164,11 +1197,6 @@ var errorCodes = errorCodeMap{
 		Description:    "The provided encryption parameters did not match the ones used originally.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrIncompatibleEncryptionMethod: {
 		Code:           "InvalidArgument",
 		Description:    "Server side encryption specified with both SSE-C and SSE-S3 headers",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrKMSNotConfigured: {
 		Code:           "NotImplemented",
 		Description:    "Server side encryption specified but KMS is not configured",
@@ -1248,11 +1276,21 @@ var errorCodes = errorCodeMap{
 		Description:    "The JSON you provided was not well-formed or did not validate against our published format.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidLifecycleQueryParameter: {
 		Code:           "XMinioInvalidLifecycleParameter",
 		Description:    "The boolean value provided for withUpdatedAt query parameter was invalid.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrAdminNoSuchUser: {
 		Code:           "XMinioAdminNoSuchUser",
 		Description:    "The specified user does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrAdminNoSuchUserLDAPWarn: {
 		Code:           "XMinioAdminNoSuchUser",
 		Description:    "The specified user does not exist. If you meant a user in LDAP, use `mc idp ldap`",
 		HTTPStatusCode: http.StatusNotFound,
 	},
 	ErrAdminNoSuchGroup: {
 		Code:           "XMinioAdminNoSuchGroup",
 		Description:    "The specified group does not exist.",
@@ -1385,7 +1423,7 @@ var errorCodes = errorCodeMap{
 		Description:    "Cannot respond to plain-text request from TLS-encrypted server",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrOperationTimedOut: {
+	ErrRequestTimedout: {
 		Code:           "RequestTimeout",
 		Description:    "A timeout occurred while trying to lock a resource, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
@@ -1395,9 +1433,9 @@ var errorCodes = errorCodeMap{
 		Description:    "Client disconnected before response was ready",
 		HTTPStatusCode: 499, // No official code, use nginx value.
 	},
-	ErrOperationMaxedOut: {
+	ErrTooManyRequests: {
-		Code:           "SlowDown",
+		Code:           "TooManyRequests",
-		Description:    "A timeout exceeded while waiting to proceed with the request, please reduce your request rate",
+		Description:    "Deadline exceeded while waiting in incoming queue, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrUnsupportedMetadata: {
@@ -1405,6 +1443,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your metadata headers are not supported.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrUnsupportedHostHeader: {
 		Code:           "InvalidArgument",
 		Description:    "Your Host header is malformed.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrObjectTampered: {
 		Code:           "XMinioObjectTampered",
 		Description:    errObjectTampered.Error(),
@@ -2008,11 +2051,14 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	}
 	// Only return ErrClientDisconnected if the provided context is actually canceled.
-	// This way downstream context.Canceled will still report ErrOperationTimedOut
+	// This way downstream context.Canceled will still report ErrRequestTimedout
 	if contextCanceled(ctx) && errors.Is(ctx.Err(), context.Canceled) {
 		return ErrClientDisconnected
 	}
 	// Unwrap the error first
 	err = unwrapAll(err)
 	switch err {
 	case errInvalidArgument:
 		apiErr = ErrAdminInvalidArgument
@@ -2020,6 +2066,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminNoSuchPolicy
 	case errNoSuchUser:
 		apiErr = ErrAdminNoSuchUser
 	case errNoSuchUserLDAPWarn:
 		apiErr = ErrAdminNoSuchUserLDAPWarn
 	case errNoSuchServiceAccount:
 		apiErr = ErrAdminServiceAccountNotFound
 	case errNoSuchGroup:
@@ -2047,9 +2095,9 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case errInvalidStorageClass:
 		apiErr = ErrInvalidStorageClass
 	case errErasureReadQuorum:
-		apiErr = ErrSlowDown
+		apiErr = ErrSlowDownRead
 	case errErasureWriteQuorum:
-		apiErr = ErrSlowDown
+		apiErr = ErrSlowDownWrite
 	// SSE errors
 	case errInvalidEncryptionParameters:
 		apiErr = ErrInvalidEncryptionParameters
@@ -2083,10 +2131,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSKeyNotFoundException
 	case errKMSDefaultKeyAlreadyConfigured:
 		apiErr = ErrKMSDefaultKeyAlreadyConfigured
-	case context.Canceled, context.DeadlineExceeded:
+	case context.Canceled:
-		apiErr = ErrOperationTimedOut
+		apiErr = ErrClientDisconnected
-	case errDiskNotFound:
+	case context.DeadlineExceeded:
-		apiErr = ErrSlowDown
+		apiErr = ErrRequestTimedout
 	case objectlock.ErrInvalidRetentionDate:
 		apiErr = ErrInvalidRetentionDate
 	case objectlock.ErrPastObjectLockRetainDate:
@@ -2097,6 +2145,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrObjectLockInvalidHeaders
 	case objectlock.ErrMalformedXML:
 		apiErr = ErrMalformedXML
 	case errInvalidMaxParts:
 		apiErr = ErrInvalidMaxParts
 	case ioutil.ErrOverread:
 		apiErr = ErrExcessData
 	}
 	// Compression errors
@@ -2161,9 +2213,9 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case InvalidPart:
 		apiErr = ErrInvalidPart
 	case InsufficientWriteQuorum:
-		apiErr = ErrSlowDown
+		apiErr = ErrSlowDownWrite
 	case InsufficientReadQuorum:
-		apiErr = ErrSlowDown
+		apiErr = ErrSlowDownRead
 	case InvalidMarkerPrefixCombination:
 		apiErr = ErrNotImplemented
 	case InvalidUploadIDKeyCombination:
@@ -2178,10 +2230,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrContentSHA256Mismatch
 	case hash.ChecksumMismatch:
 		apiErr = ErrContentChecksumMismatch
-	case ObjectTooLarge:
+	case hash.SizeTooSmall:
 		apiErr = ErrEntityTooLarge
 	case ObjectTooSmall:
 		apiErr = ErrEntityTooSmall
 	case hash.SizeTooLarge:
 		apiErr = ErrEntityTooLarge
 	case NotImplemented:
 		apiErr = ErrNotImplemented
 	case PartTooBig:
@@ -2257,7 +2309,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case *event.ErrUnsupportedConfiguration:
 		apiErr = ErrUnsupportedNotification
 	case OperationTimedOut:
-		apiErr = ErrOperationTimedOut
+		apiErr = ErrRequestTimedout
 	case BackendDown:
 		apiErr = ErrBackendDown
 	case ObjectNameTooLong:
@@ -2352,7 +2404,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 			}
 		case lifecycle.Error:
 			apiErr = APIError{
-				Code:           "InvalidRequest",
+				Code:           "InvalidArgument",
 				Description:    e.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
@@ -20,8 +20,6 @@ package cmd
 import (
 	"context"
 	"errors"
 	"os"
 	"path/filepath"
 	"testing"
 	"github.com/minio/minio/internal/crypto"
@@ -42,8 +40,8 @@ var toAPIErrorTests = []struct {
 	{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
 	{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
 	{err: InvalidPart{}, errCode: ErrInvalidPart},
-	{err: InsufficientReadQuorum{}, errCode: ErrSlowDown},
+	{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
-	{err: InsufficientWriteQuorum{}, errCode: ErrSlowDown},
+	{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
 	{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
 	{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
 	{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
@@ -67,11 +65,6 @@ var toAPIErrorTests = []struct {
 }
 func TestAPIErrCode(t *testing.T) {
 	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
 	defer os.RemoveAll(disk)
 	initFSObjects(disk, t)
 	ctx := context.Background()
 	for i, testCase := range toAPIErrorTests {
 		errCode := toAPIErrorCode(ctx, testCase.err)
@@ -203,7 +203,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}
 	// Set the relevant version ID as part of the response header.
-	if objInfo.VersionID != "" {
+	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}
@@ -37,8 +37,8 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
 		maxkeys = maxObjectList
 	}
-	prefix = trimLeadingSlash(values.Get("prefix"))
+	prefix = values.Get("prefix")
-	marker = trimLeadingSlash(values.Get("marker"))
+	marker = values.Get("marker")
 	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
 	return
@@ -57,8 +57,8 @@ func getListBucketObjectVersionsArgs(values url.Values) (prefix, marker, delimit
 		maxkeys = maxObjectList
 	}
-	prefix = trimLeadingSlash(values.Get("prefix"))
+	prefix = values.Get("prefix")
-	marker = trimLeadingSlash(values.Get("key-marker"))
+	marker = values.Get("key-marker")
 	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
 	versionIDMarker = values.Get("version-id-marker")
@@ -87,8 +87,8 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 		maxkeys = maxObjectList
 	}
-	prefix = trimLeadingSlash(values.Get("prefix"))
+	prefix = values.Get("prefix")
-	startAfter = trimLeadingSlash(values.Get("start-after"))
+	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
 	fetchOwner = values.Get("fetch-owner") == "true"
 	encodingType = values.Get("encoding-type")
@@ -118,8 +118,8 @@ func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadID
 		maxUploads = maxUploadsList
 	}
-	prefix = trimLeadingSlash(values.Get("prefix"))
+	prefix = values.Get("prefix")
-	keyMarker = trimLeadingSlash(values.Get("key-marker"))
+	keyMarker = values.Get("key-marker")
 	uploadIDMarker = values.Get("upload-id-marker")
 	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
@@ -35,6 +35,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
 	xxml "github.com/minio/xxml"
 )
@@ -84,7 +85,7 @@ type ListVersionsResponse struct {
 	VersionIDMarker string `xml:"VersionIdMarker"`
 	MaxKeys   int
-	Delimiter string
+	Delimiter string `xml:"Delimiter,omitempty"`
 	// A flag that indicates whether or not ListObjects returned all of the results
 	// that satisfied the search criteria.
 	IsTruncated bool
@@ -114,7 +115,7 @@ type ListObjectsResponse struct {
 	NextMarker string `xml:"NextMarker,omitempty"`
 	MaxKeys   int
-	Delimiter string
+	Delimiter string `xml:"Delimiter,omitempty"`
 	// A flag that indicates whether or not ListObjects returned all of the results
 	// that satisfied the search criteria.
 	IsTruncated bool
@@ -145,7 +146,7 @@ type ListObjectsV2Response struct {
 	KeyCount  int
 	MaxKeys   int
-	Delimiter string
+	Delimiter string `xml:"Delimiter,omitempty"`
 	// A flag that indicates whether or not ListObjects returned all of the results
 	// that satisfied the search criteria.
 	IsTruncated bool
@@ -204,7 +205,7 @@ type ListMultipartUploadsResponse struct {
 	UploadIDMarker     string `xml:"UploadIdMarker"`
 	NextKeyMarker      string
 	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
-	Delimiter          string
+	Delimiter          string `xml:"Delimiter,omitempty"`
 	Prefix             string
 	EncodingType       string `xml:"EncodingType,omitempty"`
 	MaxUploads         int
@@ -344,6 +345,13 @@ func (s *Metadata) MarshalXML(e *xxml.Encoder, start xxml.StartElement) error {
 	return e.EncodeToken(start.End())
 }
 // ObjectInternalInfo contains some internal information about a given
 // object, it will printed in listing calls with enabled metadata.
 type ObjectInternalInfo struct {
 	K int // Data blocks
 	M int // Parity blocks
 }
 // Object container for object metadata
 type Object struct {
 	Key          string
@@ -352,13 +360,16 @@ type Object struct {
 	Size         int64
 	// Owner of the object.
-	Owner Owner
+	Owner *Owner `xml:"Owner,omitempty"`
 	// The class of storage used to store the object.
 	StorageClass string
 	// UserMetadata user-defined metadata
 	UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
 	UserTags     string    `xml:"UserTags,omitempty"`
 	Internal *ObjectInternalInfo `xml:"Internal,omitempty"`
 }
 // CopyObjectResponse container returns ETag and LastModified of the successfully copied object
@@ -492,19 +503,27 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 }
 // generates an ListBucketVersions response for the said bucket with other enumerated options.
-func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo) ListVersionsResponse {
+func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo, metadata metaCheckFn) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))
-	owner := Owner{
+	owner := &Owner{
 		ID:          globalMinioDefaultOwnerID,
 		DisplayName: "minio",
 	}
 	data := ListVersionsResponse{}
 	var lastObjMetaName string
 	var tagErr, metaErr APIErrorCode = -1, -1
 	for _, object := range resp.Objects {
 		if object.Name == "" {
 			continue
 		}
 		// Cache checks for the same object
 		if metadata != nil && lastObjMetaName != object.Name {
 			tagErr = metadata(object.Name, policy.GetObjectTaggingAction)
 			metaErr = metadata(object.Name, policy.GetObjectAction)
 			lastObjMetaName = object.Name
 		}
 		content := ObjectVersion{}
 		content.Key = s3EncodeName(object.Name, encodingType)
 		content.LastModified = amztime.ISO8601Format(object.ModTime.UTC())
@@ -517,6 +536,36 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 		} else {
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
 		if tagErr == ErrNone {
 			content.UserTags = object.UserTags
 		}
 		if metaErr == ErrNone {
 			content.UserMetadata = &Metadata{}
 			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
 			case crypto.S3:
 				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 			case crypto.S3KMS:
 				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
 			case crypto.SSEC:
 				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 			}
 			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
 				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 					// Do not need to send any internal metadata
 					// values to client.
 					continue
 				}
 				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
 				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 					continue
 				}
 				content.UserMetadata.Set(k, v)
 			}
 			content.Internal = &ObjectInternalInfo{
 				K: object.DataBlocks,
 				M: object.ParityBlocks,
 			}
 		}
 		content.Owner = owner
 		content.VersionID = object.VersionID
 		if content.VersionID == "" {
@@ -553,7 +602,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 // generates an ListObjectsV1 response for the said bucket with other enumerated options.
 func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
 	contents := make([]Object, 0, len(resp.Objects))
-	owner := Owner{
+	owner := &Owner{
 		ID:          globalMinioDefaultOwnerID,
 		DisplayName: "minio",
 	}
@@ -600,12 +649,16 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 }
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
-func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata bool) ListObjectsV2Response {
+func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata metaCheckFn) ListObjectsV2Response {
 	contents := make([]Object, 0, len(objects))
-	owner := Owner{
+	var owner *Owner
-		ID:          globalMinioDefaultOwnerID,
+	if fetchOwner {
-		DisplayName: "minio",
+		owner = &Owner{
 			ID:          globalMinioDefaultOwnerID,
 			DisplayName: "minio",
 		}
 	}
 	data := ListObjectsV2Response{}
 	for _, object := range objects {
@@ -625,27 +678,36 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 			content.StorageClass = globalMinioDefaultStorageClass
 		}
 		content.Owner = owner
-		if metadata {
+		if metadata != nil {
-			content.UserMetadata = &Metadata{}
+			if metadata(object.Name, policy.GetObjectTaggingAction) == ErrNone {
-			switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
+				content.UserTags = object.UserTags
 			case crypto.S3:
 				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 			case crypto.S3KMS:
 				content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
 			case crypto.SSEC:
 				content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 			}
-			for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
+			if metadata(object.Name, policy.GetObjectAction) == ErrNone {
-				if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+				content.UserMetadata = &Metadata{}
-					// Do not need to send any internal metadata
+				switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind {
-					// values to client.
+				case crypto.S3:
-					continue
+					content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 				case crypto.S3KMS:
 					content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
 				case crypto.SSEC:
 					content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES)
 				}
-				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+				for k, v := range cleanMinioInternalMetadataKeys(object.UserDefined) {
-				if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
+					if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
-					continue
+						// Do not need to send any internal metadata
 						// values to client.
 						continue
 					}
 					// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
 					if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) {
 						continue
 					}
 					content.UserMetadata.Set(k, v)
 				}
 				content.Internal = &ObjectInternalInfo{
 					K: object.DataBlocks,
 					M: object.ParityBlocks,
 				}
 				content.UserMetadata.Set(k, v)
 			}
 		}
 		contents = append(contents, content)
@@ -673,6 +735,8 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 	return data
 }
 type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode)
 // generates CopyObjectResponse from etag and lastModified time.
 func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse {
 	return CopyObjectResponse{
@@ -700,7 +764,7 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi
 // generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
 func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
-	cs := oi.decryptChecksums()
+	cs := oi.decryptChecksums(0)
 	c := CompleteMultipartUploadResponse{
 		Location: location,
 		Bucket:   bucket,
@@ -881,6 +945,8 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 }
 func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) {
 	w.Header().Set(xMinIOErrCodeHeader, err.Code)
 	w.Header().Set(xMinIOErrDescHeader, "\""+err.Description+"\"")
 	writeResponse(w, err.HTTPStatusCode, nil, mimeNone)
 }
@@ -245,10 +245,10 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodPut).Path("/{object:.+}").
 			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
 			HandlerFunc(collectAPIStats("copyobjectpart", maxClients(gz(httpTraceAll(api.CopyObjectPartHandler))))).
-			Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+			Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
 		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+			collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("listobjectparts", maxClients(gz(httpTraceAll(api.ListObjectPartsHandler))))).Queries("uploadId", "{uploadId:.*}")
@@ -392,6 +392,9 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2")
 		// ListObjectVersions
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsMHandler))))).Queries("versions", "", "metadata", "true")
 		// ListObjectVersions
 		router.Methods(http.MethodGet).HandlerFunc(
 			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "")
 		// GetBucketPolicyStatus
@@ -88,6 +88,18 @@ func isRequestSignStreamingV4(r *http.Request) bool {
 		r.Method == http.MethodPut
 }
 // Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
 func isRequestSignStreamingTrailerV4(r *http.Request) bool {
 	return r.Header.Get(xhttp.AmzContentSha256) == streamingContentSHA256Trailer &&
 		r.Method == http.MethodPut
 }
 // Verify if the request has AWS Streaming Signature Version '4', with unsigned content and trailer.
 func isRequestUnsignedTrailerV4(r *http.Request) bool {
 	return r.Header.Get(xhttp.AmzContentSha256) == unsignedPayloadTrailer &&
 		r.Method == http.MethodPut && strings.Contains(r.Header.Get(xhttp.ContentEncoding), streamingContentEncoding)
 }
 // Authorization type.
 //
 //go:generate stringer -type=authType -trimprefix=authType $GOFILE
@@ -105,10 +117,12 @@ const (
 	authTypeSignedV2
 	authTypeJWT
 	authTypeSTS
 	authTypeStreamingSignedTrailer
 	authTypeStreamingUnsignedTrailer
 )
 // Get request authentication type.
-func getRequestAuthType(r *http.Request) authType {
+func getRequestAuthType(r *http.Request) (at authType) {
 	if r.URL != nil {
 		var err error
 		r.Form, err = url.ParseQuery(r.URL.RawQuery)
@@ -123,6 +137,10 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypePresignedV2
 	} else if isRequestSignStreamingV4(r) {
 		return authTypeStreamingSigned
 	} else if isRequestSignStreamingTrailerV4(r) {
 		return authTypeStreamingSignedTrailer
 	} else if isRequestUnsignedTrailerV4(r) {
 		return authTypeStreamingUnsignedTrailer
 	} else if isRequestSignatureV4(r) {
 		return authTypeSigned
 	} else if isRequestPresignedSignatureV4(r) {
@@ -560,13 +578,15 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty
 // List of all support S3 auth types.
 var supportedS3AuthTypes = map[authType]struct{}{
-	authTypeAnonymous:       {},
+	authTypeAnonymous:                {},
-	authTypePresigned:       {},
+	authTypePresigned:                {},
-	authTypePresignedV2:     {},
+	authTypePresignedV2:              {},
-	authTypeSigned:          {},
+	authTypeSigned:                   {},
-	authTypeSignedV2:        {},
+	authTypeSignedV2:                 {},
-	authTypePostPolicy:      {},
+	authTypePostPolicy:               {},
-	authTypeStreamingSigned: {},
+	authTypeStreamingSigned:          {},
 	authTypeStreamingSignedTrailer:   {},
 	authTypeStreamingUnsignedTrailer: {},
 }
 // Validate if the authType is valid and supported.
@@ -582,7 +602,8 @@ func setAuthHandler(h http.Handler) http.Handler {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)
 		aType := getRequestAuthType(r)
-		if aType == authTypeSigned || aType == authTypeSignedV2 || aType == authTypeStreamingSigned {
+		switch aType {
 		case authTypeSigned, authTypeSignedV2, authTypeStreamingSigned, authTypeStreamingSignedTrailer:
 			// Verify if date headers are set, if not reject the request
 			amzDate, errCode := parseAmzDateHeader(r)
 			if errCode != ErrNone {
@@ -594,6 +615,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 				// All our internal APIs are sensitive towards Date
 				// header, for all requests where Date header is not
 				// present we will reject such clients.
 				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(errCode), r.URL)
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
@@ -607,14 +629,21 @@ func setAuthHandler(h http.Handler) http.Handler {
 					tc.ResponseRecorder.LogErrBody = true
 				}
 				defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 				writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL)
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
 			}
 		}
 		if isSupportedS3AuthType(aType) || aType == authTypeJWT || aType == authTypeSTS {
 			h.ServeHTTP(w, r)
 			return
 		case authTypeJWT, authTypeSTS:
 			h.ServeHTTP(w, r)
 			return
 		default:
 			if isSupportedS3AuthType(aType) {
 				h.ServeHTTP(w, r)
 				return
 			}
 		}
 		if ok {
@@ -622,6 +651,7 @@ func setAuthHandler(h http.Handler) http.Handler {
 			tc.ResponseRecorder.LogErrBody = true
 		}
 		defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r))
 		writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL)
 		atomic.AddUint64(&globalHTTPStats.rejectedRequestsAuth, 1)
 	})
@@ -707,7 +737,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 		return ErrSignatureVersionNotSupported
 	case authTypeSignedV2, authTypePresignedV2:
 		cred, owner, s3Err = getReqAccessKeyV2(r)
-	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
+	case authTypeStreamingSigned, authTypePresigned, authTypeSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer:
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
@@ -375,8 +375,6 @@ func TestIsReqAuthenticated(t *testing.T) {
 	initConfigSubsystem(ctx, objLayer)
 	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
 		t.Fatalf("unable create credential, %s", err)
@@ -384,6 +382,8 @@ func TestIsReqAuthenticated(t *testing.T) {
 	globalActiveCred = creds
 	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
 	// List of test cases for validating http request authentication.
 	testCases := []struct {
 		req     *http.Request
@@ -464,9 +464,8 @@ func TestValidateAdminSignature(t *testing.T) {
 	}
 	initAllSubsystems(ctx)
 	initConfigSubsystem(ctx, objLayer)
-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+	initConfigSubsystem(ctx, objLayer)
 	creds, err := auth.CreateCredentials("admin", "mypassword")
 	if err != nil {
@@ -474,6 +473,8 @@ func TestValidateAdminSignature(t *testing.T) {
 	}
 	globalActiveCred = creds
 	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
 	testCases := []struct {
 		AccessKey string
 		SecretKey string
@@ -18,11 +18,13 @@ func _() {
 	_ = x[authTypeSignedV2-7]
 	_ = x[authTypeJWT-8]
 	_ = x[authTypeSTS-9]
 	_ = x[authTypeStreamingSignedTrailer-10]
 	_ = x[authTypeStreamingUnsignedTrailer-11]
 }
-const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingSignedSignedSignedV2JWTSTS"
+const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingSignedSignedSignedV2JWTSTSStreamingSignedTrailerStreamingUnsignedTrailer"
-var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81}
+var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81, 103, 127}
 func (i authType) String() string {
 	if i < 0 || i >= authType(len(_authType_index)-1) {
@@ -24,7 +24,7 @@ import (
 	"strconv"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/env"
 )
@@ -26,12 +26,15 @@ import (
 	"os"
 	"sort"
 	"strings"
 	"sync"
 	"time"
 	"github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/env"
 )
 const (
@@ -43,7 +46,8 @@ const (
 // healingTracker is used to persist healing information during a heal.
 type healingTracker struct {
-	disk StorageAPI `msg:"-"`
+	disk StorageAPI    `msg:"-"`
 	mu   *sync.RWMutex `msg:"-"`
 	ID         string
 	PoolIndex  int
@@ -112,22 +116,76 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
 	}
 	h.disk = disk
 	h.ID = diskID
 	h.mu = &sync.RWMutex{}
 	return &h, nil
 }
 // newHealingTracker will create a new healing tracker for the disk.
-func newHealingTracker(disk StorageAPI, healID string) *healingTracker {
+func newHealingTracker() *healingTracker {
-	diskID, _ := disk.GetDiskID()
+	return &healingTracker{
-	h := healingTracker{
+		mu: &sync.RWMutex{},
 		disk:     disk,
 		ID:       diskID,
 		HealID:   healID,
 		Path:     disk.String(),
 		Endpoint: disk.Endpoint().String(),
 		Started:  time.Now().UTC(),
 	}
 }
 func initHealingTracker(disk StorageAPI, healID string) *healingTracker {
 	h := newHealingTracker()
 	diskID, _ := disk.GetDiskID()
 	h.disk = disk
 	h.ID = diskID
 	h.HealID = healID
 	h.Path = disk.String()
 	h.Endpoint = disk.Endpoint().String()
 	h.Started = time.Now().UTC()
 	h.PoolIndex, h.SetIndex, h.DiskIndex = disk.GetDiskLoc()
-	return &h
+	return h
 }
 func (h healingTracker) getLastUpdate() time.Time {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 	return h.LastUpdate
 }
 func (h healingTracker) getBucket() string {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 	return h.Bucket
 }
 func (h *healingTracker) setBucket(bucket string) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 	h.Bucket = bucket
 }
 func (h healingTracker) getObject() string {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 	return h.Object
 }
 func (h *healingTracker) setObject(object string) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 	h.Object = object
 }
 func (h *healingTracker) updateProgress(success bool, bytes uint64) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 	if success {
 		h.ItemsHealed++
 		h.BytesDone += bytes
 	} else {
 		h.ItemsFailed++
 		h.BytesFailed += bytes
 	}
 }
 // update will update the tracker on the disk.
@@ -136,15 +194,18 @@ func (h *healingTracker) update(ctx context.Context) error {
 	if h.disk.Healing() == nil {
 		return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
 	}
 	h.mu.Lock()
 	if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
 		h.ID, _ = h.disk.GetDiskID()
 		h.PoolIndex, h.SetIndex, h.DiskIndex = h.disk.GetDiskLoc()
 	}
 	h.mu.Unlock()
 	return h.save(ctx)
 }
 // save will unconditionally save the tracker and will be created if not existing.
 func (h *healingTracker) save(ctx context.Context) error {
 	h.mu.Lock()
 	if h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
 		// Attempt to get location.
 		if api := newObjectLayerFn(); api != nil {
@@ -155,6 +216,7 @@ func (h *healingTracker) save(ctx context.Context) error {
 	}
 	h.LastUpdate = time.Now().UTC()
 	htrackerBytes, err := h.MarshalMsg(nil)
 	h.mu.Unlock()
 	if err != nil {
 		return err
 	}
@@ -176,6 +238,8 @@ func (h *healingTracker) delete(ctx context.Context) error {
 }
 func (h *healingTracker) isHealed(bucket string) bool {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 	for _, v := range h.HealedBuckets {
 		if v == bucket {
 			return true
@@ -186,6 +250,9 @@ func (h *healingTracker) isHealed(bucket string) bool {
 // resume will reset progress to the numbers at the start of the bucket.
 func (h *healingTracker) resume() {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 	h.ItemsHealed = h.ResumeItemsHealed
 	h.ItemsFailed = h.ResumeItemsFailed
 	h.BytesDone = h.ResumeBytesDone
@@ -195,6 +262,9 @@ func (h *healingTracker) resume() {
 // bucketDone should be called when a bucket is done healing.
 // Adds the bucket to the list of healed buckets and updates resume numbers.
 func (h *healingTracker) bucketDone(bucket string) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 	h.ResumeItemsHealed = h.ItemsHealed
 	h.ResumeItemsFailed = h.ItemsFailed
 	h.ResumeBytesDone = h.BytesDone
@@ -211,6 +281,9 @@ func (h *healingTracker) bucketDone(bucket string) {
 // setQueuedBuckets will add buckets, but exclude any that is already in h.HealedBuckets.
 // Order is preserved.
 func (h *healingTracker) setQueuedBuckets(buckets []BucketInfo) {
 	h.mu.Lock()
 	defer h.mu.Unlock()
 	s := set.CreateStringSet(h.HealedBuckets...)
 	h.QueuedBuckets = make([]string, 0, len(buckets))
 	for _, b := range buckets {
@@ -221,15 +294,22 @@ func (h *healingTracker) setQueuedBuckets(buckets []BucketInfo) {
 }
 func (h *healingTracker) printTo(writer io.Writer) {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 	b, err := json.MarshalIndent(h, "", "  ")
 	if err != nil {
 		writer.Write([]byte(err.Error()))
 		return
 	}
 	writer.Write(b)
 }
 // toHealingDisk converts the information to madmin.HealingDisk
 func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
 	h.mu.RLock()
 	defer h.mu.RUnlock()
 	return madmin.HealingDisk{
 		ID:                h.ID,
 		HealID:            h.HealID,
@@ -267,7 +347,9 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {
 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)
-	go monitorLocalDisksAndHeal(ctx, z)
+	if env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
 		go monitorLocalDisksAndHeal(ctx, z)
 	}
 }
 func getLocalDisksToHeal() (disksToHeal Endpoints) {
@@ -336,7 +418,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 			return nil
 		}
 		logger.LogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
-		tracker = newHealingTracker(disk, mustGetUUID())
+		tracker = initHealingTracker(disk, mustGetUUID())
 	}
 	logger.Info(fmt.Sprintf("Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint))
@@ -453,7 +535,8 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) {
 					globalBackgroundHealState.setDiskHealingStatus(disk, true)
 					if err := healFreshDisk(ctx, z, disk); err != nil {
 						globalBackgroundHealState.setDiskHealingStatus(disk, false)
-						if !errors.Is(err, context.Canceled) {
+						timedout := OperationTimedOut{}
 						if !errors.Is(err, context.Canceled) && !errors.As(err, &timedout) {
 							printEndpointError(disk, err, false)
 						}
 						return
@@ -702,6 +702,12 @@ func (z *BatchJobReplicateSource) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Endpoint")
 				return
 			}
 		case "Path":
 			z.Path, err = dc.ReadString()
 			if err != nil {
 				err = msgp.WrapError(err, "Path")
 				return
 			}
 		case "Creds":
 			var zb0003 uint32
 			zb0003, err = dc.ReadMapHeader()
@@ -756,9 +762,9 @@ func (z *BatchJobReplicateSource) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 5
+	// map header, size 6
 	// write "Type"
-	err = en.Append(0x85, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	err = en.Append(0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	if err != nil {
 		return
 	}
@@ -797,6 +803,16 @@ func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Endpoint")
 		return
 	}
 	// write "Path"
 	err = en.Append(0xa4, 0x50, 0x61, 0x74, 0x68)
 	if err != nil {
 		return
 	}
 	err = en.WriteString(z.Path)
 	if err != nil {
 		err = msgp.WrapError(err, "Path")
 		return
 	}
 	// write "Creds"
 	err = en.Append(0xa5, 0x43, 0x72, 0x65, 0x64, 0x73)
 	if err != nil {
@@ -839,9 +855,9 @@ func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobReplicateSource) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 5
+	// map header, size 6
 	// string "Type"
-	o = append(o, 0x85, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	o = append(o, 0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	o = msgp.AppendString(o, string(z.Type))
 	// string "Bucket"
 	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
@@ -852,6 +868,9 @@ func (z *BatchJobReplicateSource) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Endpoint"
 	o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
 	o = msgp.AppendString(o, z.Endpoint)
 	// string "Path"
 	o = append(o, 0xa4, 0x50, 0x61, 0x74, 0x68)
 	o = msgp.AppendString(o, z.Path)
 	// string "Creds"
 	o = append(o, 0xa5, 0x43, 0x72, 0x65, 0x64, 0x73)
 	// map header, size 3
@@ -913,6 +932,12 @@ func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error)
 				err = msgp.WrapError(err, "Endpoint")
 				return
 			}
 		case "Path":
 			z.Path, bts, err = msgp.ReadStringBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Path")
 				return
 			}
 		case "Creds":
 			var zb0003 uint32
 			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
@@ -968,7 +993,7 @@ func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error)
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobReplicateSource) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken)
+	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken)
 	return
 }
@@ -1018,6 +1043,12 @@ func (z *BatchJobReplicateTarget) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Endpoint")
 				return
 			}
 		case "Path":
 			z.Path, err = dc.ReadString()
 			if err != nil {
 				err = msgp.WrapError(err, "Path")
 				return
 			}
 		case "Creds":
 			var zb0003 uint32
 			zb0003, err = dc.ReadMapHeader()
@@ -1072,9 +1103,9 @@ func (z *BatchJobReplicateTarget) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobReplicateTarget) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 5
+	// map header, size 6
 	// write "Type"
-	err = en.Append(0x85, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	err = en.Append(0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	if err != nil {
 		return
 	}
@@ -1113,6 +1144,16 @@ func (z *BatchJobReplicateTarget) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Endpoint")
 		return
 	}
 	// write "Path"
 	err = en.Append(0xa4, 0x50, 0x61, 0x74, 0x68)
 	if err != nil {
 		return
 	}
 	err = en.WriteString(z.Path)
 	if err != nil {
 		err = msgp.WrapError(err, "Path")
 		return
 	}
 	// write "Creds"
 	err = en.Append(0xa5, 0x43, 0x72, 0x65, 0x64, 0x73)
 	if err != nil {
@@ -1155,9 +1196,9 @@ func (z *BatchJobReplicateTarget) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobReplicateTarget) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 5
+	// map header, size 6
 	// string "Type"
-	o = append(o, 0x85, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	o = append(o, 0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	o = msgp.AppendString(o, string(z.Type))
 	// string "Bucket"
 	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
@@ -1168,6 +1209,9 @@ func (z *BatchJobReplicateTarget) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Endpoint"
 	o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
 	o = msgp.AppendString(o, z.Endpoint)
 	// string "Path"
 	o = append(o, 0xa4, 0x50, 0x61, 0x74, 0x68)
 	o = msgp.AppendString(o, z.Path)
 	// string "Creds"
 	o = append(o, 0xa5, 0x43, 0x72, 0x65, 0x64, 0x73)
 	// map header, size 3
@@ -1229,6 +1273,12 @@ func (z *BatchJobReplicateTarget) UnmarshalMsg(bts []byte) (o []byte, err error)
 				err = msgp.WrapError(err, "Endpoint")
 				return
 			}
 		case "Path":
 			z.Path, bts, err = msgp.ReadStringBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "Path")
 				return
 			}
 		case "Creds":
 			var zb0003 uint32
 			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
@@ -1284,7 +1334,7 @@ func (z *BatchJobReplicateTarget) UnmarshalMsg(bts []byte) (o []byte, err error)
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobReplicateTarget) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken)
+	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken)
 	return
 }
@@ -1538,6 +1588,24 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 					return
 				}
 			}
 		case "KeyRotate":
 			if dc.IsNil() {
 				err = dc.ReadNil()
 				if err != nil {
 					err = msgp.WrapError(err, "KeyRotate")
 					return
 				}
 				z.KeyRotate = nil
 			} else {
 				if z.KeyRotate == nil {
 					z.KeyRotate = new(BatchJobKeyRotateV1)
 				}
 				err = z.KeyRotate.DecodeMsg(dc)
 				if err != nil {
 					err = msgp.WrapError(err, "KeyRotate")
 					return
 				}
 			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -1551,9 +1619,9 @@ func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 5
+	// map header, size 6
 	// write "ID"
-	err = en.Append(0x85, 0xa2, 0x49, 0x44)
+	err = en.Append(0x86, 0xa2, 0x49, 0x44)
 	if err != nil {
 		return
 	}
@@ -1609,15 +1677,32 @@ func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) {
 			return
 		}
 	}
 	// write "KeyRotate"
 	err = en.Append(0xa9, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65)
 	if err != nil {
 		return
 	}
 	if z.KeyRotate == nil {
 		err = en.WriteNil()
 		if err != nil {
 			return
 		}
 	} else {
 		err = z.KeyRotate.EncodeMsg(en)
 		if err != nil {
 			err = msgp.WrapError(err, "KeyRotate")
 			return
 		}
 	}
 	return
 }
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 5
+	// map header, size 6
 	// string "ID"
-	o = append(o, 0x85, 0xa2, 0x49, 0x44)
+	o = append(o, 0x86, 0xa2, 0x49, 0x44)
 	o = msgp.AppendString(o, z.ID)
 	// string "User"
 	o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72)
@@ -1639,6 +1724,17 @@ func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) {
 			return
 		}
 	}
 	// string "KeyRotate"
 	o = append(o, 0xa9, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65)
 	if z.KeyRotate == nil {
 		o = msgp.AppendNil(o)
 	} else {
 		o, err = z.KeyRotate.MarshalMsg(o)
 		if err != nil {
 			err = msgp.WrapError(err, "KeyRotate")
 			return
 		}
 	}
 	return
 }
@@ -1701,6 +1797,23 @@ func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) {
 					return
 				}
 			}
 		case "KeyRotate":
 			if msgp.IsNil(bts) {
 				bts, err = msgp.ReadNilBytes(bts)
 				if err != nil {
 					return
 				}
 				z.KeyRotate = nil
 			} else {
 				if z.KeyRotate == nil {
 					z.KeyRotate = new(BatchJobKeyRotateV1)
 				}
 				bts, err = z.KeyRotate.UnmarshalMsg(bts)
 				if err != nil {
 					err = msgp.WrapError(err, "KeyRotate")
 					return
 				}
 			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -1721,6 +1834,12 @@ func (z *BatchJobRequest) Msgsize() (s int) {
 	} else {
 		s += z.Replicate.Msgsize()
 	}
 	s += 10
 	if z.KeyRotate == nil {
 		s += msgp.NilSize
 	} else {
 		s += z.KeyRotate.Msgsize()
 	}
 	return
 }
@@ -0,0 +1,564 @@
 // Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU Affero General Public License for more details.
 //
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package cmd
 import (
 	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
 	"math/rand"
 	"net/http"
 	"runtime"
 	"strconv"
 	"strings"
 	"time"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/env"
 	"github.com/minio/pkg/wildcard"
 	"github.com/minio/pkg/workers"
 )
 // keyrotate:
 //   apiVersion: v1
 //   bucket: BUCKET
 //   prefix: PREFIX
 //   encryption:
 //     type: sse-s3 # valid values are sse-s3 and sse-kms
 //     key: <new-kms-key> # valid only for sse-kms
 //     context: <new-kms-key-context> # valid only for sse-kms
 // # optional flags based filtering criteria
 // # for all objects
 // flags:
 //   filter:
 //     newerThan: "7d" # match objects newer than this value (e.g. 7d10h31s)
 //     olderThan: "7d" # match objects older than this value (e.g. 7d10h31s)
 //     createdAfter: "date" # match objects created after "date"
 //     createdBefore: "date" # match objects created before "date"
 //     tags:
 //       - key: "name"
 //         value: "pick*" # match objects with tag 'name', with all values starting with 'pick'
 //     metadata:
 //       - key: "content-type"
 //         value: "image/*" # match objects with 'content-type', with all values starting with 'image/'
 //     kmskey: "key-id" # match objects with KMS key-id (applicable only for sse-kms)
 //   notify:
 //     endpoint: "https://notify.endpoint" # notification endpoint to receive job status events
 //     token: "Bearer xxxxx" # optional authentication token for the notification endpoint
 //   retry:
 //     attempts: 10 # number of retries for the job before giving up
 //     delay: "500ms" # least amount of delay between each retry
 //go:generate msgp -file $GOFILE -unexported
 // BatchKeyRotateKV is a datatype that holds key and values for filtering of objects
 // used by metadata filter as well as tags based filtering.
 type BatchKeyRotateKV struct {
 	Key   string `yaml:"key" json:"key"`
 	Value string `yaml:"value" json:"value"`
 }
 // Validate returns an error if key is empty
 func (kv BatchKeyRotateKV) Validate() error {
 	if kv.Key == "" {
 		return errInvalidArgument
 	}
 	return nil
 }
 // Empty indicates if kv is not set
 func (kv BatchKeyRotateKV) Empty() bool {
 	return kv.Key == "" && kv.Value == ""
 }
 // Match matches input kv with kv, value will be wildcard matched depending on the user input
 func (kv BatchKeyRotateKV) Match(ikv BatchKeyRotateKV) bool {
 	if kv.Empty() {
 		return true
 	}
 	if strings.EqualFold(kv.Key, ikv.Key) {
 		return wildcard.Match(kv.Value, ikv.Value)
 	}
 	return false
 }
 // BatchKeyRotateRetry datatype represents total retry attempts and delay between each retries.
 type BatchKeyRotateRetry struct {
 	Attempts int           `yaml:"attempts" json:"attempts"` // number of retry attempts
 	Delay    time.Duration `yaml:"delay" json:"delay"`       // delay between each retries
 }
 // Validate validates input replicate retries.
 func (r BatchKeyRotateRetry) Validate() error {
 	if r.Attempts < 0 {
 		return errInvalidArgument
 	}
 	if r.Delay < 0 {
 		return errInvalidArgument
 	}
 	return nil
 }
 // BatchKeyRotationType defines key rotation type
 type BatchKeyRotationType string
 const (
 	sses3  BatchKeyRotationType = "sse-s3"
 	ssekms BatchKeyRotationType = "sse-kms"
 )
 // BatchJobKeyRotateEncryption defines key rotation encryption options passed
 type BatchJobKeyRotateEncryption struct {
 	Type       BatchKeyRotationType `yaml:"type" json:"type"`
 	Key        string               `yaml:"key" json:"key"`
 	Context    string               `yaml:"context" json:"context"`
 	kmsContext kms.Context          `msg:"-"`
 }
 // Validate validates input key rotation encryption options.
 func (e BatchJobKeyRotateEncryption) Validate() error {
 	if e.Type != sses3 && e.Type != ssekms {
 		return errInvalidArgument
 	}
 	spaces := strings.HasPrefix(e.Key, " ") || strings.HasSuffix(e.Key, " ")
 	if e.Type == ssekms && spaces {
 		return crypto.ErrInvalidEncryptionKeyID
 	}
 	if e.Type == ssekms && GlobalKMS != nil {
 		ctx := kms.Context{}
 		if e.Context != "" {
 			b, err := base64.StdEncoding.DecodeString(e.Context)
 			if err != nil {
 				return err
 			}
 			json := jsoniter.ConfigCompatibleWithStandardLibrary
 			if err := json.Unmarshal(b, &ctx); err != nil {
 				return err
 			}
 		}
 		e.kmsContext = kms.Context{}
 		for k, v := range ctx {
 			e.kmsContext[k] = v
 		}
 		ctx["MinIO batch API"] = "batchrotate" // Context for a test key operation
 		if _, err := GlobalKMS.GenerateKey(GlobalContext, e.Key, ctx); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 // BatchKeyRotateFilter holds all the filters currently supported for batch replication
 type BatchKeyRotateFilter struct {
 	NewerThan     time.Duration      `yaml:"newerThan,omitempty" json:"newerThan"`
 	OlderThan     time.Duration      `yaml:"olderThan,omitempty" json:"olderThan"`
 	CreatedAfter  time.Time          `yaml:"createdAfter,omitempty" json:"createdAfter"`
 	CreatedBefore time.Time          `yaml:"createdBefore,omitempty" json:"createdBefore"`
 	Tags          []BatchKeyRotateKV `yaml:"tags,omitempty" json:"tags"`
 	Metadata      []BatchKeyRotateKV `yaml:"metadata,omitempty" json:"metadata"`
 	KMSKeyID      string             `yaml:"kmskeyid" json:"kmskey"`
 }
 // BatchKeyRotateNotification success or failure notification endpoint for each job attempts
 type BatchKeyRotateNotification struct {
 	Endpoint string `yaml:"endpoint" json:"endpoint"`
 	Token    string `yaml:"token" json:"token"`
 }
 // BatchJobKeyRotateFlags various configurations for replication job definition currently includes
 // - filter
 // - notify
 // - retry
 type BatchJobKeyRotateFlags struct {
 	Filter BatchKeyRotateFilter       `yaml:"filter" json:"filter"`
 	Notify BatchKeyRotateNotification `yaml:"notify" json:"notify"`
 	Retry  BatchKeyRotateRetry        `yaml:"retry" json:"retry"`
 }
 // BatchJobKeyRotateV1 v1 of batch key rotation job
 type BatchJobKeyRotateV1 struct {
 	APIVersion string                      `yaml:"apiVersion" json:"apiVersion"`
 	Flags      BatchJobKeyRotateFlags      `yaml:"flags" json:"flags"`
 	Bucket     string                      `yaml:"bucket" json:"bucket"`
 	Prefix     string                      `yaml:"prefix" json:"prefix"`
 	Endpoint   string                      `yaml:"endpoint" json:"endpoint"`
 	Encryption BatchJobKeyRotateEncryption `yaml:"encryption" json:"encryption"`
 }
 // Notify notifies notification endpoint if configured regarding job failure or success.
 func (r BatchJobKeyRotateV1) Notify(ctx context.Context, body io.Reader) error {
 	if r.Flags.Notify.Endpoint == "" {
 		return nil
 	}
 	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
 	defer cancel()
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.Flags.Notify.Endpoint, body)
 	if err != nil {
 		return err
 	}
 	if r.Flags.Notify.Token != "" {
 		req.Header.Set("Authorization", r.Flags.Notify.Token)
 	}
 	clnt := http.Client{Transport: getRemoteInstanceTransport}
 	resp, err := clnt.Do(req)
 	if err != nil {
 		return err
 	}
 	xhttp.DrainBody(resp.Body)
 	if resp.StatusCode != http.StatusOK {
 		return errors.New(resp.Status)
 	}
 	return nil
 }
 // KeyRotate rotates encryption key of an object
 func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo) error {
 	srcBucket := r.Bucket
 	srcObject := objInfo.Name
 	if objInfo.DeleteMarker || !objInfo.VersionPurgeStatus.Empty() {
 		return nil
 	}
 	sseKMS := crypto.S3KMS.IsEncrypted(objInfo.UserDefined)
 	sseS3 := crypto.S3.IsEncrypted(objInfo.UserDefined)
 	if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
 		return errInvalidEncryptionParameters
 	}
 	if sseKMS && r.Encryption.Type == sses3 { // previously encrypted with sse-kms, now sse-s3 disallowed
 		return errInvalidEncryptionParameters
 	}
 	versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject)
 	lock := api.NewNSLock(r.Bucket, objInfo.Name)
 	lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
 	if err != nil {
 		return err
 	}
 	ctx = lkctx.Context()
 	defer lock.Unlock(lkctx)
 	opts := ObjectOptions{
 		VersionID:        objInfo.VersionID,
 		Versioned:        versioned,
 		VersionSuspended: versionSuspended,
 		NoLock:           true,
 	}
 	obj, err := api.GetObjectInfo(ctx, r.Bucket, objInfo.Name, opts)
 	if err != nil {
 		return err
 	}
 	oi := obj.Clone()
 	var (
 		newKeyID      string
 		newKeyContext kms.Context
 	)
 	encMetadata := make(map[string]string)
 	for k, v := range oi.UserDefined {
 		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
 			encMetadata[k] = v
 		}
 	}
 	if (sseKMS || sseS3) && r.Encryption.Type == ssekms {
 		if err = r.Encryption.Validate(); err != nil {
 			return err
 		}
 		newKeyID = strings.TrimPrefix(r.Encryption.Key, crypto.ARNPrefix)
 		newKeyContext = r.Encryption.kmsContext
 	}
 	if err = rotateKey(ctx, []byte{}, newKeyID, []byte{}, r.Bucket, oi.Name, encMetadata, newKeyContext); err != nil {
 		return err
 	}
 	// Since we are rotating the keys, make sure to update the metadata.
 	oi.metadataOnly = true
 	oi.keyRotation = true
 	for k, v := range encMetadata {
 		oi.UserDefined[k] = v
 	}
 	if _, err := api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, ObjectOptions{
 		VersionID: oi.VersionID,
 	}, ObjectOptions{
 		VersionID: oi.VersionID,
 		NoLock:    true,
 	}); err != nil {
 		return err
 	}
 	return nil
 }
 const (
 	batchKeyRotationName               = "batch-rotate.bin"
 	batchKeyRotationFormat             = 1
 	batchKeyRotateVersionV1            = 1
 	batchKeyRotateVersion              = batchKeyRotateVersionV1
 	batchKeyRotateAPIVersion           = "v1"
 	batchKeyRotateJobDefaultRetries    = 3
 	batchKeyRotateJobDefaultRetryDelay = 250 * time.Millisecond
 )
 // Start the batch key rottion job, resumes if there was a pending job via "job.ID"
 func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error {
 	ri := &batchJobInfo{
 		JobID:     job.ID,
 		JobType:   string(job.Type()),
 		StartTime: job.Started,
 	}
 	if err := ri.load(ctx, api, job); err != nil {
 		return err
 	}
 	globalBatchJobsMetrics.save(job.ID, ri)
 	lastObject := ri.Object
 	delay := job.KeyRotate.Flags.Retry.Delay
 	if delay == 0 {
 		delay = batchKeyRotateJobDefaultRetryDelay
 	}
 	rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
 	skip := func(info FileInfo) (ok bool) {
 		if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan {
 			// skip all objects that are newer than specified older duration
 			return false
 		}
 		if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan {
 			// skip all objects that are older than specified newer duration
 			return false
 		}
 		if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(info.ModTime) {
 			// skip all objects that are created before the specified time.
 			return false
 		}
 		if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(info.ModTime) {
 			// skip all objects that are created after the specified time.
 			return false
 		}
 		if len(r.Flags.Filter.Tags) > 0 {
 			// Only parse object tags if tags filter is specified.
 			tagMap := map[string]string{}
 			tagStr := info.Metadata[xhttp.AmzObjectTagging]
 			if len(tagStr) != 0 {
 				t, err := tags.ParseObjectTags(tagStr)
 				if err != nil {
 					return false
 				}
 				tagMap = t.ToMap()
 			}
 			for _, kv := range r.Flags.Filter.Tags {
 				for t, v := range tagMap {
 					if kv.Match(BatchKeyRotateKV{Key: t, Value: v}) {
 						return true
 					}
 				}
 			}
 			// None of the provided tags filter match skip the object
 			return false
 		}
 		if len(r.Flags.Filter.Metadata) > 0 {
 			for _, kv := range r.Flags.Filter.Metadata {
 				for k, v := range info.Metadata {
 					if !strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") && !isStandardHeader(k) {
 						continue
 					}
 					// We only need to match x-amz-meta or standardHeaders
 					if kv.Match(BatchKeyRotateKV{Key: k, Value: v}) {
 						return true
 					}
 				}
 			}
 			// None of the provided metadata filters match skip the object.
 			return false
 		}
 		if r.Flags.Filter.KMSKeyID != "" {
 			if v, ok := info.Metadata[xhttp.AmzServerSideEncryptionKmsID]; ok && strings.TrimPrefix(v, crypto.ARNPrefix) != r.Flags.Filter.KMSKeyID {
 				return false
 			}
 		}
 		return true
 	}
 	workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_KEYROTATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2)))
 	if err != nil {
 		return err
 	}
 	wk, err := workers.New(workerSize)
 	if err != nil {
 		// invalid worker size.
 		return err
 	}
 	retryAttempts := ri.RetryAttempts
 	ctx, cancel := context.WithCancel(ctx)
 	results := make(chan ObjectInfo, 100)
 	if err := api.Walk(ctx, r.Bucket, r.Prefix, results, ObjectOptions{
 		WalkMarker: lastObject,
 		WalkFilter: skip,
 	}); err != nil {
 		cancel()
 		// Do not need to retry if we can't list objects on source.
 		return err
 	}
 	for result := range results {
 		result := result
 		sseKMS := crypto.S3KMS.IsEncrypted(result.UserDefined)
 		sseS3 := crypto.S3.IsEncrypted(result.UserDefined)
 		if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed
 			continue
 		}
 		wk.Take()
 		go func() {
 			defer wk.Give()
 			for attempts := 1; attempts <= retryAttempts; attempts++ {
 				attempts := attempts
 				stopFn := globalBatchJobsMetrics.trace(batchKeyRotationMetricObject, job.ID, attempts, result)
 				success := true
 				if err := r.KeyRotate(ctx, api, result); err != nil {
 					stopFn(err)
 					logger.LogIf(ctx, err)
 					success = false
 				} else {
 					stopFn(nil)
 				}
 				ri.trackCurrentBucketObject(r.Bucket, result, success)
 				ri.RetryAttempts = attempts
 				globalBatchJobsMetrics.save(job.ID, ri)
 				// persist in-memory state to disk after every 10secs.
 				logger.LogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job))
 				if success {
 					break
 				}
 			}
 		}()
 	}
 	wk.Wait()
 	ri.Complete = ri.ObjectsFailed == 0
 	ri.Failed = ri.ObjectsFailed > 0
 	globalBatchJobsMetrics.save(job.ID, ri)
 	// persist in-memory state to disk.
 	logger.LogIf(ctx, ri.updateAfter(ctx, api, 0, job))
 	buf, _ := json.Marshal(ri)
 	if err := r.Notify(ctx, bytes.NewReader(buf)); err != nil {
 		logger.LogIf(ctx, fmt.Errorf("unable to notify %v", err))
 	}
 	cancel()
 	if ri.Failed {
 		ri.ObjectsFailed = 0
 		ri.Bucket = ""
 		ri.Object = ""
 		ri.Objects = 0
 		time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay)))
 	}
 	return nil
 }
 //msgp:ignore batchKeyRotationJobError
 type batchKeyRotationJobError struct {
 	Code           string
 	Description    string
 	HTTPStatusCode int
 }
 func (e batchKeyRotationJobError) Error() string {
 	return e.Description
 }
 // Validate validates the job definition input
 func (r *BatchJobKeyRotateV1) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error {
 	if r == nil {
 		return nil
 	}
 	if r.APIVersion != batchKeyRotateAPIVersion {
 		return errInvalidArgument
 	}
 	if r.Bucket == "" {
 		return errInvalidArgument
 	}
 	if _, err := o.GetBucketInfo(ctx, r.Bucket, BucketOptions{}); err != nil {
 		if isErrBucketNotFound(err) {
 			return batchKeyRotationJobError{
 				Code:           "NoSuchSourceBucket",
 				Description:    "The specified source bucket does not exist",
 				HTTPStatusCode: http.StatusNotFound,
 			}
 		}
 		return err
 	}
 	if GlobalKMS == nil {
 		return errKMSNotConfigured
 	}
 	if err := r.Encryption.Validate(); err != nil {
 		return err
 	}
 	for _, tag := range r.Flags.Filter.Tags {
 		if err := tag.Validate(); err != nil {
 			return err
 		}
 	}
 	for _, meta := range r.Flags.Filter.Metadata {
 		if err := meta.Validate(); err != nil {
 			return err
 		}
 	}
 	if err := r.Flags.Retry.Validate(); err != nil {
 		return err
 	}
 	return nil
 }
@@ -0,0 +1,801 @@
 package cmd
 // Code generated by github.com/tinylib/msgp DO NOT EDIT.
 import (
 	"bytes"
 	"testing"
 	"github.com/tinylib/msgp/msgp"
 )
 func TestMarshalUnmarshalBatchJobKeyRotateEncryption(t *testing.T) {
 	v := BatchJobKeyRotateEncryption{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	left, err := v.UnmarshalMsg(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
 	}
 	left, err = msgp.Skip(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
 	}
 }
 func BenchmarkMarshalMsgBatchJobKeyRotateEncryption(b *testing.B) {
 	v := BatchJobKeyRotateEncryption{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.MarshalMsg(nil)
 	}
 }
 func BenchmarkAppendMsgBatchJobKeyRotateEncryption(b *testing.B) {
 	v := BatchJobKeyRotateEncryption{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		bts, _ = v.MarshalMsg(bts[0:0])
 	}
 }
 func BenchmarkUnmarshalBatchJobKeyRotateEncryption(b *testing.B) {
 	v := BatchJobKeyRotateEncryption{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		_, err := v.UnmarshalMsg(bts)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestEncodeDecodeBatchJobKeyRotateEncryption(t *testing.T) {
 	v := BatchJobKeyRotateEncryption{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	m := v.Msgsize()
 	if buf.Len() > m {
 		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateEncryption Msgsize() is inaccurate")
 	}
 	vn := BatchJobKeyRotateEncryption{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
 	}
 	buf.Reset()
 	msgp.Encode(&buf, &v)
 	err = msgp.NewReader(&buf).Skip()
 	if err != nil {
 		t.Error(err)
 	}
 }
 func BenchmarkEncodeBatchJobKeyRotateEncryption(b *testing.B) {
 	v := BatchJobKeyRotateEncryption{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	en := msgp.NewWriter(msgp.Nowhere)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.EncodeMsg(en)
 	}
 	en.Flush()
 }
 func BenchmarkDecodeBatchJobKeyRotateEncryption(b *testing.B) {
 	v := BatchJobKeyRotateEncryption{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	rd := msgp.NewEndlessReader(buf.Bytes(), b)
 	dc := msgp.NewReader(rd)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		err := v.DecodeMsg(dc)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestMarshalUnmarshalBatchJobKeyRotateFlags(t *testing.T) {
 	v := BatchJobKeyRotateFlags{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	left, err := v.UnmarshalMsg(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
 	}
 	left, err = msgp.Skip(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
 	}
 }
 func BenchmarkMarshalMsgBatchJobKeyRotateFlags(b *testing.B) {
 	v := BatchJobKeyRotateFlags{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.MarshalMsg(nil)
 	}
 }
 func BenchmarkAppendMsgBatchJobKeyRotateFlags(b *testing.B) {
 	v := BatchJobKeyRotateFlags{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		bts, _ = v.MarshalMsg(bts[0:0])
 	}
 }
 func BenchmarkUnmarshalBatchJobKeyRotateFlags(b *testing.B) {
 	v := BatchJobKeyRotateFlags{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		_, err := v.UnmarshalMsg(bts)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestEncodeDecodeBatchJobKeyRotateFlags(t *testing.T) {
 	v := BatchJobKeyRotateFlags{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	m := v.Msgsize()
 	if buf.Len() > m {
 		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateFlags Msgsize() is inaccurate")
 	}
 	vn := BatchJobKeyRotateFlags{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
 	}
 	buf.Reset()
 	msgp.Encode(&buf, &v)
 	err = msgp.NewReader(&buf).Skip()
 	if err != nil {
 		t.Error(err)
 	}
 }
 func BenchmarkEncodeBatchJobKeyRotateFlags(b *testing.B) {
 	v := BatchJobKeyRotateFlags{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	en := msgp.NewWriter(msgp.Nowhere)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.EncodeMsg(en)
 	}
 	en.Flush()
 }
 func BenchmarkDecodeBatchJobKeyRotateFlags(b *testing.B) {
 	v := BatchJobKeyRotateFlags{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	rd := msgp.NewEndlessReader(buf.Bytes(), b)
 	dc := msgp.NewReader(rd)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		err := v.DecodeMsg(dc)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestMarshalUnmarshalBatchJobKeyRotateV1(t *testing.T) {
 	v := BatchJobKeyRotateV1{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	left, err := v.UnmarshalMsg(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
 	}
 	left, err = msgp.Skip(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
 	}
 }
 func BenchmarkMarshalMsgBatchJobKeyRotateV1(b *testing.B) {
 	v := BatchJobKeyRotateV1{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.MarshalMsg(nil)
 	}
 }
 func BenchmarkAppendMsgBatchJobKeyRotateV1(b *testing.B) {
 	v := BatchJobKeyRotateV1{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		bts, _ = v.MarshalMsg(bts[0:0])
 	}
 }
 func BenchmarkUnmarshalBatchJobKeyRotateV1(b *testing.B) {
 	v := BatchJobKeyRotateV1{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		_, err := v.UnmarshalMsg(bts)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestEncodeDecodeBatchJobKeyRotateV1(t *testing.T) {
 	v := BatchJobKeyRotateV1{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	m := v.Msgsize()
 	if buf.Len() > m {
 		t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateV1 Msgsize() is inaccurate")
 	}
 	vn := BatchJobKeyRotateV1{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
 	}
 	buf.Reset()
 	msgp.Encode(&buf, &v)
 	err = msgp.NewReader(&buf).Skip()
 	if err != nil {
 		t.Error(err)
 	}
 }
 func BenchmarkEncodeBatchJobKeyRotateV1(b *testing.B) {
 	v := BatchJobKeyRotateV1{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	en := msgp.NewWriter(msgp.Nowhere)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.EncodeMsg(en)
 	}
 	en.Flush()
 }
 func BenchmarkDecodeBatchJobKeyRotateV1(b *testing.B) {
 	v := BatchJobKeyRotateV1{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	rd := msgp.NewEndlessReader(buf.Bytes(), b)
 	dc := msgp.NewReader(rd)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		err := v.DecodeMsg(dc)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestMarshalUnmarshalBatchKeyRotateFilter(t *testing.T) {
 	v := BatchKeyRotateFilter{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	left, err := v.UnmarshalMsg(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
 	}
 	left, err = msgp.Skip(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
 	}
 }
 func BenchmarkMarshalMsgBatchKeyRotateFilter(b *testing.B) {
 	v := BatchKeyRotateFilter{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.MarshalMsg(nil)
 	}
 }
 func BenchmarkAppendMsgBatchKeyRotateFilter(b *testing.B) {
 	v := BatchKeyRotateFilter{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		bts, _ = v.MarshalMsg(bts[0:0])
 	}
 }
 func BenchmarkUnmarshalBatchKeyRotateFilter(b *testing.B) {
 	v := BatchKeyRotateFilter{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		_, err := v.UnmarshalMsg(bts)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestEncodeDecodeBatchKeyRotateFilter(t *testing.T) {
 	v := BatchKeyRotateFilter{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	m := v.Msgsize()
 	if buf.Len() > m {
 		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateFilter Msgsize() is inaccurate")
 	}
 	vn := BatchKeyRotateFilter{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
 	}
 	buf.Reset()
 	msgp.Encode(&buf, &v)
 	err = msgp.NewReader(&buf).Skip()
 	if err != nil {
 		t.Error(err)
 	}
 }
 func BenchmarkEncodeBatchKeyRotateFilter(b *testing.B) {
 	v := BatchKeyRotateFilter{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	en := msgp.NewWriter(msgp.Nowhere)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.EncodeMsg(en)
 	}
 	en.Flush()
 }
 func BenchmarkDecodeBatchKeyRotateFilter(b *testing.B) {
 	v := BatchKeyRotateFilter{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	rd := msgp.NewEndlessReader(buf.Bytes(), b)
 	dc := msgp.NewReader(rd)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		err := v.DecodeMsg(dc)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestMarshalUnmarshalBatchKeyRotateKV(t *testing.T) {
 	v := BatchKeyRotateKV{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	left, err := v.UnmarshalMsg(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
 	}
 	left, err = msgp.Skip(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
 	}
 }
 func BenchmarkMarshalMsgBatchKeyRotateKV(b *testing.B) {
 	v := BatchKeyRotateKV{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.MarshalMsg(nil)
 	}
 }
 func BenchmarkAppendMsgBatchKeyRotateKV(b *testing.B) {
 	v := BatchKeyRotateKV{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		bts, _ = v.MarshalMsg(bts[0:0])
 	}
 }
 func BenchmarkUnmarshalBatchKeyRotateKV(b *testing.B) {
 	v := BatchKeyRotateKV{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		_, err := v.UnmarshalMsg(bts)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestEncodeDecodeBatchKeyRotateKV(t *testing.T) {
 	v := BatchKeyRotateKV{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	m := v.Msgsize()
 	if buf.Len() > m {
 		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateKV Msgsize() is inaccurate")
 	}
 	vn := BatchKeyRotateKV{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
 	}
 	buf.Reset()
 	msgp.Encode(&buf, &v)
 	err = msgp.NewReader(&buf).Skip()
 	if err != nil {
 		t.Error(err)
 	}
 }
 func BenchmarkEncodeBatchKeyRotateKV(b *testing.B) {
 	v := BatchKeyRotateKV{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	en := msgp.NewWriter(msgp.Nowhere)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.EncodeMsg(en)
 	}
 	en.Flush()
 }
 func BenchmarkDecodeBatchKeyRotateKV(b *testing.B) {
 	v := BatchKeyRotateKV{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	rd := msgp.NewEndlessReader(buf.Bytes(), b)
 	dc := msgp.NewReader(rd)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		err := v.DecodeMsg(dc)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestMarshalUnmarshalBatchKeyRotateNotification(t *testing.T) {
 	v := BatchKeyRotateNotification{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	left, err := v.UnmarshalMsg(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
 	}
 	left, err = msgp.Skip(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
 	}
 }
 func BenchmarkMarshalMsgBatchKeyRotateNotification(b *testing.B) {
 	v := BatchKeyRotateNotification{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.MarshalMsg(nil)
 	}
 }
 func BenchmarkAppendMsgBatchKeyRotateNotification(b *testing.B) {
 	v := BatchKeyRotateNotification{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		bts, _ = v.MarshalMsg(bts[0:0])
 	}
 }
 func BenchmarkUnmarshalBatchKeyRotateNotification(b *testing.B) {
 	v := BatchKeyRotateNotification{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		_, err := v.UnmarshalMsg(bts)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestEncodeDecodeBatchKeyRotateNotification(t *testing.T) {
 	v := BatchKeyRotateNotification{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	m := v.Msgsize()
 	if buf.Len() > m {
 		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateNotification Msgsize() is inaccurate")
 	}
 	vn := BatchKeyRotateNotification{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
 	}
 	buf.Reset()
 	msgp.Encode(&buf, &v)
 	err = msgp.NewReader(&buf).Skip()
 	if err != nil {
 		t.Error(err)
 	}
 }
 func BenchmarkEncodeBatchKeyRotateNotification(b *testing.B) {
 	v := BatchKeyRotateNotification{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	en := msgp.NewWriter(msgp.Nowhere)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.EncodeMsg(en)
 	}
 	en.Flush()
 }
 func BenchmarkDecodeBatchKeyRotateNotification(b *testing.B) {
 	v := BatchKeyRotateNotification{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	rd := msgp.NewEndlessReader(buf.Bytes(), b)
 	dc := msgp.NewReader(rd)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		err := v.DecodeMsg(dc)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestMarshalUnmarshalBatchKeyRotateRetry(t *testing.T) {
 	v := BatchKeyRotateRetry{}
 	bts, err := v.MarshalMsg(nil)
 	if err != nil {
 		t.Fatal(err)
 	}
 	left, err := v.UnmarshalMsg(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left)
 	}
 	left, err = msgp.Skip(bts)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(left) > 0 {
 		t.Errorf("%d bytes left over after Skip(): %q", len(left), left)
 	}
 }
 func BenchmarkMarshalMsgBatchKeyRotateRetry(b *testing.B) {
 	v := BatchKeyRotateRetry{}
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.MarshalMsg(nil)
 	}
 }
 func BenchmarkAppendMsgBatchKeyRotateRetry(b *testing.B) {
 	v := BatchKeyRotateRetry{}
 	bts := make([]byte, 0, v.Msgsize())
 	bts, _ = v.MarshalMsg(bts[0:0])
 	b.SetBytes(int64(len(bts)))
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		bts, _ = v.MarshalMsg(bts[0:0])
 	}
 }
 func BenchmarkUnmarshalBatchKeyRotateRetry(b *testing.B) {
 	v := BatchKeyRotateRetry{}
 	bts, _ := v.MarshalMsg(nil)
 	b.ReportAllocs()
 	b.SetBytes(int64(len(bts)))
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		_, err := v.UnmarshalMsg(bts)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
 func TestEncodeDecodeBatchKeyRotateRetry(t *testing.T) {
 	v := BatchKeyRotateRetry{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	m := v.Msgsize()
 	if buf.Len() > m {
 		t.Log("WARNING: TestEncodeDecodeBatchKeyRotateRetry Msgsize() is inaccurate")
 	}
 	vn := BatchKeyRotateRetry{}
 	err := msgp.Decode(&buf, &vn)
 	if err != nil {
 		t.Error(err)
 	}
 	buf.Reset()
 	msgp.Encode(&buf, &v)
 	err = msgp.NewReader(&buf).Skip()
 	if err != nil {
 		t.Error(err)
 	}
 }
 func BenchmarkEncodeBatchKeyRotateRetry(b *testing.B) {
 	v := BatchKeyRotateRetry{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	en := msgp.NewWriter(msgp.Nowhere)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		v.EncodeMsg(en)
 	}
 	en.Flush()
 }
 func BenchmarkDecodeBatchKeyRotateRetry(b *testing.B) {
 	v := BatchKeyRotateRetry{}
 	var buf bytes.Buffer
 	msgp.Encode(&buf, &v)
 	b.SetBytes(int64(buf.Len()))
 	rd := msgp.NewEndlessReader(buf.Bytes(), b)
 	dc := msgp.NewReader(rd)
 	b.ReportAllocs()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		err := v.DecodeMsg(dc)
 		if err != nil {
 			b.Fatal(err)
 		}
 	}
 }
@@ -0,0 +1,24 @@
 // Code generated by "stringer -type=batchJobMetric -trimprefix=batchJobMetric batch-handlers.go"; DO NOT EDIT.
 package cmd
 import "strconv"
 func _() {
 	// An "invalid array index" compiler error signifies that the constant values have changed.
 	// Re-run the stringer command to generate them again.
 	var x [1]struct{}
 	_ = x[batchReplicationMetricObject-0]
 	_ = x[batchKeyRotationMetricObject-1]
 }
 const _batchJobMetric_name = "batchReplicationMetricObjectbatchKeyRotationMetricObject"
 var _batchJobMetric_index = [...]uint8{0, 28, 56}
 func (i batchJobMetric) String() string {
 	if i >= batchJobMetric(len(_batchJobMetric_index)-1) {
 		return "batchJobMetric(" + strconv.FormatInt(int64(i), 10) + ")"
 	}
 	return _batchJobMetric_name[_batchJobMetric_index[i]:_batchJobMetric_index[i+1]]
 }
@@ -1,23 +0,0 @@
 // Code generated by "stringer -type=batchReplicationMetric -trimprefix=batchReplicationMetric batch-handlers.go"; DO NOT EDIT.
 package cmd
 import "strconv"
 func _() {
 	// An "invalid array index" compiler error signifies that the constant values have changed.
 	// Re-run the stringer command to generate them again.
 	var x [1]struct{}
 	_ = x[batchReplicationMetricObject-0]
 }
 const _batchReplicationMetric_name = "Object"
 var _batchReplicationMetric_index = [...]uint8{0, 6}
 func (i batchReplicationMetric) String() string {
 	if i >= batchReplicationMetric(len(_batchReplicationMetric_index)-1) {
 		return "batchReplicationMetric(" + strconv.FormatInt(int64(i), 10) + ")"
 	}
 	return _batchReplicationMetric_name[_batchReplicationMetric_index[i]:_batchReplicationMetric_index[i+1]]
 }
@@ -165,9 +165,9 @@ func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) {
 			b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset)
 			if err != nil {
 				if !IsErr(err, ignoredErrs...) {
-					logger.LogIf(GlobalContext,
+					logger.LogOnceIf(GlobalContext,
 						fmt.Errorf("Reading erasure shards at (%s: %s/%s) returned '%w', will attempt to reconstruct if we have quorum",
-							b.disk, b.volume, b.filePath, err))
+							b.disk, b.volume, b.filePath, err), "bitrot-read-file-stream"+b.volume+b.filePath)
 				}
 			}
 		} else {
@@ -23,7 +23,7 @@ import (
 	"sync"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/pubsub"
 )
@@ -62,8 +62,8 @@ func (bs *bootstrapTracer) Empty() bool {
 	return empty
 }
-func (bs *bootstrapTracer) Record(msg string) {
+func (bs *bootstrapTracer) Record(msg string, skip int) {
-	source := getSource(2)
+	source := getSource(skip + 1)
 	bs.mu.Lock()
 	now := time.Now().UTC()
 	bs.info[bs.idx] = bootstrapInfo{
@@ -28,7 +28,7 @@ func TestBootstrap(t *testing.T) {
 	// Bootstrap events exceed bootstrap messages limit
 	bsTracer := &bootstrapTracer{}
 	for i := 0; i < bootstrapMsgsLimit+10; i++ {
-		bsTracer.Record(fmt.Sprintf("msg-%d", i))
+		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
 	}
 	traceInfos := bsTracer.Events()
@@ -45,7 +45,7 @@ func TestBootstrap(t *testing.T) {
 	// Fewer than 4K bootstrap events
 	for i := 0; i < 10; i++ {
-		bsTracer.Record(fmt.Sprintf("msg-%d", i))
+		bsTracer.Record(fmt.Sprintf("msg-%d", i), 1)
 	}
 	events := bsTracer.Events()
 	if len(events) != 10 {
@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
 	"os"
 	"reflect"
 	"time"
@@ -216,6 +217,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 	for onlineServers < len(clnts)/2 {
 		for _, clnt := range clnts {
 			if err := clnt.Verify(ctx, srcCfg); err != nil {
 				bootstrapTrace(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
 				if !isNetworkError(err) {
 					logger.LogOnceIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err), clnt.String())
 					incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
@@ -264,7 +266,11 @@ func newBootstrapRESTClients(endpointServerPools EndpointServerPools) []*bootstr
 			// Only proceed for remote endpoints.
 			if !endpoint.IsLocal {
-				clnts = append(clnts, newBootstrapRESTClient(endpoint))
+				cl := newBootstrapRESTClient(endpoint)
 				if serverDebugLog {
 					cl.restClient.TraceOutput = os.Stdout
 				}
 				clnts = append(clnts, cl)
 			}
 		}
 	}
@@ -26,7 +26,7 @@ import (
 	"net/http"
 	"github.com/minio/kes-go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
@@ -21,13 +21,17 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
 	"encoding/json"
 	"encoding/xml"
 	"errors"
 	"fmt"
 	"io"
 	"mime/multipart"
 	"net/http"
 	"net/textproto"
 	"net/url"
 	"path"
 	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -35,8 +39,10 @@ import (
 	"github.com/google/uuid"
 	"github.com/minio/mux"
 	"github.com/valyala/bytebufferpool"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/auth"
@@ -49,17 +55,21 @@ import (
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/bucket/policy"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	"github.com/minio/pkg/sync/errgroup"
 )
 const (
 	objectLockConfig        = "object-lock.xml"
 	bucketTaggingConfig     = "tagging.xml"
 	bucketReplicationConfig = "replication.xml"
 	xMinIOErrCodeHeader = "x-minio-error-code"
 	xMinIOErrDescHeader = "x-minio-error-desc"
 )
 // Check if there are buckets on server without corresponding entry in etcd backend and
@@ -429,6 +439,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// The max. XML contains 100000 object names (each at most 1024 bytes long) + XML overhead
 	const maxBodySize = 2 * 100000 * 1024
 	if r.ContentLength > maxBodySize {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL)
 		return
 	}
 	// Unmarshal list of keys to be deleted.
 	deleteObjectsReq := &DeleteObjectsRequest{}
 	if err := xmlDecoder(r.Body, deleteObjectsReq, maxBodySize); err != nil {
@@ -540,6 +555,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			oss[index].SetTransitionState(goi.TransitionedObject)
 		}
 		// All deletes on directory objects needs to be for `nullVersionID`
 		if isDirObject(object.ObjectName) && object.VersionID == "" {
 			object.VersionID = nullVersionID
 		}
 		if replicateDeletes {
 			dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{
 				ObjectV: ObjectV{
@@ -634,6 +654,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		if deleteResult.errInfo.Code != "" {
 			deleteErrors = append(deleteErrors, deleteResult.errInfo)
 		} else {
 			// All deletes on directory objects was with `nullVersionID`.
 			// Remove it from response.
 			if isDirObject(deleteResult.delInfo.ObjectName) && deleteResult.delInfo.VersionID == nullVersionID {
 				deleteResult.delInfo.VersionID = ""
 			}
 			deletedObjects = append(deletedObjects, deleteResult.delInfo)
 		}
 	}
@@ -649,6 +674,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 		}
 		if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == Pending) {
 			// copy so we can re-add null ID.
 			dobj := dobj
 			if isDirObject(dobj.ObjectName) && dobj.VersionID == "" {
 				dobj.VersionID = nullVersionID
 			}
 			dv := DeletedObjectReplicationInfo{
 				DeletedObject: dobj,
 				Bucket:        bucket,
@@ -882,14 +912,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}
 	bucket := mux.Vars(r)["bucket"]
 	// Require Content-Length to be set in the request
 	size := r.ContentLength
 	if size < 0 {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
 		return
 	}
 	resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
 	if err != nil {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
@@ -904,43 +926,151 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	// Here the parameter is the size of the form data that should
 	// be loaded in memory, the remaining being put in temporary files.
-	reader, err := r.MultipartReader()
+	mp, err := r.MultipartReader()
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
-		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
+		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
-	// Read multipart data and save in memory and in the disk if needed
+	const mapEntryOverhead = 200
-	form, err := reader.ReadForm(maxFormMemory)
+
-	if err != nil {
+	var (
 		reader        io.Reader
 		fileSize      int64 = -1
 		fileName      string
 		fanOutEntries = make([]minio.PutObjectFanOutEntry, 0, 100)
 	)
 	maxParts := 1000
 	// Canonicalize the form values into http.Header.
 	formValues := make(http.Header)
 	for {
 		part, err := mp.NextRawPart()
 		if errors.Is(err, io.EOF) {
 			break
 		}
 		if err != nil {
 			apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 			apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
 			writeErrorResponse(ctx, w, apiErr, r.URL)
 			return
 		}
 		if maxParts <= 0 {
 			apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 			apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
 			writeErrorResponse(ctx, w, apiErr, r.URL)
 			return
 		}
 		maxParts--
 		name := part.FormName()
 		if name == "" {
 			continue
 		}
 		fileName = part.FileName()
 		// Multiple values for the same key (one map entry, longer slice) are cheaper
 		// than the same number of values for different keys (many map entries), but
 		// using a consistent per-value cost for overhead is simpler.
 		maxMemoryBytes := 2 * int64(10<<20)
 		maxMemoryBytes -= int64(len(name))
 		maxMemoryBytes -= mapEntryOverhead
 		if maxMemoryBytes < 0 {
 			// We can't actually take this path, since nextPart would already have
 			// rejected the MIME headers for being too large. Check anyway.
 			apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 			apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
 			writeErrorResponse(ctx, w, apiErr, r.URL)
 			return
 		}
 		var b bytes.Buffer
 		if fileName == "" {
 			if http.CanonicalHeaderKey(name) == http.CanonicalHeaderKey("x-minio-fanout-list") {
 				dec := json.NewDecoder(part)
 				// while the array contains values
 				for dec.More() {
 					var m minio.PutObjectFanOutEntry
 					if err := dec.Decode(&m); err != nil {
 						part.Close()
 						apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 						apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
 						writeErrorResponse(ctx, w, apiErr, r.URL)
 						return
 					}
 					fanOutEntries = append(fanOutEntries, m)
 				}
 				part.Close()
 				continue
 			}
 			// value, store as string in memory
 			n, err := io.CopyN(&b, part, maxMemoryBytes+1)
 			part.Close()
 			if err != nil && err != io.EOF {
 				apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 				apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
 				writeErrorResponse(ctx, w, apiErr, r.URL)
 				return
 			}
 			maxMemoryBytes -= n
 			if maxMemoryBytes < 0 {
 				apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 				apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
 				writeErrorResponse(ctx, w, apiErr, r.URL)
 				return
 			}
 			if n > maxFormFieldSize {
 				apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 				apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
 				writeErrorResponse(ctx, w, apiErr, r.URL)
 				return
 			}
 			formValues[http.CanonicalHeaderKey(name)] = append(formValues[http.CanonicalHeaderKey(name)], b.String())
 			continue
 		}
 		// In accordance with https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html
 		// The file or text content.
 		// The file or text content must be the last field in the form.
 		// You cannot upload more than one file at a time.
 		reader = part
 		// we have found the File part of the request we are done processing multipart-form
 		break
 	}
 	if _, ok := formValues["Key"]; !ok {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
-		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
+		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The name of the uploaded key is missing"))
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
-	// Remove all tmp files created during multipart upload
+	if fileName == "" {
 	defer form.RemoveAll()
 	// Extract all form fields
 	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
-		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
+		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The file or text content is missing"))
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
-
+	checksum, err := hash.GetContentChecksum(formValues)
-	// Check if file is provided, error out otherwise.
+	if err != nil {
-	if fileBody == nil {
+		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPOSTFileRequired), r.URL)
+		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, fmt.Errorf("Invalid checksum: %w", err))
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
 	if checksum != nil && checksum.Type.Trailing() {
 		// Not officially supported in POST requests.
 		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
 		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("Trailing checksums not available for POST operations"))
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
 	// Close multipart file
 	defer fileBody.Close()
 	formValues.Set("Bucket", bucket)
 	if fileName != "" && strings.Contains(formValues.Get("Key"), "${filename}") {
@@ -968,20 +1098,38 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
-	// Once signature is validated, check if the user has
+	if len(fanOutEntries) > 0 {
-	// explicit permissions for the user.
+		// Once signature is validated, check if the user has
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
+		// explicit permissions for the user.
-		AccountName:     cred.AccessKey,
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
-		Groups:          cred.Groups,
+			AccountName:     cred.AccessKey,
-		Action:          iampolicy.PutObjectAction,
+			Groups:          cred.Groups,
-		ConditionValues: getConditionValues(r, "", cred),
+			Action:          iampolicy.PutObjectFanOutAction,
-		BucketName:      bucket,
+			ConditionValues: getConditionValues(r, "", cred),
-		ObjectName:      object,
+			BucketName:      bucket,
-		IsOwner:         globalActiveCred.AccessKey == cred.AccessKey,
+			ObjectName:      object,
-		Claims:          cred.Claims,
+			IsOwner:         globalActiveCred.AccessKey == cred.AccessKey,
-	}) {
+			Claims:          cred.Claims,
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		}) {
-		return
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 			return
 		}
 	} else {
 		// Once signature is validated, check if the user has
 		// explicit permissions for the user.
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
 			Action:          iampolicy.PutObjectAction,
 			ConditionValues: getConditionValues(r, "", cred),
 			BucketName:      bucket,
 			ObjectName:      object,
 			IsOwner:         globalActiveCred.AccessKey == cred.AccessKey,
 			Claims:          cred.Claims,
 		}) {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 			return
 		}
 	}
 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
@@ -990,6 +1138,19 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	hashReader, err := hash.NewReader(reader, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 	if checksum != nil && checksum.Valid() {
 		if err = hashReader.AddChecksumNoTrailer(formValues, false); err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
 	}
 	// Handle policy if it is set.
 	if len(policyBytes) > 0 {
 		postPolicyForm, err := parsePostPolicyForm(bytes.NewReader(policyBytes))
@@ -1010,15 +1171,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		// should not exceed the maximum single Put size (5 GiB)
 		lengthRange := postPolicyForm.Conditions.ContentLengthRange
 		if lengthRange.Valid {
-			if fileSize < lengthRange.Min {
+			hashReader.SetExpectedMin(lengthRange.Min)
-				writeErrorResponse(ctx, w, toAPIError(ctx, errDataTooSmall), r.URL)
+			hashReader.SetExpectedMax(lengthRange.Max)
 				return
 			}
 			if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
 				writeErrorResponse(ctx, w, toAPIError(ctx, errDataTooLarge), r.URL)
 				return
 			}
 		}
 	}
@@ -1030,19 +1184,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 	rawReader := hashReader
 	pReader := NewPutObjReader(rawReader)
 	var objectEncryptionKey crypto.ObjectKey
 	// Check if bucket encryption is enabled
 	sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
-	sseConfig.Apply(r.Header, sse.ApplyOptions{
+	sseConfig.Apply(formValues, sse.ApplyOptions{
 		AutoEncrypt: globalAutoEncryption,
 	})
@@ -1053,12 +1201,24 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	fanOutOpts := fanOutOptions{Checksum: checksum}
 	if crypto.Requested(formValues) {
 		if crypto.SSECopy.IsRequested(r.Header) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
 			return
 		}
 		if crypto.SSEC.IsRequested(r.Header) && crypto.S3.IsRequested(r.Header) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL)
 			return
 		}
 		if crypto.SSEC.IsRequested(r.Header) && crypto.S3KMS.IsRequested(r.Header) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL)
 			return
 		}
 		if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
 			return
@@ -1085,23 +1245,147 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				return
 			}
 		}
-		reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
+
 		if len(fanOutEntries) == 0 {
 			reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
 			// do not try to verify encrypted content/
 			hashReader, err = hash.NewReader(reader, -1, "", "", -1)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
 			if checksum != nil && checksum.Valid() {
 				if err = hashReader.AddChecksumNoTrailer(formValues, true); err != nil {
 					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 					return
 				}
 			}
 			pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
 			if err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
 		} else {
 			fanOutOpts = fanOutOptions{
 				Key:      key,
 				Kind:     kind,
 				KeyID:    keyID,
 				KmsCtx:   kmsCtx,
 				Checksum: checksum,
 			}
 		}
 	}
 	if len(fanOutEntries) > 0 {
 		// Fan-out requires no copying, and must be carried from original source
 		// https://en.wikipedia.org/wiki/Copy_protection so the incoming stream
 		// is always going to be in-memory as we cannot re-read from what we
 		// wrote to disk - since that amounts to "copying" from a "copy"
 		// instead of "copying" from source, we need the stream to be seekable
 		// to ensure that we can make fan-out calls concurrently.
 		buf := bytebufferpool.Get()
 		defer bytebufferpool.Put(buf)
 		// Maximum allowed fan-out object size.
 		const maxFanOutSize = 16 << 20
 		n, err := io.Copy(buf, ioutil.HardLimitReader(pReader, maxFanOutSize))
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-		info := ObjectInfo{Size: fileSize}
+
-		// do not try to verify encrypted content
+		concurrentSize := 100
-		hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize)
+		if runtime.GOMAXPROCS(0) < concurrentSize {
-		if err != nil {
+			concurrentSize = runtime.GOMAXPROCS(0)
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-		pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
+
-		if err != nil {
+		fanOutResp := make([]minio.PutObjectFanOutResponse, 0, len(fanOutEntries))
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		eventArgsList := make([]eventArgs, 0, len(fanOutEntries))
-			return
+		for {
 			var objInfos []ObjectInfo
 			var errs []error
 			var done bool
 			if len(fanOutEntries) < concurrentSize {
 				objInfos, errs = fanOutPutObject(ctx, bucket, objectAPI, fanOutEntries, buf.Bytes()[:n], fanOutOpts)
 				done = true
 			} else {
 				objInfos, errs = fanOutPutObject(ctx, bucket, objectAPI, fanOutEntries[:concurrentSize], buf.Bytes()[:n], fanOutOpts)
 				fanOutEntries = fanOutEntries[concurrentSize:]
 			}
 			for i, objInfo := range objInfos {
 				if errs[i] != nil {
 					fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{
 						Key:   objInfo.Name,
 						Error: errs[i].Error(),
 					})
 					eventArgsList = append(eventArgsList, eventArgs{
 						EventName:    event.ObjectCreatedPost,
 						BucketName:   objInfo.Bucket,
 						Object:       ObjectInfo{Name: objInfo.Name},
 						ReqParams:    extractReqParams(r),
 						RespElements: extractRespElements(w),
 						UserAgent:    fmt.Sprintf("%s MinIO-Fan-Out (failed: %v)", r.UserAgent(), errs[i]),
 						Host:         handlers.GetSourceIP(r),
 					})
 					continue
 				}
 				fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{
 					Key:          objInfo.Name,
 					ETag:         getDecryptedETag(formValues, objInfo, false),
 					VersionID:    objInfo.VersionID,
 					LastModified: &objInfo.ModTime,
 				})
 				eventArgsList = append(eventArgsList, eventArgs{
 					EventName:    event.ObjectCreatedPost,
 					BucketName:   objInfo.Bucket,
 					Object:       objInfo,
 					ReqParams:    extractReqParams(r),
 					RespElements: extractRespElements(w),
 					UserAgent:    r.UserAgent() + " " + "MinIO-Fan-Out",
 					Host:         handlers.GetSourceIP(r),
 				})
 			}
 			if done {
 				break
 			}
 		}
 		enc := json.NewEncoder(w)
 		for i, fanOutResp := range fanOutResp {
 			if err = enc.Encode(&fanOutResp); err != nil {
 				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 				return
 			}
 			// Notify object created events.
 			sendEvent(eventArgsList[i])
 			if eventArgsList[i].Object.NumVersions > dataScannerExcessiveVersionsThreshold {
 				// Send events for excessive versions.
 				sendEvent(eventArgs{
 					EventName:    event.ObjectManyVersions,
 					BucketName:   eventArgsList[i].Object.Bucket,
 					Object:       eventArgsList[i].Object,
 					ReqParams:    extractReqParams(r),
 					RespElements: extractRespElements(w),
 					UserAgent:    r.UserAgent() + " " + "MinIO-Fan-Out",
 					Host:         handlers.GetSourceIP(r),
 				})
 			}
 		}
 		return
 	}
 	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, opts)
@@ -1116,7 +1400,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}
 	// Set the relevant version ID as part of the response header.
-	if objInfo.VersionID != "" {
+	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}
@@ -1134,6 +1418,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		UserAgent:    r.UserAgent(),
 		Host:         handlers.GetSourceIP(r),
 	})
 	if objInfo.NumVersions > dataScannerExcessiveVersionsThreshold {
 		defer sendEvent(eventArgs{
 			EventName:    event.ObjectManyVersions,
@@ -1156,6 +1441,11 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}
 	// Add checksum header.
 	if checksum != nil && checksum.Valid() {
 		hash.AddChecksumHeader(w, checksum.AsMap())
 	}
 	// Decide what http response to send depending on success_action_status parameter
 	switch successStatus {
 	case "201":
@@ -1324,14 +1614,6 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 	}
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(bucket); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually", err))
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
 	}
 	deleteBucket := objectAPI.DeleteBucket
 	// Attempt to delete bucket.
@@ -1345,15 +1627,18 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 				apiErr.Description = "The bucket you tried to delete is not empty. You must delete all versions in the bucket."
 			}
 		}
 		if globalDNSConfig != nil {
 			if err2 := globalDNSConfig.Put(bucket); err2 != nil {
 				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, please fix it manually", err2))
 			}
 		}
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
 	if globalDNSConfig != nil {
 		if err := globalDNSConfig.Delete(bucket); err != nil {
 			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually, bucket on MinIO no longer exists", err))
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
 	}
 	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
 	globalReplicationPool.deleteResyncMetadata(ctx, bucket)
@@ -1398,7 +1683,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 	config, err := objectlock.ParseObjectLockConfig(r.Body)
 	if err != nil {
-		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
+		apiErr := errorCodes.ToAPIErr(ErrInvalidArgument)
 		apiErr.Description = err.Error()
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
@@ -1412,7 +1697,11 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri
 	// Deny object locking configuration settings on existing buckets without object lock enabled.
 	if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		if _, ok := err.(BucketObjectLockConfigNotFound); ok {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed), r.URL)
 		} else {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		}
 		return
 	}
@@ -1614,5 +1903,5 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 	}))
 	// Write success response.
-	writeSuccessResponseHeadersOnly(w)
+	writeSuccessNoContent(w)
 }
@@ -657,13 +657,14 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 ) {
 	var err error
 	contentBytes := []byte("hello")
 	sha256sum := ""
 	var objectNames []string
 	for i := 0; i < 10; i++ {
 		contentBytes := []byte("hello")
 		objectName := "test-object-" + strconv.Itoa(i)
 		if i == 0 {
 			objectName += "/"
 			contentBytes = []byte{}
 		}
 		// uploading the object.
 		_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
@@ -676,6 +677,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		objectNames = append(objectNames, objectName)
 	}
 	contentBytes := []byte("hello")
 	for _, name := range []string{"private/object", "public/object"} {
 		// Uploading the object with retention enabled
 		_, err = obj.PutObject(GlobalContext, bucketName, name, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
@@ -745,8 +747,13 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	deletedObjects := make([]DeletedObject, len(requestList[0].Objects))
 	for i := range requestList[0].Objects {
 		var vid string
 		if isDirObject(requestList[0].Objects[i].ObjectName) {
 			vid = ""
 		}
 		deletedObjects[i] = DeletedObject{
 			ObjectName: requestList[0].Objects[i].ObjectName,
 			VersionID:  vid,
 		}
 	}
@@ -756,9 +763,14 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	successRequest1 := encodeResponse(requestList[1])
 	deletedObjects = make([]DeletedObject, len(requestList[1].Objects))
-	for i := range requestList[0].Objects {
+	for i := range requestList[1].Objects {
 		var vid string
 		if isDirObject(requestList[0].Objects[i].ObjectName) {
 			vid = ""
 		}
 		deletedObjects[i] = DeletedObject{
 			ObjectName: requestList[1].Objects[i].ObjectName,
 			VersionID:  vid,
 		}
 	}
@@ -0,0 +1,88 @@
 // Copyright (c) 2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU Affero General Public License for more details.
 //
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package cmd
 import "github.com/minio/minio/internal/bucket/lifecycle"
 //go:generate stringer -type lcEventSrc -trimprefix lcEventSrc_ $GOFILE
 type lcEventSrc uint8
 //revive:disable:var-naming Underscores is used here to indicate where common prefix ends and the enumeration name begins
 const (
 	lcEventSrc_None lcEventSrc = iota
 	lcEventSrc_Scanner
 	lcEventSrc_Decom
 	lcEventSrc_Rebal
 	lcEventSrc_s3HeadObject
 	lcEventSrc_s3GetObject
 	lcEventSrc_s3ListObjects
 	lcEventSrc_s3PutObject
 	lcEventSrc_s3CopyObject
 	lcEventSrc_s3CompleteMultipartUpload
 )
 //revive:enable:var-naming
 type lcAuditEvent struct {
 	lifecycle.Event
 	source lcEventSrc
 }
 func (lae lcAuditEvent) Tags() map[string]interface{} {
 	event := lae.Event
 	src := lae.source
 	const (
 		ilmSrc                     = "ilm-src"
 		ilmAction                  = "ilm-action"
 		ilmDue                     = "ilm-due"
 		ilmRuleID                  = "ilm-rule-id"
 		ilmTier                    = "ilm-tier"
 		ilmNewerNoncurrentVersions = "ilm-newer-noncurrent-versions"
 		ilmNoncurrentDays          = "ilm-noncurrent-days"
 	)
 	tags := make(map[string]interface{}, 5)
 	if src > lcEventSrc_None {
 		tags[ilmSrc] = src.String()
 	}
 	tags[ilmAction] = event.Action.String()
 	tags[ilmRuleID] = event.RuleID
 	if !event.Due.IsZero() {
 		tags[ilmDue] = event.Due
 	}
 	// rule with Transition/NoncurrentVersionTransition in effect
 	if event.StorageClass != "" {
 		tags[ilmTier] = event.StorageClass
 	}
 	// rule with NewernoncurrentVersions in effect
 	if event.NewerNoncurrentVersions > 0 {
 		tags[ilmNewerNoncurrentVersions] = event.NewerNoncurrentVersions
 	}
 	if event.NoncurrentDays > 0 {
 		tags[ilmNoncurrentDays] = event.NoncurrentDays
 	}
 	return tags
 }
 func newLifecycleAuditEvent(src lcEventSrc, event lifecycle.Event) lcAuditEvent {
 	return lcAuditEvent{
 		Event:  event,
 		source: src,
 	}
 }
@@ -21,6 +21,7 @@ import (
 	"encoding/xml"
 	"io"
 	"net/http"
 	"strconv"
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	xhttp "github.com/minio/minio/internal/http"
@@ -115,6 +116,16 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	var withUpdatedAt bool
 	if updatedAtStr := r.Form.Get("withUpdatedAt"); updatedAtStr != "" {
 		var err error
 		withUpdatedAt, err = strconv.ParseBool(updatedAtStr)
 		if err != nil {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidLifecycleQueryParameter), r.URL)
 			return
 		}
 	}
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -126,7 +137,7 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
-	config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
+	config, updatedAt, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -138,6 +149,9 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 	if withUpdatedAt {
 		w.Header().Set(xhttp.MinIOLifecycleCfgUpdatedAt, updatedAt.Format(iso8601Format))
 	}
 	// Write lifecycle configuration to client.
 	writeSuccessResponseXML(w, configData)
 }
@@ -179,7 +179,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 			lifecycleResponse:  []byte(``),
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
-				Code:     "InvalidRequest",
+				Code:     "InvalidArgument",
 				Message:  "Filter must have exactly one of Prefix, Tag, or And specified",
 			},
@@ -196,7 +196,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 			lifecycleResponse:  []byte(``),
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
-				Code:     "InvalidRequest",
+				Code:     "InvalidArgument",
 				Message:  "Date must be provided in ISO 8601 format",
 			},
@@ -24,12 +24,15 @@ import (
 	"fmt"
 	"io"
 	"net/http"
 	"runtime"
 	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 	"github.com/google/uuid"
 	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/amztime"
 	sse "github.com/minio/minio/internal/bucket/encryption"
@@ -38,6 +41,8 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
 	"github.com/minio/pkg/env"
 	"github.com/minio/pkg/workers"
 )
 const (
@@ -58,7 +63,8 @@ type LifecycleSys struct{}
 // Get - gets lifecycle config associated to a given bucket name.
 func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
-	return globalBucketMetadataSys.GetLifecycleConfig(bucketName)
+	lc, _, err = globalBucketMetadataSys.GetLifecycleConfig(bucketName)
 	return lc, err
 }
 // NewLifecycleSys - creates new lifecycle system.
@@ -66,10 +72,34 @@ func NewLifecycleSys() *LifecycleSys {
 	return &LifecycleSys{}
 }
 func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string) madmin.TraceInfo {
 	return madmin.TraceInfo{
 		TraceType: madmin.TraceILM,
 		Time:      startTime,
 		NodeName:  globalLocalNodeName,
 		FuncName:  event,
 		Duration:  duration,
 		Path:      pathJoin(oi.Bucket, oi.Name),
 		Error:     "",
 		Message:   getSource(4),
 		Custom:    map[string]string{"version-id": oi.VersionID},
 	}
 }
 func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string) {
 	startTime := time.Now()
 	return func(event string) {
 		duration := time.Since(startTime)
 		if globalTrace.NumSubscribers(madmin.TraceILM) > 0 {
 			globalTrace.Publish(ilmTrace(startTime, duration, oi, event))
 		}
 	}
 }
 type expiryTask struct {
-	objInfo        ObjectInfo
+	objInfo ObjectInfo
-	versionExpiry  bool
+	event   lifecycle.Event
-	restoredObject bool
+	src     lcEventSrc
 }
 type expiryState struct {
@@ -92,22 +122,22 @@ func (es *expiryState) close() {
 }
 // enqueueByDays enqueues object versions expired by days for expiry.
-func (es *expiryState) enqueueByDays(oi ObjectInfo, restoredObject bool, rmVersion bool) {
+func (es *expiryState) enqueueByDays(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) {
 	select {
 	case <-GlobalContext.Done():
 		es.close()
-	case es.byDaysCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion, restoredObject: restoredObject}:
+	case es.byDaysCh <- expiryTask{objInfo: oi, event: event, src: src}:
 	default:
 	}
 }
 // enqueueByNewerNoncurrent enqueues object versions expired by
 // NewerNoncurrentVersions limit for expiry.
-func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete) {
+func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete, lcEvent lifecycle.Event) {
 	select {
 	case <-GlobalContext.Done():
 		es.close()
-	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions}:
+	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions, event: lcEvent}:
 	default:
 	}
 }
@@ -116,26 +146,51 @@ var globalExpiryState *expiryState
 func newExpiryState() *expiryState {
 	return &expiryState{
-		byDaysCh:            make(chan expiryTask, 10000),
+		byDaysCh:            make(chan expiryTask, 100000),
-		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 10000),
+		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 100000),
 	}
 }
 func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 	globalExpiryState = newExpiryState()
 	workerSize, _ := strconv.Atoi(env.Get("_MINIO_ILM_EXPIRY_WORKERS", strconv.Itoa((runtime.GOMAXPROCS(0)+1)/2)))
 	if workerSize == 0 {
 		workerSize = 4
 	}
 	ewk, err := workers.New(workerSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
 	}
 	nwk, err := workers.New(workerSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
 	}
 	go func() {
 		for t := range globalExpiryState.byDaysCh {
-			if t.objInfo.TransitionedObject.Status != "" {
+			ewk.Take()
-				applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
+			go func(t expiryTask) {
-			} else {
+				defer ewk.Give()
-				applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
+				if t.objInfo.TransitionedObject.Status != "" {
-			}
+					applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.event, t.src)
 				} else {
 					applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.event, t.src)
 				}
 			}(t)
 		}
 		ewk.Wait()
 	}()
 	go func() {
 		for t := range globalExpiryState.byNewerNoncurrentCh {
-			deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
+			nwk.Take()
 			go func(t newerNoncurrentTask) {
 				defer nwk.Give()
 				deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions, t.event)
 			}(t)
 		}
 		nwk.Wait()
 	}()
 }
@@ -144,11 +199,13 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 type newerNoncurrentTask struct {
 	bucket   string
 	versions []ObjectToDelete
 	event    lifecycle.Event
 }
 type transitionTask struct {
 	tier    string
 	objInfo ObjectInfo
 	src     lcEventSrc
 	event   lifecycle.Event
 }
 type transitionState struct {
@@ -166,10 +223,10 @@ type transitionState struct {
 	lastDayStats map[string]*lastDayTierStats
 }
-func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
+func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) {
 	select {
 	case <-t.ctx.Done():
-	case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
+	case t.transitionCh <- transitionTask{objInfo: oi, event: event, src: src}:
 	default:
 	}
 }
@@ -222,9 +279,9 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 				return
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
-			if err := transitionObject(t.ctx, objectAPI, task.objInfo, task.tier); err != nil {
+			if err := transitionObject(t.ctx, objectAPI, task.objInfo, newLifecycleAuditEvent(task.src, task.event)); err != nil {
-				logger.LogIf(t.ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
+				logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
-					task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
+					task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 			} else {
 				ts := tierStats{
 					TotalSize:   uint64(task.objInfo.Size),
@@ -233,7 +290,7 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 				if task.objInfo.IsLatest {
 					ts.NumObjects = 1
 				}
-				t.addLastDayStats(task.tier, ts)
+				t.addLastDayStats(task.event.StorageClass, ts)
 			}
 			atomic.AddInt32(&t.activeTasks, -1)
 		}
@@ -304,89 +361,78 @@ func validateTransitionTier(lc *lifecycle.Lifecycle) error {
 // enqueueTransitionImmediate enqueues obj for transition if eligible.
 // This is to be called after a successful upload of an object (version).
-func enqueueTransitionImmediate(obj ObjectInfo) {
+func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) {
 	if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
-		event := lc.Eval(obj.ToLifecycleOpts())
+		switch event := lc.Eval(obj.ToLifecycleOpts()); event.Action {
 		switch event.Action {
 		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-			globalTransitionState.queueTransitionTask(obj, event.StorageClass)
+			globalTransitionState.queueTransitionTask(obj, event, src)
 		}
 	}
 }
 // expireAction represents different actions to be performed on expiry of a
 // restored/transitioned object
 type expireAction int
 const (
 	// ignore the zero value
 	_ expireAction = iota
 	// expireObj indicates expiry of 'regular' transitioned objects.
 	expireObj
 	// expireRestoredObj indicates expiry of restored objects.
 	expireRestoredObj
 )
 // expireTransitionedObject handles expiry of transitioned/restored objects
 // (versions) in one of the following situations:
 //
 // 1. when a restored (via PostRestoreObject API) object expires.
 // 2. when a transitioned object expires (based on an ILM rule).
-func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, action expireAction) error {
+func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, lcEvent lifecycle.Event, src lcEventSrc) error {
 	traceFn := globalLifecycleSys.trace(*oi)
 	var opts ObjectOptions
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name)
 	opts.VersionID = lcOpts.VersionID
 	opts.Expiration = ExpirationOptions{Expire: true}
-	switch action {
+	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
-	case expireObj:
+	if lcEvent.Action.DeleteRestored() {
 		// When an object is past expiry or when a transitioned object is being
 		// deleted, 'mark' the data in the remote tier for delete.
 		entry := jentry{
 			ObjName:   oi.TransitionedObject.Name,
 			VersionID: oi.TransitionedObject.VersionID,
 			TierName:  oi.TransitionedObject.Tier,
 		}
 		if err := globalTierJournal.AddEntry(entry); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
 		// Delete metadata on source, now that data in remote tier has been
 		// marked for deletion.
 		if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
 			logger.LogIf(ctx, err)
 			return err
 		}
 		// Send audit for the lifecycle delete operation
 		auditLogLifecycle(ctx, *oi, ILMExpiry)
 		eventName := event.ObjectRemovedDelete
 		if lcOpts.DeleteMarker {
 			eventName = event.ObjectRemovedDeleteMarkerCreated
 		}
 		objInfo := ObjectInfo{
 			Name:         oi.Name,
 			VersionID:    lcOpts.VersionID,
 			DeleteMarker: lcOpts.DeleteMarker,
 		}
 		// Notify object deleted event.
 		sendEvent(eventArgs{
 			EventName:  eventName,
 			BucketName: oi.Bucket,
 			Object:     objInfo,
 			Host:       "Internal: [ILM-Expiry]",
 		})
 	case expireRestoredObj:
 		// delete locally restored copy of object or object version
 		// from the source, while leaving metadata behind. The data on
 		// transitioned tier lies untouched and still accessible
 		opts.Transition.ExpireRestored = true
 		_, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
 		if err == nil {
 			// TODO consider including expiry of restored object to events we
 			// notify.
 			auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
 		}
 		return err
 	default:
 		return fmt.Errorf("Unknown expire action %v", action)
 	}
 	// When an object is past expiry or when a transitioned object is being
 	// deleted, 'mark' the data in the remote tier for delete.
 	entry := jentry{
 		ObjName:   oi.TransitionedObject.Name,
 		VersionID: oi.TransitionedObject.VersionID,
 		TierName:  oi.TransitionedObject.Tier,
 	}
 	if err := globalTierJournal.AddEntry(entry); err != nil {
 		logger.LogIf(ctx, err)
 		return err
 	}
 	// Delete metadata on source, now that data in remote tier has been
 	// marked for deletion.
 	if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
 		logger.LogIf(ctx, err)
 		return err
 	}
 	// Send audit for the lifecycle delete operation
 	defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
 	eventName := event.ObjectRemovedDelete
 	if lcOpts.DeleteMarker {
 		eventName = event.ObjectRemovedDeleteMarkerCreated
 	}
 	objInfo := ObjectInfo{
 		Name:         oi.Name,
 		VersionID:    lcOpts.VersionID,
 		DeleteMarker: lcOpts.DeleteMarker,
 	}
 	// Notify object deleted event.
 	sendEvent(eventArgs{
 		EventName:  eventName,
 		BucketName: oi.Bucket,
 		Object:     objInfo,
 		UserAgent:  "Internal: [ILM-Expiry]",
 		Host:       globalLocalNodeName,
 	})
 	return nil
 }
@@ -406,17 +452,18 @@ func genTransitionObjName(bucket string) (string, error) {
 // storage specified by the transition ARN, the metadata is left behind on source cluster and original content
 // is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
 // to the transition tier without decrypting or re-encrypting.
-func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, tier string) error {
+func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, lae lcAuditEvent) error {
 	opts := ObjectOptions{
 		Transition: TransitionOptions{
 			Status: lifecycle.TransitionPending,
-			Tier:   tier,
+			Tier:   lae.StorageClass,
 			ETag:   oi.ETag,
 		},
-		VersionID:        oi.VersionID,
+		LifecycleAuditEvent: lae,
-		Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+		VersionID:           oi.VersionID,
-		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
+		Versioned:           globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
-		MTime:            oi.ModTime,
+		VersionSuspended:    globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
 		MTime:               oi.ModTime,
 	}
 	return objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
 }
@@ -59,10 +59,18 @@ func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, max
 	return ErrNone
 }
-// ListObjectVersions - GET Bucket Object versions
+func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
 	api.listObjectVersionsHandler(w, r, false)
 }
 func (api objectAPIHandlers) ListObjectVersionsMHandler(w http.ResponseWriter, r *http.Request) {
 	api.listObjectVersionsHandler(w, r, true)
 }
 // ListObjectVersionsHandler - GET Bucket Object versions
 // You can use the versions subresource to list metadata about all
 // of the versions of objects in a bucket.
-func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) {
+func (api objectAPIHandlers) listObjectVersionsHandler(w http.ResponseWriter, r *http.Request, metadata bool) {
 	ctx := newContext(r, w, "ListObjectVersions")
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -80,7 +88,12 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
-
+	var checkObjMeta metaCheckFn
 	if metadata {
 		checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
 			return checkRequestAuthType(ctx, r, action, bucket, name)
 		}
 	}
 	urlValues := r.Form
 	// Extract all the listBucketVersions query params to their native values.
@@ -111,8 +124,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
+	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo, checkObjMeta)
 	response := generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo)
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponseList(response))
@@ -128,64 +140,7 @@ func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r
 // MinIO continues to support ListObjectsV1 and V2 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2M")
-
+	api.listObjectsV2Handler(ctx, w, r, true)
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]
 	objectAPI := api.ObjectAPI()
 	if objectAPI == nil {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
 		return
 	}
 	if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
 	urlValues := r.Form
 	// Extract all the listObjectsV2 query params to their native values.
 	prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues)
 	if errCode != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
 		return
 	}
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
 	if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
 	}
 	listObjectsV2 := objectAPI.ListObjectsV2
 	// Inititate a list objects operation based on the input params.
 	// On success would return back ListObjectsInfo object to be
 	// marshaled into S3 compatible XML header.
 	listObjectsV2Info, err := listObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 	if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
 	// The next continuation token has id@node_index format to optimize paginated listing
 	nextContinuationToken := listObjectsV2Info.NextContinuationToken
 	response := generateListObjectsV2Response(bucket, prefix, token, nextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
 		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, true)
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponseList(response))
 }
 // ListObjectsV2Handler - GET Bucket (List Objects) Version 2.
@@ -198,7 +153,11 @@ func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *htt
 // MinIO continues to support ListObjectsV1 for supporting legacy tools.
 func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "ListObjectsV2")
 	api.listObjectsV2Handler(ctx, w, r, false)
 }
 // listObjectsV2Handler performs listing either with or without extra metadata.
 func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.ResponseWriter, r *http.Request, metadata bool) {
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 	vars := mux.Vars(r)
@@ -215,6 +174,12 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 		return
 	}
 	var checkObjMeta metaCheckFn
 	if metadata {
 		checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) {
 			return checkRequestAuthType(ctx, r, action, bucket, name)
 		}
 	}
 	urlValues := r.Form
 	// Extract all the listObjectsV2 query params to their native values.
@@ -225,7 +190,6 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	}
 	// Validate the query params before beginning to serve the request.
 	// fetch-owner is not validated since it is a boolean
 	if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -257,7 +221,7 @@ func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http
 	response := generateListObjectsV2Response(bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter,
 		delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated,
-		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, false)
+		maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, checkObjMeta)
 	// Write success response.
 	writeSuccessResponseXML(w, encodeResponseList(response))
@@ -25,7 +25,7 @@ import (
 	"sync"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	bucketsse "github.com/minio/minio/internal/bucket/encryption"
@@ -36,8 +36,8 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/bucket/policy"
 	"github.com/minio/pkg/sync/errgroup"
 )
 // BucketMetadataSys captures all bucket metadata for a given cluster.
@@ -121,6 +121,7 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 		meta.NotificationConfigXML = configData
 	case bucketLifecycleConfig:
 		meta.LifecycleConfigXML = configData
 		meta.LifecycleConfigUpdatedAt = updatedAt
 	case bucketSSEConfig:
 		meta.EncryptionConfigXML = configData
 		meta.EncryptionConfigUpdatedAt = updatedAt
@@ -246,18 +247,18 @@ func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Co
 // GetLifecycleConfig returns configured lifecycle config
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
+func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, time.Time, error) {
 	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
-			return nil, BucketLifecycleNotFound{Bucket: bucket}
+			return nil, time.Time{}, BucketLifecycleNotFound{Bucket: bucket}
 		}
-		return nil, err
+		return nil, time.Time{}, err
 	}
 	if meta.lifecycleConfig == nil {
-		return nil, BucketLifecycleNotFound{Bucket: bucket}
+		return nil, time.Time{}, BucketLifecycleNotFound{Bucket: bucket}
 	}
-	return meta.lifecycleConfig, nil
+	return meta.lifecycleConfig, meta.LifecycleConfigUpdatedAt, nil
 }
 // GetNotificationConfig returns configured notification config
@@ -419,8 +420,8 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob
 	sys.objAPI = objAPI
-	// Load bucket metadata sys in background
+	// Load bucket metadata sys.
-	go sys.init(ctx, buckets)
+	sys.init(ctx, buckets)
 	return nil
 }
@@ -29,7 +29,7 @@ import (
 	"path"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	bucketsse "github.com/minio/minio/internal/bucket/encryption"
 	"github.com/minio/minio/internal/bucket/lifecycle"
@@ -88,6 +88,7 @@ type BucketMetadata struct {
 	QuotaConfigUpdatedAt        time.Time
 	ReplicationConfigUpdatedAt  time.Time
 	VersioningConfigUpdatedAt   time.Time
 	LifecycleConfigUpdatedAt    time.Time
 	// Unexported fields. Must be updated atomically.
 	policyConfig           *policy.Policy
@@ -119,6 +120,16 @@ func newBucketMetadata(name string) BucketMetadata {
 	}
 }
 // Versioning returns true if versioning is enabled
 func (b BucketMetadata) Versioning() bool {
 	return b.LockEnabled || (b.versioningConfig != nil && b.versioningConfig.Enabled()) || (b.objectLockConfig != nil && b.objectLockConfig.Enabled())
 }
 // ObjectLocking returns true if object locking is enabled
 func (b BucketMetadata) ObjectLocking() bool {
 	return b.LockEnabled || (b.objectLockConfig != nil && b.objectLockConfig.Enabled())
 }
 // SetCreatedAt preserves the CreatedAt time for bucket across sites in site replication. It defaults to
 // creation time of bucket on this cluster in all other cases.
 func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
@@ -417,6 +428,10 @@ func (b *BucketMetadata) defaultTimestamps() {
 	if b.VersioningConfigUpdatedAt.IsZero() {
 		b.VersioningConfigUpdatedAt = b.Created
 	}
 	if b.LifecycleConfigUpdatedAt.IsZero() {
 		b.LifecycleConfigUpdatedAt = b.Created
 	}
 }
 // Save config to supplied ObjectLayer api.
@@ -150,6 +150,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
 				return
 			}
 		case "LifecycleConfigUpdatedAt":
 			z.LifecycleConfigUpdatedAt, err = dc.ReadTime()
 			if err != nil {
 				err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
 				return
 			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -163,9 +169,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 21
+	// map header, size 22
 	// write "Name"
-	err = en.Append(0xde, 0x0, 0x15, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0xde, 0x0, 0x16, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -374,15 +380,25 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
 		return
 	}
 	// write "LifecycleConfigUpdatedAt"
 	err = en.Append(0xb8, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
 	if err != nil {
 		return
 	}
 	err = en.WriteTime(z.LifecycleConfigUpdatedAt)
 	if err != nil {
 		err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
 		return
 	}
 	return
 }
 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 21
+	// map header, size 22
 	// string "Name"
-	o = append(o, 0xde, 0x0, 0x15, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0xde, 0x0, 0x16, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -444,6 +460,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "VersioningConfigUpdatedAt"
 	o = append(o, 0xb9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
 	o = msgp.AppendTime(o, z.VersioningConfigUpdatedAt)
 	// string "LifecycleConfigUpdatedAt"
 	o = append(o, 0xb8, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
 	o = msgp.AppendTime(o, z.LifecycleConfigUpdatedAt)
 	return
 }
@@ -591,6 +610,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
 				return
 			}
 		case "LifecycleConfigUpdatedAt":
 			z.LifecycleConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
 			if err != nil {
 				err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
 				return
 			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -605,6 +630,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize
+	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize
 	return
 }
@@ -83,12 +83,6 @@ func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locke
 // governance bypass headers are set and user has governance bypass permissions.
 // Objects in "Compliance" mode can be overwritten only if retention date is past.
 func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, oi ObjectInfo, gerr error) APIErrorCode {
 	opts, err := getOpts(ctx, r, bucket, object.ObjectName)
 	if err != nil {
 		return toAPIErrorCode(ctx, err)
 	}
 	opts.VersionID = object.VersionID
 	if gerr != nil { // error from GetObjectInfo
 		if _, ok := gerr.(MethodNotAllowed); ok {
 			// This happens usually for a delete marker
@@ -312,7 +306,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
 			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
 		rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header)
-		if err != nil {
+		if err != nil && !(replica && rMode == "" && rDate.IsZero()) {
 			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
 		if retentionPermErr != ErrNone {
@@ -24,7 +24,7 @@ import (
 	"net/http"
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
 	"github.com/minio/pkg/bucket/policy"
@@ -87,7 +87,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, APIError{
-			Code:           "ErrMalformedXML",
+			Code:           "MalformedPolicy",
 			HTTPStatusCode: http.StatusBadRequest,
 			Description:    err.Error(),
 		}, r.URL)
@@ -24,7 +24,7 @@ import (
 	"fmt"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 )
@@ -47,9 +47,13 @@ func NewBucketQuotaSys() *BucketQuotaSys {
 // Init initialize bucket quota.
 func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 	sys.bucketStorageCache.Once.Do(func() {
-		sys.bucketStorageCache.TTL = 1 * time.Second
+		// Set this to 10 secs since its enough, as scanner
 		// does not update the bucket usage values frequently.
 		sys.bucketStorageCache.TTL = 10 * time.Second
 		// Rely on older value if usage loading fails from disk.
 		sys.bucketStorageCache.Relax = true
 		sys.bucketStorageCache.Update = func() (interface{}, error) {
-			ctx, done := context.WithTimeout(context.Background(), 5*time.Second)
+			ctx, done := context.WithTimeout(context.Background(), 1*time.Second)
 			defer done()
 			return loadDataUsageFromBackend(ctx, objAPI)
@@ -60,16 +64,20 @@ func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 // GetBucketUsageInfo return bucket usage info for a given bucket
 func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, error) {
 	v, err := sys.bucketStorageCache.Get()
-	if err != nil {
+	if err != nil && v != nil {
-		return BucketUsageInfo{}, err
+		logger.LogIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err))
 	}
 	if v == nil {
 		logger.LogIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"))
 	}
 	var bui BucketUsageInfo
 	dui, ok := v.(DataUsageInfo)
-	if !ok {
+	if ok {
-		return BucketUsageInfo{}, fmt.Errorf("internal error: Unexpected DUI data type: %T", v)
+		bui = dui.BucketsUsage[bucket]
 	}
 	bui := dui.BucketsUsage[bucket]
 	return bui, nil
 }
@@ -82,7 +90,7 @@ func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota,
 	if !quotaCfg.IsValid() {
 		if quotaCfg.Type == "fifo" {
 			logger.LogIf(GlobalContext, errors.New("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore. Please clear your quota configs using 'mc admin bucket quota alias/bucket --clear' and use 'mc ilm add' for expiration of objects"))
-			return quotaCfg, nil
+			return quotaCfg, fmt.Errorf("invalid quota type 'fifo'")
 		}
 		return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg)
 	}
@@ -100,6 +108,10 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
 	}
 	if q != nil && q.Type == madmin.HardQuota && q.Quota > 0 {
 		if uint64(size) >= q.Quota { // check if file size already exceeds the quota
 			return BucketQuotaExceeded{Bucket: bucket}
 		}
 		bui, err := sys.GetBucketUsageInfo(bucket)
 		if err != nil {
 			return err
@@ -277,6 +277,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 				Bucket: bucket,
 				Err:    fmt.Errorf("invalid query parameter older-than %s for %s : %w", durationStr, bucket, err),
 			}), r.URL)
 			return
 		}
 	}
 	resetBeforeDate := UTCNow().AddDate(0, 0, -1*int(days/24))
@@ -347,6 +348,7 @@ func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseW
 		default:
 			writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL)
 		}
 		return
 	}
 	targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket)
 	if err != nil {
@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -28,7 +28,7 @@ import (
 	"sync"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 )
@@ -512,7 +512,10 @@ func getHealReplicateObjectInfo(objInfo ObjectInfo, rcfg replicationConfig) Repl
 				ObjectName: oi.Name,
 				VersionID:  oi.VersionID,
 			},
-		}, oi, ObjectOptions{VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name)}, nil)
+		}, oi, ObjectOptions{
 			Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
 			VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
 		}, nil)
 	} else {
 		dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(ObjectInfo{
 			UserDefined: oi.UserDefined,
@@ -26,6 +26,7 @@ import (
 	"math"
 	"math/rand"
 	"net/http"
 	"net/url"
 	"path"
 	"reflect"
 	"strings"
@@ -34,7 +35,7 @@ import (
 	"time"
 	"github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/encrypt"
 	"github.com/minio/minio-go/v7/pkg/tags"
@@ -48,6 +49,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/workers"
 	"github.com/zeebo/xxh3"
 )
@@ -138,6 +140,13 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 		// validate replication ARN against target endpoint
 		c := globalBucketTargetSys.GetRemoteTargetClient(ctx, arnStr)
 		if c != nil {
 			if err := checkRemoteEndpoint(ctx, c.EndpointURL()); err != nil {
 				switch err.(type) {
 				case BucketRemoteIdenticalToSource:
 					return true, errorCodes.ToAPIErrWithErr(ErrBucketRemoteIdenticalToSource, fmt.Errorf("remote target endpoint %s is self referential", c.EndpointURL().String()))
 				default:
 				}
 			}
 			if c.EndpointURL().String() == clnt.EndpointURL().String() {
 				selfTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
 				if !sameTarget {
@@ -154,6 +163,45 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 	return sameTarget, toAPIError(ctx, nil)
 }
 // performs a http request to remote endpoint to check if deployment id of remote endpoint is same as
 // local cluster deployment id. This is to prevent replication to self, especially in case of a loadbalancer
 // in front of MinIO.
 func checkRemoteEndpoint(ctx context.Context, epURL *url.URL) error {
 	reqURL := &url.URL{
 		Scheme: epURL.Scheme,
 		Host:   epURL.Host,
 		Path:   healthCheckPathPrefix + healthCheckReadinessPath,
 	}
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL.String(), nil)
 	if err != nil {
 		return err
 	}
 	client := &http.Client{
 		Transport: NewHTTPTransport(),
 		Timeout:   10 * time.Second,
 	}
 	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}
 	if err == nil {
 		// Drain the connection.
 		xhttp.DrainBody(resp.Body)
 	}
 	if resp != nil {
 		amzid := resp.Header.Get(xhttp.AmzRequestHostID)
 		if _, ok := globalNodeNamesHex[amzid]; ok {
 			return BucketRemoteIdenticalToSource{
 				Endpoint: epURL.String(),
 			}
 		}
 	}
 	return nil
 }
 type mustReplicateOptions struct {
 	meta               map[string]string
 	status             replication.StatusType
@@ -387,7 +435,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
 			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -403,7 +452,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
 			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -424,7 +474,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
 			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		return
@@ -449,7 +500,8 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 					VersionID:    versionID,
 					DeleteMarker: dobj.DeleteMarker,
 				},
-				Host:      "Internal: [Replication]",
+				UserAgent: "Internal: [Replication]",
 				Host:      globalLocalNodeName,
 				EventName: event.ObjectReplicationNotTracked,
 			})
 			continue
@@ -508,7 +560,6 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 		VersionSuspended: globalBucketVersioningSys.Suspended(bucket),
 	})
 	if err != nil && !isErrVersionNotFound(err) { // VersionNotFound would be reported by pool that object version is missing on.
 		logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %s", bucket, dobj.ObjectName, versionID, err))
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object: ObjectInfo{
@@ -517,14 +568,16 @@ func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, obj
 				VersionID:    versionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
 			Host:      globalLocalNodeName,
 			EventName: eventName,
 		})
 	} else {
 		sendEvent(eventArgs{
 			BucketName: bucket,
 			Object:     dobjInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 			EventName:  eventName,
 		})
 	}
@@ -552,7 +605,7 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		return
 	}
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN))
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete"+tgt.ARN)
 		sendEvent(eventArgs{
 			BucketName: dobj.Bucket,
 			Object: ObjectInfo{
@@ -561,7 +614,8 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 				VersionID:    dobj.VersionID,
 				DeleteMarker: dobj.DeleteMarker,
 			},
-			Host:      "Internal: [Replication]",
+			UserAgent: "Internal: [Replication]",
 			Host:      globalLocalNodeName,
 			EventName: event.ObjectReplicationNotTracked,
 		})
 		if dobj.VersionID == "" {
@@ -916,12 +970,13 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
-		logger.LogIf(ctx, err)
+		logger.LogOnceIf(ctx, err, "get-replication-config"+bucket)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -939,7 +994,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
 		globalReplicationPool.queueMRFSave(ri.ToMRFEntry())
 		logger.LogIf(ctx, fmt.Errorf("failed to get lock for object: %s bucket:%s arn:%s", object, bucket, ri.TargetArn))
@@ -959,7 +1015,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
 			continue
 		}
@@ -975,11 +1032,13 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		}(i, tgt)
 	}
 	wg.Wait()
 	replicationStatus = rinfos.ReplicationStatus() // used in defer function
 	// FIXME: add support for missing replication events
 	// - event.ObjectReplicationMissedThreshold
 	// - event.ObjectReplicationReplicatedAfterThreshold
 	eventName := event.ObjectReplicationComplete
-	if rinfos.ReplicationStatus() == replication.Failed {
+	if replicationStatus == replication.Failed {
 		eventName = event.ObjectReplicationFailed
 	}
 	newReplStatusInternal := rinfos.ReplicationStatusInternal()
@@ -1005,10 +1064,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 			},
 		}
-		if _, err = objectAPI.PutObjectMetadata(ctx, bucket, object, popts); err != nil {
+		_, _ = objectAPI.PutObjectMetadata(ctx, bucket, object, popts)
 			logger.LogIf(ctx, fmt.Errorf("Unable to update replication metadata for %s/%s(%s): %w",
 				bucket, objInfo.Name, objInfo.VersionID, err))
 		}
 		opType := replication.MetadataReplicationType
 		if rinfos.Action() == replicateAll {
 			opType = replication.ObjectReplicationType
@@ -1024,7 +1080,8 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		EventName:  eventName,
 		BucketName: bucket,
 		Object:     objInfo,
-		Host:       "Internal: [Replication]",
+		UserAgent:  "Internal: [Replication]",
 		Host:       globalLocalNodeName,
 	})
 	// re-queue failures once more - keep a retry count to avoid flooding the queue if
@@ -1064,12 +1121,13 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN))
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN), "replication-target-offline-obj"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1077,7 +1135,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
-	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
+	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
 		VersionID:        objInfo.VersionID,
 		Versioned:        versioned,
 		VersionSuspended: versionSuspended,
@@ -1088,15 +1146,18 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
-			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
+			logger.LogIf(ctx, fmt.Errorf("unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
 		}
 		return
 	}
 	defer gr.Close()
 	objInfo = gr.ObjInfo
 	// make sure we have the latest metadata for metrics calculation
 	rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN)
 	size, err := objInfo.GetActualSize()
 	if err != nil {
@@ -1105,18 +1166,20 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
 		return
 	}
 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("Unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
+		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
 		return rinfo
 	}
@@ -1141,7 +1204,8 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1168,14 +1232,14 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			r, objInfo, putOpts); err != nil {
 			if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 			}
 		}
 	} else {
 		if _, err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); err != nil {
 			if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 			}
 		}
 	}
@@ -1207,12 +1271,13 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	}
 	if globalBucketTargetSys.isOffline(tgt.EndpointURL()) {
-		logger.LogIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN))
+		logger.LogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", bucket, tgt.ARN), "replication-target-offline-all"+tgt.ARN)
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1220,7 +1285,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)
-	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
+	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
 		VersionID:        objInfo.VersionID,
 		Versioned:        versioned,
 		VersionSuspended: versionSuspended,
@@ -1231,15 +1296,18 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
-			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
+			logger.LogIf(ctx, fmt.Errorf("unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
 		}
 		return
 	}
 	defer gr.Close()
 	objInfo = gr.ObjInfo
 	// make sure we have the latest metadata for metrics calculation
 	rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN)
 	// use latest ObjectInfo to check if previous replication attempt succeeded
 	if objInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
@@ -1255,7 +1323,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
 		return
 	}
@@ -1266,7 +1335,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
 			Object:     objInfo,
-			Host:       "Internal: [Replication]",
+			UserAgent:  "Internal: [Replication]",
 			Host:       globalLocalNodeName,
 		})
 		return rinfo
 	}
@@ -1296,7 +1366,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 					EventName:  event.ObjectReplicationNotTracked,
 					BucketName: bucket,
 					Object:     objInfo,
-					Host:       "Internal: [Replication]",
+					UserAgent:  "Internal: [Replication]",
 					Host:       globalLocalNodeName,
 				})
 			}
 			// object with same VersionID already exists, replication kicked off by
@@ -1313,6 +1384,25 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			return
 		}
 	}
 	// if target returns error other than NoSuchKey, defer replication attempt
 	if cerr != nil {
 		errResp := minio.ToErrorResponse(cerr)
 		switch errResp.Code {
 		case "NoSuchKey", "NoSuchVersion", "SlowDownRead":
 			rAction = replicateAll
 		default:
 			logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (%s). Target (%s) returned %s error on HEAD",
 				bucket, object, objInfo.VersionID, tgt.EndpointURL(), cerr))
 			sendEvent(eventArgs{
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
 				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
 			return
 		}
 	}
 	rinfo.ReplicationStatus = replication.Completed
 	rinfo.Size = size
 	rinfo.ReplicationAction = rAction
@@ -1344,7 +1434,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				EventName:  event.ObjectReplicationNotTracked,
 				BucketName: bucket,
 				Object:     objInfo,
-				Host:       "Internal: [Replication]",
+				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
 			return
 		}
@@ -1391,6 +1482,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, object string, r io.Reader, objInfo ObjectInfo, opts minio.PutObjectOptions) (err error) {
 	var uploadedParts []minio.CompletePart
 	// new multipart must not set mtime as it may lead to erroneous cleanups at various intervals.
 	opts.Internal.SourceMTime = time.Time{} // this value is saved properly in CompleteMultipartUpload()
 	uploadID, err := c.NewMultipartUpload(context.Background(), bucket, object, opts)
 	if err != nil {
 		return err
@@ -1420,11 +1513,16 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 	)
 	for _, partInfo := range objInfo.Parts {
-		hr, err = hash.NewReader(r, partInfo.ActualSize, "", "", partInfo.ActualSize)
+		hr, err = hash.NewReader(io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
 		if err != nil {
 			return err
 		}
-		pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, "", "", opts.ServerSideEncryption)
+
 		popts := minio.PutObjectPartOptions{
 			SSE: opts.ServerSideEncryption,
 		}
 		pInfo, err = c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, partInfo.ActualSize, popts)
 		if err != nil {
 			return err
 		}
@@ -2015,6 +2113,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 			StorageClass:              objInfo.StorageClass,
 			ReplicationStatusInternal: objInfo.ReplicationStatus,
 			UserTags:                  tags.String(),
 			ReplicationStatus:         replication.StatusType(objInfo.ReplicationStatus),
 		}
 		oi.UserDefined = make(map[string]string, len(objInfo.Metadata))
 		for k, v := range objInfo.Metadata {
@@ -2229,7 +2328,10 @@ func (s *replicationResyncer) PersistToDisk(ctx context.Context, objectAPI Objec
 	}
 }
-const resyncWorkerCnt = 50 // limit of number of bucket resyncs is progress at any given time
+const (
 	resyncWorkerCnt        = 10 // limit of number of bucket resyncs is progress at any given time
 	resyncParallelRoutines = 10 // number of parallel resync ops per bucket
 )
 func newresyncer() *replicationResyncer {
 	rs := replicationResyncer{
@@ -2244,6 +2346,36 @@ func newresyncer() *replicationResyncer {
 	return &rs
 }
 // mark status of replication resync on remote target for the bucket
 func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts) {
 	s.Lock()
 	defer s.Unlock()
 	m := s.statusMap[opts.bucket]
 	st := m.TargetsMap[opts.arn]
 	st.LastUpdate = UTCNow()
 	st.ResyncStatus = status
 	m.TargetsMap[opts.arn] = st
 	m.LastUpdate = UTCNow()
 	s.statusMap[opts.bucket] = m
 }
 // update replication resync stats for bucket's remote target
 func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts resyncOpts) {
 	s.Lock()
 	defer s.Unlock()
 	m := s.statusMap[opts.bucket]
 	st := m.TargetsMap[opts.arn]
 	st.Object = ts.Object
 	st.ReplicatedCount += ts.ReplicatedCount
 	st.FailedCount += ts.FailedCount
 	st.ReplicatedSize += ts.ReplicatedSize
 	st.FailedSize += ts.FailedSize
 	m.TargetsMap[opts.arn] = st
 	m.LastUpdate = UTCNow()
 	s.statusMap[opts.bucket] = m
 }
 // resyncBucket resyncs all qualifying objects as per replication rules for the target
 // ARN
 func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI ObjectLayer, heal bool, opts resyncOpts) {
@@ -2255,15 +2387,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	resyncStatus := ResyncFailed
 	defer func() {
-		s.Lock()
+		s.markStatus(resyncStatus, opts)
 		m := s.statusMap[opts.bucket]
 		st := m.TargetsMap[opts.arn]
 		st.LastUpdate = UTCNow()
 		st.ResyncStatus = resyncStatus
 		m.TargetsMap[opts.arn] = st
 		m.LastUpdate = UTCNow()
 		s.statusMap[opts.bucket] = m
 		s.Unlock()
 		globalSiteResyncMetrics.incBucket(opts, resyncStatus)
 		s.workerCh <- struct{}{}
 	}()
@@ -2299,15 +2423,9 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	}
 	// mark resync status as resync started
 	if !heal {
-		s.Lock()
+		s.markStatus(ResyncStarted, opts)
 		m := s.statusMap[opts.bucket]
 		st := m.TargetsMap[opts.arn]
 		st.ResyncStatus = ResyncStarted
 		m.TargetsMap[opts.arn] = st
 		m.LastUpdate = UTCNow()
 		s.statusMap[opts.bucket] = m
 		s.Unlock()
 	}
 	// Walk through all object versions - Walk() is always in ascending order needed to ensure
 	// delete marker replicated to target after object version is first created.
 	if err := objectAPI.Walk(ctx, opts.bucket, "", objInfoCh, ObjectOptions{}); err != nil {
@@ -2323,80 +2441,85 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	if st.ResyncStatus == ResyncStarted || st.ResyncStatus == ResyncFailed {
 		lastCheckpoint = st.Object
 	}
 	workers, err := workers.New(resyncParallelRoutines)
 	for obj := range objInfoCh {
 		select {
 		case <-s.resyncCancelCh:
 			resyncStatus = ResyncCanceled
 			return
 		case <-ctx.Done():
 			return
 		default:
 		}
 		if heal && lastCheckpoint != "" && lastCheckpoint != obj.Name {
 			continue
 		}
 		lastCheckpoint = ""
-
+		obj := obj
-		roi := getHealReplicateObjectInfo(obj, rcfg)
+		workers.Take()
-		if !roi.ExistingObjResync.mustResync() {
+		go func() {
-			continue
+			defer workers.Give()
-		}
+			roi := getHealReplicateObjectInfo(obj, rcfg)
-		traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
+			if !roi.ExistingObjResync.mustResync() {
-		if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
+				return
 			versionID := ""
 			dmVersionID := ""
 			if roi.VersionPurgeStatus.Empty() {
 				dmVersionID = roi.VersionID
 			} else {
 				versionID = roi.VersionID
 			}
 			traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
 			if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
 				versionID := ""
 				dmVersionID := ""
 				if roi.VersionPurgeStatus.Empty() {
 					dmVersionID = roi.VersionID
 				} else {
 					versionID = roi.VersionID
 				}
-			doi := DeletedObjectReplicationInfo{
+				doi := DeletedObjectReplicationInfo{
-				DeletedObject: DeletedObject{
+					DeletedObject: DeletedObject{
-					ObjectName:            roi.Name,
+						ObjectName:            roi.Name,
-					DeleteMarkerVersionID: dmVersionID,
+						DeleteMarkerVersionID: dmVersionID,
-					VersionID:             versionID,
+						VersionID:             versionID,
-					ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
+						ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
-					DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
+						DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
-					DeleteMarker:          roi.DeleteMarker,
+						DeleteMarker:          roi.DeleteMarker,
 					},
 					Bucket:    roi.Bucket,
 					OpType:    replication.ExistingObjectReplicationType,
 					EventType: ReplicateExistingDelete,
 				}
 				replicateDelete(ctx, doi, objectAPI)
 			} else {
 				roi.OpType = replication.ExistingObjectReplicationType
 				roi.EventType = ReplicateExisting
 				replicateObject(ctx, roi, objectAPI)
 			}
 			_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
 				VersionID: roi.VersionID,
 				Internal: minio.AdvancedGetOptions{
 					ReplicationProxyRequest: "false",
 				},
-				Bucket:    roi.Bucket,
+			})
-				OpType:    replication.ExistingObjectReplicationType,
+			st := TargetReplicationResyncStatus{
-				EventType: ReplicateExistingDelete,
+				Object: roi.Name,
 				Bucket: roi.Bucket,
 			}
-			replicateDelete(ctx, doi, objectAPI)
+			success := true
-		} else {
+			if err != nil {
-			roi.OpType = replication.ExistingObjectReplicationType
+				if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
-			roi.EventType = ReplicateExisting
+					st.ReplicatedCount++
-			replicateObject(ctx, roi, objectAPI)
+				} else {
-		}
+					st.FailedCount++
-		_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
+					success = false
-			VersionID: roi.VersionID,
+				}
 			Internal: minio.AdvancedGetOptions{
 				ReplicationProxyRequest: "false",
 			},
 		})
 		s.Lock()
 		m = s.statusMap[opts.bucket]
 		st = m.TargetsMap[opts.arn]
 		st.Object = roi.Name
 		success := true
 		if err != nil {
 			if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
 				st.ReplicatedCount++
 			} else {
-				st.FailedCount++
+				st.ReplicatedCount++
-				success = false
+				st.ReplicatedSize += roi.Size
 			}
-		} else {
+			s.incStats(st, opts)
-			st.ReplicatedCount++
+			traceFn(err)
-			st.ReplicatedSize += roi.Size
+			globalSiteResyncMetrics.updateMetric(roi, success, opts.resyncID)
-		}
+		}()
 		m.TargetsMap[opts.arn] = st
 		m.LastUpdate = UTCNow()
 		s.statusMap[opts.bucket] = m
 		s.Unlock()
 		traceFn(err)
 		globalSiteResyncMetrics.updateMetric(roi, success, opts.resyncID)
 	}
 	workers.Wait()
 	resyncStatus = ResyncCompleted
 }
@@ -3028,7 +3151,7 @@ func (p *ReplicationPool) loadStatsFromDisk() (rs map[string]BucketReplicationSt
 	data, err := readConfig(p.ctx, p.objLayer, getReplicationStatsPath())
 	if err != nil {
-		if !errors.Is(err, errConfigNotFound) {
+		if errors.Is(err, errConfigNotFound) {
 			return rs, nil
 		}
 		return rs, err
@@ -24,7 +24,7 @@ import (
 	"testing"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/replication"
 	xhttp "github.com/minio/minio/internal/http"
 )
@@ -25,7 +25,7 @@ import (
 	"time"
 	jsoniter "github.com/json-iterator/go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio/internal/bucket/replication"
@@ -208,14 +208,14 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	}
 	clnt, err := sys.getRemoteTargetClient(tgt)
 	if err != nil {
-		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+		return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket, Err: err}
 	}
 	// validate if target credentials are ok
 	exists, err := clnt.BucketExists(ctx, tgt.TargetBucket)
 	if err != nil {
 		switch minio.ToErrorResponse(err).Code {
 		case "NoSuchBucket":
-			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket, Err: err}
 		case "AccessDenied":
 			return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, AccessKey: tgt.Credentials.AccessKey, Err: err}
 		}
@@ -24,7 +24,7 @@ import (
 	"net/http"
 	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/mux"
@@ -28,7 +28,7 @@ import (
 	"net/url"
 	"time"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/logger"
 )
@@ -168,7 +168,7 @@ func sendHealthInfo(ctx context.Context, healthInfo madmin.HealthInfo) error {
 	}
 	filename := fmt.Sprintf("health_%s.json.gz", UTCNow().Format("20060102150405"))
-	url += "?filename=" + filename + "&src=callhome"
+	url += "?filename=" + filename
 	_, err := globalSubnetConfig.Upload(url, filename, createHealthJSONGzip(ctx, healthInfo))
 	return err
@@ -52,14 +52,13 @@ import (
 	"github.com/minio/console/restapi"
 	"github.com/minio/console/restapi/operations"
 	"github.com/minio/kes-go"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/certs"
@@ -133,20 +132,11 @@ func init() {
 		}
 	}()
 	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
 		PassHost:     true,
 		RoundTripper: NewHTTPTransportWithTimeout(1 * time.Hour),
 		Logger: func(err error) {
 			if err != nil && !errors.Is(err, context.Canceled) {
 				logger.LogIf(GlobalContext, err)
 			}
 		},
 	})
 	console.SetColor("Debug", fcolor.New())
 	gob.Register(StorageErr(""))
 	gob.Register(madmin.TimeInfo{})
 	gob.Register(madmin.XFSErrorConfigs{})
 	gob.Register(map[string]interface{}{})
 	defaultAWSCredProvider = []credentials.Provider{
@@ -187,7 +177,7 @@ func minioConfigToConsoleFeatures() {
 		}
 	}
 	// pass the console subpath configuration
-	if value := env.Get(config.EnvMinIOBrowserRedirectURL, ""); value != "" {
+	if value := env.Get(config.EnvBrowserRedirectURL, ""); value != "" {
 		subPath := path.Clean(pathJoin(strings.TrimSpace(globalBrowserRedirectURL.Path), SlashSeparator))
 		if subPath != SlashSeparator {
 			os.Setenv("CONSOLE_SUBPATH", subPath)
@@ -208,6 +198,11 @@ func minioConfigToConsoleFeatures() {
 	if globalIAMSys.LDAPConfig.Enabled() {
 		os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
 	}
 	// Handle animation in welcome page
 	if value := env.Get(config.EnvBrowserLoginAnimation, "on"); value != "" {
 		os.Setenv("CONSOLE_ANIMATED_LOGIN", value)
 	}
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
@@ -257,6 +252,9 @@ func initConsoleServer() (*restapi.Server, error) {
 		Path: globalCertsCADir.Get(),
 	}
 	// set certs before other console initialization
 	restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts
 	swaggerSpec, err := loads.Embedded(restapi.SwaggerJSON, restapi.FlatSwaggerJSON)
 	if err != nil {
 		return nil, err
@@ -283,8 +281,6 @@ func initConsoleServer() (*restapi.Server, error) {
 	// register all APIs
 	server.ConfigureAPI()
 	restapi.GlobalRootCAs, restapi.GlobalPublicCerts, restapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts
 	consolePort, _ := strconv.Atoi(globalMinioConsolePort)
 	server.Host = globalMinioConsoleHost
@@ -631,7 +627,7 @@ func handleCommonEnvVars() {
 		logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
 	}
 	if globalBrowserEnabled {
-		if redirectURL := env.Get(config.EnvMinIOBrowserRedirectURL, ""); redirectURL != "" {
+		if redirectURL := env.Get(config.EnvBrowserRedirectURL, ""); redirectURL != "" {
 			u, err := xnet.ParseHTTPURL(redirectURL)
 			if err != nil {
 				logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value in environment variable")
@@ -776,9 +772,12 @@ func handleCommonEnvVars() {
 			logger.Info(color.RedBold(msg))
 		}
 		globalActiveCred = cred
 		globalCredViaEnv = true
 	} else {
 		globalActiveCred = auth.DefaultCredentials
 	}
 	globalDisableFreezeOnBoot = env.Get("_MINIO_DISABLE_API_FREEZE_ON_BOOT", "") == "true" || serverDebugLog
 }
 // Initialize KMS global variable after valiadating and loading the configuration.
@@ -30,12 +30,7 @@ import (
 var errConfigNotFound = errors.New("config file not found")
 func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string, opts ObjectOptions) ([]byte, ObjectInfo, error) {
-	lockType := readLock
+	r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, opts)
 	if opts.NoLock {
 		lockType = noLock // erasureObjects.GetObjectNInfo honors lockType argument but not opts.NoLock.
 	}
 	r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, lockType, opts)
 	if err != nil {
 		if isErrObjectNotFound(err) {
 			return nil, ObjectInfo{}, errConfigNotFound
@@ -24,7 +24,7 @@ import (
 	"strings"
 	"sync"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/api"
 	"github.com/minio/minio/internal/config/cache"
@@ -283,7 +283,7 @@ var (
 	globalServerConfigMu sync.RWMutex
 )
-func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) error {
+func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, objAPI ObjectLayer) error {
 	switch subSys {
 	case config.SiteSubSys:
 		if _, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]); err != nil {
@@ -393,7 +393,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 	}
 	if config.NotifySubSystems.Contains(subSys) {
-		if err := notify.TestSubSysNotificationTargets(GlobalContext, s, subSys, NewHTTPTransport()); err != nil {
+		if err := notify.TestSubSysNotificationTargets(ctx, s, subSys, NewHTTPTransport()); err != nil {
 			return err
 		}
 	}
@@ -407,7 +407,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 	return nil
 }
-func validateConfig(s config.Config, subSys string) error {
+func validateConfig(ctx context.Context, s config.Config, subSys string) error {
 	objAPI := newObjectLayerFn()
 	// We must have a global lock for this so nobody else modifies env while we do.
@@ -419,12 +419,12 @@ func validateConfig(s config.Config, subSys string) error {
 	// Enable env values to validate KMS.
 	defer env.SetEnvOn()
 	if subSys != "" {
-		return validateSubSysConfig(s, subSys, objAPI)
+		return validateSubSysConfig(ctx, s, subSys, objAPI)
 	}
 	// No sub-system passed. Validate all of them.
 	for _, ss := range config.SubSystems.ToSlice() {
-		if err := validateSubSysConfig(s, ss, objAPI); err != nil {
+		if err := validateSubSysConfig(ctx, s, ss, objAPI); err != nil {
 			return err
 		}
 	}
@@ -588,7 +588,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			}
 			loggerCfg.HTTP[n] = l
 		}
-		if errs := logger.UpdateSystemTargets(loggerCfg); len(errs) > 0 {
+		if errs := logger.UpdateSystemTargets(ctx, loggerCfg); len(errs) > 0 {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
 		}
 	case config.AuditWebhookSubSys:
@@ -606,7 +606,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			loggerCfg.AuditWebhook[n] = l
 		}
-		if errs := logger.UpdateAuditWebhookTargets(loggerCfg); len(errs) > 0 {
+		if errs := logger.UpdateAuditWebhookTargets(ctx, loggerCfg); len(errs) > 0 {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
 		}
 	case config.AuditKafkaSubSys:
@@ -616,11 +616,14 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 		}
 		for n, l := range loggerCfg.AuditKafka {
 			if l.Enabled {
 				if l.TLS.Enable {
 					l.TLS.RootCAs = globalRootCAs
 				}
 				l.LogOnce = logger.LogOnceIf
 				loggerCfg.AuditKafka[n] = l
 			}
 		}
-		if errs := logger.UpdateAuditKafkaTargets(loggerCfg); len(errs) > 0 {
+		if errs := logger.UpdateAuditKafkaTargets(ctx, loggerCfg); len(errs) > 0 {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %v", errs))
 		}
 	case config.StorageClassSubSys:
@@ -1,215 +0,0 @@
 // Copyright (c) 2015-2021 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by
 // the Free Software Foundation, either version 3 of the License, or
 // (at your option) any later version.
 //
 // This program is distributed in the hope that it will be useful
 // but WITHOUT ANY WARRANTY; without even the implied warranty of
 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 // GNU Affero General Public License for more details.
 //
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 package cmd
 import (
 	"bytes"
 	"context"
 	"fmt"
 	"path"
 	"time"
 	"unicode/utf8"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	etcd "go.etcd.io/etcd/client/v3"
 )
 func handleEncryptedConfigBackend(objAPI ObjectLayer) error {
 	encrypted, err := checkBackendEncrypted(objAPI)
 	if err != nil {
 		return fmt.Errorf("Unable to encrypt config %w", err)
 	}
 	if !encrypted {
 		return nil
 	}
 	if err = migrateConfigPrefixToEncrypted(objAPI); err != nil {
 		return fmt.Errorf("Unable to migrate all config at .minio.sys/config/: %w", err)
 	}
 	return nil
 }
 const backendEncryptedFile = "backend-encrypted"
 var backendEncryptedMigrationComplete = []byte("encrypted")
 func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool, error) {
 	bootstrapTrace("check if etcd backend is encrypted")
 	data, err := readKeyEtcd(ctx, client, backendEncryptedFile)
 	if err != nil && err != errConfigNotFound {
 		return false, err
 	}
 	return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil
 }
 func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) {
 	bootstrapTrace("check if the config backend is encrypted")
 	data, err := readConfig(GlobalContext, objAPI, backendEncryptedFile)
 	if err != nil && err != errConfigNotFound {
 		return false, err
 	}
 	return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil
 }
 func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client) error {
 	encrypted, err := checkBackendEtcdEncrypted(ctx, client)
 	if err != nil {
 		return err
 	}
 	// If backend doesn't have this file means we have already
 	// attempted then migration
 	if !encrypted {
 		return nil
 	}
 	bootstrapTrace("encrypt etcd config")
 	if GlobalKMS != nil {
 		stat, err := GlobalKMS.Stat(ctx)
 		if err != nil {
 			return err
 		}
 		logger.Info(fmt.Sprintf("Attempting to re-encrypt IAM users and policies on etcd with %q (%s)", stat.DefaultKey, stat.Name))
 	}
 	listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
 	defer cancel()
 	r, err := client.Get(listCtx, minioConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
 	if err != nil {
 		return err
 	}
 	for _, kv := range r.Kvs {
 		data, err := readKeyEtcd(ctx, client, string(kv.Key))
 		if err == errConfigNotFound { // Perhaps not present or someone deleted it.
 			continue
 		}
 		if err != nil {
 			return err
 		}
 		if !utf8.Valid(data) {
 			pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
 			if err != nil {
 				if GlobalKMS != nil {
 					pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
 						minioMetaBucket: path.Join(minioMetaBucket, string(kv.Key)),
 					})
 					if err != nil {
 						pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
 							minioMetaBucket: string(kv.Key),
 						})
 						if err != nil {
 							return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
 						}
 					}
 				} else {
 					return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
 				}
 			}
 			data = pdata
 		}
 		if GlobalKMS != nil {
 			data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
 				minioMetaBucket: path.Join(minioMetaBucket, string(kv.Key)),
 			})
 			if err != nil {
 				return err
 			}
 		}
 		if err = saveKeyEtcd(ctx, client, string(kv.Key), data); err != nil {
 			return err
 		}
 	}
 	if GlobalKMS != nil {
 		logger.Info("Migration of encrypted IAM config data completed. All data is now encrypted on etcd.")
 	}
 	return deleteKeyEtcd(ctx, client, backendEncryptedFile)
 }
 func migrateConfigPrefixToEncrypted(objAPI ObjectLayer) error {
 	bootstrapTrace("migrating config prefix to encrypted")
 	if GlobalKMS != nil {
 		stat, err := GlobalKMS.Stat(context.Background())
 		if err != nil {
 			return err
 		}
 		logger.Info(fmt.Sprintf("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name))
 	}
 	results := make(chan ObjectInfo)
 	if err := objAPI.Walk(GlobalContext, minioMetaBucket, minioConfigPrefix, results, ObjectOptions{}); err != nil {
 		return err
 	}
 	for obj := range results {
 		data, err := readConfig(GlobalContext, objAPI, obj.Name)
 		if err != nil {
 			return err
 		}
 		if !utf8.Valid(data) {
 			pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
 			if err != nil {
 				if GlobalKMS != nil {
 					pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
 						minioMetaBucket: path.Join(minioMetaBucket, obj.Name),
 					})
 					if err != nil {
 						pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
 							minioMetaBucket: obj.Name,
 						})
 						if err != nil {
 							return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
 						}
 					}
 				} else {
 					return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
 				}
 			}
 			data = pdata
 		}
 		if GlobalKMS != nil {
 			data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
 				obj.Bucket: path.Join(obj.Bucket, obj.Name),
 			})
 			if err != nil {
 				return err
 			}
 		}
 		if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil {
 			return err
 		}
 	}
 	if GlobalKMS != nil {
 		logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
 	}
 	return deleteConfig(GlobalContext, objAPI, backendEncryptedFile)
 }
@@ -21,7 +21,7 @@ import (
 	"bytes"
 	"testing"
-	"github.com/minio/madmin-go/v2"
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio/internal/auth"
 )
@@ -18,16 +18,15 @@
 package cmd
 import (
-	"bytes"
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
 	"unicode/utf8"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/cache"
@@ -39,7 +38,6 @@ import (
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/event/target"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/target/http"
 	xnet "github.com/minio/pkg/net"
@@ -2471,284 +2469,97 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	return saveServerConfig(GlobalContext, objAPI, config)
 }
-// Migrates '.minio.sys/config.json' to v33.
+func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.Config, error) {
 func migrateMinioSysConfig(objAPI ObjectLayer) error {
 	bootstrapTrace("migrate .minio.sys/config/config.json to latest version")
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	// Check if the config version is latest, if not migrate.
+	configFiles := []string{
-	ok, _, err := checkConfigVersion(objAPI, configFile, "33")
+		getConfigFile(),
-	if err != nil {
+		getConfigFile() + ".deprecated",
-		return err
+		configFile,
 	}
 	if ok {
 		return nil
 	}
-	if err := migrateV27ToV28MinioSys(objAPI); err != nil {
+	newServerCfg := func() (config.Config, error) {
-		return err
+		// Initialize server config.
-	}
+		srvCfg := newServerConfig()
 	if err := migrateV28ToV29MinioSys(objAPI); err != nil {
 		return err
 	}
 	if err := migrateV29ToV30MinioSys(objAPI); err != nil {
 		return err
 	}
 	if err := migrateV30ToV31MinioSys(objAPI); err != nil {
 		return err
 	}
 	if err := migrateV31ToV32MinioSys(objAPI); err != nil {
 		return err
 	}
 	return migrateV32ToV33MinioSys(objAPI)
 }
-func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
+		return srvCfg, saveServerConfig(ctx, objAPI, srvCfg)
 	data, err := readConfig(GlobalContext, objAPI, configFile)
 	if err != nil {
 		return false, nil, err
 	}
-	if !utf8.Valid(data) {
+	var data []byte
-		if GlobalKMS != nil {
+	var err error
 			data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
 				minioMetaBucket: path.Join(minioMetaBucket, configFile),
 			})
 		}
-		if GlobalKMS == nil && err != nil {
+	cfg := &serverConfigV33{}
-			data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
+	for _, cfgFile := range configFiles {
-			if err != nil {
+		if _, err = Load(cfgFile, cfg); err != nil {
-				if err == madmin.ErrMaliciousData || err == madmin.ErrUnexpectedHeader {
+			if !osIsNotExist(err) && !osIsPermission(err) {
-					return false, nil, config.ErrInvalidConfigDecryptionKey(nil)
+				return nil, err
 				}
 				return false, nil, err
 			}
 			continue
 		}
 		data, _ = json.Marshal(cfg)
 		break
 	}
 	if osIsPermission(err) {
 		logger.Info("Older config found but is not readable %s, proceeding to read config from other places", err)
 	}
 	if osIsNotExist(err) || osIsPermission(err) || len(data) == 0 {
 		data, err = readConfig(GlobalContext, objAPI, configFile)
 		if err != nil {
 			// when config.json is not found, then we freshly initialize.
 			if errors.Is(err, errConfigNotFound) {
 				return newServerCfg()
 			}
 			return nil, err
 		}
 		data, err = decryptData(data, configFile)
 		if err != nil {
 			return nil, err
 		}
 		newCfg, err := readServerConfig(GlobalContext, objAPI, data)
 		if err == nil {
 			return newCfg, nil
 		}
 		// Read older `.minio.sys/config/config.json`, if not
 		// possible just fail.
 		if err = json.Unmarshal(data, cfg); err != nil {
 			// Unable to parse old JSON simply re-initialize a new one.
 			return newServerCfg()
 		}
 	}
-	if version == "kvs" {
+	if !globalCredViaEnv && cfg.Credential.IsValid() {
-		// Check api config values are present.
+		// Preserve older credential if we do not have
-		var vcfg struct {
+		// root credentials set via environment variable.
-			API struct {
+		globalActiveCred = cfg.Credential
-				E []struct {
+	}
-					Key   string `json:"key"`
+
-					Value string `json:"value"`
+	// Init compression config. For future migration, Compression config needs to be copied over from previous version.
-				} `json:"_"`
+	switch cfg.Version {
-			} `json:"api"`
+	case "29":
 		// V29 -> V30
 		cfg.Compression.Enabled = false
 		cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
 		cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
 	case "30":
 		// V30 -> V31
 		cfg.OpenID = openid.Config{}
 		cfg.Policy.OPA = opa.Args{
 			URL:       &xnet.URL{},
 			AuthToken: "",
 		}
-		if err = json.Unmarshal(data, &vcfg); err != nil {
+	case "31":
-			return false, nil, nil
+		// V31 -> V32
-		}
+		cfg.Notify.NSQ = make(map[string]target.NSQArgs)
-		return len(vcfg.API.E) > 0, data, nil
+		cfg.Notify.NSQ["1"] = target.NSQArgs{}
 	}
 	var versionConfig struct {
 		Version string `json:"version"`
 	}
 	vcfg := &versionConfig
 	if err = json.Unmarshal(data, vcfg); err != nil {
 		return false, nil, err
 	}
 	return vcfg.Version == version, data, nil
 }
 func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	ok, data, err := checkConfigVersion(objAPI, configFile, "27")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
 	}
 	cfg := &serverConfigV28{}
 	if err = json.Unmarshal(data, cfg); err != nil {
 		return err
 	}
 	cfg.Version = "28"
 	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %w", err)
 	}
 	logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
 	return nil
 }
 func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	ok, data, err := checkConfigVersion(objAPI, configFile, "28")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
 	}
 	cfg := &serverConfigV29{}
 	if err = json.Unmarshal(data, cfg); err != nil {
 		return err
 	}
 	cfg.Version = "29"
 	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %w", err)
 	}
 	logger.Info(configMigrateMSGTemplate, configFile, "28", "29")
 	return nil
 }
 func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	ok, data, err := checkConfigVersion(objAPI, configFile, "29")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
 	}
 	cfg := &serverConfigV30{}
 	if err = json.Unmarshal(data, cfg); err != nil {
 		return err
 	}
 	cfg.Version = "30"
 	// Init compression config.For future migration, Compression config needs to be copied over from previous version.
 	cfg.Compression.Enabled = false
 	cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
 	cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
 	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %w", err)
 	}
 	logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
 	return nil
 }
 func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	ok, data, err := checkConfigVersion(objAPI, configFile, "30")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
 	}
 	cfg := &serverConfigV31{}
 	if err = json.Unmarshal(data, cfg); err != nil {
 		return err
 	}
 	cfg.Version = "31"
 	cfg.OpenID = openid.Config{}
 	cfg.Policy.OPA = opa.Args{
 		URL:       &xnet.URL{},
 		AuthToken: "",
 	}
 	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %w", err)
 	}
 	logger.Info(configMigrateMSGTemplate, configFile, "30", "31")
 	return nil
 }
 func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	ok, data, err := checkConfigVersion(objAPI, configFile, "31")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
 	}
 	cfg := &serverConfigV32{}
 	if err = json.Unmarshal(data, cfg); err != nil {
 		return err
 	}
 	cfg.Version = "32"
 	cfg.Notify.NSQ = make(map[string]target.NSQArgs)
 	cfg.Notify.NSQ["1"] = target.NSQArgs{}
 	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %w", err)
 	}
 	logger.Info(configMigrateMSGTemplate, configFile, "31", "32")
 	return nil
 }
 func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	ok, data, err := checkConfigVersion(objAPI, configFile, "32")
 	if err == errConfigNotFound {
 		return nil
 	} else if err != nil {
 		return fmt.Errorf("Unable to load config file. %w", err)
 	}
 	if !ok {
 		return nil
 	}
 	cfg := &serverConfigV33{}
 	if err = json.Unmarshal(data, cfg); err != nil {
 		return err
 	}
 	// Move to latest.
 	cfg.Version = "33"
 	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
 		return fmt.Errorf("Failed to migrate config from '32' to '33' . %w", err)
 	}
 	logger.Info(configMigrateMSGTemplate, configFile, "32", "33")
 	return nil
 }
 func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
 	bootstrapTrace("migrate config to KV style")
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)
 	// Check if the config version is latest, if not migrate.
 	ok, data, err := checkConfigVersion(objAPI, configFile, "33")
 	if err != nil {
 		return err
 	}
 	if !ok {
 		return nil
 	}
 	cfg := &serverConfigV33{}
 	if err = json.Unmarshal(data, cfg); err != nil {
 		return err
 	}
 	newCfg := newServerConfig()
 	config.SetRegion(newCfg, cfg.Region)
@@ -2797,11 +2608,5 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
 		notify.SetNotifyWebhook(newCfg, k, args)
 	}
-	if err = saveServerConfig(GlobalContext, objAPI, newCfg); err != nil {
+	return newCfg, nil
 		return err
 	}
 	logger.Info("Configuration file %s migrated from version '%s' to new KV format successfully.",
 		configFile, "33")
 	return nil
 }
--- a/Show More
+++ b/Show More