fix: sio-error test to fail if commands fail (#17466 )

allow quorum fileInfo to pick same parityBlocks (#17454 )
Bonus: allow replication to proceed for 503 errors such as with error code SlowDownRead
2026-11-07 08:12:03 +03:00 · 2023-06-19 12:52:50 -07:00 · 2023-06-18 18:20:15 -07:00 · 2023-06-18 11:14:53 -07:00 · 2023-06-18 07:09:20 -07:00 · 2023-06-17 19:18:20 -07:00
302 changed files with 11394 additions and 7153 deletions
@@ -1,5 +0,0 @@
-# Config file for markdownlint-cli
-MD033:
-  allowed_elements:
-    - details
-    - summary
@@ -1,30 +0,0 @@
-name: Markdown Linter
-
-on:
-  pull_request:
-    branches:
-    - master
-
-# This ensures that previous jobs for the PR are canceled when the PR is
-# updated.
-concurrency: 
-  group: ${{ github.workflow }}-${{ github.head_ref }}
-  cancel-in-progress: true
-
-permissions:
-  contents: read
-
-jobs:
-  lint:
-    name: Lint all docs
-    runs-on: ubuntu-latest
-    steps:
-    - name: Check out code
-      uses: actions/checkout@v3
-
-    - name: Lint all docs
-      run: |
-        npm install -g markdownlint-cli
-        markdownlint --fix '**/*.md' \
-          --config /home/runner/work/minio/minio/.github/markdown-lint-cfg.yaml \
-          --disable MD013 MD040 MD051
@@ -11,8 +11,9 @@ x-minio-common: &minio-common
    MINIO_CI_CD: "on"
    MINIO_ROOT_USER: "minio"
    MINIO_ROOT_PASSWORD: "minio123"
-    MINIO_COMPRESS: "true"
-    MINIO_COMPRESS_MIMETYPES: "*"
+    MINIO_COMPRESSION_ENABLE: "on"
+    MINIO_COMPRESSION_MIME_TYPES: "*"
+    MINIO_COMPRESSION_ALLOW_ENCRYPTION: "on"
    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
@@ -11,8 +11,6 @@ x-minio-common: &minio-common
    MINIO_CI_CD: "on"
    MINIO_ROOT_USER: "minio"
    MINIO_ROOT_PASSWORD: "minio123"
-    MINIO_COMPRESS: "true"
-    MINIO_COMPRESS_MIMETYPES: "*"
    MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw="
  healthcheck:
    test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
@@ -0,0 +1,34 @@
+name: Root lockdown tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go-version: [1.20.x]
+        os: [ubuntu-latest]
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go-version }}
+          check-latest: true
+      - name: Start root lockdown tests
+        run: |
+          make test-root-disable
@@ -8,8 +8,9 @@ export SECRET_KEY="$3"
 export JOB_NAME="$4"
 export MINT_MODE="full"

-docker system prune -f
-docker volume prune -f
+docker system prune -f || true
+docker volume prune -f || true
+docker volume rm $(docker volume ls -f dangling=true) || true

 ## change working directory
 cd .github/workflows/mint
@@ -18,34 +19,20 @@ docker-compose -f minio-${MODE}.yaml up -d
 sleep 5m

 docker run --rm --net=host \
-       --name="mint-${MODE}-${JOB_NAME}" \
-       -e SERVER_ENDPOINT="127.0.0.1:9000" \
-       -e ACCESS_KEY="${ACCESS_KEY}" \
-       -e SECRET_KEY="${SECRET_KEY}" \
-       -e ENABLE_HTTPS=0 \
-       -e MINT_MODE="${MINT_MODE}" \
-       docker.io/minio/mint:edge \
-	 aws-sdk-go   \
-	 aws-sdk-java \
-	 aws-sdk-php  \
-	 aws-sdk-ruby \
-	 awscli       \
-	 healthcheck  \
-	 mc           \
-	 minio-go     \
-	 minio-java   \
-	 minio-js     \
-	 minio-py     \
-	 s3cmd        \
-	 s3select     \
-	 versioning
+	--name="mint-${MODE}-${JOB_NAME}" \
+	-e SERVER_ENDPOINT="127.0.0.1:9000" \
+	-e ACCESS_KEY="${ACCESS_KEY}" \
+	-e SECRET_KEY="${SECRET_KEY}" \
+	-e ENABLE_HTTPS=0 \
+	-e MINT_MODE="${MINT_MODE}" \
+	docker.io/minio/mint:edge

 docker-compose -f minio-${MODE}.yaml down || true
 sleep 10s

 docker system prune -f || true
 docker volume prune -f || true
+docker volume rm $(docker volume ls -f dangling=true) || true

 ## change working directory
 cd ../../../
-
@@ -0,0 +1,22 @@
+name: Shell formatting checks
+
+on:
+  pull_request:
+    branches:
+    - master
+
+permissions:
+  contents: read
+    
+jobs:
+  build:
+    name: runner / shfmt
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: luizm/action-sh-checker@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          SHFMT_OPTS: "-s"
+        with:
+          sh_checker_shellcheck_disable: true # disable for now
@@ -20,7 +20,7 @@ jobs:
      - name: Set up Go
        uses: actions/setup-go@v3
        with:
-          go-version: 1.20.x
+          go-version: 1.19.10
          check-latest: true
      - name: Get official govulncheck
        run: go install golang.org/x/vuln/cmd/govulncheck@latest
@@ -917,214 +917,6 @@ https://cloud.google.com/go/iam

 ================================================================

-cloud.google.com/go/longrunning
-https://cloud.google.com/go/longrunning
----------------------------------------------------------------
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
-
-================================================================
-
 cloud.google.com/go/storage
 https://cloud.google.com/go/storage
 ----------------------------------------------------------------
@@ -3095,7 +2887,7 @@ SOFTWARE.
 github.com/beevik/ntp
 https://github.com/beevik/ntp
 ----------------------------------------------------------------
-Copyright 2015-2017 Brett Vickers. All rights reserved.
+Copyright 2015-2023 Brett Vickers. All rights reserved.

 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions
@@ -3262,7 +3054,7 @@ https://github.com/charmbracelet/bubbletea
 ----------------------------------------------------------------
 MIT License

-Copyright (c) 2020 Charmbracelet, Inc
+Copyright (c) 2020-2023 Charmbracelet, Inc

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -14941,7 +14733,7 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 github.com/klauspost/pgzip
 https://github.com/klauspost/pgzip
 ----------------------------------------------------------------
-MIT License
+The MIT License (MIT)

 Copyright (c) 2014 Klaus Post

@@ -14963,6 +14755,7 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.

+
 ================================================================

 github.com/klauspost/readahead
@@ -15020,6 +14813,39 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.


+================================================================
+
+github.com/kr/fs
+https://github.com/kr/fs
+----------------------------------------------------------------
+Copyright (c) 2012 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 ================================================================

 github.com/kr/pretty
@@ -21937,7 +21763,7 @@ https://github.com/montanaflynn/stats
 ----------------------------------------------------------------
 The MIT License (MIT)

-Copyright (c) 2014-2020 Montana Flynn (https://montanaflynn.com)
+Copyright (c) 2014-2023 Montana Flynn (https://montanaflynn.com)

 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -23931,6 +23757,21 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 ================================================================

+github.com/pkg/sftp
+https://github.com/pkg/sftp
+----------------------------------------------------------------
+Copyright (c) 2013, Dave Cheney
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+ * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+ * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 github.com/pkg/xattr
 https://github.com/pkg/xattr
 ----------------------------------------------------------------
@@ -28535,6 +28376,32 @@ THE SOFTWARE.

 ================================================================

+goftp.io/server/v2
+https://goftp.io/server/v2
+----------------------------------------------------------------
+Copyright (c) 2018 Goftp Authors
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+================================================================
+
 golang.org/x/crypto
 https://golang.org/x/crypto
 ----------------------------------------------------------------
@@ -28568,6 +28435,39 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

 ================================================================

+golang.org/x/exp
+https://golang.org/x/exp
+----------------------------------------------------------------
+Copyright (c) 2009 The Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 golang.org/x/mod
 https://golang.org/x/mod
 ----------------------------------------------------------------
@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8

 ARG RELEASE

@@ -27,9 +27,8 @@ COPY LICENSE /licenses/LICENSE
 RUN \
     microdnf clean all && \
     microdnf update --nodocs && \
-     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install minisign --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE} -o /opt/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8

 ARG TARGETARCH

@@ -29,9 +29,8 @@ COPY LICENSE /licenses/LICENSE
 RUN \
     microdnf clean all && \
     microdnf update --nodocs && \
-     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools --nodocs && \
     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install minisign --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /opt/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
@@ -1,4 +1,4 @@
-FROM registry.access.redhat.com/ubi8/ubi-minimal:8.7
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.8

 ARG TARGETARCH

@@ -29,9 +29,8 @@ COPY LICENSE /licenses/LICENSE
 RUN \
     microdnf clean all && \
     microdnf update --nodocs && \
-     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
-     microdnf install minisign --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux gzip lsof tar net-tools iproute iputils jq minisign --nodocs && \
     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /opt/bin/minio && \
     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /opt/bin/minio.sha256sum && \
@@ -49,6 +49,10 @@ test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
 	@MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -tags kqueue ./...

+test-root-disable: install
+	@echo "Running minio root lockdown tests"
+	@env bash $(PWD)/buildscripts/disable-root.sh
+
 test-decom: install
 	@echo "Running minio decom tests"
 	@env bash $(PWD)/docs/distributed/decom.sh
@@ -70,6 +74,9 @@ test-iam: build ## verify IAM (external IDP, etcd backends)
 	@echo "Running tests for IAM (external IDP, etcd backends) with -race"
 	@MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd

+test-sio-error:
+	@(env bash $(PWD)/docs/bucket/replication/sio-error.sh)
+
 test-replication-2site:
 	@(env bash $(PWD)/docs/bucket/replication/setup_2site_existing_replication.sh)

@@ -79,7 +86,7 @@ test-replication-3site:
 test-delete-replication:
 	@(env bash $(PWD)/docs/bucket/replication/delete-replication.sh)

-test-replication: install test-replication-2site test-replication-3site test-delete-replication ## verify multi site replication
+test-replication: install test-replication-2site test-replication-3site test-delete-replication test-sio-error ## verify multi site replication
 	@echo "Running tests for replicating three sites"

 test-site-replication-ldap: install ## verify automatic site replication
@@ -3,19 +3,19 @@

 _init() {

-    shopt -s extglob
+	shopt -s extglob

-    ## Minimum required versions for build dependencies
-    GIT_VERSION="1.0"
-    GO_VERSION="1.16"
-    OSX_VERSION="10.8"
-    KNAME=$(uname -s)
-    ARCH=$(uname -m)
-    case "${KNAME}" in
-        SunOS )
-            ARCH=$(isainfo -k)
-            ;;
-    esac
+	## Minimum required versions for build dependencies
+	GIT_VERSION="1.0"
+	GO_VERSION="1.16"
+	OSX_VERSION="10.8"
+	KNAME=$(uname -s)
+	ARCH=$(uname -m)
+	case "${KNAME}" in
+	SunOS)
+		ARCH=$(isainfo -k)
+		;;
+	esac
 }

 ## FIXME:
@@ -28,24 +28,23 @@ _init() {
 ## }
 ##
 readlink() {
-    TARGET_FILE=$1
+	TARGET_FILE=$1

-    cd `dirname $TARGET_FILE`
-    TARGET_FILE=`basename $TARGET_FILE`
+	cd $(dirname $TARGET_FILE)
+	TARGET_FILE=$(basename $TARGET_FILE)

-    # Iterate down a (possible) chain of symlinks
-    while [ -L "$TARGET_FILE" ]
-    do
-        TARGET_FILE=$(env readlink $TARGET_FILE)
-        cd `dirname $TARGET_FILE`
-        TARGET_FILE=`basename $TARGET_FILE`
-    done
+	# Iterate down a (possible) chain of symlinks
+	while [ -L "$TARGET_FILE" ]; do
+		TARGET_FILE=$(env readlink $TARGET_FILE)
+		cd $(dirname $TARGET_FILE)
+		TARGET_FILE=$(basename $TARGET_FILE)
+	done

-    # Compute the canonicalized name by finding the physical path
-    # for the directory we're in and appending the target file.
-    PHYS_DIR=`pwd -P`
-    RESULT=$PHYS_DIR/$TARGET_FILE
-    echo $RESULT
+	# Compute the canonicalized name by finding the physical path
+	# for the directory we're in and appending the target file.
+	PHYS_DIR=$(pwd -P)
+	RESULT=$PHYS_DIR/$TARGET_FILE
+	echo $RESULT
 }

 ## FIXME:
@@ -59,84 +58,86 @@ readlink() {
 ## }
 ##
 check_minimum_version() {
-    IFS='.' read -r -a varray1 <<< "$1"
-    IFS='.' read -r -a varray2 <<< "$2"
+	IFS='.' read -r -a varray1 <<<"$1"
+	IFS='.' read -r -a varray2 <<<"$2"

-    for i in "${!varray1[@]}"; do
-        if [[ ${varray1[i]} -lt ${varray2[i]} ]]; then
-            return 0
-        elif [[ ${varray1[i]} -gt ${varray2[i]} ]]; then
-            return 1
-        fi
-    done
+	for i in "${!varray1[@]}"; do
+		if [[ ${varray1[i]} -lt ${varray2[i]} ]]; then
+			return 0
+		elif [[ ${varray1[i]} -gt ${varray2[i]} ]]; then
+			return 1
+		fi
+	done

-    return 0
+	return 0
 }

 assert_is_supported_arch() {
-    case "${ARCH}" in
-        x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x )
-            return
-            ;;
-        *)
-            echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x]"
-            exit 1
-    esac
+	case "${ARCH}" in
+	x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x | loong64 | loongarch64)
+		return
+		;;
+	*)
+		echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x, loong64, loongarch64]"
+		exit 1
+		;;
+	esac
 }

 assert_is_supported_os() {
-    case "${KNAME}" in
-        Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS )
-            return
-            ;;
-        Darwin )
-            osx_host_version=$(env sw_vers -productVersion)
-            if ! check_minimum_version "${OSX_VERSION}" "${osx_host_version}"; then
-                echo "OSX version '${osx_host_version}' is not supported. Minimum supported version: ${OSX_VERSION}"
-                exit 1
-            fi
-            return
-            ;;
-        *)
-            echo "OS '${KNAME}' is not supported. Supported OS: [Linux, FreeBSD, OpenBSD, NetBSD, Darwin, DragonFly]"
-            exit 1
-    esac
+	case "${KNAME}" in
+	Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS)
+		return
+		;;
+	Darwin)
+		osx_host_version=$(env sw_vers -productVersion)
+		if ! check_minimum_version "${OSX_VERSION}" "${osx_host_version}"; then
+			echo "OSX version '${osx_host_version}' is not supported. Minimum supported version: ${OSX_VERSION}"
+			exit 1
+		fi
+		return
+		;;
+	*)
+		echo "OS '${KNAME}' is not supported. Supported OS: [Linux, FreeBSD, OpenBSD, NetBSD, Darwin, DragonFly]"
+		exit 1
+		;;
+	esac
 }

 assert_check_golang_env() {
-    if ! which go >/dev/null 2>&1; then
-        echo "Cannot find go binary in your PATH configuration, please refer to Go installation document at https://golang.org/doc/install"
-        exit 1
-    fi
+	if ! which go >/dev/null 2>&1; then
+		echo "Cannot find go binary in your PATH configuration, please refer to Go installation document at https://golang.org/doc/install"
+		exit 1
+	fi

-    installed_go_version=$(go version | sed 's/^.* go\([0-9.]*\).*$/\1/')
-    if ! check_minimum_version "${GO_VERSION}" "${installed_go_version}"; then
-        echo "Go runtime version '${installed_go_version}' is unsupported. Minimum supported version: ${GO_VERSION} to compile."
-        exit 1
-    fi
+	installed_go_version=$(go version | sed 's/^.* go\([0-9.]*\).*$/\1/')
+	if ! check_minimum_version "${GO_VERSION}" "${installed_go_version}"; then
+		echo "Go runtime version '${installed_go_version}' is unsupported. Minimum supported version: ${GO_VERSION} to compile."
+		exit 1
+	fi
 }

 assert_check_deps() {
-    # support unusual Git versions such as: 2.7.4 (Apple Git-66)
-    installed_git_version=$(git version | perl -ne '$_ =~ m/git version (.*?)( |$)/; print "$1\n";')
-    if ! check_minimum_version "${GIT_VERSION}" "${installed_git_version}"; then
-        echo "Git version '${installed_git_version}' is not supported. Minimum supported version: ${GIT_VERSION}"
-        exit 1
-    fi
+	# support unusual Git versions such as: 2.7.4 (Apple Git-66)
+	installed_git_version=$(git version | perl -ne '$_ =~ m/git version (.*?)( |$)/; print "$1\n";')
+	if ! check_minimum_version "${GIT_VERSION}" "${installed_git_version}"; then
+		echo "Git version '${installed_git_version}' is not supported. Minimum supported version: ${GIT_VERSION}"
+		exit 1
+	fi
 }

 main() {
-    ## Check for supported arch
-    assert_is_supported_arch
+	## Check for supported arch
+	assert_is_supported_arch

-    ## Check for supported os
-    assert_is_supported_os
+	## Check for supported os
+	assert_is_supported_os

-    ## Check for Go environment
-    assert_check_golang_env
+	## Check for Go environment
+	assert_check_golang_env

-    ## Check for dependencies
-    assert_check_deps
+	## Check for dependencies
+	assert_check_deps
 }

 _init && main "$@"
@@ -5,33 +5,33 @@ set -e
 [ -n "$BASH_XTRACEFD" ] && set -x

 function _init() {
-    ## All binaries are static make sure to disable CGO.
-    export CGO_ENABLED=0
+	## All binaries are static make sure to disable CGO.
+	export CGO_ENABLED=0

-    ## List of architectures and OS to test coss compilation.
-    SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64"
+	## List of architectures and OS to test coss compilation.
+	SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips"
 }

 function _build() {
-    local osarch=$1
-    IFS=/ read -r -a arr <<<"$osarch"
-    os="${arr[0]}"
-    arch="${arr[1]}"
-    package=$(go list -f '{{.ImportPath}}')
-    printf -- "--> %15s:%s\n" "${osarch}" "${package}"
+	local osarch=$1
+	IFS=/ read -r -a arr <<<"$osarch"
+	os="${arr[0]}"
+	arch="${arr[1]}"
+	package=$(go list -f '{{.ImportPath}}')
+	printf -- "--> %15s:%s\n" "${osarch}" "${package}"

-    # go build -trimpath to build the binary.
-    export GOOS=$os
-    export GOARCH=$arch
-    export GO111MODULE=on
-    go build -trimpath -tags kqueue -o /dev/null
+	# go build -trimpath to build the binary.
+	export GOOS=$os
+	export GOARCH=$arch
+	export GO111MODULE=on
+	go build -trimpath -tags kqueue -o /dev/null
 }

 function main() {
-    echo "Testing builds for OS/Arch: ${SUPPORTED_OSARCH}"
-    for each_osarch in ${SUPPORTED_OSARCH}; do
-        _build "${each_osarch}"
-    done
+	echo "Testing builds for OS/Arch: ${SUPPORTED_OSARCH}"
+	for each_osarch in ${SUPPORTED_OSARCH}; do
+		_build "${each_osarch}"
+	done
 }

 _init && main "$@"
@@ -0,0 +1,119 @@
+#!/bin/bash
+
+set -x
+
+export MINIO_CI_CD=1
+killall -9 minio
+
+rm -rf ${HOME}/tmp/dist
+
+scheme="http"
+nr_servers=4
+
+addr="localhost"
+args=""
+for ((i = 0; i < $((nr_servers)); i++)); do
+	args="$args $scheme://$addr:$((9100 + i))/${HOME}/tmp/dist/path1/$i"
+done
+
+echo $args
+
+for ((i = 0; i < $((nr_servers)); i++)); do
+	(minio server --address ":$((9100 + i))" $args 2>&1 >/tmp/log$i.txt) &
+done
+
+sleep 10s
+
+if [ ! -f ./mc ]; then
+	wget --quiet -O ./mc https://dl.minio.io/client/mc/release/linux-amd64/./mc &&
+		chmod +x mc
+fi
+
+set +e
+
+export MC_HOST_minioadm=http://minioadmin:minioadmin@localhost:9100/
+
+./mc ls minioadm/
+
+./mc admin config set minioadm/ api root_access=off
+
+sleep 3s # let things settle a little
+
+./mc ls minioadm/
+if [ $? -eq 0 ]; then
+	echo "listing succeeded, 'minioadmin' was not disabled"
+	exit 1
+fi
+
+set -e
+
+killall -9 minio
+
+export MINIO_API_ROOT_ACCESS=on
+for ((i = 0; i < $((nr_servers)); i++)); do
+	(minio server --address ":$((9100 + i))" $args 2>&1 >/tmp/log$i.txt) &
+done
+
+set +e
+
+./mc ls minioadm/
+if [ $? -ne 0 ]; then
+	echo "listing failed, 'minioadmin' should be enabled"
+	exit 1
+fi
+
+killall -9 minio
+
+rm -rf /tmp/multisitea/
+rm -rf /tmp/multisiteb/
+
+echo "Setup site-replication and then disable root credentials"
+
+minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
+minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
+
+minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
+minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
+
+sleep 20s
+
+export MC_HOST_sitea=http://minioadmin:minioadmin@127.0.0.1:9001
+export MC_HOST_siteb=http://minioadmin:minioadmin@127.0.0.1:9004
+
+./mc admin replicate add sitea siteb
+
+./mc admin user add sitea foobar foo12345
+
+./mc admin policy attach sitea/ consoleAdmin --user=foobar
+
+./mc admin user info siteb foobar
+
+killall -9 minio
+
+echo "turning off root access, however site replication must continue"
+export MINIO_API_ROOT_ACCESS=off
+
+minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 &
+minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 &
+
+minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 &
+minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \
+	"http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 &
+
+sleep 20s
+
+export MC_HOST_sitea=http://foobar:foo12345@127.0.0.1:9001
+export MC_HOST_siteb=http://foobar:foo12345@127.0.0.1:9004
+
+./mc admin user add sitea foobar-admin foo12345
+
+sleep 2s
+
+./mc admin user info siteb foobar-admin
@@ -6,88 +6,87 @@ set -x

 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)

 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
-    exit 1
+	echo "minio executable binary not found in current directory"
+	exit 1
 fi

 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
-    exit 1
+	echo "minio executable binary not found in current directory"
+	exit 1
 fi

 function start_minio_4drive() {
-    start_port=$1
+	start_port=$1

-    export MINIO_ROOT_USER=minio
-    export MINIO_ROOT_PASSWORD=minio123
-    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
-    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
-    export MINIO_CI_CD=1
+	export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_PASSWORD=minio123
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+	export MINIO_CI_CD=1

-    mkdir ${WORK_DIR}
-    C_PWD=${PWD}
-    if [ ! -x "$PWD/mc" ]; then
-	MC_BUILD_DIR="mc-$RANDOM"
-	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-	    echo "failed to download https://github.com/minio/mc"
-	    purge "${MC_BUILD_DIR}"
-	    exit 1
+	mkdir ${WORK_DIR}
+	C_PWD=${PWD}
+	if [ ! -x "$PWD/mc" ]; then
+		MC_BUILD_DIR="mc-$RANDOM"
+		if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+			echo "failed to download https://github.com/minio/mc"
+			purge "${MC_BUILD_DIR}"
+			exit 1
+		fi
+
+		(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
+
+		# remove mc source.
+		purge "${MC_BUILD_DIR}"
 	fi

-	(cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc")
+	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" >"${WORK_DIR}/server1.log" 2>&1 &
+	pid=$!
+	disown $pid
+	sleep 5

-	# remove mc source.
-	purge "${MC_BUILD_DIR}"
-    fi
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" > "${WORK_DIR}/server1.log" 2>&1 &
-    pid=$!
-    disown $pid
-    sleep 5
+	"${PWD}/mc" mb --with-versioning minio/bucket

-    if ! ps -p ${pid} 1>&2 >/dev/null; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/server1.log"
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi
+	for i in $(seq 1 4); do
+		"${PWD}/mc" cp /etc/hosts minio/bucket/testobj

-    "${PWD}/mc" mb --with-versioning minio/bucket
+		sudo chown -R root. "${WORK_DIR}/disk${i}"

-    for i in $(seq 1 4); do
-	"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
+		"${PWD}/mc" cp /etc/hosts minio/bucket/testobj

-	sudo chown -R root. "${WORK_DIR}/disk${i}"
+		sudo chown -R ${USER}. "${WORK_DIR}/disk${i}"
+	done

-	"${PWD}/mc" cp /etc/hosts minio/bucket/testobj
+	for vid in $("${PWD}/mc" ls --json --versions minio/bucket/testobj | jq -r .versionId); do
+		"${PWD}/mc" cat --vid "${vid}" minio/bucket/testobj | md5sum
+	done

-	sudo chown -R ${USER}. "${WORK_DIR}/disk${i}"
-    done
-
-    for vid in $("${PWD}/mc" ls --json --versions minio/bucket/testobj | jq -r .versionId); do
-	"${PWD}/mc" cat --vid "${vid}" minio/bucket/testobj | md5sum
-    done
-
-    pkill minio
-    sleep 3
+	pkill minio
+	sleep 3
 }

 function main() {
-    start_port=$(shuf -i 10000-65000 -n 1)
+	start_port=$(shuf -i 10000-65000 -n 1)

-    start_minio_4drive ${start_port}
+	start_minio_4drive ${start_port}
 }

-function purge()
-{
-    rm -rf "$1"
+function purge() {
+	rm -rf "$1"
 }

-( main "$@" )
+(main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -4,89 +4,89 @@ trap 'cleanup $LINENO' ERR

 # shellcheck disable=SC2120
 cleanup() {
-    MINIO_VERSION=dev docker-compose \
-                 -f "buildscripts/upgrade-tests/compose.yml" \
-                 rm -s -f
-    docker volume prune -f
+	MINIO_VERSION=dev docker-compose \
+		-f "buildscripts/upgrade-tests/compose.yml" \
+		rm -s -f
+	docker volume prune -f
 }

 verify_checksum_after_heal() {
-    local sum1
-    sum1=$(curl -s "$2" | sha256sum);
-    mc admin heal --json -r "$1" >/dev/null; # test after healing
-    local sum1_heal
-    sum1_heal=$(curl -s "$2" | sha256sum);
+	local sum1
+	sum1=$(curl -s "$2" | sha256sum)
+	mc admin heal --json -r "$1" >/dev/null # test after healing
+	local sum1_heal
+	sum1_heal=$(curl -s "$2" | sha256sum)

-    if [ "${sum1_heal}" != "${sum1}" ]; then
-        echo "mismatch expected ${sum1_heal}, got ${sum1}"
-        exit 1;
-    fi
+	if [ "${sum1_heal}" != "${sum1}" ]; then
+		echo "mismatch expected ${sum1_heal}, got ${sum1}"
+		exit 1
+	fi
 }

 verify_checksum_mc() {
-    local expected
-    expected=$(mc cat "$1" | sha256sum)
-    local got
-    got=$(mc cat "$2" | sha256sum)
+	local expected
+	expected=$(mc cat "$1" | sha256sum)
+	local got
+	got=$(mc cat "$2" | sha256sum)

-    if [ "${expected}" != "${got}" ]; then
-        echo "mismatch - expected ${expected}, got ${got}"
-        exit 1;
-    fi
-    echo "matches - ${expected}, got ${got}"
+	if [ "${expected}" != "${got}" ]; then
+		echo "mismatch - expected ${expected}, got ${got}"
+		exit 1
+	fi
+	echo "matches - ${expected}, got ${got}"
 }

 add_alias() {
-    for i in $(seq 1 4); do
-        echo "... attempting to add alias $i"
-        until (mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin); do
-            echo "...waiting... for 5secs" && sleep 5
-        done
-    done
+	for i in $(seq 1 4); do
+		echo "... attempting to add alias $i"
+		until (mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin); do
+			echo "...waiting... for 5secs" && sleep 5
+		done
+	done

-    echo "Sleeping for nginx"
-    sleep 20
+	echo "Sleeping for nginx"
+	sleep 20
 }

 __init__() {
-    sudo apt install curl -y
-    export GOPATH=/tmp/gopath
-    export PATH=${PATH}:${GOPATH}/bin
+	sudo apt install curl -y
+	export GOPATH=/tmp/gopath
+	export PATH=${PATH}:${GOPATH}/bin

-    go install github.com/minio/mc@latest
+	go install github.com/minio/mc@latest

-    TAG=minio/minio:dev make docker
+	TAG=minio/minio:dev make docker

-    MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \
-                 -f "buildscripts/upgrade-tests/compose.yml" \
-                 up -d --build
+	MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \
+		-f "buildscripts/upgrade-tests/compose.yml" \
+		up -d --build

-    add_alias
+	add_alias

-    mc mb minio/minio-test/
-    mc cp ./minio minio/minio-test/to-read/
-    mc cp /etc/hosts minio/minio-test/to-read/hosts
-    mc anonymous set download minio/minio-test
+	mc mb minio/minio-test/
+	mc cp ./minio minio/minio-test/to-read/
+	mc cp /etc/hosts minio/minio-test/to-read/hosts
+	mc anonymous set download minio/minio-test

-    verify_checksum_mc ./minio minio/minio-test/to-read/minio
+	verify_checksum_mc ./minio minio/minio-test/to-read/minio

-    curl -s http://127.0.0.1:9000/minio-test/to-read/hosts | sha256sum
+	curl -s http://127.0.0.1:9000/minio-test/to-read/hosts | sha256sum

-    MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop
+	MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop
 }

 main() {
-    MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build
+	MINIO_VERSION=dev docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build

-    add_alias
+	add_alias

-    verify_checksum_after_heal minio/minio-test http://127.0.0.1:9000/minio-test/to-read/hosts
+	verify_checksum_after_heal minio/minio-test http://127.0.0.1:9000/minio-test/to-read/hosts

-    verify_checksum_mc ./minio minio/minio-test/to-read/minio
+	verify_checksum_mc ./minio minio/minio-test/to-read/minio

-    verify_checksum_mc /etc/hosts minio/minio-test/to-read/hosts
+	verify_checksum_mc /etc/hosts minio/minio-test/to-read/hosts

-    cleanup
+	cleanup
 }

-( __init__ "$@" && main "$@" )
+(__init__ "$@" && main "$@")
@@ -6,5 +6,5 @@ export GORACE="history_size=7"
 export MINIO_API_REQUESTS_MAX=10000

 for d in $(go list ./...); do
-    CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
+	CGO_ENABLED=1 go test -v -race --timeout 100m "$d"
 done
@@ -6,67 +6,66 @@ set -x

 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)

 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
-    exit 1
+	echo "minio executable binary not found in current directory"
+	exit 1
 fi

 function start_minio_5drive() {
-    start_port=$1
+	start_port=$1

-    export MINIO_ROOT_USER=minio
-    export MINIO_ROOT_PASSWORD=minio123
-    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
-    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
-    export MINIO_CI_CD=1
+	export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_PASSWORD=minio123
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+	export MINIO_CI_CD=1

-    MC_BUILD_DIR="mc-$RANDOM"
-    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-	echo "failed to download https://github.com/minio/mc"
+	MC_BUILD_DIR="mc-$RANDOM"
+	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+		echo "failed to download https://github.com/minio/mc"
+		purge "${MC_BUILD_DIR}"
+		exit 1
+	fi
+
+	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+
+	# remove mc source.
 	purge "${MC_BUILD_DIR}"
-	exit 1
-    fi

-    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+	"${WORK_DIR}/mc" cp --quiet -r "buildscripts/cicd-corpus/" "${WORK_DIR}/cicd-corpus/"

-    # remove mc source.
-    purge "${MC_BUILD_DIR}"
+	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" >"${WORK_DIR}/server1.log" 2>&1 &
+	pid=$!
+	disown $pid
+	sleep 5

-    "${WORK_DIR}/mc" cp --quiet -r "buildscripts/cicd-corpus/" "${WORK_DIR}/cicd-corpus/"
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" > "${WORK_DIR}/server1.log" 2>&1 &
-    pid=$!
-    disown $pid
-    sleep 5
+	"${WORK_DIR}/mc" stat minio/bucket/testobj

-    if ! ps -p ${pid} 1>&2 >/dev/null; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/server1.log"
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi
-
-    "${WORK_DIR}/mc" stat minio/bucket/testobj
-
-    pkill minio
-    sleep 3
+	pkill minio
+	sleep 3
 }

 function main() {
-    start_port=$(shuf -i 10000-65000 -n 1)
+	start_port=$(shuf -i 10000-65000 -n 1)

-    start_minio_5drive ${start_port}
+	start_minio_5drive ${start_port}
 }

-function purge()
-{
-    rm -rf "$1"
+function purge() {
+	rm -rf "$1"
 }

-( main "$@" )
+(main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -6,146 +6,151 @@ set -x

 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO_OLD=( "$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server )
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO_OLD=("$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server)
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)

 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
-    exit 1
+	echo "minio executable binary not found in current directory"
+	exit 1
 fi

 function download_old_release() {
-    if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then
-	curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z
-	chmod a+x minio.RELEASE.2020-10-28T08-16-50Z
-    fi
+	if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then
+		curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z
+		chmod a+x minio.RELEASE.2020-10-28T08-16-50Z
+	fi
 }

 function verify_rewrite() {
-    start_port=$1
+	start_port=$1

-    export MINIO_ACCESS_KEY=minio
-    export MINIO_SECRET_KEY=minio123
-    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
-    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
-    export MINIO_CI_CD=1
+	export MINIO_ACCESS_KEY=minio
+	export MINIO_SECRET_KEY=minio123
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+	export MINIO_CI_CD=1

-    MC_BUILD_DIR="mc-$RANDOM"
-    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-	echo "failed to download https://github.com/minio/mc"
+	MC_BUILD_DIR="mc-$RANDOM"
+	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+		echo "failed to download https://github.com/minio/mc"
+		purge "${MC_BUILD_DIR}"
+		exit 1
+	fi
+
+	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+
+	# remove mc source.
 	purge "${MC_BUILD_DIR}"
-	exit 1
-    fi

-    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+	"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
+	pid=$!
+	disown $pid
+	sleep 10

-    # remove mc source.
-    purge "${MC_BUILD_DIR}"
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
-    pid=$!
-    disown $pid
-    sleep 10
+	"${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock
+	"${WORK_DIR}/mc" cp \
+		buildscripts/verify-build.sh \
+		minio/healing-rewrite-bucket/ \
+		--disable-multipart --quiet

-    if ! ps -p ${pid} 1>&2 >/dev/null; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/server1.log"
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi
+	"${WORK_DIR}/mc" cp \
+		buildscripts/verify-build.sh \
+		minio/healing-rewrite-bucket/ \
+		--disable-multipart --quiet

-    "${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock
-    "${WORK_DIR}/mc" cp \
-		     buildscripts/verify-build.sh \
-		     minio/healing-rewrite-bucket/ \
-		     --disable-multipart --quiet
+	"${WORK_DIR}/mc" cp \
+		buildscripts/verify-build.sh \
+		minio/healing-rewrite-bucket/ \
+		--disable-multipart --quiet

-    "${WORK_DIR}/mc" cp \
-		     buildscripts/verify-build.sh \
-		     minio/healing-rewrite-bucket/ \
-		     --disable-multipart --quiet
+	kill ${pid}
+	sleep 3

-    "${WORK_DIR}/mc" cp \
-		     buildscripts/verify-build.sh \
-		     minio/healing-rewrite-bucket/ \
-		     --disable-multipart --quiet
+	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
+	pid=$!
+	disown $pid
+	sleep 10

-    kill ${pid}
-    sleep 3
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
-    pid=$!
-    disown $pid
-    sleep 10
+	go build ./docs/debugging/s3-check-md5/
+	if ! ./s3-check-md5 \
+		-debug \
+		-versions \
+		-access-key minio \
+		-secret-key minio123 \
+		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		mkdir -p inspects
+		(
+			cd inspects
+			"${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**
+		)

-    if ! ps -p ${pid} 1>&2 >/dev/null; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/server1.log"
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi
+		"${WORK_DIR}/mc" mb play/inspects
+		"${WORK_DIR}/mc" mirror inspects play/inspects

-    go build ./docs/debugging/s3-check-md5/
-    if ! ./s3-check-md5 \
-	 -debug \
-	 -versions \
-	 -access-key minio \
-	 -secret-key minio123 \
-	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/server1.log"
-	echo "FAILED"
-	mkdir -p inspects
-	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
+		purge "$WORK_DIR"
+		exit 1
+	fi

-	"${WORK_DIR}/mc" mb play/inspects
-	"${WORK_DIR}/mc" mirror inspects play/inspects
+	go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
+	sleep 1

-	purge "$WORK_DIR"
-	exit 1
-    fi
+	if ! ./s3-check-md5 \
+		-debug \
+		-versions \
+		-access-key minio \
+		-secret-key minio123 \
+		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		mkdir -p inspects
+		(
+			cd inspects
+			"${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**
+		)

-    go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123"
-    sleep 1
+		"${WORK_DIR}/mc" mb play/inspects
+		"${WORK_DIR}/mc" mirror inspects play/inspects

-    if ! ./s3-check-md5 \
-	 -debug \
-	 -versions \
-	 -access-key minio \
-	 -secret-key minio123 \
-	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/server1.log"
-	echo "FAILED"
-	mkdir -p inspects
-	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/**)
+		purge "$WORK_DIR"
+		exit 1
+	fi

-	"${WORK_DIR}/mc" mb play/inspects
-	"${WORK_DIR}/mc" mirror inspects play/inspects
-
-	purge "$WORK_DIR"
-	exit 1
-    fi
-
-    kill ${pid}
+	kill ${pid}
 }

 function main() {
-    download_old_release
+	download_old_release

-    start_port=$(shuf -i 10000-65000 -n 1)
+	start_port=$(shuf -i 10000-65000 -n 1)

-    verify_rewrite ${start_port}
+	verify_rewrite ${start_port}
 }

-function purge()
-{
-    rm -rf "$1"
+function purge() {
+	rm -rf "$1"
 }

-( main "$@" )
+(main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -6,167 +6,172 @@ set -o pipefail
 set -x

 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
-    exit 1
+	echo "minio executable binary not found in current directory"
+	exit 1
 fi

 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO_OLD=( "$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server )
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO_OLD=("$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server)
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)

 function download_old_release() {
-    if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
-	curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
-	chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
-    fi
+	if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
+		curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
+		chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
+	fi
 }

 function start_minio_16drive() {
-    start_port=$1
+	start_port=$1

-    export MINIO_ROOT_USER=minio
-    export MINIO_ROOT_PASSWORD=minio123
-    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
-    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
-    export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
-    export MINIO_CI_CD=1
+	export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_PASSWORD=minio123
+	export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+	unset MINIO_KMS_AUTO_ENCRYPTION         # do not auto-encrypt objects
+	export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
+	export MINIO_CI_CD=1

-    MC_BUILD_DIR="mc-$RANDOM"
-    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-	echo "failed to download https://github.com/minio/mc"
+	MC_BUILD_DIR="mc-$RANDOM"
+	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+		echo "failed to download https://github.com/minio/mc"
+		purge "${MC_BUILD_DIR}"
+		exit 1
+	fi
+
+	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+
+	# remove mc source.
 	purge "${MC_BUILD_DIR}"
-	exit 1
-    fi

-    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+	"${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
+	pid=$!
+	disown $pid
+	sleep 30

-    # remove mc source.
-    purge "${MC_BUILD_DIR}"
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
-    pid=$!
-    disown $pid
-    sleep 30
+	shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
+	"${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
+	"${WORK_DIR}/mc" cp \
+		"${WORK_DIR}/unaligned" \
+		minio/healing-shard-bucket/unaligned \
+		--disable-multipart --quiet

-    if ! ps -p ${pid} 1>&2 >/dev/null; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/server1.log"
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi
+	## "unaligned" object name gets consistently distributed
+	## to disks in following distribution order
+	##
+	## NOTE: if you change the name make sure to change the
+	## distribution order present here
+	##
+	## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]

-    shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
-    "${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
-    "${WORK_DIR}/mc" cp \
-		     "${WORK_DIR}/unaligned" \
-		     minio/healing-shard-bucket/unaligned \
-		     --disable-multipart --quiet
+	## make sure to remove the "last" data shard
+	rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
+	sleep 10
+	## Heal the shard
+	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
+	## then remove any other data shard let's pick first disk
+	## - 1st data shard.
+	rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
+	sleep 10

-    ## "unaligned" object name gets consistently distributed
-    ## to disks in following distribution order
-    ##
-    ## NOTE: if you change the name make sure to change the
-    ## distribution order present here
-    ##
-    ## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+	go build ./docs/debugging/s3-check-md5/
+	if ! ./s3-check-md5 \
+		-debug \
+		-access-key minio \
+		-secret-key minio123 \
+		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    ## make sure to remove the "last" data shard
-    rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
-    sleep 10
-    ## Heal the shard
-    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
-    ## then remove any other data shard let's pick first disk
-    ## - 1st data shard.
-    rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
-    sleep 10
+	pkill minio
+	sleep 3

-    go build ./docs/debugging/s3-check-md5/
-    if ! ./s3-check-md5 \
-	 -debug \
-	 -access-key minio \
-	 -secret-key minio123 \
-	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/server1.log"
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi
+	"${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 &
+	pid=$!
+	disown $pid
+	sleep 30

-    pkill minio
-    sleep 3
+	if ! ps -p ${pid} 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
-    pid=$!
-    disown $pid
-    sleep 30
+	if ! ./s3-check-md5 \
+		-debug \
+		-access-key minio \
+		-secret-key minio123 \
+		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		mkdir -p inspects
+		(
+			cd inspects
+			"${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**
+		)

-    if ! ps -p ${pid} 1>&2 >/dev/null; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/server1.log"
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi
+		"${WORK_DIR}/mc" mb play/inspects
+		"${WORK_DIR}/mc" mirror inspects play/inspects

-    if ! ./s3-check-md5 \
-	 -debug \
-	 -access-key minio \
-	 -secret-key minio123 \
-	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/server1.log"
-	echo "FAILED"
-	mkdir -p inspects
-	(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
+		purge "$WORK_DIR"
+		exit 1
+	fi

-	"${WORK_DIR}/mc" mb play/inspects
-	"${WORK_DIR}/mc" mirror inspects play/inspects
+	"${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket

-	purge "$WORK_DIR"
-	exit 1
-    fi
+	if ! ./s3-check-md5 \
+		-debug \
+		-access-key minio \
+		-secret-key minio123 \
+		-endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/server1.log"
+		echo "FAILED"
+		mkdir -p inspects
+		(
+			cd inspects
+			"${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**
+		)

-    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
+		"${WORK_DIR}/mc" mb play/inspects
+		"${WORK_DIR}/mc" mirror inspects play/inspects

-    if ! ./s3-check-md5 \
-	 -debug \
-	 -access-key minio \
-	 -secret-key minio123 \
-	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/server1.log"
-	echo "FAILED"
-	mkdir -p inspects
-	(cd inspects; "${WORK_DIR}/mc" support inspect minio/healing-shard-bucket/unaligned/**)
+		purge "$WORK_DIR"
+		exit 1
+	fi

-	"${WORK_DIR}/mc" mb play/inspects
-	"${WORK_DIR}/mc" mirror inspects play/inspects
-
-	purge "$WORK_DIR"
-	exit 1
-    fi
-
-    pkill minio
-    sleep 3
+	pkill minio
+	sleep 3
 }

 function main() {
-    download_old_release
+	download_old_release

-    start_port=$(shuf -i 10000-65000 -n 1)
+	start_port=$(shuf -i 10000-65000 -n 1)

-    start_minio_16drive ${start_port}
+	start_minio_16drive ${start_port}
 }

-function purge()
-{
-    rm -rf "$1"
+function purge() {
+	rm -rf "$1"
 }

-( main "$@" )
+(main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -6,8 +6,8 @@ set -E
 set -o pipefail

 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
-    exit 1
+	echo "minio executable binary not found in current directory"
+	exit 1
 fi

 WORK_DIR="$PWD/.verify-$RANDOM"
@@ -25,285 +25,270 @@ export ENABLE_ADMIN=1
 export MINIO_CI_CD=1

 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" )
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR")

 FILE_1_MB="$MINT_DATA_DIR/datafile-1-MB"
 FILE_65_MB="$MINT_DATA_DIR/datafile-65-MB"

 FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh"

-function start_minio_fs()
-{
-    export MINIO_ROOT_USER=$ACCESS_KEY
-    export MINIO_ROOT_PASSWORD=$SECRET_KEY
-    "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
-    sleep 10
+function start_minio_fs() {
+	export MINIO_ROOT_USER=$ACCESS_KEY
+	export MINIO_ROOT_PASSWORD=$SECRET_KEY
+	"${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 &
+	sleep 10
 }

-function start_minio_erasure()
-{
-    "${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 &
-    sleep 15
+function start_minio_erasure() {
+	"${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 &
+	sleep 15
 }

-function start_minio_erasure_sets()
-{
-    export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}"
-    "${MINIO[@]}" server > "$WORK_DIR/erasure-minio-sets.log" 2>&1 &
-    sleep 15
+function start_minio_erasure_sets() {
+	export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}"
+	"${MINIO[@]}" server >"$WORK_DIR/erasure-minio-sets.log" 2>&1 &
+	sleep 15
 }

-function start_minio_pool_erasure_sets()
-{
-    export MINIO_ROOT_USER=$ACCESS_KEY
-    export MINIO_ROOT_PASSWORD=$SECRET_KEY
-    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}"
-    "${MINIO[@]}" server --address ":9000" > "$WORK_DIR/pool-minio-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address ":9001" > "$WORK_DIR/pool-minio-9001.log" 2>&1 &
+function start_minio_pool_erasure_sets() {
+	export MINIO_ROOT_USER=$ACCESS_KEY
+	export MINIO_ROOT_PASSWORD=$SECRET_KEY
+	export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}"
+	"${MINIO[@]}" server --address ":9000" >"$WORK_DIR/pool-minio-9000.log" 2>&1 &
+	"${MINIO[@]}" server --address ":9001" >"$WORK_DIR/pool-minio-9001.log" 2>&1 &

-    sleep 40
+	sleep 40
 }

-function start_minio_pool_erasure_sets_ipv6()
-{
-    export MINIO_ROOT_USER=$ACCESS_KEY
-    export MINIO_ROOT_PASSWORD=$SECRET_KEY
-    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}"
-    "${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
-    "${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
+function start_minio_pool_erasure_sets_ipv6() {
+	export MINIO_ROOT_USER=$ACCESS_KEY
+	export MINIO_ROOT_PASSWORD=$SECRET_KEY
+	export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}"
+	"${MINIO[@]}" server --address="[::1]:9000" >"$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
+	"${MINIO[@]}" server --address="[::1]:9001" >"$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &

-    sleep 40
+	sleep 40
 }

-function start_minio_dist_erasure()
-{
-    export MINIO_ROOT_USER=$ACCESS_KEY
-    export MINIO_ROOT_PASSWORD=$SECRET_KEY
-    export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4"
-    for i in $(seq 0 3); do
-        "${MINIO[@]}" server --address ":900${i}" > "$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
-    done
+function start_minio_dist_erasure() {
+	export MINIO_ROOT_USER=$ACCESS_KEY
+	export MINIO_ROOT_PASSWORD=$SECRET_KEY
+	export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4"
+	for i in $(seq 0 3); do
+		"${MINIO[@]}" server --address ":900${i}" >"$WORK_DIR/dist-minio-900${i}.log" 2>&1 &
+	done

-    sleep 40
+	sleep 40
 }

-function run_test_fs()
-{
-    start_minio_fs
+function run_test_fs() {
+	start_minio_fs

-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	rv=$?

-    pkill minio
-    sleep 3
+	pkill minio
+	sleep 3

-    if [ "$rv" -ne 0 ]; then
-        cat "$WORK_DIR/fs-minio.log"
-    fi
-    rm -f "$WORK_DIR/fs-minio.log"
+	if [ "$rv" -ne 0 ]; then
+		cat "$WORK_DIR/fs-minio.log"
+	fi
+	rm -f "$WORK_DIR/fs-minio.log"

-    return "$rv"
+	return "$rv"
 }

-function run_test_erasure_sets()
-{
-    start_minio_erasure_sets
+function run_test_erasure_sets() {
+	start_minio_erasure_sets

-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	rv=$?

-    pkill minio
-    sleep 3
+	pkill minio
+	sleep 3

-    if [ "$rv" -ne 0 ]; then
-        cat "$WORK_DIR/erasure-minio-sets.log"
-    fi
-    rm -f "$WORK_DIR/erasure-minio-sets.log"
+	if [ "$rv" -ne 0 ]; then
+		cat "$WORK_DIR/erasure-minio-sets.log"
+	fi
+	rm -f "$WORK_DIR/erasure-minio-sets.log"

-    return "$rv"
+	return "$rv"
 }

-function run_test_pool_erasure_sets()
-{
-    start_minio_pool_erasure_sets
+function run_test_pool_erasure_sets() {
+	start_minio_pool_erasure_sets

-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	rv=$?

-    pkill minio
-    sleep 3
+	pkill minio
+	sleep 3

-    if [ "$rv" -ne 0 ]; then
-        for i in $(seq 0 1); do
-            echo "server$i log:"
-            cat "$WORK_DIR/pool-minio-900$i.log"
-        done
-    fi
+	if [ "$rv" -ne 0 ]; then
+		for i in $(seq 0 1); do
+			echo "server$i log:"
+			cat "$WORK_DIR/pool-minio-900$i.log"
+		done
+	fi

-    for i in $(seq 0 1); do
-        rm -f "$WORK_DIR/pool-minio-900$i.log"
-    done
+	for i in $(seq 0 1); do
+		rm -f "$WORK_DIR/pool-minio-900$i.log"
+	done

-    return "$rv"
+	return "$rv"
 }

-function run_test_pool_erasure_sets_ipv6()
-{
-    start_minio_pool_erasure_sets_ipv6
+function run_test_pool_erasure_sets_ipv6() {
+	start_minio_pool_erasure_sets_ipv6

-    export SERVER_ENDPOINT="[::1]:9000"
+	export SERVER_ENDPOINT="[::1]:9000"

-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	rv=$?

-    pkill minio
-    sleep 3
+	pkill minio
+	sleep 3

-    if [ "$rv" -ne 0 ]; then
-        for i in $(seq 0 1); do
-            echo "server$i log:"
-            cat "$WORK_DIR/pool-minio-ipv6-900$i.log"
-        done
-    fi
+	if [ "$rv" -ne 0 ]; then
+		for i in $(seq 0 1); do
+			echo "server$i log:"
+			cat "$WORK_DIR/pool-minio-ipv6-900$i.log"
+		done
+	fi

-    for i in $(seq 0 1); do
-        rm -f "$WORK_DIR/pool-minio-ipv6-900$i.log"
-    done
+	for i in $(seq 0 1); do
+		rm -f "$WORK_DIR/pool-minio-ipv6-900$i.log"
+	done

-    return "$rv"
+	return "$rv"
 }

-function run_test_erasure()
-{
-    start_minio_erasure
+function run_test_erasure() {
+	start_minio_erasure

-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	rv=$?

-    pkill minio
-    sleep 3
+	pkill minio
+	sleep 3

-    if [ "$rv" -ne 0 ]; then
-        cat "$WORK_DIR/erasure-minio.log"
-    fi
-    rm -f "$WORK_DIR/erasure-minio.log"
+	if [ "$rv" -ne 0 ]; then
+		cat "$WORK_DIR/erasure-minio.log"
+	fi
+	rm -f "$WORK_DIR/erasure-minio.log"

-    return "$rv"
+	return "$rv"
 }

-function run_test_dist_erasure()
-{
-    start_minio_dist_erasure
+function run_test_dist_erasure() {
+	start_minio_dist_erasure

-    (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
-    rv=$?
+	(cd "$WORK_DIR" && "$FUNCTIONAL_TESTS")
+	rv=$?

-    pkill minio
-    sleep 3
+	pkill minio
+	sleep 3

-    if [ "$rv" -ne 0 ]; then
-        echo "server1 log:"
-        cat "$WORK_DIR/dist-minio-9000.log"
-        echo "server2 log:"
-        cat "$WORK_DIR/dist-minio-9001.log"
-        echo "server3 log:"
-        cat "$WORK_DIR/dist-minio-9002.log"
-        echo "server4 log:"
-        cat "$WORK_DIR/dist-minio-9003.log"
-    fi
+	if [ "$rv" -ne 0 ]; then
+		echo "server1 log:"
+		cat "$WORK_DIR/dist-minio-9000.log"
+		echo "server2 log:"
+		cat "$WORK_DIR/dist-minio-9001.log"
+		echo "server3 log:"
+		cat "$WORK_DIR/dist-minio-9002.log"
+		echo "server4 log:"
+		cat "$WORK_DIR/dist-minio-9003.log"
+	fi

-    rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log"
+	rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log"

-    return "$rv"
+	return "$rv"
 }

-function purge()
-{
-    rm -rf "$1"
+function purge() {
+	rm -rf "$1"
 }

-function __init__()
-{
-    echo "Initializing environment"
-    mkdir -p "$WORK_DIR"
-    mkdir -p "$MINIO_CONFIG_DIR"
-    mkdir -p "$MINT_DATA_DIR"
+function __init__() {
+	echo "Initializing environment"
+	mkdir -p "$WORK_DIR"
+	mkdir -p "$MINIO_CONFIG_DIR"
+	mkdir -p "$MINT_DATA_DIR"

-    MC_BUILD_DIR="mc-$RANDOM"
-    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
-        echo "failed to download https://github.com/minio/mc"
-        purge "${MC_BUILD_DIR}"
-        exit 1
-    fi
+	MC_BUILD_DIR="mc-$RANDOM"
+	if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+		echo "failed to download https://github.com/minio/mc"
+		purge "${MC_BUILD_DIR}"
+		exit 1
+	fi

-    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+	(cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")

-    # remove mc source.
-    purge "${MC_BUILD_DIR}"
+	# remove mc source.
+	purge "${MC_BUILD_DIR}"

-    shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null
-    shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null
+	shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null
+	shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null

-    ## version is purposefully set to '3' for minio to migrate configuration file
-    echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
+	## version is purposefully set to '3' for minio to migrate configuration file
+	echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json"

-    if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then
-        echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh"
-        exit 1
-    fi
+	if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then
+		echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh"
+		exit 1
+	fi

-    sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS"
-    chmod a+x "$FUNCTIONAL_TESTS"
+	sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS"
+	chmod a+x "$FUNCTIONAL_TESTS"
 }

-function main()
-{
-    echo "Testing in FS setup"
-    if ! run_test_fs; then
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
-    fi
+function main() {
+	echo "Testing in FS setup"
+	if ! run_test_fs; then
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    echo "Testing in Erasure setup"
-    if ! run_test_erasure; then
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
-    fi
+	echo "Testing in Erasure setup"
+	if ! run_test_erasure; then
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    echo "Testing in Distributed Erasure setup"
-    if ! run_test_dist_erasure; then
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
-    fi
+	echo "Testing in Distributed Erasure setup"
+	if ! run_test_dist_erasure; then
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    echo "Testing in Erasure setup as sets"
-    if ! run_test_erasure_sets; then
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
-    fi
+	echo "Testing in Erasure setup as sets"
+	if ! run_test_erasure_sets; then
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    echo "Testing in Distributed Eraure expanded setup"
-    if ! run_test_pool_erasure_sets; then
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
-    fi
+	echo "Testing in Distributed Eraure expanded setup"
+	if ! run_test_pool_erasure_sets; then
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    echo "Testing in Distributed Erasure expanded setup with ipv6"
-    if ! run_test_pool_erasure_sets_ipv6; then
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
-    fi
+	echo "Testing in Distributed Erasure expanded setup with ipv6"
+	if ! run_test_pool_erasure_sets_ipv6; then
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi

-    purge "$WORK_DIR"
+	purge "$WORK_DIR"
 }

-( __init__ "$@" && main "$@" )
+(__init__ "$@" && main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -4,96 +4,93 @@ set -E
 set -o pipefail
 set -x

-
 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
-    exit 1
+	echo "minio executable binary not found in current directory"
+	exit 1
 fi

 WORK_DIR="$(mktemp -d)"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
-
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)

 function start_minio() {
-    start_port=$1
+	start_port=$1

-    export MINIO_ROOT_USER=minio
-    export MINIO_ROOT_PASSWORD=minio123
-    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
-    unset MINIO_CI_CD
-    unset CI
+	export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_PASSWORD=minio123
+	unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+	unset MINIO_CI_CD
+	unset CI

-    args=()
-    for i in $(seq 1 4); do
-            args+=("http://localhost:$[${start_port}+$i]${WORK_DIR}/mnt/disk$i/ ")
-    done
+	args=()
+	for i in $(seq 1 4); do
+		args+=("http://localhost:$((start_port + i))${WORK_DIR}/mnt/disk$i/ ")
+	done

-    for i in $(seq 1 4); do
-            "${MINIO[@]}" --address ":$[$start_port+$i]" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" &
-    done
+	for i in $(seq 1 4); do
+		"${MINIO[@]}" --address ":$((start_port + i))" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" &
+	done

-    # Wait until all nodes return 403
-    for i in $(seq 1 4); do
-	    while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$[$start_port+$i])" -ne "403" ]; do
-		    echo -n ".";
-                    sleep 1;
-            done
-    done
+	# Wait until all nodes return 403
+	for i in $(seq 1 4); do
+		while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$((start_port + i)))" -ne "403" ]; do
+			echo -n "."
+			sleep 1
+		done
+	done

- }
+}

 # Prepare fake disks with losetup
 function prepare_block_devices() {
-    set -e
-    mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
-    sudo modprobe loop
-    for i in 1 2 3 4; do
-        dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000
-        device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i})
-        sudo mkfs.ext4 -F ${device}
-        mkdir -p ${WORK_DIR}/mnt/disk${i}/
-        sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/
-        sudo chown "$(id -u):$(id -g)" ${device} ${WORK_DIR}/mnt/disk${i}/
-    done
-    set +e
+	set -e
+	mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/
+	sudo modprobe loop
+	for i in 1 2 3 4; do
+		dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000
+		device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i})
+		sudo mkfs.ext4 -F ${device}
+		mkdir -p ${WORK_DIR}/mnt/disk${i}/
+		sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/
+		sudo chown "$(id -u):$(id -g)" ${device} ${WORK_DIR}/mnt/disk${i}/
+	done
+	set +e
 }

 # Start a distributed MinIO setup, unmount one disk and check if it is formatted
 function main() {
-    start_port=$(shuf -i 10000-65000 -n 1)
-    start_minio ${start_port}
+	start_port=$(shuf -i 10000-65000 -n 1)
+	start_minio ${start_port}

-    # Unmount the disk, after the unmount the device id
-    # /tmp/xxx/mnt/disk4 will be the same as '/' and it
-    # will be detected as root disk
-    while [ "$u" != "0" ]; do
-	    sudo umount ${WORK_DIR}/mnt/disk4/
-	    u=$?
-	    sleep 1
-    done
+	# Unmount the disk, after the unmount the device id
+	# /tmp/xxx/mnt/disk4 will be the same as '/' and it
+	# will be detected as root disk
+	while [ "$u" != "0" ]; do
+		sudo umount ${WORK_DIR}/mnt/disk4/
+		u=$?
+		sleep 1
+	done

-    # Wait until MinIO self heal kicks in
-    sleep 60
+	# Wait until MinIO self heal kicks in
+	sleep 60

-    if [ -f ${WORK_DIR}/mnt/disk4/.minio.sys/format.json ]; then
-	    echo "A root disk is formatted unexpectedely"
-            cat "${WORK_DIR}/server4.log"
-            exit -1
-    fi
+	if [ -f ${WORK_DIR}/mnt/disk4/.minio.sys/format.json ]; then
+		echo "A root disk is formatted unexpectedely"
+		cat "${WORK_DIR}/server4.log"
+		exit -1
+	fi
 }

 function cleanup() {
-    pkill minio
-    sudo umount ${WORK_DIR}/mnt/disk{1..3}/
-    sudo rm /dev/minio-loopdisk*
-    rm -rf "$WORK_DIR"
+	pkill minio
+	sudo umount ${WORK_DIR}/mnt/disk{1..3}/
+	sudo rm /dev/minio-loopdisk*
+	rm -rf "$WORK_DIR"
 }

-( prepare_block_devices )
-( main "$@" )
+(prepare_block_devices)
+(main "$@")
 rv=$?

 cleanup
 exit "$rv"
-
@@ -5,139 +5,135 @@ set -E
 set -o pipefail

 if [ ! -x "$PWD/minio" ]; then
-    echo "minio executable binary not found in current directory"
-    exit 1
+	echo "minio executable binary not found in current directory"
+	exit 1
 fi

 WORK_DIR="$PWD/.verify-$RANDOM"
 MINIO_CONFIG_DIR="$WORK_DIR/.minio"
-MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server)

 function start_minio_3_node() {
-    export MINIO_ROOT_USER=minio
-    export MINIO_ROOT_PASSWORD=minio123
-    export MINIO_ERASURE_SET_DRIVE_COUNT=6
-    export MINIO_CI_CD=1
+	export MINIO_ROOT_USER=minio
+	export MINIO_ROOT_PASSWORD=minio123
+	export MINIO_ERASURE_SET_DRIVE_COUNT=6
+	export MINIO_CI_CD=1

-    start_port=$2
-    args=""
-    for i in $(seq 1 3); do
-	args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
-    done
-
-    "${MINIO[@]}" --address ":$[$start_port+1]" $args > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
-    pid1=$!
-    disown ${pid1}
-
-    "${MINIO[@]}" --address ":$[$start_port+2]" $args > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
-    pid2=$!
-    disown $pid2
-
-    "${MINIO[@]}" --address ":$[$start_port+3]" $args > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
-    pid3=$!
-    disown $pid3
-
-    sleep "$1"
-
-    if ! ps -p $pid1 1>&2 > /dev/null; then
-	echo "server1 log:"
-	cat "${WORK_DIR}/dist-minio-server1.log"
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi
-
-    if ! ps -p $pid2 1>&2 > /dev/null; then
-	echo "server2 log:"
-	cat "${WORK_DIR}/dist-minio-server2.log"
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi
-
-    if ! ps -p $pid3 1>&2 > /dev/null; then
-	echo "server3 log:"
-	cat "${WORK_DIR}/dist-minio-server3.log"
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi
-
-    if ! pkill minio; then
+	start_port=$2
+	args=""
 	for i in $(seq 1 3); do
-	    echo "server$i log:"
-	    cat "${WORK_DIR}/dist-minio-server$i.log"
+		args="$args http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/1/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/2/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/3/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/4/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/5/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/6/"
 	done
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi

-    sleep 1;
-    if pgrep minio; then
-	# forcibly killing, to proceed further properly.
-	if ! pkill -9 minio; then
-	    echo "no minio process running anymore, proceed."
+	"${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 &
+	pid1=$!
+	disown ${pid1}
+
+	"${MINIO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-minio-server2.log" 2>&1 &
+	pid2=$!
+	disown $pid2
+
+	"${MINIO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-minio-server3.log" 2>&1 &
+	pid3=$!
+	disown $pid3
+
+	sleep "$1"
+
+	if ! ps -p $pid1 1>&2 >/dev/null; then
+		echo "server1 log:"
+		cat "${WORK_DIR}/dist-minio-server1.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
 	fi
-    fi
-}

+	if ! ps -p $pid2 1>&2 >/dev/null; then
+		echo "server2 log:"
+		cat "${WORK_DIR}/dist-minio-server2.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+
+	if ! ps -p $pid3 1>&2 >/dev/null; then
+		echo "server3 log:"
+		cat "${WORK_DIR}/dist-minio-server3.log"
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+
+	if ! pkill minio; then
+		for i in $(seq 1 3); do
+			echo "server$i log:"
+			cat "${WORK_DIR}/dist-minio-server$i.log"
+		done
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
+
+	sleep 1
+	if pgrep minio; then
+		# forcibly killing, to proceed further properly.
+		if ! pkill -9 minio; then
+			echo "no minio process running anymore, proceed."
+		fi
+	fi
+}

 function check_online() {
-    if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
-	echo "1"
-    fi
+	if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
+		echo "1"
+	fi
 }

-function purge()
-{
-    rm -rf "$1"
+function purge() {
+	rm -rf "$1"
 }

-function __init__()
-{
-    echo "Initializing environment"
-    mkdir -p "$WORK_DIR"
-    mkdir -p "$MINIO_CONFIG_DIR"
+function __init__() {
+	echo "Initializing environment"
+	mkdir -p "$WORK_DIR"
+	mkdir -p "$MINIO_CONFIG_DIR"

-    ## version is purposefully set to '3' for minio to migrate configuration file
-    echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' > "$MINIO_CONFIG_DIR/config.json"
+	## version is purposefully set to '3' for minio to migrate configuration file
+	echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json"
 }

 function perform_test() {
-    start_minio_3_node 120 $2
+	start_minio_3_node 120 $2

-    echo "Testing Distributed Erasure setup healing of drives"
-    echo "Remove the contents of the disks belonging to '${1}' erasure set"
+	echo "Testing Distributed Erasure setup healing of drives"
+	echo "Remove the contents of the disks belonging to '${1}' erasure set"

-    rm -rf ${WORK_DIR}/${1}/*/
+	rm -rf ${WORK_DIR}/${1}/*/

-    start_minio_3_node 120 $2
+	start_minio_3_node 120 $2

-    rv=$(check_online)
-    if [ "$rv" == "1" ]; then
-	for i in $(seq 1 3); do
-	    echo "server$i log:"
-	    cat "${WORK_DIR}/dist-minio-server$i.log"
-	done
-	pkill -9 minio
-	echo "FAILED"
-	purge "$WORK_DIR"
-	exit 1
-    fi
+	rv=$(check_online)
+	if [ "$rv" == "1" ]; then
+		for i in $(seq 1 3); do
+			echo "server$i log:"
+			cat "${WORK_DIR}/dist-minio-server$i.log"
+		done
+		pkill -9 minio
+		echo "FAILED"
+		purge "$WORK_DIR"
+		exit 1
+	fi
 }

-function main()
-{
-    # use same ports for all tests
-    start_port=$(shuf -i 10000-65000 -n 1)
+function main() {
+	# use same ports for all tests
+	start_port=$(shuf -i 10000-65000 -n 1)

-    perform_test "2" ${start_port}
-    perform_test "1" ${start_port}
-    perform_test "3" ${start_port}
+	perform_test "2" ${start_port}
+	perform_test "1" ${start_port}
+	perform_test "3" ${start_port}
 }

-( __init__ "$@" && main "$@" )
+(__init__ "$@" && main "$@")
 rv=$?
 purge "$WORK_DIR"
 exit "$rv"
@@ -91,7 +91,7 @@ func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.
 		acl := &accessControlPolicy{}
 		if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil {
 			if terr, ok := err.(*xml.SyntaxError); ok && terr.Msg == io.EOF.Error() {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingSecurityHeader),
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML),
 					r.URL)
 				return
 			}
@@ -85,11 +85,6 @@ func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *
 		return
 	}

-	if quotaConfig.Type == "fifo" {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
-		return
-	}
-
 	updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
@@ -215,6 +210,11 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL)
 		return
 	}
+	if globalSiteReplicationSys.isEnabled() && !update {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetDenyAddError, err), r.URL)
+		return
+	}
+
 	if update {
 		// overlay the updates on existing target
 		tgt := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, target.Arn)
@@ -225,14 +225,14 @@ func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.
 		for _, op := range ops {
 			switch op {
 			case madmin.CredentialsUpdateType:
-				if globalSiteReplicationSys.isEnabled() {
-					writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetDenyEditError, err), r.URL)
-					return
+				if !globalSiteReplicationSys.isEnabled() {
+					// credentials update is possible only in bucket replication. User will never
+					// know the site replicator creds.
+					tgt.Credentials = target.Credentials
+					tgt.TargetBucket = target.TargetBucket
+					tgt.Secure = target.Secure
+					tgt.Endpoint = target.Endpoint
 				}
-				tgt.Credentials = target.Credentials
-				tgt.TargetBucket = target.TargetBucket
-				tgt.Secure = target.Secure
-				tgt.Endpoint = target.Endpoint
 			case madmin.SyncUpdateType:
 				tgt.ReplicationSync = target.ReplicationSync
 			case madmin.ProxyUpdateType:
@@ -460,7 +460,7 @@ func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *
 					return
 				}
 			case bucketLifecycleConfig:
-				config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
+				config, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
 				if err != nil {
 					if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) {
 						continue
@@ -705,7 +705,7 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 			}
 			if _, ok := bucketMap[bucket]; !ok {
 				opts := MakeBucketOptions{
-					LockEnabled: config.ObjectLockEnabled == "Enabled",
+					LockEnabled: config.Enabled(),
 				}
 				err = objectAPI.MakeBucket(ctx, bucket, opts)
 				if err != nil {
@@ -1032,11 +1032,6 @@ func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *
 				continue
 			}

-			if quotaConfig.Type == "fifo" {
-				rpt.SetStatus(bucket, fileName, fmt.Errorf("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore"))
-				continue
-			}
-
 			updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data)
 			if err != nil {
 				rpt.SetStatus(bucket, fileName, err)
@@ -168,10 +168,16 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			}
 		case errors.Is(err, errPolicyInUse):
 			apiErr = APIError{
-				Code:           "XMinioAdminPolicyInUse",
+				Code:           "XMinioIAMPolicyInUse",
 				Description:    "The policy cannot be removed, as it is in use",
 				HTTPStatusCode: http.StatusBadRequest,
 			}
+		case errors.Is(err, errSessionPolicyTooLarge):
+			apiErr = APIError{
+				Code:           "XMinioIAMServiceAccountSessionPolicyTooLarge",
+				Description:    err.Error(),
+				HTTPStatusCode: http.StatusBadRequest,
+			}
 		case errors.Is(err, kes.ErrKeyExists):
 			apiErr = APIError{
 				Code:           "XMinioKMSKeyExists",
@@ -220,12 +226,10 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 // toAdminAPIErrCode - converts errErasureWriteQuorum error to admin API
 // specific error.
 func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode {
-	switch err {
-	case errErasureWriteQuorum:
+	if errors.Is(err, errErasureWriteQuorum) {
 		return ErrAdminConfigNoQuorum
-	default:
-		return toAPIErrorCode(ctx, err)
 	}
+	return toAPIErrorCode(ctx, err)
 }

 // wraps export error for more context
@@ -83,7 +83,7 @@ func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}

-	if err = validateConfig(cfg, subSys); err != nil {
+	if err = validateConfig(ctx, cfg, subSys); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -211,7 +211,12 @@ func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (re
 		return
 	}

-	if verr := validateConfig(result.Cfg, result.SubSys); verr != nil {
+	tgts, err := config.ParseConfigTargetID(bytes.NewReader(kvBytes))
+	if err != nil {
+		return
+	}
+	ctx = context.WithValue(ctx, config.ContextKeyForTargetFromConfig, tgts)
+	if verr := validateConfig(ctx, result.Cfg, result.SubSys); verr != nil {
 		err = badConfigErr{Err: verr}
 		return
 	}
@@ -353,7 +358,7 @@ func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r
 		return
 	}

-	if err = validateConfig(cfg, ""); err != nil {
+	if err = validateConfig(ctx, cfg, ""); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -464,7 +469,7 @@ func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}

-	if err = validateConfig(cfg, ""); err != nil {
+	if err = validateConfig(ctx, cfg, ""); err != nil {
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -541,7 +546,7 @@ func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Reques
 // setLoggerWebhookSubnetProxy - Sets the logger webhook's subnet proxy value to
 // one being set for subnet proxy
 func setLoggerWebhookSubnetProxy(subSys string, cfg config.Config) bool {
-	if subSys == config.SubnetSubSys {
+	if subSys == config.SubnetSubSys || subSys == config.LoggerWebhookSubSys {
 		subnetWebhookCfg := cfg[config.LoggerWebhookSubSys][subnet.LoggerWebhookName]
 		loggerWebhookSubnetProxy := subnetWebhookCfg.Get(logger.Proxy)
 		subnetProxy := cfg[config.SubnetSubSys][config.Default].Get(logger.Proxy)
@@ -125,7 +125,7 @@ func (a adminAPIHandlers) addOrUpdateIDPHandler(ctx context.Context, w http.Resp
 		return
 	}

-	if err = validateConfig(cfg, subSys); err != nil {
+	if err = validateConfig(ctx, cfg, subSys); err != nil {

 		var validationErr ldap.Validation
 		if errors.As(err, &validationErr) {
@@ -422,7 +422,17 @@ func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *ht
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	if err = validateConfig(cfg, subSys); err != nil {
+	if err = validateConfig(ctx, cfg, subSys); err != nil {
+
+		var validationErr ldap.Validation
+		if errors.As(err, &validationErr) {
+			// If we got an LDAP validation error, we need to send appropriate
+			// error message back to client (likely mc).
+			writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPValidation),
+				validationErr.FormatError(), r.URL)
+			return
+		}
+
 		writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL)
 		return
 	}
@@ -95,7 +95,7 @@ func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Reque
 		poolIndices = append(poolIndices, idx)
 	}

-	if len(poolIndices) > 0 && globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
+	if len(poolIndices) > 0 && !globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
 		ep := globalEndpoints[poolIndices[0]].Endpoints[0]
 		for nodeIdx, proxyEp := range globalProxyEndpoints {
 			if proxyEp.Endpoint.Host == ep.Host {
@@ -32,7 +32,7 @@ import (

 	"github.com/minio/madmin-go/v2"
 	minio "github.com/minio/minio-go/v7"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 )

 func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) {
@@ -44,7 +44,7 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {

 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
+	objectAPI, cred := validateAdminReq(ctx, w, r, iampolicy.DeleteUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -62,6 +62,13 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}

+	// When the user is root credential you are not allowed to
+	// remove the root user. Also you cannot delete yourself.
+	if accessKey == globalActiveCred.AccessKey || accessKey == cred.AccessKey {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+		return
+	}
+
 	if err := globalIAMSys.DeleteUser(ctx, accessKey, true); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -239,6 +246,26 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 		return
 	}
+
+	// Reject if the group add and remove are temporary credentials, or root credential.
+	for _, member := range updReq.Members {
+		ok, _, err := globalIAMSys.IsTempUser(member)
+		if err != nil && err != errNoSuchUser {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		if ok {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+			return
+		}
+		// When the user is root credential you are not allowed to
+		// add policies for root user.
+		if member == globalActiveCred.AccessKey {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+			return
+		}
+	}
+
 	var updatedAt time.Time
 	if updReq.IsRemove {
 		updatedAt, err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members)
@@ -374,7 +401,7 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)

 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))

-	objectAPI, _ := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
+	objectAPI, creds := validateAdminReq(ctx, w, r, iampolicy.EnableUserAdminAction)
 	if objectAPI == nil {
 		return
 	}
@@ -383,9 +410,9 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 	accessKey := vars["accessKey"]
 	status := vars["status"]

-	// This API is not allowed to lookup master access key user status
-	if accessKey == globalActiveCred.AccessKey {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+	// you cannot enable or disable yourself.
+	if accessKey == creds.AccessKey {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL)
 		return
 	}

@@ -638,6 +665,13 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}

+	if err := createReq.Validate(); err != nil {
+		// Since this validation would happen client side as well, we only send
+		// a generic error message here.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
+		return
+	}
+
 	var (
 		targetUser   string
 		targetGroups []string
@@ -650,12 +684,17 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		targetUser = cred.AccessKey
 	}

+	description := createReq.Description
+	if description == "" {
+		description = createReq.Comment
+	}
 	opts := newServiceAccountOpts{
-		accessKey:  createReq.AccessKey,
-		secretKey:  createReq.SecretKey,
-		comment:    createReq.Comment,
-		expiration: createReq.Expiration,
-		claims:     make(map[string]interface{}),
+		accessKey:   createReq.AccessKey,
+		secretKey:   createReq.SecretKey,
+		name:        createReq.Name,
+		description: description,
+		expiration:  createReq.Expiration,
+		claims:      make(map[string]interface{}),
 	}

 	// Find the user for the request sender (as it may be sent via a service
@@ -808,7 +847,8 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 					AccessKey:     newCred.AccessKey,
 					SecretKey:     newCred.SecretKey,
 					Groups:        newCred.Groups,
-					Comment:       newCred.Comment,
+					Name:          newCred.Name,
+					Description:   newCred.Description,
 					Claims:        opts.claims,
 					SessionPolicy: createReq.Policy,
 					Status:        auth.AccountOn,
@@ -883,6 +923,13 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}

+	if err := updateReq.Validate(); err != nil {
+		// Since this validation would happen client side as well, we only send
+		// a generic error message here.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL)
+		return
+	}
+
 	var sp *iampolicy.Policy
 	if len(updateReq.NewPolicy) > 0 {
 		sp, err = iampolicy.ParseConfig(bytes.NewReader(updateReq.NewPolicy))
@@ -894,7 +941,8 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 	opts := updateServiceAccountOpts{
 		secretKey:     updateReq.NewSecretKey,
 		status:        updateReq.NewStatus,
-		comment:       updateReq.NewComment,
+		name:          updateReq.NewName,
+		description:   updateReq.NewDescription,
 		expiration:    updateReq.NewExpiration,
 		sessionPolicy: sp,
 	}
@@ -913,7 +961,8 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 					AccessKey:     accessKey,
 					SecretKey:     opts.secretKey,
 					Status:        opts.status,
-					Comment:       opts.comment,
+					Name:          opts.name,
+					Description:   opts.description,
 					SessionPolicy: updateReq.NewPolicy,
 					Expiration:    updateReq.NewExpiration,
 				},
@@ -1001,7 +1050,8 @@ func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Requ

 	infoResp := madmin.InfoServiceAccountResp{
 		ParentUser:    svcAccount.ParentUser,
-		Comment:       svcAccount.Comment,
+		Name:          svcAccount.Name,
+		Description:   svcAccount.Description,
 		AccountStatus: svcAccount.Status,
 		ImpliedPolicy: policy == nil,
 		Policy:        string(policyJSON),
@@ -1073,14 +1123,18 @@ func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Req
 		return
 	}

-	var serviceAccountsNames []string
+	var serviceAccountList []madmin.ServiceAccountInfo

 	for _, svc := range serviceAccounts {
-		serviceAccountsNames = append(serviceAccountsNames, svc.AccessKey)
+		expiryTime := svc.Expiration
+		serviceAccountList = append(serviceAccountList, madmin.ServiceAccountInfo{
+			AccessKey:  svc.AccessKey,
+			Expiration: &expiryTime,
+		})
 	}

 	listResp := madmin.ListServiceAccountsResp{
-		Accounts: serviceAccountsNames,
+		Accounts: serviceAccountList,
 	}

 	data, err := json.Marshal(listResp)
@@ -1627,6 +1681,12 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
 			return
 		}
+		// When the user is root credential you are not allowed to
+		// add policies for root user.
+		if entityName == globalActiveCred.AccessKey {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+			return
+		}
 	}

 	// Validate that user or group exists.
@@ -1711,7 +1771,7 @@ func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *ht
 	writeSuccessResponseJSON(w, econfigData)
 }

-// AttachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/attach
+// AttachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/policy/attach
 func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "AttachPolicyBuiltin")

@@ -1771,21 +1831,27 @@ func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 			return
 		}

+		// When the user is root credential you are not allowed to
+		// add policies for root user.
+		if userOrGroup == globalActiveCred.AccessKey {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL)
+			return
+		}
+
 		// Validate that user exists.
-		if globalIAMSys.GetUsersSysType() == MinIOUsersSysType {
-			_, ok := globalIAMSys.GetUser(ctx, userOrGroup)
-			if !ok {
-				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
+		_, ok = globalIAMSys.GetUser(ctx, userOrGroup)
+		if !ok {
+			if globalIAMSys.LDAPConfig.Enabled() {
+				// When LDAP is enabled, warn user that they are using the wrong
+				// API.
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUserLDAPWarn), r.URL)
 				return
 			}
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
+			return
 		}
 	}

-	userType := regUser
-	if globalIAMSys.GetUsersSysType() == LDAPUsersSysType {
-		userType = stsUser
-	}
-
 	var existingPolicies []string
 	if isGroup {
 		existingPolicies, err = globalIAMSys.PolicyDBGet(userOrGroup, true)
@@ -1815,6 +1881,7 @@ func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 	existingPolicies = append(existingPolicies, policiesToAttach...)
 	newPolicies := strings.Join(existingPolicies, ",")

+	userType := regUser
 	updatedAt, err := globalIAMSys.PolicyDBSet(ctx, userOrGroup, newPolicies, userType, isGroup)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1835,7 +1902,7 @@ func (a adminAPIHandlers) AttachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 	writeResponse(w, http.StatusCreated, nil, mimeNone)
 }

-// DetachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/detach
+// DetachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/policy/detach
 func (a adminAPIHandlers) DetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "DetachPolicyBuiltin")

@@ -1896,12 +1963,16 @@ func (a adminAPIHandlers) DetachPolicyBuiltin(w http.ResponseWriter, r *http.Req
 		}

 		// Validate that user exists.
-		if globalIAMSys.GetUsersSysType() == MinIOUsersSysType {
-			_, ok := globalIAMSys.GetUser(ctx, userOrGroup)
-			if !ok {
-				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
+		_, ok = globalIAMSys.GetUser(ctx, userOrGroup)
+		if !ok {
+			if globalIAMSys.LDAPConfig.Enabled() {
+				// When LDAP is enabled, warn user that they are using the wrong
+				// API.
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUserLDAPWarn), r.URL)
 				return
 			}
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL)
+			return
 		}
 	}

@@ -2056,13 +2127,15 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 			userAccounts := make(map[string]madmin.AddOrUpdateUserReq)
 			for u, uid := range userIdentities {
-				status := madmin.AccountDisabled
-				if uid.Credentials.IsValid() {
-					status = madmin.AccountEnabled
-				}
 				userAccounts[u] = madmin.AddOrUpdateUserReq{
 					SecretKey: uid.Credentials.SecretKey,
-					Status:    status,
+					Status: func() madmin.AccountStatus {
+						// Export current credential status
+						if uid.Credentials.Status == auth.AccountOff {
+							return madmin.AccountDisabled
+						}
+						return madmin.AccountEnabled
+					}(),
 				}
 			}
 			userData, err := json.Marshal(userAccounts)
@@ -2101,6 +2174,10 @@ func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) {
 			}
 			svcAccts := make(map[string]madmin.SRSvcAccCreate)
 			for user, acc := range serviceAccounts {
+				if user == siteReplicatorSvcAcc {
+					// skip site-replication service account.
+					continue
+				}
 				claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.Credentials.AccessKey)
 				if err != nil {
 					writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL)
@@ -2449,7 +2526,8 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					opts := updateServiceAccountOpts{
 						secretKey:     svcAcctReq.SecretKey,
 						status:        svcAcctReq.Status,
-						comment:       svcAcctReq.Comment,
+						name:          svcAcctReq.Name,
+						description:   svcAcctReq.Description,
 						expiration:    svcAcctReq.Expiration,
 						sessionPolicy: sp,
 					}
@@ -2461,12 +2539,14 @@ func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) {
 					continue
 				}
 				opts := newServiceAccountOpts{
-					accessKey:     user,
-					secretKey:     svcAcctReq.SecretKey,
-					sessionPolicy: sp,
-					claims:        svcAcctReq.Claims,
-					comment:       svcAcctReq.Comment,
-					expiration:    svcAcctReq.Expiration,
+					accessKey:                  user,
+					secretKey:                  svcAcctReq.SecretKey,
+					sessionPolicy:              sp,
+					claims:                     svcAcctReq.Claims,
+					name:                       svcAcctReq.Name,
+					description:                svcAcctReq.Description,
+					expiration:                 svcAcctReq.Expiration,
+					allowSiteReplicatorAccount: false,
 				}

 				// In case of LDAP we need to resolve the targetUser to a DN and
@@ -1329,7 +1329,11 @@ func (c *check) assertSvcAccAppearsInListing(ctx context.Context, madmClient *ma
 	if err != nil {
 		c.Fatalf("unable to list svc accounts: %v", err)
 	}
-	if !set.CreateStringSet(listResp.Accounts...).Contains(svcAK) {
+	var accessKeys []string
+	for _, item := range listResp.Accounts {
+		accessKeys = append(accessKeys, item.AccessKey)
+	}
+	if !set.CreateStringSet(accessKeys...).Contains(svcAK) {
 		c.Fatalf("service account did not appear in listing!")
 	}
 }
@@ -770,7 +770,7 @@ func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request)
 		var err error
 		duration, err = time.ParseDuration(dstr)
 		if err != nil {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL)
+			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
 			return
 		}
 	}
@@ -1525,7 +1525,7 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 		return shouldTrace(entry, traceOpts)
 	})
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

@@ -1605,7 +1605,7 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque

 	err = globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil)
 	if err != nil {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrSlowDown), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

@@ -1855,7 +1855,7 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 	assignPoolNumbers(servers)

 	var poolsInfo map[int]map[int]madmin.ErasureSetInfo
-	var backend interface{}
+	var backend madmin.ErasureBackend

 	mode := madmin.ItemInitializing

@@ -1896,6 +1896,8 @@ func getServerInfo(ctx context.Context, poolsInfoEnabled bool, r *http.Request)
 			OfflineDisks:     offlineDisks.Sum(),
 			StandardSCParity: backendInfo.StandardSCParity,
 			RRSCParity:       backendInfo.RRSCParity,
+			TotalSets:        backendInfo.TotalSets,
+			DrivesPerSet:     backendInfo.DrivesPerSet,
 		}

 		if poolsInfoEnabled {
@@ -2509,13 +2511,13 @@ func fetchLoggerInfo() ([]madmin.Logger, []madmin.Audit) {
 	var auditloggerInfo []madmin.Audit
 	for _, tgt := range logger.SystemTargets() {
 		if tgt.Endpoint() != "" {
-			loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): logger.TargetStatus(tgt)})
+			loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): logger.TargetStatus(GlobalContext, tgt)})
 		}
 	}

 	for _, tgt := range logger.AuditTargets() {
 		if tgt.Endpoint() != "" {
-			auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): logger.TargetStatus(tgt)})
+			auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): logger.TargetStatus(GlobalContext, tgt)})
 		}
 	}

@@ -479,8 +479,8 @@ func TestTopLockEntries(t *testing.T) {
 		if len(exp) != len(got) {
 			return 0, false
 		}
-		sort.Sort(byResourceUID{exp})
-		sort.Sort(byResourceUID{got})
+		sort.Slice(exp, byResourceUID{exp}.Less)
+		sort.Slice(got, byResourceUID{got}.Less)
 		// printEntries(exp)
 		// printEntries(got)
 		for i, e := range exp {
@@ -784,6 +784,11 @@ func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error {
 }

 func (h *healSequence) healItems(objAPI ObjectLayer, bucketsOnly bool) error {
+	if h.clientToken == bgHealingUUID {
+		// For background heal do nothing.
+		return nil
+	}
+
 	if err := h.healDiskMeta(objAPI); err != nil {
 		return err
 	}
@@ -869,13 +874,8 @@ func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly

 	if !h.settings.Recursive {
 		if h.object != "" {
-			// Check if an object named as the objPrefix exists,
-			// and if so heal it.
-			oi, err := objAPI.GetObjectInfo(h.ctx, bucket, h.object, ObjectOptions{})
-			if err == nil {
-				if err = h.healObject(bucket, h.object, oi.VersionID); err != nil {
-					return err
-				}
+			if err := h.healObject(bucket, h.object, ""); err != nil {
+				return err
 			}
 		}

@@ -87,7 +87,6 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 	}

 	props := madmin.ServerProperties{
-		State:    string(madmin.ItemInitializing),
 		Endpoint: addr,
 		Uptime:   UTCNow().Unix() - globalBootTime.Unix(),
 		Version:  Version,
@@ -146,7 +145,8 @@ func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Req
 		props.State = string(madmin.ItemOnline)
 		props.Disks = storageInfo.Disks
 	} else {
-		props.State = string(madmin.ItemOffline)
+		props.State = string(madmin.ItemInitializing)
+		props.Disks = getOfflineDisks("", globalEndpoints)
 	}

 	return props
@@ -28,6 +28,7 @@ import (
 	"strings"

 	"github.com/Azure/azure-storage-blob-go/azblob"
+	"github.com/minio/minio/internal/ioutil"
 	"google.golang.org/api/googleapi"

 	"github.com/minio/madmin-go/v2"
@@ -86,6 +87,7 @@ const (
 	ErrInternalError
 	ErrInvalidAccessKeyID
 	ErrAccessKeyDisabled
+	ErrInvalidArgument
 	ErrInvalidBucketName
 	ErrInvalidDigest
 	ErrInvalidRange
@@ -134,7 +136,7 @@ const (
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
 	ErrReplicationDenyEditError
-	ErrRemoteTargetDenyEditError
+	ErrRemoteTargetDenyAddError
 	ErrReplicationNoExistingObjects
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
@@ -148,6 +150,7 @@ const (
 	ErrMethodNotAllowed
 	ErrInvalidPart
 	ErrInvalidPartOrder
+	ErrMissingPart
 	ErrAuthorizationHeaderMalformed
 	ErrMalformedPOSTRequest
 	ErrPOSTFileRequired
@@ -182,7 +185,8 @@ const (
 	ErrMetadataTooLarge
 	ErrUnsupportedMetadata
 	ErrMaximumExpires
-	ErrSlowDown
+	ErrSlowDownRead
+	ErrSlowDownWrite
 	ErrInvalidPrefixMarker
 	ErrBadRequest
 	ErrKeyTooLongError
@@ -199,6 +203,7 @@ const (
 	ErrInvalidTagDirective
 	ErrPolicyAlreadyAttached
 	ErrPolicyNotAttached
+	ErrExcessData
 	// Add new error codes here.

 	// SSE-S3/SSE-KMS related API errors
@@ -251,10 +256,11 @@ const (
 	ErrInvalidObjectName
 	ErrInvalidObjectNamePrefixSlash
 	ErrInvalidResourceName
+	ErrInvalidLifecycleQueryParameter
 	ErrServerNotInitialized
-	ErrOperationTimedOut
+	ErrRequestTimedout
 	ErrClientDisconnected
-	ErrOperationMaxedOut
+	ErrTooManyRequests
 	ErrInvalidRequest
 	ErrTransitionStorageClassNotFoundError
 	// MinIO storage class error codes
@@ -266,6 +272,7 @@ const (

 	ErrMalformedJSON
 	ErrAdminNoSuchUser
+	ErrAdminNoSuchUserLDAPWarn
 	ErrAdminNoSuchGroup
 	ErrAdminGroupNotEmpty
 	ErrAdminGroupDisabled
@@ -527,6 +534,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your proposed upload exceeds the maximum allowed object size.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrExcessData: {
+		Code:           "ExcessData",
+		Description:    "More data provided than indicated content length",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrPolicyTooLarge: {
 		Code:           "PolicyTooLarge",
 		Description:    "Policy exceeds the maximum allowed document size.",
@@ -552,6 +564,11 @@ var errorCodes = errorCodeMap{
 		Description:    "Your account is disabled; please contact your administrator.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrInvalidArgument: {
+		Code:           "InvalidArgument",
+		Description:    "Invalid argument",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidBucketName: {
 		Code:           "InvalidBucketName",
 		Description:    "The specified bucket is not valid.",
@@ -677,6 +694,11 @@ var errorCodes = errorCodeMap{
 		Description:    "One or more of the specified parts could not be found.  The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrMissingPart: {
+		Code:           "InvalidRequest",
+		Description:    "You must specify at least one part",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrInvalidPartOrder: {
 		Code:           "InvalidPartOrder",
 		Description:    "The list of parts was not in ascending order. The parts list must be specified in order by part number.",
@@ -822,11 +844,16 @@ var errorCodes = errorCodeMap{
 		Description:    "Request is not valid yet",
 		HTTPStatusCode: http.StatusForbidden,
 	},
-	ErrSlowDown: {
-		Code:           "SlowDown",
+	ErrSlowDownRead: {
+		Code:           "SlowDownRead",
 		Description:    "Resource requested is unreadable, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
+	ErrSlowDownWrite: {
+		Code:           "SlowDownWrite",
+		Description:    "Resource requested is unwritable, please reduce your request rate",
+		HTTPStatusCode: http.StatusServiceUnavailable,
+	},
 	ErrInvalidPrefixMarker: {
 		Code:           "InvalidPrefixMarker",
 		Description:    "Invalid marker prefix combination",
@@ -927,9 +954,9 @@ var errorCodes = errorCodeMap{
 		Description:    "No matching ExistingsObjects rule enabled",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrRemoteTargetDenyEditError: {
-		Code:           "XMinioAdminRemoteTargetDenyEdit",
-		Description:    "Cannot alter remote target endpoint since this server is in a cluster replication setup. use `mc admin replicate update`",
+	ErrRemoteTargetDenyAddError: {
+		Code:           "XMinioAdminRemoteTargetDenyAdd",
+		Description:    "Cannot add remote target endpoint since this server is in a cluster replication setup",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrReplicationDenyEditError: {
@@ -1100,8 +1127,13 @@ var errorCodes = errorCodeMap{
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidEncryptionMethod: {
-		Code:           "InvalidRequest",
-		Description:    "The encryption method specified is not supported",
+		Code:           "InvalidArgument",
+		Description:    "Server Side Encryption with AWS KMS managed key requires HTTP header x-amz-server-side-encryption : aws:kms",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
+	ErrIncompatibleEncryptionMethod: {
+		Code:           "InvalidArgument",
+		Description:    "Server Side Encryption with Customer provided key is incompatible with the encryption method specified",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
 	ErrInvalidEncryptionKeyID: {
@@ -1164,11 +1196,6 @@ var errorCodes = errorCodeMap{
 		Description:    "The provided encryption parameters did not match the ones used originally.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrIncompatibleEncryptionMethod: {
-		Code:           "InvalidArgument",
-		Description:    "Server side encryption specified with both SSE-C and SSE-S3 headers",
-		HTTPStatusCode: http.StatusBadRequest,
-	},
 	ErrKMSNotConfigured: {
 		Code:           "NotImplemented",
 		Description:    "Server side encryption specified but KMS is not configured",
@@ -1248,11 +1275,21 @@ var errorCodes = errorCodeMap{
 		Description:    "The JSON you provided was not well-formed or did not validate against our published format.",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrInvalidLifecycleQueryParameter: {
+		Code:           "XMinioInvalidLifecycleParameter",
+		Description:    "The boolean value provided for withUpdatedAt query parameter was invalid.",
+		HTTPStatusCode: http.StatusBadRequest,
+	},
 	ErrAdminNoSuchUser: {
 		Code:           "XMinioAdminNoSuchUser",
 		Description:    "The specified user does not exist.",
 		HTTPStatusCode: http.StatusNotFound,
 	},
+	ErrAdminNoSuchUserLDAPWarn: {
+		Code:           "XMinioAdminNoSuchUser",
+		Description:    "The specified user does not exist. If you meant a user in LDAP, use `mc idp ldap`",
+		HTTPStatusCode: http.StatusNotFound,
+	},
 	ErrAdminNoSuchGroup: {
 		Code:           "XMinioAdminNoSuchGroup",
 		Description:    "The specified group does not exist.",
@@ -1385,7 +1422,7 @@ var errorCodes = errorCodeMap{
 		Description:    "Cannot respond to plain-text request from TLS-encrypted server",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
-	ErrOperationTimedOut: {
+	ErrRequestTimedout: {
 		Code:           "RequestTimeout",
 		Description:    "A timeout occurred while trying to lock a resource, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
@@ -1395,9 +1432,9 @@ var errorCodes = errorCodeMap{
 		Description:    "Client disconnected before response was ready",
 		HTTPStatusCode: 499, // No official code, use nginx value.
 	},
-	ErrOperationMaxedOut: {
-		Code:           "SlowDown",
-		Description:    "A timeout exceeded while waiting to proceed with the request, please reduce your request rate",
+	ErrTooManyRequests: {
+		Code:           "TooManyRequests",
+		Description:    "Deadline exceeded while waiting in incoming queue, please reduce your request rate",
 		HTTPStatusCode: http.StatusServiceUnavailable,
 	},
 	ErrUnsupportedMetadata: {
@@ -2008,11 +2045,14 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	}

 	// Only return ErrClientDisconnected if the provided context is actually canceled.
-	// This way downstream context.Canceled will still report ErrOperationTimedOut
+	// This way downstream context.Canceled will still report ErrRequestTimedout
 	if contextCanceled(ctx) && errors.Is(ctx.Err(), context.Canceled) {
 		return ErrClientDisconnected
 	}

+	// Unwrap the error first
+	err = unwrapAll(err)
+
 	switch err {
 	case errInvalidArgument:
 		apiErr = ErrAdminInvalidArgument
@@ -2020,6 +2060,8 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrAdminNoSuchPolicy
 	case errNoSuchUser:
 		apiErr = ErrAdminNoSuchUser
+	case errNoSuchUserLDAPWarn:
+		apiErr = ErrAdminNoSuchUserLDAPWarn
 	case errNoSuchServiceAccount:
 		apiErr = ErrAdminServiceAccountNotFound
 	case errNoSuchGroup:
@@ -2047,9 +2089,9 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case errInvalidStorageClass:
 		apiErr = ErrInvalidStorageClass
 	case errErasureReadQuorum:
-		apiErr = ErrSlowDown
+		apiErr = ErrSlowDownRead
 	case errErasureWriteQuorum:
-		apiErr = ErrSlowDown
+		apiErr = ErrSlowDownWrite
 	// SSE errors
 	case errInvalidEncryptionParameters:
 		apiErr = ErrInvalidEncryptionParameters
@@ -2083,10 +2125,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrKMSKeyNotFoundException
 	case errKMSDefaultKeyAlreadyConfigured:
 		apiErr = ErrKMSDefaultKeyAlreadyConfigured
-	case context.Canceled, context.DeadlineExceeded:
-		apiErr = ErrOperationTimedOut
-	case errDiskNotFound:
-		apiErr = ErrSlowDown
+	case context.Canceled:
+		apiErr = ErrClientDisconnected
+	case context.DeadlineExceeded:
+		apiErr = ErrRequestTimedout
 	case objectlock.ErrInvalidRetentionDate:
 		apiErr = ErrInvalidRetentionDate
 	case objectlock.ErrPastObjectLockRetainDate:
@@ -2097,6 +2139,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrObjectLockInvalidHeaders
 	case objectlock.ErrMalformedXML:
 		apiErr = ErrMalformedXML
+	case errInvalidMaxParts:
+		apiErr = ErrInvalidMaxParts
+	case ioutil.ErrOverread:
+		apiErr = ErrExcessData
 	}

 	// Compression errors
@@ -2161,9 +2207,9 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case InvalidPart:
 		apiErr = ErrInvalidPart
 	case InsufficientWriteQuorum:
-		apiErr = ErrSlowDown
+		apiErr = ErrSlowDownWrite
 	case InsufficientReadQuorum:
-		apiErr = ErrSlowDown
+		apiErr = ErrSlowDownRead
 	case InvalidMarkerPrefixCombination:
 		apiErr = ErrNotImplemented
 	case InvalidUploadIDKeyCombination:
@@ -2178,10 +2224,10 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 		apiErr = ErrContentSHA256Mismatch
 	case hash.ChecksumMismatch:
 		apiErr = ErrContentChecksumMismatch
-	case ObjectTooLarge:
-		apiErr = ErrEntityTooLarge
-	case ObjectTooSmall:
+	case hash.SizeTooSmall:
 		apiErr = ErrEntityTooSmall
+	case hash.SizeTooLarge:
+		apiErr = ErrEntityTooLarge
 	case NotImplemented:
 		apiErr = ErrNotImplemented
 	case PartTooBig:
@@ -2257,7 +2303,7 @@ func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) {
 	case *event.ErrUnsupportedConfiguration:
 		apiErr = ErrUnsupportedNotification
 	case OperationTimedOut:
-		apiErr = ErrOperationTimedOut
+		apiErr = ErrRequestTimedout
 	case BackendDown:
 		apiErr = ErrBackendDown
 	case ObjectNameTooLong:
@@ -2352,7 +2398,7 @@ func toAPIError(ctx context.Context, err error) APIError {
 			}
 		case lifecycle.Error:
 			apiErr = APIError{
-				Code:           "InvalidRequest",
+				Code:           "InvalidArgument",
 				Description:    e.Error(),
 				HTTPStatusCode: http.StatusBadRequest,
 			}
@@ -20,8 +20,6 @@ package cmd
 import (
 	"context"
 	"errors"
-	"os"
-	"path/filepath"
 	"testing"

 	"github.com/minio/minio/internal/crypto"
@@ -42,8 +40,8 @@ var toAPIErrorTests = []struct {
 	{err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName},
 	{err: InvalidUploadID{}, errCode: ErrNoSuchUpload},
 	{err: InvalidPart{}, errCode: ErrInvalidPart},
-	{err: InsufficientReadQuorum{}, errCode: ErrSlowDown},
-	{err: InsufficientWriteQuorum{}, errCode: ErrSlowDown},
+	{err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead},
+	{err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite},
 	{err: InvalidMarkerPrefixCombination{}, errCode: ErrNotImplemented},
 	{err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented},
 	{err: MalformedUploadID{}, errCode: ErrNoSuchUpload},
@@ -67,11 +65,6 @@ var toAPIErrorTests = []struct {
 }

 func TestAPIErrCode(t *testing.T) {
-	disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix())
-	defer os.RemoveAll(disk)
-
-	initFSObjects(disk, t)
-
 	ctx := context.Background()
 	for i, testCase := range toAPIErrorTests {
 		errCode := toAPIErrorCode(ctx, testCase.err)
@@ -203,7 +203,7 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 	}

 	// Set the relevant version ID as part of the response header.
-	if objInfo.VersionID != "" {
+	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}

@@ -37,8 +37,8 @@ func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string,
 		maxkeys = maxObjectList
 	}

-	prefix = trimLeadingSlash(values.Get("prefix"))
-	marker = trimLeadingSlash(values.Get("marker"))
+	prefix = values.Get("prefix")
+	marker = values.Get("marker")
 	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
 	return
@@ -57,8 +57,8 @@ func getListBucketObjectVersionsArgs(values url.Values) (prefix, marker, delimit
 		maxkeys = maxObjectList
 	}

-	prefix = trimLeadingSlash(values.Get("prefix"))
-	marker = trimLeadingSlash(values.Get("key-marker"))
+	prefix = values.Get("prefix")
+	marker = values.Get("key-marker")
 	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
 	versionIDMarker = values.Get("version-id-marker")
@@ -87,8 +87,8 @@ func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimit
 		maxkeys = maxObjectList
 	}

-	prefix = trimLeadingSlash(values.Get("prefix"))
-	startAfter = trimLeadingSlash(values.Get("start-after"))
+	prefix = values.Get("prefix")
+	startAfter = values.Get("start-after")
 	delimiter = values.Get("delimiter")
 	fetchOwner = values.Get("fetch-owner") == "true"
 	encodingType = values.Get("encoding-type")
@@ -118,8 +118,8 @@ func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadID
 		maxUploads = maxUploadsList
 	}

-	prefix = trimLeadingSlash(values.Get("prefix"))
-	keyMarker = trimLeadingSlash(values.Get("key-marker"))
+	prefix = values.Get("prefix")
+	keyMarker = values.Get("key-marker")
 	uploadIDMarker = values.Get("upload-id-marker")
 	delimiter = values.Get("delimiter")
 	encodingType = values.Get("encoding-type")
@@ -85,7 +85,7 @@ type ListVersionsResponse struct {
 	VersionIDMarker string `xml:"VersionIdMarker"`

 	MaxKeys   int
-	Delimiter string
+	Delimiter string `xml:"Delimiter,omitempty"`
 	// A flag that indicates whether or not ListObjects returned all of the results
 	// that satisfied the search criteria.
 	IsTruncated bool
@@ -115,7 +115,7 @@ type ListObjectsResponse struct {
 	NextMarker string `xml:"NextMarker,omitempty"`

 	MaxKeys   int
-	Delimiter string
+	Delimiter string `xml:"Delimiter,omitempty"`
 	// A flag that indicates whether or not ListObjects returned all of the results
 	// that satisfied the search criteria.
 	IsTruncated bool
@@ -146,7 +146,7 @@ type ListObjectsV2Response struct {

 	KeyCount  int
 	MaxKeys   int
-	Delimiter string
+	Delimiter string `xml:"Delimiter,omitempty"`
 	// A flag that indicates whether or not ListObjects returned all of the results
 	// that satisfied the search criteria.
 	IsTruncated bool
@@ -205,7 +205,7 @@ type ListMultipartUploadsResponse struct {
 	UploadIDMarker     string `xml:"UploadIdMarker"`
 	NextKeyMarker      string
 	NextUploadIDMarker string `xml:"NextUploadIdMarker"`
-	Delimiter          string
+	Delimiter          string `xml:"Delimiter,omitempty"`
 	Prefix             string
 	EncodingType       string `xml:"EncodingType,omitempty"`
 	MaxUploads         int
@@ -345,6 +345,13 @@ func (s *Metadata) MarshalXML(e *xxml.Encoder, start xxml.StartElement) error {
 	return e.EncodeToken(start.End())
 }

+// ObjectInternalInfo contains some internal information about a given
+// object, it will printed in listing calls with enabled metadata.
+type ObjectInternalInfo struct {
+	K int // Data blocks
+	M int // Parity blocks
+}
+
 // Object container for object metadata
 type Object struct {
 	Key          string
@@ -353,7 +360,7 @@ type Object struct {
 	Size         int64

 	// Owner of the object.
-	Owner Owner
+	Owner *Owner `xml:"Owner,omitempty"`

 	// The class of storage used to store the object.
 	StorageClass string
@@ -361,6 +368,8 @@ type Object struct {
 	// UserMetadata user-defined metadata
 	UserMetadata *Metadata `xml:"UserMetadata,omitempty"`
 	UserTags     string    `xml:"UserTags,omitempty"`
+
+	Internal *ObjectInternalInfo `xml:"Internal,omitempty"`
 }

 // CopyObjectResponse container returns ETag and LastModified of the successfully copied object
@@ -497,7 +506,7 @@ func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse {
 func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo, metadata metaCheckFn) ListVersionsResponse {
 	versions := make([]ObjectVersion, 0, len(resp.Objects))

-	owner := Owner{
+	owner := &Owner{
 		ID:          globalMinioDefaultOwnerID,
 		DisplayName: "minio",
 	}
@@ -552,6 +561,10 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 				}
 				content.UserMetadata.Set(k, v)
 			}
+			content.Internal = &ObjectInternalInfo{
+				K: object.DataBlocks,
+				M: object.ParityBlocks,
+			}
 		}
 		content.Owner = owner
 		content.VersionID = object.VersionID
@@ -589,7 +602,7 @@ func generateListVersionsResponse(bucket, prefix, marker, versionIDMarker, delim
 // generates an ListObjectsV1 response for the said bucket with other enumerated options.
 func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse {
 	contents := make([]Object, 0, len(resp.Objects))
-	owner := Owner{
+	owner := &Owner{
 		ID:          globalMinioDefaultOwnerID,
 		DisplayName: "minio",
 	}
@@ -638,10 +651,14 @@ func generateListObjectsV1Response(bucket, prefix, marker, delimiter, encodingTy
 // generates an ListObjectsV2 response for the said bucket with other enumerated options.
 func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata metaCheckFn) ListObjectsV2Response {
 	contents := make([]Object, 0, len(objects))
-	owner := Owner{
-		ID:          globalMinioDefaultOwnerID,
-		DisplayName: "minio",
+	var owner *Owner
+	if fetchOwner {
+		owner = &Owner{
+			ID:          globalMinioDefaultOwnerID,
+			DisplayName: "minio",
+		}
 	}
+
 	data := ListObjectsV2Response{}

 	for _, object := range objects {
@@ -687,6 +704,10 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 					}
 					content.UserMetadata.Set(k, v)
 				}
+				content.Internal = &ObjectInternalInfo{
+					K: object.DataBlocks,
+					M: object.ParityBlocks,
+				}
 			}
 		}
 		contents = append(contents, content)
@@ -743,7 +764,7 @@ func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) Initi

 // generates CompleteMultipartUploadResponse for given bucket, key, location and ETag.
 func generateCompleteMultpartUploadResponse(bucket, key, location string, oi ObjectInfo) CompleteMultipartUploadResponse {
-	cs := oi.decryptChecksums()
+	cs := oi.decryptChecksums(0)
 	c := CompleteMultipartUploadResponse{
 		Location: location,
 		Bucket:   bucket,
@@ -924,6 +945,8 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 }

 func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) {
+	w.Header().Set(xMinIOErrCodeHeader, err.Code)
+	w.Header().Set(xMinIOErrDescHeader, "\""+err.Description+"\"")
 	writeResponse(w, err.HTTPStatusCode, nil, mimeNone)
 }

@@ -245,10 +245,10 @@ func registerAPIRouter(router *mux.Router) {
 		router.Methods(http.MethodPut).Path("/{object:.+}").
 			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
 			HandlerFunc(collectAPIStats("copyobjectpart", maxClients(gz(httpTraceAll(api.CopyObjectPartHandler))))).
-			Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+			Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
 		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+			collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("listobjectparts", maxClients(gz(httpTraceAll(api.ListObjectPartsHandler))))).Queries("uploadId", "{uploadId:.*}")
@@ -88,6 +88,18 @@ func isRequestSignStreamingV4(r *http.Request) bool {
 		r.Method == http.MethodPut
 }

+// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation.
+func isRequestSignStreamingTrailerV4(r *http.Request) bool {
+	return r.Header.Get(xhttp.AmzContentSha256) == streamingContentSHA256Trailer &&
+		r.Method == http.MethodPut
+}
+
+// Verify if the request has AWS Streaming Signature Version '4', with unsigned content and trailer.
+func isRequestUnsignedTrailerV4(r *http.Request) bool {
+	return r.Header.Get(xhttp.AmzContentSha256) == unsignedPayloadTrailer &&
+		r.Method == http.MethodPut && strings.Contains(r.Header.Get(xhttp.ContentEncoding), streamingContentEncoding)
+}
+
 // Authorization type.
 //
 //go:generate stringer -type=authType -trimprefix=authType $GOFILE
@@ -105,10 +117,12 @@ const (
 	authTypeSignedV2
 	authTypeJWT
 	authTypeSTS
+	authTypeStreamingSignedTrailer
+	authTypeStreamingUnsignedTrailer
 )

 // Get request authentication type.
-func getRequestAuthType(r *http.Request) authType {
+func getRequestAuthType(r *http.Request) (at authType) {
 	if r.URL != nil {
 		var err error
 		r.Form, err = url.ParseQuery(r.URL.RawQuery)
@@ -123,6 +137,10 @@ func getRequestAuthType(r *http.Request) authType {
 		return authTypePresignedV2
 	} else if isRequestSignStreamingV4(r) {
 		return authTypeStreamingSigned
+	} else if isRequestSignStreamingTrailerV4(r) {
+		return authTypeStreamingSignedTrailer
+	} else if isRequestUnsignedTrailerV4(r) {
+		return authTypeStreamingUnsignedTrailer
 	} else if isRequestSignatureV4(r) {
 		return authTypeSigned
 	} else if isRequestPresignedSignatureV4(r) {
@@ -560,13 +578,15 @@ func isReqAuthenticated(ctx context.Context, r *http.Request, region string, sty

 // List of all support S3 auth types.
 var supportedS3AuthTypes = map[authType]struct{}{
-	authTypeAnonymous:       {},
-	authTypePresigned:       {},
-	authTypePresignedV2:     {},
-	authTypeSigned:          {},
-	authTypeSignedV2:        {},
-	authTypePostPolicy:      {},
-	authTypeStreamingSigned: {},
+	authTypeAnonymous:                {},
+	authTypePresigned:                {},
+	authTypePresignedV2:              {},
+	authTypeSigned:                   {},
+	authTypeSignedV2:                 {},
+	authTypePostPolicy:               {},
+	authTypeStreamingSigned:          {},
+	authTypeStreamingSignedTrailer:   {},
+	authTypeStreamingUnsignedTrailer: {},
 }

 // Validate if the authType is valid and supported.
@@ -582,7 +602,8 @@ func setAuthHandler(h http.Handler) http.Handler {
 		tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt)

 		aType := getRequestAuthType(r)
-		if aType == authTypeSigned || aType == authTypeSignedV2 || aType == authTypeStreamingSigned {
+		switch aType {
+		case authTypeSigned, authTypeSignedV2, authTypeStreamingSigned, authTypeStreamingSignedTrailer:
 			// Verify if date headers are set, if not reject the request
 			amzDate, errCode := parseAmzDateHeader(r)
 			if errCode != ErrNone {
@@ -613,10 +634,16 @@ func setAuthHandler(h http.Handler) http.Handler {
 				atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1)
 				return
 			}
-		}
-		if isSupportedS3AuthType(aType) || aType == authTypeJWT || aType == authTypeSTS {
 			h.ServeHTTP(w, r)
 			return
+		case authTypeJWT, authTypeSTS:
+			h.ServeHTTP(w, r)
+			return
+		default:
+			if isSupportedS3AuthType(aType) {
+				h.ServeHTTP(w, r)
+				return
+			}
 		}

 		if ok {
@@ -710,7 +737,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 		return ErrSignatureVersionNotSupported
 	case authTypeSignedV2, authTypePresignedV2:
 		cred, owner, s3Err = getReqAccessKeyV2(r)
-	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
+	case authTypeStreamingSigned, authTypePresigned, authTypeSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer:
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {
@@ -375,8 +375,6 @@ func TestIsReqAuthenticated(t *testing.T) {

 	initConfigSubsystem(ctx, objLayer)

-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
-
 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
 		t.Fatalf("unable create credential, %s", err)
@@ -384,6 +382,8 @@ func TestIsReqAuthenticated(t *testing.T) {

 	globalActiveCred = creds

+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+
 	// List of test cases for validating http request authentication.
 	testCases := []struct {
 		req     *http.Request
@@ -464,9 +464,8 @@ func TestValidateAdminSignature(t *testing.T) {
 	}

 	initAllSubsystems(ctx)
-	initConfigSubsystem(ctx, objLayer)

-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+	initConfigSubsystem(ctx, objLayer)

 	creds, err := auth.CreateCredentials("admin", "mypassword")
 	if err != nil {
@@ -474,6 +473,8 @@ func TestValidateAdminSignature(t *testing.T) {
 	}
 	globalActiveCred = creds

+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+
 	testCases := []struct {
 		AccessKey string
 		SecretKey string
@@ -18,11 +18,13 @@ func _() {
 	_ = x[authTypeSignedV2-7]
 	_ = x[authTypeJWT-8]
 	_ = x[authTypeSTS-9]
+	_ = x[authTypeStreamingSignedTrailer-10]
+	_ = x[authTypeStreamingUnsignedTrailer-11]
 }

-const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingSignedSignedSignedV2JWTSTS"
+const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingSignedSignedSignedV2JWTSTSStreamingSignedTrailerStreamingUnsignedTrailer"

-var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81}
+var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81, 103, 127}

 func (i authType) String() string {
 	if i < 0 || i >= authType(len(_authType_index)-1) {
@@ -26,12 +26,15 @@ import (
 	"os"
 	"sort"
 	"strings"
+	"sync"
 	"time"

 	"github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/env"
 )

 const (
@@ -43,7 +46,8 @@ const (

 // healingTracker is used to persist healing information during a heal.
 type healingTracker struct {
-	disk StorageAPI `msg:"-"`
+	disk StorageAPI    `msg:"-"`
+	mu   *sync.RWMutex `msg:"-"`

 	ID         string
 	PoolIndex  int
@@ -112,22 +116,76 @@ func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker,
 	}
 	h.disk = disk
 	h.ID = diskID
+	h.mu = &sync.RWMutex{}
 	return &h, nil
 }

 // newHealingTracker will create a new healing tracker for the disk.
-func newHealingTracker(disk StorageAPI, healID string) *healingTracker {
-	diskID, _ := disk.GetDiskID()
-	h := healingTracker{
-		disk:     disk,
-		ID:       diskID,
-		HealID:   healID,
-		Path:     disk.String(),
-		Endpoint: disk.Endpoint().String(),
-		Started:  time.Now().UTC(),
+func newHealingTracker() *healingTracker {
+	return &healingTracker{
+		mu: &sync.RWMutex{},
 	}
+}
+
+func initHealingTracker(disk StorageAPI, healID string) *healingTracker {
+	h := newHealingTracker()
+	diskID, _ := disk.GetDiskID()
+	h.disk = disk
+	h.ID = diskID
+	h.HealID = healID
+	h.Path = disk.String()
+	h.Endpoint = disk.Endpoint().String()
+	h.Started = time.Now().UTC()
 	h.PoolIndex, h.SetIndex, h.DiskIndex = disk.GetDiskLoc()
-	return &h
+	return h
+}
+
+func (h healingTracker) getLastUpdate() time.Time {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	return h.LastUpdate
+}
+
+func (h healingTracker) getBucket() string {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	return h.Bucket
+}
+
+func (h *healingTracker) setBucket(bucket string) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	h.Bucket = bucket
+}
+
+func (h healingTracker) getObject() string {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
+	return h.Object
+}
+
+func (h *healingTracker) setObject(object string) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	h.Object = object
+}
+
+func (h *healingTracker) updateProgress(success bool, bytes uint64) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	if success {
+		h.ItemsHealed++
+		h.BytesDone += bytes
+	} else {
+		h.ItemsFailed++
+		h.BytesFailed += bytes
+	}
 }

 // update will update the tracker on the disk.
@@ -136,15 +194,18 @@ func (h *healingTracker) update(ctx context.Context) error {
 	if h.disk.Healing() == nil {
 		return fmt.Errorf("healingTracker: drive %q is not marked as healing", h.ID)
 	}
+	h.mu.Lock()
 	if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
 		h.ID, _ = h.disk.GetDiskID()
 		h.PoolIndex, h.SetIndex, h.DiskIndex = h.disk.GetDiskLoc()
 	}
+	h.mu.Unlock()
 	return h.save(ctx)
 }

 // save will unconditionally save the tracker and will be created if not existing.
 func (h *healingTracker) save(ctx context.Context) error {
+	h.mu.Lock()
 	if h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 {
 		// Attempt to get location.
 		if api := newObjectLayerFn(); api != nil {
@@ -155,6 +216,7 @@ func (h *healingTracker) save(ctx context.Context) error {
 	}
 	h.LastUpdate = time.Now().UTC()
 	htrackerBytes, err := h.MarshalMsg(nil)
+	h.mu.Unlock()
 	if err != nil {
 		return err
 	}
@@ -176,6 +238,8 @@ func (h *healingTracker) delete(ctx context.Context) error {
 }

 func (h *healingTracker) isHealed(bucket string) bool {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
 	for _, v := range h.HealedBuckets {
 		if v == bucket {
 			return true
@@ -186,6 +250,9 @@ func (h *healingTracker) isHealed(bucket string) bool {

 // resume will reset progress to the numbers at the start of the bucket.
 func (h *healingTracker) resume() {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
 	h.ItemsHealed = h.ResumeItemsHealed
 	h.ItemsFailed = h.ResumeItemsFailed
 	h.BytesDone = h.ResumeBytesDone
@@ -195,6 +262,9 @@ func (h *healingTracker) resume() {
 // bucketDone should be called when a bucket is done healing.
 // Adds the bucket to the list of healed buckets and updates resume numbers.
 func (h *healingTracker) bucketDone(bucket string) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
 	h.ResumeItemsHealed = h.ItemsHealed
 	h.ResumeItemsFailed = h.ItemsFailed
 	h.ResumeBytesDone = h.BytesDone
@@ -211,6 +281,9 @@ func (h *healingTracker) bucketDone(bucket string) {
 // setQueuedBuckets will add buckets, but exclude any that is already in h.HealedBuckets.
 // Order is preserved.
 func (h *healingTracker) setQueuedBuckets(buckets []BucketInfo) {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
 	s := set.CreateStringSet(h.HealedBuckets...)
 	h.QueuedBuckets = make([]string, 0, len(buckets))
 	for _, b := range buckets {
@@ -221,15 +294,22 @@ func (h *healingTracker) setQueuedBuckets(buckets []BucketInfo) {
 }

 func (h *healingTracker) printTo(writer io.Writer) {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
 	b, err := json.MarshalIndent(h, "", "  ")
 	if err != nil {
 		writer.Write([]byte(err.Error()))
+		return
 	}
 	writer.Write(b)
 }

 // toHealingDisk converts the information to madmin.HealingDisk
 func (h *healingTracker) toHealingDisk() madmin.HealingDisk {
+	h.mu.RLock()
+	defer h.mu.RUnlock()
+
 	return madmin.HealingDisk{
 		ID:                h.ID,
 		HealID:            h.HealID,
@@ -267,7 +347,9 @@ func initAutoHeal(ctx context.Context, objAPI ObjectLayer) {

 	globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...)

-	go monitorLocalDisksAndHeal(ctx, z)
+	if env.Get("_MINIO_AUTO_DISK_HEALING", config.EnableOn) == config.EnableOn {
+		go monitorLocalDisksAndHeal(ctx, z)
+	}
 }

 func getLocalDisksToHeal() (disksToHeal Endpoints) {
@@ -336,7 +418,7 @@ func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint
 			return nil
 		}
 		logger.LogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err))
-		tracker = newHealingTracker(disk, mustGetUUID())
+		tracker = initHealingTracker(disk, mustGetUUID())
 	}

 	logger.Info(fmt.Sprintf("Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint))
@@ -48,11 +48,11 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/workers"
 	"github.com/minio/pkg/console"
 	"github.com/minio/pkg/env"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	"github.com/minio/pkg/wildcard"
+	"github.com/minio/pkg/workers"
 	"gopkg.in/yaml.v2"
 )

@@ -86,6 +86,7 @@ import (
 //     type: "minio"
 //     bucket: "testbucket1"
 //     endpoint: "https://play.min.io"
+//     path: "on"
 //     credentials:
 //       accessKey: "minioadmin"
 //       secretKey: "minioadmin"
@@ -174,15 +175,22 @@ type BatchJobReplicateResourceType string
 func (t BatchJobReplicateResourceType) Validate() error {
 	switch t {
 	case BatchJobReplicateResourceMinIO:
+	case BatchJobReplicateResourceS3:
 	default:
 		return errInvalidArgument
 	}
 	return nil
 }

+func (t BatchJobReplicateResourceType) isMinio() bool {
+	return t == BatchJobReplicateResourceMinIO
+}
+
 // Different types of batch jobs..
 const (
 	BatchJobReplicateResourceMinIO BatchJobReplicateResourceType = "minio"
+	BatchJobReplicateResourceS3    BatchJobReplicateResourceType = "s3"
+
 	// add future targets
 )

@@ -214,9 +222,15 @@ type BatchJobReplicateTarget struct {
 	Bucket   string                        `yaml:"bucket" json:"bucket"`
 	Prefix   string                        `yaml:"prefix" json:"prefix"`
 	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
+	Path     string                        `yaml:"path" json:"path"`
 	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
 }

+// ValidPath returns true if path is valid
+func (t BatchJobReplicateTarget) ValidPath() bool {
+	return t.Path == "on" || t.Path == "off" || t.Path == "auto" || t.Path == ""
+}
+
 // BatchJobReplicateSource describes source element of the replication job that is
 // the source of the data for the target
 type BatchJobReplicateSource struct {
@@ -224,9 +238,20 @@ type BatchJobReplicateSource struct {
 	Bucket   string                        `yaml:"bucket" json:"bucket"`
 	Prefix   string                        `yaml:"prefix" json:"prefix"`
 	Endpoint string                        `yaml:"endpoint" json:"endpoint"`
+	Path     string                        `yaml:"path" json:"path"`
 	Creds    BatchJobReplicateCredentials  `yaml:"credentials" json:"credentials"`
 }

+// ValidPath returns true if path is valid
+func (s BatchJobReplicateSource) ValidPath() bool {
+	switch s.Path {
+	case "on", "off", "auto", "":
+		return true
+	default:
+		return false
+	}
+}
+
 // BatchJobReplicateV1 v1 of batch job replication
 type BatchJobReplicateV1 struct {
 	APIVersion string                  `yaml:"apiVersion" json:"apiVersion"`
@@ -297,10 +322,13 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec

 	versioned := globalBucketVersioningSys.PrefixEnabled(tgtBucket, tgtObject)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(tgtBucket, tgtObject)
-
+	versionID := srcObjInfo.VersionID
+	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
+		versionID = ""
+	}
 	if srcObjInfo.DeleteMarker {
 		_, err := api.DeleteObject(ctx, tgtBucket, tgtObject, ObjectOptions{
-			VersionID:          srcObjInfo.VersionID,
+			VersionID:          versionID,
 			VersionSuspended:   versionSuspended,
 			Versioned:          versioned,
 			MTime:              srcObjInfo.ModTime,
@@ -318,6 +346,9 @@ func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api Objec
 		PreserveETag:     srcObjInfo.ETag,
 		UserDefined:      srcObjInfo.UserDefined,
 	}
+	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
+		opts.VersionID = ""
+	}
 	if crypto.S3.IsEncrypted(srcObjInfo.UserDefined) {
 		opts.ServerSideEncryption = encrypt.NewSSE()
 	}
@@ -358,7 +389,9 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 	if r.Target.Prefix != "" {
 		tgtObject = path.Join(r.Target.Prefix, srcObjInfo.Name)
 	}
-
+	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
+		opts.VersionID = ""
+	}
 	var uploadedParts []CompletePart
 	res, err := api.NewMultipartUpload(context.Background(), tgtBucket, tgtObject, opts)
 	if err != nil {
@@ -402,7 +435,7 @@ func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, a
 		}
 		defer rd.Close()

-		hr, err = hash.NewReader(rd, objInfo.Size, "", "", objInfo.Size)
+		hr, err = hash.NewReader(io.LimitReader(rd, objInfo.Size), objInfo.Size, "", "", objInfo.Size)
 		if err != nil {
 			return err
 		}
@@ -513,9 +546,10 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	cred := r.Source.Creds

 	c, err := miniogo.New(u.Host, &miniogo.Options{
-		Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
-		Secure:    u.Scheme == "https",
-		Transport: getRemoteInstanceTransport,
+		Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+		Secure:       u.Scheme == "https",
+		Transport:    getRemoteInstanceTransport,
+		BucketLookup: lookupStyle(r.Source.Path),
 	})
 	if err != nil {
 		return err
@@ -539,19 +573,47 @@ func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLay
 	retry := false
 	for attempts := 1; attempts <= retryAttempts; attempts++ {
 		attempts := attempts
-
+		// one of source/target is s3, skip delete marker and all versions under the same object name.
+		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3
+		minioSrc := r.Source.Type == BatchJobReplicateResourceMinIO
 		ctx, cancel := context.WithCancel(ctx)
 		objInfoCh := c.ListObjects(ctx, r.Source.Bucket, minio.ListObjectsOptions{
 			Prefix:       r.Source.Prefix,
-			WithVersions: true,
+			WithVersions: minioSrc,
 			Recursive:    true,
 			WithMetadata: true,
 		})
+		prevObj := ""
+		skipReplicate := false
+
 		for obj := range objInfoCh {
 			oi := toObjectInfo(r.Source.Bucket, obj.Key, obj)
+			if !minioSrc {
+				oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, minio.StatObjectOptions{})
+				if err == nil {
+					oi = toObjectInfo(r.Source.Bucket, obj.Key, oi2)
+				} else {
+					if isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) ||
+						isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) {
+						continue
+					}
+					logger.LogIf(ctx, err)
+					cancel()
+					return err
+				}
+			}
 			if skip(oi) {
 				continue
 			}
+			if obj.Key != prevObj {
+				prevObj = obj.Key
+				// skip replication of delete marker and all versions under the same object name if one of source or target is s3.
+				skipReplicate = obj.IsDeleteMarker && s3Type
+			}
+			if skipReplicate {
+				continue
+			}
+
 			wk.Take()
 			go func() {
 				defer wk.Give()
@@ -646,9 +708,10 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
 	tgtBucket := r.Target.Bucket
 	tgtPrefix := r.Target.Prefix
 	srcObject := srcObjInfo.Name
+	s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3

 	if srcObjInfo.DeleteMarker || !srcObjInfo.VersionPurgeStatus.Empty() {
-		if retry {
+		if retry && !s3Type {
 			if _, err := c.StatObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), miniogo.StatObjectOptions{
 				VersionID: srcObjInfo.VersionID,
 				Internal: miniogo.AdvancedGetOptions{
@@ -664,7 +727,10 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
 		if srcObjInfo.VersionPurgeStatus.Empty() {
 			dmVersionID = srcObjInfo.VersionID
 		}
-
+		if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
+			dmVersionID = ""
+			versionID = ""
+		}
 		return c.RemoveObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), miniogo.RemoveObjectOptions{
 			VersionID: versionID,
 			Internal: miniogo.AdvancedRemoveOptions{
@@ -676,7 +742,7 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
 		})
 	}

-	if retry { // when we are retrying avoid copying if necessary.
+	if retry && !s3Type { // when we are retrying avoid copying if necessary.
 		gopts := miniogo.GetObjectOptions{}
 		if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil {
 			return err
@@ -694,7 +760,7 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
 		Versioned:        versioned,
 		VersionSuspended: versionSuspended,
 	}
-	rd, err := api.GetObjectNInfo(ctx, srcBucket, srcObject, nil, http.Header{}, readLock, opts)
+	rd, err := api.GetObjectNInfo(ctx, srcBucket, srcObject, nil, http.Header{}, opts)
 	if err != nil {
 		return err
 	}
@@ -710,7 +776,9 @@ func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectL
 	if err != nil {
 		return err
 	}
-
+	if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 {
+		putOpts.Internal = miniogo.AdvancedPutOptions{}
+	}
 	if objInfo.isMultipart() {
 		if err := replicateObjectWithMultipart(ctx, c, tgtBucket, pathJoin(tgtPrefix, objInfo.Name), rd, objInfo, putOpts); err != nil {
 			return err
@@ -1021,6 +1089,10 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			// None of the provided metadata filters match skip the object.
 			return false
 		}
+		// if one of source or target is non MinIO, just replicate the top most version like `mc mirror`
+		if (r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3) && !info.IsLatest {
+			return false
+		}

 		return true
 	}
@@ -1033,9 +1105,10 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 	cred := r.Target.Creds

 	c, err := miniogo.NewCore(u.Host, &miniogo.Options{
-		Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
-		Secure:    u.Scheme == "https",
-		Transport: getRemoteInstanceTransport,
+		Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+		Secure:       u.Scheme == "https",
+		Transport:    getRemoteInstanceTransport,
+		BucketLookup: lookupStyle(r.Target.Path),
 	})
 	if err != nil {
 		return err
@@ -1059,6 +1132,8 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 		attempts := attempts

 		ctx, cancel := context.WithCancel(ctx)
+		// one of source/target is s3, skip delete marker and all versions under the same object name.
+		s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3

 		results := make(chan ObjectInfo, 100)
 		if err := api.Walk(ctx, r.Source.Bucket, r.Source.Prefix, results, ObjectOptions{
@@ -1070,8 +1145,18 @@ func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job Ba
 			return err
 		}

+		prevObj := ""
+
+		skipReplicate := false
 		for result := range results {
 			result := result
+			if result.Name != prevObj {
+				prevObj = result.Name
+				skipReplicate = result.DeleteMarker && s3Type
+			}
+			if skipReplicate {
+				continue
+			}
 			wk.Take()
 			go func() {
 				defer wk.Give()
@@ -1195,6 +1280,13 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 		return errInvalidArgument
 	}

+	if r.Source.Endpoint != "" && !r.Source.Type.isMinio() && !r.Source.ValidPath() {
+		return errInvalidArgument
+	}
+
+	if r.Target.Endpoint != "" && !r.Target.Type.isMinio() && !r.Target.ValidPath() {
+		return errInvalidArgument
+	}
 	if r.Target.Bucket == "" {
 		return errInvalidArgument
 	}
@@ -1232,11 +1324,14 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	remoteEp := r.Target.Endpoint
 	remoteBkt := r.Target.Bucket
 	cred := r.Target.Creds
+	pathStyle := r.Target.Path

 	if r.Source.Endpoint != "" {
 		remoteEp = r.Source.Endpoint
 		cred = r.Source.Creds
 		remoteBkt = r.Source.Bucket
+		pathStyle = r.Source.Path
+
 	}

 	u, err := url.Parse(remoteEp)
@@ -1245,9 +1340,10 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 	}

 	c, err := miniogo.NewCore(u.Host, &miniogo.Options{
-		Creds:     credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
-		Secure:    u.Scheme == "https",
-		Transport: getRemoteInstanceTransport,
+		Creds:        credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken),
+		Secure:       u.Scheme == "https",
+		Transport:    getRemoteInstanceTransport,
+		BucketLookup: lookupStyle(pathStyle),
 	})
 	if err != nil {
 		return err
@@ -1265,8 +1361,10 @@ func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest,
 		}
 		return err
 	}
+	// if both source and target are minio instances
+	minioType := r.Target.Type == BatchJobReplicateResourceMinIO && r.Source.Type == BatchJobReplicateResourceMinIO
 	// If source has versioning enabled, target must have versioning enabled
-	if (info.Versioning && !vcfg.Enabled() && !isRemoteToLocal) || (!info.Versioning && vcfg.Enabled() && isRemoteToLocal) {
+	if minioType && ((info.Versioning && !vcfg.Enabled() && !isRemoteToLocal) || (!info.Versioning && vcfg.Enabled() && isRemoteToLocal)) {
 		return batchReplicationJobError{
 			Code: "InvalidBucketState",
 			Description: fmt.Sprintf("The source '%s' has versioning enabled, target '%s' must have versioning enabled",
@@ -1355,9 +1453,10 @@ func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (p
 		return putOpts, err
 	}
 	putOpts.Internal = miniogo.AdvancedPutOptions{
-		SourceVersionID: objInfo.VersionID,
-		SourceMTime:     objInfo.ModTime,
-		SourceETag:      objInfo.ETag,
+		SourceVersionID:    objInfo.VersionID,
+		SourceMTime:        objInfo.ModTime,
+		SourceETag:         objInfo.ETag,
+		ReplicationRequest: true,
 	}
 	return putOpts, nil
 }
@@ -1810,3 +1909,17 @@ func (m *batchJobMetrics) trace(d batchJobMetric, job string, attempts int, info
 		}
 	}
 }
+
+func lookupStyle(s string) miniogo.BucketLookupType {
+	var lookup miniogo.BucketLookupType
+	switch s {
+	case "on":
+		lookup = miniogo.BucketLookupPath
+	case "off":
+		lookup = miniogo.BucketLookupDNS
+	default:
+		lookup = miniogo.BucketLookupAuto
+
+	}
+	return lookup
+}
@@ -702,6 +702,12 @@ func (z *BatchJobReplicateSource) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Endpoint")
 				return
 			}
+		case "Path":
+			z.Path, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Path")
+				return
+			}
 		case "Creds":
 			var zb0003 uint32
 			zb0003, err = dc.ReadMapHeader()
@@ -756,9 +762,9 @@ func (z *BatchJobReplicateSource) DecodeMsg(dc *msgp.Reader) (err error) {

 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 5
+	// map header, size 6
 	// write "Type"
-	err = en.Append(0x85, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	err = en.Append(0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	if err != nil {
 		return
 	}
@@ -797,6 +803,16 @@ func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Endpoint")
 		return
 	}
+	// write "Path"
+	err = en.Append(0xa4, 0x50, 0x61, 0x74, 0x68)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Path)
+	if err != nil {
+		err = msgp.WrapError(err, "Path")
+		return
+	}
 	// write "Creds"
 	err = en.Append(0xa5, 0x43, 0x72, 0x65, 0x64, 0x73)
 	if err != nil {
@@ -839,9 +855,9 @@ func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobReplicateSource) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 5
+	// map header, size 6
 	// string "Type"
-	o = append(o, 0x85, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	o = append(o, 0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	o = msgp.AppendString(o, string(z.Type))
 	// string "Bucket"
 	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
@@ -852,6 +868,9 @@ func (z *BatchJobReplicateSource) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Endpoint"
 	o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
 	o = msgp.AppendString(o, z.Endpoint)
+	// string "Path"
+	o = append(o, 0xa4, 0x50, 0x61, 0x74, 0x68)
+	o = msgp.AppendString(o, z.Path)
 	// string "Creds"
 	o = append(o, 0xa5, 0x43, 0x72, 0x65, 0x64, 0x73)
 	// map header, size 3
@@ -913,6 +932,12 @@ func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error)
 				err = msgp.WrapError(err, "Endpoint")
 				return
 			}
+		case "Path":
+			z.Path, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Path")
+				return
+			}
 		case "Creds":
 			var zb0003 uint32
 			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
@@ -968,7 +993,7 @@ func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error)

 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobReplicateSource) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken)
+	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken)
 	return
 }

@@ -1018,6 +1043,12 @@ func (z *BatchJobReplicateTarget) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "Endpoint")
 				return
 			}
+		case "Path":
+			z.Path, err = dc.ReadString()
+			if err != nil {
+				err = msgp.WrapError(err, "Path")
+				return
+			}
 		case "Creds":
 			var zb0003 uint32
 			zb0003, err = dc.ReadMapHeader()
@@ -1072,9 +1103,9 @@ func (z *BatchJobReplicateTarget) DecodeMsg(dc *msgp.Reader) (err error) {

 // EncodeMsg implements msgp.Encodable
 func (z *BatchJobReplicateTarget) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 5
+	// map header, size 6
 	// write "Type"
-	err = en.Append(0x85, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	err = en.Append(0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	if err != nil {
 		return
 	}
@@ -1113,6 +1144,16 @@ func (z *BatchJobReplicateTarget) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "Endpoint")
 		return
 	}
+	// write "Path"
+	err = en.Append(0xa4, 0x50, 0x61, 0x74, 0x68)
+	if err != nil {
+		return
+	}
+	err = en.WriteString(z.Path)
+	if err != nil {
+		err = msgp.WrapError(err, "Path")
+		return
+	}
 	// write "Creds"
 	err = en.Append(0xa5, 0x43, 0x72, 0x65, 0x64, 0x73)
 	if err != nil {
@@ -1155,9 +1196,9 @@ func (z *BatchJobReplicateTarget) EncodeMsg(en *msgp.Writer) (err error) {
 // MarshalMsg implements msgp.Marshaler
 func (z *BatchJobReplicateTarget) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 5
+	// map header, size 6
 	// string "Type"
-	o = append(o, 0x85, 0xa4, 0x54, 0x79, 0x70, 0x65)
+	o = append(o, 0x86, 0xa4, 0x54, 0x79, 0x70, 0x65)
 	o = msgp.AppendString(o, string(z.Type))
 	// string "Bucket"
 	o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74)
@@ -1168,6 +1209,9 @@ func (z *BatchJobReplicateTarget) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "Endpoint"
 	o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74)
 	o = msgp.AppendString(o, z.Endpoint)
+	// string "Path"
+	o = append(o, 0xa4, 0x50, 0x61, 0x74, 0x68)
+	o = msgp.AppendString(o, z.Path)
 	// string "Creds"
 	o = append(o, 0xa5, 0x43, 0x72, 0x65, 0x64, 0x73)
 	// map header, size 3
@@ -1229,6 +1273,12 @@ func (z *BatchJobReplicateTarget) UnmarshalMsg(bts []byte) (o []byte, err error)
 				err = msgp.WrapError(err, "Endpoint")
 				return
 			}
+		case "Path":
+			z.Path, bts, err = msgp.ReadStringBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "Path")
+				return
+			}
 		case "Creds":
 			var zb0003 uint32
 			zb0003, bts, err = msgp.ReadMapHeaderBytes(bts)
@@ -1284,7 +1334,7 @@ func (z *BatchJobReplicateTarget) UnmarshalMsg(bts []byte) (o []byte, err error)

 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BatchJobReplicateTarget) Msgsize() (s int) {
-	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken)
+	s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken)
 	return
 }

@@ -38,9 +38,9 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/workers"
 	"github.com/minio/pkg/env"
 	"github.com/minio/pkg/wildcard"
+	"github.com/minio/pkg/workers"
 )

 // keyrotate:
@@ -24,6 +24,7 @@ import (
 	"io"
 	"net/http"
 	"net/url"
+	"os"
 	"reflect"
 	"time"

@@ -216,6 +217,7 @@ func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointS
 	for onlineServers < len(clnts)/2 {
 		for _, clnt := range clnts {
 			if err := clnt.Verify(ctx, srcCfg); err != nil {
+				bootstrapTrace(fmt.Sprintf("clnt.Verify: %v, endpoint: %v", err, clnt.endpoint))
 				if !isNetworkError(err) {
 					logger.LogOnceIf(ctx, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err), clnt.String())
 					incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt.String(), err))
@@ -264,7 +266,11 @@ func newBootstrapRESTClients(endpointServerPools EndpointServerPools) []*bootstr

 			// Only proceed for remote endpoints.
 			if !endpoint.IsLocal {
-				clnts = append(clnts, newBootstrapRESTClient(endpoint))
+				cl := newBootstrapRESTClient(endpoint)
+				if serverDebugLog {
+					cl.restClient.TraceOutput = os.Stdout
+				}
+				clnts = append(clnts, cl)
 			}
 		}
 	}
@@ -21,13 +21,17 @@ import (
 	"bytes"
 	"context"
 	"encoding/base64"
+	"encoding/json"
 	"encoding/xml"
+	"errors"
 	"fmt"
 	"io"
+	"mime/multipart"
 	"net/http"
 	"net/textproto"
 	"net/url"
 	"path"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -35,8 +39,10 @@ import (

 	"github.com/google/uuid"
 	"github.com/minio/mux"
+	"github.com/valyala/bytebufferpool"

 	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	"github.com/minio/minio/internal/auth"
@@ -49,17 +55,21 @@ import (
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
+	"github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/bucket/policy"
 	iampolicy "github.com/minio/pkg/iam/policy"
+	"github.com/minio/pkg/sync/errgroup"
 )

 const (
 	objectLockConfig        = "object-lock.xml"
 	bucketTaggingConfig     = "tagging.xml"
 	bucketReplicationConfig = "replication.xml"
+
+	xMinIOErrCodeHeader = "x-minio-error-code"
+	xMinIOErrDescHeader = "x-minio-error-desc"
 )

 // Check if there are buckets on server without corresponding entry in etcd backend and
@@ -429,6 +439,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 	// The max. XML contains 100000 object names (each at most 1024 bytes long) + XML overhead
 	const maxBodySize = 2 * 100000 * 1024

+	if r.ContentLength > maxBodySize {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL)
+		return
+	}
+
 	// Unmarshal list of keys to be deleted.
 	deleteObjectsReq := &DeleteObjectsRequest{}
 	if err := xmlDecoder(r.Body, deleteObjectsReq, maxBodySize); err != nil {
@@ -540,6 +555,11 @@ func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter,
 			oss[index].SetTransitionState(goi.TransitionedObject)
 		}

+		// All deletes on directory objects needs to be for `nullVersionID`
+		if isDirObject(object.ObjectName) && object.VersionID == "" {
+			object.VersionID = nullVersionID
+		}
+
 		if replicateDeletes {
 			dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{
 				ObjectV: ObjectV{
@@ -882,14 +902,6 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	}

 	bucket := mux.Vars(r)["bucket"]
-
-	// Require Content-Length to be set in the request
-	size := r.ContentLength
-	if size < 0 {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL)
-		return
-	}
-
 	resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
 	if err != nil {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
@@ -904,43 +916,151 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h

 	// Here the parameter is the size of the form data that should
 	// be loaded in memory, the remaining being put in temporary files.
-	reader, err := r.MultipartReader()
+	mp, err := r.MultipartReader()
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
-		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
+		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}

-	// Read multipart data and save in memory and in the disk if needed
-	form, err := reader.ReadForm(maxFormMemory)
-	if err != nil {
+	const mapEntryOverhead = 200
+
+	var (
+		reader        io.Reader
+		fileSize      int64 = -1
+		fileName      string
+		fanOutEntries = make([]minio.PutObjectFanOutEntry, 0, 100)
+	)
+
+	maxParts := 1000
+	// Canonicalize the form values into http.Header.
+	formValues := make(http.Header)
+	for {
+		part, err := mp.NextRawPart()
+		if errors.Is(err, io.EOF) {
+			break
+		}
+		if err != nil {
+			apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+			apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
+			writeErrorResponse(ctx, w, apiErr, r.URL)
+			return
+		}
+		if maxParts <= 0 {
+			apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+			apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
+			writeErrorResponse(ctx, w, apiErr, r.URL)
+			return
+		}
+		maxParts--
+
+		name := part.FormName()
+		if name == "" {
+			continue
+		}
+
+		fileName = part.FileName()
+
+		// Multiple values for the same key (one map entry, longer slice) are cheaper
+		// than the same number of values for different keys (many map entries), but
+		// using a consistent per-value cost for overhead is simpler.
+		maxMemoryBytes := 2 * int64(10<<20)
+		maxMemoryBytes -= int64(len(name))
+		maxMemoryBytes -= mapEntryOverhead
+		if maxMemoryBytes < 0 {
+			// We can't actually take this path, since nextPart would already have
+			// rejected the MIME headers for being too large. Check anyway.
+			apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+			apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
+			writeErrorResponse(ctx, w, apiErr, r.URL)
+			return
+		}
+
+		var b bytes.Buffer
+		if fileName == "" {
+			if http.CanonicalHeaderKey(name) == http.CanonicalHeaderKey("x-minio-fanout-list") {
+				dec := json.NewDecoder(part)
+
+				// while the array contains values
+				for dec.More() {
+					var m minio.PutObjectFanOutEntry
+					if err := dec.Decode(&m); err != nil {
+						part.Close()
+						apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+						apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
+						writeErrorResponse(ctx, w, apiErr, r.URL)
+						return
+					}
+					fanOutEntries = append(fanOutEntries, m)
+				}
+				part.Close()
+				continue
+			}
+
+			// value, store as string in memory
+			n, err := io.CopyN(&b, part, maxMemoryBytes+1)
+			part.Close()
+
+			if err != nil && err != io.EOF {
+				apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+				apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err)
+				writeErrorResponse(ctx, w, apiErr, r.URL)
+				return
+			}
+			maxMemoryBytes -= n
+			if maxMemoryBytes < 0 {
+				apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+				apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
+				writeErrorResponse(ctx, w, apiErr, r.URL)
+				return
+			}
+			if n > maxFormFieldSize {
+				apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+				apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge)
+				writeErrorResponse(ctx, w, apiErr, r.URL)
+				return
+			}
+			formValues[http.CanonicalHeaderKey(name)] = append(formValues[http.CanonicalHeaderKey(name)], b.String())
+			continue
+		}
+
+		// In accordance with https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html
+		// The file or text content.
+		// The file or text content must be the last field in the form.
+		// You cannot upload more than one file at a time.
+		reader = part
+		// we have found the File part of the request we are done processing multipart-form
+		break
+	}
+
+	if _, ok := formValues["Key"]; !ok {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
-		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
+		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The name of the uploaded key is missing"))
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}

-	// Remove all tmp files created during multipart upload
-	defer form.RemoveAll()
-
-	// Extract all form fields
-	fileBody, fileName, fileSize, formValues, err := extractPostPolicyFormValues(ctx, form)
-	if err != nil {
+	if fileName == "" {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
-		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
+		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The file or text content is missing"))
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
-
-	// Check if file is provided, error out otherwise.
-	if fileBody == nil {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPOSTFileRequired), r.URL)
+	checksum, err := hash.GetContentChecksum(formValues)
+	if err != nil {
+		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, fmt.Errorf("Invalid checksum: %w", err))
+		writeErrorResponse(ctx, w, apiErr, r.URL)
+		return
+	}
+	if checksum != nil && checksum.Type.Trailing() {
+		// Not officially supported in POST requests.
+		apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest)
+		apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("Trailing checksums not available for POST operations"))
+		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
-
-	// Close multipart file
-	defer fileBody.Close()

 	formValues.Set("Bucket", bucket)
 	if fileName != "" && strings.Contains(formValues.Get("Key"), "${filename}") {
@@ -968,20 +1088,38 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}

-	// Once signature is validated, check if the user has
-	// explicit permissions for the user.
-	if !globalIAMSys.IsAllowed(iampolicy.Args{
-		AccountName:     cred.AccessKey,
-		Groups:          cred.Groups,
-		Action:          iampolicy.PutObjectAction,
-		ConditionValues: getConditionValues(r, "", cred),
-		BucketName:      bucket,
-		ObjectName:      object,
-		IsOwner:         globalActiveCred.AccessKey == cred.AccessKey,
-		Claims:          cred.Claims,
-	}) {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-		return
+	if len(fanOutEntries) > 0 {
+		// Once signature is validated, check if the user has
+		// explicit permissions for the user.
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          iampolicy.PutObjectFanOutAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			BucketName:      bucket,
+			ObjectName:      object,
+			IsOwner:         globalActiveCred.AccessKey == cred.AccessKey,
+			Claims:          cred.Claims,
+		}) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
+	} else {
+		// Once signature is validated, check if the user has
+		// explicit permissions for the user.
+		if !globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     cred.AccessKey,
+			Groups:          cred.Groups,
+			Action:          iampolicy.PutObjectAction,
+			ConditionValues: getConditionValues(r, "", cred),
+			BucketName:      bucket,
+			ObjectName:      object,
+			IsOwner:         globalActiveCred.AccessKey == cred.AccessKey,
+			Claims:          cred.Claims,
+		}) {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+			return
+		}
 	}

 	policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy"))
@@ -990,6 +1128,19 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}

+	hashReader, err := hash.NewReader(reader, fileSize, "", "", fileSize)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		return
+	}
+	if checksum != nil && checksum.Valid() {
+		if err = hashReader.AddChecksumNoTrailer(formValues, false); err != nil {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+	}
+
 	// Handle policy if it is set.
 	if len(policyBytes) > 0 {
 		postPolicyForm, err := parsePostPolicyForm(bytes.NewReader(policyBytes))
@@ -1010,15 +1161,8 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		// should not exceed the maximum single Put size (5 GiB)
 		lengthRange := postPolicyForm.Conditions.ContentLengthRange
 		if lengthRange.Valid {
-			if fileSize < lengthRange.Min {
-				writeErrorResponse(ctx, w, toAPIError(ctx, errDataTooSmall), r.URL)
-				return
-			}
-
-			if fileSize > lengthRange.Max || isMaxObjectSize(fileSize) {
-				writeErrorResponse(ctx, w, toAPIError(ctx, errDataTooLarge), r.URL)
-				return
-			}
+			hashReader.SetExpectedMin(lengthRange.Min)
+			hashReader.SetExpectedMax(lengthRange.Max)
 		}
 	}

@@ -1030,19 +1174,13 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}

-	hashReader, err := hash.NewReader(fileBody, fileSize, "", "", fileSize)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
 	rawReader := hashReader
 	pReader := NewPutObjReader(rawReader)
 	var objectEncryptionKey crypto.ObjectKey

 	// Check if bucket encryption is enabled
 	sseConfig, _ := globalBucketSSEConfigSys.Get(bucket)
-	sseConfig.Apply(r.Header, sse.ApplyOptions{
+	sseConfig.Apply(formValues, sse.ApplyOptions{
 		AutoEncrypt: globalAutoEncryption,
 	})

@@ -1053,12 +1191,24 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}

+	fanOutOpts := fanOutOptions{Checksum: checksum}
+
 	if crypto.Requested(formValues) {
 		if crypto.SSECopy.IsRequested(r.Header) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
 			return
 		}

+		if crypto.SSEC.IsRequested(r.Header) && crypto.S3.IsRequested(r.Header) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL)
+			return
+		}
+
+		if crypto.SSEC.IsRequested(r.Header) && crypto.S3KMS.IsRequested(r.Header) {
+			writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL)
+			return
+		}
+
 		if crypto.SSEC.IsRequested(r.Header) && isReplicationEnabled(ctx, bucket) {
 			writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParametersSSEC), r.URL)
 			return
@@ -1085,23 +1235,147 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 				return
 			}
 		}
-		reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
+
+		if len(fanOutEntries) == 0 {
+			reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx)
+			if err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				return
+			}
+			// do not try to verify encrypted content/
+			hashReader, err = hash.NewReader(reader, -1, "", "", -1)
+			if err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				return
+			}
+			if checksum != nil && checksum.Valid() {
+				if err = hashReader.AddChecksumNoTrailer(formValues, true); err != nil {
+					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+					return
+				}
+			}
+			pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
+			if err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				return
+			}
+		} else {
+			fanOutOpts = fanOutOptions{
+				Key:      key,
+				Kind:     kind,
+				KeyID:    keyID,
+				KmsCtx:   kmsCtx,
+				Checksum: checksum,
+			}
+		}
+	}
+
+	if len(fanOutEntries) > 0 {
+		// Fan-out requires no copying, and must be carried from original source
+		// https://en.wikipedia.org/wiki/Copy_protection so the incoming stream
+		// is always going to be in-memory as we cannot re-read from what we
+		// wrote to disk - since that amounts to "copying" from a "copy"
+		// instead of "copying" from source, we need the stream to be seekable
+		// to ensure that we can make fan-out calls concurrently.
+		buf := bytebufferpool.Get()
+		defer bytebufferpool.Put(buf)
+
+		// Maximum allowed fan-out object size.
+		const maxFanOutSize = 16 << 20
+
+		n, err := io.Copy(buf, ioutil.HardLimitReader(pReader, maxFanOutSize))
 		if err != nil {
 			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 			return
 		}
-		info := ObjectInfo{Size: fileSize}
-		// do not try to verify encrypted content
-		hashReader, err = hash.NewReader(reader, info.EncryptedSize(), "", "", fileSize)
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
+
+		concurrentSize := 100
+		if runtime.GOMAXPROCS(0) < concurrentSize {
+			concurrentSize = runtime.GOMAXPROCS(0)
 		}
-		pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
-		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
+
+		fanOutResp := make([]minio.PutObjectFanOutResponse, 0, len(fanOutEntries))
+		eventArgsList := make([]eventArgs, 0, len(fanOutEntries))
+		for {
+			var objInfos []ObjectInfo
+			var errs []error
+
+			var done bool
+			if len(fanOutEntries) < concurrentSize {
+				objInfos, errs = fanOutPutObject(ctx, bucket, objectAPI, fanOutEntries, buf.Bytes()[:n], fanOutOpts)
+				done = true
+			} else {
+				objInfos, errs = fanOutPutObject(ctx, bucket, objectAPI, fanOutEntries[:concurrentSize], buf.Bytes()[:n], fanOutOpts)
+				fanOutEntries = fanOutEntries[concurrentSize:]
+			}
+
+			for i, objInfo := range objInfos {
+				if errs[i] != nil {
+					fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{
+						Key:   objInfo.Name,
+						Error: errs[i].Error(),
+					})
+
+					eventArgsList = append(eventArgsList, eventArgs{
+						EventName:    event.ObjectCreatedPost,
+						BucketName:   objInfo.Bucket,
+						Object:       ObjectInfo{Name: objInfo.Name},
+						ReqParams:    extractReqParams(r),
+						RespElements: extractRespElements(w),
+						UserAgent:    fmt.Sprintf("%s MinIO-Fan-Out (failed: %v)", r.UserAgent(), errs[i]),
+						Host:         handlers.GetSourceIP(r),
+					})
+					continue
+				}
+
+				fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{
+					Key:          objInfo.Name,
+					ETag:         getDecryptedETag(formValues, objInfo, false),
+					VersionID:    objInfo.VersionID,
+					LastModified: &objInfo.ModTime,
+				})
+
+				eventArgsList = append(eventArgsList, eventArgs{
+					EventName:    event.ObjectCreatedPost,
+					BucketName:   objInfo.Bucket,
+					Object:       objInfo,
+					ReqParams:    extractReqParams(r),
+					RespElements: extractRespElements(w),
+					UserAgent:    r.UserAgent() + " " + "MinIO-Fan-Out",
+					Host:         handlers.GetSourceIP(r),
+				})
+			}
+
+			if done {
+				break
+			}
 		}
+
+		enc := json.NewEncoder(w)
+		for i, fanOutResp := range fanOutResp {
+			if err = enc.Encode(&fanOutResp); err != nil {
+				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+				return
+			}
+
+			// Notify object created events.
+			sendEvent(eventArgsList[i])
+
+			if eventArgsList[i].Object.NumVersions > dataScannerExcessiveVersionsThreshold {
+				// Send events for excessive versions.
+				sendEvent(eventArgs{
+					EventName:    event.ObjectManyVersions,
+					BucketName:   eventArgsList[i].Object.Bucket,
+					Object:       eventArgsList[i].Object,
+					ReqParams:    extractReqParams(r),
+					RespElements: extractRespElements(w),
+					UserAgent:    r.UserAgent() + " " + "MinIO-Fan-Out",
+					Host:         handlers.GetSourceIP(r),
+				})
+			}
+		}
+
+		return
 	}

 	objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, opts)
@@ -1116,7 +1390,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 	w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`}

 	// Set the relevant version ID as part of the response header.
-	if objInfo.VersionID != "" {
+	if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID {
 		w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID}
 	}

@@ -1134,6 +1408,7 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		UserAgent:    r.UserAgent(),
 		Host:         handlers.GetSourceIP(r),
 	})
+
 	if objInfo.NumVersions > dataScannerExcessiveVersionsThreshold {
 		defer sendEvent(eventArgs{
 			EventName:    event.ObjectManyVersions,
@@ -1156,6 +1431,11 @@ func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *h
 		return
 	}

+	// Add checksum header.
+	if checksum != nil && checksum.Valid() {
+		hash.AddChecksumHeader(w, checksum.AsMap())
+	}
+
 	// Decide what http response to send depending on success_action_status parameter
 	switch successStatus {
 	case "201":
@@ -1324,14 +1604,6 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 	}

-	if globalDNSConfig != nil {
-		if err := globalDNSConfig.Delete(bucket); err != nil {
-			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually", err))
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
-		}
-	}
-
 	deleteBucket := objectAPI.DeleteBucket

 	// Attempt to delete bucket.
@@ -1345,15 +1617,18 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 				apiErr.Description = "The bucket you tried to delete is not empty. You must delete all versions in the bucket."
 			}
 		}
-		if globalDNSConfig != nil {
-			if err2 := globalDNSConfig.Put(bucket); err2 != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, please fix it manually", err2))
-			}
-		}
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}

+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(bucket); err != nil {
+			logger.LogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually, bucket on MinIO no longer exists", err))
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+			return
+		}
+	}
+
 	globalNotificationSys.DeleteBucketMetadata(ctx, bucket)
 	globalReplicationPool.deleteResyncMetadata(ctx, bucket)

@@ -1398,7 +1673,7 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri

 	config, err := objectlock.ParseObjectLockConfig(r.Body)
 	if err != nil {
-		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
+		apiErr := errorCodes.ToAPIErr(ErrInvalidArgument)
 		apiErr.Description = err.Error()
 		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
@@ -1412,7 +1687,11 @@ func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWri

 	// Deny object locking configuration settings on existing buckets without object lock enabled.
 	if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		if _, ok := err.(BucketObjectLockConfigNotFound); ok {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed), r.URL)
+		} else {
+			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		}
 		return
 	}

@@ -1614,5 +1893,5 @@ func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r
 	}))

 	// Write success response.
-	writeSuccessResponseHeadersOnly(w)
+	writeSuccessNoContent(w)
 }
@@ -657,13 +657,14 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 ) {
 	var err error

-	contentBytes := []byte("hello")
 	sha256sum := ""
 	var objectNames []string
 	for i := 0; i < 10; i++ {
+		contentBytes := []byte("hello")
 		objectName := "test-object-" + strconv.Itoa(i)
 		if i == 0 {
 			objectName += "/"
+			contentBytes = []byte{}
 		}
 		// uploading the object.
 		_, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
@@ -676,6 +677,7 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 		objectNames = append(objectNames, objectName)
 	}

+	contentBytes := []byte("hello")
 	for _, name := range []string{"private/object", "public/object"} {
 		// Uploading the object with retention enabled
 		_, err = obj.PutObject(GlobalContext, bucketName, name, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{})
@@ -745,8 +747,13 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa

 	deletedObjects := make([]DeletedObject, len(requestList[0].Objects))
 	for i := range requestList[0].Objects {
+		var vid string
+		if isDirObject(requestList[0].Objects[i].ObjectName) {
+			vid = nullVersionID
+		}
 		deletedObjects[i] = DeletedObject{
 			ObjectName: requestList[0].Objects[i].ObjectName,
+			VersionID:  vid,
 		}
 	}

@@ -756,9 +763,14 @@ func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketNa
 	successRequest1 := encodeResponse(requestList[1])

 	deletedObjects = make([]DeletedObject, len(requestList[1].Objects))
-	for i := range requestList[0].Objects {
+	for i := range requestList[1].Objects {
+		var vid string
+		if isDirObject(requestList[0].Objects[i].ObjectName) {
+			vid = nullVersionID
+		}
 		deletedObjects[i] = DeletedObject{
 			ObjectName: requestList[1].Objects[i].ObjectName,
+			VersionID:  vid,
 		}
 	}

@@ -0,0 +1,88 @@
+// Copyright (c) 2023 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import "github.com/minio/minio/internal/bucket/lifecycle"
+
+//go:generate stringer -type lcEventSrc -trimprefix lcEventSrc_ $GOFILE
+type lcEventSrc uint8
+
+//revive:disable:var-naming Underscores is used here to indicate where common prefix ends and the enumeration name begins
+const (
+	lcEventSrc_None lcEventSrc = iota
+	lcEventSrc_Scanner
+	lcEventSrc_Decom
+	lcEventSrc_Rebal
+	lcEventSrc_s3HeadObject
+	lcEventSrc_s3GetObject
+	lcEventSrc_s3ListObjects
+	lcEventSrc_s3PutObject
+	lcEventSrc_s3CopyObject
+	lcEventSrc_s3CompleteMultipartUpload
+)
+
+//revive:enable:var-naming
+type lcAuditEvent struct {
+	lifecycle.Event
+	source lcEventSrc
+}
+
+func (lae lcAuditEvent) Tags() map[string]interface{} {
+	event := lae.Event
+	src := lae.source
+	const (
+		ilmSrc                     = "ilm-src"
+		ilmAction                  = "ilm-action"
+		ilmDue                     = "ilm-due"
+		ilmRuleID                  = "ilm-rule-id"
+		ilmTier                    = "ilm-tier"
+		ilmNewerNoncurrentVersions = "ilm-newer-noncurrent-versions"
+		ilmNoncurrentDays          = "ilm-noncurrent-days"
+	)
+	tags := make(map[string]interface{}, 5)
+	if src > lcEventSrc_None {
+		tags[ilmSrc] = src.String()
+	}
+	tags[ilmAction] = event.Action.String()
+	tags[ilmRuleID] = event.RuleID
+
+	if !event.Due.IsZero() {
+		tags[ilmDue] = event.Due
+	}
+
+	// rule with Transition/NoncurrentVersionTransition in effect
+	if event.StorageClass != "" {
+		tags[ilmTier] = event.StorageClass
+	}
+
+	// rule with NewernoncurrentVersions in effect
+	if event.NewerNoncurrentVersions > 0 {
+		tags[ilmNewerNoncurrentVersions] = event.NewerNoncurrentVersions
+	}
+	if event.NoncurrentDays > 0 {
+		tags[ilmNoncurrentDays] = event.NoncurrentDays
+	}
+	return tags
+}
+
+func newLifecycleAuditEvent(src lcEventSrc, event lifecycle.Event) lcAuditEvent {
+	return lcAuditEvent{
+		Event:  event,
+		source: src,
+	}
+}
@@ -21,6 +21,7 @@ import (
 	"encoding/xml"
 	"io"
 	"net/http"
+	"strconv"

 	"github.com/minio/minio/internal/bucket/lifecycle"
 	xhttp "github.com/minio/minio/internal/http"
@@ -115,6 +116,16 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 	vars := mux.Vars(r)
 	bucket := vars["bucket"]

+	var withUpdatedAt bool
+	if updatedAtStr := r.Form.Get("withUpdatedAt"); updatedAtStr != "" {
+		var err error
+		withUpdatedAt, err = strconv.ParseBool(updatedAtStr)
+		if err != nil {
+			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidLifecycleQueryParameter), r.URL)
+			return
+		}
+	}
+
 	if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLifecycleAction, bucket, ""); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -126,7 +137,7 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}

-	config, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
+	config, updatedAt, err := globalBucketMetadataSys.GetLifecycleConfig(bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
@@ -138,6 +149,9 @@ func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}

+	if withUpdatedAt {
+		w.Header().Set(xhttp.MinIOLifecycleCfgUpdatedAt, updatedAt.Format(iso8601Format))
+	}
 	// Write lifecycle configuration to client.
 	writeSuccessResponseXML(w, configData)
 }
@@ -179,7 +179,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 			lifecycleResponse:  []byte(``),
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
-				Code:     "InvalidRequest",
+				Code:     "InvalidArgument",
 				Message:  "Filter must have exactly one of Prefix, Tag, or And specified",
 			},

@@ -196,7 +196,7 @@ func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName strin
 			lifecycleResponse:  []byte(``),
 			errorResponse: APIErrorResponse{
 				Resource: SlashSeparator + bucketName + SlashSeparator,
-				Code:     "InvalidRequest",
+				Code:     "InvalidArgument",
 				Message:  "Date must be provided in ISO 8601 format",
 			},

@@ -41,8 +41,8 @@ import (
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/s3select"
-	"github.com/minio/minio/internal/workers"
 	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/workers"
 )

 const (
@@ -63,7 +63,8 @@ type LifecycleSys struct{}

 // Get - gets lifecycle config associated to a given bucket name.
 func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) {
-	return globalBucketMetadataSys.GetLifecycleConfig(bucketName)
+	lc, _, err = globalBucketMetadataSys.GetLifecycleConfig(bucketName)
+	return lc, err
 }

 // NewLifecycleSys - creates new lifecycle system.
@@ -96,9 +97,9 @@ func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string) {
 }

 type expiryTask struct {
-	objInfo        ObjectInfo
-	versionExpiry  bool
-	restoredObject bool
+	objInfo ObjectInfo
+	event   lifecycle.Event
+	src     lcEventSrc
 }

 type expiryState struct {
@@ -121,22 +122,22 @@ func (es *expiryState) close() {
 }

 // enqueueByDays enqueues object versions expired by days for expiry.
-func (es *expiryState) enqueueByDays(oi ObjectInfo, restoredObject bool, rmVersion bool) {
+func (es *expiryState) enqueueByDays(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) {
 	select {
 	case <-GlobalContext.Done():
 		es.close()
-	case es.byDaysCh <- expiryTask{objInfo: oi, versionExpiry: rmVersion, restoredObject: restoredObject}:
+	case es.byDaysCh <- expiryTask{objInfo: oi, event: event, src: src}:
 	default:
 	}
 }

 // enqueueByNewerNoncurrent enqueues object versions expired by
 // NewerNoncurrentVersions limit for expiry.
-func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete) {
+func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete, lcEvent lifecycle.Event) {
 	select {
 	case <-GlobalContext.Done():
 		es.close()
-	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions}:
+	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions, event: lcEvent}:
 	default:
 	}
 }
@@ -154,7 +155,9 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 	globalExpiryState = newExpiryState()

 	workerSize, _ := strconv.Atoi(env.Get("_MINIO_ILM_EXPIRY_WORKERS", strconv.Itoa((runtime.GOMAXPROCS(0)+1)/2)))
-
+	if workerSize == 0 {
+		workerSize = 4
+	}
 	ewk, err := workers.New(workerSize)
 	if err != nil {
 		logger.LogIf(ctx, err)
@@ -171,9 +174,9 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 			go func(t expiryTask) {
 				defer ewk.Give()
 				if t.objInfo.TransitionedObject.Status != "" {
-					applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.restoredObject)
+					applyExpiryOnTransitionedObject(ctx, objectAPI, t.objInfo, t.event, t.src)
 				} else {
-					applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.versionExpiry)
+					applyExpiryOnNonTransitionedObjects(ctx, objectAPI, t.objInfo, t.event, t.src)
 				}
 			}(t)
 		}
@@ -184,7 +187,7 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 			nwk.Take()
 			go func(t newerNoncurrentTask) {
 				defer nwk.Give()
-				deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
+				deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions, t.event)
 			}(t)
 		}
 		nwk.Wait()
@@ -196,11 +199,13 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 type newerNoncurrentTask struct {
 	bucket   string
 	versions []ObjectToDelete
+	event    lifecycle.Event
 }

 type transitionTask struct {
-	tier    string
 	objInfo ObjectInfo
+	src     lcEventSrc
+	event   lifecycle.Event
 }

 type transitionState struct {
@@ -218,10 +223,10 @@ type transitionState struct {
 	lastDayStats map[string]*lastDayTierStats
 }

-func (t *transitionState) queueTransitionTask(oi ObjectInfo, sc string) {
+func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) {
 	select {
 	case <-t.ctx.Done():
-	case t.transitionCh <- transitionTask{objInfo: oi, tier: sc}:
+	case t.transitionCh <- transitionTask{objInfo: oi, event: event, src: src}:
 	default:
 	}
 }
@@ -274,9 +279,9 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 				return
 			}
 			atomic.AddInt32(&t.activeTasks, 1)
-			if err := transitionObject(t.ctx, objectAPI, task.objInfo, task.tier); err != nil {
-				logger.LogIf(t.ctx, fmt.Errorf("Transition failed for %s/%s version:%s with %w",
-					task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
+			if err := transitionObject(t.ctx, objectAPI, task.objInfo, newLifecycleAuditEvent(task.src, task.event)); err != nil {
+				logger.LogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w",
+					task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err))
 			} else {
 				ts := tierStats{
 					TotalSize:   uint64(task.objInfo.Size),
@@ -285,7 +290,7 @@ func (t *transitionState) worker(objectAPI ObjectLayer) {
 				if task.objInfo.IsLatest {
 					ts.NumObjects = 1
 				}
-				t.addLastDayStats(task.tier, ts)
+				t.addLastDayStats(task.event.StorageClass, ts)
 			}
 			atomic.AddInt32(&t.activeTasks, -1)
 		}
@@ -356,91 +361,78 @@ func validateTransitionTier(lc *lifecycle.Lifecycle) error {

 // enqueueTransitionImmediate enqueues obj for transition if eligible.
 // This is to be called after a successful upload of an object (version).
-func enqueueTransitionImmediate(obj ObjectInfo) {
+func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) {
 	if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil {
-		event := lc.Eval(obj.ToLifecycleOpts())
-		switch event.Action {
+		switch event := lc.Eval(obj.ToLifecycleOpts()); event.Action {
 		case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-			globalTransitionState.queueTransitionTask(obj, event.StorageClass)
+			globalTransitionState.queueTransitionTask(obj, event, src)
 		}
 	}
 }

-// expireAction represents different actions to be performed on expiry of a
-// restored/transitioned object
-type expireAction int
-
-const (
-	// ignore the zero value
-	_ expireAction = iota
-	// expireObj indicates expiry of 'regular' transitioned objects.
-	expireObj
-	// expireRestoredObj indicates expiry of restored objects.
-	expireRestoredObj
-)
-
 // expireTransitionedObject handles expiry of transitioned/restored objects
 // (versions) in one of the following situations:
 //
 // 1. when a restored (via PostRestoreObject API) object expires.
 // 2. when a transitioned object expires (based on an ILM rule).
-func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, action expireAction) error {
+func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcOpts lifecycle.ObjectOpts, lcEvent lifecycle.Event, src lcEventSrc) error {
 	traceFn := globalLifecycleSys.trace(*oi)
 	var opts ObjectOptions
 	opts.Versioned = globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name)
 	opts.VersionID = lcOpts.VersionID
 	opts.Expiration = ExpirationOptions{Expire: true}
-	switch action {
-	case expireObj:
-		// When an object is past expiry or when a transitioned object is being
-		// deleted, 'mark' the data in the remote tier for delete.
-		entry := jentry{
-			ObjName:   oi.TransitionedObject.Name,
-			VersionID: oi.TransitionedObject.VersionID,
-			TierName:  oi.TransitionedObject.Tier,
-		}
-		if err := globalTierJournal.AddEntry(entry); err != nil {
-			logger.LogIf(ctx, err)
-			return err
-		}
-		// Delete metadata on source, now that data in remote tier has been
-		// marked for deletion.
-		if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
-			logger.LogIf(ctx, err)
-			return err
-		}
-
-		// Send audit for the lifecycle delete operation
-		defer auditLogLifecycle(ctx, *oi, ILMExpiry, traceFn)
-
-		eventName := event.ObjectRemovedDelete
-		if lcOpts.DeleteMarker {
-			eventName = event.ObjectRemovedDeleteMarkerCreated
-		}
-		objInfo := ObjectInfo{
-			Name:         oi.Name,
-			VersionID:    lcOpts.VersionID,
-			DeleteMarker: lcOpts.DeleteMarker,
-		}
-		// Notify object deleted event.
-		sendEvent(eventArgs{
-			EventName:  eventName,
-			BucketName: oi.Bucket,
-			Object:     objInfo,
-			UserAgent:  "Internal: [ILM-Expiry]",
-			Host:       globalLocalNodeName,
-		})
-
-	case expireRestoredObj:
+	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
+	if lcEvent.Action.DeleteRestored() {
 		// delete locally restored copy of object or object version
 		// from the source, while leaving metadata behind. The data on
 		// transitioned tier lies untouched and still accessible
 		opts.Transition.ExpireRestored = true
 		_, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts)
+		if err == nil {
+			// TODO consider including expiry of restored object to events we
+			// notify.
+			auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
+		}
 		return err
-	default:
-		return fmt.Errorf("Unknown expire action %v", action)
 	}
+	// When an object is past expiry or when a transitioned object is being
+	// deleted, 'mark' the data in the remote tier for delete.
+	entry := jentry{
+		ObjName:   oi.TransitionedObject.Name,
+		VersionID: oi.TransitionedObject.VersionID,
+		TierName:  oi.TransitionedObject.Tier,
+	}
+	if err := globalTierJournal.AddEntry(entry); err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+	// Delete metadata on source, now that data in remote tier has been
+	// marked for deletion.
+	if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil {
+		logger.LogIf(ctx, err)
+		return err
+	}
+
+	// Send audit for the lifecycle delete operation
+	defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn)
+
+	eventName := event.ObjectRemovedDelete
+	if lcOpts.DeleteMarker {
+		eventName = event.ObjectRemovedDeleteMarkerCreated
+	}
+	objInfo := ObjectInfo{
+		Name:         oi.Name,
+		VersionID:    lcOpts.VersionID,
+		DeleteMarker: lcOpts.DeleteMarker,
+	}
+	// Notify object deleted event.
+	sendEvent(eventArgs{
+		EventName:  eventName,
+		BucketName: oi.Bucket,
+		Object:     objInfo,
+		UserAgent:  "Internal: [ILM-Expiry]",
+		Host:       globalLocalNodeName,
+	})

 	return nil
 }
@@ -460,17 +452,18 @@ func genTransitionObjName(bucket string) (string, error) {
 // storage specified by the transition ARN, the metadata is left behind on source cluster and original content
 // is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved
 // to the transition tier without decrypting or re-encrypting.
-func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, tier string) error {
+func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, lae lcAuditEvent) error {
 	opts := ObjectOptions{
 		Transition: TransitionOptions{
 			Status: lifecycle.TransitionPending,
-			Tier:   tier,
+			Tier:   lae.StorageClass,
 			ETag:   oi.ETag,
 		},
-		VersionID:        oi.VersionID,
-		Versioned:        globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
-		VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
-		MTime:            oi.ModTime,
+		LifecycleAuditEvent: lae,
+		VersionID:           oi.VersionID,
+		Versioned:           globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name),
+		VersionSuspended:    globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name),
+		MTime:               oi.ModTime,
 	}
 	return objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts)
 }
@@ -190,7 +190,6 @@ func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.Re
 	}

 	// Validate the query params before beginning to serve the request.
-	// fetch-owner is not validated since it is a boolean
 	if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 		return
@@ -36,8 +36,8 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/sync/errgroup"
 )

 // BucketMetadataSys captures all bucket metadata for a given cluster.
@@ -121,6 +121,7 @@ func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string,
 		meta.NotificationConfigXML = configData
 	case bucketLifecycleConfig:
 		meta.LifecycleConfigXML = configData
+		meta.LifecycleConfigUpdatedAt = updatedAt
 	case bucketSSEConfig:
 		meta.EncryptionConfigXML = configData
 		meta.EncryptionConfigUpdatedAt = updatedAt
@@ -246,18 +247,18 @@ func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Co

 // GetLifecycleConfig returns configured lifecycle config
 // The returned object may not be modified.
-func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, error) {
+func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, time.Time, error) {
 	meta, _, err := sys.GetConfig(GlobalContext, bucket)
 	if err != nil {
 		if errors.Is(err, errConfigNotFound) {
-			return nil, BucketLifecycleNotFound{Bucket: bucket}
+			return nil, time.Time{}, BucketLifecycleNotFound{Bucket: bucket}
 		}
-		return nil, err
+		return nil, time.Time{}, err
 	}
 	if meta.lifecycleConfig == nil {
-		return nil, BucketLifecycleNotFound{Bucket: bucket}
+		return nil, time.Time{}, BucketLifecycleNotFound{Bucket: bucket}
 	}
-	return meta.lifecycleConfig, nil
+	return meta.lifecycleConfig, meta.LifecycleConfigUpdatedAt, nil
 }

 // GetNotificationConfig returns configured notification config
@@ -419,8 +420,8 @@ func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []BucketInfo, ob

 	sys.objAPI = objAPI

-	// Load bucket metadata sys in background
-	go sys.init(ctx, buckets)
+	// Load bucket metadata sys.
+	sys.init(ctx, buckets)
 	return nil
 }

@@ -88,6 +88,7 @@ type BucketMetadata struct {
 	QuotaConfigUpdatedAt        time.Time
 	ReplicationConfigUpdatedAt  time.Time
 	VersioningConfigUpdatedAt   time.Time
+	LifecycleConfigUpdatedAt    time.Time

 	// Unexported fields. Must be updated atomically.
 	policyConfig           *policy.Policy
@@ -119,6 +120,16 @@ func newBucketMetadata(name string) BucketMetadata {
 	}
 }

+// Versioning returns true if versioning is enabled
+func (b BucketMetadata) Versioning() bool {
+	return b.LockEnabled || (b.versioningConfig != nil && b.versioningConfig.Enabled()) || (b.objectLockConfig != nil && b.objectLockConfig.Enabled())
+}
+
+// ObjectLocking returns true if object locking is enabled
+func (b BucketMetadata) ObjectLocking() bool {
+	return b.LockEnabled || (b.objectLockConfig != nil && b.objectLockConfig.Enabled())
+}
+
 // SetCreatedAt preserves the CreatedAt time for bucket across sites in site replication. It defaults to
 // creation time of bucket on this cluster in all other cases.
 func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) {
@@ -417,6 +428,10 @@ func (b *BucketMetadata) defaultTimestamps() {
 	if b.VersioningConfigUpdatedAt.IsZero() {
 		b.VersioningConfigUpdatedAt = b.Created
 	}
+
+	if b.LifecycleConfigUpdatedAt.IsZero() {
+		b.LifecycleConfigUpdatedAt = b.Created
+	}
 }

 // Save config to supplied ObjectLayer api.
@@ -150,6 +150,12 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {
 				err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
 				return
 			}
+		case "LifecycleConfigUpdatedAt":
+			z.LifecycleConfigUpdatedAt, err = dc.ReadTime()
+			if err != nil {
+				err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
+				return
+			}
 		default:
 			err = dc.Skip()
 			if err != nil {
@@ -163,9 +169,9 @@ func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) {

 // EncodeMsg implements msgp.Encodable
 func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
-	// map header, size 21
+	// map header, size 22
 	// write "Name"
-	err = en.Append(0xde, 0x0, 0x15, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	err = en.Append(0xde, 0x0, 0x16, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	if err != nil {
 		return
 	}
@@ -374,15 +380,25 @@ func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) {
 		err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
 		return
 	}
+	// write "LifecycleConfigUpdatedAt"
+	err = en.Append(0xb8, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	if err != nil {
+		return
+	}
+	err = en.WriteTime(z.LifecycleConfigUpdatedAt)
+	if err != nil {
+		err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
+		return
+	}
 	return
 }

 // MarshalMsg implements msgp.Marshaler
 func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	o = msgp.Require(b, z.Msgsize())
-	// map header, size 21
+	// map header, size 22
 	// string "Name"
-	o = append(o, 0xde, 0x0, 0x15, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
+	o = append(o, 0xde, 0x0, 0x16, 0xa4, 0x4e, 0x61, 0x6d, 0x65)
 	o = msgp.AppendString(o, z.Name)
 	// string "Created"
 	o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64)
@@ -444,6 +460,9 @@ func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) {
 	// string "VersioningConfigUpdatedAt"
 	o = append(o, 0xb9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
 	o = msgp.AppendTime(o, z.VersioningConfigUpdatedAt)
+	// string "LifecycleConfigUpdatedAt"
+	o = append(o, 0xb8, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74)
+	o = msgp.AppendTime(o, z.LifecycleConfigUpdatedAt)
 	return
 }

@@ -591,6 +610,12 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {
 				err = msgp.WrapError(err, "VersioningConfigUpdatedAt")
 				return
 			}
+		case "LifecycleConfigUpdatedAt":
+			z.LifecycleConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts)
+			if err != nil {
+				err = msgp.WrapError(err, "LifecycleConfigUpdatedAt")
+				return
+			}
 		default:
 			bts, err = msgp.Skip(bts)
 			if err != nil {
@@ -605,6 +630,6 @@ func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) {

 // Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message
 func (z *BucketMetadata) Msgsize() (s int) {
-	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize
+	s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize
 	return
 }
@@ -312,7 +312,7 @@ func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, ob
 			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
 		rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header)
-		if err != nil {
+		if err != nil && !(replica && rMode == "" && rDate.IsZero()) {
 			return mode, retainDate, legalHold, toAPIErrorCode(ctx, err)
 		}
 		if retentionPermErr != ErrNone {
@@ -87,7 +87,7 @@ func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *ht
 	bucketPolicy, err := policy.ParseConfig(bytes.NewReader(bucketPolicyBytes), bucket)
 	if err != nil {
 		writeErrorResponse(ctx, w, APIError{
-			Code:           "ErrMalformedXML",
+			Code:           "MalformedPolicy",
 			HTTPStatusCode: http.StatusBadRequest,
 			Description:    err.Error(),
 		}, r.URL)
@@ -47,9 +47,13 @@ func NewBucketQuotaSys() *BucketQuotaSys {
 // Init initialize bucket quota.
 func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 	sys.bucketStorageCache.Once.Do(func() {
-		sys.bucketStorageCache.TTL = 1 * time.Second
+		// Set this to 10 secs since its enough, as scanner
+		// does not update the bucket usage values frequently.
+		sys.bucketStorageCache.TTL = 10 * time.Second
+		// Rely on older value if usage loading fails from disk.
+		sys.bucketStorageCache.Relax = true
 		sys.bucketStorageCache.Update = func() (interface{}, error) {
-			ctx, done := context.WithTimeout(context.Background(), 5*time.Second)
+			ctx, done := context.WithTimeout(context.Background(), 1*time.Second)
 			defer done()

 			return loadDataUsageFromBackend(ctx, objAPI)
@@ -60,16 +64,20 @@ func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) {
 // GetBucketUsageInfo return bucket usage info for a given bucket
 func (sys *BucketQuotaSys) GetBucketUsageInfo(bucket string) (BucketUsageInfo, error) {
 	v, err := sys.bucketStorageCache.Get()
-	if err != nil {
-		return BucketUsageInfo{}, err
+	if err != nil && v != nil {
+		logger.LogIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err))
 	}
+	if v == nil {
+		logger.LogIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"))
+	}
+
+	var bui BucketUsageInfo

 	dui, ok := v.(DataUsageInfo)
-	if !ok {
-		return BucketUsageInfo{}, fmt.Errorf("internal error: Unexpected DUI data type: %T", v)
+	if ok {
+		bui = dui.BucketsUsage[bucket]
 	}

-	bui := dui.BucketsUsage[bucket]
 	return bui, nil
 }

@@ -82,7 +90,7 @@ func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota,
 	if !quotaCfg.IsValid() {
 		if quotaCfg.Type == "fifo" {
 			logger.LogIf(GlobalContext, errors.New("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore. Please clear your quota configs using 'mc admin bucket quota alias/bucket --clear' and use 'mc ilm add' for expiration of objects"))
-			return quotaCfg, nil
+			return quotaCfg, fmt.Errorf("invalid quota type 'fifo'")
 		}
 		return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg)
 	}
@@ -100,6 +108,10 @@ func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string,
 	}

 	if q != nil && q.Type == madmin.HardQuota && q.Quota > 0 {
+		if uint64(size) >= q.Quota { // check if file size already exceeds the quota
+			return BucketQuotaExceeded{Bucket: bucket}
+		}
+
 		bui, err := sys.GetBucketUsageInfo(bucket)
 		if err != nil {
 			return err
@@ -26,6 +26,7 @@ import (
 	"math"
 	"math/rand"
 	"net/http"
+	"net/url"
 	"path"
 	"reflect"
 	"strings"
@@ -48,6 +49,7 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/pkg/workers"
 	"github.com/zeebo/xxh3"
 )

@@ -138,6 +140,13 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 		// validate replication ARN against target endpoint
 		c := globalBucketTargetSys.GetRemoteTargetClient(ctx, arnStr)
 		if c != nil {
+			if err := checkRemoteEndpoint(ctx, c.EndpointURL()); err != nil {
+				switch err.(type) {
+				case BucketRemoteIdenticalToSource:
+					return true, errorCodes.ToAPIErrWithErr(ErrBucketRemoteIdenticalToSource, fmt.Errorf("remote target endpoint %s is self referential", c.EndpointURL().String()))
+				default:
+				}
+			}
 			if c.EndpointURL().String() == clnt.EndpointURL().String() {
 				selfTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort)
 				if !sameTarget {
@@ -154,6 +163,45 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 	return sameTarget, toAPIError(ctx, nil)
 }

+// performs a http request to remote endpoint to check if deployment id of remote endpoint is same as
+// local cluster deployment id. This is to prevent replication to self, especially in case of a loadbalancer
+// in front of MinIO.
+func checkRemoteEndpoint(ctx context.Context, epURL *url.URL) error {
+	reqURL := &url.URL{
+		Scheme: epURL.Scheme,
+		Host:   epURL.Host,
+		Path:   healthCheckReadinessPath,
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL.String(), nil)
+	if err != nil {
+		return err
+	}
+
+	client := &http.Client{
+		Transport: NewHTTPTransport(),
+		Timeout:   10 * time.Second,
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+	if err == nil {
+		// Drain the connection.
+		xhttp.DrainBody(resp.Body)
+	}
+	if resp != nil {
+		amzid := resp.Header.Get(xhttp.AmzRequestHostID)
+		if _, ok := globalNodeNamesHex[amzid]; ok {
+			return BucketRemoteIdenticalToSource{
+				Endpoint: epURL.String(),
+			}
+		}
+	}
+	return nil
+}
+
 type mustReplicateOptions struct {
 	meta               map[string]string
 	status             replication.StatusType
@@ -984,11 +1032,13 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		}(i, tgt)
 	}
 	wg.Wait()
+
+	replicationStatus = rinfos.ReplicationStatus() // used in defer function
 	// FIXME: add support for missing replication events
 	// - event.ObjectReplicationMissedThreshold
 	// - event.ObjectReplicationReplicatedAfterThreshold
 	eventName := event.ObjectReplicationComplete
-	if rinfos.ReplicationStatus() == replication.Failed {
+	if replicationStatus == replication.Failed {
 		eventName = event.ObjectReplicationFailed
 	}
 	newReplStatusInternal := rinfos.ReplicationStatusInternal()
@@ -1085,7 +1135,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)

-	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
+	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
 		VersionID:        objInfo.VersionID,
 		Versioned:        versioned,
 		VersionSuspended: versionSuspended,
@@ -1099,13 +1149,15 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
-			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
+			logger.LogIf(ctx, fmt.Errorf("unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
 		}
 		return
 	}
 	defer gr.Close()

 	objInfo = gr.ObjInfo
+	// make sure we have the latest metadata for metrics calculation
+	rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN)

 	size, err := objInfo.GetActualSize()
 	if err != nil {
@@ -1121,7 +1173,7 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 	}

 	if tgt.Bucket == "" {
-		logger.LogIf(ctx, fmt.Errorf("Unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
+		logger.LogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty", objInfo.Name, objInfo.VersionID))
 		sendEvent(eventArgs{
 			EventName:  event.ObjectReplicationNotTracked,
 			BucketName: bucket,
@@ -1180,14 +1232,14 @@ func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI Obj
 			r, objInfo, putOpts); err != nil {
 			if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 			}
 		}
 	} else {
 		if _, err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts); err != nil {
 			if minio.ToErrorResponse(err).Code != "PreconditionFailed" {
 				rinfo.ReplicationStatus = replication.Failed
-				logger.LogIf(ctx, fmt.Errorf("Unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
+				logger.LogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): %s", bucket, objInfo.Name, objInfo.VersionID, err))
 			}
 		}
 	}
@@ -1233,7 +1285,7 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 	versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object)
 	versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object)

-	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, readLock, ObjectOptions{
+	gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{
 		VersionID:        objInfo.VersionID,
 		Versioned:        versioned,
 		VersionSuspended: versionSuspended,
@@ -1247,13 +1299,15 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 				UserAgent:  "Internal: [Replication]",
 				Host:       globalLocalNodeName,
 			})
-			logger.LogIf(ctx, fmt.Errorf("Unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
+			logger.LogIf(ctx, fmt.Errorf("unable to update replicate metadata for %s/%s(%s): %w", bucket, object, objInfo.VersionID, err))
 		}
 		return
 	}
 	defer gr.Close()

 	objInfo = gr.ObjInfo
+	// make sure we have the latest metadata for metrics calculation
+	rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN)

 	// use latest ObjectInfo to check if previous replication attempt succeeded
 	if objInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) {
@@ -1330,6 +1384,25 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object
 			return
 		}
 	}
+	// if target returns error other than NoSuchKey, defer replication attempt
+	if cerr != nil {
+		errResp := minio.ToErrorResponse(cerr)
+		switch errResp.Code {
+		case "NoSuchKey", "NoSuchVersion", "SlowDownRead":
+			rAction = replicateAll
+		default:
+			logger.LogIf(ctx, fmt.Errorf("unable to replicate %s/%s (%s). Target (%s) returned %s error on HEAD",
+				bucket, object, objInfo.VersionID, tgt.EndpointURL(), cerr))
+			sendEvent(eventArgs{
+				EventName:  event.ObjectReplicationNotTracked,
+				BucketName: bucket,
+				Object:     objInfo,
+				UserAgent:  "Internal: [Replication]",
+				Host:       globalLocalNodeName,
+			})
+			return
+		}
+	}
 	rinfo.ReplicationStatus = replication.Completed
 	rinfo.Size = size
 	rinfo.ReplicationAction = rAction
@@ -1409,6 +1482,8 @@ func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI Object

 func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, object string, r io.Reader, objInfo ObjectInfo, opts minio.PutObjectOptions) (err error) {
 	var uploadedParts []minio.CompletePart
+	// new multipart must not set mtime as it may lead to erroneous cleanups at various intervals.
+	opts.Internal.SourceMTime = time.Time{} // this value is saved properly in CompleteMultipartUpload()
 	uploadID, err := c.NewMultipartUpload(context.Background(), bucket, object, opts)
 	if err != nil {
 		return err
@@ -1438,7 +1513,7 @@ func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, ob
 	)

 	for _, partInfo := range objInfo.Parts {
-		hr, err = hash.NewReader(r, partInfo.ActualSize, "", "", partInfo.ActualSize)
+		hr, err = hash.NewReader(io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize)
 		if err != nil {
 			return err
 		}
@@ -2038,6 +2113,7 @@ func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRa
 			StorageClass:              objInfo.StorageClass,
 			ReplicationStatusInternal: objInfo.ReplicationStatus,
 			UserTags:                  tags.String(),
+			ReplicationStatus:         replication.StatusType(objInfo.ReplicationStatus),
 		}
 		oi.UserDefined = make(map[string]string, len(objInfo.Metadata))
 		for k, v := range objInfo.Metadata {
@@ -2252,7 +2328,10 @@ func (s *replicationResyncer) PersistToDisk(ctx context.Context, objectAPI Objec
 	}
 }

-const resyncWorkerCnt = 50 // limit of number of bucket resyncs is progress at any given time
+const (
+	resyncWorkerCnt        = 10 // limit of number of bucket resyncs is progress at any given time
+	resyncParallelRoutines = 10 // number of parallel resync ops per bucket
+)

 func newresyncer() *replicationResyncer {
 	rs := replicationResyncer{
@@ -2267,6 +2346,36 @@ func newresyncer() *replicationResyncer {
 	return &rs
 }

+// mark status of replication resync on remote target for the bucket
+func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts) {
+	s.Lock()
+	defer s.Unlock()
+
+	m := s.statusMap[opts.bucket]
+	st := m.TargetsMap[opts.arn]
+	st.LastUpdate = UTCNow()
+	st.ResyncStatus = status
+	m.TargetsMap[opts.arn] = st
+	m.LastUpdate = UTCNow()
+	s.statusMap[opts.bucket] = m
+}
+
+// update replication resync stats for bucket's remote target
+func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts resyncOpts) {
+	s.Lock()
+	defer s.Unlock()
+	m := s.statusMap[opts.bucket]
+	st := m.TargetsMap[opts.arn]
+	st.Object = ts.Object
+	st.ReplicatedCount += ts.ReplicatedCount
+	st.FailedCount += ts.FailedCount
+	st.ReplicatedSize += ts.ReplicatedSize
+	st.FailedSize += ts.FailedSize
+	m.TargetsMap[opts.arn] = st
+	m.LastUpdate = UTCNow()
+	s.statusMap[opts.bucket] = m
+}
+
 // resyncBucket resyncs all qualifying objects as per replication rules for the target
 // ARN
 func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI ObjectLayer, heal bool, opts resyncOpts) {
@@ -2278,15 +2387,7 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object

 	resyncStatus := ResyncFailed
 	defer func() {
-		s.Lock()
-		m := s.statusMap[opts.bucket]
-		st := m.TargetsMap[opts.arn]
-		st.LastUpdate = UTCNow()
-		st.ResyncStatus = resyncStatus
-		m.TargetsMap[opts.arn] = st
-		m.LastUpdate = UTCNow()
-		s.statusMap[opts.bucket] = m
-		s.Unlock()
+		s.markStatus(resyncStatus, opts)
 		globalSiteResyncMetrics.incBucket(opts, resyncStatus)
 		s.workerCh <- struct{}{}
 	}()
@@ -2322,15 +2423,9 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	}
 	// mark resync status as resync started
 	if !heal {
-		s.Lock()
-		m := s.statusMap[opts.bucket]
-		st := m.TargetsMap[opts.arn]
-		st.ResyncStatus = ResyncStarted
-		m.TargetsMap[opts.arn] = st
-		m.LastUpdate = UTCNow()
-		s.statusMap[opts.bucket] = m
-		s.Unlock()
+		s.markStatus(ResyncStarted, opts)
 	}
+
 	// Walk through all object versions - Walk() is always in ascending order needed to ensure
 	// delete marker replicated to target after object version is first created.
 	if err := objectAPI.Walk(ctx, opts.bucket, "", objInfoCh, ObjectOptions{}); err != nil {
@@ -2346,80 +2441,85 @@ func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI Object
 	if st.ResyncStatus == ResyncStarted || st.ResyncStatus == ResyncFailed {
 		lastCheckpoint = st.Object
 	}
+	workers, err := workers.New(resyncParallelRoutines)
 	for obj := range objInfoCh {
 		select {
 		case <-s.resyncCancelCh:
 			resyncStatus = ResyncCanceled
 			return
+		case <-ctx.Done():
+			return
 		default:
 		}
 		if heal && lastCheckpoint != "" && lastCheckpoint != obj.Name {
 			continue
 		}
 		lastCheckpoint = ""
-
-		roi := getHealReplicateObjectInfo(obj, rcfg)
-		if !roi.ExistingObjResync.mustResync() {
-			continue
-		}
-		traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
-		if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
-			versionID := ""
-			dmVersionID := ""
-			if roi.VersionPurgeStatus.Empty() {
-				dmVersionID = roi.VersionID
-			} else {
-				versionID = roi.VersionID
+		obj := obj
+		workers.Take()
+		go func() {
+			defer workers.Give()
+			roi := getHealReplicateObjectInfo(obj, rcfg)
+			if !roi.ExistingObjResync.mustResync() {
+				return
 			}
+			traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID))
+			if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() {
+				versionID := ""
+				dmVersionID := ""
+				if roi.VersionPurgeStatus.Empty() {
+					dmVersionID = roi.VersionID
+				} else {
+					versionID = roi.VersionID
+				}

-			doi := DeletedObjectReplicationInfo{
-				DeletedObject: DeletedObject{
-					ObjectName:            roi.Name,
-					DeleteMarkerVersionID: dmVersionID,
-					VersionID:             versionID,
-					ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
-					DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
-					DeleteMarker:          roi.DeleteMarker,
+				doi := DeletedObjectReplicationInfo{
+					DeletedObject: DeletedObject{
+						ObjectName:            roi.Name,
+						DeleteMarkerVersionID: dmVersionID,
+						VersionID:             versionID,
+						ReplicationState:      roi.getReplicationState(roi.Dsc.String(), versionID, true),
+						DeleteMarkerMTime:     DeleteMarkerMTime{roi.ModTime},
+						DeleteMarker:          roi.DeleteMarker,
+					},
+					Bucket:    roi.Bucket,
+					OpType:    replication.ExistingObjectReplicationType,
+					EventType: ReplicateExistingDelete,
+				}
+				replicateDelete(ctx, doi, objectAPI)
+			} else {
+				roi.OpType = replication.ExistingObjectReplicationType
+				roi.EventType = ReplicateExisting
+				replicateObject(ctx, roi, objectAPI)
+			}
+			_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
+				VersionID: roi.VersionID,
+				Internal: minio.AdvancedGetOptions{
+					ReplicationProxyRequest: "false",
 				},
-				Bucket:    roi.Bucket,
-				OpType:    replication.ExistingObjectReplicationType,
-				EventType: ReplicateExistingDelete,
+			})
+			st := TargetReplicationResyncStatus{
+				Object: roi.Name,
+				Bucket: roi.Bucket,
 			}
-			replicateDelete(ctx, doi, objectAPI)
-		} else {
-			roi.OpType = replication.ExistingObjectReplicationType
-			roi.EventType = ReplicateExisting
-			replicateObject(ctx, roi, objectAPI)
-		}
-		_, err = tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{
-			VersionID: roi.VersionID,
-			Internal: minio.AdvancedGetOptions{
-				ReplicationProxyRequest: "false",
-			},
-		})
-		s.Lock()
-		m = s.statusMap[opts.bucket]
-		st = m.TargetsMap[opts.arn]
-		st.Object = roi.Name
-		success := true
-		if err != nil {
-			if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
-				st.ReplicatedCount++
+			success := true
+			if err != nil {
+				if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) {
+					st.ReplicatedCount++
+				} else {
+					st.FailedCount++
+					success = false
+				}
 			} else {
-				st.FailedCount++
-				success = false
+				st.ReplicatedCount++
+				st.ReplicatedSize += roi.Size
 			}
-		} else {
-			st.ReplicatedCount++
-			st.ReplicatedSize += roi.Size
-		}
-		m.TargetsMap[opts.arn] = st
-		m.LastUpdate = UTCNow()
-		s.statusMap[opts.bucket] = m
-		s.Unlock()
-		traceFn(err)
-		globalSiteResyncMetrics.updateMetric(roi, success, opts.resyncID)
+			s.incStats(st, opts)
+			traceFn(err)
+			globalSiteResyncMetrics.updateMetric(roi, success, opts.resyncID)
+		}()
 	}
+	workers.Wait()
 	resyncStatus = ResyncCompleted
 }

@@ -3051,7 +3151,7 @@ func (p *ReplicationPool) loadStatsFromDisk() (rs map[string]BucketReplicationSt

 	data, err := readConfig(p.ctx, p.objLayer, getReplicationStatsPath())
 	if err != nil {
-		if !errors.Is(err, errConfigNotFound) {
+		if errors.Is(err, errConfigNotFound) {
 			return rs, nil
 		}
 		return rs, err
@@ -215,7 +215,7 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 	if err != nil {
 		switch minio.ToErrorResponse(err).Code {
 		case "NoSuchBucket":
-			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
+			return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket, Err: err}
 		case "AccessDenied":
 			return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, AccessKey: tgt.Credentials.AccessKey, Err: err}
 		}
@@ -59,7 +59,6 @@ import (
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/certs"
@@ -133,20 +132,11 @@ func init() {
 		}
 	}()

-	globalForwarder = handlers.NewForwarder(&handlers.Forwarder{
-		PassHost:     true,
-		RoundTripper: NewHTTPTransportWithTimeout(1 * time.Hour),
-		Logger: func(err error) {
-			if err != nil && !errors.Is(err, context.Canceled) {
-				logger.LogIf(GlobalContext, err)
-			}
-		},
-	})
-
 	console.SetColor("Debug", fcolor.New())

 	gob.Register(StorageErr(""))
 	gob.Register(madmin.TimeInfo{})
+	gob.Register(madmin.XFSErrorConfigs{})
 	gob.Register(map[string]interface{}{})

 	defaultAWSCredProvider = []credentials.Provider{
@@ -187,7 +177,7 @@ func minioConfigToConsoleFeatures() {
 		}
 	}
 	// pass the console subpath configuration
-	if value := env.Get(config.EnvMinIOBrowserRedirectURL, ""); value != "" {
+	if value := env.Get(config.EnvBrowserRedirectURL, ""); value != "" {
 		subPath := path.Clean(pathJoin(strings.TrimSpace(globalBrowserRedirectURL.Path), SlashSeparator))
 		if subPath != SlashSeparator {
 			os.Setenv("CONSOLE_SUBPATH", subPath)
@@ -208,6 +198,11 @@ func minioConfigToConsoleFeatures() {
 	if globalIAMSys.LDAPConfig.Enabled() {
 		os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn)
 	}
+	// Handle animation in welcome page
+	if value := env.Get(config.EnvBrowserLoginAnimation, "on"); value != "" {
+		os.Setenv("CONSOLE_ANIMATED_LOGIN", value)
+	}
+
 	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))

@@ -632,7 +627,7 @@ func handleCommonEnvVars() {
 		logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
 	}
 	if globalBrowserEnabled {
-		if redirectURL := env.Get(config.EnvMinIOBrowserRedirectURL, ""); redirectURL != "" {
+		if redirectURL := env.Get(config.EnvBrowserRedirectURL, ""); redirectURL != "" {
 			u, err := xnet.ParseHTTPURL(redirectURL)
 			if err != nil {
 				logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value in environment variable")
@@ -777,9 +772,12 @@ func handleCommonEnvVars() {
 			logger.Info(color.RedBold(msg))
 		}
 		globalActiveCred = cred
+		globalCredViaEnv = true
 	} else {
 		globalActiveCred = auth.DefaultCredentials
 	}
+
+	globalDisableFreezeOnBoot = env.Get("_MINIO_DISABLE_API_FREEZE_ON_BOOT", "") == "true" || serverDebugLog
 }

 // Initialize KMS global variable after valiadating and loading the configuration.
@@ -30,12 +30,7 @@ import (
 var errConfigNotFound = errors.New("config file not found")

 func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string, opts ObjectOptions) ([]byte, ObjectInfo, error) {
-	lockType := readLock
-	if opts.NoLock {
-		lockType = noLock // erasureObjects.GetObjectNInfo honors lockType argument but not opts.NoLock.
-	}
-
-	r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, lockType, opts)
+	r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, opts)
 	if err != nil {
 		if isErrObjectNotFound(err) {
 			return nil, ObjectInfo{}, errConfigNotFound
@@ -283,7 +283,7 @@ var (
 	globalServerConfigMu sync.RWMutex
 )

-func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) error {
+func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, objAPI ObjectLayer) error {
 	switch subSys {
 	case config.SiteSubSys:
 		if _, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]); err != nil {
@@ -393,7 +393,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 	}

 	if config.NotifySubSystems.Contains(subSys) {
-		if err := notify.TestSubSysNotificationTargets(GlobalContext, s, subSys, NewHTTPTransport()); err != nil {
+		if err := notify.TestSubSysNotificationTargets(ctx, s, subSys, NewHTTPTransport()); err != nil {
 			return err
 		}
 	}
@@ -407,7 +407,7 @@ func validateSubSysConfig(s config.Config, subSys string, objAPI ObjectLayer) er
 	return nil
 }

-func validateConfig(s config.Config, subSys string) error {
+func validateConfig(ctx context.Context, s config.Config, subSys string) error {
 	objAPI := newObjectLayerFn()

 	// We must have a global lock for this so nobody else modifies env while we do.
@@ -419,12 +419,12 @@ func validateConfig(s config.Config, subSys string) error {
 	// Enable env values to validate KMS.
 	defer env.SetEnvOn()
 	if subSys != "" {
-		return validateSubSysConfig(s, subSys, objAPI)
+		return validateSubSysConfig(ctx, s, subSys, objAPI)
 	}

 	// No sub-system passed. Validate all of them.
 	for _, ss := range config.SubSystems.ToSlice() {
-		if err := validateSubSysConfig(s, ss, objAPI); err != nil {
+		if err := validateSubSysConfig(ctx, s, ss, objAPI); err != nil {
 			return err
 		}
 	}
@@ -588,7 +588,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			}
 			loggerCfg.HTTP[n] = l
 		}
-		if errs := logger.UpdateSystemTargets(loggerCfg); len(errs) > 0 {
+		if errs := logger.UpdateSystemTargets(ctx, loggerCfg); len(errs) > 0 {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs))
 		}
 	case config.AuditWebhookSubSys:
@@ -606,7 +606,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 			loggerCfg.AuditWebhook[n] = l
 		}

-		if errs := logger.UpdateAuditWebhookTargets(loggerCfg); len(errs) > 0 {
+		if errs := logger.UpdateAuditWebhookTargets(ctx, loggerCfg); len(errs) > 0 {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs))
 		}
 	case config.AuditKafkaSubSys:
@@ -620,7 +620,7 @@ func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s conf
 				loggerCfg.AuditKafka[n] = l
 			}
 		}
-		if errs := logger.UpdateAuditKafkaTargets(loggerCfg); len(errs) > 0 {
+		if errs := logger.UpdateAuditKafkaTargets(ctx, loggerCfg); len(errs) > 0 {
 			logger.LogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %v", errs))
 		}
 	case config.StorageClassSubSys:
@@ -1,215 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-	"path"
-	"time"
-	"unicode/utf8"
-
-	"github.com/minio/madmin-go/v2"
-	"github.com/minio/minio/internal/config"
-	"github.com/minio/minio/internal/kms"
-	"github.com/minio/minio/internal/logger"
-	etcd "go.etcd.io/etcd/client/v3"
-)
-
-func handleEncryptedConfigBackend(objAPI ObjectLayer) error {
-	encrypted, err := checkBackendEncrypted(objAPI)
-	if err != nil {
-		return fmt.Errorf("Unable to encrypt config %w", err)
-	}
-	if !encrypted {
-		return nil
-	}
-	if err = migrateConfigPrefixToEncrypted(objAPI); err != nil {
-		return fmt.Errorf("Unable to migrate all config at .minio.sys/config/: %w", err)
-	}
-	return nil
-}
-
-const backendEncryptedFile = "backend-encrypted"
-
-var backendEncryptedMigrationComplete = []byte("encrypted")
-
-func checkBackendEtcdEncrypted(ctx context.Context, client *etcd.Client) (bool, error) {
-	bootstrapTrace("check if etcd backend is encrypted")
-	data, err := readKeyEtcd(ctx, client, backendEncryptedFile)
-	if err != nil && err != errConfigNotFound {
-		return false, err
-	}
-	return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil
-}
-
-func checkBackendEncrypted(objAPI ObjectLayer) (bool, error) {
-	bootstrapTrace("check if the config backend is encrypted")
-	data, err := readConfig(GlobalContext, objAPI, backendEncryptedFile)
-	if err != nil && err != errConfigNotFound {
-		return false, err
-	}
-	return err == nil && bytes.Equal(data, backendEncryptedMigrationComplete), nil
-}
-
-func migrateIAMConfigsEtcdToEncrypted(ctx context.Context, client *etcd.Client) error {
-	encrypted, err := checkBackendEtcdEncrypted(ctx, client)
-	if err != nil {
-		return err
-	}
-
-	// If backend doesn't have this file means we have already
-	// attempted then migration
-	if !encrypted {
-		return nil
-	}
-
-	bootstrapTrace("encrypt etcd config")
-
-	if GlobalKMS != nil {
-		stat, err := GlobalKMS.Stat(ctx)
-		if err != nil {
-			return err
-		}
-		logger.Info(fmt.Sprintf("Attempting to re-encrypt IAM users and policies on etcd with %q (%s)", stat.DefaultKey, stat.Name))
-	}
-
-	listCtx, cancel := context.WithTimeout(ctx, 1*time.Minute)
-	defer cancel()
-
-	r, err := client.Get(listCtx, minioConfigPrefix, etcd.WithPrefix(), etcd.WithKeysOnly())
-	if err != nil {
-		return err
-	}
-
-	for _, kv := range r.Kvs {
-		data, err := readKeyEtcd(ctx, client, string(kv.Key))
-		if err == errConfigNotFound { // Perhaps not present or someone deleted it.
-			continue
-		}
-		if err != nil {
-			return err
-		}
-
-		if !utf8.Valid(data) {
-			pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
-			if err != nil {
-				if GlobalKMS != nil {
-					pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-						minioMetaBucket: path.Join(minioMetaBucket, string(kv.Key)),
-					})
-					if err != nil {
-						pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-							minioMetaBucket: string(kv.Key),
-						})
-						if err != nil {
-							return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
-						}
-					}
-				} else {
-					return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
-				}
-			}
-			data = pdata
-		}
-
-		if GlobalKMS != nil {
-			data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
-				minioMetaBucket: path.Join(minioMetaBucket, string(kv.Key)),
-			})
-			if err != nil {
-				return err
-			}
-		}
-
-		if err = saveKeyEtcd(ctx, client, string(kv.Key), data); err != nil {
-			return err
-		}
-	}
-
-	if GlobalKMS != nil {
-		logger.Info("Migration of encrypted IAM config data completed. All data is now encrypted on etcd.")
-	}
-
-	return deleteKeyEtcd(ctx, client, backendEncryptedFile)
-}
-
-func migrateConfigPrefixToEncrypted(objAPI ObjectLayer) error {
-	bootstrapTrace("migrating config prefix to encrypted")
-	if GlobalKMS != nil {
-		stat, err := GlobalKMS.Stat(context.Background())
-		if err != nil {
-			return err
-		}
-		logger.Info(fmt.Sprintf("Attempting to re-encrypt config, IAM users and policies on MinIO with %q (%s)", stat.DefaultKey, stat.Name))
-	}
-
-	results := make(chan ObjectInfo)
-	if err := objAPI.Walk(GlobalContext, minioMetaBucket, minioConfigPrefix, results, ObjectOptions{}); err != nil {
-		return err
-	}
-
-	for obj := range results {
-		data, err := readConfig(GlobalContext, objAPI, obj.Name)
-		if err != nil {
-			return err
-		}
-
-		if !utf8.Valid(data) {
-			pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
-			if err != nil {
-				if GlobalKMS != nil {
-					pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-						minioMetaBucket: path.Join(minioMetaBucket, obj.Name),
-					})
-					if err != nil {
-						pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-							minioMetaBucket: obj.Name,
-						})
-						if err != nil {
-							return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
-						}
-					}
-				} else {
-					return fmt.Errorf("Decrypting IAM config failed %w, possibly credentials are incorrect", err)
-				}
-			}
-			data = pdata
-		}
-
-		if GlobalKMS != nil {
-			data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{
-				obj.Bucket: path.Join(obj.Bucket, obj.Name),
-			})
-			if err != nil {
-				return err
-			}
-		}
-
-		if err = saveConfig(GlobalContext, objAPI, obj.Name, data); err != nil {
-			return err
-		}
-	}
-
-	if GlobalKMS != nil {
-		logger.Info("Migration of encrypted config data completed. All config data is now encrypted.")
-	}
-
-	return deleteConfig(GlobalContext, objAPI, backendEncryptedFile)
-}
@@ -18,16 +18,15 @@
 package cmd

 import (
-	"bytes"
+	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
-	"unicode/utf8"

-	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/config/cache"
@@ -39,7 +38,6 @@ import (
 	"github.com/minio/minio/internal/config/storageclass"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/event/target"
-	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/target/http"
 	xnet "github.com/minio/pkg/net"
@@ -2471,284 +2469,97 @@ func migrateConfigToMinioSys(objAPI ObjectLayer) (err error) {
 	return saveServerConfig(GlobalContext, objAPI, config)
 }

-// Migrates '.minio.sys/config.json' to v33.
-func migrateMinioSysConfig(objAPI ObjectLayer) error {
-	bootstrapTrace("migrate .minio.sys/config/config.json to latest version")
+func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.Config, error) {
 	// Construct path to config.json for the given bucket.
 	configFile := path.Join(minioConfigPrefix, minioConfigFile)

-	// Check if the config version is latest, if not migrate.
-	ok, _, err := checkConfigVersion(objAPI, configFile, "33")
-	if err != nil {
-		return err
-	}
-	if ok {
-		return nil
+	configFiles := []string{
+		getConfigFile(),
+		getConfigFile() + ".deprecated",
+		configFile,
 	}

-	if err := migrateV27ToV28MinioSys(objAPI); err != nil {
-		return err
-	}
-	if err := migrateV28ToV29MinioSys(objAPI); err != nil {
-		return err
-	}
-	if err := migrateV29ToV30MinioSys(objAPI); err != nil {
-		return err
-	}
-	if err := migrateV30ToV31MinioSys(objAPI); err != nil {
-		return err
-	}
-	if err := migrateV31ToV32MinioSys(objAPI); err != nil {
-		return err
-	}
-	return migrateV32ToV33MinioSys(objAPI)
-}
+	newServerCfg := func() (config.Config, error) {
+		// Initialize server config.
+		srvCfg := newServerConfig()

-func checkConfigVersion(objAPI ObjectLayer, configFile string, version string) (bool, []byte, error) {
-	data, err := readConfig(GlobalContext, objAPI, configFile)
-	if err != nil {
-		return false, nil, err
+		return srvCfg, saveServerConfig(ctx, objAPI, srvCfg)
 	}

-	if !utf8.Valid(data) {
-		if GlobalKMS != nil {
-			data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-				minioMetaBucket: path.Join(minioMetaBucket, configFile),
-			})
-		}
+	var data []byte
+	var err error

-		if GlobalKMS == nil && err != nil {
-			data, err = madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data))
-			if err != nil {
-				if err == madmin.ErrMaliciousData || err == madmin.ErrUnexpectedHeader {
-					return false, nil, config.ErrInvalidConfigDecryptionKey(nil)
-				}
-				return false, nil, err
+	cfg := &serverConfigV33{}
+	for _, cfgFile := range configFiles {
+		if _, err = Load(cfgFile, cfg); err != nil {
+			if !osIsNotExist(err) && !osIsPermission(err) {
+				return nil, err
 			}
+			continue
+		}
+		data, _ = json.Marshal(cfg)
+		break
+	}
+	if osIsPermission(err) {
+		logger.Info("Older config found but is not readable %s, proceeding to read config from other places", err)
+	}
+	if osIsNotExist(err) || osIsPermission(err) || len(data) == 0 {
+		data, err = readConfig(GlobalContext, objAPI, configFile)
+		if err != nil {
+			// when config.json is not found, then we freshly initialize.
+			if errors.Is(err, errConfigNotFound) {
+				return newServerCfg()
+			}
+			return nil, err
+		}
+
+		data, err = decryptData(data, configFile)
+		if err != nil {
+			return nil, err
+		}
+
+		newCfg, err := readServerConfig(GlobalContext, objAPI, data)
+		if err == nil {
+			return newCfg, nil
+		}
+
+		// Read older `.minio.sys/config/config.json`, if not
+		// possible just fail.
+		if err = json.Unmarshal(data, cfg); err != nil {
+			// Unable to parse old JSON simply re-initialize a new one.
+			return newServerCfg()
 		}
 	}

-	if version == "kvs" {
-		// Check api config values are present.
-		var vcfg struct {
-			API struct {
-				E []struct {
-					Key   string `json:"key"`
-					Value string `json:"value"`
-				} `json:"_"`
-			} `json:"api"`
+	if !globalCredViaEnv && cfg.Credential.IsValid() {
+		// Preserve older credential if we do not have
+		// root credentials set via environment variable.
+		globalActiveCred = cfg.Credential
+	}
+
+	// Init compression config. For future migration, Compression config needs to be copied over from previous version.
+	switch cfg.Version {
+	case "29":
+		// V29 -> V30
+		cfg.Compression.Enabled = false
+		cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
+		cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
+	case "30":
+		// V30 -> V31
+		cfg.OpenID = openid.Config{}
+		cfg.Policy.OPA = opa.Args{
+			URL:       &xnet.URL{},
+			AuthToken: "",
 		}
-		if err = json.Unmarshal(data, &vcfg); err != nil {
-			return false, nil, nil
-		}
-		return len(vcfg.API.E) > 0, data, nil
-	}
-	var versionConfig struct {
-		Version string `json:"version"`
-	}
-
-	vcfg := &versionConfig
-	if err = json.Unmarshal(data, vcfg); err != nil {
-		return false, nil, err
-	}
-	return vcfg.Version == version, data, nil
-}
-
-func migrateV27ToV28MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-	ok, data, err := checkConfigVersion(objAPI, configFile, "27")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV28{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
-	cfg.Version = "28"
-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘27’ to ‘28’. %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "27", "28")
-	return nil
-}
-
-func migrateV28ToV29MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	ok, data, err := checkConfigVersion(objAPI, configFile, "28")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV29{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
-	cfg.Version = "29"
-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘28’ to ‘29’. %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "28", "29")
-	return nil
-}
-
-func migrateV29ToV30MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	ok, data, err := checkConfigVersion(objAPI, configFile, "29")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV30{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
-	cfg.Version = "30"
-	// Init compression config.For future migration, Compression config needs to be copied over from previous version.
-	cfg.Compression.Enabled = false
-	cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator)
-	cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator)
-
-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘29’ to ‘30’. %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "29", "30")
-	return nil
-}
-
-func migrateV30ToV31MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	ok, data, err := checkConfigVersion(objAPI, configFile, "30")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV31{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
-	cfg.Version = "31"
-	cfg.OpenID = openid.Config{}
-
-	cfg.Policy.OPA = opa.Args{
-		URL:       &xnet.URL{},
-		AuthToken: "",
-	}
-
-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘30’ to ‘31’. %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "30", "31")
-	return nil
-}
-
-func migrateV31ToV32MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	ok, data, err := checkConfigVersion(objAPI, configFile, "31")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV32{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
-	cfg.Version = "32"
-	cfg.Notify.NSQ = make(map[string]target.NSQArgs)
-	cfg.Notify.NSQ["1"] = target.NSQArgs{}
-
-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from ‘31’ to ‘32’. %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "31", "32")
-	return nil
-}
-
-func migrateV32ToV33MinioSys(objAPI ObjectLayer) error {
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	ok, data, err := checkConfigVersion(objAPI, configFile, "32")
-	if err == errConfigNotFound {
-		return nil
-	} else if err != nil {
-		return fmt.Errorf("Unable to load config file. %w", err)
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV33{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
+	case "31":
+		// V31 -> V32
+		cfg.Notify.NSQ = make(map[string]target.NSQArgs)
+		cfg.Notify.NSQ["1"] = target.NSQArgs{}
 	}

+	// Move to latest.
 	cfg.Version = "33"

-	if err = saveServerConfig(GlobalContext, objAPI, cfg); err != nil {
-		return fmt.Errorf("Failed to migrate config from '32' to '33' . %w", err)
-	}
-
-	logger.Info(configMigrateMSGTemplate, configFile, "32", "33")
-	return nil
-}
-
-func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
-	bootstrapTrace("migrate config to KV style")
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	// Check if the config version is latest, if not migrate.
-	ok, data, err := checkConfigVersion(objAPI, configFile, "33")
-	if err != nil {
-		return err
-	}
-	if !ok {
-		return nil
-	}
-
-	cfg := &serverConfigV33{}
-	if err = json.Unmarshal(data, cfg); err != nil {
-		return err
-	}
-
 	newCfg := newServerConfig()

 	config.SetRegion(newCfg, cfg.Region)
@@ -2797,11 +2608,5 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
 		notify.SetNotifyWebhook(newCfg, k, args)
 	}

-	if err = saveServerConfig(GlobalContext, objAPI, newCfg); err != nil {
-		return err
-	}
-
-	logger.Info("Configuration file %s migrated from version '%s' to new KV format successfully.",
-		configFile, "33")
-	return nil
+	return newCfg, nil
 }
@@ -196,11 +196,12 @@ func TestServerConfigMigrateV2toV33(t *testing.T) {
 		t.Fatal("Unexpected error: ", err)
 	}

-	if err := migrateMinioSysConfig(objLayer); err != nil {
+	srvCfg, err := readConfigWithoutMigrate(context.Background(), objLayer)
+	if err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}

-	if err := migrateMinioSysConfigToKV(objLayer); err != nil {
+	if err = saveServerConfig(GlobalContext, objLayer, srvCfg); err != nil {
 		t.Fatal("Unexpected error: ", err)
 	}

@@ -717,107 +717,6 @@ type serverConfigV28 struct {
 	Logger logger.Config `json:"logger"`
 }

-// serverConfigV29 is just like version '28'.
-type serverConfigV29 serverConfigV28
-
-// serverConfigV30 is just like version '29', stores additionally
-// extensions and mimetypes fields for compression.
-type serverConfigV30 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Worm       config.BoolFlag  `json:"worm"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notifierV3 `json:"notify"`
-
-	// Logger configuration
-	Logger logger.Config `json:"logger"`
-
-	// Compression configuration
-	Compression compress.Config `json:"compress"`
-}
-
-// serverConfigV31 is just like version '30', with OPA and OpenID configuration.
-type serverConfigV31 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Worm       config.BoolFlag  `json:"worm"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notifierV3 `json:"notify"`
-
-	// Logger configuration
-	Logger logger.Config `json:"logger"`
-
-	// Compression configuration
-	Compression compress.Config `json:"compress"`
-
-	// OpenID configuration
-	OpenID openid.Config `json:"openid"`
-
-	// External policy enforcements.
-	Policy struct {
-		// OPA configuration.
-		OPA opa.Args `json:"opa"`
-
-		// Add new external policy enforcements here.
-	} `json:"policy"`
-}
-
-// serverConfigV32 is just like version '31' with added nsq notifer.
-type serverConfigV32 struct {
-	Version string `json:"version"`
-
-	// S3 API configuration.
-	Credential auth.Credentials `json:"credential"`
-	Region     string           `json:"region"`
-	Worm       config.BoolFlag  `json:"worm"`
-
-	// Storage class configuration
-	StorageClass storageclass.Config `json:"storageclass"`
-
-	// Cache configuration
-	Cache cache.Config `json:"cache"`
-
-	// Notification queue configuration.
-	Notify notify.Config `json:"notify"`
-
-	// Logger configuration
-	Logger logger.Config `json:"logger"`
-
-	// Compression configuration
-	Compression compress.Config `json:"compress"`
-
-	// OpenID configuration
-	OpenID openid.Config `json:"openid"`
-
-	// External policy enforcements.
-	Policy struct {
-		// OPA configuration.
-		OPA opa.Args `json:"opa"`
-
-		// Add new external policy enforcements here.
-	} `json:"policy"`
-}
-
 // serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in all notification targets.
 type serverConfigV33 struct {
 	quick.Config `json:"-"` // ignore interfaces
@@ -25,7 +25,6 @@ import (
 	"path"
 	"sort"
 	"strings"
-	"unicode/utf8"

 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go/v2"
@@ -65,16 +64,16 @@ func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData b
 			if withData {
 				data, err := readConfig(ctx, objAPI, obj.Name)
 				if err != nil {
-					return nil, err
+					// ignore history file if not readable.
+					continue
 				}
-				if GlobalKMS != nil {
-					data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-						obj.Bucket: path.Join(obj.Bucket, obj.Name),
-					})
-					if err != nil {
-						return nil, err
-					}
+
+				data, err = decryptData(data, obj.Name)
+				if err != nil {
+					// ignore history file that cannot be loaded.
+					continue
 				}
+
 				cfgEntry.Data = string(data)
 			}
 			configHistory = append(configHistory, cfgEntry)
@@ -110,12 +109,7 @@ func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV str
 		return nil, err
 	}

-	if GlobalKMS != nil {
-		data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-			minioMetaBucket: path.Join(minioMetaBucket, historyFile),
-		})
-	}
-	return data, err
+	return decryptData(data, historyFile)
 }

 func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error {
@@ -167,14 +161,10 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer, data []byte) (con
 			return nil, err
 		}

-		if GlobalKMS != nil && !utf8.Valid(data) {
-			data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
-				minioMetaBucket: path.Join(minioMetaBucket, configFile),
-			})
-			if err != nil {
-				lookupConfigs(srvCfg, objAPI)
-				return nil, err
-			}
+		data, err = decryptData(data, configFile)
+		if err != nil {
+			lookupConfigs(srvCfg, objAPI)
+			return nil, err
 		}
 	}

@@ -205,47 +195,32 @@ func NewConfigSys() *ConfigSys {
 }

 // Initialize and load config from remote etcd or local config directory
-func initConfig(objAPI ObjectLayer) error {
+func initConfig(objAPI ObjectLayer) (err error) {
 	bootstrapTrace("load the configuration")
+	defer func() {
+		if err != nil {
+			bootstrapTrace(fmt.Sprintf("loading configuration failed: %v", err))
+		}
+	}()

 	if objAPI == nil {
 		return errServerNotInitialized
 	}

-	if isFile(getConfigFile()) {
-		if err := migrateConfig(); err != nil {
-			return err
-		}
-	}
-
-	// Check if the config version is latest (kvs), if not migrate.
-	ok, data, err := checkConfigVersion(objAPI, path.Join(minioConfigPrefix, minioConfigFile), "kvs")
-	if err != nil && !errors.Is(err, errConfigNotFound) {
+	srvCfg, err := readConfigWithoutMigrate(GlobalContext, objAPI)
+	if err != nil {
 		return err
 	}
-	if ok {
-		return loadConfig(objAPI, data)
-	}

-	// Migrates ${HOME}/.minio/config.json or config.json.deprecated
-	// to '<export_path>/.minio.sys/config/config.json'
-	// ignore if the file doesn't exist.
-	// If etcd is set then migrates /config/config.json
-	// to '<export_path>/.minio.sys/config/config.json'
-	if err := migrateConfigToMinioSys(objAPI); err != nil {
-		return fmt.Errorf("migrateConfigToMinioSys: %w", err)
-	}
+	bootstrapTrace("lookup the configuration")

-	// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
-	if err := migrateMinioSysConfig(objAPI); err != nil {
-		return fmt.Errorf("migrateMinioSysConfig: %w", err)
-	}
+	// Override any values from ENVs.
+	lookupConfigs(srvCfg, objAPI)

-	// Migrates backend '<export_path>/.minio.sys/config/config.json' to
-	// latest config format.
-	if err := migrateMinioSysConfigToKV(objAPI); err != nil {
-		return fmt.Errorf("migrateMinioSysConfigToKV: %w", err)
-	}
+	// hold the mutex lock before a new config is assigned.
+	globalServerConfigMu.Lock()
+	globalServerConfig = srvCfg
+	globalServerConfigMu.Unlock()

-	return loadConfig(objAPI, nil)
+	return nil
 }
@@ -58,7 +58,7 @@ func NewConsoleLogger(ctx context.Context) *HTTPConsoleLoggerSys {
 }

 // IsOnline always true in case of console logger
-func (sys *HTTPConsoleLoggerSys) IsOnline() bool {
+func (sys *HTTPConsoleLoggerSys) IsOnline(_ context.Context) bool {
 	return true
 }

@@ -87,7 +87,7 @@ func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool {
 func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan log.Info, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry log.Info) bool) error {
 	// Enable console logging for remote client.
 	if !sys.HasLogListeners() {
-		logger.AddSystemTarget(sys)
+		logger.AddSystemTarget(GlobalContext, sys)
 	}

 	cnt := 0
@@ -128,7 +128,7 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan log.Info, doneCh <-chan st
 }

 // Init if HTTPConsoleLoggerSys is valid, always returns nil right now
-func (sys *HTTPConsoleLoggerSys) Init() error {
+func (sys *HTTPConsoleLoggerSys) Init(_ context.Context) error {
 	return nil
 }

@@ -180,7 +180,7 @@ func (sys *HTTPConsoleLoggerSys) Type() types.TargetType {

 // Send log message 'e' to console and publish to console
 // log pubsub system
-func (sys *HTTPConsoleLoggerSys) Send(entry interface{}) error {
+func (sys *HTTPConsoleLoggerSys) Send(ctx context.Context, entry interface{}) error {
 	var lg log.Info
 	switch e := entry.(type) {
 	case log.Entry:
@@ -373,7 +373,7 @@ func (f *folderScanner) sendUpdate() {
 	}
 	if flat := f.updateCache.sizeRecursive(f.newCache.Info.Name); flat != nil {
 		select {
-		case f.updates <- *flat:
+		case f.updates <- flat.clone():
 		default:
 		}
 		f.lastUpdate = time.Now()
@@ -945,9 +945,9 @@ func (i *scannerItem) applyLifecycle(ctx context.Context, o ObjectLayer, oi Obje

 	switch lcEvt.Action {
 	case lifecycle.DeleteAction, lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-		return applyLifecycleAction(lcEvt.Action, oi, ""), 0
+		return applyLifecycleAction(lcEvt, lcEventSrc_Scanner, oi), 0
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		return applyLifecycleAction(lcEvt.Action, oi, lcEvt.StorageClass), size
+		return applyLifecycleAction(lcEvt, lcEventSrc_Scanner, oi), size
 	default:
 		// No action.
 		return false, size
@@ -983,7 +983,7 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 		InclFreeVersions: true,
 	})
 	if err == nil {
-		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete, traceFn)
+		auditLogLifecycle(ctx, oi, ILMFreeVersionDelete, nil, traceFn)
 	}
 	if ignoreNotFoundErr(err) != nil {
 		logger.LogIf(ctx, err)
@@ -1011,7 +1011,8 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 		return objectInfos, nil
 	}

-	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
+	event := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
+	lim := event.NewerNoncurrentVersions
 	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
 		for _, fi := range fivs {
 			objectInfos = append(objectInfos, fi.ToObjectInfo(i.bucket, i.objectPath(), versioned))
@@ -1040,7 +1041,7 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 		}

 		// NoncurrentDays not passed yet.
-		if time.Now().UTC().Before(lifecycle.ExpectedExpiryTime(obj.SuccessorModTime, days)) {
+		if time.Now().UTC().Before(lifecycle.ExpectedExpiryTime(obj.SuccessorModTime, event.NoncurrentDays)) {
 			// add this version back to remaining versions for
 			// subsequent lifecycle policy applications
 			fivs = append(fivs, fi)
@@ -1055,7 +1056,7 @@ func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ Ob
 		})
 	}

-	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel)
+	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel, event)
 	return objectInfos, nil
 }

@@ -1153,20 +1154,16 @@ func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr loc
 	return event
 }

-func applyTransitionRule(obj ObjectInfo, storageClass string) bool {
+func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) bool {
 	if obj.DeleteMarker {
 		return false
 	}
-	globalTransitionState.queueTransitionTask(obj, storageClass)
+	globalTransitionState.queueTransitionTask(obj, event, src)
 	return true
 }

-func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, restoredObject bool) bool {
-	action := expireObj
-	if restoredObject {
-		action = expireRestoredObj
-	}
-	if err := expireTransitionedObject(ctx, objLayer, &obj, obj.ToLifecycleOpts(), action); err != nil {
+func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) bool {
+	if err := expireTransitionedObject(ctx, objLayer, &obj, obj.ToLifecycleOpts(), lcEvent, src); err != nil {
 		if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 			return false
 		}
@@ -1177,13 +1174,13 @@ func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer,
 	return true
 }

-func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, applyOnVersion bool) bool {
+func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) bool {
 	traceFn := globalLifecycleSys.trace(obj)
 	opts := ObjectOptions{
 		Expiration: ExpirationOptions{Expire: true},
 	}

-	if applyOnVersion {
+	if lcEvent.Action.DeleteVersioned() {
 		opts.VersionID = obj.VersionID
 	}
 	if opts.VersionID == "" {
@@ -1201,8 +1198,9 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 		return false
 	}

+	tags := newLifecycleAuditEvent(src, lcEvent).Tags()
 	// Send audit for the lifecycle delete operation
-	auditLogLifecycle(ctx, obj, ILMExpiry, traceFn)
+	auditLogLifecycle(ctx, obj, ILMExpiry, tags, traceFn)

 	eventName := event.ObjectRemovedDelete
 	if obj.DeleteMarker {
@@ -1222,20 +1220,19 @@ func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLay
 }

 // Apply object, object version, restored object or restored object version action on the given object
-func applyExpiryRule(obj ObjectInfo, restoredObject, applyOnVersion bool) bool {
-	globalExpiryState.enqueueByDays(obj, restoredObject, applyOnVersion)
+func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) bool {
+	globalExpiryState.enqueueByDays(obj, event, src)
 	return true
 }

 // Perform actions (removal or transitioning of objects), return true the action is successfully performed
-func applyLifecycleAction(action lifecycle.Action, obj ObjectInfo, storageClass string) (success bool) {
-	switch action {
-	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction:
-		success = applyExpiryRule(obj, false, action == lifecycle.DeleteVersionAction)
-	case lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
-		success = applyExpiryRule(obj, true, action == lifecycle.DeleteRestoredVersionAction)
+func applyLifecycleAction(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) (success bool) {
+	switch action := event.Action; action {
+	case lifecycle.DeleteVersionAction, lifecycle.DeleteAction,
+		lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction:
+		success = applyExpiryRule(event, src, obj)
 	case lifecycle.TransitionAction, lifecycle.TransitionVersionAction:
-		success = applyTransitionRule(obj, storageClass)
+		success = applyTransitionRule(event, src, obj)
 	}
 	return
 }
@@ -1349,11 +1346,11 @@ func (d *dynamicSleeper) Timer(ctx context.Context) func() {
 			select {
 			case <-ctx.Done():
 				if !timer.Stop() {
-					if d.isScanner {
-						globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
-					}
 					<-timer.C
 				}
+				if d.isScanner {
+					globalScannerMetrics.incTime(scannerMetricYield, wantSleep)
+				}
 				return
 			case <-timer.C:
 				if d.isScanner {
@@ -1445,7 +1442,7 @@ const (
 	ILMTransition = " ilm:transition"
 )

-func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, traceFn func(event string)) {
+func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]interface{}, traceFn func(event string)) {
 	var apiName string
 	switch event {
 	case ILMExpiry:
@@ -1461,6 +1458,7 @@ func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, traceFn
 		Bucket:    oi.Bucket,
 		Object:    oi.Name,
 		VersionID: oi.VersionID,
+		Tags:      tags,
 	})
 	traceFn(event)
 }
@@ -88,6 +88,20 @@ func (ats *allTierStats) merge(other *allTierStats) {
 	}
 }

+func (ats *allTierStats) clone() *allTierStats {
+	if ats == nil {
+		return nil
+	}
+	dst := *ats
+	if dst.Tiers != nil {
+		dst.Tiers = make(map[string]tierStats, len(dst.Tiers))
+		for tier, st := range dst.Tiers {
+			dst.Tiers[tier] = st
+		}
+	}
+	return &dst
+}
+
 func (ats *allTierStats) adminStats(stats map[string]madmin.TierStats) map[string]madmin.TierStats {
 	if ats == nil {
 		return stats
@@ -168,6 +182,25 @@ type replicationAllStatsV1 struct {
 	ReplicaSize uint64 `msg:"ReplicaSize,omitempty"`
 }

+// clone creates a deep-copy clone.
+func (r *replicationAllStats) clone() *replicationAllStats {
+	if r == nil {
+		return nil
+	}
+
+	// Shallow copy
+	dst := *r
+
+	// Copy individual targets.
+	if dst.Targets != nil {
+		dst.Targets = make(map[string]replicationStats, len(dst.Targets))
+		for k, v := range r.Targets {
+			dst.Targets[k] = v
+		}
+	}
+	return &dst
+}
+
 //msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6
 //msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6

@@ -413,14 +446,11 @@ func (e dataUsageEntry) clone() dataUsageEntry {
 		e.Children = ch
 	}
 	if e.ReplicationStats != nil {
-		// Copy to new struct
-		r := *e.ReplicationStats
-		e.ReplicationStats = &r
+		// Clone ReplicationStats
+		e.ReplicationStats = e.ReplicationStats.clone()
 	}
 	if e.AllTierStats != nil {
-		ats := newAllTierStats()
-		ats.merge(e.AllTierStats)
-		e.AllTierStats = ats
+		e.AllTierStats = e.AllTierStats.clone()
 	}
 	return e
 }
@@ -873,7 +903,7 @@ func (d *dataUsageCache) merge(other dataUsageCache) {
 }

 type objectIO interface {
-	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (reader *GetObjectReader, err error)
+	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (reader *GetObjectReader, err error)
 	PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
 }

@@ -889,7 +919,7 @@ func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string)
 	// Caches are read+written without locks,
 	retries := 0
 	for retries < 5 {
-		r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, noLock, ObjectOptions{NoLock: true})
+		r, err := store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, ObjectOptions{NoLock: true})
 		if err != nil {
 			switch err.(type) {
 			case ObjectNotFound, BucketNotFound:
@@ -81,7 +81,7 @@ func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket

 				for id, usageInfo := range cache.flattenChildrens(*root) {
 					prefix := decodeDirObject(strings.TrimPrefix(id, bucket+slashSeparator))
-					// decodeDirObject to avoid any __XL_DIR__ objects
+					// decodeDirObject to avoid any __XLDIR__ objects
 					m[prefix] += uint64(usageInfo.Size)
 				}
 			}
@@ -33,8 +33,8 @@ import (
 	"github.com/minio/minio/internal/disk"
 	"github.com/minio/minio/internal/hash"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	xnet "github.com/minio/pkg/net"
+	"github.com/minio/pkg/sync/errgroup"
 	"github.com/minio/pkg/wildcard"
 )

@@ -80,7 +80,7 @@ type CacheStorageInfo struct {
 // CacheObjectLayer implements primitives for cache object API layer.
 type CacheObjectLayer interface {
 	// Object operations.
-	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
+	GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error)
 	GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error)
 	DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error)
@@ -117,7 +117,7 @@ type cacheObjects struct {
 	// Cache stats
 	cacheStats *CacheStats

-	InnerGetObjectNInfoFn          func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error)
+	InnerGetObjectNInfoFn          func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error)
 	InnerGetObjectInfoFn           func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	InnerDeleteObjectFn            func(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error)
 	InnerPutObjectFn               func(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error)
@@ -231,16 +231,16 @@ func (c *cacheObjects) incCacheStats(size int64) {
 	c.cacheStats.incBytesServed(size)
 }

-func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	if c.isCacheExclude(bucket, object) || c.skipCache() {
-		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, opts)
 	}
 	var cc *cacheControl
 	var cacheObjSize int64
 	// fetch diskCache if object is currently cached or nearest available cache drive
 	dcache, err := c.getCacheToLoc(ctx, bucket, object)
 	if err != nil {
-		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, opts)
 	}

 	cacheReader, numCacheHits, cacheErr := dcache.Get(ctx, bucket, object, rs, h, opts)
@@ -269,7 +269,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 		if cc != nil && cc.noStore {
 			cacheReader.Close()
 			c.cacheStats.incMiss()
-			bReader, err := c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+			bReader, err := c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, opts)
 			bReader.ObjInfo.CacheLookupStatus = CacheHit
 			bReader.ObjInfo.CacheStatus = CacheMiss
 			return bReader, err
@@ -304,7 +304,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			cacheReader.Close()
 		}
 		c.cacheStats.incMiss()
-		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, opts)
 	}
 	// skip cache for objects with locks
 	objRetention := objectlock.GetObjectRetentionMeta(objInfo.UserDefined)
@@ -314,7 +314,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			cacheReader.Close()
 		}
 		c.cacheStats.incMiss()
-		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+		return c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, opts)
 	}
 	if cacheErr == nil {
 		// if ETag matches for stale cache entry, serve from cache
@@ -332,7 +332,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 	// Reaching here implies cache miss
 	c.cacheStats.incMiss()

-	bkReader, bkErr := c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, lockType, opts)
+	bkReader, bkErr := c.InnerGetObjectNInfoFn(ctx, bucket, object, rs, h, opts)

 	if bkErr != nil {
 		return bkReader, bkErr
@@ -359,7 +359,7 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			// if range caching is disabled, download entire object.
 			rs = nil
 			// fill cache in the background for range GET requests
-			bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, rs, h, lockType, opts)
+			bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, rs, h, opts)
 			if bErr != nil {
 				return
 			}
@@ -713,7 +713,7 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 		if err == nil {
 			go func() {
 				// fill cache in the background
-				bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
+				bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, nil, http.Header{}, ObjectOptions{})
 				if bErr != nil {
 					return
 				}
@@ -857,8 +857,8 @@ func newServerCacheObjects(ctx context.Context, config cache.Config) (CacheObjec
 		InnerGetObjectInfoFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().GetObjectInfo(ctx, bucket, object, opts)
 		},
-		InnerGetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
-			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, lockType, opts)
+		InnerGetObjectNInfoFn: func(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
+			return newObjectLayerFn().GetObjectNInfo(ctx, bucket, object, rs, h, opts)
 		},
 		InnerDeleteObjectFn: func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
 			return newObjectLayerFn().DeleteObject(ctx, bucket, object, opts)
@@ -1143,7 +1143,7 @@ func (c *cacheObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject,
 			End:            startOffset + length,
 		}
 		// fill cache in the background
-		bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, srcBucket, srcObject, rs, http.Header{}, readLock, ObjectOptions{})
+		bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, srcBucket, srcObject, rs, http.Header{}, ObjectOptions{})
 		if bErr != nil {
 			return
 		}
@@ -1176,7 +1176,7 @@ func (c *cacheObjects) CompleteMultipartUpload(ctx context.Context, bucket, obje
 			_, err := dcache.CompleteMultipartUpload(bgContext(ctx), bucket, object, uploadID, uploadedParts, oi, opts)
 			if err != nil {
 				// fill cache in the background
-				bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, nil, http.Header{}, readLock, ObjectOptions{})
+				bReader, bErr := c.InnerGetObjectNInfoFn(GlobalContext, bucket, object, nil, http.Header{}, ObjectOptions{})
 				if bErr != nil {
 					return
 				}
@@ -818,8 +818,8 @@ func getDecryptedETag(headers http.Header, objInfo ObjectInfo, copySource bool)

 	// As per AWS S3 Spec, ETag for SSE-C encrypted objects need not be MD5Sum of the data.
 	// Since server side copy with same source and dest just replaces the ETag, we save
-	// encrypted content MD5Sum as ETag for both SSE-C and SSE-S3, we standardize the ETag
-	// encryption across SSE-C and SSE-S3, and only return last 32 bytes for SSE-C
+	// encrypted content MD5Sum as ETag for both SSE-C and SSE-KMS, we standardize the ETag
+	// encryption across SSE-C and SSE-KMS, and only return last 32 bytes for SSE-C
 	if (crypto.SSEC.IsEncrypted(objInfo.UserDefined) || crypto.S3KMS.IsEncrypted(objInfo.UserDefined)) && !copySource {
 		return objInfo.ETag[len(objInfo.ETag)-32:]
 	}
@@ -828,15 +828,15 @@ func getDecryptedETag(headers http.Header, objInfo ObjectInfo, copySource bool)
 	if err != nil {
 		return objInfo.ETag
 	}
-	return tryDecryptETag(objectEncryptionKey, objInfo.ETag, false)
+	return tryDecryptETag(objectEncryptionKey, objInfo.ETag, true)
 }

 // helper to decrypt Etag given object encryption key and encrypted ETag
-func tryDecryptETag(key []byte, encryptedETag string, ssec bool) string {
-	// ETag for SSE-C encrypted objects need not be content MD5Sum.While encrypted
+func tryDecryptETag(key []byte, encryptedETag string, sses3 bool) string {
+	// ETag for SSE-C or SSE-KMS encrypted objects need not be content MD5Sum.While encrypted
 	// md5sum is stored internally, return just the last 32 bytes of hex-encoded and
 	// encrypted md5sum string for SSE-C
-	if ssec {
+	if !sses3 {
 		return encryptedETag[len(encryptedETag)-32:]
 	}
 	var objectKey crypto.ObjectKey
@@ -1124,7 +1124,8 @@ func (o *ObjectInfo) metadataEncryptFn(headers http.Header) (objectMetaEncryptFn
 }

 // decryptChecksums will attempt to decode checksums and return it/them if set.
-func (o *ObjectInfo) decryptChecksums() map[string]string {
+// if part > 0, and we have the checksum for the part that will be returned.
+func (o *ObjectInfo) decryptChecksums(part int) map[string]string {
 	data := o.Checksum
 	if len(data) == 0 {
 		return nil
@@ -1137,5 +1138,5 @@ func (o *ObjectInfo) decryptChecksums() map[string]string {
 		}
 		data = decrypted
 	}
-	return hash.ReadCheckSums(data)
+	return hash.ReadCheckSums(data, part)
 }
@@ -355,7 +355,7 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		if err != nil {
 			return nil, -1, err
 		}
-		endpointList, newSetupType, err := CreateEndpoints(serverAddr, false, setArgs...)
+		endpointList, newSetupType, err := CreateEndpoints(serverAddr, setArgs...)
 		if err != nil {
 			return nil, -1, err
 		}
@@ -371,37 +371,36 @@ func createServerEndpoints(serverAddr string, args ...string) (
 		return endpointServerPools, setupType, nil
 	}

-	var foundPrevLocal bool
+	var poolArgs [][][]string
 	for _, arg := range args {
 		if !ellipses.HasEllipses(arg) && len(args) > 1 {
 			// TODO: support SNSD deployments to be decommissioned in future
 			return nil, -1, fmt.Errorf("all args must have ellipses for pool expansion (%w) args: %s", errInvalidArgument, args)
 		}
+
 		setArgs, err := GetAllSets(arg)
 		if err != nil {
 			return nil, -1, err
 		}

-		endpointList, gotSetupType, err := CreateEndpoints(serverAddr, foundPrevLocal, setArgs...)
-		if err != nil {
-			return nil, -1, err
-		}
+		poolArgs = append(poolArgs, setArgs)
+	}
+
+	poolEndpoints, setupType, err := CreatePoolEndpoints(serverAddr, poolArgs...)
+	if err != nil {
+		return nil, -1, err
+	}
+
+	for i, endpointList := range poolEndpoints {
 		if err = endpointServerPools.Add(PoolEndpoints{
-			SetCount:     len(setArgs),
-			DrivesPerSet: len(setArgs[0]),
+			SetCount:     len(poolArgs[i]),
+			DrivesPerSet: len(poolArgs[i][0]),
 			Endpoints:    endpointList,
-			CmdLine:      arg,
 			Platform:     fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH),
+			CmdLine:      args[i],
 		}); err != nil {
 			return nil, -1, err
 		}
-		foundPrevLocal = endpointList.atleastOneEndpointLocal()
-		if setupType == UnknownSetupType {
-			setupType = gotSetupType
-		}
-		if setupType == ErasureSetupType && gotSetupType == DistErasureSetupType {
-			setupType = DistErasureSetupType
-		}
 	}

 	return endpointServerPools, setupType, nil
@@ -409,15 +409,6 @@ func (endpoints Endpoints) GetAllStrings() (all []string) {
 	return
 }

-func (endpoints Endpoints) atleastOneEndpointLocal() bool {
-	for _, endpoint := range endpoints {
-		if endpoint.IsLocal {
-			return true
-		}
-	}
-	return false
-}
-
 func hostResolveToLocalhost(endpoint Endpoint) bool {
 	hostIPs, err := getHostIP(endpoint.Hostname())
 	if err != nil {
@@ -440,7 +431,7 @@ func hostResolveToLocalhost(endpoint Endpoint) bool {
 }

 // UpdateIsLocal - resolves the host and discovers the local host.
-func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
+func (endpoints Endpoints) UpdateIsLocal() error {
 	orchestrated := IsDocker() || IsKubernetes()

 	var epsResolved int
@@ -448,13 +439,14 @@ func (endpoints Endpoints) UpdateIsLocal(foundPrevLocal bool) error {
 	resolvedList := make([]bool, len(endpoints))
 	// Mark the starting time
 	startTime := time.Now()
-	keepAliveTicker := time.NewTicker(10 * time.Millisecond)
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
 	defer keepAliveTicker.Stop()
 	for {
 		// Break if the local endpoint is found already Or all the endpoints are resolved.
 		if foundLocal || (epsResolved == len(endpoints)) {
 			break
 		}
+
 		// Retry infinitely on Kubernetes and Docker swarm.
 		// This is needed as the remote hosts are sometime
 		// not available immediately.
@@ -610,8 +602,353 @@ func checkCrossDeviceMounts(endpoints Endpoints) (err error) {
 	return mountinfo.CheckCrossDevice(absPaths)
 }

+// PoolEndpointList is a temporary type to holds the list of endpoints
+type PoolEndpointList []Endpoints
+
+// UpdateIsLocal - resolves all hosts and discovers which are local
+func (p PoolEndpointList) UpdateIsLocal() error {
+	orchestrated := IsDocker() || IsKubernetes()
+
+	var epsResolved int
+	var epCount int
+
+	for _, endpoints := range p {
+		epCount += len(endpoints)
+	}
+
+	var foundLocal bool
+	resolvedList := make(map[Endpoint]bool)
+
+	// Mark the starting time
+	startTime := time.Now()
+	keepAliveTicker := time.NewTicker(1 * time.Second)
+	defer keepAliveTicker.Stop()
+	for {
+		// Break if the local endpoint is found already Or all the endpoints are resolved.
+		if foundLocal || (epsResolved == epCount) {
+			break
+		}
+
+		// Retry infinitely on Kubernetes and Docker swarm.
+		// This is needed as the remote hosts are sometime
+		// not available immediately.
+		select {
+		case <-globalOSSignalCh:
+			return fmt.Errorf("The endpoint resolution got interrupted")
+		default:
+			for i, endpoints := range p {
+				for j, endpoint := range endpoints {
+					if resolvedList[endpoint] {
+						// Continue if host is already resolved.
+						continue
+					}
+
+					// Log the message to console about the host resolving
+					reqInfo := (&logger.ReqInfo{}).AppendTags(
+						"host",
+						endpoint.Hostname(),
+					)
+
+					if orchestrated && hostResolveToLocalhost(endpoint) {
+						// time elapsed
+						timeElapsed := time.Since(startTime)
+						// log error only if more than a second has elapsed
+						if timeElapsed > time.Second {
+							reqInfo.AppendTags("elapsedTime",
+								humanize.RelTime(startTime,
+									startTime.Add(timeElapsed),
+									"elapsed",
+									"",
+								))
+							ctx := logger.SetReqInfo(GlobalContext,
+								reqInfo)
+							logger.LogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP",
+								endpoint.Hostname()), endpoint.Hostname(), logger.Application)
+						}
+						continue
+					}
+
+					// return err if not Docker or Kubernetes
+					// We use IsDocker() to check for Docker environment
+					// We use IsKubernetes() to check for Kubernetes environment
+					isLocal, err := isLocalHost(endpoint.Hostname(),
+						endpoint.Port(),
+						globalMinioPort,
+					)
+					if err != nil && !orchestrated {
+						return err
+					}
+					if err != nil {
+						// time elapsed
+						timeElapsed := time.Since(startTime)
+						// log error only if more than a second has elapsed
+						if timeElapsed > time.Second {
+							reqInfo.AppendTags("elapsedTime",
+								humanize.RelTime(startTime,
+									startTime.Add(timeElapsed),
+									"elapsed",
+									"",
+								))
+							ctx := logger.SetReqInfo(GlobalContext,
+								reqInfo)
+							logger.LogOnceIf(ctx, err, endpoint.Hostname(), logger.Application)
+						}
+					} else {
+						resolvedList[endpoint] = true
+						endpoint.IsLocal = isLocal
+						epsResolved++
+						if !foundLocal {
+							foundLocal = isLocal
+						}
+						endpoints[j] = endpoint
+					}
+				}
+
+				p[i] = endpoints
+
+				// Wait for the tick, if the there exist a local endpoint in discovery.
+				// Non docker/kubernetes environment we do not need to wait.
+				if !foundLocal && orchestrated {
+					<-keepAliveTicker.C
+				}
+			}
+		}
+	}
+
+	// On Kubernetes/Docker setups DNS resolves inappropriately sometimes
+	// where there are situations same endpoints with multiple disks
+	// come online indicating either one of them is local and some
+	// of them are not local. This situation can never happen and
+	// its only a possibility in orchestrated deployments with dynamic
+	// DNS. Following code ensures that we treat if one of the endpoint
+	// says its local for a given host - it is true for all endpoints
+	// for the same host. Following code ensures that this assumption
+	// is true and it works in all scenarios and it is safe to assume
+	// for a given host.
+	for i, endpoints := range p {
+		endpointLocalMap := make(map[string]bool)
+		for _, ep := range endpoints {
+			if ep.IsLocal {
+				endpointLocalMap[ep.Host] = ep.IsLocal
+			}
+		}
+		for i := range endpoints {
+			endpoints[i].IsLocal = endpointLocalMap[endpoints[i].Host]
+		}
+		p[i] = endpoints
+	}
+
+	return nil
+}
+
+// CreatePoolEndpoints creates a list of endpoints per pool, resolves their relevant hostnames and
+// discovers those are local or remote.
+func CreatePoolEndpoints(serverAddr string, poolArgs ...[][]string) ([]Endpoints, SetupType, error) {
+	var setupType SetupType
+
+	// Check whether serverAddr is valid for this host.
+	if err := CheckLocalServerAddr(serverAddr); err != nil {
+		return nil, setupType, err
+	}
+
+	_, serverAddrPort := mustSplitHostPort(serverAddr)
+
+	poolEndpoints := make(PoolEndpointList, len(poolArgs))
+
+	// For single arg, return single drive EC setup.
+	if len(poolArgs) == 1 && len(poolArgs[0]) == 1 && len(poolArgs[0][0]) == 1 && len(poolArgs[0][0][0]) == 1 {
+		endpoint, err := NewEndpoint(poolArgs[0][0][0])
+		if err != nil {
+			return nil, setupType, err
+		}
+		if err := endpoint.UpdateIsLocal(); err != nil {
+			return nil, setupType, err
+		}
+		if endpoint.Type() != PathEndpointType {
+			return nil, setupType, config.ErrInvalidEndpoint(nil).Msg("use path style endpoint for single node setup")
+		}
+
+		var endpoints Endpoints
+		endpoints = append(endpoints, endpoint)
+		setupType = ErasureSDSetupType
+
+		poolEndpoints[0] = endpoints
+		// Check for cross device mounts if any.
+		if err = checkCrossDeviceMounts(endpoints); err != nil {
+			return nil, setupType, config.ErrInvalidEndpoint(nil).Msg(err.Error())
+		}
+
+		return poolEndpoints, setupType, nil
+	}
+
+	for i, args := range poolArgs {
+		var endpoints Endpoints
+		for _, iargs := range args {
+			// Convert args to endpoints
+			eps, err := NewEndpoints(iargs...)
+			if err != nil {
+				return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
+			}
+
+			// Check for cross device mounts if any.
+			if err = checkCrossDeviceMounts(eps); err != nil {
+				return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
+			}
+
+			endpoints = append(endpoints, eps...)
+		}
+
+		if len(endpoints) == 0 {
+			return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("invalid number of endpoints")
+		}
+
+		poolEndpoints[i] = endpoints
+	}
+
+	for _, endpoints := range poolEndpoints {
+		// Return Erasure setup when all endpoints are path style.
+		if endpoints[0].Type() == PathEndpointType {
+			setupType = ErasureSetupType
+		}
+		if endpoints[0].Type() == URLEndpointType && setupType != DistErasureSetupType {
+			setupType = DistErasureSetupType
+		}
+	}
+
+	if setupType == ErasureSetupType {
+		return poolEndpoints, setupType, nil
+	}
+
+	if err := poolEndpoints.UpdateIsLocal(); err != nil {
+		return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
+	}
+
+	uniqueArgs := set.NewStringSet()
+
+	for i, endpoints := range poolEndpoints {
+		// Here all endpoints are URL style.
+		endpointPathSet := set.NewStringSet()
+		localEndpointCount := 0
+		localServerHostSet := set.NewStringSet()
+		localPortSet := set.NewStringSet()
+
+		for _, endpoint := range endpoints {
+			endpointPathSet.Add(endpoint.Path)
+			if endpoint.IsLocal {
+				localServerHostSet.Add(endpoint.Hostname())
+
+				_, port, err := net.SplitHostPort(endpoint.Host)
+				if err != nil {
+					port = serverAddrPort
+				}
+				localPortSet.Add(port)
+
+				localEndpointCount++
+			}
+		}
+
+		orchestrated := IsKubernetes() || IsDocker()
+		if !orchestrated {
+			// Check whether same path is not used in endpoints of a host on different port.
+			// Only verify this on baremetal setups, DNS is not available in orchestrated
+			// environments so we can't do much here.
+			{
+				pathIPMap := make(map[string]set.StringSet)
+				hostIPCache := make(map[string]set.StringSet)
+				for _, endpoint := range endpoints {
+					host := endpoint.Hostname()
+					hostIPSet, ok := hostIPCache[host]
+					if !ok {
+						var err error
+						hostIPSet, err = getHostIP(host)
+						if err != nil {
+							return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err))
+						}
+						hostIPCache[host] = hostIPSet
+					}
+					if IPSet, ok := pathIPMap[endpoint.Path]; ok {
+						if !IPSet.Intersection(hostIPSet).IsEmpty() {
+							return nil, setupType,
+								config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path))
+						}
+						pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet)
+					} else {
+						pathIPMap[endpoint.Path] = hostIPSet
+					}
+				}
+			}
+		}
+
+		// Check whether same path is used for more than 1 local endpoints.
+		{
+			localPathSet := set.CreateStringSet()
+			for _, endpoint := range endpoints {
+				if !endpoint.IsLocal {
+					continue
+				}
+				if localPathSet.Contains(endpoint.Path) {
+					return nil, setupType,
+						config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path))
+				}
+				localPathSet.Add(endpoint.Path)
+			}
+		}
+
+		// Add missing port in all endpoints.
+		for i := range endpoints {
+			_, port, err := net.SplitHostPort(endpoints[i].Host)
+			if err != nil {
+				endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort)
+			} else if endpoints[i].IsLocal && serverAddrPort != port {
+				// If endpoint is local, but port is different than serverAddrPort, then make it as remote.
+				endpoints[i].IsLocal = false
+			}
+		}
+
+		// All endpoints are pointing to local host
+		if len(endpoints) == localEndpointCount {
+			// If all endpoints have same port number, Just treat it as local erasure setup
+			// using URL style endpoints.
+			if len(localPortSet) == 1 {
+				if len(localServerHostSet) > 1 {
+					return nil, setupType,
+						config.ErrInvalidErasureEndpoints(nil).Msg("all local endpoints should not have different hostnames/ips")
+				}
+			}
+
+			// Even though all endpoints are local, but those endpoints use different ports.
+			// This means it is DistErasure setup.
+		}
+
+		poolEndpoints[i] = endpoints
+
+		for _, endpoint := range endpoints {
+			uniqueArgs.Add(endpoint.Host)
+		}
+
+		poolEndpoints[i] = endpoints
+	}
+
+	publicIPs := env.Get(config.EnvPublicIPs, "")
+	if len(publicIPs) == 0 {
+		updateDomainIPs(uniqueArgs)
+	}
+
+	for _, endpoints := range poolEndpoints {
+		// Return Erasure setup when all endpoints are path style.
+		if endpoints[0].Type() == PathEndpointType {
+			setupType = ErasureSetupType
+		}
+		if endpoints[0].Type() == URLEndpointType && setupType != DistErasureSetupType {
+			setupType = DistErasureSetupType
+		}
+	}
+
+	return poolEndpoints, setupType, nil
+}
+
 // CreateEndpoints - validates and creates new endpoints for given args.
-func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endpoints, SetupType, error) {
+func CreateEndpoints(serverAddr string, args ...[]string) (Endpoints, SetupType, error) {
 	var endpoints Endpoints
 	var setupType SetupType
 	var err error
@@ -672,7 +1009,7 @@ func CreateEndpoints(serverAddr string, foundLocal bool, args ...[]string) (Endp
 		return endpoints, setupType, nil
 	}

-	if err = endpoints.UpdateIsLocal(foundLocal); err != nil {
+	if err = endpoints.UpdateIsLocal(); err != nil {
 		return endpoints, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error())
 	}

@@ -314,7 +314,7 @@ func TestCreateEndpoints(t *testing.T) {
 	for _, testCase := range testCases {
 		testCase := testCase
 		t.Run("", func(t *testing.T) {
-			endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, false, testCase.args...)
+			endpoints, setupType, err := CreateEndpoints(testCase.serverAddr, testCase.args...)
 			if err == nil && testCase.expectedErr != nil {
 				t.Errorf("error: expected = %v, got = <nil>", testCase.expectedErr)
 			}
@@ -374,7 +374,7 @@ func TestGetLocalPeer(t *testing.T) {
 	for i, testCase := range testCases {
 		zendpoints := mustGetPoolEndpoints(testCase.endpointArgs...)
 		if !zendpoints[0].Endpoints[0].IsLocal {
-			if err := zendpoints[0].Endpoints.UpdateIsLocal(false); err != nil {
+			if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil {
 				t.Fatalf("error: expected = <nil>, got = %v", err)
 			}
 		}
@@ -407,7 +407,7 @@ func TestGetRemotePeers(t *testing.T) {
 	for _, testCase := range testCases {
 		zendpoints := mustGetPoolEndpoints(testCase.endpointArgs...)
 		if !zendpoints[0].Endpoints[0].IsLocal {
-			if err := zendpoints[0].Endpoints.UpdateIsLocal(false); err != nil {
+			if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil {
 				t.Errorf("error: expected = <nil>, got = %v", err)
 			}
 		}
@@ -23,7 +23,7 @@ import (
 	"sync"

 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 )

 func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) {
@@ -20,6 +20,7 @@ package cmd
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"sync"
 	"sync/atomic"
@@ -124,6 +125,7 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 		readTriggerCh <- true
 	}

+	disksNotFound := int32(0)
 	bitrotHeal := int32(0)       // Atomic bool flag.
 	missingPartsHeal := int32(0) // Atomic bool flag.
 	readerIndex := 0
@@ -164,10 +166,13 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 			p.buf[bufIdx] = p.buf[bufIdx][:p.shardSize]
 			n, err := rr.ReadAt(p.buf[bufIdx], p.offset)
 			if err != nil {
-				if errors.Is(err, errFileNotFound) {
+				switch {
+				case errors.Is(err, errFileNotFound):
 					atomic.StoreInt32(&missingPartsHeal, 1)
-				} else if errors.Is(err, errFileCorrupt) {
+				case errors.Is(err, errFileCorrupt):
 					atomic.StoreInt32(&bitrotHeal, 1)
+				case errors.Is(err, errDiskNotFound):
+					atomic.AddInt32(&disksNotFound, 1)
 				}

 				// This will be communicated upstream.
@@ -189,16 +194,16 @@ func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) {
 	wg.Wait()
 	if p.canDecode(newBuf) {
 		p.offset += p.shardSize
-		if atomic.LoadInt32(&missingPartsHeal) == 1 {
+		if missingPartsHeal == 1 {
 			return newBuf, errFileNotFound
-		} else if atomic.LoadInt32(&bitrotHeal) == 1 {
+		} else if bitrotHeal == 1 {
 			return newBuf, errFileCorrupt
 		}
 		return newBuf, nil
 	}

 	// If we cannot decode, just return read quorum error.
-	return nil, errErasureReadQuorum
+	return nil, fmt.Errorf("%w (offline-disks=%d/%d)", errErasureReadQuorum, disksNotFound, len(p.readers))
 }

 // Decode reads from readers, reconstructs data if needed and writes the data to the writer.
@@ -19,6 +19,7 @@ package cmd

 import (
 	"context"
+	"fmt"
 	"io"
 	"sync"

@@ -70,7 +71,9 @@ func (p *parallelWriter) Write(ctx context.Context, blocks [][]byte) error {
 	if nilCount >= p.writeQuorum {
 		return nil
 	}
-	return reduceWriteQuorumErrs(ctx, p.errs, objectOpIgnoredErrs, p.writeQuorum)
+
+	writeErr := reduceWriteQuorumErrs(ctx, p.errs, objectOpIgnoredErrs, p.writeQuorum)
+	return fmt.Errorf("%w (offline-disks=%d/%d)", writeErr, countErrs(p.errs, errDiskNotFound), len(p.writers))
 }

 // Encode reads from the reader, erasure-encodes the data and writes to the writers.
@@ -25,6 +25,38 @@ import (
 	"github.com/minio/madmin-go/v2"
 )

+func commonETags(etags []string) (etag string, maxima int) {
+	etagOccurrenceMap := make(map[string]int, len(etags))
+
+	// Ignore the uuid sentinel and count the rest.
+	for _, etag := range etags {
+		if etag == "" {
+			continue
+		}
+		etagOccurrenceMap[etag]++
+	}
+
+	maxima = 0 // Counter for remembering max occurrence of elements.
+	latest := ""
+
+	// Find the common cardinality from previously collected
+	// occurrences of elements.
+	for etag, count := range etagOccurrenceMap {
+		if count < maxima {
+			continue
+		}
+
+		// We are at or above maxima
+		if count > maxima {
+			maxima = count
+			latest = etag
+		}
+	}
+
+	// Return the collected common max time, with maxima
+	return latest, maxima
+}
+
 // commonTime returns a maximally occurring time from a list of time.
 func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime time.Time, maxima int) {
 	timeOccurenceMap := make(map[int64]int, len(times))
@@ -76,10 +108,21 @@ func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime tim
 	return time.Unix(0, latest).UTC(), maxima
 }

-// commonTime returns a maximally occurring time from a list of time.
-func commonTime(modTimes []time.Time) (modTime time.Time) {
-	modTime, _ = commonTimeAndOccurence(modTimes, 0)
-	return modTime
+// commonTime returns a maximally occurring time from a list of time if it
+// occurs >= quorum, else return timeSentinel
+func commonTime(modTimes []time.Time, quorum int) time.Time {
+	if modTime, count := commonTimeAndOccurence(modTimes, 0); count >= quorum {
+		return modTime
+	}
+
+	return timeSentinel
+}
+
+func commonETag(etags []string, quorum int) string {
+	if etag, count := commonETags(etags); count >= quorum {
+		return etag
+	}
+	return ""
 }

 // Beginning of unix time is treated as sentinel value here.
@@ -98,6 +141,33 @@ func bootModtimes(diskCount int) []time.Time {
 	return modTimes
 }

+func listObjectETags(partsMetadata []FileInfo, errs []error, quorum int) (etags []string) {
+	etags = make([]string, len(partsMetadata))
+	vidMap := map[string]int{}
+	for index, metadata := range partsMetadata {
+		if errs[index] != nil {
+			continue
+		}
+		vid := metadata.VersionID
+		if metadata.VersionID == "" {
+			vid = nullVersionID
+		}
+		vidMap[vid]++
+		etags[index] = metadata.Metadata["etag"]
+	}
+
+	for _, count := range vidMap {
+		// do we have enough common versions
+		// that have enough quorum to satisfy
+		// the etag.
+		if count >= quorum {
+			return etags
+		}
+	}
+
+	return make([]string, len(partsMetadata))
+}
+
 // Extracts list of times from FileInfo slice and returns, skips
 // slice elements which have errors.
 func listObjectModtimes(partsMetadata []FileInfo, errs []error) (modTimes []time.Time) {
@@ -157,15 +227,32 @@ func listObjectDiskMtimes(partsMetadata []FileInfo) (diskMTimes []time.Time) {
 // listOnlineDisks - returns
 // - a slice of disks where disk having 'older' xl.meta (or nothing)
 // are set to nil.
-// - latest (in time) of the maximally occurring modTime(s).
-func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error) (onlineDisks []StorageAPI, modTime time.Time) {
+// - latest (in time) of the maximally occurring modTime(s), which has at least quorum occurrences.
+func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error, quorum int) (onlineDisks []StorageAPI, modTime time.Time, etag string) {
 	onlineDisks = make([]StorageAPI, len(disks))

 	// List all the file commit ids from parts metadata.
 	modTimes := listObjectModtimes(partsMetadata, errs)

 	// Reduce list of UUIDs to a single common value.
-	modTime = commonTime(modTimes)
+	modTime = commonTime(modTimes, quorum)
+
+	if modTime.IsZero() || modTime.Equal(timeSentinel) {
+		etags := listObjectETags(partsMetadata, errs, quorum)
+
+		etag = commonETag(etags, quorum)
+
+		if etag != "" { // allow this fallback only if a non-empty etag is found.
+			for index, e := range etags {
+				if partsMetadata[index].IsValid() && e == etag {
+					onlineDisks[index] = disks[index]
+				} else {
+					onlineDisks[index] = nil
+				}
+			}
+			return onlineDisks, modTime, etag
+		}
+	}

 	// Create a new online disks slice, which have common uuid.
 	for index, t := range modTimes {
@@ -176,7 +263,7 @@ func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error)
 		}
 	}

-	return onlineDisks, modTime
+	return onlineDisks, modTime, ""
 }

 // disksWithAllParts - This function needs to be called with
@@ -52,7 +52,7 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultPar
 	var latestFileInfo FileInfo

 	// Reduce list of UUIDs to a single common value - i.e. the last updated Time
-	modTime := commonTime(modTimes)
+	modTime := commonTime(modTimes, expectedRQuorum)

 	if modTime.IsZero() || modTime.Equal(timeSentinel) {
 		return FileInfo{}, errErasureReadQuorum
@@ -82,8 +82,9 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultPar
 func TestCommonTime(t *testing.T) {
 	// List of test cases for common modTime.
 	testCases := []struct {
-		times []time.Time
-		time  time.Time
+		times  []time.Time
+		time   time.Time
+		quorum int
 	}{
 		{
 			// 1. Tests common times when slice has varying time elements.
@@ -97,6 +98,7 @@ func TestCommonTime(t *testing.T) {
 				time.Unix(0, 1).UTC(),
 			},
 			time.Unix(0, 3).UTC(),
+			3,
 		},
 		{
 			// 2. Tests common time obtained when all elements are equal.
@@ -110,10 +112,11 @@ func TestCommonTime(t *testing.T) {
 				time.Unix(0, 3).UTC(),
 			},
 			time.Unix(0, 3).UTC(),
+			4,
 		},
 		{
-			// 3. Tests common time obtained when elements have a mixture
-			// of sentinel values.
+			// 3. Tests common time obtained when elements have a mixture of
+			// sentinel values and don't have read quorum on any of the values.
 			[]time.Time{
 				time.Unix(0, 3).UTC(),
 				time.Unix(0, 3).UTC(),
@@ -126,7 +129,8 @@ func TestCommonTime(t *testing.T) {
 				timeSentinel,
 				timeSentinel,
 			},
-			time.Unix(0, 3).UTC(),
+			timeSentinel,
+			5,
 		},
 	}

@@ -134,7 +138,7 @@ func TestCommonTime(t *testing.T) {
 	// common modtime. Tests fail if modtime does not match.
 	for i, testCase := range testCases {
 		// Obtain a common mod time from modTimes slice.
-		ctime := commonTime(testCase.times)
+		ctime := commonTime(testCase.times, testCase.quorum)
 		if !testCase.time.Equal(ctime) {
 			t.Errorf("Test case %d, expect to pass but failed. Wanted modTime: %s, got modTime: %s\n", i+1, testCase.time, ctime)
 		}
@@ -151,30 +155,34 @@ func TestListOnlineDisks(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Prepare Erasure backend failed - %v", err)
 	}
+	setObjectLayer(obj)
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(disks)

 	type tamperKind int
 	const (
-		noTamper    tamperKind = iota
-		deletePart  tamperKind = iota
-		corruptPart tamperKind = iota
+		noTamper tamperKind = iota
+		deletePart
+		corruptPart
 	)
-	threeNanoSecs := time.Unix(0, 3).UTC()
-	fourNanoSecs := time.Unix(0, 4).UTC()
-	modTimesThreeNone := []time.Time{
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		timeSentinel, timeSentinel, timeSentinel, timeSentinel,
-		timeSentinel, timeSentinel, timeSentinel, timeSentinel,
-		timeSentinel,
-	}
-	modTimesThreeFour := []time.Time{
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		fourNanoSecs, fourNanoSecs, fourNanoSecs, fourNanoSecs,
-		fourNanoSecs, fourNanoSecs, fourNanoSecs, fourNanoSecs,
+
+	timeSentinel := time.Unix(1, 0).UTC()
+	threeNanoSecs := time.Unix(3, 0).UTC()
+	fourNanoSecs := time.Unix(4, 0).UTC()
+	modTimesThreeNone := make([]time.Time, 16)
+	modTimesThreeFour := make([]time.Time, 16)
+	for i := 0; i < 16; i++ {
+		// Have 13 good xl.meta, 12 for default parity count = 4 (EC:4) and one
+		// to be tampered with.
+		if i > 12 {
+			modTimesThreeFour[i] = fourNanoSecs
+			modTimesThreeNone[i] = timeSentinel
+			continue
+		}
+		modTimesThreeFour[i] = threeNanoSecs
+		modTimesThreeNone[i] = threeNanoSecs
 	}
+
 	testCases := []struct {
 		modTimes       []time.Time
 		expectedTime   time.Time
@@ -183,10 +191,10 @@ func TestListOnlineDisks(t *testing.T) {
 	}{
 		{
 			modTimes:     modTimesThreeFour,
-			expectedTime: fourNanoSecs,
+			expectedTime: threeNanoSecs,
 			errs: []error{
-				nil, nil, nil, nil, nil, nil, nil, nil, nil,
-				nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil, nil, nil, nil,
 			},
 			_tamperBackend: noTamper,
 		},
@@ -195,13 +203,10 @@ func TestListOnlineDisks(t *testing.T) {
 			expectedTime: threeNanoSecs,
 			errs: []error{
 				// Disks that have a valid xl.meta.
-				nil, nil, nil, nil, nil, nil, nil,
-				// Majority of disks don't have xl.meta.
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errDiskAccessDenied,
-				errDiskNotFound, errFileNotFound,
-				errFileNotFound,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil,
+				// Some disks can't access xl.meta.
+				errFileNotFound, errDiskAccessDenied, errDiskNotFound,
 			},
 			_tamperBackend: deletePart,
 		},
@@ -210,13 +215,10 @@ func TestListOnlineDisks(t *testing.T) {
 			expectedTime: threeNanoSecs,
 			errs: []error{
 				// Disks that have a valid xl.meta.
-				nil, nil, nil, nil, nil, nil, nil,
-				// Majority of disks don't have xl.meta.
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errDiskAccessDenied,
-				errDiskNotFound, errFileNotFound,
-				errFileNotFound,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil,
+				// Some disks don't have xl.meta.
+				errDiskNotFound, errFileNotFound, errFileNotFound,
 			},
 			_tamperBackend: corruptPart,
 		},
@@ -296,7 +298,8 @@ func TestListOnlineDisks(t *testing.T) {

 			}

-			onlineDisks, modTime := listOnlineDisks(erasureDisks, partsMetadata, test.errs)
+			rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
+			onlineDisks, modTime, _ := listOnlineDisks(erasureDisks, partsMetadata, test.errs, rQuorum)
 			if !modTime.Equal(test.expectedTime) {
 				t.Fatalf("Expected modTime to be equal to %v but was found to be %v",
 					test.expectedTime, modTime)
@@ -325,6 +328,7 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Prepare Erasure backend failed - %v", err)
 	}
+	setObjectLayer(obj)
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(disks)

@@ -337,19 +341,20 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 	timeSentinel := time.Unix(1, 0).UTC()
 	threeNanoSecs := time.Unix(3, 0).UTC()
 	fourNanoSecs := time.Unix(4, 0).UTC()
-	modTimesThreeNone := []time.Time{
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		timeSentinel, timeSentinel, timeSentinel, timeSentinel,
-		timeSentinel, timeSentinel, timeSentinel, timeSentinel,
-		timeSentinel,
-	}
-	modTimesThreeFour := []time.Time{
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		threeNanoSecs, threeNanoSecs, threeNanoSecs, threeNanoSecs,
-		fourNanoSecs, fourNanoSecs, fourNanoSecs, fourNanoSecs,
-		fourNanoSecs, fourNanoSecs, fourNanoSecs, fourNanoSecs,
+	modTimesThreeNone := make([]time.Time, 16)
+	modTimesThreeFour := make([]time.Time, 16)
+	for i := 0; i < 16; i++ {
+		// Have 13 good xl.meta, 12 for default parity count = 4 (EC:4) and one
+		// to be tampered with.
+		if i > 12 {
+			modTimesThreeFour[i] = fourNanoSecs
+			modTimesThreeNone[i] = timeSentinel
+			continue
+		}
+		modTimesThreeFour[i] = threeNanoSecs
+		modTimesThreeNone[i] = threeNanoSecs
 	}
+
 	testCases := []struct {
 		modTimes       []time.Time
 		expectedTime   time.Time
@@ -358,10 +363,10 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 	}{
 		{
 			modTimes:     modTimesThreeFour,
-			expectedTime: fourNanoSecs,
+			expectedTime: threeNanoSecs,
 			errs: []error{
-				nil, nil, nil, nil, nil, nil, nil, nil, nil,
-				nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil, nil, nil, nil,
 			},
 			_tamperBackend: noTamper,
 		},
@@ -370,13 +375,10 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 			expectedTime: threeNanoSecs,
 			errs: []error{
 				// Disks that have a valid xl.meta.
-				nil, nil, nil, nil, nil, nil, nil,
-				// Majority of disks don't have xl.meta.
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errDiskAccessDenied,
-				errDiskNotFound, errFileNotFound,
-				errFileNotFound,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil,
+				// Some disks can't access xl.meta.
+				errFileNotFound, errDiskAccessDenied, errDiskNotFound,
 			},
 			_tamperBackend: deletePart,
 		},
@@ -385,13 +387,10 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 			expectedTime: threeNanoSecs,
 			errs: []error{
 				// Disks that have a valid xl.meta.
-				nil, nil, nil, nil, nil, nil, nil,
-				// Majority of disks don't have xl.meta.
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errFileNotFound,
-				errFileNotFound, errDiskAccessDenied,
-				errDiskNotFound, errFileNotFound,
-				errFileNotFound,
+				nil, nil, nil, nil, nil, nil, nil, nil,
+				nil, nil, nil, nil, nil,
+				// Some disks don't have xl.meta.
+				errDiskNotFound, errFileNotFound, errFileNotFound,
 			},
 			_tamperBackend: corruptPart,
 		},
@@ -481,7 +480,8 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 				t.Fatalf("Failed to getLatestFileInfo, expected %v, got %v", errErasureReadQuorum, err)
 			}

-			onlineDisks, modTime := listOnlineDisks(erasureDisks, partsMetadata, test.errs)
+			rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount
+			onlineDisks, modTime, _ := listOnlineDisks(erasureDisks, partsMetadata, test.errs, rQuorum)
 			if !modTime.Equal(test.expectedTime) {
 				t.Fatalf("Expected modTime to be equal to %v but was found to be %v",
 					test.expectedTime, modTime)
@@ -508,6 +508,7 @@ func TestDisksWithAllParts(t *testing.T) {
 	if err != nil {
 		t.Fatalf("Prepare Erasure backend failed - %v", err)
 	}
+	setObjectLayer(obj)
 	defer obj.Shutdown(context.Background())
 	defer removeRoots(disks)

@@ -547,7 +548,7 @@ func TestDisksWithAllParts(t *testing.T) {
 		t.Fatalf("Failed to get quorum consistent fileInfo %v", err)
 	}

-	erasureDisks, _ = listOnlineDisks(erasureDisks, partsMetadata, errs)
+	erasureDisks, _, _ = listOnlineDisks(erasureDisks, partsMetadata, errs, readQuorum)

 	filteredDisks, errs, _ := disksWithAllParts(ctx, erasureDisks, partsMetadata,
 		errs, fi, bucket, object, madmin.HealDeepScan)
@@ -655,3 +656,133 @@ func TestDisksWithAllParts(t *testing.T) {
 		}
 	}
 }
+
+func TestCommonParities(t *testing.T) {
+	// This test uses two FileInfo values that represent the same object but
+	// have different parities. They occur in equal number of drives, but only
+	// one has read quorum. commonParity should pick the parity corresponding to
+	// the FileInfo which has read quorum.
+	fi1 := FileInfo{
+		Volume:         "mybucket",
+		Name:           "myobject",
+		VersionID:      "",
+		IsLatest:       true,
+		Deleted:        false,
+		ExpireRestored: false,
+		DataDir:        "4a01d9dd-0c5e-4103-88f8-b307c57d212e",
+		XLV1:           false,
+		ModTime:        time.Date(2023, time.March, 15, 11, 18, 4, 989906961, time.UTC),
+		Size:           329289, Mode: 0x0, WrittenByVersion: 0x63c77756,
+		Metadata: map[string]string{
+			"content-type": "application/octet-stream", "etag": "f205307ef9f50594c4b86d9c246bee86", "x-minio-internal-erasure-upgraded": "5->6", "x-minio-internal-inline-data": "true",
+		},
+		Parts: []ObjectPartInfo{
+			{
+				ETag:       "",
+				Number:     1,
+				Size:       329289,
+				ActualSize: 329289,
+				ModTime:    time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
+				Index:      []uint8(nil),
+				Checksums:  map[string]string(nil),
+			},
+		},
+		Erasure: ErasureInfo{
+			Algorithm:    "ReedSolomon",
+			DataBlocks:   6,
+			ParityBlocks: 6,
+			BlockSize:    1048576,
+			Index:        1,
+			Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+			Checksums:    []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}},
+		},
+		NumVersions: 1,
+		Idx:         0,
+	}
+
+	fi2 := FileInfo{
+		Volume:           "mybucket",
+		Name:             "myobject",
+		VersionID:        "",
+		IsLatest:         true,
+		Deleted:          false,
+		DataDir:          "6f5c106d-9d28-4c85-a7f4-eac56225876b",
+		ModTime:          time.Date(2023, time.March, 15, 19, 57, 30, 492530160, time.UTC),
+		Size:             329289,
+		Mode:             0x0,
+		WrittenByVersion: 0x63c77756,
+		Metadata:         map[string]string{"content-type": "application/octet-stream", "etag": "f205307ef9f50594c4b86d9c246bee86", "x-minio-internal-inline-data": "true"},
+		Parts: []ObjectPartInfo{
+			{
+				ETag:       "",
+				Number:     1,
+				Size:       329289,
+				ActualSize: 329289,
+				ModTime:    time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC),
+				Index:      []uint8(nil),
+				Checksums:  map[string]string(nil),
+			},
+		},
+		Erasure: ErasureInfo{
+			Algorithm:    "ReedSolomon",
+			DataBlocks:   7,
+			ParityBlocks: 5,
+			BlockSize:    1048576,
+			Index:        2,
+			Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12},
+			Checksums: []ChecksumInfo{
+				{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}},
+			},
+		},
+		NumVersions: 1,
+		Idx:         0,
+	}
+
+	fiDel := FileInfo{
+		Volume:           "mybucket",
+		Name:             "myobject",
+		VersionID:        "",
+		IsLatest:         true,
+		Deleted:          true,
+		ModTime:          time.Date(2023, time.March, 15, 19, 57, 30, 492530160, time.UTC),
+		Mode:             0x0,
+		WrittenByVersion: 0x63c77756,
+		NumVersions:      1,
+		Idx:              0,
+	}
+
+	tests := []struct {
+		fi1, fi2 FileInfo
+	}{
+		{
+			fi1: fi1,
+			fi2: fi2,
+		},
+		{
+			fi1: fi1,
+			fi2: fiDel,
+		},
+	}
+	for idx, test := range tests {
+		var metaArr []FileInfo
+		for i := 0; i < 12; i++ {
+			fi := test.fi1
+			if i%2 == 0 {
+				fi = test.fi2
+			}
+			metaArr = append(metaArr, fi)
+		}
+
+		parities := listObjectParities(metaArr, make([]error, len(metaArr)))
+		parity := commonParity(parities, 5)
+		var match int
+		for _, fi := range metaArr {
+			if fi.Erasure.ParityBlocks == parity {
+				match++
+			}
+		}
+		if match < len(metaArr)-parity {
+			t.Fatalf("Test %d: Expected %d drives with parity=%d, but got %d", idx, len(metaArr)-parity, parity, match)
+		}
+	}
+}
@@ -28,7 +28,7 @@ import (

 	"github.com/minio/madmin-go/v2"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 )

 const reservedMetadataPrefixLowerDataShardFix = ReservedMetadataPrefixLower + "data-shard-fix"
@@ -386,11 +386,11 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object

 	// List of disks having latest version of the object xl.meta
 	// (by modtime).
-	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
+	onlineDisks, modTime, etag := listOnlineDisks(storageDisks, partsMetadata, errs, readQuorum)

 	// Latest FileInfo for reference. If a valid metadata is not
 	// present, it is as good as object not found.
-	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, readQuorum)
+	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, etag, readQuorum)
 	if err != nil {
 		return result, err
 	}
@@ -503,7 +503,7 @@ func (er *erasureObjects) healObject(ctx context.Context, bucket string, object

 	if !latestMeta.XLV1 && !latestMeta.Deleted && !recreate && disksToHealCount > latestMeta.Erasure.ParityBlocks {
 		// When disk to heal count is greater than parity blocks we should simply error out.
-		err := fmt.Errorf("more drives are expected to heal than parity, returned errors: %v (dataErrs %v) -> %s/%s(%s)", errs, dataErrs, bucket, object, versionID)
+		err := fmt.Errorf("(%d > %d) more drives are expected to heal than parity, returned errors: %v (dataErrs %v) -> %s/%s(%s)", disksToHealCount, latestMeta.Erasure.ParityBlocks, errs, dataErrs, bucket, object, versionID)
 		logger.LogIf(ctx, err)
 		return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs,
 			bucket, object, versionID), err
@@ -1012,9 +1012,6 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 	notFoundMetaErrs, corruptedMetaErrs, driveNotFoundMetaErrs := danglingErrsCount(errs)
 	notFoundPartsErrs, corruptedPartsErrs, driveNotFoundPartsErrs := danglingErrsCount(ndataErrs)

-	if driveNotFoundMetaErrs > 0 || driveNotFoundPartsErrs > 0 {
-		return validMeta, false
-	}
 	for _, m := range metaArr {
 		if m.IsValid() {
 			validMeta = m
@@ -1027,6 +1024,10 @@ func isObjectDangling(metaArr []FileInfo, errs []error, dataErrs []error) (valid
 		return validMeta, false
 	}

+	if driveNotFoundMetaErrs > 0 || driveNotFoundPartsErrs > 0 {
+		return validMeta, false
+	}
+
 	if validMeta.Deleted {
 		// notFoundPartsErrs is ignored since
 		// - delete marker does not have any parts
@@ -33,6 +33,7 @@ import (
 	"github.com/dustin/go-humanize"
 	uuid2 "github.com/google/uuid"
 	"github.com/minio/madmin-go/v2"
+	"github.com/minio/minio/internal/config/storageclass"
 )

 // Tests isObjectDangling function
@@ -563,6 +564,17 @@ func TestHealingDanglingObject(t *testing.T) {
 	resetGlobalHealState()
 	defer resetGlobalHealState()

+	// Set globalStoragClass.STANDARD to EC:4 for this test
+	saveSC := globalStorageClass
+	defer func() {
+		globalStorageClass.Update(saveSC)
+	}()
+	globalStorageClass.Update(storageclass.Config{
+		Standard: storageclass.StorageClass{
+			Parity: 4,
+		},
+	})
+
 	nDisks := 16
 	fsDirs, err := getRandomDisks(nDisks)
 	if err != nil {
@@ -577,6 +589,8 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}

+	setObjectLayer(objLayer)
+
 	bucket := getRandomBucketName()
 	object := getRandomObjectName()
 	data := bytes.Repeat([]byte("a"), 128*1024)
@@ -1620,7 +1634,7 @@ func TestHealLastDataShard(t *testing.T) {
 				t.Fatal(err)
 			}

-			firstGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
+			firstGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, ObjectOptions{NoLock: true})
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -1650,7 +1664,7 @@ func TestHealLastDataShard(t *testing.T) {
 				t.Fatal(err)
 			}

-			secondGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
+			secondGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, ObjectOptions{NoLock: true})
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -23,10 +23,9 @@ import (
 	"fmt"
 	"hash/crc32"
 	"io"
-	"strings"

 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 )

 // figure out the most commonVersions across disk that satisfies
@@ -173,10 +172,8 @@ func readAllFileInfo(ctx context.Context, disks []StorageAPI, bucket, object, ve
 		errFileVersionNotFound,
 		errDiskNotFound,
 		errUnformattedDisk,
-	}
-	if strings.HasPrefix(bucket, minioMetaBucket) {
-		// listing object might be truncated, ignore such errors from logging.
-		ignoredErrs = append(ignoredErrs, io.ErrUnexpectedEOF)
+		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
+		io.EOF,              // some times we would read without locks, ignore these errors
 	}
 	errs := g.Wait()
 	for index, err := range errs {
@@ -33,7 +33,7 @@ import (
 	"github.com/minio/minio/internal/hash/sha256"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
+	"github.com/minio/pkg/sync/errgroup"
 	"github.com/minio/sio"
 )

@@ -42,13 +42,6 @@ const minIOErasureUpgraded = "x-minio-internal-erasure-upgraded"

 const erasureAlgorithm = "rs-vandermonde"

-// byObjectPartNumber is a collection satisfying sort.Interface.
-type byObjectPartNumber []ObjectPartInfo
-
-func (t byObjectPartNumber) Len() int           { return len(t) }
-func (t byObjectPartNumber) Swap(i, j int)      { t[i], t[j] = t[j], t[i] }
-func (t byObjectPartNumber) Less(i, j int) bool { return t[i].Number < t[j].Number }
-
 // AddChecksumInfo adds a checksum of a part.
 func (e *ErasureInfo) AddChecksumInfo(ckSumInfo ChecksumInfo) {
 	for i, sum := range e.Checksums {
@@ -165,6 +158,8 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 		IsDir:            HasSuffix(object, SlashSeparator),
 		Bucket:           bucket,
 		Name:             object,
+		ParityBlocks:     fi.Erasure.ParityBlocks,
+		DataBlocks:       fi.Erasure.DataBlocks,
 		VersionID:        versionID,
 		IsLatest:         fi.IsLatest,
 		DeleteMarker:     fi.Deleted,
@@ -234,6 +229,7 @@ func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInf
 		}
 	}
 	objInfo.Checksum = fi.Checksum
+	objInfo.Inlined = fi.InlineData()
 	// Success.
 	return objInfo
 }
@@ -307,7 +303,7 @@ func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize, act
 	fi.Parts = append(fi.Parts, partInfo)

 	// Parts in FileInfo should be in sorted order by part number.
-	sort.Sort(byObjectPartNumber(fi.Parts))
+	sort.Slice(fi.Parts, func(i, j int) bool { return fi.Parts[i].Number < fi.Parts[j].Number })
 }

 // ObjectToPartOffset - translate offset of an object to offset of its individual part.
@@ -332,7 +328,7 @@ func (fi FileInfo) ObjectToPartOffset(ctx context.Context, offset int64) (partIn
 	return 0, 0, InvalidRange{}
 }

-func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.Time, quorum int) (FileInfo, error) {
+func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.Time, etag string, quorum int) (FileInfo, error) {
 	// with less quorum return error.
 	if quorum < 1 {
 		return FileInfo{}, errErasureReadQuorum
@@ -340,12 +336,21 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 	metaHashes := make([]string, len(metaArr))
 	h := sha256.New()
 	for i, meta := range metaArr {
-		if meta.IsValid() && meta.ModTime.Equal(modTime) {
+		if !meta.IsValid() {
+			continue
+		}
+		etagOnly := modTime.Equal(timeSentinel) && (etag != "" && etag == meta.Metadata["etag"])
+		mtimeValid := meta.ModTime.Equal(modTime)
+		if mtimeValid || etagOnly {
 			fmt.Fprintf(h, "%v", meta.XLV1)
-			fmt.Fprintf(h, "%v", meta.GetDataDir())
+			if !etagOnly {
+				// Verify dataDir is same only when mtime is valid and etag is not considered.
+				fmt.Fprintf(h, "%v", meta.GetDataDir())
+			}
 			for _, part := range meta.Parts {
 				fmt.Fprintf(h, "part.%d", part.Number)
 			}
+			fmt.Fprintf(h, "%v+%v", meta.Erasure.DataBlocks, meta.Erasure.ParityBlocks)
 			fmt.Fprintf(h, "%v", meta.Erasure.Distribution)

 			// ILM transition fields
@@ -412,8 +417,8 @@ func pickValidDiskTimeWithQuorum(metaArr []FileInfo, quorum int) time.Time {

 // pickValidFileInfo - picks one valid FileInfo content and returns from a
 // slice of FileInfo.
-func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Time, quorum int) (FileInfo, error) {
-	return findFileInfoInQuorum(ctx, metaArr, modTime, quorum)
+func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Time, etag string, quorum int) (FileInfo, error) {
+	return findFileInfoInQuorum(ctx, metaArr, modTime, etag, quorum)
 }

 // writeUniqueFileInfo - writes unique `xl.meta` content for each disk concurrently.
@@ -444,22 +449,35 @@ func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, bucket, prefix
 	return evalDisks(disks, mErrs), err
 }

-func commonParity(parities []int) int {
+func commonParity(parities []int, defaultParityCount int) int {
+	N := len(parities)
+
 	occMap := make(map[int]int)
 	for _, p := range parities {
 		occMap[p]++
 	}

-	var maxOcc, commonParity int
-
+	var maxOcc, cparity int
 	for parity, occ := range occMap {
 		if parity == -1 {
 			// Ignore non defined parity
 			continue
 		}
-		if occ >= maxOcc {
+
+		readQuorum := N - parity
+		if defaultParityCount > 0 && parity == 0 {
+			// In this case, parity == 0 implies that this object version is a
+			// delete marker
+			readQuorum = N/2 + 1
+		}
+		if occ < readQuorum {
+			// Ignore this parity since we don't have enough shards for read quorum
+			continue
+		}
+
+		if occ > maxOcc {
 			maxOcc = occ
-			commonParity = parity
+			cparity = parity
 		}
 	}

@@ -467,7 +485,7 @@ func commonParity(parities []int) int {
 		// Did not found anything useful
 		return -1
 	}
-	return commonParity
+	return cparity
 }

 func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int) {
@@ -481,7 +499,9 @@ func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int)
 			parities[index] = -1
 			continue
 		}
-		if !metadata.Deleted {
+		if metadata.Deleted {
+			parities[index] = len(partsMetadata) / 2
+		} else {
 			parities[index] = metadata.Erasure.ParityBlocks
 		}
 	}
@@ -510,8 +530,7 @@ func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []
 	}

 	parities := listObjectParities(partsMetaData, errs)
-	parityBlocks := commonParity(parities)
-
+	parityBlocks := commonParity(parities, defaultParityCount)
 	if parityBlocks < 0 {
 		return -1, -1, errErasureReadQuorum
 	}
@@ -616,11 +635,9 @@ func getInternalReplicationState(m map[string]string) ReplicationState {
 	for k, v := range m {
 		switch {
 		case equals(k, ReservedMetadataPrefixLower+ReplicationTimestamp):
-			tm, _ := time.Parse(http.TimeFormat, v)
-			d.ReplicationTimeStamp = tm
+			d.ReplicaTimeStamp, _ = amztime.ParseReplicationTS(v)
 		case equals(k, ReservedMetadataPrefixLower+ReplicaTimestamp):
-			tm, _ := time.Parse(http.TimeFormat, v)
-			d.ReplicaTimeStamp = tm
+			d.ReplicaTimeStamp, _ = amztime.ParseReplicationTS(v)
 		case equals(k, ReservedMetadataPrefixLower+ReplicaStatus):
 			d.ReplicaStatus = replication.StatusType(v)
 		case equals(k, ReservedMetadataPrefixLower+ReplicationStatus):
@@ -204,7 +204,7 @@ func TestFindFileInfoInQuorum(t *testing.T) {
 	for _, test := range tests {
 		test := test
 		t.Run("", func(t *testing.T) {
-			_, err := findFileInfoInQuorum(context.Background(), test.fis, test.modTime, test.expectedQuorum)
+			_, err := findFileInfoInQuorum(context.Background(), test.fis, test.modTime, "", test.expectedQuorum)
 			if err != test.expectedErr {
 				t.Errorf("Expected %s, got %s", test.expectedErr, err)
 			}
@@ -37,8 +37,9 @@ import (
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/mimedb"
+	"github.com/minio/pkg/sync/errgroup"
+	uatomic "go.uber.org/atomic"
 )

 func (er erasureObjects) getUploadIDDir(bucket, object, uploadID string) string {
@@ -78,28 +79,27 @@ func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object
 		return fi, nil, err
 	}

-	// List all online disks.
-	_, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
+	quorum := readQuorum
+	if write {
+		quorum = writeQuorum
+	}
+
+	// List all online disks.
+	_, modTime, etag := listOnlineDisks(storageDisks, partsMetadata, errs, quorum)

-	var quorum int
 	if write {
 		reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-		if reducedErr == errErasureWriteQuorum {
+		if errors.Is(reducedErr, errErasureWriteQuorum) {
 			return fi, nil, reducedErr
 		}
-
-		quorum = writeQuorum
 	} else {
 		if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil {
 			return fi, nil, reducedErr
 		}
-
-		// Pick one from the first valid metadata.
-		quorum = readQuorum
 	}

 	// Pick one from the first valid metadata.
-	fi, err = pickValidFileInfo(ctx, partsMetadata, modTime, quorum)
+	fi, err = pickValidFileInfo(ctx, partsMetadata, modTime, etag, quorum)

 	return fi, partsMetadata, err
 }
@@ -199,20 +199,31 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 	diskPath := disk.Endpoint().Path

 	readDirFn(pathJoin(diskPath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error {
-		return readDirFn(pathJoin(diskPath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error {
+		readDirFn(pathJoin(diskPath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error {
 			uploadIDPath := pathJoin(shaDir, uploadIDDir)
 			fi, err := disk.ReadVersion(ctx, minioMetaMultipartBucket, uploadIDPath, "", false)
 			if err != nil {
-				er.deleteAll(ctx, minioMetaMultipartBucket, uploadIDPath)
 				return nil
 			}
 			wait := deletedCleanupSleeper.Timer(ctx)
 			if now.Sub(fi.ModTime) > expiry {
-				er.deleteAll(ctx, minioMetaMultipartBucket, uploadIDPath)
+				removeAll(pathJoin(diskPath, minioMetaMultipartBucket, uploadIDPath))
 			}
 			wait()
 			return nil
 		})
+		vi, err := disk.StatVol(ctx, pathJoin(minioMetaMultipartBucket, shaDir))
+		if err != nil {
+			return nil
+		}
+		wait := deletedCleanupSleeper.Timer(ctx)
+		if now.Sub(vi.Created) > expiry {
+			// We are not deleting shaDir recursively here, if shaDir is empty
+			// and its older then we can happily delete it.
+			Remove(pathJoin(diskPath, minioMetaMultipartBucket, shaDir))
+		}
+		wait()
+		return nil
 	})

 	readDirFn(pathJoin(diskPath, minioMetaTmpBucket), func(tmpDir string, typ os.FileMode) error {
@@ -225,7 +236,7 @@ func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk Sto
 		}
 		wait := deletedCleanupSleeper.Timer(ctx)
 		if now.Sub(vi.Created) > expiry {
-			er.deleteAll(ctx, minioMetaTmpBucket, tmpDir)
+			removeAll(pathJoin(diskPath, minioMetaTmpBucket, tmpDir))
 		}
 		wait()
 		return nil
@@ -363,29 +374,61 @@ func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string,
 		userDefined["etag"] = opts.PreserveETag
 	}
 	onlineDisks := er.getDisks()
+
+	// Get parity and data drive count based on storage class metadata
 	parityDrives := globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass])
 	if parityDrives < 0 {
 		parityDrives = er.defaultParityCount
 	}

+	// If we have offline disks upgrade the number of erasure codes for this object.
 	parityOrig := parityDrives
+
+	atomicParityDrives := uatomic.NewInt64(0)
+	atomicOfflineDrives := uatomic.NewInt64(0)
+
+	// Start with current parityDrives
+	atomicParityDrives.Store(int64(parityDrives))
+
+	var wg sync.WaitGroup
 	for _, disk := range onlineDisks {
-		if parityDrives >= len(onlineDisks)/2 {
-			parityDrives = len(onlineDisks) / 2
-			break
-		}
 		if disk == nil {
-			parityDrives++
+			atomicParityDrives.Inc()
+			atomicOfflineDrives.Inc()
 			continue
 		}
-		di, err := disk.DiskInfo(ctx)
-		if err != nil || di.ID == "" {
-			parityDrives++
+		if !disk.IsOnline() {
+			atomicParityDrives.Inc()
+			atomicOfflineDrives.Inc()
+			continue
 		}
+		wg.Add(1)
+		go func(disk StorageAPI) {
+			defer wg.Done()
+			di, err := disk.DiskInfo(ctx)
+			if err != nil || di.ID == "" {
+				atomicOfflineDrives.Inc()
+				atomicParityDrives.Inc()
+			}
+		}(disk)
+	}
+	wg.Wait()
+
+	if int(atomicOfflineDrives.Load()) >= (len(onlineDisks)+1)/2 {
+		// if offline drives are more than 50% of the drives
+		// we have no quorum, we shouldn't proceed just
+		// fail at that point.
+		return nil, toObjectErr(errErasureWriteQuorum, bucket, object)
+	}
+
+	parityDrives = int(atomicParityDrives.Load())
+	if parityDrives >= len(onlineDisks)/2 {
+		parityDrives = len(onlineDisks) / 2
 	}
 	if parityOrig != parityDrives {
 		userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)
 	}
+
 	dataDrives := len(onlineDisks) - parityDrives

 	// we now know the number of blocks this object needs for data and parity.
@@ -458,21 +501,6 @@ func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object
 	return er.newMultipartUpload(ctx, bucket, object, opts)
 }

-// CopyObjectPart - reads incoming stream and internally erasure codes
-// them. This call is similar to put object part operation but the source
-// data is read from an existing object.
-//
-// Implements S3 compatible Upload Part Copy API.
-func (er erasureObjects) CopyObjectPart(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (pi PartInfo, e error) {
-	partInfo, err := er.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, NewPutObjReader(srcInfo.Reader), dstOpts)
-	if err != nil {
-		return pi, toObjectErr(err, dstBucket, dstObject)
-	}
-
-	// Success.
-	return partInfo, nil
-}
-
 // renamePart - renames multipart part to its relevant location under uploadID.
 func renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, error) {
 	g := errgroup.WithNErrs(len(disks))
@@ -522,7 +550,7 @@ func writeAllDisks(ctx context.Context, disks []StorageAPI, dstBucket, dstEntry
 	// We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum
 	// otherwise return failure. Cleanup successful renames.
 	err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
-	if err == errErasureWriteQuorum {
+	if errors.Is(err, errErasureWriteQuorum) {
 		// Remove all written
 		g := errgroup.WithNErrs(len(disks))
 		for index := range disks {
@@ -593,7 +621,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 			return pi, InvalidArgument{
 				Bucket: bucket,
 				Object: fi.Name,
-				Err:    fmt.Errorf("checksum missing, want %s, got %s", cs, r.ContentCRCType().String()),
+				Err:    fmt.Errorf("checksum missing, want %q, got %q", cs, r.ContentCRCType().String()),
 			}
 		}
 	}
@@ -626,7 +654,8 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
 	case size == -1:
 		if size := data.ActualSize(); size > 0 && size < fi.Erasure.BlockSize {
-			buffer = make([]byte, data.ActualSize()+256, data.ActualSize()*2+512)
+			// Account for padding and forced compression overhead and encryption.
+			buffer = make([]byte, data.ActualSize()+256+32+32, data.ActualSize()*2+512)
 		} else {
 			buffer = er.bp.Get()
 			defer er.bp.Put(buffer)
@@ -709,6 +738,7 @@ func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uplo
 		Index:      index,
 		Checksums:  r.ContentCRC(),
 	}
+
 	fi.Parts = []ObjectPartInfo{partInfo}
 	partFI, err := fi.MarshalMsg(nil)
 	if err != nil {
@@ -1065,7 +1095,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str

 		// ensure that part ETag is canonicalized to strip off extraneous quotes
 		part.ETag = canonicalizeETag(part.ETag)
-		expETag := tryDecryptETag(objectEncryptionKey, expPart.ETag, kind != crypto.S3)
+		expETag := tryDecryptETag(objectEncryptionKey, expPart.ETag, kind == crypto.S3)
 		if expETag != part.ETag {
 			invp := InvalidPart{
 				PartNumber: part.PartNumber,
@@ -1137,8 +1167,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		}
 	}
 	if checksumType.IsSet() {
+		checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart
 		cs := hash.NewChecksumFromData(checksumType, checksumCombined)
-		fi.Checksum = cs.AppendTo(nil)
+		fi.Checksum = cs.AppendTo(nil, checksumCombined)
 		if opts.EncryptFn != nil {
 			fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
 		}
@@ -1226,7 +1257,7 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 		if disk != nil && disk.IsOnline() {
 			continue
 		}
-		er.addPartial(bucket, object, fi.VersionID, fi.Size)
+		er.addPartial(bucket, object, fi.VersionID)
 		break
 	}

@@ -29,6 +29,7 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"time"

 	"github.com/klauspost/readahead"
 	"github.com/minio/madmin-go/v2"
@@ -36,13 +37,14 @@ import (
 	"github.com/minio/minio/internal/bucket/lifecycle"
 	"github.com/minio/minio/internal/bucket/object/lock"
 	"github.com/minio/minio/internal/bucket/replication"
+	"github.com/minio/minio/internal/crypto"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
-	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/mimedb"
+	"github.com/minio/pkg/sync/errgroup"
 	"github.com/minio/pkg/wildcard"
 	uatomic "go.uber.org/atomic"
 )
@@ -106,10 +108,10 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d
 	}

 	// List all online disks.
-	onlineDisks, modTime := listOnlineDisks(storageDisks, metaArr, errs)
+	onlineDisks, modTime, etag := listOnlineDisks(storageDisks, metaArr, errs, readQuorum)

 	// Pick latest valid metadata.
-	fi, err := pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
+	fi, err := pickValidFileInfo(ctx, metaArr, modTime, etag, readQuorum)
 	if err != nil {
 		return oi, toObjectErr(err, srcBucket, srcObject)
 	}
@@ -183,7 +185,7 @@ func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, d

 // GetObjectNInfo - returns object info and an object
 // Read(Closer). When err != nil, the returned reader is always nil.
-func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, lockType LockType, opts ObjectOptions) (gr *GetObjectReader, err error) {
+func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) {
 	auditObjectErasureSet(ctx, object, &er)

 	// This is a special call attempted first to check for SOS-API calls.
@@ -203,25 +205,26 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 	}()

 	// Acquire lock
-	if lockType != noLock {
+	if !opts.NoLock {
 		lock := er.NewNSLock(bucket, object)
-		switch lockType {
-		case writeLock:
-			lkctx, err := lock.GetLock(ctx, globalOperationTimeout)
-			if err != nil {
-				return nil, err
-			}
-			ctx = lkctx.Context()
-			nsUnlocker = func() { lock.Unlock(lkctx) }
-		case readLock:
-			lkctx, err := lock.GetRLock(ctx, globalOperationTimeout)
-			if err != nil {
-				return nil, err
-			}
-			ctx = lkctx.Context()
-			nsUnlocker = func() { lock.RUnlock(lkctx) }
+		lkctx, err := lock.GetRLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return nil, err
 		}
+		ctx = lkctx.Context()
+
+		// Release lock when the metadata is verified, and reader
+		// is ready to be read.
+		//
+		// This is possible to be lock free because
+		// - xl.meta for inlined objects has already read the data
+		//   into memory, any mutation on xl.meta subsequently is
+		//   inconsequential to the overall read operation.
+		// - xl.meta metadata is still verified for quorum under lock()
+		//   however writing the response doesn't need to serialize
+		//   concurrent writers
 		unlockOnDefer = true
+		nsUnlocker = func() { lock.RUnlock(lkctx) }
 	}

 	fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true)
@@ -265,6 +268,7 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 			ObjInfo: objInfo,
 		}, toObjectErr(errMethodNotAllowed, bucket, object)
 	}
+
 	if objInfo.IsRemote() {
 		gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, h, objInfo, opts)
 		if err != nil {
@@ -278,7 +282,10 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 	if err != nil {
 		return nil, err
 	}
-	unlockOnDefer = false
+
+	if unlockOnDefer {
+		unlockOnDefer = fi.InlineData()
+	}

 	pr, pw := xioutil.WaitPipe()
 	go func() {
@@ -291,7 +298,11 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		pr.CloseWithError(nil)
 	}

-	return fn(pr, h, pipeCloser, nsUnlocker)
+	if !unlockOnDefer {
+		return fn(pr, h, pipeCloser, nsUnlocker)
+	}
+
+	return fn(pr, h, pipeCloser)
 }

 func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI) error {
@@ -336,11 +347,6 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje

 	var healOnce sync.Once

-	// once we have obtained a common FileInfo i.e latest, we should stick
-	// to single dataDir to read the content to avoid reading from some other
-	// dataDir that has stale FileInfo{} to ensure that we fail appropriately
-	// during reads and expect the same dataDir everywhere.
-	dataDir := fi.DataDir
 	for ; partIndex <= lastPartIndex; partIndex++ {
 		if length == totalBytesRead {
 			break
@@ -368,8 +374,11 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 			if !metaArr[index].IsValid() {
 				continue
 			}
+			if !metaArr[index].Erasure.Equal(fi.Erasure) {
+				continue
+			}
 			checksumInfo := metaArr[index].Erasure.GetChecksumInfo(partNumber)
-			partPath := pathJoin(object, dataDir, fmt.Sprintf("part.%d", partNumber))
+			partPath := pathJoin(object, metaArr[index].DataDir, fmt.Sprintf("part.%d", partNumber))
 			readers[index] = newBitrotReader(disk, metaArr[index].Data, bucket, partPath, tillOffset,
 				checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize())

@@ -432,6 +441,14 @@ func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, obje
 func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) {
 	auditObjectErasureSet(ctx, object, &er)

+	// This is a special call attempted first to check for SOS-API calls.
+	info, err = veeamSOSAPIHeadObject(ctx, bucket, object, opts)
+	if err == nil {
+		return info, nil
+	}
+	// reset any error to 'nil'
+	err = nil
+
 	if !opts.NoLock {
 		// Lock the object before reading.
 		lk := er.NewNSLock(bucket, object)
@@ -542,6 +559,8 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		errVolumeNotFound,
 		errFileVersionNotFound,
 		errDiskNotFound,
+		io.ErrUnexpectedEOF, // some times we would read without locks, ignore these errors
+		io.EOF,              // some times we would read without locks, ignore these errors
 	}

 	errs := g.Wait()
@@ -549,13 +568,6 @@ func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, r
 		if err == nil {
 			continue
 		}
-		if bucket == minioMetaBucket {
-			// minioMetaBucket "reads" for .metacache are not written with O_SYNC
-			// so there is a potential for them to not fully committed to stable
-			// storage leading to unexpected EOFs. Allow these failures to
-			// be ignored since the caller already ignores them in streamMetadataParts()
-			ignoredErrs = append(ignoredErrs, io.ErrUnexpectedEOF, io.EOF)
-		}
 		if !IsErr(err, ignoredErrs...) {
 			logger.LogOnceIf(ctx, fmt.Errorf("Drive %s, path (%s/%s) returned an error (%w)",
 				disks[index], bucket, object, err),
@@ -650,10 +662,10 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 	}

 	// List all online disks.
-	onlineDisks, modTime := listOnlineDisks(disks, metaArr, errs)
+	onlineDisks, modTime, etag := listOnlineDisks(disks, metaArr, errs, readQuorum)

 	// Pick latest valid metadata.
-	fi, err = pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
+	fi, err = pickValidFileInfo(ctx, metaArr, modTime, etag, readQuorum)
 	if err != nil {
 		return fi, nil, nil, err
 	}
@@ -679,7 +691,7 @@ func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object s
 			missingBlocks++
 			continue
 		}
-		if metaArr[i].IsValid() && metaArr[i].ModTime.Equal(fi.ModTime) {
+		if metaArr[i].IsValid() && metaArr[i].ModTime.Equal(modTime) {
 			continue
 		}
 		metaArr[i] = FileInfo{}
@@ -996,17 +1008,34 @@ func healObjectVersionsDisparity(bucket string, entry metaCacheEntry) error {
 func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) {
 	auditObjectErasureSet(ctx, object, &er)

+	data := r.Reader
+
 	if opts.CheckPrecondFn != nil {
-		obj, err := er.getObjectInfo(ctx, bucket, object, opts)
-		if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) {
-			return objInfo, err
+		if !opts.NoLock {
+			ns := er.NewNSLock(bucket, object)
+			lkctx, err := ns.GetLock(ctx, globalOperationTimeout)
+			if err != nil {
+				return ObjectInfo{}, err
+			}
+			ctx = lkctx.Context()
+			defer ns.Unlock(lkctx)
+			opts.NoLock = true
 		}
-		if opts.CheckPrecondFn(obj) {
+
+		obj, err := er.getObjectInfo(ctx, bucket, object, opts)
+		if err == nil && opts.CheckPrecondFn(obj) {
 			return objInfo, PreConditionFailed{}
 		}
+		if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) && !isErrReadQuorum(err) {
+			return objInfo, err
+		}
 	}

-	data := r.Reader
+	// Validate input data size and it can never be less than -1.
+	if data.Size() < -1 {
+		logger.LogIf(ctx, errInvalidArgument, logger.Application)
+		return ObjectInfo{}, toObjectErr(errInvalidArgument)
+	}

 	userDefined := cloneMSS(opts.UserDefined)

@@ -1024,6 +1053,8 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		parityOrig := parityDrives

 		atomicParityDrives := uatomic.NewInt64(0)
+		atomicOfflineDrives := uatomic.NewInt64(0)
+
 		// Start with current parityDrives
 		atomicParityDrives.Store(int64(parityDrives))

@@ -1031,10 +1062,12 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		for _, disk := range storageDisks {
 			if disk == nil {
 				atomicParityDrives.Inc()
+				atomicOfflineDrives.Inc()
 				continue
 			}
 			if !disk.IsOnline() {
 				atomicParityDrives.Inc()
+				atomicOfflineDrives.Inc()
 				continue
 			}
 			wg.Add(1)
@@ -1042,12 +1075,20 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 				defer wg.Done()
 				di, err := disk.DiskInfo(ctx)
 				if err != nil || di.ID == "" {
+					atomicOfflineDrives.Inc()
 					atomicParityDrives.Inc()
 				}
 			}(disk)
 		}
 		wg.Wait()

+		if int(atomicOfflineDrives.Load()) >= (len(storageDisks)+1)/2 {
+			// if offline drives are more than 50% of the drives
+			// we have no quorum, we shouldn't proceed just
+			// fail at that point.
+			return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object)
+		}
+
 		parityDrives = int(atomicParityDrives.Load())
 		if parityDrives >= len(storageDisks)/2 {
 			parityDrives = len(storageDisks) / 2
@@ -1065,12 +1106,6 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 		writeQuorum++
 	}

-	// Validate input data size and it can never be less than zero.
-	if data.Size() < -1 {
-		logger.LogIf(ctx, errInvalidArgument, logger.Application)
-		return ObjectInfo{}, toObjectErr(errInvalidArgument)
-	}
-
 	// Initialize parts metadata
 	partsMetadata := make([]FileInfo, len(storageDisks))

@@ -1081,7 +1116,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}

 	fi.DataDir = mustGetUUID()
-	fi.Checksum = opts.WantChecksum.AppendTo(nil)
+	fi.Checksum = opts.WantChecksum.AppendTo(nil, nil)
 	if opts.EncryptFn != nil {
 		fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum)
 	}
@@ -1238,8 +1273,16 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 	}

 	userDefined["etag"] = r.MD5CurrentHexString()
+	kind, _ := crypto.IsEncrypted(userDefined)
 	if opts.PreserveETag != "" {
-		userDefined["etag"] = opts.PreserveETag
+		if !opts.ReplicationRequest {
+			userDefined["etag"] = opts.PreserveETag
+		} else if kind != crypto.S3 {
+			// if we have a replication request
+			// and SSE-S3 is specified do not preserve
+			// the incoming etag.
+			userDefined["etag"] = opts.PreserveETag
+		}
 	}

 	// Guess content-type from the extension if possible.
@@ -1290,7 +1333,7 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 				continue
 			}

-			er.addPartial(bucket, object, fi.VersionID, fi.Size)
+			er.addPartial(bucket, object, fi.VersionID)
 			break
 		}

@@ -1482,7 +1525,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		if errs[i] != nil && !isErrVersionNotFound(errs[i]) && !isErrObjectNotFound(errs[i]) {
 			// all other direct versionId references we should
 			// ensure no dangling file is left over.
-			er.addPartial(bucket, dobj.ObjectName, dobj.VersionID, -1)
+			er.addPartial(bucket, dobj.ObjectName, dobj.VersionID)
 			continue
 		}

@@ -1495,7 +1538,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec

 			// all other direct versionId references we should
 			// ensure no dangling file is left over.
-			er.addPartial(bucket, dobj.ObjectName, dobj.VersionID, -1)
+			er.addPartial(bucket, dobj.ObjectName, dobj.VersionID)
 			break
 		}
 	}
@@ -1658,7 +1701,7 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 			if disk != nil && disk.IsOnline() {
 				continue
 			}
-			er.addPartial(bucket, object, opts.VersionID, -1)
+			er.addPartial(bucket, object, opts.VersionID)
 			break
 		}
 	}()
@@ -1715,14 +1758,12 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string

 // Send the successful but partial upload/delete, however ignore
 // if the channel is blocked by other items.
-func (er erasureObjects) addPartial(bucket, object, versionID string, size int64) {
+func (er erasureObjects) addPartial(bucket, object, versionID string) {
 	globalMRFState.addPartialOp(partialOperation{
 		bucket:    bucket,
 		object:    object,
 		versionID: versionID,
-		size:      size,
-		setIndex:  er.setIndex,
-		poolIndex: er.poolIndex,
+		queued:    time.Now(),
 	})
 }

@@ -1762,10 +1803,10 @@ func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object s
 	}

 	// List all online disks.
-	onlineDisks, modTime := listOnlineDisks(disks, metaArr, errs)
+	onlineDisks, modTime, etag := listOnlineDisks(disks, metaArr, errs, readQuorum)

 	// Pick latest valid metadata.
-	fi, err := pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
+	fi, err := pickValidFileInfo(ctx, metaArr, modTime, etag, readQuorum)
 	if err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
@@ -1835,10 +1876,10 @@ func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object strin
 	}

 	// List all online disks.
-	onlineDisks, modTime := listOnlineDisks(disks, metaArr, errs)
+	onlineDisks, modTime, etag := listOnlineDisks(disks, metaArr, errs, readQuorum)

 	// Pick latest valid metadata.
-	fi, err := pickValidFileInfo(ctx, metaArr, modTime, readQuorum)
+	fi, err := pickValidFileInfo(ctx, metaArr, modTime, etag, readQuorum)
 	if err != nil {
 		return ObjectInfo{}, toObjectErr(err, bucket, object)
 	}
@@ -1989,7 +2030,7 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		if disk != nil && disk.IsOnline() {
 			continue
 		}
-		er.addPartial(bucket, object, opts.VersionID, -1)
+		er.addPartial(bucket, object, opts.VersionID)
 		break
 	}

@@ -2001,7 +2042,8 @@ func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object st
 		UserAgent:  "Internal: [ILM-Transition]",
 		Host:       globalLocalNodeName,
 	})
-	auditLogLifecycle(ctx, objInfo, ILMTransition, traceFn)
+	tags := opts.LifecycleAuditEvent.Tags()
+	auditLogLifecycle(ctx, objInfo, ILMTransition, tags, traceFn)
 	return err
 }

@@ -2084,7 +2126,7 @@ func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket s

 	// rehydrate the parts back on disk as per the original xl.meta prior to transition
 	for _, partInfo := range oi.Parts {
-		hr, err := hash.NewReader(gr, partInfo.Size, "", "", partInfo.Size)
+		hr, err := hash.NewReader(io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size)
 		if err != nil {
 			return setRestoreHeaderFn(oi, err)
 		}
@@ -532,7 +532,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 		}
 	}

-	gr, err := xl.GetObjectNInfo(ctx, bucket, object, nil, nil, readLock, opts)
+	gr, err := xl.GetObjectNInfo(ctx, bucket, object, nil, nil, opts)
 	if err != nil {
 		if err != toObjectErr(errErasureReadQuorum, bucket, object) {
 			t.Errorf("Expected GetObject to fail with %v, but failed with %v", toObjectErr(errErasureReadQuorum, bucket, object), err)
@@ -577,7 +577,7 @@ func TestGetObjectNoQuorum(t *testing.T) {
 		}
 		z.serverPools[0].erasureDisksMu.Unlock()
 		// Fetch object from store.
-		gr, err := xl.GetObjectNInfo(ctx, bucket, object, nil, nil, readLock, opts)
+		gr, err := xl.GetObjectNInfo(ctx, bucket, object, nil, nil, opts)
 		if err != nil {
 			if err != toObjectErr(errErasureReadQuorum, bucket, object) {
 				t.Errorf("Expected GetObject to fail with %v, but failed with %v", toObjectErr(errErasureReadQuorum, bucket, object), err)
@@ -831,7 +831,7 @@ func TestPutObjectSmallInlineData(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	gr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, readLock, ObjectOptions{})
+	gr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Expected GetObject to succeed, but failed with %v", err)
 	}
@@ -855,7 +855,7 @@ func TestPutObjectSmallInlineData(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	gr, err = obj.GetObjectNInfo(ctx, bucket, object, nil, nil, readLock, ObjectOptions{})
+	gr, err = obj.GetObjectNInfo(ctx, bucket, object, nil, nil, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Expected GetObject to succeed, but failed with %v", err)
 	}
@@ -895,14 +895,14 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin

 	// Object for test case 1 - No StorageClass defined, no MetaData in PutObject
 	object1 := "object1"
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		RRS: storageclass.StorageClass{
 			Parity: 2,
 		},
 		Standard: storageclass.StorageClass{
 			Parity: 4,
 		},
-	}
+	})
 	_, err = obj.PutObject(ctx, bucket, object1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
 	if err != nil {
 		t.Fatalf("Failed to putObject %v", err)
@@ -939,11 +939,11 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object4 := "object4"
 	metadata4 := make(map[string]string)
 	metadata4["x-amz-storage-class"] = storageclass.STANDARD
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 6,
 		},
-	}
+	})

 	_, err = obj.PutObject(ctx, bucket, object4, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata4})
 	if err != nil {
@@ -962,11 +962,11 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object5 := "object5"
 	metadata5 := make(map[string]string)
 	metadata5["x-amz-storage-class"] = storageclass.RRS
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		RRS: storageclass.StorageClass{
 			Parity: 2,
 		},
-	}
+	})

 	_, err = obj.PutObject(ctx, bucket, object5, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata5})
 	if err != nil {
@@ -980,14 +980,14 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object6 := "object6"
 	metadata6 := make(map[string]string)
 	metadata6["x-amz-storage-class"] = storageclass.STANDARD
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 4,
 		},
 		RRS: storageclass.StorageClass{
 			Parity: 2,
 		},
-	}
+	})

 	_, err = obj.PutObject(ctx, bucket, object6, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata6})
 	if err != nil {
@@ -1006,11 +1006,11 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	object7 := "object7"
 	metadata7 := make(map[string]string)
 	metadata7["x-amz-storage-class"] = storageclass.STANDARD
-	globalStorageClass = storageclass.Config{
+	globalStorageClass.Update(storageclass.Config{
 		Standard: storageclass.StorageClass{
 			Parity: 5,
 		},
-	}
+	})

 	_, err = obj.PutObject(ctx, bucket, object7, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata7})
 	if err != nil {
@@ -1043,7 +1043,7 @@ func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []strin
 	for _, tt := range tests {
 		tt := tt
 		t.(*testing.T).Run("", func(t *testing.T) {
-			globalStorageClass = tt.storageClassCfg
+			globalStorageClass.Update(tt.storageClassCfg)
 			actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(ctx, tt.parts, tt.errs, storageclass.DefaultParityBlocks(len(erasureDisks)))
 			if tt.expectedError != nil && err == nil {
 				t.Errorf("Expected %s, got %s", tt.expectedError, err)
@@ -1099,7 +1099,7 @@ func TestGetObjectInlineNotInline(t *testing.T) {
 	}

 	// Try to read the object and check its md5sum
-	gr, err := objLayer.GetObjectNInfo(ctx, "testbucket", "file", nil, nil, readLock, ObjectOptions{})
+	gr, err := objLayer.GetObjectNInfo(ctx, "testbucket", "file", nil, nil, ObjectOptions{})
 	if err != nil {
 		t.Fatalf("Expected GetObject to succeed, but failed with %v", err)
 	}
@@ -1193,7 +1193,7 @@ func TestGetObjectWithOutdatedDisks(t *testing.T) {
 		sets.erasureDisksMu.Lock()
 		xl.getDisks = func() []StorageAPI { return origErasureDisks }
 		sets.erasureDisksMu.Unlock()
-		gr, err := z.GetObjectNInfo(ctx, testCase.bucket, testCase.object, nil, nil, readLock, ObjectOptions{VersionID: got.VersionID})
+		gr, err := z.GetObjectNInfo(ctx, testCase.bucket, testCase.object, nil, nil, ObjectOptions{VersionID: got.VersionID})
 		if err != nil {
 			t.Fatalf("Expected GetObject to succeed, but failed with %v", err)
 		}
@@ -1,4 +1,4 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
+// Copyright (c) 2015-2023 MinIO, Inc.
 //
 // This file is part of MinIO Object Storage stack
 //
@@ -22,12 +22,12 @@ import (
 	"encoding/binary"
 	"errors"
 	"fmt"
+	"io"
 	"math/rand"
 	"net/http"
 	"sort"
 	"strconv"
 	"strings"
-	"sync"
 	"time"

 	"github.com/dustin/go-humanize"
@@ -36,6 +36,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/console"
 	"github.com/minio/pkg/env"
+	"github.com/minio/pkg/workers"
 )

 // PoolDecommissionInfo currently decommissioning information
@@ -300,81 +301,31 @@ func (p *poolMeta) validate(pools []*erasureSets) (bool, error) {
 		specifiedPools[pool.endpoints.CmdLine] = idx
 	}

-	replaceScheme := func(k string) string {
-		// This is needed as fallback when users are updating
-		// from http->https or https->http, we need to verify
-		// both because MinIO remembers the command-line in
-		// "exact" order - as long as this order is not disturbed
-		// we allow changing the "scheme" i.e internode communication
-		// from plain-text to TLS or from TLS to plain-text.
-		if strings.HasPrefix(k, "http://") {
-			k = strings.ReplaceAll(k, "http://", "https://")
-		} else if strings.HasPrefix(k, "https://") {
-			k = strings.ReplaceAll(k, "https://", "http://")
-		}
-		return k
-	}
-
 	var update bool
-	// Check if specified pools need to remove decommissioned pool.
+	// Check if specified pools need to be removed from decommissioned pool.
 	for k := range specifiedPools {
 		pi, ok := rememberedPools[k]
 		if !ok {
-			pi, ok = rememberedPools[replaceScheme(k)]
-			if ok {
-				update = true // Looks like user is changing from http->https or https->http
-			}
+			// we do not have the pool anymore that we previously remembered, since all
+			// the CLI checks out we can allow updates since we are mostly adding a pool here.
+			update = true
 		}
 		if ok && pi.completed {
 			return false, fmt.Errorf("pool(%s) = %s is decommissioned, please remove from server command line", humanize.Ordinal(pi.position+1), k)
 		}
 	}

-	// check if remembered pools are in right position or missing from command line.
-	for k, pi := range rememberedPools {
-		if pi.completed {
-			continue
-		}
-		_, ok := specifiedPools[k]
-		if !ok {
-			_, ok = specifiedPools[replaceScheme(k)]
-			if ok {
-				update = true // Looks like user is changing from http->https or https->http
-			}
-		}
-		if !ok {
-			update = true
-		}
-	}
-
-	// check when remembered pools and specified pools are same they are at the expected position
-	if len(rememberedPools) == len(specifiedPools) {
+	if len(specifiedPools) == len(rememberedPools) {
 		for k, pi := range rememberedPools {
 			pos, ok := specifiedPools[k]
-			if !ok {
-				pos, ok = specifiedPools[replaceScheme(k)]
-				if ok {
-					update = true // Looks like user is changing from http->https or https->http
-				}
-			}
-			if !ok {
-				update = true
-			}
 			if ok && pos != pi.position {
-				return false, fmt.Errorf("pool order change detected for %s, expected position is (%s) but found (%s)", k, humanize.Ordinal(pi.position+1), humanize.Ordinal(pos+1))
+				update = true // pool order is changing, its okay to allow it.
 			}
 		}
 	}

 	if !update {
-		update = len(rememberedPools) != len(specifiedPools)
-	}
-	if update {
-		for k, pi := range rememberedPools {
-			if pi.decomStarted && !pi.completed {
-				return false, fmt.Errorf("pool(%s) = %s is being decommissioned, No changes should be made to the command line arguments. Please complete the decommission in progress", humanize.Ordinal(pi.position+1), k)
-			}
-		}
+		update = len(specifiedPools) != len(rememberedPools)
 	}

 	return update, nil
@@ -466,8 +417,9 @@ func (p poolMeta) save(ctx context.Context, pools []*erasureSets) error {
 	}

 	// Saves on all pools to make sure decommissioning of first pool is allowed.
-	for _, eset := range pools {
+	for i, eset := range pools {
 		if err = saveConfig(ctx, eset, poolMetaName, buf); err != nil {
+			logger.LogIf(ctx, fmt.Errorf("saving pool.bin for pool index %d failed with: %v", i, err))
 			return err
 		}
 	}
@@ -506,60 +458,59 @@ func (z *erasureServerPools) Init(ctx context.Context) error {
 	// if no update is needed return right away.
 	if !update {
 		z.poolMeta = meta
-
-		pools := meta.returnResumablePools()
-		poolIndices := make([]int, 0, len(pools))
-		for _, pool := range pools {
-			idx := globalEndpoints.GetPoolIdx(pool.CmdLine)
-			if idx == -1 {
-				return fmt.Errorf("unexpected state present for decommission status pool(%s) not found", pool.CmdLine)
-			}
-			poolIndices = append(poolIndices, idx)
+	} else {
+		meta = poolMeta{} // to update write poolMeta fresh.
+		// looks like new pool was added we need to update,
+		// or this is a fresh installation (or an existing
+		// installation with pool removed)
+		meta.Version = poolMetaVersion
+		for idx, pool := range z.serverPools {
+			meta.Pools = append(meta.Pools, PoolStatus{
+				CmdLine:    pool.endpoints.CmdLine,
+				ID:         idx,
+				LastUpdate: UTCNow(),
+			})
 		}
+		if err = meta.save(ctx, z.serverPools); err != nil {
+			return err
+		}
+		z.poolMeta = meta
+	}

-		if len(poolIndices) > 0 && globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
-			go func() {
-				r := rand.New(rand.NewSource(time.Now().UnixNano()))
-				for {
-					if err := z.Decommission(ctx, poolIndices...); err != nil {
-						if errors.Is(err, errDecommissionAlreadyRunning) {
-							// A previous decommission running found restart it.
-							for _, idx := range poolIndices {
-								z.doDecommissionInRoutine(ctx, idx)
-							}
-							return
+	pools := meta.returnResumablePools()
+	poolIndices := make([]int, 0, len(pools))
+	for _, pool := range pools {
+		idx := globalEndpoints.GetPoolIdx(pool.CmdLine)
+		if idx == -1 {
+			return fmt.Errorf("unexpected state present for decommission status pool(%s) not found", pool.CmdLine)
+		}
+		poolIndices = append(poolIndices, idx)
+	}
+
+	if len(poolIndices) > 0 && globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal {
+		go func() {
+			r := rand.New(rand.NewSource(time.Now().UnixNano()))
+			for {
+				if err := z.Decommission(ctx, poolIndices...); err != nil {
+					if errors.Is(err, errDecommissionAlreadyRunning) {
+						// A previous decommission running found restart it.
+						for _, idx := range poolIndices {
+							z.doDecommissionInRoutine(ctx, idx)
 						}
-						if configRetriableErrors(err) {
-							logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pools %v: %w: retrying..", pools, err))
-							time.Sleep(time.Second + time.Duration(r.Float64()*float64(5*time.Second)))
-							continue
-						}
-						logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pools, err))
 						return
 					}
+					if configRetriableErrors(err) {
+						logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pools %v: %w: retrying..", pools, err))
+						time.Sleep(time.Second + time.Duration(r.Float64()*float64(5*time.Second)))
+						continue
+					}
+					logger.LogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pools, err))
+					return
 				}
-			}()
-		}
-
-		return nil
+			}
+		}()
 	}

-	meta = poolMeta{} // to update write poolMeta fresh.
-	// looks like new pool was added we need to update,
-	// or this is a fresh installation (or an existing
-	// installation with pool removed)
-	meta.Version = poolMetaVersion
-	for idx, pool := range z.serverPools {
-		meta.Pools = append(meta.Pools, PoolStatus{
-			CmdLine:    pool.endpoints.CmdLine,
-			ID:         idx,
-			LastUpdate: UTCNow(),
-		})
-	}
-	if err = meta.save(ctx, z.serverPools); err != nil {
-		return err
-	}
-	z.poolMeta = meta
 	return nil
 }

@@ -595,7 +546,6 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 	if objInfo.isMultipart() {
 		res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{
 			VersionID:   objInfo.VersionID,
-			MTime:       objInfo.ModTime,
 			UserDefined: objInfo.UserDefined,
 		})
 		if err != nil {
@@ -604,7 +554,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, ObjectOptions{})
 		parts := make([]CompletePart, len(objInfo.Parts))
 		for i, part := range objInfo.Parts {
-			hr, err := hash.NewReader(gr, part.Size, "", "", part.ActualSize)
+			hr, err := hash.NewReader(io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize)
 			if err != nil {
 				return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 			}
@@ -638,7 +588,7 @@ func (z *erasureServerPools) decommissionObject(ctx context.Context, bucket stri
 		return err
 	}

-	hr, err := hash.NewReader(gr, objInfo.Size, "", "", actualSize)
+	hr, err := hash.NewReader(io.LimitReader(gr, objInfo.Size), objInfo.Size, "", "", actualSize)
 	if err != nil {
 		return fmt.Errorf("decommissionObject: hash.NewReader() %w", err)
 	}
@@ -708,26 +658,32 @@ func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi dec
 func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bi decomBucketInfo) error {
 	ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{})

-	var wg sync.WaitGroup
 	wStr := env.Get("_MINIO_DECOMMISSION_WORKERS", strconv.Itoa(len(pool.sets)))
 	workerSize, err := strconv.Atoi(wStr)
 	if err != nil {
 		return err
 	}

-	parallelWorkers := make(chan struct{}, workerSize)
+	// each set get its own thread separate from the concurrent
+	// objects/versions being decommissioned.
+	workerSize += len(pool.sets)
+
+	wk, err := workers.New(workerSize)
+	if err != nil {
+		return err
+	}
+
+	vc, _ := globalBucketVersioningSys.Get(bi.Name)
+
+	// Check if the current bucket has a configured lifecycle policy
+	lc, _ := globalLifecycleSys.Get(bi.Name)
+
+	// Check if bucket is object locked.
+	lr, _ := globalBucketObjectLockSys.Get(bi.Name)

 	for _, set := range pool.sets {
 		set := set

-		vc, _ := globalBucketVersioningSys.Get(bi.Name)
-
-		// Check if the current bucket has a configured lifecycle policy
-		lc, _ := globalLifecycleSys.Get(bi.Name)
-
-		// Check if bucket is object locked.
-		lr, _ := globalBucketObjectLockSys.Get(bi.Name)
-
 		filterLifecycle := func(bucket, object string, fi FileInfo) bool {
 			if lc == nil {
 				return false
@@ -738,10 +694,10 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
 			switch {
 			case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
-				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt.Action.DeleteRestored())
+				applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)
 				return false
 			case evt.Action.Delete():
-				globalExpiryState.enqueueByDays(objInfo, evt.Action.DeleteRestored(), evt.Action.DeleteVersioned())
+				globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Decom)
 				return true
 			default:
 				return false
@@ -749,10 +705,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 		}

 		decommissionEntry := func(entry metaCacheEntry) {
-			defer func() {
-				<-parallelWorkers
-				wg.Done()
-			}()
+			defer wk.Give()

 			if entry.isDir() {
 				return
@@ -771,34 +724,37 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			for _, version := range fivs.Versions {
 				// Apply lifecycle rules on the objects that are expired.
 				if filterLifecycle(bi.Name, version.Name, version) {
-					logger.LogIf(ctx, fmt.Errorf("found %s/%s (%s) expired object based on ILM rules, skipping and scheduled for deletion", bi.Name, version.Name, version.VersionID))
+					decommissionedCount++
 					continue
 				}

-				// We will skip decommissioning delete markers
-				// with single version, its as good as there
-				// is no data associated with the object.
+				// any object with only single DEL marker we don't need
+				// to decommission, just skip it.
 				if version.Deleted && len(fivs.Versions) == 1 {
-					logger.LogIf(ctx, fmt.Errorf("found %s/%s delete marked object with no other versions, skipping since there is no content left", bi.Name, version.Name))
+					decommissionedCount++
 					continue
 				}

+				versionID := version.VersionID
+				if versionID == "" {
+					versionID = nullVersionID
+				}
+
 				if version.Deleted {
 					_, err := z.DeleteObject(ctx,
 						bi.Name,
 						version.Name,
 						ObjectOptions{
-							Versioned:          vc.PrefixEnabled(version.Name),
-							VersionID:          version.VersionID,
+							// Since we are preserving a delete marker, we have to make sure this is always true.
+							// regardless of the current configuration of the bucket we must preserve all versions
+							// on the pool being decommissioned.
+							Versioned:          true,
+							VersionID:          versionID,
 							MTime:              version.ModTime,
 							DeleteReplication:  version.ReplicationState,
 							DeleteMarker:       true, // make sure we create a delete marker
 							SkipDecommissioned: true, // make sure we skip the decommissioned pool
 						})
-					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
-						// object/version already deleted by the application, nothing to do here we move on.
-						continue
-					}
 					var failure bool
 					if err != nil {
 						logger.LogIf(ctx, err)
@@ -820,7 +776,7 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 					if version.IsRemote() {
 						stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
 						if err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{
-							VersionID:   version.VersionID,
+							VersionID:   versionID,
 							MTime:       version.ModTime,
 							UserDefined: version.Metadata,
 						}); err != nil {
@@ -833,27 +789,38 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 						failure = false
 						break
 					}
+					stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
 					gr, err := set.GetObjectNInfo(ctx,
 						bi.Name,
 						encodeDirObject(version.Name),
 						nil,
 						http.Header{},
-						noLock, // all mutations are blocked reads are safe without locks.
 						ObjectOptions{
-							VersionID:    version.VersionID,
+							VersionID:    versionID,
 							NoDecryption: true,
+							NoLock:       true,
 						})
 					if isErrObjectNotFound(err) || isErrVersionNotFound(err) {
 						// object deleted by the application, nothing to do here we move on.
 						ignore = true
+						stopFn(nil)
 						break
 					}
+					if err != nil && !ignore {
+						// if usage-cache.bin is not readable log and ignore it.
+						if bi.Name == minioMetaBucket && strings.Contains(version.Name, dataUsageCacheName) {
+							ignore = true
+							stopFn(err)
+							logger.LogIf(ctx, err)
+							break
+						}
+					}
 					if err != nil {
 						failure = true
 						logger.LogIf(ctx, err)
+						stopFn(err)
 						continue
 					}
-					stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID)
 					if err = z.decommissionObject(ctx, bi.Name, gr); err != nil {
 						stopFn(err)
 						failure = true
@@ -902,21 +869,19 @@ func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool
 			z.poolMetaMutex.Unlock()
 		}

-		wg.Add(1)
+		wk.Take()
 		go func() {
-			defer wg.Done()
+			defer wk.Give()
 			err := set.listObjectsToDecommission(ctx, bi,
 				func(entry metaCacheEntry) {
-					// Wait must be synchronized here.
-					wg.Add(1)
-					parallelWorkers <- struct{}{}
+					wk.Take()
 					go decommissionEntry(entry)
 				},
 			)
 			logger.LogIf(ctx, err)
 		}()
 	}
-	wg.Wait()
+	wk.Wait()
 	return nil
 }

@@ -1002,17 +967,70 @@ func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error
 	pool := z.serverPools[idx]
 	for _, set := range pool.sets {
 		for _, bi := range buckets {
-			var objectsFound int
+			vc, _ := globalBucketVersioningSys.Get(bi.Name)
+
+			// Check if the current bucket has a configured lifecycle policy
+			lc, _ := globalLifecycleSys.Get(bi.Name)
+
+			// Check if bucket is object locked.
+			lr, _ := globalBucketObjectLockSys.Get(bi.Name)
+
+			filterLifecycle := func(bucket, object string, fi FileInfo) bool {
+				if lc == nil {
+					return false
+				}
+				versioned := vc != nil && vc.Versioned(object)
+				objInfo := fi.ToObjectInfo(bucket, object, versioned)
+
+				evt := evalActionFromLifecycle(ctx, *lc, lr, objInfo)
+				switch {
+				case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously
+					applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom)
+					return false
+				case evt.Action.Delete():
+					globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Decom)
+					return true
+				default:
+					return false
+				}
+			}
+
+			var versionsFound int
 			err := set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) {
-				if entry.isObject() {
-					objectsFound++
+				if !entry.isObject() {
+					return
+				}
+
+				fivs, err := entry.fileInfoVersions(bi.Name)
+				if err != nil {
+					return
+				}
+
+				// We need a reversed order for decommissioning,
+				// to create the appropriate stack.
+				versionsSorter(fivs.Versions).reverse()
+
+				for _, version := range fivs.Versions {
+					// Apply lifecycle rules on the objects that are expired.
+					if filterLifecycle(bi.Name, version.Name, version) {
+						continue
+					}
+
+					// `.usage-cache.bin` still exists, must be not readable ignore it.
+					if bi.Name == minioMetaBucket && strings.Contains(version.Name, dataUsageCacheName) {
+						// skipping bucket usage cache name, as its autogenerated.
+						continue
+					}
+
+					versionsFound++
 				}
 			})
 			if err != nil {
 				return err
 			}
-			if objectsFound > 0 {
-				return fmt.Errorf("at least %d objects were found in bucket `%s` after decommissioning", objectsFound, bi.Name)
+
+			if versionsFound > 0 {
+				return fmt.Errorf("at least %d object(s)/version(s) were found in bucket `%s` after decommissioning", versionsFound, bi.Name)
 			}
 		}
 	}
@@ -1079,11 +1097,9 @@ func (z *erasureServerPools) Decommission(ctx context.Context, indices ...int) e
 		return err
 	}

-	go func() {
-		for _, idx := range indices {
-			z.doDecommissionInRoutine(ctx, idx)
-		}
-	}()
+	for _, idx := range indices {
+		go z.doDecommissionInRoutine(ctx, idx)
+	}

 	// Successfully started decommissioning.
 	return nil
@@ -1280,7 +1296,7 @@ func (z *erasureServerPools) StartDecommission(ctx context.Context, indices ...i
 		z.HealBucket(ctx, bucket.Name, madmin.HealOpts{})
 	}

-	// Create .minio.sys/conifg, .minio.sys/buckets paths if missing,
+	// Create .minio.sys/config, .minio.sys/buckets paths if missing,
 	// this code is present to avoid any missing meta buckets on other
 	// pools.
 	for _, metaBucket := range []string{
@@ -127,8 +127,8 @@ func TestPoolMetaValidate(t *testing.T) {
 			meta:           meta,
 			pools:          orderChangePools,
 			name:           "Invalid-Orderchange",
-			expectedErr:    true,
-			expectedUpdate: false,
+			expectedErr:    false,
+			expectedUpdate: true,
 		},
 		{
 			meta:           nmeta1,
@@ -148,8 +148,8 @@ func TestPoolMetaValidate(t *testing.T) {
 			meta:           nmeta2,
 			pools:          reducedPools,
 			name:           "Invalid-Decom-Pending-Pool-Removal",
-			expectedErr:    true,
-			expectedUpdate: false,
+			expectedErr:    false,
+			expectedUpdate: true,
 		},
 		{
 			meta:           nmeta1,
@@ -169,8 +169,8 @@ func TestPoolMetaValidate(t *testing.T) {
 			meta:           nmeta2,
 			pools:          orderChangePools,
 			name:           "Invalid-Orderchange-Decom",
-			expectedErr:    true,
-			expectedUpdate: false,
+			expectedErr:    false,
+			expectedUpdate: true,
 		},
 	}

--- a/Show More
+++ b/Show More