fix: update storage-class properly

fixes #14005

.github/workflows/go-healing.yml

@@ -0,0 +1,48 @@
+name: Healing Functional Tests
+
+on:
+  pull_request:
+    branches:
+    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        go-version: [1.17.x]
+        os: [ubuntu-latest]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-go@v2
+        with:
+          go-version: ${{ matrix.go-version }}
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go-version }}-go-
+      - name: Build on ${{ matrix.os }}
+        if: matrix.os == 'ubuntu-latest'
+        env:
+          CGO_ENABLED: 0
+          GO111MODULE: on
+          MINIO_KMS_KES_CERT_FILE: /home/runner/work/minio/minio/.github/workflows/root.cert
+          MINIO_KMS_KES_KEY_FILE: /home/runner/work/minio/minio/.github/workflows/root.key
+          MINIO_KMS_KES_ENDPOINT: "https://play.min.io:7373"
+          MINIO_KMS_KES_KEY_NAME: "my-minio-key"
+          MINIO_KMS_AUTO_ENCRYPTION: on
+        run: |
+          sudo sysctl net.ipv6.conf.all.disable_ipv6=0
+          sudo sysctl net.ipv6.conf.default.disable_ipv6=0
+          make verify-healing

.github/workflows/go.yml

@@ -13,7 +13,7 @@ concurrency:
 
 jobs:
   build:
-    name: Go ${{ matrix.go-version }} on ${{ matrix.os }}
+    name: Go ${{ matrix.go-version }} on ${{ matrix.os }} - healing
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
@@ -46,4 +46,3 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make verify
-          make verify-healing

.github/workflows/iam-integrations.yaml

@@ -85,3 +85,7 @@ jobs:
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
           make test-iam
+      - name: Test LDAP for automatic site replication
+        if: matrix.ldap == 'localhost:389'
+        run: |
+          make test-site-replication

.github/workflows/replication.yaml

@@ -37,5 +37,4 @@ jobs:
         run: |
           sudo sysctl net.ipv6.conf.all.disable_ipv6=0
           sudo sysctl net.ipv6.conf.default.disable_ipv6=0
-          go install -v github.com/minio/mc@latest
           make test-replication

.gitignore

@@ -21,4 +21,15 @@ prime/
 stage/
 .sia_temp/
 config.json
-node_modules/
+node_modules/
+mc.*
+s3-check-md5*
+xl-meta*
+healing-*
+inspect*
+200M*
+hash-set
+minio.RELEASE*
+mc
+nancy
+inspects/*

COMPLIANCE.md

@@ -0,0 +1,6 @@
+# AGPLv3 Compliance
+We have designed MinIO as an Open Source software for the Open Source software community. This requires applications to consider whether their usage of MinIO is in compliance with the GNU AGPLv3 [license](https://github.com/minio/minio/blob/master/LICENSE).
+
+MinIO cannot make the determination as to whether your application's usage of MinIO is in compliance with the AGPLv3 license requirements. You should instead rely on your own legal counsel or licensing specialists to audit and ensure your application is in compliance with the licenses of MinIO and all other open-source projects with which your application integrates or interacts. We understand that AGPLv3 licensing is complex and nuanced. It is for that reason we strongly encourage using experts in licensing to make any such determinations around compliance instead of relying on apocryphal or anecdotal advice.
+
+[MinIO Commercial Licensing](https://min.io/pricing) is the best option for applications that trigger AGPLv3 obligations (e.g. open sourcing your application). Applications using MinIO - or any other OSS-licensed code - without validating their usage do so at their own risk.

CREDITS

@@ -2412,6 +2412,33 @@ https://github.com/briandowns/spinner
 
 ================================================================
 
+github.com/buger/jsonparser
+https://github.com/buger/jsonparser
+----------------------------------------------------------------
+MIT License
+
+Copyright (c) 2016 Leonid Bugaev
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+================================================================
+
 github.com/cespare/xxhash
 https://github.com/cespare/xxhash
 ----------------------------------------------------------------
@@ -2928,6 +2955,214 @@ https://github.com/coredns/coredns
 
 ================================================================
 
+github.com/coreos/go-oidc
+https://github.com/coreos/go-oidc
+----------------------------------------------------------------
+Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+
+================================================================
+
 github.com/coreos/go-semver
 https://github.com/coreos/go-semver
 ----------------------------------------------------------------
@@ -23378,6 +23613,214 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 ================================================================
 
+github.com/pquerna/cachecontrol
+https://github.com/pquerna/cachecontrol
+----------------------------------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
 github.com/prometheus/client_golang
 https://github.com/prometheus/client_golang
 ----------------------------------------------------------------
@@ -25523,6 +25966,37 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI
 
 ================================================================
 
+github.com/zeebo/xxh3
+https://github.com/zeebo/xxh3
+----------------------------------------------------------------
+xxHash Library
+Copyright (c) 2012-2014, Yann Collet
+Copyright (c) 2019, Jeff Wendling
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification,
+are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice, this
+  list of conditions and the following disclaimer in the documentation and/or
+  other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+================================================================
+
 go.etcd.io/bbolt
 https://go.etcd.io/bbolt
 ----------------------------------------------------------------
@@ -29211,6 +29685,214 @@ https://gopkg.in/jcmturner/rpc.v1
 
 ================================================================
 
+gopkg.in/square/go-jose.v2
+https://gopkg.in/square/go-jose.v2
+----------------------------------------------------------------
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+
+================================================================
+
 gopkg.in/yaml.v2
 https://gopkg.in/yaml.v2
 ----------------------------------------------------------------

Dockerfile

@@ -1,5 +1,8 @@
 FROM minio/minio:latest
 
+COPY ./minio /opt/bin/minio
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
+
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 
 VOLUME ["/data"]

Dockerfile.dev

@@ -1,17 +1,7 @@
-FROM minio/minio
+FROM minio/minio:latest
 
-LABEL maintainer="MinIO Inc <dev@min.io>"
-
-ENV PATH=/opt/bin:$PATH
-
-COPY minio /opt/bin
-COPY dockerscripts/docker-entrypoint.sh /usr/bin/
-
-RUN mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
-    chmod +x /opt/bin/minio && \
-    chmod +x /usr/bin/docker-entrypoint.sh
-
-EXPOSE 9000
+COPY ./minio /opt/bin/minio
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
 
 ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
 

Dockerfile.hotfix

@@ -0,0 +1,50 @@
+FROM registry.access.redhat.com/ubi8/ubi-minimal:8.4
+
+ARG RELEASE
+
+LABEL name="MinIO" \
+      vendor="MinIO Inc <dev@min.io>" \
+      maintainer="MinIO Inc <dev@min.io>" \
+      version="${RELEASE}" \
+      release="${RELEASE}" \
+      summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \
+      description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads."
+
+ENV MINIO_ACCESS_KEY_FILE=access_key \
+    MINIO_SECRET_KEY_FILE=secret_key \
+    MINIO_ROOT_USER_FILE=access_key \
+    MINIO_ROOT_PASSWORD_FILE=secret_key \
+    MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
+    MINIO_CONFIG_ENV_FILE=config.env \
+    PATH=/opt/bin:$PATH
+
+COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
+COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
+COPY CREDITS /licenses/CREDITS
+COPY LICENSE /licenses/LICENSE
+
+RUN \
+     microdnf clean all && \
+     microdnf update --nodocs && \
+     microdnf install curl ca-certificates shadow-utils util-linux --nodocs && \
+     rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
+     microdnf install minisign --nodocs && \
+     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
+     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE} -o /opt/bin/minio && \
+     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/hotfixes/linux-amd64/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
+     microdnf clean all && \
+     chmod +x /opt/bin/minio && \
+     chmod +x /usr/bin/docker-entrypoint.sh && \
+     chmod +x /usr/bin/verify-minio.sh && \
+     /usr/bin/verify-minio.sh && \
+     microdnf clean all
+
+EXPOSE 9000
+
+ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"]
+
+VOLUME ["/data"]
+
+CMD ["minio"]

Makefile

@@ -40,7 +40,7 @@ lint: ## runs golangci-lint suite of linters
 check: test
 test: verifiers build ## builds minio, runs linters, tests
 	@echo "Running unit tests"
-	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./... 1>/dev/null
+	@GO111MODULE=on CGO_ENABLED=0 go test -tags kqueue ./...
 
 test-upgrade: build
 	@echo "Running minio upgrade tests"
@@ -57,9 +57,13 @@ test-iam: build ## verify IAM (external IDP, etcd backends)
 	@CGO_ENABLED=1 go test -race -tags kqueue -v -run TestIAM* ./cmd
 
 test-replication: install ## verify multi site replication
-	@echo "Running tests for Replication three sites"
+	@echo "Running tests for replicating three sites"
 	@(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh)
 
+test-site-replication: install ## verify automatic site replication
+	@echo "Running tests for automatic site replication of IAM"
+	@(env bash $(PWD)/docs/site-replication/run-multi-site.sh)
+
 verify: ## verify minio various setups
 	@echo "Verifying build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
@@ -69,6 +73,7 @@ verify-healing: ## verify healing and replacing disks with minio binary
 	@echo "Verify healing build with race"
 	@GO111MODULE=on CGO_ENABLED=1 go build -race -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null
 	@(env bash $(PWD)/buildscripts/verify-healing.sh)
+	@(env bash $(PWD)/buildscripts/unaligned-healing.sh)
 
 build: checks ## builds minio to $(PWD)
 	@echo "Building minio binary to './minio'"
@@ -77,16 +82,28 @@ build: checks ## builds minio to $(PWD)
 hotfix-vars:
 	$(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \
     sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#')))
-	$(eval TAG := "minio/minio:$(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD)")
-hotfix: hotfix-vars install ## builds minio binary with hotfix tags
+	$(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD))
+	$(eval TAG := "minio/minio:$(VERSION)")
 
-docker-hotfix: hotfix checks ## builds minio docker container with hotfix tags
+hotfix: hotfix-vars install ## builds minio binary with hotfix tags
+	@mv -f ./minio ./minio.$(VERSION)
+	@minisign -qQSm ./minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase"
+	@sha256sum < ./minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio.$(VERSION).sha256sum
+
+hotfix-push: hotfix
+	@scp -r minio.$(VERSION)* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+	@scp -r minio.$(VERSION)* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-amd64/archive/
+
+docker-hotfix-push: docker-hotfix
+	@docker push $(TAG)
+
+docker-hotfix: hotfix-push checks ## builds minio docker container with hotfix tags
 	@echo "Building minio docker image '$(TAG)'"
-	@docker build -t $(TAG) . -f Dockerfile.dev
+	@docker build -t $(TAG) --build-arg RELEASE=$(VERSION) . -f Dockerfile.hotfix
 
 docker: build checks ## builds minio docker container
 	@echo "Building minio docker image '$(TAG)'"
-	@docker build -t $(TAG) . -f Dockerfile.dev
+	@docker build -t $(TAG) . -f Dockerfile
 
 install: build ## builds minio and installs it to $GOPATH/bin.
 	@echo "Installing minio binary to '$(GOPATH)/bin/minio'"

README.md

@@ -237,13 +237,11 @@ mc admin update <minio alias, e.g., myminio>
 
 ## Important things to remember during MinIO upgrades
 
+- Container environments are advised to update the container images instead of updating binaries inside the container.
+- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
 - `mc admin update` will only work if the user running MinIO has write access to the parent directory where the binary is located, for example if the current binary is at `/usr/local/bin/minio`, you would need write access to `/usr/local/bin`.
 - `mc admin update` updates and restarts all servers simultaneously, applications would retry and continue their respective operations upon upgrade.
-- `mc admin update` is disabled in kubernetes/container environments, container environments provide their own mechanisms to rollout of updates.
-- In the case of federated setups `mc admin update` should be run against each cluster individually. Avoid updating `mc` to any new releases until all clusters have been successfully updated.
 - If using `kes` as KMS with MinIO, just replace the binary and restart `kes` more information about `kes` can be found [here](https://github.com/minio/kes/wiki)
-- If using Vault as KMS with MinIO, ensure you have followed the Vault upgrade procedure outlined here: https://www.vaultproject.io/docs/upgrading/index.html
-- If using etcd with MinIO for the federation, ensure you have followed the etcd upgrade procedure outlined here: https://github.com/etcd-io/etcd/blob/master/Documentation/upgrades/upgrading-etcd.md
 
 # Explore Further
 - [MinIO Erasure Code QuickStart Guide](https://docs.min.io/docs/minio-erasure-code-quickstart-guide)
@@ -257,5 +255,6 @@ mc admin update <minio alias, e.g., myminio>
 Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md)
 
 # License
-MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
-MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+- MinIO source is licensed under the GNU AGPLv3 license that can be found in the [LICENSE](https://github.com/minio/minio/blob/master/LICENSE) file.
+- MinIO [Documentation](https://github.com/minio/minio/tree/master/docs) © 2021 by MinIO, Inc is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/).
+- [License Compliance](https://github.com/minio/minio/blob/master/COMPLIANCE.md)

buildscripts/minio-upgrade.sh

@@ -43,6 +43,9 @@ add_alias() {
             echo "...waiting... for 5secs" && sleep 5
         done
     done
+
+    echo "Sleeping for nginx"
+    sleep 20
 }
 
 __init__() {

buildscripts/unaligned-healing.sh

@@ -0,0 +1,171 @@
+#!/bin/bash -e
+#
+
+set -E
+set -o pipefail
+set -x
+
+if [ ! -x "$PWD/minio" ]; then
+    echo "minio executable binary not found in current directory"
+    exit 1
+fi
+
+WORK_DIR="$PWD/.verify-$RANDOM"
+MINIO_CONFIG_DIR="$WORK_DIR/.minio"
+MINIO_OLD=( "$PWD/minio.RELEASE.2021-11-24T23-19-33Z" --config-dir "$MINIO_CONFIG_DIR" server )
+MINIO=( "$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server )
+
+function download_old_release() {
+    if [ ! -f minio.RELEASE.2021-11-24T23-19-33Z ]; then
+	curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2021-11-24T23-19-33Z
+	chmod a+x minio.RELEASE.2021-11-24T23-19-33Z
+    fi
+}
+
+function start_minio_16drive() {
+    start_port=$1
+
+    export MINIO_ROOT_USER=minio
+    export MINIO_ROOT_PASSWORD=minio123
+    export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/"
+    unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects
+    export _MINIO_SHARD_DISKTIME_DELTA="5s" # do not change this as its needed for tests
+
+    MC_BUILD_DIR="mc-$RANDOM"
+    if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then
+	echo "failed to download https://github.com/minio/mc"
+	purge "${MC_BUILD_DIR}"
+	exit 1
+    fi
+
+    (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc")
+
+    # remove mc source.
+    purge "${MC_BUILD_DIR}"
+
+    "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 30
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    shred --iterations=1 --size=5241856 - 1>"${WORK_DIR}/unaligned" 2>/dev/null
+    "${WORK_DIR}/mc" mb minio/healing-shard-bucket --quiet
+    "${WORK_DIR}/mc" cp \
+		     "${WORK_DIR}/unaligned" \
+		     minio/healing-shard-bucket/unaligned \
+		     --disable-multipart --quiet
+
+    ## "unaligned" object name gets consistently distributed
+    ## to disks in following distribution order
+    ##
+    ## NOTE: if you change the name make sure to change the
+    ## distribution order present here
+    ##
+    ## [15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14]
+
+    ## make sure to remove the "last" data shard
+    rm -rf "${WORK_DIR}/xl14/healing-shard-bucket/unaligned"
+    sleep 10
+    ## Heal the shard
+    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
+    ## then remove any other data shard let's pick first disk
+    ## - 1st data shard.
+    rm -rf "${WORK_DIR}/xl3/healing-shard-bucket/unaligned"
+    sleep 10
+
+    go build ./docs/debugging/s3-check-md5/
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep CORRUPTED; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    pkill minio
+    sleep 3
+
+    "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" > "${WORK_DIR}/server1.log" 2>&1 &
+    pid=$!
+    disown $pid
+    sleep 30
+
+    if ! ps -p ${pid} 1>&2 >/dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    "${WORK_DIR}/mc" admin heal --quiet --recursive minio/healing-shard-bucket
+
+    if ! ./s3-check-md5 \
+	 -debug \
+	 -access-key minio \
+	 -secret-key minio123 \
+	 -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/server1.log"
+	echo "FAILED"
+	mkdir -p inspects
+	(cd inspects; "${WORK_DIR}/mc" admin inspect minio/healing-shard-bucket/unaligned/**)
+
+	"${WORK_DIR}/mc" mb play/inspects
+	"${WORK_DIR}/mc" mirror inspects play/inspects
+
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    pkill minio
+    sleep 3
+}
+
+function main() {
+    download_old_release
+
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    start_minio_16drive ${start_port}
+}
+
+function purge()
+{
+    rm -rf "$1"
+}
+
+( main "$@" )
+rv=$?
+purge "$WORK_DIR"
+exit "$rv"

buildscripts/verify-build.sh

@@ -66,7 +66,7 @@ function start_minio_pool_erasure_sets_ipv6()
 {
     export MINIO_ROOT_USER=$ACCESS_KEY
     export MINIO_ROOT_PASSWORD=$SECRET_KEY
-    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets{5...8}"
+    export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}"
     "${MINIO[@]}" server --address="[::1]:9000" > "$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 &
     "${MINIO[@]}" server --address="[::1]:9001" > "$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 &
 

buildscripts/verify-healing.sh

@@ -18,54 +18,73 @@ function start_minio_3_node() {
     export MINIO_ROOT_PASSWORD=minio123
     export MINIO_ERASURE_SET_DRIVE_COUNT=6
 
-    start_port=$(shuf -i 10000-65000 -n 1)
+    start_port=$2
     args=""
     for i in $(seq 1 3); do
-        args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
+	args="$args http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/1/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/2/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/3/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/4/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/5/ http://127.0.0.1:$[$start_port+$i]${WORK_DIR}/$i/6/"
     done
 
     "${MINIO[@]}" --address ":$[$start_port+1]" $args > "${WORK_DIR}/dist-minio-server1.log" 2>&1 &
-    disown $!
+    pid1=$!
+    disown ${pid1}
 
     "${MINIO[@]}" --address ":$[$start_port+2]" $args > "${WORK_DIR}/dist-minio-server2.log" 2>&1 &
-    disown $!
+    pid2=$!
+    disown $pid2
 
     "${MINIO[@]}" --address ":$[$start_port+3]" $args > "${WORK_DIR}/dist-minio-server3.log" 2>&1 &
-    disown $!
+    pid3=$!
+    disown $pid3
 
     sleep "$1"
-    if [ "$(pgrep -c minio)" -ne 3 ]; then
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-server$i.log"
-        done
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
+
+    if ! ps -p $pid1 1>&2 > /dev/null; then
+	echo "server1 log:"
+	cat "${WORK_DIR}/dist-minio-server1.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
     fi
+
+    if ! ps -p $pid2 1>&2 > /dev/null; then
+	echo "server2 log:"
+	cat "${WORK_DIR}/dist-minio-server2.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
+    if ! ps -p $pid3 1>&2 > /dev/null; then
+	echo "server3 log:"
+	cat "${WORK_DIR}/dist-minio-server3.log"
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
+    fi
+
     if ! pkill minio; then
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-server$i.log"
-        done
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
+	for i in $(seq 1 3); do
+	    echo "server$i log:"
+	    cat "${WORK_DIR}/dist-minio-server$i.log"
+	done
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
     fi
 
     sleep 1;
     if pgrep minio; then
-        # forcibly killing, to proceed further properly.
-        if ! pkill -9 minio; then
-            echo "no minio process running anymore, proceed."
-        fi
+	# forcibly killing, to proceed further properly.
+	if ! pkill -9 minio; then
+	    echo "no minio process running anymore, proceed."
+	fi
     fi
 }
 
 
 function check_online() {
     if grep -q 'Unable to initialize sub-systems' ${WORK_DIR}/dist-minio-*.log; then
-        echo "1"
+	echo "1"
     fi
 }
 
@@ -85,33 +104,36 @@ function __init__()
 }
 
 function perform_test() {
-    start_minio_3_node 120
+    start_minio_3_node 120 $2
 
     echo "Testing Distributed Erasure setup healing of drives"
     echo "Remove the contents of the disks belonging to '${1}' erasure set"
 
     rm -rf ${WORK_DIR}/${1}/*/
 
-    start_minio_3_node 120
+    start_minio_3_node 120 $2
 
     rv=$(check_online)
     if [ "$rv" == "1" ]; then
-        for i in $(seq 1 3); do
-            echo "server$i log:"
-            cat "${WORK_DIR}/dist-minio-server$i.log"
-        done
-        pkill -9 minio
-        echo "FAILED"
-        purge "$WORK_DIR"
-        exit 1
+	for i in $(seq 1 3); do
+	    echo "server$i log:"
+	    cat "${WORK_DIR}/dist-minio-server$i.log"
+	done
+	pkill -9 minio
+	echo "FAILED"
+	purge "$WORK_DIR"
+	exit 1
     fi
 }
 
 function main()
 {
-    perform_test "2"
-    perform_test "1"
-    perform_test "3"
+    # use same ports for all tests
+    start_port=$(shuf -i 10000-65000 -n 1)
+
+    perform_test "2" ${start_port}
+    perform_test "1" ${start_port}
+    perform_test "3" ${start_port}
 }
 
 ( __init__ "$@" && main "$@" )

cmd/admin-handler-utils.go

@@ -86,6 +86,8 @@ func toAdminAPIErr(ctx context.Context, err error) APIError {
 			Description:    e.Message,
 			HTTPStatusCode: e.StatusCode,
 		}
+	case SRError:
+		apiErr = errorCodes.ToAPIErrWithErr(e.Code, e.Cause)
 	default:
 		switch {
 		case errors.Is(err, errConfigNotFound):

cmd/admin-handlers-site-replication.go

@@ -45,16 +45,16 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	}
 
 	var sites []madmin.PeerSite
-	errCode := readJSONBody(ctx, r.Body, &sites, cred.SecretKey)
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	err := parseJSONBody(ctx, r.Body, &sites, cred.SecretKey)
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	status, errInfo := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
-	if errInfo.Code != ErrNone {
-		logger.LogIf(ctx, errInfo)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(errInfo.Code, errInfo.Cause), r.URL)
+	status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
@@ -67,12 +67,12 @@ func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Requ
 	writeSuccessResponseJSON(w, body)
 }
 
-// SRInternalJoin - PUT /minio/admin/v3/site-replication/join
+// SRPeerJoin - PUT /minio/admin/v3/site-replication/join
 //
 // used internally to tell current cluster to enable SR with
 // the provided peer clusters and service account.
-func (a adminAPIHandlers) SRInternalJoin(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalJoin")
+func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerJoin")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -81,24 +81,22 @@ func (a adminAPIHandlers) SRInternalJoin(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	var joinArg madmin.SRInternalJoinReq
-	errCode := readJSONBody(ctx, r.Body, &joinArg, cred.SecretKey)
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	var joinArg madmin.SRPeerJoinReq
+	if err := parseJSONBody(ctx, r.Body, &joinArg, cred.SecretKey); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	errInfo := globalSiteReplicationSys.InternalJoinReq(ctx, joinArg)
-	if errInfo.Code != ErrNone {
-		logger.LogIf(ctx, errInfo)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(errInfo.Code, errInfo.Cause), r.URL)
+	if err := globalSiteReplicationSys.InternalJoinReq(ctx, joinArg); err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 }
 
-// SRInternalBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
-func (a adminAPIHandlers) SRInternalBucketOps(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalBucketOps")
+// SRPeerBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y
+func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerBucketOps")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -113,6 +111,8 @@ func (a adminAPIHandlers) SRInternalBucketOps(w http.ResponseWriter, r *http.Req
 
 	var err error
 	switch operation {
+	default:
+		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.MakeWithVersioningBktOp:
 		_, isLockEnabled := r.Form["lockEnabled"]
 		_, isVersioningEnabled := r.Form["versioningEnabled"]
@@ -128,21 +128,18 @@ func (a adminAPIHandlers) SRInternalBucketOps(w http.ResponseWriter, r *http.Req
 		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, false)
 	case madmin.ForceDeleteBucketBktOp:
 		err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, true)
-	default:
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
-		return
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 }
 
-// SRInternalReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
-func (a adminAPIHandlers) SRInternalReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalReplicateIAMItem")
+// SRPeerReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item
+func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerReplicateIAMItem")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -152,45 +149,47 @@ func (a adminAPIHandlers) SRInternalReplicateIAMItem(w http.ResponseWriter, r *h
 	}
 
 	var item madmin.SRIAMItem
-	errCode := readJSONBody(ctx, r.Body, &item, "")
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	var err error
 	switch item.Type {
+	default:
+		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRIAMItemPolicy:
-		var policy *iampolicy.Policy
-		if len(item.Policy) > 0 {
-			policy, err = iampolicy.ParseConfig(bytes.NewReader(item.Policy))
-			if err != nil {
-				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		if item.Policy == nil {
+			err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+		} else {
+			policy, perr := iampolicy.ParseConfig(bytes.NewReader(item.Policy))
+			if perr != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL)
 				return
 			}
+			if policy.IsEmpty() {
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil)
+			} else {
+				err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy)
+			}
 		}
-		err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy)
 	case madmin.SRIAMItemSvcAcc:
-		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, *item.SvcAccChange)
+		err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange)
 	case madmin.SRIAMItemPolicyMapping:
-		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, *item.PolicyMapping)
+		err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping)
 	case madmin.SRIAMItemSTSAcc:
-		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, *item.STSCredential)
-
-	default:
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
-		return
+		err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential)
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 }
 
-// SRInternalReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
-func (a adminAPIHandlers) SRInternalReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "SRInternalReplicateIAMItem")
+// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/bucket-meta
+func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "SRPeerReplicateBucketItem")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
 
@@ -200,38 +199,40 @@ func (a adminAPIHandlers) SRInternalReplicateBucketItem(w http.ResponseWriter, r
 	}
 
 	var item madmin.SRBucketMeta
-	errCode := readJSONBody(ctx, r.Body, &item, "")
-	if errCode != ErrNone {
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL)
+	if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
 	var err error
 	switch item.Type {
+	default:
+		err = errSRInvalidRequest(errInvalidArgument)
 	case madmin.SRBucketMetaTypePolicy:
-		var bktPolicy *policy.Policy
-		if len(item.Policy) > 0 {
-			bktPolicy, err = policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
-			if err != nil {
-				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		if item.Policy == nil {
+			err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+		} else {
+			bktPolicy, berr := policy.ParseConfig(bytes.NewReader(item.Policy), item.Bucket)
+			if berr != nil {
+				writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL)
 				return
 			}
+			if bktPolicy.IsEmpty() {
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil)
+			} else {
+				err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
+			}
 		}
-		err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy)
 	case madmin.SRBucketMetaTypeTags:
 		err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags)
 	case madmin.SRBucketMetaTypeObjectLockConfig:
 		err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig)
 	case madmin.SRBucketMetaTypeSSEConfig:
 		err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig)
-
-	default:
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL)
-		return
 	}
 	if err != nil {
 		logger.LogIf(ctx, err)
-		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 }
@@ -271,7 +272,7 @@ func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Req
 	}
 }
 
-func (a adminAPIHandlers) SRInternalGetIDPSettings(w http.ResponseWriter, r *http.Request) {
+func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "SiteReplicationGetIDPSettings")
 
 	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
@@ -288,24 +289,25 @@ func (a adminAPIHandlers) SRInternalGetIDPSettings(w http.ResponseWriter, r *htt
 	}
 }
 
-func readJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) APIErrorCode {
+func parseJSONBody(ctx context.Context, body io.Reader, v interface{}, encryptionKey string) error {
 	data, err := ioutil.ReadAll(body)
 	if err != nil {
-		return ErrInvalidRequest
+		return SRError{
+			Cause: err,
+			Code:  ErrSiteReplicationInvalidRequest,
+		}
 	}
 
 	if encryptionKey != "" {
 		data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data))
 		if err != nil {
 			logger.LogIf(ctx, err)
-			return ErrInvalidRequest
+			return SRError{
+				Cause: err,
+				Code:  ErrSiteReplicationInvalidRequest,
+			}
 		}
 	}
 
-	err = json.Unmarshal(data, v)
-	if err != nil {
-		return ErrAdminConfigBadJSON
-	}
-
-	return ErrNone
+	return json.Unmarshal(data, v)
 }

cmd/admin-handlers-users.go

@@ -58,18 +58,10 @@ func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if err := globalIAMSys.DeleteUser(ctx, accessKey); err != nil {
+	if err := globalIAMSys.DeleteUser(ctx, accessKey, true); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other MinIO peers to delete user.
-	for _, nerr := range globalNotificationSys.DeleteUser(accessKey) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
 }
 
 // ListUsers - GET /minio/admin/v3/list-users?bucket={bucket}
@@ -164,31 +156,24 @@ func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	accessKey := cred.ParentUser
-	if accessKey == "" {
-		accessKey = cred.AccessKey
+	checkDenyOnly := false
+	if name == cred.AccessKey {
+		// Check that there is no explicit deny - otherwise it's allowed
+		// to view one's own info.
+		checkDenyOnly = true
 	}
 
-	// For temporary credentials always
-	// the temporary credentials to check
-	// policy without implicit permissions.
-	if cred.IsTemp() && cred.ParentUser == globalActiveCred.AccessKey {
-		accessKey = cred.AccessKey
-	}
-
-	implicitPerm := name == accessKey
-	if !implicitPerm {
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     accessKey,
-			Groups:          cred.Groups,
-			Action:          iampolicy.GetUserAdminAction,
-			ConditionValues: getConditionValues(r, "", accessKey, claims),
-			IsOwner:         owner,
-			Claims:          claims,
-		}) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
+	if !globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
+		Groups:          cred.Groups,
+		Action:          iampolicy.GetUserAdminAction,
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
+		IsOwner:         owner,
+		Claims:          claims,
+		DenyOnly:        checkDenyOnly,
+	}) {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
+		return
 	}
 
 	userInfo, err := globalIAMSys.GetUserInfo(ctx, name)
@@ -240,16 +225,6 @@ func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Requ
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other MinIO peers to load group.
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadGroup(updReq.Group) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
 }
 
 // GetGroup - /minio/admin/v3/group?group=mygroup1
@@ -335,16 +310,6 @@ func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other MinIO peers to reload user.
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadGroup(group) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
 }
 
 // SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=<access_key>&status=[enabled|disabled]
@@ -372,16 +337,6 @@ func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other MinIO peers to reload user.
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
 }
 
 // AddUser - PUT /minio/admin/v3/add-user?accessKey=<access_key>
@@ -412,6 +367,14 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	userCred, exists := globalIAMSys.GetUser(ctx, accessKey)
+	if exists && (userCred.IsTemp() || userCred.IsServiceAccount()) {
+		// Updating STS credential is not allowed, and this API does not
+		// support updating service accounts.
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL)
+		return
+	}
+
 	if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey {
 		// Incoming access key matches parent user then we should
 		// reject password change requests.
@@ -419,39 +382,21 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	implicitPerm := accessKey == cred.AccessKey
-	if !implicitPerm {
-		parentUser := cred.ParentUser
-		if parentUser == "" {
-			parentUser = cred.AccessKey
-		}
-		// For temporary credentials always
-		// the temporary credentials to check
-		// policy without implicit permissions.
-		if cred.IsTemp() && cred.ParentUser == globalActiveCred.AccessKey {
-			parentUser = cred.AccessKey
-		}
-		if !globalIAMSys.IsAllowed(iampolicy.Args{
-			AccountName:     parentUser,
-			Groups:          cred.Groups,
-			Action:          iampolicy.CreateUserAdminAction,
-			ConditionValues: getConditionValues(r, "", parentUser, claims),
-			IsOwner:         owner,
-			Claims:          claims,
-		}) {
-			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
-			return
-		}
+	checkDenyOnly := false
+	if accessKey == cred.AccessKey {
+		// Check that there is no explicit deny - otherwise it's allowed
+		// to change one's own password.
+		checkDenyOnly = true
 	}
 
-	if implicitPerm && !globalIAMSys.IsAllowed(iampolicy.Args{
-		AccountName:     accessKey,
+	if !globalIAMSys.IsAllowed(iampolicy.Args{
+		AccountName:     cred.AccessKey,
 		Groups:          cred.Groups,
 		Action:          iampolicy.CreateUserAdminAction,
-		ConditionValues: getConditionValues(r, "", accessKey, claims),
+		ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 		IsOwner:         owner,
 		Claims:          claims,
-		DenyOnly:        true, // check if changing password is explicitly denied.
+		DenyOnly:        checkDenyOnly,
 	}) {
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 		return
@@ -471,27 +416,17 @@ func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	var uinfo madmin.UserInfo
-	if err = json.Unmarshal(configBytes, &uinfo); err != nil {
+	var ureq madmin.AddOrUpdateUserReq
+	if err = json.Unmarshal(configBytes, &ureq); err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL)
 		return
 	}
 
-	if err = globalIAMSys.CreateUser(ctx, accessKey, uinfo); err != nil {
+	if err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	// Notify all other Minio peers to reload user
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadUser(accessKey, false) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
 }
 
 // AddServiceAccount - PUT /minio/admin/v3/add-service-account
@@ -571,6 +506,7 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		// if there is no deny statement this call is implicitly enabled.
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     requestorUser,
+			Groups:          requestorGroups,
 			Action:          iampolicy.CreateServiceAccountAdminAction,
 			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			IsOwner:         owner,
@@ -604,10 +540,12 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		// user <> to the request sender
 		if !globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     requestorUser,
+			Groups:          requestorGroups,
 			Action:          iampolicy.CreateServiceAccountAdminAction,
 			ConditionValues: getConditionValues(r, "", cred.AccessKey, claims),
 			IsOwner:         owner,
 			Claims:          claims,
+			DenyOnly:        true,
 		}) {
 			writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL)
 			return
@@ -647,16 +585,6 @@ func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Reque
 		return
 	}
 
-	// Notify all other Minio peers to reload user the service account
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadServiceAccount(newCred.AccessKey) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
-
 	// Call hook for cluster-replication.
 	//
 	// FIXME: This wont work in an OpenID situation as the parent credential
@@ -788,16 +716,6 @@ func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	// Notify all other Minio peers to reload user the service account
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadServiceAccount(accessKey) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
-
 	// Call site replication hook. Only LDAP accounts are supported for
 	// replication operations.
 	svcAccClaims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, accessKey)
@@ -1052,17 +970,11 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount)
+	err = globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-	for _, nerr := range globalNotificationSys.DeleteServiceAccount(serviceAccount) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
 
 	// Call site replication hook. Only LDAP accounts are supported for
 	// replication operations.
@@ -1181,19 +1093,11 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	}
 
 	accountName := cred.AccessKey
-	var policies []string
-	switch globalIAMSys.usersSysType {
-	case MinIOUsersSysType:
-		policies, err = globalIAMSys.PolicyDBGet(accountName, false)
-	case LDAPUsersSysType:
-		parentUser := accountName
-		if cred.ParentUser != "" {
-			parentUser = cred.ParentUser
-		}
-		policies, err = globalIAMSys.PolicyDBGet(parentUser, false, cred.Groups...)
-	default:
-		err = errors.New("should never happen")
+	if cred.IsTemp() || cred.IsServiceAccount() {
+		// For derived credentials, check the parent user's permissions.
+		accountName = cred.ParentUser
 	}
+	policies, err := globalIAMSys.PolicyDBGet(accountName, false, cred.Groups...)
 	if err != nil {
 		logger.LogIf(ctx, err)
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
@@ -1272,6 +1176,16 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 }
 
 // InfoCannedPolicy - GET /minio/admin/v3/info-canned-policy?name={policyName}
+//
+// Newer API response with policy timestamps is returned with query parameter
+// `v=2` like:
+//
+// GET /minio/admin/v3/info-canned-policy?name={policyName}&v=2
+//
+// The newer API will eventually become the default (and only) one. The older
+// response is to return only the policy JSON. The newer response returns
+// timestamps along with the policy JSON. Both versions are supported for now,
+// for smooth transition to new API.
 func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "InfoCannedPolicy")
 
@@ -1282,13 +1196,36 @@ func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	policy, err := globalIAMSys.InfoPolicy(mux.Vars(r)["name"])
+	name := mux.Vars(r)["name"]
+	policies := newMappedPolicy(name).toSlice()
+	if len(policies) != 1 {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errTooManyPolicies), r.URL)
+		return
+	}
+
+	policyDoc, err := globalIAMSys.InfoPolicy(name)
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	buf, err := json.MarshalIndent(policy, "", " ")
+	// Is the new API version being requested?
+	infoPolicyAPIVersion := r.Form.Get("v")
+	if infoPolicyAPIVersion == "2" {
+		buf, err := json.MarshalIndent(policyDoc, "", " ")
+		if err != nil {
+			writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+			return
+		}
+		w.Write(buf)
+		return
+	} else if infoPolicyAPIVersion != "" {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("invalid version parameter 'v' supplied")), r.URL)
+		return
+	}
+
+	// Return the older API response value of just the policy json.
+	buf, err := json.MarshalIndent(policyDoc.Policy, "", " ")
 	if err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
@@ -1375,19 +1312,11 @@ func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Requ
 	vars := mux.Vars(r)
 	policyName := vars["name"]
 
-	if err := globalIAMSys.DeletePolicy(ctx, policyName); err != nil {
+	if err := globalIAMSys.DeletePolicy(ctx, policyName, true); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
 
-	// Notify all other MinIO peers to delete policy
-	for _, nerr := range globalNotificationSys.DeletePolicy(policyName) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-
 	// Call cluster-replication policy creation hook to replicate policy deletion to
 	// other minio clusters.
 	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
@@ -1448,16 +1377,6 @@ func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request
 		return
 	}
 
-	// Notify all other MinIO peers to reload policy
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
-
 	// Call cluster-replication policy creation hook to replicate policy to
 	// other minio clusters.
 	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
@@ -1503,16 +1422,6 @@ func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http
 		return
 	}
 
-	// Notify all other MinIO peers to reload policy
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadPolicyMapping(entityName, isGroup) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
-
 	if err := globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{
 		Type: madmin.SRIAMItemPolicyMapping,
 		PolicyMapping: &madmin.SRPolicyMapping{

cmd/admin-handlers-users_test.go

@@ -18,8 +18,15 @@
 package cmd
 
 import (
+	"bytes"
 	"context"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
 	"os"
 	"strings"
 	"testing"
@@ -29,7 +36,9 @@ import (
 	minio "github.com/minio/minio-go/v7"
 	"github.com/minio/minio-go/v7/pkg/credentials"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
+	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio-go/v7/pkg/signer"
 	"github.com/minio/minio/internal/auth"
 )
 
@@ -177,6 +186,7 @@ func TestIAMInternalIDPServerSuite(t *testing.T) {
 
 				suite.SetUpSuite(c)
 				suite.TestUserCreate(c)
+				suite.TestUserPolicyEscalationBug(c)
 				suite.TestPolicyCreate(c)
 				suite.TestCannedPolicies(c)
 				suite.TestGroupAddRemove(c)
@@ -222,6 +232,24 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
 		c.Fatalf("user could not create bucket: %v", err)
 	}
 
+	// 3.10. Check that user's password can be updated.
+	_, newSecretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, newSecretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to update user's secret key: %v", err)
+	}
+	// 3.10.1 Check that old password no longer works.
+	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
+	if err == nil {
+		c.Fatalf("user was unexpectedly able to create bucket with bad password!")
+	}
+	// 3.10.2 Check that new password works.
+	client = s.getUserClient(c, accessKey, newSecretKey, "")
+	err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("user could not create bucket: %v", err)
+	}
+
 	// 4. Check that user can be disabled and verify it.
 	err = s.adm.SetUserStatus(ctx, accessKey, madmin.AccountDisabled)
 	if err != nil {
@@ -260,6 +288,108 @@ func (s *TestSuiteIAM) TestUserCreate(c *check) {
 	}
 }
 
+func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) {
+	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
+	defer cancel()
+
+	bucket := getRandomBucketName()
+	err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{})
+	if err != nil {
+		c.Fatalf("bucket creat error: %v", err)
+	}
+
+	// 2. Create a user, associate policy and verify access
+	accessKey, secretKey := mustGenerateCredentials(c)
+	err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled)
+	if err != nil {
+		c.Fatalf("Unable to set user: %v", err)
+	}
+	// 2.1 check that user does not have any access to the bucket
+	uClient := s.getUserClient(c, accessKey, secretKey, "")
+	c.mustNotListObjects(ctx, uClient, bucket)
+
+	// 2.2 create and associate policy to user
+	policy := "mypolicy-test-user-update"
+	policyBytes := []byte(fmt.Sprintf(`{
+ "Version": "2012-10-17",
+ "Statement": [
+  {
+   "Effect": "Allow",
+   "Action": [
+    "s3:PutObject",
+    "s3:GetObject",
+    "s3:ListBucket"
+   ],
+   "Resource": [
+    "arn:aws:s3:::%s/*"
+   ]
+  }
+ ]
+}`, bucket))
+	err = s.adm.AddCannedPolicy(ctx, policy, policyBytes)
+	if err != nil {
+		c.Fatalf("policy add error: %v", err)
+	}
+	err = s.adm.SetPolicy(ctx, policy, accessKey, false)
+	if err != nil {
+		c.Fatalf("Unable to set policy: %v", err)
+	}
+	// 2.3 check user has access to bucket
+	c.mustListObjects(ctx, uClient, bucket)
+	// 2.3 check that user cannot delete the bucket
+	err = uClient.RemoveBucket(ctx, bucket)
+	if err == nil || err.Error() != "Access Denied." {
+		c.Fatalf("bucket was deleted unexpectedly or got unexpected err: %v", err)
+	}
+
+	// 3. Craft a request to update the user's permissions
+	ep := s.adm.GetEndpointURL()
+	urlValue := url.Values{}
+	urlValue.Add("accessKey", accessKey)
+	u, err := url.Parse(fmt.Sprintf("%s://%s/minio/admin/v3/add-user?%s", ep.Scheme, ep.Host, s3utils.QueryEncode(urlValue)))
+	if err != nil {
+		c.Fatalf("unexpected url parse err: %v", err)
+	}
+	req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), nil)
+	if err != nil {
+		c.Fatalf("unexpected new request err: %v", err)
+	}
+	reqBodyArg := madmin.UserInfo{
+		SecretKey:  secretKey,
+		PolicyName: "consoleAdmin",
+		Status:     madmin.AccountEnabled,
+	}
+	buf, err := json.Marshal(reqBodyArg)
+	if err != nil {
+		c.Fatalf("unexpected json encode err: %v", err)
+	}
+	buf, err = madmin.EncryptData(secretKey, buf)
+	if err != nil {
+		c.Fatalf("unexpected encryption err: %v", err)
+	}
+
+	req.ContentLength = int64(len(buf))
+	sum := sha256.Sum256(buf)
+	req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum[:]))
+	req.Body = ioutil.NopCloser(bytes.NewReader(buf))
+	req = signer.SignV4(*req, accessKey, secretKey, "", "")
+
+	// 3.1 Execute the request.
+	resp, err := s.TestSuiteCommon.client.Do(req)
+	if err != nil {
+		c.Fatalf("unexpected request err: %v", err)
+	}
+	if resp.StatusCode != 200 {
+		c.Fatalf("got unexpected response: %#v\n", resp)
+	}
+
+	// 3.2 check that user cannot delete the bucket
+	err = uClient.RemoveBucket(ctx, bucket)
+	if err == nil || err.Error() != "Access Denied." {
+		c.Fatalf("User was able to escalate privileges (Err=%v)!", err)
+	}
+}
+
 func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) {
 	ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout)
 	defer cancel()
@@ -831,6 +961,36 @@ func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) {
 	c.assertSvcAccDeletion(ctx, s, s.adm, accessKey, bucket)
 }
 
+func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) madmin.Credentials {
+	randUser := mustGetUUID()
+	randPass := mustGetUUID()
+	err := admClnt.AddUser(ctx, randUser, randPass)
+	if err != nil {
+		c.Fatalf("should be able to create a user: %v", err)
+	}
+	return madmin.Credentials{
+		AccessKey: randUser,
+		SecretKey: randPass,
+	}
+}
+
+func (c *check) mustGetIAMUserInfo(ctx context.Context, admClnt *madmin.AdminClient, accessKey string) madmin.UserInfo {
+	ui, err := admClnt.GetUserInfo(ctx, accessKey)
+	if err != nil {
+		c.Fatalf("should be able to get user info: %v", err)
+	}
+	return ui
+}
+
+func (c *check) mustNotCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) {
+	randUser := mustGetUUID()
+	randPass := mustGetUUID()
+	err := admClnt.AddUser(ctx, randUser, randPass)
+	if err == nil {
+		c.Fatalf("should not be able to create a user")
+	}
+}
+
 func (c *check) mustCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) madmin.Credentials {
 	cr, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{
 		TargetUser: tgtUser,

cmd/admin-handlers.go

@@ -957,6 +957,7 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
 	autotune := r.Form.Get("autotune") == "true"
+	storageClass := r.Form.Get("storage-class")
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -968,6 +969,10 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 		concurrent = 32
 	}
 
+	if runtime.GOMAXPROCS(0) < concurrent {
+		concurrent = runtime.GOMAXPROCS(0)
+	}
+
 	duration, err := time.ParseDuration(durationStr)
 	if err != nil {
 		duration = time.Second * 10
@@ -980,6 +985,7 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 			NoRecreate: true,
 		})
 	}
+	defer deleteBucket()
 
 	// Freeze all incoming S3 API calls before running speedtest.
 	globalNotificationSys.ServiceFreeze(ctx, true)
@@ -991,7 +997,7 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 	defer keepAliveTicker.Stop()
 
 	enc := json.NewEncoder(w)
-	ch := speedTest(ctx, size, concurrent, duration, autotune)
+	ch := speedTest(ctx, speedTestOpts{size, concurrent, duration, autotune, storageClass})
 	for {
 		select {
 		case <-ctx.Done():
@@ -1004,7 +1010,6 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 			w.(http.Flusher).Flush()
 		case result, ok := <-ch:
 			if !ok {
-				deleteBucket()
 				return
 			}
 			if err := enc.Encode(result); err != nil {
@@ -1466,7 +1471,7 @@ func getServerInfo(ctx context.Context, r *http.Request) madmin.InfoMessage {
 	return madmin.InfoMessage{
 		Mode:         string(mode),
 		Domain:       domain,
-		Region:       globalServerRegion,
+		Region:       globalSite.Region,
 		SQSARN:       globalNotificationSys.GetARNList(false),
 		DeploymentID: globalDeploymentID,
 		Buckets:      buckets,
@@ -1864,6 +1869,8 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			}
 
 			tls := getTLSInfo()
+			isK8s := IsKubernetes()
+			isDocker := IsDocker()
 			healthInfo.Minio.Info = madmin.MinioInfo{
 				Mode:         infoMessage.Mode,
 				Domain:       infoMessage.Domain,
@@ -1877,6 +1884,8 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 				Backend:      infoMessage.Backend,
 				Servers:      servers,
 				TLS:          &tls,
+				IsKubernetes: &isK8s,
+				IsDocker:     &isDocker,
 			}
 			partialWrite(healthInfo)
 		}

cmd/admin-handlers_test.go

@@ -41,11 +41,13 @@ type adminErasureTestBed struct {
 	erasureDirs []string
 	objLayer    ObjectLayer
 	router      *mux.Router
+	done        context.CancelFunc
 }
 
 // prepareAdminErasureTestBed - helper function that setups a single-node
 // Erasure backend for admin-handler tests.
 func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, error) {
+	ctx, cancel := context.WithCancel(ctx)
 
 	// reset global variables to start afresh.
 	resetTestGlobals()
@@ -57,11 +59,13 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 	// Initializing objectLayer for HealFormatHandler.
 	objLayer, erasureDirs, xlErr := initTestErasureObjLayer(ctx)
 	if xlErr != nil {
+		cancel()
 		return nil, xlErr
 	}
 
 	// Initialize minio server config.
 	if err := newTestConfig(globalMinioDefaultRegion, objLayer); err != nil {
+		cancel()
 		return nil, err
 	}
 
@@ -74,7 +78,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 
 	initConfigSubsystem(ctx, objLayer)
 
-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
@@ -84,12 +88,14 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 		erasureDirs: erasureDirs,
 		objLayer:    objLayer,
 		router:      adminRouter,
+		done:        cancel,
 	}, nil
 }
 
 // TearDown - method that resets the test bed for subsequent unit
 // tests to start afresh.
 func (atb *adminErasureTestBed) TearDown() {
+	atb.done()
 	removeRoots(atb.erasureDirs)
 	resetTestGlobals()
 }

cmd/admin-heal-ops.go

@@ -710,6 +710,9 @@ func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItem
 
 	select {
 	case globalBackgroundHealRoutine.tasks <- task:
+		if serverDebugLog {
+			logger.Info("Task in the queue: %#v", task)
+		}
 	case <-h.ctx.Done():
 		return nil
 	}
@@ -885,6 +888,7 @@ func (h *healSequence) healObject(bucket, object, versionID string) error {
 		bucket:    bucket,
 		object:    object,
 		versionID: versionID,
+		opts:      &h.settings,
 	}, madmin.HealItemObject)
 
 	// Wait and proceed if there are active requests

cmd/admin-router.go

@@ -194,11 +194,11 @@ func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationAdd)))
 			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/disable").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationDisable)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(gz(httpTraceHdrs(adminAPI.SiteReplicationInfo)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalJoin)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalReplicateIAMItem)))
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalReplicateBucketItem)))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRInternalGetIDPSettings)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerJoin)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerBucketOps))).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateIAMItem)))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerReplicateBucketItem)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(gz(httpTraceHdrs(adminAPI.SRPeerGetIDPSettings)))
 		}
 
 		if globalIsDistErasure {

cmd/api-errors.go

@@ -82,6 +82,7 @@ const (
 	ErrIncompleteBody
 	ErrInternalError
 	ErrInvalidAccessKeyID
+	ErrAccessKeyDisabled
 	ErrInvalidBucketName
 	ErrInvalidDigest
 	ErrInvalidRange
@@ -129,6 +130,7 @@ const (
 	ErrReplicationSourceNotVersionedError
 	ErrReplicationNeedsVersioningError
 	ErrReplicationBucketNeedsVersioningError
+	ErrReplicationDenyEditError
 	ErrReplicationNoMatchingRuleError
 	ErrObjectRestoreAlreadyInProgress
 	ErrNoSuchKey
@@ -396,10 +398,10 @@ func (e errorCodeMap) ToAPIErrWithErr(errCode APIErrorCode, err error) APIError
 	if err != nil {
 		apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err)
 	}
-	if globalServerRegion != "" {
+	if globalSite.Region != "" {
 		switch errCode {
 		case ErrAuthorizationHeaderMalformed:
-			apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalServerRegion)
+			apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
 			return apiErr
 		}
 	}
@@ -513,6 +515,11 @@ var errorCodes = errorCodeMap{
 		Description:    "The Access Key Id you provided does not exist in our records.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
+	ErrAccessKeyDisabled: {
+		Code:           "InvalidAccessKeyId",
+		Description:    "Your account is disabled; please contact your administrator.",
+		HTTPStatusCode: http.StatusForbidden,
+	},
 	ErrInvalidBucketName: {
 		Code:           "InvalidBucketName",
 		Description:    "The specified bucket is not valid.",
@@ -680,7 +687,7 @@ var errorCodes = errorCodeMap{
 	},
 	ErrAllAccessDisabled: {
 		Code:           "AllAccessDisabled",
-		Description:    "All access to this bucket has been disabled.",
+		Description:    "All access to this resource has been disabled.",
 		HTTPStatusCode: http.StatusForbidden,
 	},
 	ErrMalformedPolicy: {
@@ -888,6 +895,11 @@ var errorCodes = errorCodeMap{
 		Description:    "No matching replication rule found for this object prefix",
 		HTTPStatusCode: http.StatusBadRequest,
 	},
+	ErrReplicationDenyEditError: {
+		Code:           "XMinioReplicationDenyEdit",
+		Description:    "Cannot alter local replication config since this server is in a cluster replication setup",
+		HTTPStatusCode: http.StatusConflict,
+	},
 	ErrBucketRemoteIdenticalToSource: {
 		Code:           "XMinioAdminRemoteIdenticalToSource",
 		Description:    "The remote target cannot be identical to source",
@@ -2286,7 +2298,7 @@ func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID,
 		BucketName: reqInfo.BucketName,
 		Key:        reqInfo.ObjectName,
 		Resource:   resource,
-		Region:     globalServerRegion,
+		Region:     globalSite.Region,
 		RequestID:  requestID,
 		HostID:     hostID,
 	}

cmd/api-headers.go

@@ -52,7 +52,7 @@ func setCommonHeaders(w http.ResponseWriter) {
 
 	// Set `x-amz-bucket-region` only if region is set on the server
 	// by default minio uses an empty region.
-	if region := globalServerRegion; region != "" {
+	if region := globalSite.Region; region != "" {
 		w.Header().Set(xhttp.AmzBucketRegion, region)
 	}
 	w.Header().Set(xhttp.AcceptRanges, "bytes")

cmd/api-response.go

@@ -787,9 +787,9 @@ func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError
 		// https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After
 		w.Header().Set(xhttp.RetryAfter, "120")
 	case "InvalidRegion":
-		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalServerRegion)
+		err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalSite.Region)
 	case "AuthorizationHeaderMalformed":
-		err.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalServerRegion)
+		err.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region)
 	}
 
 	// Generate error response.

cmd/auth-handler.go

@@ -238,19 +238,6 @@ func getClaimsFromToken(token string) (map[string]interface{}, error) {
 		claims.MapClaims[iampolicy.SessionPolicyName] = string(spBytes)
 	}
 
-	// If LDAP claim key is set, return here.
-	if _, ok := claims.MapClaims[ldapUser]; ok {
-		return claims.Map(), nil
-	}
-
-	// Session token must have a policy, reject requests without policy
-	// claim.
-	_, pokOpenID := claims.MapClaims[iamPolicyClaimNameOpenID()]
-	_, pokSA := claims.MapClaims[iamPolicyClaimNameSA()]
-	if !pokOpenID && !pokSA {
-		return nil, errAuthentication
-	}
-
 	return claims.Map(), nil
 }
 
@@ -299,7 +286,7 @@ func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action
 		}
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeSigned, authTypePresigned:
-		region := globalServerRegion
+		region := globalSite.Region
 		switch action {
 		case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction:
 			region = ""
@@ -529,7 +516,7 @@ func validateSignature(atype authType, r *http.Request) (auth.Credentials, bool,
 		}
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypePresigned, authTypeSigned:
-		region := globalServerRegion
+		region := globalSite.Region
 		if s3Err = isReqAuthenticated(GlobalContext, r, region, serviceS3); s3Err != ErrNone {
 			return cred, owner, s3Err
 		}
@@ -596,7 +583,7 @@ func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectN
 	case authTypeSignedV2, authTypePresignedV2:
 		cred, owner, s3Err = getReqAccessKeyV2(r)
 	case authTypeStreamingSigned, authTypePresigned, authTypeSigned:
-		region := globalServerRegion
+		region := globalSite.Region
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 	}
 	if s3Err != ErrNone {

cmd/auth-handler_test.go

@@ -369,7 +369,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 
 	initConfigSubsystem(ctx, objLayer)
 
-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
 
 	creds, err := auth.CreateCredentials("myuser", "mypassword")
 	if err != nil {
@@ -397,7 +397,7 @@ func TestIsReqAuthenticated(t *testing.T) {
 
 	// Validates all testcases.
 	for i, testCase := range testCases {
-		s3Error := isReqAuthenticated(ctx, testCase.req, globalServerRegion, serviceS3)
+		s3Error := isReqAuthenticated(ctx, testCase.req, globalSite.Region, serviceS3)
 		if s3Error != testCase.s3Error {
 			if _, err := ioutil.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error {
 				t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code)
@@ -435,7 +435,7 @@ func TestCheckAdminRequestAuthType(t *testing.T) {
 	}
 	ctx := context.Background()
 	for i, testCase := range testCases {
-		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalServerRegion); s3Error != testCase.ErrCode {
+		if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, iampolicy.AllAdminActions, globalSite.Region); s3Error != testCase.ErrCode {
 			t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error)
 		}
 	}
@@ -459,7 +459,7 @@ func TestValidateAdminSignature(t *testing.T) {
 
 	initConfigSubsystem(ctx, objLayer)
 
-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
 
 	creds, err := auth.CreateCredentials("admin", "mypassword")
 	if err != nil {

cmd/background-newdisks-heal-ops.go

@@ -375,15 +375,13 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 
 			buckets, _ := z.ListBuckets(ctx)
 
-			buckets = append(buckets, BucketInfo{
-				Name: pathJoin(minioMetaBucket, minioConfigPrefix),
-			})
-
 			// Buckets data are dispersed in multiple zones/sets, make
 			// sure to heal all bucket metadata configuration.
-			buckets = append(buckets, []BucketInfo{
-				{Name: pathJoin(minioMetaBucket, bucketMetaPrefix)},
-			}...)
+			buckets = append(buckets, BucketInfo{
+				Name: pathJoin(minioMetaBucket, minioConfigPrefix),
+			}, BucketInfo{
+				Name: pathJoin(minioMetaBucket, bucketMetaPrefix),
+			})
 
 			// Heal latest buckets first.
 			sort.Slice(buckets, func(i, j int) bool {
@@ -441,10 +439,12 @@ func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools, bgSeq
 								continue
 							}
 
-							logger.Info("Healing disk '%s' on %s pool complete", disk, humanize.Ordinal(i+1))
+							logger.Info("Healing disk '%s' on %s pool, %s set complete", disk,
+								humanize.Ordinal(i+1), humanize.Ordinal(setIndex+1))
 							logger.Info("Summary:\n")
 							tracker.printTo(os.Stdout)
 							logger.LogIf(ctx, tracker.delete(ctx))
+							logger.Info("\n")
 
 							// Only upon success pop the healed disk.
 							globalBackgroundHealState.popHealLocalDisks(disk.Endpoint())

cmd/bootstrap-peer-server.go

@@ -86,14 +86,28 @@ func (s1 ServerSystemConfig) Diff(s2 ServerSystemConfig) error {
 		}
 	}
 	if !reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) {
-		return fmt.Errorf("Expected same MINIO_ environment variables and values")
+		var missing []string
+		var mismatching []string
+		for k, v := range s1.MinioEnv {
+			ev, ok := s2.MinioEnv[k]
+			if !ok {
+				missing = append(missing, k)
+			} else if v != ev {
+				mismatching = append(mismatching, k)
+			}
+		}
+		if len(mismatching) > 0 {
+			return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s / Mismatch environment values: %s`, missing, mismatching)
+		}
+		return fmt.Errorf(`Expected same MINIO_ environment variables and values across all servers: Missing environment values: %s`, missing)
 	}
 	return nil
 }
 
 var skipEnvs = map[string]struct{}{
-	"MINIO_OPTS":        {},
-	"MINIO_CERT_PASSWD": {},
+	"MINIO_OPTS":         {},
+	"MINIO_CERT_PASSWD":  {},
+	"MINIO_SERVER_DEBUG": {},
 }
 
 func getServerSystemCfg() ServerSystemConfig {

cmd/bucket-handlers.go

@@ -211,7 +211,7 @@ func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *
 	// Generate response.
 	encodedSuccessResponse := encodeResponse(LocationResponse{})
 	// Get current region.
-	region := globalServerRegion
+	region := globalSite.Region
 	if region != globalMinioDefaultRegion {
 		encodedSuccessResponse = encodeResponse(LocationResponse{
 			Location: region,
@@ -1210,7 +1210,7 @@ func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	writeSuccessResponseHeadersOnly(w)
+	writeResponse(w, http.StatusOK, nil, mimeXML)
 }
 
 // DeleteBucketHandler - Delete bucket
@@ -1280,7 +1280,7 @@ func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.
 		}
 		if globalDNSConfig != nil {
 			if err2 := globalDNSConfig.Put(bucket); err2 != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, pl1ease fix it manually", err2))
+				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, please fix it manually", err2))
 			}
 		}
 		writeErrorResponse(ctx, w, apiErr, r.URL)
@@ -1586,7 +1586,10 @@ func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWr
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
+		return
+	}
 	if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned {
 		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL)
 		return
@@ -1688,6 +1691,10 @@ func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.Respons
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL)
+		return
+	}
 	if err := globalBucketMetadataSys.Update(bucket, bucketReplicationConfig, nil); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return

cmd/bucket-lifecycle-handlers.go

@@ -24,7 +24,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/minio/internal/bucket/lifecycle"
-	"github.com/minio/minio/internal/bucket/object/lock"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
@@ -80,17 +79,6 @@ func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r
 		return
 	}
 
-	// Disallow MaxNoncurrentVersions if bucket has object locking enabled
-	var rCfg lock.Retention
-	if rCfg, err = globalBucketObjectLockSys.Get(bucket); err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-		return
-	}
-	if rCfg.LockEnabled && bucketLifecycle.HasMaxNoncurrentVersions() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidLifecycleWithObjectLock), r.URL)
-		return
-	}
-
 	// Validate the transition storage ARNs
 	if err = validateTransitionTier(bucketLifecycle); err != nil {
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)

cmd/bucket-lifecycle.go

@@ -81,21 +81,21 @@ type expiryTask struct {
 }
 
 type expiryState struct {
-	once              sync.Once
-	byDaysCh          chan expiryTask
-	byMaxNoncurrentCh chan maxNoncurrentTask
+	once                sync.Once
+	byDaysCh            chan expiryTask
+	byNewerNoncurrentCh chan newerNoncurrentTask
 }
 
 // PendingTasks returns the number of pending ILM expiry tasks.
 func (es *expiryState) PendingTasks() int {
-	return len(es.byDaysCh) + len(es.byMaxNoncurrentCh)
+	return len(es.byDaysCh) + len(es.byNewerNoncurrentCh)
 }
 
 // close closes work channels exactly once.
 func (es *expiryState) close() {
 	es.once.Do(func() {
 		close(es.byDaysCh)
-		close(es.byMaxNoncurrentCh)
+		close(es.byNewerNoncurrentCh)
 	})
 }
 
@@ -109,13 +109,13 @@ func (es *expiryState) enqueueByDays(oi ObjectInfo, restoredObject bool, rmVersi
 	}
 }
 
-// enqueueByMaxNoncurrent enqueues object versions expired by
-// MaxNoncurrentVersions limit for expiry.
-func (es *expiryState) enqueueByMaxNoncurrent(bucket string, versions []ObjectToDelete) {
+// enqueueByNewerNoncurrent enqueues object versions expired by
+// NewerNoncurrentVersions limit for expiry.
+func (es *expiryState) enqueueByNewerNoncurrent(bucket string, versions []ObjectToDelete) {
 	select {
 	case <-GlobalContext.Done():
 		es.close()
-	case es.byMaxNoncurrentCh <- maxNoncurrentTask{bucket: bucket, versions: versions}:
+	case es.byNewerNoncurrentCh <- newerNoncurrentTask{bucket: bucket, versions: versions}:
 	default:
 	}
 }
@@ -124,8 +124,8 @@ var globalExpiryState *expiryState
 
 func newExpiryState() *expiryState {
 	return &expiryState{
-		byDaysCh:          make(chan expiryTask, 10000),
-		byMaxNoncurrentCh: make(chan maxNoncurrentTask, 10000),
+		byDaysCh:            make(chan expiryTask, 10000),
+		byNewerNoncurrentCh: make(chan newerNoncurrentTask, 10000),
 	}
 }
 
@@ -141,15 +141,15 @@ func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) {
 		}
 	}()
 	go func() {
-		for t := range globalExpiryState.byMaxNoncurrentCh {
+		for t := range globalExpiryState.byNewerNoncurrentCh {
 			deleteObjectVersions(ctx, objectAPI, t.bucket, t.versions)
 		}
 	}()
 }
 
-// maxNoncurrentTask encapsulates arguments required by worker to expire objects
-// by MaxNoncurrentVersions
-type maxNoncurrentTask struct {
+// newerNoncurrentTask encapsulates arguments required by worker to expire objects
+// by NewerNoncurrentVersions
+type newerNoncurrentTask struct {
 	bucket   string
 	versions []ObjectToDelete
 }

cmd/bucket-notification-handlers.go

@@ -72,8 +72,8 @@ func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter,
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-	config.SetRegion(globalServerRegion)
-	if err = config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil {
+	config.SetRegion(globalSite.Region)
+	if err = config.Validate(globalSite.Region, globalNotificationSys.targetList); err != nil {
 		arnErr, ok := err.(*event.ErrARNNotFound)
 		if ok {
 			for i, queue := range config.QueueList {
@@ -145,7 +145,7 @@ func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter,
 		return
 	}
 
-	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalServerRegion, globalNotificationSys.targetList)
+	config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region, globalNotificationSys.targetList)
 	if err != nil {
 		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
 		if event.IsEventError(err) {

cmd/bucket-replication-stats.go

@@ -51,6 +51,9 @@ func (r *ReplicationStats) Delete(bucket string) {
 	r.Lock()
 	defer r.Unlock()
 	delete(r.Cache, bucket)
+
+	r.ulock.Lock()
+	defer r.ulock.Unlock()
 	delete(r.UsageCache, bucket)
 
 }
@@ -82,10 +85,12 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
 	bs, ok := r.Cache[bucket]
 	if !ok {
 		bs = &BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+		r.Cache[bucket] = bs
 	}
 	b, ok := bs.Stats[arn]
 	if !ok {
 		b = &BucketReplicationStat{}
+		bs.Stats[arn] = b
 	}
 	switch status {
 	case replication.Completed:
@@ -115,9 +120,6 @@ func (r *ReplicationStats) Update(bucket string, arn string, n int64, duration t
 			b.ReplicaSize += n
 		}
 	}
-
-	bs.Stats[arn] = b
-	r.Cache[bucket] = bs
 }
 
 // GetInitialUsage get replication metrics available at the time of cluster initialization
@@ -128,10 +130,10 @@ func (r *ReplicationStats) GetInitialUsage(bucket string) BucketReplicationStats
 	r.ulock.RLock()
 	defer r.ulock.RUnlock()
 	st, ok := r.UsageCache[bucket]
-	if ok {
-		return st.Clone()
+	if !ok {
+		return BucketReplicationStats{}
 	}
-	return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+	return st.Clone()
 }
 
 // Get replication metrics for a bucket from this node since this node came up.
@@ -145,7 +147,7 @@ func (r *ReplicationStats) Get(bucket string) BucketReplicationStats {
 
 	st, ok := r.Cache[bucket]
 	if !ok {
-		return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)}
+		return BucketReplicationStats{}
 	}
 	return st.Clone()
 }
@@ -162,41 +164,46 @@ func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *Replicatio
 func (r *ReplicationStats) loadInitialReplicationMetrics(ctx context.Context) {
 	rTimer := time.NewTimer(time.Minute * 1)
 	defer rTimer.Stop()
+	var (
+		dui DataUsageInfo
+		err error
+	)
+outer:
 	for {
 		select {
 		case <-ctx.Done():
 			return
 		case <-rTimer.C:
-			dui, err := loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
+			dui, err = loadDataUsageFromBackend(GlobalContext, newObjectLayerFn())
 			if err != nil {
 				continue
 			}
-			// data usage has not captured any data yet.
-			if dui.LastUpdate.IsZero() {
-				continue
+			// If LastUpdate is set, data usage is available.
+			if !dui.LastUpdate.IsZero() {
+				break outer
 			}
-			m := make(map[string]*BucketReplicationStats)
-			for bucket, usage := range dui.BucketsUsage {
-				b := &BucketReplicationStats{
-					Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
-				}
-				for arn, uinfo := range usage.ReplicationInfo {
-					b.Stats[arn] = &BucketReplicationStat{
-						FailedSize:     int64(uinfo.ReplicationFailedSize),
-						ReplicatedSize: int64(uinfo.ReplicatedSize),
-						ReplicaSize:    int64(uinfo.ReplicaSize),
-						FailedCount:    int64(uinfo.ReplicationFailedCount),
-					}
-				}
-				b.ReplicaSize += int64(usage.ReplicaSize)
-				if b.hasReplicationUsage() {
-					m[bucket] = b
-				}
-			}
-			r.ulock.Lock()
-			defer r.ulock.Unlock()
-			r.UsageCache = m
-			return
 		}
 	}
+
+	m := make(map[string]*BucketReplicationStats)
+	for bucket, usage := range dui.BucketsUsage {
+		b := &BucketReplicationStats{
+			Stats: make(map[string]*BucketReplicationStat, len(usage.ReplicationInfo)),
+		}
+		for arn, uinfo := range usage.ReplicationInfo {
+			b.Stats[arn] = &BucketReplicationStat{
+				FailedSize:     int64(uinfo.ReplicationFailedSize),
+				ReplicatedSize: int64(uinfo.ReplicatedSize),
+				ReplicaSize:    int64(uinfo.ReplicaSize),
+				FailedCount:    int64(uinfo.ReplicationFailedCount),
+			}
+		}
+		b.ReplicaSize += int64(usage.ReplicaSize)
+		if b.hasReplicationUsage() {
+			m[bucket] = b
+		}
+	}
+	r.ulock.Lock()
+	defer r.ulock.Unlock()
+	r.UsageCache = m
 }

cmd/bucket-replication.go

@@ -62,6 +62,8 @@ const (
 	ObjectLockRetentionTimestamp = "objectlock-retention-timestamp"
 	// ObjectLockLegalHoldTimestamp - the last time a legal hold metadata modification happened on this cluster for this object version
 	ObjectLockLegalHoldTimestamp = "objectlock-legalhold-timestamp"
+	// ReplicationWorkerMultiplier is suggested worker multiplier if traffic exceeds replication worker capacity
+	ReplicationWorkerMultiplier = 1.5
 )
 
 // gets replication config associated to a given bucket name.
@@ -515,8 +517,8 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 		}
 		return
 	}
-	// early return if already replicated delete marker for existing object replication
-	if dobj.DeleteMarkerVersionID != "" && dobj.OpType == replication.ExistingObjectReplicationType {
+	// early return if already replicated delete marker for existing object replication/ healing delete markers
+	if dobj.DeleteMarkerVersionID != "" && (dobj.OpType == replication.ExistingObjectReplicationType || dobj.OpType == replication.HealReplicationType) {
 		if _, err := tgt.StatObject(ctx, tgt.Bucket, dobj.ObjectName, miniogo.StatObjectOptions{
 			VersionID: versionID,
 			Internal: miniogo.AdvancedGetOptions{
@@ -524,10 +526,8 @@ func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationI
 			}}); isErrMethodNotAllowed(ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName)) {
 			if dobj.VersionID == "" {
 				rinfo.ReplicationStatus = replication.Completed
-			} else {
-				rinfo.VersionPurgeStatus = Complete
+				return
 			}
-			return
 		}
 	}
 
@@ -1412,6 +1412,14 @@ func (p *ReplicationPool) ResizeFailedWorkers(n int) {
 	}
 }
 
+// suggestedWorkers recommends an increase in number of workers to meet replication load.
+func (p *ReplicationPool) suggestedWorkers(failQueue bool) int {
+	if failQueue {
+		return int(float64(p.mrfWorkerSize) * ReplicationWorkerMultiplier)
+	}
+	return int(float64(p.workerSize) * ReplicationWorkerMultiplier)
+}
+
 func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 	if p == nil {
 		return
@@ -1425,6 +1433,7 @@ func (p *ReplicationPool) queueReplicaFailedTask(ri ReplicateObjectInfo) {
 		})
 	case p.mrfReplicaCh <- ri:
 	default:
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication failed workers could not keep up with healing failures - consider increasing number of replication failed workers with `mc admin config set api replication_failed_workers=%d`", p.suggestedWorkers(true)), replicationSubsystem)
 	}
 }
 
@@ -1450,6 +1459,7 @@ func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) {
 		})
 	case ch <- ri:
 	default:
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication workers could not keep up with incoming traffic - consider increasing number of replication workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
 	}
 }
 
@@ -1486,6 +1496,7 @@ func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInf
 		})
 	case ch <- doi:
 	default:
+		logger.LogOnceIf(GlobalContext, fmt.Errorf("WARNING: Replication workers could not keep up with incoming traffic - consider increasing number of replication workers with `mc admin config set api replication_workers=%d`", p.suggestedWorkers(false)), replicationSubsystem)
 	}
 }
 
@@ -1797,22 +1808,15 @@ func getLatestReplicationStats(bucket string, u BucketUsageInfo) (s BucketReplic
 	// add initial usage stat to cluster stats
 	usageStat := globalReplicationStats.GetInitialUsage(bucket)
 	totReplicaSize += usageStat.ReplicaSize
-	if usageStat.Stats != nil {
-		for arn, stat := range usageStat.Stats {
-			st := stats[arn]
-			if st == nil {
-				st = &BucketReplicationStat{
-					ReplicatedSize: stat.ReplicatedSize,
-					FailedSize:     stat.FailedSize,
-					FailedCount:    stat.FailedCount,
-				}
-			} else {
-				st.ReplicatedSize += stat.ReplicatedSize
-				st.FailedSize += stat.FailedSize
-				st.FailedCount += stat.FailedCount
-			}
+	for arn, stat := range usageStat.Stats {
+		st, ok := stats[arn]
+		if !ok {
+			st = &BucketReplicationStat{}
 			stats[arn] = st
 		}
+		st.ReplicatedSize += stat.ReplicatedSize
+		st.FailedSize += stat.FailedSize
+		st.FailedCount += stat.FailedCount
 	}
 	s = BucketReplicationStats{
 		Stats: make(map[string]*BucketReplicationStat, len(stats)),

cmd/bucket-stats.go

@@ -18,7 +18,6 @@
 package cmd
 
 import (
-	"sync/atomic"
 	"time"
 )
 
@@ -52,13 +51,6 @@ func (rl *ReplicationLatency) update(size int64, duration time.Duration) {
 	rl.UploadHistogram.Add(size, duration)
 }
 
-// Clone replication latency
-func (rl ReplicationLatency) clone() ReplicationLatency {
-	return ReplicationLatency{
-		UploadHistogram: rl.UploadHistogram.Clone(),
-	}
-}
-
 // BucketStats bucket statistics
 type BucketStats struct {
 	ReplicationStats BucketReplicationStats
@@ -88,29 +80,18 @@ func (brs *BucketReplicationStats) Empty() bool {
 }
 
 // Clone creates a new BucketReplicationStats copy
-func (brs BucketReplicationStats) Clone() BucketReplicationStats {
-	c := BucketReplicationStats{
-		Stats: make(map[string]*BucketReplicationStat, len(brs.Stats)),
-	}
-	// This is called only by replicationStats cache and already holds a read lock before calling Clone()
+func (brs BucketReplicationStats) Clone() (c BucketReplicationStats) {
+	// This is called only by replicationStats cache and already holds a
+	// read lock before calling Clone()
+
+	c = brs
+	// We need to copy the map, so we do not reference the one in `brs`.
+	c.Stats = make(map[string]*BucketReplicationStat, len(brs.Stats))
 	for arn, st := range brs.Stats {
-		c.Stats[arn] = &BucketReplicationStat{
-			FailedSize:     atomic.LoadInt64(&st.FailedSize),
-			ReplicatedSize: atomic.LoadInt64(&st.ReplicatedSize),
-			ReplicaSize:    atomic.LoadInt64(&st.ReplicaSize),
-			FailedCount:    atomic.LoadInt64(&st.FailedCount),
-			PendingSize:    atomic.LoadInt64(&st.PendingSize),
-			PendingCount:   atomic.LoadInt64(&st.PendingCount),
-			Latency:        st.Latency.clone(),
-		}
+		// make a copy of `*st`
+		s := *st
+		c.Stats[arn] = &s
 	}
-	// update total counts across targets
-	c.FailedSize = atomic.LoadInt64(&brs.FailedSize)
-	c.FailedCount = atomic.LoadInt64(&brs.FailedCount)
-	c.PendingCount = atomic.LoadInt64(&brs.PendingCount)
-	c.PendingSize = atomic.LoadInt64(&brs.PendingSize)
-	c.ReplicaSize = atomic.LoadInt64(&brs.ReplicaSize)
-	c.ReplicatedSize = atomic.LoadInt64(&brs.ReplicatedSize)
 	return c
 }
 

cmd/bucket-versioning-handler.go

@@ -63,6 +63,15 @@ func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r
 		return
 	}
 
+	if globalSiteReplicationSys.isEnabled() {
+		writeErrorResponse(ctx, w, APIError{
+			Code:           "InvalidBucketState",
+			Description:    "Cluster replication is enabled for this site, so the versioning state cannot be changed.",
+			HTTPStatusCode: http.StatusConflict,
+		}, r.URL)
+		return
+	}
+
 	if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() {
 		writeErrorResponse(ctx, w, APIError{
 			Code:           "InvalidBucketState",

cmd/common-main.go

@@ -18,12 +18,15 @@
 package cmd
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/gob"
 	"errors"
 	"fmt"
+	"io/ioutil"
 	"math/rand"
 	"net"
 	"net/http"
@@ -34,6 +37,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"syscall"
 	"time"
 
 	fcolor "github.com/fatih/color"
@@ -64,6 +68,7 @@ import (
 
 // serverDebugLog will enable debug printing
 var serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn
+var shardDiskTimeDelta time.Duration
 var defaultAWSCredProvider []credentials.Provider
 
 func init() {
@@ -133,6 +138,12 @@ func init() {
 		},
 	}
 
+	var err error
+	shardDiskTimeDelta, err = time.ParseDuration(env.Get("_MINIO_SHARD_DISKTIME_DELTA", "1m"))
+	if err != nil {
+		shardDiskTimeDelta = 1 * time.Minute
+	}
+
 	// All minio-go API operations shall be performed only once,
 	// another way to look at this is we are turning off retries.
 	minio.MaxRetry = 1
@@ -187,11 +198,14 @@ func minioConfigToConsoleFeatures() {
 			os.Setenv("CONSOLE_IDP_CALLBACK", getConsoleEndpoints()[0]+"/oauth_callback")
 		}
 	}
-	os.Setenv("CONSOLE_MINIO_REGION", globalServerRegion)
+	os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
 	if globalSubnetConfig.License != "" {
 		os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetConfig.License)
 	}
+	if globalSubnetConfig.APIKey != "" {
+		os.Setenv("CONSOLE_SUBNET_API_KEY", globalSubnetConfig.APIKey)
+	}
 }
 
 func initConsoleServer() (*restapi.Server, error) {
@@ -303,7 +317,7 @@ func checkUpdate(mode string) {
 		return
 	}
 
-	logStartupMessage(prepareUpdateMessage("Run `mc admin update`", lrTime.Sub(crTime)))
+	logStartupMessage(prepareUpdateMessage("\nRun `mc admin update`", lrTime.Sub(crTime)))
 }
 
 func newConfigDirFromCtx(ctx *cli.Context, option string, getDefaultDir func() string) (*ConfigDir, bool) {
@@ -429,7 +443,166 @@ func handleCommonCmdArgs(ctx *cli.Context) {
 	logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get())
 }
 
+type envKV struct {
+	Key   string
+	Value string
+	Skip  bool
+}
+
+func (e envKV) String() string {
+	if e.Skip {
+		return ""
+	}
+	return fmt.Sprintf("%s=%s", e.Key, e.Value)
+}
+
+func parsEnvEntry(envEntry string) (envKV, error) {
+	envEntry = strings.TrimSpace(envEntry)
+	if envEntry == "" {
+		// Skip all empty lines
+		return envKV{
+			Skip: true,
+		}, nil
+	}
+	const envSeparator = "="
+	envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), envSeparator, 2)
+	if len(envTokens) != 2 {
+		return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry)
+	}
+
+	key := envTokens[0]
+	val := envTokens[1]
+
+	// Remove quotes from the value if found
+	if len(val) >= 2 {
+		quote := val[0]
+		if (quote == '"' || quote == '\'') && val[len(val)-1] == quote {
+			val = val[1 : len(val)-1]
+		}
+	}
+
+	return envKV{
+		Key:   key,
+		Value: val,
+	}, nil
+}
+
+// Similar to os.Environ returns a copy of strings representing
+// the environment values from a file, in the form "key, value".
+// in a structured form.
+func minioEnvironFromFile(envConfigFile string) ([]envKV, error) {
+	f, err := os.Open(envConfigFile)
+	if err != nil {
+		if os.IsNotExist(err) { // ignore if file doesn't exist.
+			return nil, nil
+		}
+		return nil, err
+	}
+	defer f.Close()
+	var ekvs []envKV
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		ekv, err := parsEnvEntry(scanner.Text())
+		if err != nil {
+			return nil, err
+		}
+		if ekv.Skip {
+			// Skips empty lines
+			continue
+		}
+		ekvs = append(ekvs, ekv)
+	}
+	if err = scanner.Err(); err != nil {
+		return nil, err
+	}
+	return ekvs, nil
+}
+
+func readFromSecret(sp string) (string, error) {
+	// Supports reading path from docker secrets, filename is
+	// relative to /run/secrets/ position.
+	if isFile(pathJoin("/run/secrets/", sp)) {
+		sp = pathJoin("/run/secrets/", sp)
+	}
+	credBuf, err := ioutil.ReadFile(sp)
+	if err != nil {
+		if os.IsNotExist(err) { // ignore if file doesn't exist.
+			return "", nil
+		}
+		return "", err
+	}
+	return string(bytes.TrimSpace(credBuf)), nil
+}
+
+func loadEnvVarsFromFiles() {
+	if env.IsSet(config.EnvAccessKeyFile) {
+		accessKey, err := readFromSecret(env.Get(config.EnvAccessKeyFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if accessKey != "" {
+			os.Setenv(config.EnvRootUser, accessKey)
+		}
+	}
+
+	if env.IsSet(config.EnvSecretKeyFile) {
+		secretKey, err := readFromSecret(env.Get(config.EnvSecretKeyFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if secretKey != "" {
+			os.Setenv(config.EnvRootPassword, secretKey)
+		}
+	}
+
+	if env.IsSet(config.EnvRootUserFile) {
+		rootUser, err := readFromSecret(env.Get(config.EnvRootUserFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if rootUser != "" {
+			os.Setenv(config.EnvRootUser, rootUser)
+		}
+	}
+
+	if env.IsSet(config.EnvRootPasswordFile) {
+		rootPassword, err := readFromSecret(env.Get(config.EnvRootPasswordFile, ""))
+		if err != nil {
+			logger.Fatal(config.ErrInvalidCredentials(err),
+				"Unable to validate credentials inherited from the secret file(s)")
+		}
+		if rootPassword != "" {
+			os.Setenv(config.EnvRootPassword, rootPassword)
+		}
+	}
+
+	if env.IsSet(config.EnvKMSSecretKeyFile) {
+		kmsSecret, err := readFromSecret(env.Get(config.EnvKMSSecretKeyFile, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to read the KMS secret key inherited from secret file")
+		}
+		if kmsSecret != "" {
+			os.Setenv(config.EnvKMSSecretKey, kmsSecret)
+		}
+	}
+
+	if env.IsSet(config.EnvConfigEnvFile) {
+		ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, ""))
+		if err != nil {
+			logger.Fatal(err, "Unable to read the config environment file")
+		}
+		for _, ekv := range ekvs {
+			os.Setenv(ekv.Key, ekv.Value)
+		}
+	}
+}
+
 func handleCommonEnvVars() {
+	loadEnvVarsFromFiles()
+
 	var err error
 	globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn))
 	if err != nil {
@@ -526,8 +699,8 @@ func handleCommonEnvVars() {
 	// in-place update is off.
 	globalInplaceUpdateDisabled = strings.EqualFold(env.Get(config.EnvUpdate, config.EnableOn), config.EnableOff)
 
-	// Check if the supported credential env vars, "MINIO_ROOT_USER" and
-	// "MINIO_ROOT_PASSWORD" are provided
+	// Check if the supported credential env vars,
+	// "MINIO_ROOT_USER" and "MINIO_ROOT_PASSWORD" are provided
 	// Warn user if deprecated environment variables,
 	// "MINIO_ACCESS_KEY" and "MINIO_SECRET_KEY", are defined
 	// Check all error conditions first
@@ -548,25 +721,24 @@ func handleCommonEnvVars() {
 	// are defined or both are not defined.
 	// Check both cases and authenticate them if correctly defined
 	var user, password string
-	haveRootCredentials := false
-	haveAccessCredentials := false
+	var hasCredentials bool
 	//nolint:gocritic
 	if env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) {
 		user = env.Get(config.EnvRootUser, "")
 		password = env.Get(config.EnvRootPassword, "")
-		haveRootCredentials = true
+		hasCredentials = true
 	} else if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 		user = env.Get(config.EnvAccessKey, "")
 		password = env.Get(config.EnvSecretKey, "")
-		haveAccessCredentials = true
+		hasCredentials = true
 	}
-	if haveRootCredentials || haveAccessCredentials {
+	if hasCredentials {
 		cred, err := auth.CreateCredentials(user, password)
 		if err != nil {
 			logger.Fatal(config.ErrInvalidCredentials(err),
 				"Unable to validate credentials inherited from the shell environment")
 		}
-		if haveAccessCredentials {
+		if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) {
 			msg := fmt.Sprintf("WARNING: %s and %s are deprecated.\n"+
 				"         Please use %s and %s",
 				config.EnvAccessKey, config.EnvSecretKey,
@@ -718,6 +890,10 @@ func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secu
 		}
 	}
 	secureConn = true
+
+	// syscall.SIGHUP to reload the certs.
+	manager.ReloadOnSignal(syscall.SIGHUP)
+
 	return x509Certs, manager, secureConn, nil
 }
 

cmd/common-main_test.go

@@ -0,0 +1,156 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"errors"
+	"io/ioutil"
+	"reflect"
+	"testing"
+)
+
+func Test_readFromSecret(t *testing.T) {
+	testCases := []struct {
+		content       string
+		expectedErr   bool
+		expectedValue string
+	}{
+		{
+			"value\n",
+			false,
+			"value",
+		},
+		{
+			" \t\n Hello, Gophers \n\t\r\n",
+			false,
+			"Hello, Gophers",
+		},
+	}
+
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			tmpfile, err := ioutil.TempFile("", "testfile")
+			if err != nil {
+				t.Error(err)
+			}
+			tmpfile.WriteString(testCase.content)
+			tmpfile.Sync()
+			tmpfile.Close()
+
+			value, err := readFromSecret(tmpfile.Name())
+			if err != nil && !testCase.expectedErr {
+				t.Error(err)
+			}
+			if err == nil && testCase.expectedErr {
+				t.Error(errors.New("expected error, found success"))
+			}
+			if value != testCase.expectedValue {
+				t.Errorf("Expected %s, got %s", testCase.expectedValue, value)
+			}
+		})
+	}
+}
+
+func Test_minioEnvironFromFile(t *testing.T) {
+	testCases := []struct {
+		content      string
+		expectedErr  bool
+		expectedEkvs []envKV
+	}{
+		{`
+export MINIO_ROOT_USER=minio
+export MINIO_ROOT_PASSWORD=minio123`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+				{
+					Key:   "MINIO_ROOT_PASSWORD",
+					Value: "minio123",
+				},
+			},
+		},
+		// Value with double quotes
+		{`export MINIO_ROOT_USER="minio"`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+			},
+		},
+		// Value with single quotes
+		{`export MINIO_ROOT_USER='minio'`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+			},
+		},
+		{`
+MINIO_ROOT_USER=minio
+MINIO_ROOT_PASSWORD=minio123`,
+			false,
+			[]envKV{
+				{
+					Key:   "MINIO_ROOT_USER",
+					Value: "minio",
+				},
+				{
+					Key:   "MINIO_ROOT_PASSWORD",
+					Value: "minio123",
+				},
+			},
+		},
+		{`
+export MINIO_ROOT_USERminio
+export MINIO_ROOT_PASSWORD=minio123`,
+			true,
+			nil,
+		},
+	}
+	for _, testCase := range testCases {
+		testCase := testCase
+		t.Run("", func(t *testing.T) {
+			tmpfile, err := ioutil.TempFile("", "testfile")
+			if err != nil {
+				t.Error(err)
+			}
+			tmpfile.WriteString(testCase.content)
+			tmpfile.Sync()
+			tmpfile.Close()
+
+			ekvs, err := minioEnvironFromFile(tmpfile.Name())
+			if err != nil && !testCase.expectedErr {
+				t.Error(err)
+			}
+			if err == nil && testCase.expectedErr {
+				t.Error(errors.New("expected error, found success"))
+			}
+			if !reflect.DeepEqual(ekvs, testCase.expectedEkvs) {
+				t.Errorf("expected %v, got %v", testCase.expectedEkvs, ekvs)
+			}
+		})
+	}
+}

cmd/config-common.go

@@ -29,27 +29,31 @@ import (
 
 var errConfigNotFound = errors.New("config file not found")
 
-func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
-	// Read entire content by setting size to -1
+func readConfigWithMetadata(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, ObjectInfo, error) {
 	r, err := objAPI.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {
 		// Treat object not found as config not found.
 		if isErrObjectNotFound(err) {
-			return nil, errConfigNotFound
+			return nil, ObjectInfo{}, errConfigNotFound
 		}
 
-		return nil, err
+		return nil, ObjectInfo{}, err
 	}
 	defer r.Close()
 
 	buf, err := ioutil.ReadAll(r)
 	if err != nil {
-		return nil, err
+		return nil, ObjectInfo{}, err
 	}
 	if len(buf) == 0 {
-		return nil, errConfigNotFound
+		return nil, ObjectInfo{}, errConfigNotFound
 	}
-	return buf, nil
+	return buf, r.ObjInfo, nil
+}
+
+func readConfig(ctx context.Context, objAPI ObjectLayer, configFile string) ([]byte, error) {
+	buf, _, err := readConfigWithMetadata(ctx, objAPI, configFile)
+	return buf, err
 }
 
 type objectDeleter interface {

cmd/config-current.go

@@ -58,10 +58,11 @@ func initHelp() {
 		config.IdentityOpenIDSubSys: openid.DefaultKVS,
 		config.IdentityTLSSubSys:    xtls.DefaultKVS,
 		config.PolicyOPASubSys:      opa.DefaultKVS,
+		config.SiteSubSys:           config.DefaultSiteKVS,
 		config.RegionSubSys:         config.DefaultRegionKVS,
 		config.APISubSys:            api.DefaultKVS,
 		config.CredentialsSubSys:    config.DefaultCredentialKVS,
-		config.LoggerWebhookSubSys:  logger.DefaultKVS,
+		config.LoggerWebhookSubSys:  logger.DefaultLoggerWebhookKVS,
 		config.AuditWebhookSubSys:   logger.DefaultAuditWebhookKVS,
 		config.AuditKafkaSubSys:     logger.DefaultAuditKafkaKVS,
 		config.HealSubSys:           heal.DefaultKVS,
@@ -79,8 +80,8 @@ func initHelp() {
 	// Captures help for each sub-system
 	var helpSubSys = config.HelpKVS{
 		config.HelpKV{
-			Key:         config.RegionSubSys,
-			Description: "label the location of the server",
+			Key:         config.SiteSubSys,
+			Description: "label the server and its location",
 		},
 		config.HelpKV{
 			Key:         config.CacheSubSys,
@@ -190,7 +191,7 @@ func initHelp() {
 		config.HelpKV{
 			Key:         config.SubnetSubSys,
 			Type:        "string",
-			Description: "set subnet config for the cluster e.g. license token",
+			Description: "set subnet config for the cluster e.g. api key",
 			Optional:    true,
 		},
 	}
@@ -206,6 +207,7 @@ func initHelp() {
 
 	var helpMap = map[string]config.HelpKVS{
 		"":                          helpSubSys, // Help for all sub-systems.
+		config.SiteSubSys:           config.SiteHelp,
 		config.RegionSubSys:         config.RegionHelp,
 		config.APISubSys:            api.Help,
 		config.StorageClassSubSys:   storageclass.Help,
@@ -231,10 +233,20 @@ func initHelp() {
 		config.NotifyRedisSubSys:    notify.HelpRedis,
 		config.NotifyWebhookSubSys:  notify.HelpWebhook,
 		config.NotifyESSubSys:       notify.HelpES,
-		config.SubnetSubSys:         subnet.HelpLicense,
+		config.SubnetSubSys:         subnet.HelpSubnet,
 	}
 
 	config.RegisterHelpSubSys(helpMap)
+
+	// save top-level help for deprecated sub-systems in a separate map.
+	deprecatedHelpKVMap := map[string]config.HelpKV{
+		config.RegionSubSys: {
+			Key:         config.RegionSubSys,
+			Description: "[DEPRECATED - use `site` instead] label the location of the server",
+		},
+	}
+
+	config.RegisterHelpDeprecatedSubSys(deprecatedHelpKVMap)
 }
 
 var (
@@ -259,7 +271,7 @@ func validateConfig(s config.Config) error {
 		return err
 	}
 
-	if _, err := config.LookupRegion(s[config.RegionSubSys][config.Default]); err != nil {
+	if _, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]); err != nil {
 		return err
 	}
 
@@ -315,7 +327,7 @@ func validateConfig(s config.Config) error {
 		}
 	}
 	if _, err := openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody); err != nil {
+		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region); err != nil {
 		return err
 	}
 
@@ -437,9 +449,9 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	// but not federation.
 	globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled
 
-	globalServerRegion, err = config.LookupRegion(s[config.RegionSubSys][config.Default])
+	globalSite, err = config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default])
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Invalid region configuration: %w", err))
+		logger.LogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err))
 	}
 
 	apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default])
@@ -501,7 +513,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 	}
 
 	globalOpenIDConfig, err = openid.LookupConfig(s[config.IdentityOpenIDSubSys][config.Default],
-		NewGatewayHTTPTransport(), xhttp.DrainBody)
+		NewGatewayHTTPTransport(), xhttp.DrainBody, globalSite.Region)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err))
 	}
@@ -531,7 +543,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 
 	loggerCfg, err := logger.LookupConfig(s)
 	if err != nil {
-		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger: %w", err))
+		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger/audit targets: %w", err))
 	}
 
 	for _, l := range loggerCfg.HTTP {
@@ -541,7 +553,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			// Enable http logging
 			if err = logger.AddTarget(http.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize console HTTP target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server logger HTTP target: %w", err))
 			}
 		}
 	}
@@ -553,7 +565,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			// Enable http audit logging
 			if err = logger.AddAuditTarget(http.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit HTTP target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server audit HTTP target: %w", err))
 			}
 		}
 	}
@@ -563,7 +575,7 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 			l.LogOnce = logger.LogOnceIf
 			// Enable Kafka audit logging
 			if err = logger.AddAuditTarget(kafka.New(l)); err != nil {
-				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit Kafka target: %w", err))
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize server audit Kafka target: %w", err))
 			}
 		}
 	}
@@ -674,7 +686,10 @@ func GetHelp(subSys, key string, envOnly bool) (Help, error) {
 
 	subSysHelp, ok := config.HelpSubSysMap[""].Lookup(subSys)
 	if !ok {
-		return Help{}, config.Errorf("unknown sub-system %s", subSys)
+		subSysHelp, ok = config.HelpDeprecatedSubSysMap[subSys]
+		if !ok {
+			return Help{}, config.Errorf("unknown sub-system %s", subSys)
+		}
 	}
 
 	h, ok := config.HelpSubSysMap[subSys]

cmd/config-current_test.go

@@ -36,18 +36,21 @@ func TestServerConfig(t *testing.T) {
 		t.Fatalf("Init Test config failed")
 	}
 
-	if globalServerRegion != globalMinioDefaultRegion {
-		t.Errorf("Expecting region `us-east-1` found %s", globalServerRegion)
+	if globalSite.Region != globalMinioDefaultRegion {
+		t.Errorf("Expecting region `us-east-1` found %s", globalSite.Region)
 	}
 
 	// Set new region and verify.
 	config.SetRegion(globalServerConfig, "us-west-1")
-	region, err := config.LookupRegion(globalServerConfig[config.RegionSubSys][config.Default])
+	site, err := config.LookupSite(
+		globalServerConfig[config.SiteSubSys][config.Default],
+		globalServerConfig[config.RegionSubSys][config.Default],
+	)
 	if err != nil {
 		t.Fatal(err)
 	}
-	if region != "us-west-1" {
-		t.Errorf("Expecting region `us-west-1` found %s", globalServerRegion)
+	if site.Region != "us-west-1" {
+		t.Errorf("Expecting region `us-west-1` found %s", globalSite.Region)
 	}
 
 	if err := saveServerConfig(context.Background(), objLayer, globalServerConfig); err != nil {

cmd/config.go

@@ -186,11 +186,6 @@ func readServerConfig(ctx context.Context, objAPI ObjectLayer) (config.Config, e
 // ConfigSys - config system.
 type ConfigSys struct{}
 
-// Load - load config.json.
-func (sys *ConfigSys) Load(objAPI ObjectLayer) error {
-	return sys.Init(objAPI)
-}
-
 // Init - initializes config system from config.json.
 func (sys *ConfigSys) Init(objAPI ObjectLayer) error {
 	if objAPI == nil {

cmd/data-scanner.go

@@ -71,7 +71,17 @@ var (
 
 // initDataScanner will start the scanner in the background.
 func initDataScanner(ctx context.Context, objAPI ObjectLayer) {
-	go runDataScanner(ctx, objAPI)
+	go func() {
+		// Run the data scanner in a loop
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+				runDataScanner(ctx, objAPI)
+			}
+		}
+	}()
 }
 
 type safeDuration struct {
@@ -656,6 +666,7 @@ func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, int
 			dirQuorum: getReadQuorum(len(f.disks)),
 			objQuorum: getReadQuorum(len(f.disks)),
 			bucket:    "",
+			strict:    false,
 		}
 
 		healObjectsPrefix := color.Green("healObjects:")
@@ -971,14 +982,14 @@ func (i *scannerItem) applyTierObjSweep(ctx context.Context, o ObjectLayer, oi O
 
 }
 
-// applyMaxNoncurrentVersionLimit removes noncurrent versions older than the most recent MaxNoncurrentVersions configured.
+// applyNewerNoncurrentVersionLimit removes noncurrent versions older than the most recent NewerNoncurrentVersions configured.
 // Note: This function doesn't update sizeSummary since it always removes versions that it doesn't return.
-func (i *scannerItem) applyMaxNoncurrentVersionLimit(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
+func (i *scannerItem) applyNewerNoncurrentVersionLimit(ctx context.Context, _ ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
 	if i.lifeCycle == nil {
 		return fivs, nil
 	}
 
-	lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
+	_, days, lim := i.lifeCycle.NoncurrentVersionsExpirationLimit(lifecycle.ObjectOpts{Name: i.objectPath()})
 	if lim == 0 || len(fivs) <= lim+1 { // fewer than lim _noncurrent_ versions
 		return fivs, nil
 	}
@@ -991,6 +1002,7 @@ func (i *scannerItem) applyMaxNoncurrentVersionLimit(ctx context.Context, o Obje
 	toDel := make([]ObjectToDelete, 0, len(overflowVersions))
 	for _, fi := range overflowVersions {
 		obj := fi.ToObjectInfo(i.bucket, i.objectPath())
+		// skip versions with object locking enabled
 		if rcfg.LockEnabled && enforceRetentionForDeletion(ctx, obj) {
 			if i.debug {
 				if obj.VersionID != "" {
@@ -999,22 +1011,34 @@ func (i *scannerItem) applyMaxNoncurrentVersionLimit(ctx context.Context, o Obje
 					console.Debugf(applyVersionActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name)
 				}
 			}
+			// add this version back to remaining versions for
+			// subsequent lifecycle policy applications
+			fivs = append(fivs, fi)
 			continue
 		}
+
+		// NoncurrentDays not passed yet.
+		if time.Now().UTC().Before(lifecycle.ExpectedExpiryTime(obj.SuccessorModTime, days)) {
+			// add this version back to remaining versions for
+			// subsequent lifecycle policy applications
+			fivs = append(fivs, fi)
+			continue
+		}
+
 		toDel = append(toDel, ObjectToDelete{
 			ObjectName: fi.Name,
 			VersionID:  fi.VersionID,
 		})
 	}
 
-	globalExpiryState.enqueueByMaxNoncurrent(i.bucket, toDel)
+	globalExpiryState.enqueueByNewerNoncurrent(i.bucket, toDel)
 	return fivs, nil
 }
 
 // applyVersionActions will apply lifecycle checks on all versions of a scanned item. Returns versions that remain
 // after applying lifecycle checks configured.
 func (i *scannerItem) applyVersionActions(ctx context.Context, o ObjectLayer, fivs []FileInfo) ([]FileInfo, error) {
-	return i.applyMaxNoncurrentVersionLimit(ctx, o, fivs)
+	return i.applyNewerNoncurrentVersionLimit(ctx, o, fivs)
 }
 
 // applyActions will apply lifecycle checks on to a scanned item.
@@ -1245,6 +1269,7 @@ func (i *scannerItem) healReplicationDeletes(ctx context.Context, o ObjectLayer,
 				DeleteMarker:          roi.DeleteMarker,
 			},
 			Bucket: roi.Bucket,
+			OpType: replication.HealReplicationType,
 		}
 		if roi.ExistingObjResync.mustResync() {
 			doi.OpType = replication.ExistingObjectReplicationType

cmd/data-update-tracker.go

@@ -473,27 +473,32 @@ func (d *dataUpdateTracker) deserialize(src io.Reader, newerThan time.Time) erro
 // start a collector that picks up entries from objectUpdatedCh
 // and adds them  to the current bloom filter.
 func (d *dataUpdateTracker) startCollector(ctx context.Context) {
-	for in := range d.input {
-		bucket, _ := path2BucketObjectWithBasePath("", in)
-		if bucket == "" {
-			if d.debug && len(in) > 0 {
-				console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case in := <-d.input:
+			bucket, _ := path2BucketObjectWithBasePath("", in)
+			if bucket == "" {
+				if d.debug && len(in) > 0 {
+					console.Debugf(color.Green("dataUpdateTracker:")+" no bucket (%s)\n", in)
+				}
+				continue
 			}
-			continue
-		}
 
-		if isReservedOrInvalidBucket(bucket, false) {
-			continue
-		}
-		split := splitPathDeterministic(in)
+			if isReservedOrInvalidBucket(bucket, false) {
+				continue
+			}
+			split := splitPathDeterministic(in)
 
-		// Add all paths until done.
-		d.mu.Lock()
-		for i := range split {
-			d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+			// Add all paths until done.
+			d.mu.Lock()
+			for i := range split {
+				d.Current.bf.AddString(hashPath(path.Join(split[:i+1]...)).String())
+			}
+			d.dirty = d.dirty || len(split) > 0
+			d.mu.Unlock()
 		}
-		d.dirty = d.dirty || len(split) > 0
-		d.mu.Unlock()
 	}
 }
 

cmd/disk-cache-backend.go

@@ -59,10 +59,13 @@ const (
 	cacheExpiryDays  = 90 * time.Hour * 24 // defaults to 90 days
 	// SSECacheEncrypted is the metadata key indicating that the object
 	// is a cache entry encrypted with cache KMS master key in globalCacheKMS.
-	SSECacheEncrypted               = "X-Minio-Internal-Encrypted-Cache"
-	cacheMultipartDir               = "multipart"
+	SSECacheEncrypted = "X-Minio-Internal-Encrypted-Cache"
+	cacheMultipartDir = "multipart"
+	cacheWritebackDir = "writeback"
+
 	cacheStaleUploadCleanupInterval = time.Hour * 24
 	cacheStaleUploadExpiry          = time.Hour * 24
+	cacheWBStaleUploadExpiry        = time.Hour * 24 * 7
 )
 
 // CacheChecksumInfoV1 - carries checksums of individual blocks on disk.
@@ -105,15 +108,15 @@ func (r *RangeInfo) Empty() bool {
 	return r.Range == "" && r.File == "" && r.Size == 0
 }
 
-func (m *cacheMeta) ToObjectInfo(bucket, object string) (o ObjectInfo) {
+func (m *cacheMeta) ToObjectInfo() (o ObjectInfo) {
 	if len(m.Meta) == 0 {
 		m.Meta = make(map[string]string)
 		m.Stat.ModTime = timeSentinel
 	}
 
 	o = ObjectInfo{
-		Bucket:            bucket,
-		Name:              object,
+		Bucket:            m.Bucket,
+		Name:              m.Object,
 		CacheStatus:       CacheHit,
 		CacheLookupStatus: CacheHit,
 	}
@@ -325,7 +328,11 @@ func (c *diskCache) purge(ctx context.Context) {
 	expiry := UTCNow().Add(-cacheExpiryDays)
 	// defaulting max hits count to 100
 	// ignore error we know what value we are passing.
-	scorer, _ := newFileScorer(toFree, time.Now().Unix(), 100)
+	scorer, err := newFileScorer(toFree, time.Now().Unix(), 100)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		return
+	}
 
 	// this function returns FileInfo for cached range files.
 	fiStatRangesFn := func(ranges map[string]string, pathPrefix string) map[string]os.FileInfo {
@@ -377,7 +384,7 @@ func (c *diskCache) purge(ctx context.Context) {
 		lastAtime := lastAtimeFn(meta.PartNumbers, pathJoin(c.dir, name))
 		// stat all cached file ranges.
 		cachedRngFiles := fiStatRangesFn(meta.Ranges, pathJoin(c.dir, name))
-		objInfo := meta.ToObjectInfo("", "")
+		objInfo := meta.ToObjectInfo()
 		// prevent gc from clearing un-synced commits. This metadata is present when
 		// cache writeback commit setting is enabled.
 		status, ok := objInfo.UserDefined[writeBackStatusHeader]
@@ -388,9 +395,7 @@ func (c *diskCache) purge(ctx context.Context) {
 		switch {
 		case cc != nil:
 			if cc.isStale(objInfo.ModTime) {
-				if err = removeAll(cacheDir); err != nil {
-					logger.LogIf(ctx, err)
-				}
+				removeAll(cacheDir)
 				scorer.adjustSaveBytes(-objInfo.Size)
 				// break early if sufficient disk space reclaimed.
 				if c.diskUsageLow() {
@@ -406,11 +411,12 @@ func (c *diskCache) purge(ctx context.Context) {
 		}
 
 		for fname, fi := range cachedRngFiles {
+			if fi == nil {
+				continue
+			}
 			if cc != nil {
 				if cc.isStale(objInfo.ModTime) {
-					if err = removeAll(fname); err != nil {
-						logger.LogIf(ctx, err)
-					}
+					removeAll(fname)
 					scorer.adjustSaveBytes(-fi.Size())
 
 					// break early if sufficient disk space reclaimed.
@@ -425,9 +431,11 @@ func (c *diskCache) purge(ctx context.Context) {
 		}
 		// clean up stale cache.json files for objects that never got cached but access count was maintained in cache.json
 		fi, err := os.Stat(pathJoin(cacheDir, cacheMetaJSONFile))
-		if err != nil || (fi.ModTime().Before(expiry) && len(cachedRngFiles) == 0) {
+		if err != nil || (fi != nil && fi.ModTime().Before(expiry) && len(cachedRngFiles) == 0) {
 			removeAll(cacheDir)
-			scorer.adjustSaveBytes(-fi.Size())
+			if fi != nil {
+				scorer.adjustSaveBytes(-fi.Size())
+			}
 			// Proceed to next file.
 			return nil
 		}
@@ -486,7 +494,7 @@ func (c *diskCache) Stat(ctx context.Context, bucket, object string) (oi ObjectI
 	if partial {
 		return oi, numHits, errFileNotFound
 	}
-	oi = meta.ToObjectInfo("", "")
+	oi = meta.ToObjectInfo()
 	oi.Bucket = bucket
 	oi.Name = object
 
@@ -521,7 +529,7 @@ func (c *diskCache) statRange(ctx context.Context, bucket, object string, rs *HT
 		return
 	}
 
-	oi = meta.ToObjectInfo("", "")
+	oi = meta.ToObjectInfo()
 	oi.Bucket = bucket
 	oi.Name = object
 	if !partial {
@@ -573,12 +581,16 @@ func (c *diskCache) statCache(ctx context.Context, cacheObjPath string) (meta *c
 	if _, err := os.Stat(pathJoin(cacheObjPath, cacheDataFile)); err == nil {
 		partial = false
 	}
+	if writebackInProgress(meta.Meta) {
+		partial = false
+	}
 	return meta, partial, meta.Hits, nil
 }
 
 // saves object metadata to disk cache
 // incHitsOnly is true if metadata update is incrementing only the hit counter
-func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly bool) error {
+// finalizeWB is true only if metadata update accompanied by moving part from temp location to cache dir.
+func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, meta map[string]string, actualSize int64, rs *HTTPRangeSpec, rsFileName string, incHitsOnly, finalizeWB bool) error {
 	cachedPath := getCacheSHADir(c.dir, bucket, object)
 	cLock := c.NewNSLockFn(cachedPath)
 	lkctx, err := cLock.GetLock(ctx, globalOperationTimeout)
@@ -587,7 +599,18 @@ func (c *diskCache) SaveMetadata(ctx context.Context, bucket, object string, met
 	}
 	ctx = lkctx.Context()
 	defer cLock.Unlock(lkctx.Cancel)
-	return c.saveMetadata(ctx, bucket, object, meta, actualSize, rs, rsFileName, incHitsOnly)
+	if err = c.saveMetadata(ctx, bucket, object, meta, actualSize, rs, rsFileName, incHitsOnly); err != nil {
+		return err
+	}
+	// move part saved in writeback directory and cache.json atomically
+	if finalizeWB {
+		wbdir := getCacheWriteBackSHADir(c.dir, bucket, object)
+		if err = renameAll(pathJoin(wbdir, cacheDataFile), pathJoin(cachedPath, cacheDataFile)); err != nil {
+			return err
+		}
+		removeAll(wbdir) // cleanup writeback/shadir
+	}
+	return nil
 }
 
 // saves object metadata to disk cache
@@ -702,6 +725,11 @@ func getCacheSHADir(dir, bucket, object string) string {
 	return pathJoin(dir, getSHA256Hash([]byte(pathJoin(bucket, object))))
 }
 
+// returns temporary writeback cache location.
+func getCacheWriteBackSHADir(dir, bucket, object string) string {
+	return pathJoin(dir, minioMetaBucket, "writeback", getSHA256Hash([]byte(pathJoin(bucket, object))))
+}
+
 // Cache data to disk with bitrot checksum added for each block of 1MB
 func (c *diskCache) bitrotWriteToCache(cachePath, fileName string, reader io.Reader, size uint64) (int64, string, error) {
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
@@ -808,10 +836,6 @@ func (c *diskCache) GetLockContext(ctx context.Context, bucket, object string) (
 
 // Caches the object to disk
 func (c *diskCache) Put(ctx context.Context, bucket, object string, data io.Reader, size int64, rs *HTTPRangeSpec, opts ObjectOptions, incHitsOnly, writeback bool) (oi ObjectInfo, err error) {
-	if !c.diskSpaceAvailable(size) {
-		io.Copy(ioutil.Discard, data)
-		return oi, errDiskFull
-	}
 	cLock, lkctx, err := c.GetLockContext(ctx, bucket, object)
 	if err != nil {
 		return oi, err
@@ -850,6 +874,11 @@ func (c *diskCache) put(ctx context.Context, bucket, object string, data io.Read
 	if !c.diskSpaceAvailable(size) {
 		return oi, errDiskFull
 	}
+
+	if writeback {
+		cachePath = getCacheWriteBackSHADir(c.dir, bucket, object)
+	}
+
 	if err := os.MkdirAll(cachePath, 0777); err != nil {
 		return oi, err
 	}
@@ -1135,6 +1164,9 @@ func (c *diskCache) Get(ctx context.Context, bucket, object string, rs *HTTPRang
 		}()
 	} else {
 		go func() {
+			if writebackInProgress(objInfo.UserDefined) {
+				cacheObjPath = getCacheWriteBackSHADir(c.dir, bucket, object)
+			}
 			filePath := pathJoin(cacheObjPath, cacheFile)
 			err := c.bitrotReadFromCache(ctx, filePath, startOffset, length, pw)
 			if err != nil {
@@ -1203,7 +1235,7 @@ func (c *diskCache) scanCacheWritebackFailures(ctx context.Context) {
 			return nil
 		}
 
-		objInfo := meta.ToObjectInfo("", "")
+		objInfo := meta.ToObjectInfo()
 		status, ok := objInfo.UserDefined[writeBackStatusHeader]
 		if !ok || status == CommitComplete.String() {
 			return nil
@@ -1501,6 +1533,8 @@ func (c *diskCache) CompleteMultipartUpload(ctx context.Context, bucket, object,
 	}
 	uploadMeta.Stat.Size = objectSize
 	uploadMeta.Stat.ModTime = roi.ModTime
+	uploadMeta.Bucket = bucket
+	uploadMeta.Object = object
 	// if encrypted - make sure ETag updated
 
 	uploadMeta.Meta["etag"] = roi.ETag
@@ -1536,7 +1570,7 @@ func (c *diskCache) CompleteMultipartUpload(ctx context.Context, bucket, object,
 	}
 	renameAll(pathJoin(uploadIDDir, cacheMetaJSONFile), pathJoin(cachePath, cacheMetaJSONFile))
 	removeAll(uploadIDDir) // clean up any unused parts in the uploadIDDir
-	return uploadMeta.ToObjectInfo(bucket, object), nil
+	return uploadMeta.ToObjectInfo(), nil
 }
 
 func (c *diskCache) AbortUpload(bucket, object, uploadID string) (err error) {
@@ -1583,9 +1617,6 @@ func getMultipartCacheSHADir(dir, bucket, object string) string {
 
 // clean up stale cache multipart uploads according to cleanup interval.
 func (c *diskCache) cleanupStaleUploads(ctx context.Context) {
-	if !c.commitWritethrough {
-		return
-	}
 	timer := time.NewTimer(cacheStaleUploadCleanupInterval)
 	defer timer.Stop()
 	for {
@@ -1609,6 +1640,22 @@ func (c *diskCache) cleanupStaleUploads(ctx context.Context) {
 					return nil
 				})
 			})
+			// clean up of writeback folder where cache.json no longer exists in the main c.dir/<sha256(bucket,object> path
+			// and if past upload expiry window.
+			readDirFn(pathJoin(c.dir, minioMetaBucket, cacheWritebackDir), func(shaDir string, typ os.FileMode) error {
+				wbdir := pathJoin(c.dir, minioMetaBucket, cacheWritebackDir, shaDir)
+				cachedir := pathJoin(c.dir, shaDir)
+				if _, err := os.Stat(cachedir); os.IsNotExist(err) {
+					fi, err := os.Stat(wbdir)
+					if err != nil {
+						return nil
+					}
+					if now.Sub(fi.ModTime()) > cacheWBStaleUploadExpiry {
+						return removeAll(wbdir)
+					}
+				}
+				return nil
+			})
 		}
 	}
 }

cmd/disk-cache-utils.go

@@ -419,6 +419,9 @@ func (f *fileScorer) addFileWithObjInfo(objInfo ObjectInfo, hits int) {
 // Returns true if there still is a need to delete files (n+saveBytes >0),
 // false if no more bytes needs to be saved.
 func (f *fileScorer) adjustSaveBytes(n int64) bool {
+	if f == nil {
+		return false
+	}
 	if int64(f.saveBytes)+n <= 0 {
 		f.saveBytes = 0
 		f.trimQueue()
@@ -581,3 +584,14 @@ func (t *multiWriter) Write(p []byte) (n int, err error) {
 func cacheMultiWriter(w1 io.Writer, w2 *io.PipeWriter) io.Writer {
 	return &multiWriter{backendWriter: w1, cacheWriter: w2}
 }
+
+// writebackInProgress returns true if writeback commit is not complete
+func writebackInProgress(m map[string]string) bool {
+	if v, ok := m[writeBackStatusHeader]; ok {
+		switch cacheCommitStatus(v) {
+		case CommitPending, CommitFailed:
+			return true
+		}
+	}
+	return false
+}

cmd/disk-cache.go

@@ -132,7 +132,7 @@ type cacheObjects struct {
 
 func (c *cacheObjects) incHitsToMeta(ctx context.Context, dcache *diskCache, bucket, object string, size int64, eTag string, rs *HTTPRangeSpec) error {
 	metadata := map[string]string{"etag": eTag}
-	return dcache.SaveMetadata(ctx, bucket, object, metadata, size, rs, "", true)
+	return dcache.SaveMetadata(ctx, bucket, object, metadata, size, rs, "", true, false)
 }
 
 // Backend metadata could have changed through server side copy - reset cache metadata if that is the case
@@ -159,7 +159,7 @@ func (c *cacheObjects) updateMetadataIfChanged(ctx context.Context, dcache *disk
 		bkObjectInfo.ETag != cacheObjInfo.ETag ||
 		bkObjectInfo.ContentType != cacheObjInfo.ContentType ||
 		!bkObjectInfo.Expires.Equal(cacheObjInfo.Expires) {
-		return dcache.SaveMetadata(ctx, bucket, object, getMetadata(bkObjectInfo), bkObjectInfo.Size, nil, "", false)
+		return dcache.SaveMetadata(ctx, bucket, object, getMetadata(bkObjectInfo), bkObjectInfo.Size, nil, "", false, false)
 	}
 	return c.incHitsToMeta(ctx, dcache, bucket, object, cacheObjInfo.Size, cacheObjInfo.ETag, rs)
 }
@@ -276,6 +276,10 @@ func (c *cacheObjects) GetObjectNInfo(ctx context.Context, bucket, object string
 			bReader.ObjInfo.CacheStatus = CacheMiss
 			return bReader, err
 		}
+		// serve cached content without ETag verification if writeback commit is not yet complete
+		if writebackInProgress(cacheReader.ObjInfo.UserDefined) {
+			return cacheReader, nil
+		}
 	}
 
 	objInfo, err := c.InnerGetObjectInfoFn(ctx, bucket, object, opts)
@@ -422,6 +426,10 @@ func (c *cacheObjects) GetObjectInfo(ctx context.Context, bucket, object string,
 			c.cacheStats.incHit()
 			return cachedObjInfo, nil
 		}
+		// serve cache metadata without ETag verification if writeback commit is not yet complete
+		if writebackInProgress(cachedObjInfo.UserDefined) {
+			return cachedObjInfo, nil
+		}
 	}
 
 	objInfo, err := getObjectInfoFn(ctx, bucket, object, opts)
@@ -698,7 +706,7 @@ func (c *cacheObjects) PutObject(ctx context.Context, bucket, object string, r *
 				oi, _, err := dcache.Stat(GlobalContext, bucket, object)
 				// avoid cache overwrite if another background routine filled cache
 				if err != nil || oi.ETag != bReader.ObjInfo.ETag {
-					dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false, true)
+					dcache.Put(GlobalContext, bucket, object, bReader, bReader.ObjInfo.Size, nil, ObjectOptions{UserDefined: getMetadata(bReader.ObjInfo)}, false, false)
 				}
 			}()
 		}
@@ -786,6 +794,7 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 	opts.UserDefined[xhttp.ContentMD5] = oi.UserDefined["content-md5"]
 	objInfo, err := c.InnerPutObjectFn(ctx, oi.Bucket, oi.Name, NewPutObjReader(hashReader), opts)
 	wbCommitStatus := CommitComplete
+	size := objInfo.Size
 	if err != nil {
 		wbCommitStatus = CommitFailed
 	}
@@ -796,12 +805,13 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 		retryCnt, _ = strconv.Atoi(meta[writeBackRetryHeader])
 		retryCnt++
 		meta[writeBackRetryHeader] = strconv.Itoa(retryCnt)
+		size = cReader.ObjInfo.Size
 	} else {
 		delete(meta, writeBackRetryHeader)
 	}
 	meta[writeBackStatusHeader] = wbCommitStatus.String()
 	meta["etag"] = oi.ETag
-	dcache.SaveMetadata(ctx, oi.Bucket, oi.Name, meta, objInfo.Size, nil, "", false)
+	dcache.SaveMetadata(ctx, oi.Bucket, oi.Name, meta, size, nil, "", false, wbCommitStatus == CommitComplete)
 	if retryCnt > 0 {
 		// slow down retries
 		time.Sleep(time.Second * time.Duration(retryCnt%10+1))
@@ -811,6 +821,8 @@ func (c *cacheObjects) uploadObject(ctx context.Context, oi ObjectInfo) {
 
 func (c *cacheObjects) queueWritebackRetry(oi ObjectInfo) {
 	select {
+	case <-GlobalContext.Done():
+		return
 	case c.wbRetryCh <- oi:
 		c.uploadObject(GlobalContext, oi)
 	default:

cmd/disk-cache_test.go

@@ -25,7 +25,7 @@ import (
 // Tests ToObjectInfo function.
 func TestCacheMetadataObjInfo(t *testing.T) {
 	m := cacheMeta{Meta: nil}
-	objInfo := m.ToObjectInfo("testbucket", "testobject")
+	objInfo := m.ToObjectInfo()
 	if objInfo.Size != 0 {
 		t.Fatal("Unexpected object info value for Size", objInfo.Size)
 	}

cmd/encryption-v1.go

@@ -68,6 +68,21 @@ const (
 
 )
 
+// KMSKeyID returns in AWS compatible KMS KeyID() format.
+func (o ObjectInfo) KMSKeyID() string {
+	if len(o.UserDefined) == 0 {
+		return ""
+	}
+	kmsID, ok := o.UserDefined[crypto.MetaKeyID]
+	if !ok {
+		return ""
+	}
+	if strings.HasPrefix(kmsID, "arn:aws:kms:") {
+		return kmsID
+	}
+	return "arn:aws:kms:" + kmsID
+}
+
 // isMultipart returns true if the current object is
 // uploaded by the user using multipart mechanism:
 // initiate new multipart, upload part, complete upload
@@ -76,15 +91,18 @@ func (o *ObjectInfo) isMultipart() bool {
 		return false
 	}
 	_, encrypted := crypto.IsEncrypted(o.UserDefined)
-	if encrypted && !crypto.IsMultiPart(o.UserDefined) {
-		return false
-	}
-	for _, part := range o.Parts {
-		_, err := sio.DecryptedSize(uint64(part.Size))
-		if err != nil {
+	if encrypted {
+		if !crypto.IsMultiPart(o.UserDefined) {
 			return false
 		}
+		for _, part := range o.Parts {
+			_, err := sio.DecryptedSize(uint64(part.Size))
+			if err != nil {
+				return false
+			}
+		}
 	}
+
 	// Further check if this object is uploaded using multipart mechanism
 	// by the user and it is not about Erasure internally splitting the
 	// object into parts in PutObject()

cmd/erasure-bucket.go

@@ -64,8 +64,7 @@ func (er erasureObjects) MakeBucketWithLocation(ctx context.Context, bucket stri
 		}, index)
 	}
 
-	writeQuorum := getWriteQuorum(len(storageDisks))
-	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, writeQuorum)
+	err := reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, er.defaultWQuorum())
 	return toObjectErr(err, bucket)
 }
 
@@ -121,8 +120,7 @@ func (er erasureObjects) getBucketInfo(ctx context.Context, bucketName string) (
 	// reduce to one error based on read quorum.
 	// `nil` is deliberately passed for ignoredErrs
 	// because these errors were already ignored.
-	readQuorum := getReadQuorum(len(storageDisks))
-	return BucketInfo{}, reduceReadQuorumErrs(ctx, errs, nil, readQuorum)
+	return BucketInfo{}, reduceReadQuorumErrs(ctx, errs, nil, er.defaultRQuorum())
 }
 
 // GetBucketInfo - returns BucketInfo for a bucket.
@@ -167,8 +165,7 @@ func (er erasureObjects) DeleteBucket(ctx context.Context, bucket string, opts D
 		return nil
 	}
 
-	writeQuorum := getWriteQuorum(len(storageDisks))
-	err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, writeQuorum)
+	err := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, er.defaultWQuorum())
 	if err == errErasureWriteQuorum && !opts.NoRecreate {
 		undoDeleteBucket(storageDisks, bucket)
 	}

cmd/erasure-decode.go

@@ -282,3 +282,51 @@ func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.Read
 
 	return bytesWritten, derr
 }
+
+// Heal reads from readers, reconstruct shards and writes the data to the writers.
+func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.ReaderAt, totalLength int64) (derr error) {
+	if len(writers) != e.parityBlocks+e.dataBlocks {
+		return errInvalidArgument
+	}
+
+	reader := newParallelReader(readers, e, 0, totalLength)
+
+	startBlock := int64(0)
+	endBlock := totalLength / e.blockSize
+	if totalLength%e.blockSize != 0 {
+		endBlock++
+	}
+
+	var bufs [][]byte
+	for block := startBlock; block < endBlock; block++ {
+		var err error
+		bufs, err = reader.Read(bufs)
+		if len(bufs) > 0 {
+			if errors.Is(err, errFileNotFound) || errors.Is(err, errFileCorrupt) {
+				if derr == nil {
+					derr = err
+				}
+			}
+		} else if err != nil {
+			return err
+		}
+
+		if err = e.DecodeDataAndParityBlocks(ctx, bufs); err != nil {
+			logger.LogIf(ctx, err)
+			return err
+		}
+
+		w := parallelWriter{
+			writers:     writers,
+			writeQuorum: 1,
+			errs:        make([]error, len(writers)),
+		}
+
+		err = w.Write(ctx, bufs)
+		if err != nil {
+			return err
+		}
+	}
+
+	return derr
+}

cmd/erasure-heal_test.go

@@ -24,9 +24,6 @@ import (
 	"io"
 	"os"
 	"testing"
-
-	humanize "github.com/dustin/go-humanize"
-	"github.com/minio/minio/internal/bpool"
 )
 
 var erasureHealTests = []struct {
@@ -137,15 +134,8 @@ func TestErasureHeal(t *testing.T) {
 			staleWriters[i] = newBitrotWriter(disk, "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize())
 		}
 
-		// Number of buffers, max 2GB
-		n := (2 * humanize.GiByte) / (int(test.blocksize) * 2)
-
-		// Initialize byte pool once for all sets, bpool size is set to
-		// setCount * setDriveCount with each memory upto blockSizeV2.
-		bp := bpool.NewBytePoolCap(n, int(test.blocksize), int(test.blocksize)*2)
-
 		// test case setup is complete - now call Heal()
-		err = erasure.Heal(context.Background(), readers, staleWriters, test.size, bp)
+		err = erasure.Heal(context.Background(), staleWriters, readers, test.size)
 		closeBitrotReaders(readers)
 		closeBitrotWriters(staleWriters)
 		if err != nil && !test.shouldFail {

cmd/erasure-healing-common.go

@@ -26,29 +26,59 @@ import (
 )
 
 // commonTime returns a maximally occurring time from a list of time.
-func commonTime(modTimes []time.Time) (modTime time.Time) {
-	timeOccurenceMap := make(map[int64]int, len(modTimes))
+func commonTimeAndOccurence(times []time.Time, group time.Duration) (maxTime time.Time, maxima int) {
+	timeOccurenceMap := make(map[int64]int, len(times))
+	groupNano := group.Nanoseconds()
 	// Ignore the uuid sentinel and count the rest.
-	for _, t := range modTimes {
+	for _, t := range times {
 		if t.Equal(timeSentinel) {
 			continue
 		}
-		timeOccurenceMap[t.UnixNano()]++
+		nano := t.UnixNano()
+		if group > 0 {
+			for k := range timeOccurenceMap {
+				if k == nano {
+					// We add to ourself later
+					continue
+				}
+				diff := k - nano
+				if diff < 0 {
+					diff = -diff
+				}
+				// We are within the limit
+				if diff < groupNano {
+					timeOccurenceMap[k]++
+				}
+			}
+		}
+		// Add ourself...
+		timeOccurenceMap[nano]++
 	}
 
-	var maxima int // Counter for remembering max occurrence of elements.
+	maxima = 0 // Counter for remembering max occurrence of elements.
+	latest := int64(0)
 
 	// Find the common cardinality from previously collected
 	// occurrences of elements.
 	for nano, count := range timeOccurenceMap {
-		t := time.Unix(0, nano).UTC()
-		if count > maxima || (count == maxima && t.After(modTime)) {
+		if count < maxima {
+			continue
+		}
+
+		// We are at or above maxima
+		if count > maxima || nano > latest {
 			maxima = count
-			modTime = t
+			latest = nano
 		}
 	}
 
-	// Return the collected common modTime.
+	// Return the collected common max time, with maxima
+	return time.Unix(0, latest).UTC(), maxima
+}
+
+// commonTime returns a maximally occurring time from a list of time.
+func commonTime(modTimes []time.Time) (modTime time.Time) {
+	modTime, _ = commonTimeAndOccurence(modTimes, 0)
 	return modTime
 }
 
@@ -88,6 +118,19 @@ func filterOnlineDisksInplace(fi FileInfo, partsMetadata []FileInfo, onlineDisks
 	}
 }
 
+// Extracts list of disk mtimes from FileInfo slice and returns, skips
+// slice elements that have errors.
+func listObjectDiskMtimes(partsMetadata []FileInfo) (diskMTimes []time.Time) {
+	diskMTimes = bootModtimes(len(partsMetadata))
+	for index, metadata := range partsMetadata {
+		if metadata.IsValid() {
+			// Once the file is found, save the disk mtime saved on disk.
+			diskMTimes[index] = metadata.DiskMTime
+		}
+	}
+	return diskMTimes
+}
+
 // Notes:
 // There are 5 possible states a disk could be in,
 // 1. __online__             - has the latest copy of xl.meta - returned by listOnlineDisks
@@ -182,8 +225,15 @@ func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, errs []err
 // - slice of errors about the state of data files on disk - can have
 //   a not-found error or a hash-mismatch error.
 func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo,
-	errs []error, latestMeta FileInfo,
-	bucket, object string, scanMode madmin.HealScanMode) ([]StorageAPI, []error) {
+	errs []error, latestMeta FileInfo, bucket, object string,
+	scanMode madmin.HealScanMode) ([]StorageAPI, []error, time.Time) {
+
+	var diskMTime time.Time
+	var shardFix bool
+	if !latestMeta.DataShardFixed() {
+		diskMTime = pickValidDiskTimeWithQuorum(partsMetadata,
+			latestMeta.Erasure.DataBlocks)
+	}
 
 	availableDisks := make([]StorageAPI, len(onlineDisks))
 	dataErrs := make([]error, len(onlineDisks))
@@ -234,6 +284,8 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 
 		if erasureDistributionReliable {
 			if !meta.IsValid() {
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
 				continue
 			}
 
@@ -256,6 +308,23 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 			}
 		}
 
+		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
+			if !partsMetadata[i].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
+				// not with in acceptable delta, skip.
+				// If disk mTime mismatches it is considered outdated
+				// https://github.com/minio/minio/pull/13803
+				//
+				// This check only is active if we could find maximally
+				// occurring disk mtimes that are somewhat same across
+				// the quorum. Allowing to skip those shards which we
+				// might think are wrong.
+				shardFix = true
+				partsMetadata[i] = FileInfo{}
+				dataErrs[i] = errFileCorrupt
+				continue
+			}
+		}
+
 		// Always check data, if we got it.
 		if (len(meta.Data) > 0 || meta.Size == 0) && len(meta.Parts) > 0 {
 			checksumInfo := meta.Erasure.GetChecksumInfo(meta.Parts[0].Number)
@@ -298,5 +367,10 @@ func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetad
 		}
 	}
 
-	return availableDisks, dataErrs
+	if shardFix {
+		// Only when shard is fixed return an appropriate disk mtime value.
+		return availableDisks, dataErrs, diskMTime
+	} // else return timeSentinel for disk mtime
+
+	return availableDisks, dataErrs, timeSentinel
 }

cmd/erasure-healing-common_test.go

@@ -251,7 +251,8 @@ func TestListOnlineDisks(t *testing.T) {
 				t.Fatalf("Expected modTime to be equal to %v but was found to be %v",
 					test.expectedTime, modTime)
 			}
-			availableDisks, newErrs := disksWithAllParts(ctx, onlineDisks, partsMetadata, test.errs, fi, bucket, object, madmin.HealDeepScan)
+			availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+				test.errs, fi, bucket, object, madmin.HealDeepScan)
 			test.errs = newErrs
 
 			if test._tamperBackend != noTamper {
@@ -433,7 +434,8 @@ func TestListOnlineDisksSmallObjects(t *testing.T) {
 					test.expectedTime, modTime)
 			}
 
-			availableDisks, newErrs := disksWithAllParts(ctx, onlineDisks, partsMetadata, test.errs, fi, bucket, object, madmin.HealDeepScan)
+			availableDisks, newErrs, _ := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+				test.errs, fi, bucket, object, madmin.HealDeepScan)
 			test.errs = newErrs
 
 			if test._tamperBackend != noTamper {
@@ -494,7 +496,8 @@ func TestDisksWithAllParts(t *testing.T) {
 
 	erasureDisks, _ = listOnlineDisks(erasureDisks, partsMetadata, errs)
 
-	filteredDisks, errs := disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, errs, _ := disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
@@ -515,7 +518,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	partsMetadata[0].ModTime = partsMetadata[0].ModTime.Add(-1 * time.Hour)
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
@@ -535,7 +539,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	partsMetadata[1].DataDir = "foo-random"
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, _, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))
@@ -571,7 +576,8 @@ func TestDisksWithAllParts(t *testing.T) {
 	}
 
 	errs = make([]error, len(erasureDisks))
-	filteredDisks, errs = disksWithAllParts(ctx, erasureDisks, partsMetadata, errs, fi, bucket, object, madmin.HealDeepScan)
+	filteredDisks, errs, _ = disksWithAllParts(ctx, erasureDisks, partsMetadata,
+		errs, fi, bucket, object, madmin.HealDeepScan)
 
 	if len(filteredDisks) != len(erasureDisks) {
 		t.Errorf("Unexpected number of disks: %d", len(filteredDisks))

cmd/erasure-healing.go

@@ -24,12 +24,38 @@ import (
 	"fmt"
 	"io"
 	"sync"
+	"time"
 
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
 )
 
+const reservedMetadataPrefixLowerDataShardFix = ReservedMetadataPrefixLower + "data-shard-fix"
+
+// AcceptableDelta returns 'true' if the fi.DiskMTime is under
+// acceptable delta of "delta" duration with maxTime.
+//
+// This code is primarily used for heuristic detection of
+// incorrect shards, as per https://github.com/minio/minio/pull/13803
+//
+// This check only is active if we could find maximally
+// occurring disk mtimes that are somewhat same across
+// the quorum. Allowing to skip those shards which we
+// might think are wrong.
+func (fi FileInfo) AcceptableDelta(maxTime time.Time, delta time.Duration) bool {
+	diff := maxTime.Sub(fi.DiskMTime)
+	if diff < 0 {
+		diff = -diff
+	}
+	return diff < delta
+}
+
+// DataShardFixed - data shard fixed?
+func (fi FileInfo) DataShardFixed() bool {
+	return fi.Metadata[reservedMetadataPrefixLowerDataShardFix] == "true"
+}
+
 // Heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
@@ -224,6 +250,11 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 		return true
 	}
 	if erErr == nil {
+		if meta.XLV1 {
+			// Legacy means heal always
+			// always check first.
+			return true
+		}
 		if !meta.Deleted && !meta.IsRemote() {
 			// If xl.meta was read fine but there may be problem with the part.N files.
 			if IsErr(dataErr, []error{
@@ -234,19 +265,7 @@ func shouldHealObjectOnDisk(erErr, dataErr error, meta FileInfo, latestMeta File
 				return true
 			}
 		}
-		if !latestMeta.MetadataEquals(meta) {
-			return true
-		}
-		if !latestMeta.TransitionInfoEquals(meta) {
-			return true
-		}
-		if !latestMeta.ReplicationInfoEquals(meta) {
-			return true
-		}
-		if !latestMeta.ModTime.Equal(meta.ModTime) {
-			return true
-		}
-		if meta.XLV1 {
+		if !latestMeta.Equals(meta) {
 			return true
 		}
 	}
@@ -267,12 +286,10 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	// Initialize heal result object
 	result = madmin.HealResultItem{
-		Type:         madmin.HealItemObject,
-		Bucket:       bucket,
-		Object:       object,
-		DiskCount:    len(storageDisks),
-		ParityBlocks: er.defaultParityCount,
-		DataBlocks:   len(storageDisks) - er.defaultParityCount,
+		Type:      madmin.HealItemObject,
+		Bucket:    bucket,
+		Object:    object,
+		DiskCount: len(storageDisks),
 	}
 
 	if !opts.NoLock {
@@ -287,18 +304,27 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 
 	// Re-read when we have lock...
 	partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, object, versionID, true)
+	if isAllNotFound(errs) {
+		// Nothing to do, file is already gone.
+		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
+			errs, bucket, object, versionID), nil
+	}
 
-	if _, err = getLatestFileInfo(ctx, partsMetadata, errs); err != nil {
+	readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount)
+	if err != nil {
 		return er.purgeObjectDangling(ctx, bucket, object, versionID, partsMetadata, errs, []error{}, opts)
 	}
 
+	result.ParityBlocks = result.DiskCount - readQuorum
+	result.DataBlocks = readQuorum
+
 	// List of disks having latest version of the object er.meta
 	// (by modtime).
 	onlineDisks, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
 
 	// Latest FileInfo for reference. If a valid metadata is not
 	// present, it is as good as object not found.
-	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, result.DataBlocks)
+	latestMeta, err := pickValidFileInfo(ctx, partsMetadata, modTime, readQuorum)
 	if err != nil {
 		return result, toObjectErr(err, bucket, object, versionID)
 	}
@@ -312,7 +338,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	// used here for reconstruction. This is done to ensure that
 	// we do not skip drives that have inconsistent metadata to be
 	// skipped from purging when they are stale.
-	availableDisks, dataErrs := disksWithAllParts(ctx, onlineDisks, partsMetadata,
+	availableDisks, dataErrs, diskMTime := disksWithAllParts(ctx, onlineDisks, partsMetadata,
 		errs, latestMeta, bucket, object, scanMode)
 
 	// Loop to find number of disks with valid data, per-drive
@@ -373,13 +399,9 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 	}
 
 	if isAllNotFound(errs) {
-		err = toObjectErr(errFileNotFound, bucket, object)
-		if versionID != "" {
-			err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
-		}
 		// File is fully gone, fileInfo is empty.
 		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, errs,
-			bucket, object, versionID), err
+			bucket, object, versionID), nil
 	}
 
 	// If less than read quorum number of disks have all the parts
@@ -457,10 +479,6 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		}
 
 		erasureInfo := latestMeta.Erasure
-		bp := er.bp
-		if erasureInfo.BlockSize == blockSizeV1 {
-			bp = er.bpOld
-		}
 		for partIndex := 0; partIndex < len(latestMeta.Parts); partIndex++ {
 			partSize := latestMeta.Parts[partIndex].Size
 			partActualSize := latestMeta.Parts[partIndex].ActualSize
@@ -491,7 +509,7 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 						tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize())
 				}
 			}
-			err = erasure.Heal(ctx, readers, writers, partSize, bp)
+			err = erasure.Heal(ctx, writers, readers, partSize)
 			closeBitrotReaders(readers)
 			closeBitrotWriters(writers)
 			if err != nil {
@@ -566,6 +584,20 @@ func (er erasureObjects) healObject(ctx context.Context, bucket string, object s
 		}
 	}
 
+	if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
+		// Update metadata to indicate special fix.
+		_, err = er.PutObjectMetadata(ctx, bucket, object, ObjectOptions{
+			NoLock: true,
+			UserDefined: map[string]string{
+				reservedMetadataPrefixLowerDataShardFix: "true",
+				// another reserved metadata to capture original disk-mtime
+				// captured for this version of the object, to be used
+				// possibly in future to heal other versions if possible.
+				ReservedMetadataPrefixLower + "disk-mtime": diskMTime.String(),
+			},
+		})
+	}
+
 	// Set the size of the object in the heal result
 	result.ObjectSize = latestMeta.Size
 
@@ -634,7 +666,7 @@ func (er erasureObjects) healObjectDir(ctx context.Context, bucket, object strin
 	}
 	if dryRun || danglingObject || isAllNotFound(errs) {
 		// Nothing to do, file is already gone.
-		return hr, toObjectErr(errFileNotFound, bucket, object)
+		return hr, nil
 	}
 	for i, err := range errs {
 		if err == errVolumeNotFound || err == errFileNotFound {
@@ -739,8 +771,15 @@ func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix
 // A 0 length slice will always return false.
 func isAllNotFound(errs []error) bool {
 	for _, err := range errs {
-		if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) || errors.Is(err, errFileVersionNotFound) {
-			continue
+		if err != nil {
+			switch err.Error() {
+			case errFileNotFound.Error():
+				fallthrough
+			case errVolumeNotFound.Error():
+				fallthrough
+			case errFileVersionNotFound.Error():
+				continue
+			}
 		}
 		return false
 	}
@@ -925,13 +964,9 @@ func (er erasureObjects) HealObject(ctx context.Context, bucket, object, version
 	// This allows to quickly check if all is ok or all are missing.
 	_, errs := readAllFileInfo(healCtx, storageDisks, bucket, object, versionID, false)
 	if isAllNotFound(errs) {
-		err = toObjectErr(errFileNotFound, bucket, object)
-		if versionID != "" {
-			err = toObjectErr(errFileVersionNotFound, bucket, object, versionID)
-		}
 		// Nothing to do, file is already gone.
 		return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints,
-			errs, bucket, object, versionID), err
+			errs, bucket, object, versionID), nil
 	}
 
 	// Heal the object.

cmd/erasure-healing_test.go

@@ -21,10 +21,12 @@ import (
 	"bytes"
 	"context"
 	"crypto/rand"
+	"crypto/sha256"
+	"errors"
+	"io"
 	"os"
 	"path"
 	"reflect"
-	"runtime"
 	"testing"
 	"time"
 
@@ -92,16 +94,16 @@ func TestHealing(t *testing.T) {
 	}
 
 	// After heal the meta file should be as expected.
-	if !reflect.DeepEqual(fileInfoPreHeal, fileInfoPostHeal) {
+	if !fileInfoPreHeal.Equals(fileInfoPostHeal) {
 		t.Fatal("HealObject failed")
 	}
 
-	err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "er.meta"))
+	err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "xl.meta"))
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	// Write er.meta with different modtime to simulate the case where a disk had
+	// Write xl.meta with different modtime to simulate the case where a disk had
 	// gone down when an object was replaced by a new object.
 	fileInfoOutDated := fileInfoPreHeal
 	fileInfoOutDated.ModTime = time.Now()
@@ -121,7 +123,7 @@ func TestHealing(t *testing.T) {
 	}
 
 	// After heal the meta file should be as expected.
-	if !reflect.DeepEqual(fileInfoPreHeal, fileInfoPostHeal) {
+	if !fileInfoPreHeal.Equals(fileInfoPostHeal) {
 		t.Fatal("HealObject failed")
 	}
 
@@ -147,10 +149,6 @@ func TestHealing(t *testing.T) {
 }
 
 func TestHealingDanglingObject(t *testing.T) {
-	if runtime.GOOS == "windows" {
-		t.Skip()
-	}
-
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
@@ -180,6 +178,9 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatalf("Failed to make a bucket - %v", err)
 	}
 
+	disks = objLayer.(*erasureServerPools).serverPools[0].erasureDisks[0]
+	orgDisks := append([]StorageAPI{}, disks...)
+
 	// Enable versioning.
 	globalBucketMetadataSys.Update(bucket, bucketVersioningConfig, []byte(`<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>`))
 
@@ -190,11 +191,13 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0400); err != nil {
-			t.Fatal(err)
-		}
+	setDisks := func(newDisks ...StorageAPI) {
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Lock()
+		copy(disks, newDisks)
+		objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock()
 	}
+	// Remove 4 disks.
+	setDisks(nil, nil, nil, nil)
 
 	// Create delete marker under quorum.
 	objInfo, err := objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{Versioned: true})
@@ -202,11 +205,8 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0755); err != nil {
-			t.Fatal(err)
-		}
-	}
+	// Restore...
+	setDisks(orgDisks[:4]...)
 
 	fileInfoPreHeal, err := disks[0].ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
@@ -243,11 +243,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		}
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0400); err != nil {
-			t.Fatal(err)
-		}
-	}
+	setDisks(nil, nil, nil, nil)
 
 	rd := mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", "")
 	_, err = objLayer.PutObject(ctx, bucket, object, rd, ObjectOptions{
@@ -257,11 +253,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0755); err != nil {
-			t.Fatal(err)
-		}
-	}
+	setDisks(orgDisks[:4]...)
 
 	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
@@ -297,11 +289,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0400); err != nil {
-			t.Fatal(err)
-		}
-	}
+	setDisks(nil, nil, nil, nil)
 
 	// Create delete marker under quorum.
 	_, err = objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{
@@ -312,11 +300,7 @@ func TestHealingDanglingObject(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	for _, fsDir := range fsDirs[:4] {
-		if err = os.Chmod(fsDir, 0755); err != nil {
-			t.Fatal(err)
-		}
-	}
+	setDisks(orgDisks[:4]...)
 
 	fileInfoPreHeal, err = disks[0].ReadVersion(context.Background(), bucket, object, "", false)
 	if err != nil {
@@ -345,6 +329,298 @@ func TestHealingDanglingObject(t *testing.T) {
 	}
 }
 
+func TestHealCorrectQuorum(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resetGlobalHealState()
+	defer resetGlobalHealState()
+
+	nDisks := 32
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	pools := mustGetPoolEndpoints(fsDirs[:16]...)
+	pools = append(pools, mustGetPoolEndpoints(fsDirs[16:]...)...)
+
+	// Everything is fine, should return nil
+	objLayer, _, err := initObjectLayer(ctx, pools)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+	data := bytes.Repeat([]byte("a"), 5*1024*1024)
+	var opts ObjectOptions
+
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	// Create an object with multiple parts uploaded in decreasing
+	// part number.
+	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	if err != nil {
+		t.Fatalf("Failed to create a multipart upload - %v", err)
+	}
+
+	var uploadedParts []CompletePart
+	for _, partID := range []int{2, 1} {
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		if err1 != nil {
+			t.Fatalf("Failed to upload a part - %v", err1)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Failed to complete multipart upload - %v", err)
+	}
+
+	cfgFile := pathJoin(bucketConfigPrefix, bucket, ".test.bin")
+	if err = saveConfig(ctx, objLayer, cfgFile, data); err != nil {
+		t.Fatal(err)
+	}
+
+	hopts := madmin.HealOpts{
+		DryRun:   false,
+		Remove:   true,
+		ScanMode: madmin.HealNormalScan,
+	}
+
+	// Test 1: Remove the object backend files from the first disk.
+	z := objLayer.(*erasureServerPools)
+	for _, set := range z.serverPools {
+		er := set.sets[0]
+		erasureDisks := er.getDisks()
+
+		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+		if errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		for i := 0; i < nfi.Erasure.ParityBlocks; i++ {
+			erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		}
+
+		// Try healing now, it should heal the content properly.
+		_, err = objLayer.HealObject(ctx, bucket, object, "", hopts)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		if countErrs(errs, nil) != len(fileInfos) {
+			t.Fatal("Expected all xl.meta healed, but partial heal detected")
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, minioMetaBucket, cfgFile, "", false)
+		nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+		if errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		for i := 0; i < nfi.Erasure.ParityBlocks; i++ {
+			erasureDisks[i].Delete(context.Background(), minioMetaBucket, pathJoin(cfgFile, xlStorageFormatFile), false)
+		}
+
+		// Try healing now, it should heal the content properly.
+		_, err = objLayer.HealObject(ctx, minioMetaBucket, cfgFile, "", hopts)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, minioMetaBucket, cfgFile, "", false)
+		if countErrs(errs, nil) != len(fileInfos) {
+			t.Fatal("Expected all xl.meta healed, but partial heal detected")
+		}
+	}
+}
+
+func TestHealObjectCorruptedPools(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	resetGlobalHealState()
+	defer resetGlobalHealState()
+
+	nDisks := 32
+	fsDirs, err := getRandomDisks(nDisks)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	defer removeRoots(fsDirs)
+
+	pools := mustGetPoolEndpoints(fsDirs[:16]...)
+	pools = append(pools, mustGetPoolEndpoints(fsDirs[16:]...)...)
+
+	// Everything is fine, should return nil
+	objLayer, _, err := initObjectLayer(ctx, pools)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	bucket := getRandomBucketName()
+	object := getRandomObjectName()
+	data := bytes.Repeat([]byte("a"), 5*1024*1024)
+	var opts ObjectOptions
+
+	err = objLayer.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+	if err != nil {
+		t.Fatalf("Failed to make a bucket - %v", err)
+	}
+
+	// Create an object with multiple parts uploaded in decreasing
+	// part number.
+	uploadID, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts)
+	if err != nil {
+		t.Fatalf("Failed to create a multipart upload - %v", err)
+	}
+
+	var uploadedParts []CompletePart
+	for _, partID := range []int{2, 1} {
+		pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+		if err1 != nil {
+			t.Fatalf("Failed to upload a part - %v", err1)
+		}
+		uploadedParts = append(uploadedParts, CompletePart{
+			PartNumber: pInfo.PartNumber,
+			ETag:       pInfo.ETag,
+		})
+	}
+
+	_, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{})
+	if err != nil {
+		t.Fatalf("Failed to complete multipart upload - %v", err)
+	}
+
+	// Test 1: Remove the object backend files from the first disk.
+	z := objLayer.(*erasureServerPools)
+	for _, set := range z.serverPools {
+		er := set.sets[0]
+		erasureDisks := er.getDisks()
+		firstDisk := erasureDisks[0]
+		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		if err != nil {
+			t.Fatalf("Failed to delete a file - %v", err)
+		}
+
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan})
+		if err != nil {
+			t.Fatalf("Failed to heal object - %v", err)
+		}
+
+		fileInfos, errs := readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		fi, err := getLatestFileInfo(ctx, fileInfos, errs)
+		if errors.Is(err, errFileNotFound) {
+			continue
+		}
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
+			t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
+		}
+
+		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+		if err != nil {
+			t.Errorf("Failure during deleting part.1 - %v", err)
+		}
+
+		err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{})
+		if err != nil {
+			t.Errorf("Failure during creating part.1 - %v", err)
+		}
+
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+		if err != nil {
+			t.Errorf("Expected nil but received %v", err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		nfi, err := getLatestFileInfo(ctx, fileInfos, errs)
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		if !reflect.DeepEqual(fi, nfi) {
+			t.Fatalf("FileInfo not equal after healing")
+		}
+
+		err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
+		if err != nil {
+			t.Errorf("Failure during deleting part.1 - %v", err)
+		}
+
+		bdata := bytes.Repeat([]byte("b"), int(nfi.Size))
+		err = firstDisk.WriteAll(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata)
+		if err != nil {
+			t.Errorf("Failure during creating part.1 - %v", err)
+		}
+
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+		if err != nil {
+			t.Errorf("Expected nil but received %v", err)
+		}
+
+		fileInfos, errs = readAllFileInfo(ctx, erasureDisks, bucket, object, "", false)
+		nfi, err = getLatestFileInfo(ctx, fileInfos, errs)
+		if err != nil {
+			t.Fatalf("Failed to getLatestFileInfo - %v", err)
+		}
+
+		if !reflect.DeepEqual(fi, nfi) {
+			t.Fatalf("FileInfo not equal after healing")
+		}
+
+		// Test 4: checks if HealObject returns an error when xl.meta is not found
+		// in more than read quorum number of disks, to create a corrupted situation.
+		for i := 0; i <= nfi.Erasure.DataBlocks; i++ {
+			erasureDisks[i].Delete(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+		}
+
+		// Try healing now, expect to receive errFileNotFound.
+		_, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan})
+		if err != nil {
+			if _, ok := err.(ObjectNotFound); !ok {
+				t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+			}
+		}
+
+		// since majority of xl.meta's are not available, object should be successfully deleted.
+		_, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{})
+		if _, ok := err.(ObjectNotFound); !ok {
+			t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err)
+		}
+
+		for i := 0; i < (nfi.Erasure.DataBlocks + nfi.Erasure.ParityBlocks); i++ {
+			_, err = erasureDisks[i].StatInfoFile(context.Background(), bucket, pathJoin(object, xlStorageFormatFile), false)
+			if err == nil {
+				t.Errorf("Expected xl.meta file to be not present, but succeeeded")
+			}
+		}
+	}
+}
+
 func TestHealObjectCorrupted(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -422,7 +698,7 @@ func TestHealObjectCorrupted(t *testing.T) {
 	}
 
 	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
-		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
+		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
 	}
 
 	err = firstDisk.Delete(context.Background(), bucket, pathJoin(object, fi.DataDir, "part.1"), false)
@@ -566,7 +842,7 @@ func TestHealObjectErasure(t *testing.T) {
 	}
 
 	if _, err = firstDisk.StatInfoFile(context.Background(), bucket, object+"/"+xlStorageFormatFile, false); err != nil {
-		t.Errorf("Expected er.meta file to be present but stat failed - %v", err)
+		t.Errorf("Expected xl.meta file to be present but stat failed - %v", err)
 	}
 
 	erasureDisks := er.getDisks()
@@ -582,7 +858,7 @@ func TestHealObjectErasure(t *testing.T) {
 
 	// Try healing now, expect to receive errDiskNotFound.
 	_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan})
-	// since majority of er.meta's are not available, object quorum can't be read properly and error will be errErasureReadQuorum
+	// since majority of xl.meta's are not available, object quorum can't be read properly and error will be errErasureReadQuorum
 	if _, ok := err.(InsufficientReadQuorum); !ok {
 		t.Errorf("Expected %v but received %v", InsufficientReadQuorum{}, err)
 	}
@@ -668,3 +944,134 @@ func TestHealEmptyDirectoryErasure(t *testing.T) {
 		}
 	}
 }
+
+func TestHealLastDataShard(t *testing.T) {
+	tests := []struct {
+		name     string
+		dataSize int64
+	}{
+		{"4KiB", 4 * humanize.KiByte},
+		{"64KiB", 64 * humanize.KiByte},
+		{"128KiB", 128 * humanize.KiByte},
+		{"1MiB", 1 * humanize.MiByte},
+		{"5MiB", 5 * humanize.MiByte},
+		{"10MiB", 10 * humanize.MiByte},
+		{"5MiB-1KiB", 5*humanize.MiByte - 1*humanize.KiByte},
+		{"10MiB-1Kib", 10*humanize.MiByte - 1*humanize.KiByte},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ctx, cancel := context.WithCancel(context.Background())
+			defer cancel()
+
+			nDisks := 16
+			fsDirs, err := getRandomDisks(nDisks)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			defer removeRoots(fsDirs)
+
+			obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(fsDirs...))
+			if err != nil {
+				t.Fatal(err)
+			}
+			bucket := "bucket"
+			object := "object"
+
+			data := make([]byte, test.dataSize)
+			_, err = rand.Read(data)
+			if err != nil {
+				t.Fatal(err)
+			}
+			var opts ObjectOptions
+
+			err = obj.MakeBucketWithLocation(ctx, bucket, BucketOptions{})
+			if err != nil {
+				t.Fatalf("Failed to make a bucket - %v", err)
+			}
+
+			_, err = obj.PutObject(ctx, bucket, object,
+				mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts)
+
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			actualH := sha256.New()
+			_, err = io.Copy(actualH, bytes.NewReader(data))
+			if err != nil {
+				return
+			}
+			actualSha256 := actualH.Sum(nil)
+
+			z := obj.(*erasureServerPools)
+			er := z.serverPools[0].getHashedSet(object)
+
+			disks := er.getDisks()
+			distribution := hashOrder(pathJoin(bucket, object), nDisks)
+			shuffledDisks := shuffleDisks(disks, distribution)
+
+			// remove last data shard
+			err = removeAll(pathJoin(shuffledDisks[11].String(), bucket, object))
+			if err != nil {
+				t.Fatalf("Failed to delete a file - %v", err)
+			}
+			_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{
+				ScanMode: madmin.HealNormalScan,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			firstGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
+			defer firstGr.Close()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			firstHealedH := sha256.New()
+			_, err = io.Copy(firstHealedH, firstGr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			firstHealedDataSha256 := firstHealedH.Sum(nil)
+
+			if !bytes.Equal(actualSha256, firstHealedDataSha256) {
+				t.Fatalf("object healed wrong, expected %x, got %x",
+					actualSha256, firstHealedDataSha256)
+			}
+
+			// remove another data shard
+			if err = removeAll(pathJoin(shuffledDisks[1].String(), bucket, object)); err != nil {
+				t.Fatalf("Failed to delete a file - %v", err)
+			}
+
+			_, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{
+				ScanMode: madmin.HealNormalScan,
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			secondGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, noLock, ObjectOptions{})
+			defer secondGr.Close()
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			secondHealedH := sha256.New()
+			_, err = io.Copy(secondHealedH, secondGr)
+			if err != nil {
+				t.Fatal(err)
+			}
+			secondHealedDataSha256 := secondHealedH.Sum(nil)
+
+			if !bytes.Equal(actualSha256, secondHealedDataSha256) {
+				t.Fatalf("object healed wrong, expected %x, got %x",
+					actualSha256, secondHealedDataSha256)
+			}
+		})
+	}
+}

cmd/erasure-lowlevel-heal.go

@@ -1,59 +0,0 @@
-// Copyright (c) 2015-2021 MinIO, Inc.
-//
-// This file is part of MinIO Object Storage stack
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package cmd
-
-import (
-	"context"
-	"io"
-
-	"github.com/minio/minio/internal/bpool"
-	xioutil "github.com/minio/minio/internal/ioutil"
-	"github.com/minio/minio/internal/logger"
-)
-
-// Heal heals the shard files on non-nil writers. Note that the quorum passed is 1
-// as healing should continue even if it has been successful healing only one shard file.
-func (e Erasure) Heal(ctx context.Context, readers []io.ReaderAt, writers []io.Writer, size int64, bp *bpool.BytePoolCap) error {
-	r, w := xioutil.WaitPipe()
-	go func() {
-		_, err := e.Decode(ctx, w, readers, 0, size, size, nil)
-		w.CloseWithError(err)
-	}()
-
-	// Fetch buffer for I/O, returns from the pool if not allocates a new one and returns.
-	var buffer []byte
-	switch {
-	case size == 0:
-		buffer = make([]byte, 1) // Allocate atleast a byte to reach EOF
-	case size >= e.blockSize:
-		buffer = bp.Get()
-		defer bp.Put(buffer)
-	case size < e.blockSize:
-		// No need to allocate fully blockSizeV1 buffer if the incoming data is smaller.
-		buffer = make([]byte, size, 2*size+int64(e.parityBlocks+e.dataBlocks-1))
-	}
-
-	// quorum is 1 because CreateFile should continue writing as long as we are writing to even 1 disk.
-	n, err := e.Encode(ctx, r, writers, buffer, 1)
-	if err == nil && n != size {
-		logger.LogIf(ctx, errLessData)
-		err = errLessData
-	}
-	r.CloseWithError(err)
-	return err
-}

cmd/erasure-metadata.go

@@ -351,6 +351,17 @@ func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.
 	return FileInfo{}, errErasureReadQuorum
 }
 
+func pickValidDiskTimeWithQuorum(metaArr []FileInfo, quorum int) time.Time {
+	diskMTimes := listObjectDiskMtimes(metaArr)
+
+	diskMTime, diskMaxima := commonTimeAndOccurence(diskMTimes, shardDiskTimeDelta)
+	if diskMaxima >= quorum {
+		return diskMTime
+	}
+
+	return timeSentinel
+}
+
 // pickValidFileInfo - picks one valid FileInfo content and returns from a
 // slice of FileInfo.
 func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Time, quorum int) (FileInfo, error) {

cmd/erasure-object.go

@@ -179,6 +179,30 @@ func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object stri
 		return nil, toObjectErr(err, bucket, object)
 	}
 
+	if !fi.DataShardFixed() {
+		diskMTime := pickValidDiskTimeWithQuorum(metaArr, fi.Erasure.DataBlocks)
+		if !diskMTime.Equal(timeSentinel) && !diskMTime.IsZero() {
+			for index := range onlineDisks {
+				if onlineDisks[index] == OfflineDisk {
+					continue
+				}
+				if !metaArr[index].IsValid() {
+					continue
+				}
+				if !metaArr[index].AcceptableDelta(diskMTime, shardDiskTimeDelta) {
+					// If disk mTime mismatches it is considered outdated
+					// https://github.com/minio/minio/pull/13803
+					//
+					// This check only is active if we could find maximally
+					// occurring disk mtimes that are somewhat same across
+					// the quorum. Allowing to skip those shards which we
+					// might think are wrong.
+					onlineDisks[index] = OfflineDisk
+				}
+			}
+		}
+	}
+
 	objInfo := fi.ToObjectInfo(bucket, object)
 	if objInfo.DeleteMarker {
 		if opts.VersionID == "" {
@@ -474,7 +498,7 @@ func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object strin
 func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, wquorum int, err error) {
 	fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false)
 	if err != nil {
-		return objInfo, getWriteQuorum(len(er.getDisks())), toObjectErr(err, bucket, object)
+		return objInfo, er.defaultWQuorum(), toObjectErr(err, bucket, object)
 	}
 
 	wquorum = fi.Erasure.DataBlocks
@@ -1089,7 +1113,7 @@ func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objec
 		// class for objects which have reduced quorum
 		// storage class only needs to be honored for
 		// Read() requests alone which we already do.
-		writeQuorums[i] = getWriteQuorum(len(storageDisks))
+		writeQuorums[i] = er.defaultWQuorum()
 	}
 
 	versionsMap := make(map[string]FileInfoVersions, len(objects))
@@ -1365,6 +1389,11 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 	fvID := mustGetUUID()
 	if markDelete {
 		if opts.Versioned || opts.VersionSuspended {
+			if !deleteMarker {
+				// versioning suspended means we add `null` version as
+				// delete marker, if its not decided already.
+				deleteMarker = opts.VersionSuspended && opts.VersionID == ""
+			}
 			fi := FileInfo{
 				Name:             object,
 				Deleted:          deleteMarker,
@@ -1381,10 +1410,10 @@ func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string
 					fi.VersionID = opts.VersionID
 				}
 			}
-			// versioning suspended means we add `null`
-			// version as delete marker
-			// Add delete marker, since we don't have any version specified explicitly.
-			// Or if a particular version id needs to be replicated.
+			// versioning suspended means we add `null` version as
+			// delete marker. Add delete marker, since we don't have
+			// any version specified explicitly. Or if a particular
+			// version id needs to be replicated.
 			if err = er.deleteObjectVersion(ctx, bucket, object, writeQuorum, fi, opts.DeleteMarker); err != nil {
 				return objInfo, toObjectErr(err, bucket, object)
 			}
@@ -1439,14 +1468,16 @@ func (er erasureObjects) addPartial(bucket, object, versionID string, size int64
 }
 
 func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) {
-	// Lock the object before updating tags.
-	lk := er.NewNSLock(bucket, object)
-	lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
-	if err != nil {
-		return ObjectInfo{}, err
+	if !opts.NoLock {
+		// Lock the object before updating metadata.
+		lk := er.NewNSLock(bucket, object)
+		lkctx, err := lk.GetLock(ctx, globalOperationTimeout)
+		if err != nil {
+			return ObjectInfo{}, err
+		}
+		ctx = lkctx.Context()
+		defer lk.Unlock(lkctx.Cancel)
 	}
-	ctx = lkctx.Context()
-	defer lk.Unlock(lkctx.Cancel)
 
 	disks := er.getDisks()
 
@@ -1569,7 +1600,7 @@ func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object st
 	// Wait for all the routines.
 	mErrs := g.Wait()
 
-	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, getWriteQuorum(len(onlineDisks)))
+	return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, er.defaultWQuorum())
 }
 
 // DeleteObjectTags - delete object tags from an existing object

cmd/erasure-server-pool.go

@@ -647,6 +647,10 @@ func (z *erasureServerPools) MakeBucketWithLocation(ctx context.Context, bucket
 		meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
 	}
 
+	if opts.VersioningEnabled {
+		meta.VersioningConfigXML = enabledBucketVersioningConfig
+	}
+
 	if err := meta.Save(context.Background(), z); err != nil {
 		return toObjectErr(err, bucket)
 	}
@@ -1050,6 +1054,7 @@ func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, pre
 		Marker:      marker,
 		InclDeleted: true,
 		AskDisks:    globalAPIConfig.getListQuorum(),
+		Versioned:   true,
 	}
 
 	merged, err := z.listPath(ctx, &opts)
@@ -1124,7 +1129,9 @@ func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, ma
 	}
 	merged, err := z.listPath(ctx, &opts)
 	if err != nil && err != io.EOF {
-		logger.LogIf(ctx, err)
+		if !isErrBucketNotFound(err) {
+			logger.LogIf(ctx, err)
+		}
 		return loi, err
 	}
 
@@ -1677,111 +1684,120 @@ func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, re
 // HealObjectFn closure function heals the object.
 type HealObjectFn func(bucket, object, versionID string) error
 
-func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObject HealObjectFn) error {
-	errCh := make(chan error)
+func listAndHeal(ctx context.Context, bucket, prefix string, set *erasureObjects, healEntry func(metaCacheEntry) error, errCh chan<- error) {
 	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	disks, _ := set.getOnlineDisksWithHealing()
+	if len(disks) == 0 {
+		errCh <- errors.New("listAndHeal: No non-healing disks found")
+		return
+	}
+
+	// How to resolve partial results.
+	resolver := metadataResolutionParams{
+		dirQuorum: 1,
+		objQuorum: 1,
+		bucket:    bucket,
+		strict:    false, // Allow less strict matching.
+	}
+
+	path := baseDirFromPrefix(prefix)
+	if path == "" {
+		path = prefix
+	}
+
+	lopts := listPathRawOptions{
+		disks:          disks,
+		bucket:         bucket,
+		path:           path,
+		recursive:      true,
+		forwardTo:      "",
+		minDisks:       1,
+		reportNotFound: false,
+		agreed: func(entry metaCacheEntry) {
+			if err := healEntry(entry); err != nil {
+				errCh <- err
+				return
+			}
+		},
+		partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
+			entry, ok := entries.resolve(&resolver)
+			if !ok {
+				// check if we can get one entry atleast
+				// proceed to heal nonetheless.
+				entry, _ = entries.firstFound()
+			}
+
+			if err := healEntry(*entry); err != nil {
+				errCh <- err
+				return
+			}
+		},
+		finished: nil,
+	}
+
+	if err := listPathRaw(ctx, lopts); err != nil {
+		errCh <- fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
+		return
+	}
+}
+
+func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error {
+	errCh := make(chan error)
+	healEntry := func(entry metaCacheEntry) error {
+		if entry.isDir() {
+			return nil
+		}
+		// We might land at .metacache, .trash, .multipart
+		// no need to heal them skip, only when bucket
+		// is '.minio.sys'
+		if bucket == minioMetaBucket {
+			if wildcard.Match("buckets/*/.metacache/*", entry.name) {
+				return nil
+			}
+			if wildcard.Match("tmp/*", entry.name) {
+				return nil
+			}
+			if wildcard.Match("multipart/*", entry.name) {
+				return nil
+			}
+			if wildcard.Match("tmp-old/*", entry.name) {
+				return nil
+			}
+		}
+		fivs, err := entry.fileInfoVersions(bucket)
+		if err != nil {
+			return healObjectFn(bucket, entry.name, "")
+		}
+
+		for _, version := range fivs.Versions {
+			if err := healObjectFn(bucket, version.Name, version.VersionID); err != nil {
+				return err
+			}
+		}
+
+		return nil
+	}
+
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
 	go func() {
 		defer close(errCh)
-		defer cancel()
 
+		var wg sync.WaitGroup
 		for _, erasureSet := range z.serverPools {
-			var wg sync.WaitGroup
 			for _, set := range erasureSet.sets {
-				set := set
 				wg.Add(1)
-				go func() {
+				go func(set *erasureObjects) {
 					defer wg.Done()
 
-					disks, _ := set.getOnlineDisksWithHealing()
-					if len(disks) == 0 {
-						cancel()
-						errCh <- errors.New("HealObjects: No non-healing disks found")
-						return
-					}
-
-					healEntry := func(entry metaCacheEntry) {
-						if entry.isDir() {
-							return
-						}
-						// We might land at .metacache, .trash, .multipart
-						// no need to heal them skip, only when bucket
-						// is '.minio.sys'
-						if bucket == minioMetaBucket {
-							if wildcard.Match("buckets/*/.metacache/*", entry.name) {
-								return
-							}
-							if wildcard.Match("tmp/*", entry.name) {
-								return
-							}
-							if wildcard.Match("multipart/*", entry.name) {
-								return
-							}
-							if wildcard.Match("tmp-old/*", entry.name) {
-								return
-							}
-						}
-						fivs, err := entry.fileInfoVersions(bucket)
-						if err != nil {
-							if err := healObject(bucket, entry.name, ""); err != nil {
-								cancel()
-								errCh <- err
-								return
-							}
-							return
-						}
-
-						for _, version := range fivs.Versions {
-							if err := healObject(bucket, version.Name, version.VersionID); err != nil {
-								cancel()
-								errCh <- err
-								return
-							}
-						}
-					}
-
-					// How to resolve partial results.
-					resolver := metadataResolutionParams{
-						dirQuorum: 1,
-						objQuorum: 1,
-						bucket:    bucket,
-					}
-
-					path := baseDirFromPrefix(prefix)
-					if path == "" {
-						path = prefix
-					}
-
-					lopts := listPathRawOptions{
-						disks:          disks,
-						bucket:         bucket,
-						path:           path,
-						recursive:      true,
-						forwardTo:      "",
-						minDisks:       1,
-						reportNotFound: false,
-						agreed:         healEntry,
-						partial: func(entries metaCacheEntries, nAgreed int, errs []error) {
-							entry, ok := entries.resolve(&resolver)
-							if !ok {
-								// check if we can get one entry atleast
-								// proceed to heal nonetheless.
-								entry, _ = entries.firstFound()
-							}
-
-							healEntry(*entry)
-						},
-						finished: nil,
-					}
-
-					if err := listPathRaw(ctx, lopts); err != nil {
-						cancel()
-						errCh <- fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)
-						return
-					}
-				}()
+					listAndHeal(ctx, bucket, prefix, set, healEntry, errCh)
+				}(set)
 			}
-			wg.Wait()
 		}
+		wg.Wait()
 	}()
 	return <-errCh
 }
@@ -1789,14 +1805,33 @@ func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix str
 func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) {
 	object = encodeDirObject(object)
 
-	for _, pool := range z.serverPools {
-		result, err := pool.HealObject(ctx, bucket, object, versionID, opts)
-		result.Object = decodeDirObject(result.Object)
-		if err != nil {
-			return result, err
-		}
-		return result, nil
+	errs := make([]error, len(z.serverPools))
+	results := make([]madmin.HealResultItem, len(z.serverPools))
+	var wg sync.WaitGroup
+	for idx, pool := range z.serverPools {
+		wg.Add(1)
+		go func(idx int, pool *erasureSets) {
+			defer wg.Done()
+			result, err := pool.HealObject(ctx, bucket, object, versionID, opts)
+			result.Object = decodeDirObject(result.Object)
+			errs[idx] = err
+			results[idx] = result
+		}(idx, pool)
 	}
+	wg.Wait()
+
+	for _, err := range errs {
+		if err != nil {
+			return madmin.HealResultItem{}, err
+		}
+	}
+
+	for _, result := range results {
+		if result.Object != "" {
+			return result, nil
+		}
+	}
+
 	if versionID != "" {
 		return madmin.HealResultItem{}, VersionNotFound{
 			Bucket:    bucket,

cmd/erasure.go

@@ -84,6 +84,19 @@ func (er erasureObjects) Shutdown(ctx context.Context) error {
 	return nil
 }
 
+// defaultWQuorum write quorum based on setDriveCount and defaultParityCount
+func (er erasureObjects) defaultWQuorum() int {
+	dataCount := er.setDriveCount - er.defaultParityCount
+	if dataCount == er.defaultParityCount {
+		return dataCount + 1
+	}
+	return dataCount
+}
+
+func (er erasureObjects) defaultRQuorum() int {
+	return er.setDriveCount - er.defaultParityCount
+}
+
 // byDiskTotal is a collection satisfying sort.Interface.
 type byDiskTotal []madmin.Disk
 

cmd/gateway-main.go

@@ -20,7 +20,8 @@ package cmd
 import (
 	"context"
 	"fmt"
-	"net"
+	"io/ioutil"
+	"log"
 	"net/url"
 	"os"
 	"os/signal"
@@ -269,10 +270,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		addrs = append(addrs, globalMinioAddr)
 	}
 
-	httpServer := xhttp.NewServer(addrs, setCriticalErrorHandler(corsHandler(router)), getCert)
-	httpServer.BaseContext = func(listener net.Listener) context.Context {
-		return GlobalContext
-	}
+	httpServer := xhttp.NewServer(addrs).
+		UseHandler(setCriticalErrorHandler(corsHandler(router))).
+		UseTLSConfig(newTLSConfig(getCert)).
+		UseShutdownTimeout(ctx.Duration("shutdown-timeout")).
+		UseBaseContext(GlobalContext).
+		UseCustomLogger(log.New(ioutil.Discard, "", 0)) // Turn-off random logging by Go stdlib
+
 	go func() {
 		globalHTTPServerErrorCh <- httpServer.Start(GlobalContext)
 	}()
@@ -304,7 +308,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(globalNotificationSys.Init(GlobalContext, buckets, newObject), "Unable to initialize notification system")
 	}
 
-	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
+	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalNotificationSys, globalRefreshIAMInterval)
 
 	if globalCacheConfig.Enabled {
 		// initialize the new disk cache objects.
@@ -331,11 +335,13 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// - compression
 	verifyObjectLayerFeatures("gateway "+gatewayName, newObject)
 
-	// Prints the formatted startup message once object layer is initialized.
-	if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
-		// Check update mode.
-		checkUpdate(globalMinioModeGatewayPrefix + gatewayName)
-	}
+	// Check for updates in non-blocking manner.
+	go func() {
+		if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
+			// Check for new updates from dl.min.io.
+			checkUpdate(getMinioMode())
+		}
+	}()
 
 	if !globalCLIContext.Quiet {
 		// Print gateway startup message.

cmd/global-heal.go

@@ -168,9 +168,14 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 	bgSeq := mustGetHealSequence(ctx)
 	scanMode := globalHealConfig.ScanMode()
 
+	// Make sure to copy since `buckets slice`
+	// is modified in place by tracker.
+	healBuckets := make([]string, len(buckets))
+	copy(healBuckets, buckets)
+
 	var retErr error
 	// Heal all buckets with all objects
-	for _, bucket := range buckets {
+	for _, bucket := range healBuckets {
 		if tracker.isHealed(bucket) {
 			continue
 		}
@@ -318,6 +323,8 @@ func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string,
 		default:
 			tracker.bucketDone(bucket)
 			logger.LogIf(ctx, tracker.update(ctx))
+			logger.Info("Healing bucket %s content on %s erasure set complete",
+				bucket, humanize.Ordinal(tracker.SetIndex+1))
 		}
 	}
 	tracker.Object = ""

cmd/globals.go

@@ -29,6 +29,7 @@ import (
 	"github.com/minio/console/restapi"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/bucket/bandwidth"
+	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/handlers"
 	"github.com/minio/minio/internal/kms"
 	"github.com/rs/dnscache"
@@ -148,8 +149,9 @@ var (
 	// This flag is set to 'true' when MINIO_UPDATE env is set to 'off'. Default is false.
 	globalInplaceUpdateDisabled = false
 
-	// This flag is set to 'us-east-1' by default
-	globalServerRegion = globalMinioDefaultRegion
+	globalSite = config.Site{
+		Region: globalMinioDefaultRegion,
+	}
 
 	// MinIO local server address (in `host:port` format)
 	globalMinioAddr = ""

cmd/handler-utils.go

@@ -58,7 +58,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 	} // else for both err as nil or io.EOF
 	location = locationConstraint.Location
 	if location == "" {
-		location = globalServerRegion
+		location = globalSite.Region
 	}
 	return location, ErrNone
 }
@@ -66,7 +66,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 // Validates input location is same as configured region
 // of MinIO server.
 func isValidLocation(location string) bool {
-	return globalServerRegion == "" || globalServerRegion == location
+	return globalSite.Region == "" || globalSite.Region == location
 }
 
 // Supported headers that needs to be extracted.
@@ -222,7 +222,7 @@ func extractReqParams(r *http.Request) map[string]string {
 		return nil
 	}
 
-	region := globalServerRegion
+	region := globalSite.Region
 	cred := getReqAccessCred(r, region)
 
 	principalID := cred.AccessKey
@@ -429,6 +429,18 @@ func extractAPIVersion(r *http.Request) string {
 	return "unknown"
 }
 
+func generateUnexpectedRPCMsg(rpcPath, subsystem string, expectedVersion, gotVersion string) string {
+	var reason string
+	switch {
+	case expectedVersion != gotVersion:
+		reason = fmt.Sprintf("Server expects '%s' API version '%s', instead found '%s'", subsystem, expectedVersion, gotVersion)
+	default:
+		reason = fmt.Sprintf("Unexpected RPC call at this path '%s'", rpcPath)
+	}
+
+	return fmt.Sprintf("%s - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", reason, ReleaseTag)
+}
+
 func methodNotAllowedHandler(api string) func(w http.ResponseWriter, r *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if r.Method == http.MethodOptions {
@@ -437,21 +449,21 @@ func methodNotAllowedHandler(api string) func(w http.ResponseWriter, r *http.Req
 		version := extractAPIVersion(r)
 		switch {
 		case strings.HasPrefix(r.URL.Path, peerRESTPrefix):
-			desc := fmt.Sprintf("Server expects 'peer' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", peerRESTVersion, version, ReleaseTag)
+			desc := generateUnexpectedRPCMsg(r.URL.Path, "peer", peerRESTVersion, version)
 			writeErrorResponseString(r.Context(), w, APIError{
 				Code:           "XMinioPeerVersionMismatch",
 				Description:    desc,
 				HTTPStatusCode: http.StatusUpgradeRequired,
 			}, r.URL)
 		case strings.HasPrefix(r.URL.Path, storageRESTPrefix):
-			desc := fmt.Sprintf("Server expects 'storage' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", storageRESTVersion, version, ReleaseTag)
+			desc := generateUnexpectedRPCMsg(r.URL.Path, "storage", storageRESTVersion, version)
 			writeErrorResponseString(r.Context(), w, APIError{
 				Code:           "XMinioStorageVersionMismatch",
 				Description:    desc,
 				HTTPStatusCode: http.StatusUpgradeRequired,
 			}, r.URL)
 		case strings.HasPrefix(r.URL.Path, lockRESTPrefix):
-			desc := fmt.Sprintf("Server expects 'lock' API version '%s', instead found '%s' - *rolling upgrade is not allowed* - please make sure all servers are running the same MinIO version (%s)", lockRESTVersion, version, ReleaseTag)
+			desc := generateUnexpectedRPCMsg(r.URL.Path, "lock", lockRESTVersion, version)
 			writeErrorResponseString(r.Context(), w, APIError{
 				Code:           "XMinioLockVersionMismatch",
 				Description:    desc,

cmd/iam-dummy-store.go

@@ -22,7 +22,6 @@ import (
 	"sync"
 
 	"github.com/minio/minio/internal/auth"
-	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 type iamDummyStore struct {
@@ -64,7 +63,7 @@ func (ids *iamDummyStore) migrateBackendFormat(context.Context) error {
 	return nil
 }
 
-func (ids *iamDummyStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]iampolicy.Policy) error {
+func (ids *iamDummyStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
 	v, ok := ids.iamPolicyDocsMap[policy]
 	if !ok {
 		return errNoSuchPolicy
@@ -73,7 +72,7 @@ func (ids *iamDummyStore) loadPolicyDoc(ctx context.Context, policy string, m ma
 	return nil
 }
 
-func (ids *iamDummyStore) loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error {
+func (ids *iamDummyStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error {
 	for k, v := range ids.iamPolicyDocsMap {
 		m[k] = v
 	}
@@ -154,7 +153,7 @@ func (ids *iamDummyStore) deleteIAMConfig(ctx context.Context, path string) erro
 	return nil
 }
 
-func (ids *iamDummyStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
+func (ids *iamDummyStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
 	return nil
 }
 

cmd/iam-etcd-store.go

@@ -32,7 +32,6 @@ import (
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	iampolicy "github.com/minio/pkg/iam/policy"
 	"go.etcd.io/etcd/api/v3/mvccpb"
 	etcd "go.etcd.io/etcd/client/v3"
 )
@@ -115,7 +114,7 @@ func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item interface{}, it
 	return saveKeyEtcd(ctx, ies.client, itemPath, data, opts...)
 }
 
-func getIAMConfig(item interface{}, data []byte, itemPath string) error {
+func decryptData(data []byte, itemPath string) ([]byte, error) {
 	var err error
 	if !utf8.Valid(data) && GlobalKMS != nil {
 		data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
@@ -128,10 +127,18 @@ func getIAMConfig(item interface{}, data []byte, itemPath string) error {
 				minioMetaBucket: itemPath,
 			})
 			if err != nil {
-				return err
+				return nil, err
 			}
 		}
 	}
+	return data, nil
+}
+
+func getIAMConfig(item interface{}, data []byte, itemPath string) error {
+	data, err := decryptData(data, itemPath)
+	if err != nil {
+		return err
+	}
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	return json.Unmarshal(data, item)
 }
@@ -144,6 +151,14 @@ func (ies *IAMEtcdStore) loadIAMConfig(ctx context.Context, item interface{}, pa
 	return getIAMConfig(item, data, path)
 }
 
+func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([]byte, error) {
+	data, err := readKeyEtcd(ctx, ies.client, path)
+	if err != nil {
+		return nil, err
+	}
+	return decryptData(data, path)
+}
+
 func (ies *IAMEtcdStore) deleteIAMConfig(ctx context.Context, path string) error {
 	return deleteKeyEtcd(ctx, ies.client, path)
 }
@@ -263,34 +278,46 @@ func (ies *IAMEtcdStore) migrateBackendFormat(ctx context.Context) error {
 	return ies.migrateToV1(ctx)
 }
 
-func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]iampolicy.Policy) error {
-	var p iampolicy.Policy
-	err := ies.loadIAMConfig(ctx, &p, getPolicyDocPath(policy))
+func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
+	data, err := ies.loadIAMConfigBytes(ctx, getPolicyDocPath(policy))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
 		}
 		return err
 	}
+
+	var p PolicyDoc
+	err = p.parseJSON(data)
+	if err != nil {
+		return err
+	}
+
 	m[policy] = p
 	return nil
 }
 
-func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValue, m map[string]iampolicy.Policy) error {
-	var p iampolicy.Policy
-	err := getIAMConfig(&p, kvs.Value, string(kvs.Key))
+func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValue, m map[string]PolicyDoc) error {
+	data, err := decryptData(kvs.Value, string(kvs.Key))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
 		}
 		return err
 	}
+
+	var p PolicyDoc
+	err = p.parseJSON(data)
+	if err != nil {
+		return err
+	}
+
 	policy := extractPathPrefixAndSuffix(string(kvs.Key), iamConfigPoliciesPrefix, path.Base(string(kvs.Key)))
 	m[policy] = p
 	return nil
 }
 
-func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error {
+func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error {
 	ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout)
 	defer cancel()
 	//  Retrieve all keys and values to avoid too many calls to etcd in case of
@@ -473,7 +500,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 
 }
 
-func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
+func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
 	return ies.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
 }
 

cmd/iam-object-store.go

@@ -29,7 +29,6 @@ import (
 	"github.com/minio/minio/internal/config"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
-	iampolicy "github.com/minio/pkg/iam/policy"
 )
 
 // IAMObjectStore implements IAMStorageAPI
@@ -218,19 +217,27 @@ func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item interface{}
 	return saveConfig(ctx, iamOS.objAPI, objPath, data)
 }
 
-func (iamOS *IAMObjectStore) loadIAMConfig(ctx context.Context, item interface{}, objPath string) error {
-	data, err := readConfig(ctx, iamOS.objAPI, objPath)
+func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context, objPath string) ([]byte, ObjectInfo, error) {
+	data, meta, err := readConfigWithMetadata(ctx, iamOS.objAPI, objPath)
 	if err != nil {
-		return err
+		return nil, meta, err
 	}
 	if !utf8.Valid(data) && GlobalKMS != nil {
 		data, err = config.DecryptBytes(GlobalKMS, data, kms.Context{
 			minioMetaBucket: path.Join(minioMetaBucket, objPath),
 		})
 		if err != nil {
-			return err
+			return nil, meta, err
 		}
 	}
+	return data, meta, nil
+}
+
+func (iamOS *IAMObjectStore) loadIAMConfig(ctx context.Context, item interface{}, objPath string) error {
+	data, _, err := iamOS.loadIAMConfigBytesWithMetadata(ctx, objPath)
+	if err != nil {
+		return err
+	}
 	var json = jsoniter.ConfigCompatibleWithStandardLibrary
 	return json.Unmarshal(data, item)
 }
@@ -239,20 +246,34 @@ func (iamOS *IAMObjectStore) deleteIAMConfig(ctx context.Context, path string) e
 	return deleteConfig(ctx, iamOS.objAPI, path)
 }
 
-func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]iampolicy.Policy) error {
-	var p iampolicy.Policy
-	err := iamOS.loadIAMConfig(ctx, &p, getPolicyDocPath(policy))
+func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error {
+	data, objInfo, err := iamOS.loadIAMConfigBytesWithMetadata(ctx, getPolicyDocPath(policy))
 	if err != nil {
 		if err == errConfigNotFound {
 			return errNoSuchPolicy
 		}
 		return err
 	}
+
+	var p PolicyDoc
+	err = p.parseJSON(data)
+	if err != nil {
+		return err
+	}
+
+	if p.Version == 0 {
+		// This means that policy was in the old version (without any
+		// timestamp info). We fetch the mod time of the file and save
+		// that as create and update date.
+		p.CreateDate = objInfo.ModTime
+		p.UpdateDate = objInfo.ModTime
+	}
+
 	m[policy] = p
 	return nil
 }
 
-func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error {
+func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error {
 	for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPoliciesPrefix) {
 		if item.Err != nil {
 			return item.Err
@@ -385,7 +406,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
 	return nil
 }
 
-func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
+func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error {
 	return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName))
 }
 

cmd/iam-store.go

@@ -24,8 +24,10 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/dustin/go-humanize"
+	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
@@ -169,6 +171,64 @@ func newMappedPolicy(policy string) MappedPolicy {
 	return MappedPolicy{Version: 1, Policies: policy}
 }
 
+// PolicyDoc represents an IAM policy with some metadata.
+type PolicyDoc struct {
+	Version    int `json:",omitempty"`
+	Policy     iampolicy.Policy
+	CreateDate time.Time `json:",omitempty"`
+	UpdateDate time.Time `json:",omitempty"`
+}
+
+func newPolicyDoc(p iampolicy.Policy) PolicyDoc {
+	now := UTCNow().Round(time.Millisecond)
+	return PolicyDoc{
+		Version:    1,
+		Policy:     p,
+		CreateDate: now,
+		UpdateDate: now,
+	}
+}
+
+// defaultPolicyDoc - used to wrap a default policy as PolicyDoc.
+func defaultPolicyDoc(p iampolicy.Policy) PolicyDoc {
+	return PolicyDoc{
+		Version: 1,
+		Policy:  p,
+	}
+}
+
+func (d *PolicyDoc) update(p iampolicy.Policy) {
+	now := UTCNow().Round(time.Millisecond)
+	d.UpdateDate = now
+	if d.CreateDate.IsZero() {
+		d.CreateDate = now
+	}
+	d.Policy = p
+}
+
+// parseJSON parses both the old and the new format for storing policy
+// definitions.
+//
+// The on-disk format of policy definitions has changed (around early 12/2021)
+// from iampolicy.Policy to PolicyDoc. To avoid a migration, loading supports
+// both the old and the new formats.
+func (d *PolicyDoc) parseJSON(data []byte) error {
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	var doc PolicyDoc
+	err := json.Unmarshal(data, &doc)
+	if err != nil {
+		err2 := json.Unmarshal(data, &doc.Policy)
+		if err2 != nil {
+			// Just return the first error.
+			return err
+		}
+		d.Policy = doc.Policy
+		return nil
+	}
+	*d = doc
+	return nil
+}
+
 // key options
 type options struct {
 	ttl int64 // expiry in seconds
@@ -182,7 +242,7 @@ type iamWatchEvent struct {
 // iamCache contains in-memory cache of IAM data.
 type iamCache struct {
 	// map of policy names to policy definitions
-	iamPolicyDocsMap map[string]iampolicy.Policy
+	iamPolicyDocsMap map[string]PolicyDoc
 	// map of usernames to credentials
 	iamUsersMap map[string]auth.Credentials
 	// map of group names to group info
@@ -197,7 +257,7 @@ type iamCache struct {
 
 func newIamCache() *iamCache {
 	return &iamCache{
-		iamPolicyDocsMap:        map[string]iampolicy.Policy{},
+		iamPolicyDocsMap:        map[string]PolicyDoc{},
 		iamUsersMap:             map[string]auth.Credentials{},
 		iamGroupsMap:            map[string]GroupInfo{},
 		iamUserGroupMemberships: map[string]set.StringSet{},
@@ -332,8 +392,8 @@ type IAMStorageAPI interface {
 
 	getUsersSysType() UsersSysType
 
-	loadPolicyDoc(ctx context.Context, policy string, m map[string]iampolicy.Policy) error
-	loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error
+	loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error
+	loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error
 
 	loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error
 	loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error
@@ -348,7 +408,7 @@ type IAMStorageAPI interface {
 	loadIAMConfig(ctx context.Context, item interface{}, path string) error
 	deleteIAMConfig(ctx context.Context, path string) error
 
-	savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error
+	savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error
 	saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error
 	saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error
 	saveGroupInfo(ctx context.Context, group string, gi GroupInfo) error
@@ -366,10 +426,10 @@ type iamStorageWatcher interface {
 }
 
 // Set default canned policies only if not already overridden by users.
-func setDefaultCannedPolicies(policies map[string]iampolicy.Policy) {
+func setDefaultCannedPolicies(policies map[string]PolicyDoc) {
 	for _, v := range iampolicy.DefaultPolicies {
 		if _, ok := policies[v.Name]; !ok {
-			policies[v.Name] = v.Definition
+			policies[v.Name] = defaultPolicyDoc(v.Definition)
 		}
 	}
 }
@@ -846,12 +906,14 @@ func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy
 			if !pset.Contains(policy) {
 				continue
 			}
-			_, ok := cache.iamUsersMap[u]
-			if !ok {
-				// happens when account is deleted or
-				// expired.
-				delete(cache.iamUserPolicyMap, u)
-				continue
+			if store.getUsersSysType() == MinIOUsersSysType {
+				_, ok := cache.iamUsersMap[u]
+				if !ok {
+					// happens when account is deleted or
+					// expired.
+					delete(cache.iamUserPolicyMap, u)
+					continue
+				}
 			}
 			pset.Remove(policy)
 			cache.iamUserPolicyMap[u] = newMappedPolicy(strings.Join(pset.ToSlice(), ","))
@@ -886,11 +948,13 @@ func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string) error
 	groups := []string{}
 	for u, mp := range cache.iamUserPolicyMap {
 		pset := mp.policySet()
-		if _, ok := cache.iamUsersMap[u]; !ok {
-			// This case can happen when a temporary account is
-			// deleted or expired - remove it from userPolicyMap.
-			delete(cache.iamUserPolicyMap, u)
-			continue
+		if store.getUsersSysType() == MinIOUsersSysType {
+			if _, ok := cache.iamUsersMap[u]; !ok {
+				// This case can happen when a temporary account is
+				// deleted or expired - remove it from userPolicyMap.
+				delete(cache.iamUserPolicyMap, u)
+				continue
+			}
 		}
 		if pset.Contains(policy) {
 			users = append(users, u)
@@ -943,13 +1007,31 @@ func (store *IAMStoreSys) GetPolicy(name string) (iampolicy.Policy, error) {
 		}
 		v, ok := cache.iamPolicyDocsMap[policy]
 		if !ok {
-			return v, errNoSuchPolicy
+			return v.Policy, errNoSuchPolicy
 		}
-		combinedPolicy = combinedPolicy.Merge(v)
+		combinedPolicy = combinedPolicy.Merge(v.Policy)
 	}
 	return combinedPolicy, nil
 }
 
+// GetPolicyDoc - gets the policy doc which has the policy and some metadata.
+// Exactly one policy must be specified here.
+func (store *IAMStoreSys) GetPolicyDoc(name string) (r PolicyDoc, err error) {
+	name = strings.TrimSpace(name)
+	if name == "" {
+		return r, errInvalidArgument
+	}
+
+	cache := store.rlock()
+	defer store.runlock()
+
+	v, ok := cache.iamPolicyDocsMap[name]
+	if !ok {
+		return r, errNoSuchPolicy
+	}
+	return v, nil
+}
+
 // SetPolicy - creates a policy with name.
 func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iampolicy.Policy) error {
 
@@ -960,11 +1042,21 @@ func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy iam
 	cache := store.lock()
 	defer store.unlock()
 
-	if err := store.savePolicyDoc(ctx, name, policy); err != nil {
+	var (
+		d  PolicyDoc
+		ok bool
+	)
+	if d, ok = cache.iamPolicyDocsMap[name]; ok {
+		d.update(policy)
+	} else {
+		d = newPolicyDoc(policy)
+	}
+
+	if err := store.savePolicyDoc(ctx, name, d); err != nil {
 		return err
 	}
 
-	cache.iamPolicyDocsMap[name] = policy
+	cache.iamPolicyDocsMap[name] = d
 	return nil
 
 }
@@ -975,7 +1067,7 @@ func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (
 	cache := store.lock()
 	defer store.unlock()
 
-	m := map[string]iampolicy.Policy{}
+	m := map[string]PolicyDoc{}
 	err := store.loadPolicyDocs(ctx, m)
 	if err != nil {
 		return nil, err
@@ -988,8 +1080,8 @@ func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (
 
 	ret := map[string]iampolicy.Policy{}
 	for k, v := range m {
-		if bucketName == "" || v.MatchResource(bucketName) {
-			ret[k] = v
+		if bucketName == "" || v.Policy.MatchResource(bucketName) {
+			ret[k] = v.Policy
 		}
 	}
 
@@ -1007,9 +1099,9 @@ func filterPolicies(cache *iamCache, policyName string, bucketName string) (stri
 		}
 		p, found := cache.iamPolicyDocsMap[policy]
 		if found {
-			if bucketName == "" || p.MatchResource(bucketName) {
+			if bucketName == "" || p.Policy.MatchResource(bucketName) {
 				policies = append(policies, policy)
-				combinedPolicy = combinedPolicy.Merge(p)
+				combinedPolicy = combinedPolicy.Merge(p.Policy)
 			}
 		}
 	}
@@ -1095,15 +1187,6 @@ func (store *IAMStoreSys) GetUsers() map[string]madmin.UserInfo {
 		}
 	}
 
-	if store.getUsersSysType() == LDAPUsersSysType {
-		for k, v := range cache.iamUserPolicyMap {
-			result[k] = madmin.UserInfo{
-				PolicyName: v.Policies,
-				Status:     madmin.AccountEnabled,
-			}
-		}
-	}
-
 	return result
 }
 
@@ -1309,9 +1392,11 @@ func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, user
 	return err
 }
 
-// SetTempUser - saves temporary credential to storage and cache.
+// SetTempUser - saves temporary (STS) credential to storage and cache. If a
+// policy name is given, it is associated with the parent user specified in the
+// credential.
 func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) error {
-	if accessKey == "" || !cred.IsTemp() || cred.IsExpired() {
+	if accessKey == "" || !cred.IsTemp() || cred.IsExpired() || cred.ParentUser == "" {
 		return errInvalidArgument
 	}
 
@@ -1328,26 +1413,12 @@ func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cre
 			return fmt.Errorf("specified policy %s, not found %w", policyName, errNoSuchPolicy)
 		}
 
-		err := store.saveMappedPolicy(ctx, accessKey, stsUser, false, mp, options{ttl: ttl})
+		err := store.saveMappedPolicy(ctx, cred.ParentUser, stsUser, false, mp, options{ttl: ttl})
 		if err != nil {
 			return err
 		}
 
-		cache.iamUserPolicyMap[accessKey] = mp
-
-		// We are on purpose not persisting the policy map for parent
-		// user, although this is a hack, it is a good enough hack
-		// at this point in time - we need to overhaul our OIDC
-		// usage with service accounts with a more cleaner implementation
-		//
-		// This mapping is necessary to ensure that valid credentials
-		// have necessary ParentUser present - this is mainly for only
-		// webIdentity based STS tokens.
-		if cred.ParentUser != "" && cred.ParentUser != globalActiveCred.AccessKey {
-			if _, ok := cache.iamUserPolicyMap[cred.ParentUser]; !ok {
-				cache.iamUserPolicyMap[cred.ParentUser] = mp
-			}
-		}
+		cache.iamUserPolicyMap[cred.ParentUser] = mp
 	}
 
 	u := newUserIdentity(cred)
@@ -1558,7 +1629,7 @@ func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey str
 }
 
 // AddUser - adds/updates long term user account to storage.
-func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, uinfo madmin.UserInfo) error {
+func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) error {
 	cache := store.lock()
 	defer store.unlock()
 
@@ -1571,9 +1642,9 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, uinfo m
 
 	u := newUserIdentity(auth.Credentials{
 		AccessKey: accessKey,
-		SecretKey: uinfo.SecretKey,
+		SecretKey: ureq.SecretKey,
 		Status: func() string {
-			if uinfo.Status == madmin.AccountEnabled {
+			if ureq.Status == madmin.AccountEnabled {
 				return auth.AccountOn
 			}
 			return auth.AccountOff
@@ -1586,25 +1657,7 @@ func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, uinfo m
 
 	cache.iamUsersMap[accessKey] = u.Credentials
 
-	// Set policy if specified.
-	if uinfo.PolicyName != "" {
-		policy := uinfo.PolicyName
-		// Handle policy mapping set/update
-		mp := newMappedPolicy(policy)
-		for _, p := range mp.toSlice() {
-			if _, found := cache.iamPolicyDocsMap[policy]; !found {
-				logger.LogIf(GlobalContext, fmt.Errorf("%w: (%s)", errNoSuchPolicy, p))
-				return errNoSuchPolicy
-			}
-		}
-
-		if err := store.saveMappedPolicy(ctx, accessKey, regUser, false, mp); err != nil {
-			return err
-		}
-		cache.iamUserPolicyMap[accessKey] = mp
-	}
 	return nil
-
 }
 
 // UpdateUserSecretKey - sets user secret key to storage.

cmd/iam.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"math/rand"
 	"path"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -33,7 +34,9 @@ import (
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio-go/v7/pkg/set"
+	"github.com/minio/minio/internal/arn"
 	"github.com/minio/minio/internal/auth"
+	"github.com/minio/minio/internal/color"
 	"github.com/minio/minio/internal/logger"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	etcd "go.etcd.io/etcd/client/v3"
@@ -63,9 +66,12 @@ type IAMSys struct {
 	sync.Mutex
 
 	iamRefreshInterval time.Duration
+	notificationSys    *NotificationSys
 
 	usersSysType UsersSysType
 
+	rolesMap map[arn.ARN]string
+
 	// Persistence layer for IAM subsystem
 	store *IAMStoreSys
 
@@ -193,11 +199,12 @@ func (sys *IAMSys) Load(ctx context.Context) error {
 }
 
 // Init - initializes config system by reading entries from config/iam
-func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, iamRefreshInterval time.Duration) {
+func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, nSys *NotificationSys, iamRefreshInterval time.Duration) {
 	sys.Lock()
 	defer sys.Unlock()
 
 	sys.iamRefreshInterval = iamRefreshInterval
+	sys.notificationSys = nSys
 
 	// Initialize IAM store
 	sys.initStore(objAPI, etcdClient)
@@ -215,6 +222,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 
 	r := rand.New(rand.NewSource(time.Now().UnixNano()))
 
+	// Migrate storage format if needed.
 	for {
 		// let one of the server acquire the lock, if not let them timeout.
 		// which shall be retried again by this loop.
@@ -263,6 +271,7 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		break
 	}
 
+	// Load IAM data from storage.
 	for {
 		if err := sys.Load(retryCtx); err != nil {
 			if configRetriableErrors(err) {
@@ -308,7 +317,42 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etc
 		}()
 	}
 
+	// Start watching changes to storage.
 	go sys.watch(ctx)
+
+	// Load RoleARN
+	if roleARN, rolePolicy, enabled := globalOpenIDConfig.GetRoleInfo(); enabled {
+		numPolicies := len(strings.Split(rolePolicy, ","))
+		validPolicies, _ := sys.store.FilterPolicies(rolePolicy, "")
+		numValidPolicies := len(strings.Split(validPolicies, ","))
+		if numPolicies != numValidPolicies {
+			logger.LogIf(ctx, fmt.Errorf("Some specified role policies (%s) were not defined - role based policies will not be enabled.", rolePolicy))
+			return
+		}
+		sys.rolesMap = map[arn.ARN]string{
+			roleARN: rolePolicy,
+		}
+	}
+
+	sys.printIAMRoles()
+}
+
+// Prints IAM role ARNs.
+func (sys *IAMSys) printIAMRoles() {
+	if len(sys.rolesMap) == 0 {
+		return
+	}
+	var arns []string
+	for arn := range sys.rolesMap {
+		arns = append(arns, arn.String())
+	}
+	sort.Strings(arns)
+	msgs := make([]string, 0, len(arns))
+	for _, arn := range arns {
+		msgs = append(msgs, color.Bold(arn))
+	}
+
+	logStartupMessage(fmt.Sprintf("%s %s", color.Blue("IAM Roles:"), strings.Join(msgs, " ")))
 }
 
 // HasWatcher - returns if the IAM system has a watcher to be notified of
@@ -389,22 +433,72 @@ func (sys *IAMSys) loadWatchedEvent(ctx context.Context, event iamWatchEvent) (e
 	return err
 }
 
+// HasRolePolicy - returns if a role policy is configured for IAM.
+func (sys *IAMSys) HasRolePolicy() bool {
+	return len(sys.rolesMap) > 0
+}
+
+// GetRolePolicy - returns policies associated with a role ARN.
+func (sys *IAMSys) GetRolePolicy(arnStr string) (string, error) {
+	arn, err := arn.Parse(arnStr)
+	if err != nil {
+		return "", fmt.Errorf("RoleARN parse err: %v", err)
+	}
+	rolePolicy, ok := sys.rolesMap[arn]
+	if !ok {
+		return "", fmt.Errorf("RoleARN %s is not defined.", arnStr)
+	}
+	return rolePolicy, nil
+}
+
 // DeletePolicy - deletes a canned policy from backend or etcd.
-func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string) error {
+func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPeers bool) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
-	return sys.store.DeletePolicy(ctx, policyName)
-}
-
-// InfoPolicy - expands the canned policy into its JSON structure.
-func (sys *IAMSys) InfoPolicy(policyName string) (iampolicy.Policy, error) {
-	if !sys.Initialized() {
-		return iampolicy.Policy{}, errServerNotInitialized
+	err := sys.store.DeletePolicy(ctx, policyName)
+	if err != nil {
+		return err
 	}
 
-	return sys.store.GetPolicy(policyName)
+	if !notifyPeers || sys.HasWatcher() {
+		return nil
+	}
+
+	// Notify all other MinIO peers to delete policy
+	for _, nerr := range sys.notificationSys.DeletePolicy(policyName) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	return nil
+}
+
+// InfoPolicy - returns the policy definition with some metadata.
+func (sys *IAMSys) InfoPolicy(policyName string) (*madmin.PolicyInfo, error) {
+	if !sys.Initialized() {
+		return nil, errServerNotInitialized
+	}
+
+	d, err := sys.store.GetPolicyDoc(policyName)
+	if err != nil {
+		return nil, err
+	}
+
+	pdata, err := json.Marshal(d.Policy)
+	if err != nil {
+		return nil, err
+	}
+
+	return &madmin.PolicyInfo{
+		PolicyName: policyName,
+		Policy:     pdata,
+		CreateDate: d.CreateDate,
+		UpdateDate: d.UpdateDate,
+	}, nil
 }
 
 // ListPolicies - lists all canned policies.
@@ -424,16 +518,44 @@ func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p iampolicy
 		return errServerNotInitialized
 	}
 
-	return sys.store.SetPolicy(ctx, policyName, p)
+	err := sys.store.SetPolicy(ctx, policyName, p)
+	if err != nil {
+		return err
+	}
+
+	if !sys.HasWatcher() {
+		// Notify all other MinIO peers to reload policy
+		for _, nerr := range sys.notificationSys.LoadPolicy(policyName) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
+	return nil
 }
 
 // DeleteUser - delete user (only for long-term users not STS users).
-func (sys *IAMSys) DeleteUser(ctx context.Context, accessKey string) error {
+func (sys *IAMSys) DeleteUser(ctx context.Context, accessKey string, notifyPeers bool) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
 
-	return sys.store.DeleteUser(ctx, accessKey, regUser)
+	if err := sys.store.DeleteUser(ctx, accessKey, regUser); err != nil {
+		return err
+	}
+
+	// Notify all other MinIO peers to delete user.
+	if notifyPeers && !sys.HasWatcher() {
+		for _, nerr := range sys.notificationSys.DeleteUser(accessKey) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
+
+	return nil
 }
 
 // CurrentPolicies - returns comma separated policy string, from
@@ -448,7 +570,57 @@ func (sys *IAMSys) CurrentPolicies(policyName string) string {
 	return policies
 }
 
-// SetTempUser - set temporary user credentials, these credentials have an expiry.
+func (sys *IAMSys) notifyForUser(ctx context.Context, accessKey string, isTemp bool) {
+	// Notify all other MinIO peers to reload user.
+	if !sys.HasWatcher() {
+		for _, nerr := range sys.notificationSys.LoadUser(accessKey, isTemp) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
+}
+
+// SetTempUser - set temporary user credentials, these credentials have an
+// expiry. The permissions for these STS credentials is determined in one of the
+// following ways:
+//
+// - RoleARN - if a role-arn is specified in the request, the STS credential's
+// policy is the role's policy.
+//
+// - inherited from parent - this is the case for AssumeRole API, where the
+// parent user is an actual real user with their own (permanent) credentials and
+// policy association.
+//
+// - inherited from "virtual" parent - this is the case for AssumeRoleWithLDAP
+// where the parent user is the DN of the actual LDAP user. The parent user
+// itself cannot login, but the policy associated with them determines the base
+// policy for the STS credential. The policy mapping can be updated by the
+// administrator.
+//
+// - from `Subject.CommonName` field from the STS request for
+// AssumeRoleWithCertificate. In this case, the policy for the STS credential
+// has the same name as the value of this field.
+//
+// - from special JWT claim from STS request for AssumeRoleWithOIDC API (when
+// not using RoleARN). The claim value can be a string or a list and refers to
+// the names of access policies.
+//
+// For all except the RoleARN case, the implementation is the same - the policy
+// for the STS credential is associated with a parent user. For the
+// AssumeRoleWithCertificate case, the "virtual" parent user is the value of the
+// `Subject.CommonName` field. For the OIDC (without RoleARN) case the "virtual"
+// parent is derived as a concatenation of the `sub` and `iss` fields. The
+// policies applicable to the STS credential are associated with this "virtual"
+// parent.
+//
+// When a policyName is given to this function, the policy association is
+// created and stored in the IAM store. Thus, it should NOT be given for the
+// role-arn case (because the role-to-policy mapping is separately stored
+// elsewhere), the AssumeRole case (because the parent user is real and their
+// policy is associated via policy-set API) and the AssumeRoleWithLDAP case
+// (because the policy association is made via policy-set API).
 func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
@@ -459,7 +631,13 @@ func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.
 		policyName = ""
 	}
 
-	return sys.store.SetTempUser(ctx, accessKey, cred, policyName)
+	err := sys.store.SetTempUser(ctx, accessKey, cred, policyName)
+	if err != nil {
+		return err
+	}
+
+	sys.notifyForUser(ctx, cred.AccessKey, true)
+	return nil
 }
 
 // ListBucketUsers - list all users who can access this 'bucket'
@@ -545,7 +723,25 @@ func (sys *IAMSys) SetUserStatus(ctx context.Context, accessKey string, status m
 		return errIAMActionNotAllowed
 	}
 
-	return sys.store.SetUserStatus(ctx, accessKey, status)
+	err := sys.store.SetUserStatus(ctx, accessKey, status)
+	if err != nil {
+		return err
+	}
+
+	sys.notifyForUser(ctx, accessKey, false)
+	return nil
+}
+
+func (sys *IAMSys) notifyForServiceAccount(ctx context.Context, accessKey string) {
+	// Notify all other Minio peers to reload the service account
+	if !sys.HasWatcher() {
+		for _, nerr := range sys.notificationSys.LoadServiceAccount(accessKey) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
 }
 
 type newServiceAccountOpts struct {
@@ -626,6 +822,8 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 	if err != nil {
 		return auth.Credentials{}, err
 	}
+
+	sys.notifyForServiceAccount(ctx, cred.AccessKey)
 	return cred, nil
 }
 
@@ -641,7 +839,13 @@ func (sys *IAMSys) UpdateServiceAccount(ctx context.Context, accessKey string, o
 		return errServerNotInitialized
 	}
 
-	return sys.store.UpdateServiceAccount(ctx, accessKey, opts)
+	err := sys.store.UpdateServiceAccount(ctx, accessKey, opts)
+	if err != nil {
+		return err
+	}
+
+	sys.notifyForServiceAccount(ctx, accessKey)
+	return nil
 }
 
 // ListServiceAccounts - lists all services accounts associated to a specific user
@@ -655,8 +859,20 @@ func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([
 	return sys.store.ListServiceAccounts(ctx, accessKey)
 }
 
-// GetServiceAccount - gets information about a service account
+// GetServiceAccount - wrapper method to get information about a service account
 func (sys *IAMSys) GetServiceAccount(ctx context.Context, accessKey string) (auth.Credentials, *iampolicy.Policy, error) {
+	sa, embeddedPolicy, err := sys.getServiceAccount(ctx, accessKey)
+	if err != nil {
+		return sa, embeddedPolicy, err
+	}
+	// Hide secret & session keys
+	sa.SecretKey = ""
+	sa.SessionToken = ""
+	return sa, embeddedPolicy, nil
+}
+
+// getServiceAccount - gets information about a service account
+func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (auth.Credentials, *iampolicy.Policy, error) {
 	if !sys.Initialized() {
 		return auth.Credentials{}, nil, errServerNotInitialized
 	}
@@ -684,10 +900,6 @@ func (sys *IAMSys) GetServiceAccount(ctx context.Context, accessKey string) (aut
 		}
 	}
 
-	// Hide secret & session keys
-	sa.SecretKey = ""
-	sa.SessionToken = ""
-
 	return sa, embeddedPolicy, nil
 }
 
@@ -714,7 +926,7 @@ func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (ma
 }
 
 // DeleteServiceAccount - delete a service account
-func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) error {
+func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, notifyPeers bool) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
@@ -724,12 +936,25 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) e
 		return nil
 	}
 
-	return sys.store.DeleteUser(ctx, accessKey, svcUser)
+	if err := sys.store.DeleteUser(ctx, accessKey, svcUser); err != nil {
+		return err
+	}
+
+	if notifyPeers && !sys.HasWatcher() {
+		for _, nerr := range sys.notificationSys.DeleteServiceAccount(accessKey) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
+
+	return nil
 }
 
 // CreateUser - create new user credentials and policy, if user already exists
 // they shall be rewritten with new inputs.
-func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, uinfo madmin.UserInfo) error {
+func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) error {
 	if !sys.Initialized() {
 		return errServerNotInitialized
 	}
@@ -742,11 +967,17 @@ func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, uinfo madmi
 		return auth.ErrInvalidAccessKeyLength
 	}
 
-	if !auth.IsSecretKeyValid(uinfo.SecretKey) {
+	if !auth.IsSecretKeyValid(ureq.SecretKey) {
 		return auth.ErrInvalidSecretKeyLength
 	}
 
-	return sys.store.AddUser(ctx, accessKey, uinfo)
+	err := sys.store.AddUser(ctx, accessKey, ureq)
+	if err != nil {
+		return err
+	}
+
+	sys.notifyForUser(ctx, accessKey, false)
+	return nil
 }
 
 // SetUserSecretKey - sets user secret key
@@ -920,6 +1151,18 @@ func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (cred auth.Cre
 	return cred, ok && cred.IsValid()
 }
 
+// Notify all other MinIO peers to load group.
+func (sys *IAMSys) notifyForGroup(ctx context.Context, group string) {
+	if !sys.HasWatcher() {
+		for _, nerr := range sys.notificationSys.LoadGroup(group) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
+}
+
 // AddUsersToGroup - adds users to a group, creating the group if
 // needed. No error if user(s) already are in the group.
 func (sys *IAMSys) AddUsersToGroup(ctx context.Context, group string, members []string) error {
@@ -931,7 +1174,13 @@ func (sys *IAMSys) AddUsersToGroup(ctx context.Context, group string, members []
 		return errIAMActionNotAllowed
 	}
 
-	return sys.store.AddUsersToGroup(ctx, group, members)
+	err := sys.store.AddUsersToGroup(ctx, group, members)
+	if err != nil {
+		return err
+	}
+
+	sys.notifyForGroup(ctx, group)
+	return nil
 }
 
 // RemoveUsersFromGroup - remove users from group. If no users are
@@ -945,7 +1194,13 @@ func (sys *IAMSys) RemoveUsersFromGroup(ctx context.Context, group string, membe
 		return errIAMActionNotAllowed
 	}
 
-	return sys.store.RemoveUsersFromGroup(ctx, group, members)
+	err := sys.store.RemoveUsersFromGroup(ctx, group, members)
+	if err != nil {
+		return err
+	}
+
+	sys.notifyForGroup(ctx, group)
+	return nil
 }
 
 // SetGroupStatus - enable/disabled a group
@@ -958,7 +1213,13 @@ func (sys *IAMSys) SetGroupStatus(ctx context.Context, group string, enabled boo
 		return errIAMActionNotAllowed
 	}
 
-	return sys.store.SetGroupStatus(ctx, group, enabled)
+	err := sys.store.SetGroupStatus(ctx, group, enabled)
+	if err != nil {
+		return err
+	}
+
+	sys.notifyForGroup(ctx, group)
+	return nil
 }
 
 // GetGroupDescription - builds up group description
@@ -993,7 +1254,22 @@ func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, isGroup
 		userType = stsUser
 	}
 
-	return sys.store.PolicyDBSet(ctx, name, policy, userType, isGroup)
+	err := sys.store.PolicyDBSet(ctx, name, policy, userType, isGroup)
+	if err != nil {
+		return err
+	}
+
+	// Notify all other MinIO peers to reload policy
+	if !sys.HasWatcher() {
+		for _, nerr := range sys.notificationSys.LoadPolicyMapping(name, isGroup) {
+			if nerr.Err != nil {
+				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+				logger.LogIf(ctx, nerr.Err)
+			}
+		}
+	}
+
+	return nil
 }
 
 // PolicyDBGet - gets policy set on a user or group. If a list of groups is
@@ -1029,7 +1305,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 		return false
 	}
 
-	// Check policy for this service account.
+	// Check policy for parent user of service account.
 	svcPolicies, err := sys.PolicyDBGet(parentUser, false, args.Groups...)
 	if err != nil {
 		logger.LogIf(GlobalContext, err)
@@ -1037,9 +1313,20 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser strin
 	}
 
 	if len(svcPolicies) == 0 {
-		// If parent user has no policies, look in OpenID claims in case it exists.
-		policySet, ok := iampolicy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID())
-		if ok {
+		// If parent user has no policies, check for OpenID
+		// claims/RoleARN in case it exists.
+		roleArn := args.GetRoleArn()
+		if roleArn != "" {
+			arn, err := arn.Parse(roleArn)
+			if err != nil {
+				logger.LogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
+				return false
+			}
+			svcPolicies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
+		} else {
+			// If there is no roleArn claim, check the OpenID
+			// provider's policy claim.
+			policySet, _ := iampolicy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID())
 			svcPolicies = policySet.ToSlice()
 		}
 		if len(svcPolicies) == 0 {
@@ -1156,35 +1443,43 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 		return sys.IsAllowedLDAPSTS(args, parentUser)
 	}
 
-	policies, ok := args.GetPolicies(iamPolicyClaimNameOpenID())
-	if !ok {
-		// When claims are set, it should have a policy claim field.
-		return false
+	var policies []string
+	roleArn := args.GetRoleArn()
+	if roleArn != "" {
+		arn, err := arn.Parse(roleArn)
+		if err != nil {
+			logger.LogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err))
+			return false
+		}
+		policies = newMappedPolicy(sys.rolesMap[arn]).toSlice()
+	} else {
+		// Lookup the parent user's mapping if there's no role-ARN.
+		var err error
+		policies, err = sys.store.PolicyDBGet(parentUser, false, args.Groups...)
+		if err != nil {
+			logger.LogIf(GlobalContext, fmt.Errorf("error fetching policies on %s: %v", parentUser, err))
+			return false
+		}
+		if len(policies) == 0 {
+			// TODO (deprecated in Dec 2021): Only need to handle
+			// behavior for STS credentials created in older
+			// releases. Otherwise, reject such cases, once older
+			// behavior is deprecated.
+
+			// If there is no parent policy mapping, we fall back to
+			// using policy claim from JWT.
+			policySet, ok := args.GetPolicies(iamPolicyClaimNameOpenID())
+			if !ok {
+				// When claims are set, it should have a policy claim field.
+				return false
+			}
+			policies = policySet.ToSlice()
+		}
 	}
 
-	// When claims are set, it should have policies as claim.
-	if policies.IsEmpty() {
-		// No policy, no access!
-		return false
-	}
-
-	// If policy is available for given user, check the policy.
-	mp, ok := sys.store.GetMappedPolicy(args.AccountName, false)
-	if !ok {
-		// No policy set for the user that we can find, no access!
-		return false
-	}
-
-	if !policies.Equals(mp.policySet()) {
-		// When claims has a policy, it should match the
-		// policy of args.AccountName which server remembers.
-		// if not reject such requests.
-		return false
-	}
-
-	combinedPolicy, err := sys.store.GetPolicy(strings.Join(policies.ToSlice(), ","))
+	combinedPolicy, err := sys.store.GetPolicy(strings.Join(policies, ","))
 	if err == errNoSuchPolicy {
-		for pname := range policies {
+		for _, pname := range policies {
 			_, err := sys.store.GetPolicy(pname)
 			if err == errNoSuchPolicy {
 				// all policies presented in the claim should exist

cmd/last-minute.go

@@ -102,39 +102,21 @@ type LastMinuteLatencies struct {
 	LastSec int64
 }
 
-// Clone safely returns a copy for a LastMinuteLatencies structure
-func (l *LastMinuteLatencies) Clone() LastMinuteLatencies {
-	n := LastMinuteLatencies{}
-	n.LastSec = l.LastSec
-	for i := range l.Totals {
-		for j := range l.Totals[i] {
-			n.Totals[i][j] = AccElem{
-				Total: l.Totals[i][j].Total,
-				N:     l.Totals[i][j].N,
-			}
-		}
-	}
-	return n
-}
-
 // Merge safely merges two LastMinuteLatencies structures into one
 func (l LastMinuteLatencies) Merge(o LastMinuteLatencies) (merged LastMinuteLatencies) {
-	cl := l.Clone()
-	co := o.Clone()
-
-	if cl.LastSec > co.LastSec {
-		co.forwardTo(cl.LastSec)
-		merged.LastSec = cl.LastSec
+	if l.LastSec > o.LastSec {
+		o.forwardTo(l.LastSec)
+		merged.LastSec = l.LastSec
 	} else {
-		cl.forwardTo(co.LastSec)
-		merged.LastSec = co.LastSec
+		l.forwardTo(o.LastSec)
+		merged.LastSec = o.LastSec
 	}
 
-	for i := range cl.Totals {
-		for j := range cl.Totals[i] {
+	for i := range merged.Totals {
+		for j := range merged.Totals[i] {
 			merged.Totals[i][j] = AccElem{
-				Total: cl.Totals[i][j].Total + co.Totals[i][j].Total,
-				N:     cl.Totals[i][j].N + co.Totals[i][j].N,
+				Total: l.Totals[i][j].Total + o.Totals[i][j].Total,
+				N:     l.Totals[i][j].N + o.Totals[i][j].N,
 			}
 		}
 	}

cmd/metacache-entries.go

@@ -37,7 +37,7 @@ type metaCacheEntry struct {
 	metadata []byte
 
 	// cached contains the metadata if decoded.
-	cached *FileInfo
+	cached *xlMetaV2
 
 	// Indicates the entry can be reused and only one reference to metadata is expected.
 	reusable bool
@@ -53,73 +53,90 @@ func (e metaCacheEntry) isObject() bool {
 	return len(e.metadata) > 0
 }
 
+// isObjectDir returns if the entry is representing an object__XL_DIR__
+func (e metaCacheEntry) isObjectDir() bool {
+	return len(e.metadata) > 0 && strings.HasSuffix(e.name, slashSeparator)
+}
+
 // hasPrefix returns whether an entry has a specific prefix
 func (e metaCacheEntry) hasPrefix(s string) bool {
 	return strings.HasPrefix(e.name, s)
 }
 
-// matches returns if the entries match by comparing their latest version fileinfo.
-func (e *metaCacheEntry) matches(other *metaCacheEntry, bucket string) bool {
+// matches returns if the entries have the same versions.
+// If strict is false we allow signatures to mismatch.
+func (e *metaCacheEntry) matches(other *metaCacheEntry, strict bool) (prefer *metaCacheEntry, matches bool) {
 	if e == nil && other == nil {
-		return true
+		return nil, true
 	}
-	if e == nil || other == nil {
-		return false
+	if e == nil {
+		return other, false
+	}
+	if other == nil {
+		return e, false
 	}
 
-	// This should reject 99%
-	if len(e.metadata) != len(other.metadata) || e.name != other.name {
-		return false
+	// Name should match...
+	if e.name != other.name {
+		if e.name < other.name {
+			return e, false
+		}
+		return other, false
 	}
 
-	eFi, eErr := e.fileInfo(bucket)
-	oFi, oErr := other.fileInfo(bucket)
+	eVers, eErr := e.xlmeta()
+	oVers, oErr := other.xlmeta()
 	if eErr != nil || oErr != nil {
-		return eErr == oErr
+		return nil, false
 	}
 
 	// check both fileInfo's have same number of versions, if not skip
 	// the `other` entry.
-	if eFi.NumVersions != oFi.NumVersions {
-		return false
-	}
-
-	return eFi.ModTime.Equal(oFi.ModTime) && eFi.Size == oFi.Size && eFi.VersionID == oFi.VersionID
-}
-
-// resolveEntries returns if the entries match by comparing their latest version fileinfo.
-func resolveEntries(a, b *metaCacheEntry, bucket string) *metaCacheEntry {
-	if b == nil {
-		return a
-	}
-	if a == nil {
-		return b
-	}
-
-	aFi, err := a.fileInfo(bucket)
-	if err != nil {
-		return b
-	}
-	bFi, err := b.fileInfo(bucket)
-	if err != nil {
-		return a
-	}
-
-	if aFi.NumVersions == bFi.NumVersions {
-		if aFi.ModTime.Equal(bFi.ModTime) {
-			return a
+	if len(eVers.versions) != len(oVers.versions) {
+		eTime := eVers.latestModtime()
+		oTime := oVers.latestModtime()
+		if !eTime.Equal(oTime) {
+			if eTime.After(oTime) {
+				return e, false
+			}
+			return other, false
 		}
-		if aFi.ModTime.After(bFi.ModTime) {
-			return a
+		// Tiebreak on version count.
+		if len(eVers.versions) > len(oVers.versions) {
+			return e, false
 		}
-		return b
+		return other, false
 	}
 
-	if bFi.NumVersions > aFi.NumVersions {
-		return b
-	}
+	// Check if each version matches...
+	for i, eVer := range eVers.versions {
+		oVer := oVers.versions[i]
+		if eVer.header != oVer.header {
+			if !strict && eVer.header.matchesNotStrict(oVer.header) {
+				if prefer == nil {
+					if eVer.header.sortsBefore(oVer.header) {
+						prefer = e
+					} else {
+						prefer = other
+					}
+				}
+				continue
+			}
+			if prefer != nil {
+				return prefer, false
+			}
 
-	return a
+			if eVer.header.sortsBefore(oVer.header) {
+				return e, false
+			}
+			return other, false
+		}
+	}
+	// If we match, return e
+	if prefer == nil {
+		prefer = e
+	}
+	return prefer, true
 }
 
 // isInDir returns whether the entry is in the dir when considering the separator.
@@ -143,7 +160,10 @@ func (e metaCacheEntry) isInDir(dir, separator string) bool {
 // If v2 and UNABLE to load metadata true will be returned.
 func (e *metaCacheEntry) isLatestDeletemarker() bool {
 	if e.cached != nil {
-		return e.cached.Deleted
+		if len(e.cached.versions) == 0 {
+			return true
+		}
+		return e.cached.versions[0].header.Type == DeleteType
 	}
 	if !isXL2V1Format(e.metadata) {
 		return false
@@ -152,8 +172,8 @@ func (e *metaCacheEntry) isLatestDeletemarker() bool {
 		return meta.IsLatestDeleteMarker()
 	}
 	// Fall back...
-	var xlMeta xlMetaV2
-	if err := xlMeta.Load(e.metadata); err != nil || len(xlMeta.versions) == 0 {
+	xlMeta, err := e.xlmeta()
+	if err != nil || len(xlMeta.versions) == 0 {
 		return true
 	}
 	return xlMeta.versions[0].header.Type == DeleteType
@@ -162,24 +182,37 @@ func (e *metaCacheEntry) isLatestDeletemarker() bool {
 // fileInfo returns the decoded metadata.
 // If entry is a directory it is returned as that.
 // If versioned the latest version will be returned.
-func (e *metaCacheEntry) fileInfo(bucket string) (*FileInfo, error) {
+func (e *metaCacheEntry) fileInfo(bucket string) (FileInfo, error) {
 	if e.isDir() {
-		return &FileInfo{
+		return FileInfo{
 			Volume: bucket,
 			Name:   e.name,
 			Mode:   uint32(os.ModeDir),
 		}, nil
 	}
+	if e.cached != nil {
+		return e.cached.ToFileInfo(bucket, e.name, "")
+	}
+	return getFileInfo(e.metadata, bucket, e.name, "", false)
+}
+
+// xlmeta returns the decoded metadata.
+// This should not be called on directories.
+func (e *metaCacheEntry) xlmeta() (*xlMetaV2, error) {
+	if e.isDir() {
+		return nil, errFileNotFound
+	}
 	if e.cached == nil {
 		if len(e.metadata) == 0 {
 			// only happens if the entry is not found.
 			return nil, errFileNotFound
 		}
-		fi, err := getFileInfo(e.metadata, bucket, e.name, "", false)
+		var xl xlMetaV2
+		err := xl.LoadOrConvert(e.metadata)
 		if err != nil {
 			return nil, err
 		}
-		e.cached = &fi
+		e.cached = &xl
 	}
 	return e.cached, nil
 }
@@ -200,6 +233,7 @@ func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, erro
 			},
 		}, nil
 	}
+	// Too small gains to reuse cache here.
 	return getFileInfoVersions(e.metadata, bucket, e.name)
 }
 
@@ -240,16 +274,15 @@ type metadataResolutionParams struct {
 	dirQuorum int    // Number if disks needed for a directory to 'exist'.
 	objQuorum int    // Number of disks needed for an object to 'exist'.
 	bucket    string // Name of the bucket. Used for generating cached fileinfo.
+	strict    bool   // Versions must match exactly, including all metadata.
 
 	// Reusable slice for resolution
-	candidates []struct {
-		n int
-		e *metaCacheEntry
-	}
+	candidates [][]xlMetaV2ShallowVersion
 }
 
 // resolve multiple entries.
 // entries are resolved by majority, then if tied by mod-time and versions.
+// Names must match on all entries in m.
 func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCacheEntry, ok bool) {
 	if len(m) == 0 {
 		return nil, false
@@ -257,14 +290,14 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 
 	dirExists := 0
 	if cap(r.candidates) < len(m) {
-		r.candidates = make([]struct {
-			n int
-			e *metaCacheEntry
-		}, 0, len(m))
+		r.candidates = make([][]xlMetaV2ShallowVersion, 0, len(m))
 	}
 	r.candidates = r.candidates[:0]
+	objsAgree := 0
+	objsValid := 0
 	for i := range m {
 		entry := &m[i]
+		// Empty entry
 		if entry.name == "" {
 			continue
 		}
@@ -275,72 +308,73 @@ func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCa
 			continue
 		}
 
-		// Get new entry metadata
-		if _, err := entry.fileInfo(r.bucket); err != nil {
-			logger.LogIf(context.Background(), err)
+		// Get new entry metadata,
+		// shallow decode.
+		xl, err := entry.xlmeta()
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
 			continue
 		}
+		objsValid++
 
-		found := false
-		for i, c := range r.candidates {
-			if c.e.matches(entry, r.bucket) {
-				c.n++
-				r.candidates[i] = c
-				found = true
-				break
-			}
-		}
-		if !found {
-			r.candidates = append(r.candidates, struct {
-				n int
-				e *metaCacheEntry
-			}{n: 1, e: entry})
-		}
-	}
-	if selected != nil && selected.isDir() && dirExists > r.dirQuorum {
-		return selected, true
-	}
+		// Add all valid to candidates.
+		r.candidates = append(r.candidates, xl.versions)
 
-	switch len(r.candidates) {
-	case 0:
+		// We select the first object we find as a candidate and see if all match that.
+		// This is to quickly identify if all agree.
 		if selected == nil {
-			return nil, false
+			selected = entry
+			objsAgree = 1
+			continue
 		}
-		if !selected.isDir() || dirExists < r.dirQuorum {
-			return nil, false
+		// Names match, check meta...
+		if prefer, ok := entry.matches(selected, r.strict); ok {
+			selected = prefer
+			objsAgree++
+			continue
 		}
-		return selected, true
-	case 1:
-		cand := r.candidates[0]
-		if cand.n < r.objQuorum {
-			return nil, false
-		}
-		return cand.e, true
-	default:
-		// Sort by matches....
-		sort.Slice(r.candidates, func(i, j int) bool {
-			return r.candidates[i].n > r.candidates[j].n
-		})
-
-		// Check if we have enough.
-		if r.candidates[0].n < r.objQuorum {
-			return nil, false
-		}
-
-		// if r.objQuorum == 1 then it is guaranteed that
-		// this resolver is for HealObjects(), so use resolveEntries()
-		// instead to resolve candidates, this check is only useful
-		// for regular cases of ListObjects()
-		if r.candidates[0].n > r.candidates[1].n && r.objQuorum > 1 {
-			ok := r.candidates[0].e != nil && r.candidates[0].e.name != ""
-			return r.candidates[0].e, ok
-		}
-
-		e := resolveEntries(r.candidates[0].e, r.candidates[1].e, r.bucket)
-		// Tie between two, resolve using modtime+versions.
-		ok := e != nil && e.name != ""
-		return e, ok
 	}
+
+	// Return dir entries, if enough...
+	if selected != nil && selected.isDir() && dirExists >= r.dirQuorum {
+		return selected, true
+	}
+
+	// If we would never be able to reach read quorum.
+	if objsValid < r.objQuorum {
+		return nil, false
+	}
+
+	// If all objects agree.
+	if selected != nil && objsAgree == objsValid {
+		return selected, true
+	}
+
+	// If cached is nil we shall skip the entry.
+	if selected.cached == nil {
+		return nil, false
+	}
+
+	// Merge if we have disagreement.
+	// Create a new merged result.
+	selected = &metaCacheEntry{
+		name:     selected.name,
+		reusable: true,
+		cached:   &xlMetaV2{metaV: selected.cached.metaV},
+	}
+	selected.cached.versions = mergeXLV2Versions(r.objQuorum, r.strict, r.candidates...)
+	if len(selected.cached.versions) == 0 {
+		return nil, false
+	}
+
+	// Reserialize
+	var err error
+	selected.metadata, err = selected.cached.AppendTo(metaDataPoolGet())
+	if err != nil {
+		logger.LogIf(context.Background(), err)
+		return nil, false
+	}
+	return selected, true
 }
 
 // firstFound returns the first found and the number of set entries.

cmd/metacache-entries_test.go

@@ -18,9 +18,12 @@
 package cmd
 
 import (
+	"fmt"
+	"math/rand"
 	"reflect"
 	"sort"
 	"testing"
+	"time"
 )
 
 func Test_metaCacheEntries_sort(t *testing.T) {
@@ -219,3 +222,438 @@ func Test_metaCacheEntry_isInDir(t *testing.T) {
 		})
 	}
 }
+
+func Test_metaCacheEntries_resolve(t *testing.T) {
+	baseTime, err := time.Parse("2006/01/02", "2015/02/25")
+	if err != nil {
+		t.Fatal(err)
+	}
+	var inputs = []xlMetaV2{
+		0: {
+			versions: []xlMetaV2ShallowVersion{
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+					ModTime:   baseTime.Add(30 * time.Minute).UnixNano(),
+					Signature: [4]byte{1, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+			},
+		},
+		// Mismatches Modtime+Signature and older...
+		1: {
+			versions: []xlMetaV2ShallowVersion{
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+					ModTime:   baseTime.Add(15 * time.Minute).UnixNano(),
+					Signature: [4]byte{2, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+			},
+		},
+		// Has another version prior to the one we want.
+		2: {
+			versions: []xlMetaV2ShallowVersion{
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+					ModTime:   baseTime.Add(60 * time.Minute).UnixNano(),
+					Signature: [4]byte{2, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+					ModTime:   baseTime.Add(30 * time.Minute).UnixNano(),
+					Signature: [4]byte{1, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+			},
+		},
+		// Has a completely different version id
+		3: {
+			versions: []xlMetaV2ShallowVersion{
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+					ModTime:   baseTime.Add(60 * time.Minute).UnixNano(),
+					Signature: [4]byte{1, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+			},
+		},
+		4: {
+			versions: []xlMetaV2ShallowVersion{},
+		},
+		// Has a zero version id
+		5: {
+			versions: []xlMetaV2ShallowVersion{
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{},
+					ModTime:   baseTime.Add(60 * time.Minute).UnixNano(),
+					Signature: [4]byte{5, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+			},
+		},
+		// Zero version, modtime newer..
+		6: {
+			versions: []xlMetaV2ShallowVersion{
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{},
+					ModTime:   baseTime.Add(90 * time.Minute).UnixNano(),
+					Signature: [4]byte{6, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+			},
+		},
+		7: {
+			versions: []xlMetaV2ShallowVersion{
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{},
+					ModTime:   baseTime.Add(90 * time.Minute).UnixNano(),
+					Signature: [4]byte{6, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+					ModTime:   baseTime.Add(60 * time.Minute).UnixNano(),
+					Signature: [4]byte{2, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+					ModTime:   baseTime.Add(60 * time.Minute).UnixNano(),
+					Signature: [4]byte{1, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+					ModTime:   baseTime.Add(30 * time.Minute).UnixNano(),
+					Signature: [4]byte{1, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+			},
+		},
+		// Delete marker.
+		8: {
+			versions: []xlMetaV2ShallowVersion{
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7},
+					ModTime:   baseTime.Add(90 * time.Minute).UnixNano(),
+					Signature: [4]byte{6, 1, 1, 1},
+					Type:      DeleteType,
+					Flags:     0,
+				}},
+			},
+		},
+		// Delete marker and version from 1
+		9: {
+			versions: []xlMetaV2ShallowVersion{
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7},
+					ModTime:   baseTime.Add(90 * time.Minute).UnixNano(),
+					Signature: [4]byte{6, 1, 1, 1},
+					Type:      DeleteType,
+					Flags:     0,
+				}},
+				{header: xlMetaV2VersionHeader{
+					VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
+					ModTime:   baseTime.Add(15 * time.Minute).UnixNano(),
+					Signature: [4]byte{2, 1, 1, 1},
+					Type:      ObjectType,
+					Flags:     0,
+				}},
+			},
+		},
+	}
+	inputSerialized := make([]metaCacheEntry, len(inputs))
+	for i, xl := range inputs {
+		xl.sortByModTime()
+		var err error
+		var entry = metaCacheEntry{
+			name: "testobject",
+		}
+		entry.metadata, err = xl.AppendTo(nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		inputSerialized[i] = entry
+	}
+
+	tests := []struct {
+		name         string
+		m            metaCacheEntries
+		r            metadataResolutionParams
+		wantSelected *metaCacheEntry
+		wantOk       bool
+	}{
+		{
+			name:         "consistent",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[0]},
+			r:            metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			name:         "consistent-strict",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[0]},
+			r:            metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			name:         "one zero, below quorum",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}},
+			r:            metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
+			wantSelected: nil,
+			wantOk:       false,
+		},
+		{
+			name:         "one zero, below quorum, strict",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}},
+			r:            metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: true},
+			wantSelected: nil,
+			wantOk:       false,
+		},
+		{
+			name:         "one zero, at quorum",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}},
+			r:            metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			name:         "one zero, at quorum, strict",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}},
+			r:            metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: true},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			name:         "modtime, signature mismatch",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]},
+			r:            metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			name:         "modtime,signature mismatch, strict",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]},
+			r:            metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: true},
+			wantSelected: nil,
+			wantOk:       false,
+		},
+		{
+			name:         "modtime, signature mismatch, at quorum",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]},
+			r:            metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			name:         "modtime,signature mismatch, at quorum, strict",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]},
+			r:            metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: true},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			name:         "additional version",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]},
+			r:            metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			// Since we have the same version in all inputs, that is strictly ok.
+			name:         "additional version, strict",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]},
+			r:            metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: true},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			// Since we have the same version in all inputs, that is strictly ok.
+			name: "additional version, quorum one",
+			m:    metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]},
+			r:    metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: true},
+			// We get the both versions, since we only request quorum 1
+			wantSelected: &inputSerialized[2],
+			wantOk:       true,
+		},
+		{
+			name:         "additional version, quorum two",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]},
+			r:            metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			name:         "2 additional versions, quorum two",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[2], inputSerialized[2]},
+			r:            metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true},
+			wantSelected: &inputSerialized[2],
+			wantOk:       true,
+		},
+		{
+			// inputSerialized[1] have older versions of the second in inputSerialized[2]
+			name:         "modtimemismatch",
+			m:            metaCacheEntries{inputSerialized[1], inputSerialized[1], inputSerialized[2], inputSerialized[2]},
+			r:            metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false},
+			wantSelected: &inputSerialized[2],
+			wantOk:       true,
+		},
+		{
+			// inputSerialized[1] have older versions of the second in inputSerialized[2]
+			name:         "modtimemismatch,strict",
+			m:            metaCacheEntries{inputSerialized[1], inputSerialized[1], inputSerialized[2], inputSerialized[2]},
+			r:            metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true},
+			wantSelected: &inputSerialized[2],
+			wantOk:       true,
+		},
+		{
+			// inputSerialized[1] have older versions of the second in inputSerialized[2], but
+			// since it is not strict, we should get it that one (with latest modtime)
+			name:         "modtimemismatch,not strict",
+			m:            metaCacheEntries{inputSerialized[1], inputSerialized[1], inputSerialized[2], inputSerialized[2]},
+			r:            metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			name:         "one-q1",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]},
+			r:            metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: false},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			name:         "one-q1-strict",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]},
+			r:            metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: true},
+			wantSelected: &inputSerialized[0],
+			wantOk:       true,
+		},
+		{
+			name:         "one-q2",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]},
+			r:            metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false},
+			wantSelected: nil,
+			wantOk:       false,
+		},
+		{
+			name:         "one-q2-strict",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]},
+			r:            metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true},
+			wantSelected: nil,
+			wantOk:       false,
+		},
+		{
+			name:         "two-diff-q2",
+			m:            metaCacheEntries{inputSerialized[0], inputSerialized[3], inputSerialized[4], inputSerialized[4]},
+			r:            metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false},
+			wantSelected: nil,
+			wantOk:       false,
+		},
+		{
+			name:         "zeroid",
+			m:            metaCacheEntries{inputSerialized[5], inputSerialized[5], inputSerialized[6], inputSerialized[6]},
+			r:            metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false},
+			wantSelected: &inputSerialized[6],
+			wantOk:       true,
+		},
+		{
+			// When ID is zero, do not allow non-strict matches to reach quorum.
+			name:         "zeroid-belowq",
+			m:            metaCacheEntries{inputSerialized[5], inputSerialized[5], inputSerialized[6], inputSerialized[6]},
+			r:            metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
+			wantSelected: nil,
+			wantOk:       false,
+		},
+		{
+			name:         "merge4",
+			m:            metaCacheEntries{inputSerialized[2], inputSerialized[3], inputSerialized[5], inputSerialized[6]},
+			r:            metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: false},
+			wantSelected: &inputSerialized[7],
+			wantOk:       true,
+		},
+		{
+			name:         "deletemarker",
+			m:            metaCacheEntries{inputSerialized[8], inputSerialized[4], inputSerialized[4], inputSerialized[4]},
+			r:            metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: false},
+			wantSelected: &inputSerialized[8],
+			wantOk:       true,
+		},
+		{
+			name:         "deletemarker-nonq",
+			m:            metaCacheEntries{inputSerialized[8], inputSerialized[8], inputSerialized[4], inputSerialized[4]},
+			r:            metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
+			wantSelected: nil,
+			wantOk:       false,
+		},
+		{
+			name:         "deletemarker-nonq",
+			m:            metaCacheEntries{inputSerialized[8], inputSerialized[8], inputSerialized[8], inputSerialized[1]},
+			r:            metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
+			wantSelected: &inputSerialized[8],
+			wantOk:       true,
+		},
+		{
+			name:         "deletemarker-mixed",
+			m:            metaCacheEntries{inputSerialized[8], inputSerialized[8], inputSerialized[1], inputSerialized[1]},
+			r:            metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false},
+			wantSelected: &inputSerialized[9],
+			wantOk:       true,
+		},
+		{
+			name:         "deletemarker-q3",
+			m:            metaCacheEntries{inputSerialized[8], inputSerialized[9], inputSerialized[9], inputSerialized[1]},
+			r:            metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false},
+			wantSelected: &inputSerialized[9],
+			wantOk:       true,
+		},
+		{
+			name:         "deletemarker-q3-strict",
+			m:            metaCacheEntries{inputSerialized[8], inputSerialized[9], inputSerialized[9], inputSerialized[1]},
+			r:            metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: true},
+			wantSelected: &inputSerialized[9],
+			wantOk:       true,
+		},
+	}
+
+	for testID, tt := range tests {
+		rng := rand.New(rand.NewSource(0))
+		// Run for a number of times, shuffling the input to ensure that output is consistent.
+		for i := 0; i < 10; i++ {
+			t.Run(fmt.Sprintf("test-%d-%s-run-%d", testID, tt.name, i), func(t *testing.T) {
+				if i > 0 {
+					rng.Shuffle(len(tt.m), func(i, j int) {
+						tt.m[i], tt.m[j] = tt.m[j], tt.m[i]
+					})
+				}
+				gotSelected, gotOk := tt.m.resolve(&tt.r)
+				if gotOk != tt.wantOk {
+					t.Errorf("resolve() gotOk = %v, want %v", gotOk, tt.wantOk)
+				}
+				if gotSelected != nil {
+					gotSelected.cached = nil
+					gotSelected.reusable = false
+				}
+				if !reflect.DeepEqual(gotSelected, tt.wantSelected) {
+					wantM, _ := tt.wantSelected.xlmeta()
+					gotM, _ := gotSelected.xlmeta()
+					t.Errorf("resolve() gotSelected = \n%#v, want \n%#v", *gotM, *wantM)
+				}
+			})
+		}
+	}
+}

cmd/metacache-server-pool.go

@@ -199,13 +199,15 @@ func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (
 		entries.truncate(0)
 		go func() {
 			rpc := globalNotificationSys.restClientFromHash(o.Bucket)
-			ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
-			defer cancel()
-			c, err := rpc.GetMetacacheListing(ctx, *o)
-			if err == nil {
-				c.error = "no longer used"
-				c.status = scanStateError
-				rpc.UpdateMetacacheListing(ctx, *c)
+			if rpc != nil {
+				ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second)
+				defer cancel()
+				c, err := rpc.GetMetacacheListing(ctx, *o)
+				if err == nil {
+					c.error = "no longer used"
+					c.status = scanStateError
+					rpc.UpdateMetacacheListing(ctx, *c)
+				}
 			}
 		}()
 		o.ID = ""

cmd/metacache-set.go

@@ -91,6 +91,9 @@ type listPathOptions struct {
 	// A transient result will never be returned from the cache so knowing the list id is required.
 	Transient bool
 
+	// Versioned is this a ListObjectVersions call.
+	Versioned bool
+
 	// pool and set of where the cache is located.
 	pool, set int
 }
@@ -156,7 +159,7 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
 				resCh = nil
 				continue
 			}
-			if !o.IncludeDirectories && entry.isDir() {
+			if !o.IncludeDirectories && (entry.isDir() || (!o.Versioned && entry.isObjectDir() && entry.isLatestDeletemarker())) {
 				continue
 			}
 			if o.Marker != "" && entry.name < o.Marker {
@@ -168,7 +171,7 @@ func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCache
 			if !o.Recursive && !entry.isInDir(o.Prefix, o.Separator) {
 				continue
 			}
-			if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() {
+			if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() && !entry.isObjectDir() {
 				continue
 			}
 			if o.Limit > 0 && results.len() >= o.Limit {
@@ -324,13 +327,13 @@ func (r *metacacheReader) filter(o listPathOptions) (entries metaCacheEntriesSor
 				pastPrefix = true
 				return false
 			}
-			if !o.IncludeDirectories && entry.isDir() {
+			if !o.IncludeDirectories && (entry.isDir() || (!o.Versioned && entry.isObjectDir() && entry.isLatestDeletemarker())) {
 				return true
 			}
 			if !entry.isInDir(o.Prefix, o.Separator) {
 				return true
 			}
-			if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() {
+			if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() && !entry.isObjectDir() {
 				return entries.len() < o.Limit
 			}
 			entries.o = append(entries.o, entry)
@@ -343,7 +346,7 @@ func (r *metacacheReader) filter(o listPathOptions) (entries metaCacheEntriesSor
 	}
 
 	// We should not need to filter more.
-	return r.readN(o.Limit, o.InclDeleted, o.IncludeDirectories, o.Prefix)
+	return r.readN(o.Limit, o.InclDeleted, o.IncludeDirectories, o.Versioned, o.Prefix)
 }
 
 func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) {
@@ -541,7 +544,7 @@ func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, resul
 	// Special case: ask all disks if the drive count is 4
 	if askDisks == -1 || er.setDriveCount == 4 {
 		askDisks = len(disks) // with 'strict' quorum list on all online disks.
-		listingQuorum = getReadQuorum(er.setDriveCount)
+		listingQuorum = er.defaultRQuorum()
 	}
 	if askDisks == 0 {
 		askDisks = globalAPIConfig.getListQuorum()
@@ -764,7 +767,7 @@ type listPathRawOptions struct {
 	// agreed is called if all disks agreed.
 	agreed func(entry metaCacheEntry)
 
-	// partial will be returned when there is disagreement between disks.
+	// partial will be called when there is disagreement between disks.
 	// if disk did not return any result, but also haven't errored
 	// the entry will be empty and errs will
 	partial func(entries metaCacheEntries, nAgreed int, errs []error)
@@ -905,7 +908,7 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 				continue
 			}
 			// If exact match, we agree.
-			if current.matches(&entry, opts.bucket) {
+			if _, ok := current.matches(&entry, true); ok {
 				topEntries[i] = entry
 				agree++
 				continue

cmd/metacache-stream.go

@@ -457,7 +457,7 @@ func (r *metacacheReader) forwardTo(s string) error {
 // Will return io.EOF if end of stream is reached.
 // If requesting 0 objects nil error will always be returned regardless of at end of stream.
 // Use peek to determine if at end of stream.
-func (r *metacacheReader) readN(n int, inclDeleted, inclDirs bool, prefix string) (metaCacheEntriesSorted, error) {
+func (r *metacacheReader) readN(n int, inclDeleted, inclDirs, inclVersions bool, prefix string) (metaCacheEntriesSorted, error) {
 	r.checkInit()
 	if n == 0 {
 		return metaCacheEntriesSorted{}, nil
@@ -528,10 +528,10 @@ func (r *metacacheReader) readN(n int, inclDeleted, inclDirs bool, prefix string
 			metaDataPoolPut(meta.metadata)
 			meta.metadata = nil
 		}
-		if !inclDirs && meta.isDir() {
+		if !inclDirs && (meta.isDir() || (!inclVersions && meta.isObjectDir() && meta.isLatestDeletemarker())) {
 			continue
 		}
-		if !inclDeleted && meta.isLatestDeletemarker() {
+		if !inclDeleted && meta.isLatestDeletemarker() && meta.isObject() && !meta.isObjectDir() {
 			continue
 		}
 		res = append(res, meta)

cmd/metacache-stream_test.go

@@ -40,7 +40,7 @@ func loadMetacacheSample(t testing.TB) *metacacheReader {
 func loadMetacacheSampleEntries(t testing.TB) metaCacheEntriesSorted {
 	r := loadMetacacheSample(t)
 	defer r.Close()
-	entries, err := r.readN(-1, false, true, "")
+	entries, err := r.readN(-1, false, true, false, "")
 	if err != io.EOF {
 		t.Fatal(err)
 	}
@@ -64,7 +64,7 @@ func Test_metacacheReader_readNames(t *testing.T) {
 func Test_metacacheReader_readN(t *testing.T) {
 	r := loadMetacacheSample(t)
 	defer r.Close()
-	entries, err := r.readN(-1, false, true, "")
+	entries, err := r.readN(-1, false, true, false, "")
 	if err != io.EOF {
 		t.Fatal(err, entries.len())
 	}
@@ -79,7 +79,7 @@ func Test_metacacheReader_readN(t *testing.T) {
 	}
 
 	want = want[:0]
-	entries, err = r.readN(0, false, true, "")
+	entries, err = r.readN(0, false, true, false, "")
 	if err != nil {
 		t.Fatal(err, entries.len())
 	}
@@ -90,7 +90,7 @@ func Test_metacacheReader_readN(t *testing.T) {
 	// Reload.
 	r = loadMetacacheSample(t)
 	defer r.Close()
-	entries, err = r.readN(0, false, true, "")
+	entries, err = r.readN(0, false, true, false, "")
 	if err != nil {
 		t.Fatal(err, entries.len())
 	}
@@ -98,7 +98,7 @@ func Test_metacacheReader_readN(t *testing.T) {
 		t.Fatal("unexpected length:", entries.len(), "want:", len(want))
 	}
 
-	entries, err = r.readN(5, false, true, "")
+	entries, err = r.readN(5, false, true, false, "")
 	if err != nil {
 		t.Fatal(err, entries.len())
 	}
@@ -117,7 +117,7 @@ func Test_metacacheReader_readN(t *testing.T) {
 func Test_metacacheReader_readNDirs(t *testing.T) {
 	r := loadMetacacheSample(t)
 	defer r.Close()
-	entries, err := r.readN(-1, false, true, "")
+	entries, err := r.readN(-1, false, true, false, "")
 	if err != io.EOF {
 		t.Fatal(err, entries.len())
 	}
@@ -138,7 +138,7 @@ func Test_metacacheReader_readNDirs(t *testing.T) {
 	want = noDirs
 	r = loadMetacacheSample(t)
 	defer r.Close()
-	entries, err = r.readN(-1, false, false, "")
+	entries, err = r.readN(-1, false, false, false, "")
 	if err != io.EOF {
 		t.Fatal(err, entries.len())
 	}
@@ -152,7 +152,7 @@ func Test_metacacheReader_readNDirs(t *testing.T) {
 	}
 
 	want = want[:0]
-	entries, err = r.readN(0, false, false, "")
+	entries, err = r.readN(0, false, false, false, "")
 	if err != nil {
 		t.Fatal(err, entries.len())
 	}
@@ -163,7 +163,7 @@ func Test_metacacheReader_readNDirs(t *testing.T) {
 	// Reload.
 	r = loadMetacacheSample(t)
 	defer r.Close()
-	entries, err = r.readN(0, false, false, "")
+	entries, err = r.readN(0, false, false, false, "")
 	if err != nil {
 		t.Fatal(err, entries.len())
 	}
@@ -171,7 +171,7 @@ func Test_metacacheReader_readNDirs(t *testing.T) {
 		t.Fatal("unexpected length:", entries.len(), "want:", len(want))
 	}
 
-	entries, err = r.readN(5, false, false, "")
+	entries, err = r.readN(5, false, false, false, "")
 	if err != nil {
 		t.Fatal(err, entries.len())
 	}
@@ -190,7 +190,7 @@ func Test_metacacheReader_readNDirs(t *testing.T) {
 func Test_metacacheReader_readNPrefix(t *testing.T) {
 	r := loadMetacacheSample(t)
 	defer r.Close()
-	entries, err := r.readN(-1, false, true, "src/compress/bzip2/")
+	entries, err := r.readN(-1, false, true, false, "src/compress/bzip2/")
 	if err != io.EOF {
 		t.Fatal(err, entries.len())
 	}
@@ -206,7 +206,7 @@ func Test_metacacheReader_readNPrefix(t *testing.T) {
 
 	r = loadMetacacheSample(t)
 	defer r.Close()
-	entries, err = r.readN(-1, false, true, "src/nonexist")
+	entries, err = r.readN(-1, false, true, false, "src/nonexist")
 	if err != io.EOF {
 		t.Fatal(err, entries.len())
 	}
@@ -222,7 +222,7 @@ func Test_metacacheReader_readNPrefix(t *testing.T) {
 
 	r = loadMetacacheSample(t)
 	defer r.Close()
-	entries, err = r.readN(-1, false, true, "src/a")
+	entries, err = r.readN(-1, false, true, false, "src/a")
 	if err != io.EOF {
 		t.Fatal(err, entries.len())
 	}
@@ -238,7 +238,7 @@ func Test_metacacheReader_readNPrefix(t *testing.T) {
 
 	r = loadMetacacheSample(t)
 	defer r.Close()
-	entries, err = r.readN(-1, false, true, "src/compress/zlib/e")
+	entries, err = r.readN(-1, false, true, false, "src/compress/zlib/e")
 	if err != io.EOF {
 		t.Fatal(err, entries.len())
 	}

cmd/metacache-walk.go

@@ -28,7 +28,6 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/gorilla/mux"
 	xhttp "github.com/minio/minio/internal/http"
 	xioutil "github.com/minio/minio/internal/ioutil"
 	"github.com/minio/minio/internal/logger"
@@ -342,10 +341,9 @@ func (s *storageRESTServer) WalkDirHandler(w http.ResponseWriter, r *http.Reques
 	if !s.IsValid(w, r) {
 		return
 	}
-	vars := mux.Vars(r)
-	volume := vars[storageRESTVolume]
-	dirPath := vars[storageRESTDirPath]
-	recursive, err := strconv.ParseBool(vars[storageRESTRecursive])
+	volume := r.Form.Get(storageRESTVolume)
+	dirPath := r.Form.Get(storageRESTDirPath)
+	recursive, err := strconv.ParseBool(r.Form.Get(storageRESTRecursive))
 	if err != nil {
 		s.writeErrorResponse(w, err)
 		return

cmd/notification.go

@@ -63,7 +63,7 @@ func (sys *NotificationSys) GetARNList(onlyActive bool) []string {
 	if sys == nil {
 		return arns
 	}
-	region := globalServerRegion
+	region := globalSite.Region
 	for targetID, target := range sys.targetList.TargetMap() {
 		// httpclient target is part of ListenNotification
 		// which doesn't need to be listed as part of the ARN list
@@ -643,8 +643,8 @@ func (sys *NotificationSys) set(bucket BucketInfo, meta BucketMetadata) {
 	if config == nil {
 		return
 	}
-	config.SetRegion(globalServerRegion)
-	if err := config.Validate(globalServerRegion, globalNotificationSys.targetList); err != nil {
+	config.SetRegion(globalSite.Region)
+	if err := config.Validate(globalSite.Region, globalNotificationSys.targetList); err != nil {
 		if _, ok := err.(*event.ErrARNNotFound); !ok {
 			logger.LogIf(GlobalContext, err)
 		}
@@ -1534,7 +1534,8 @@ func (sys *NotificationSys) ServiceFreeze(ctx context.Context, freeze bool) []No
 
 // Speedtest run GET/PUT tests at input concurrency for requested object size,
 // optionally you can extend the tests longer with time.Duration.
-func (sys *NotificationSys) Speedtest(ctx context.Context, size int, concurrent int, duration time.Duration) []SpeedtestResult {
+func (sys *NotificationSys) Speedtest(ctx context.Context, size int,
+	concurrent int, duration time.Duration, storageClass string) []SpeedtestResult {
 	length := len(sys.allPeerClients)
 	if length == 0 {
 		// For single node erasure setup.
@@ -1555,7 +1556,8 @@ func (sys *NotificationSys) Speedtest(ctx context.Context, size int, concurrent
 		wg.Add(1)
 		go func(index int) {
 			defer wg.Done()
-			r, err := sys.peerClients[index].Speedtest(ctx, size, concurrent, duration)
+			r, err := sys.peerClients[index].Speedtest(ctx, size,
+				concurrent, duration, storageClass)
 			u := &url.URL{
 				Scheme: scheme,
 				Host:   sys.peerClients[index].host.String(),
@@ -1572,7 +1574,7 @@ func (sys *NotificationSys) Speedtest(ctx context.Context, size int, concurrent
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		r, err := selfSpeedtest(ctx, size, concurrent, duration)
+		r, err := selfSpeedtest(ctx, size, concurrent, duration, storageClass)
 		u := &url.URL{
 			Scheme: scheme,
 			Host:   globalLocalNodeName,

cmd/object-api-common.go

@@ -247,6 +247,21 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 		return loi, nil
 	}
 
+	if len(prefix) > 0 && maxKeys == 1 && delimiter == "" && marker == "" {
+		// Optimization for certain applications like
+		// - Cohesity
+		// - Actifio, Splunk etc.
+		// which send ListObjects requests where the actual object
+		// itself is the prefix and max-keys=1 in such scenarios
+		// we can simply verify locally if such an object exists
+		// to avoid the need for ListObjects().
+		objInfo, err := obj.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
+		if err == nil {
+			loi.Objects = append(loi.Objects, objInfo)
+			return loi, nil
+		}
+	}
+
 	// For delimiter and prefix as '/' we do not list anything at all
 	// since according to s3 spec we stop at the 'delimiter'
 	// along // with the prefix. On a flat namespace with 'prefix'
@@ -290,6 +305,13 @@ func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, d
 		i := i
 		walkResult, ok := <-walkResultCh
 		if !ok {
+			if HasSuffix(prefix, SlashSeparator) {
+				objInfo, err := obj.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
+				if err == nil {
+					loi.Objects = append(loi.Objects, objInfo)
+					return loi, nil
+				}
+			}
 			// Closed channel.
 			eof = true
 			break

cmd/object-api-listobjects_test.go

@@ -30,13 +30,296 @@ import (
 	"testing"
 )
 
-// Wrapper for calling ListObjects tests for both Erasure multiple disks and single node setup.
+func TestListObjectsVersionedFolders(t *testing.T) {
+	ExecObjectLayerTest(t, testListObjectsVersionedFolders)
+}
+
+func testListObjectsVersionedFolders(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
+	if instanceType == FSTestStr {
+		return
+	}
+	t, _ := t1.(*testing.T)
+	testBuckets := []string{
+		// This bucket is used for testing ListObject operations.
+		"test-bucket-folders",
+		// This bucket has file delete marker.
+		"test-bucket-files",
+	}
+	for _, bucket := range testBuckets {
+		err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{
+			VersioningEnabled: true,
+		})
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err.Error())
+		}
+	}
+
+	var err error
+	testObjects := []struct {
+		parentBucket    string
+		name            string
+		content         string
+		meta            map[string]string
+		addDeleteMarker bool
+	}{
+		{testBuckets[0], "unique/folder/", "", nil, true},
+		{testBuckets[0], "unique/folder/1.txt", "content", nil, false},
+		{testBuckets[1], "unique/folder/1.txt", "content", nil, true},
+	}
+	for _, object := range testObjects {
+		md5Bytes := md5.Sum([]byte(object.content))
+		_, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content),
+			int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{
+			Versioned:   globalBucketVersioningSys.Enabled(object.parentBucket),
+			UserDefined: object.meta,
+		})
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err.Error())
+		}
+		if object.addDeleteMarker {
+			oi, err := obj.DeleteObject(context.Background(), object.parentBucket, object.name, ObjectOptions{
+				Versioned: globalBucketVersioningSys.Enabled(object.parentBucket),
+			})
+			if err != nil {
+				t.Fatalf("%s : %s", instanceType, err.Error())
+			}
+			if oi.DeleteMarker != object.addDeleteMarker {
+				t.Fatalf("Expected, marker %t : got %t", object.addDeleteMarker, oi.DeleteMarker)
+			}
+		}
+	}
+
+	// Formulating the result data set to be expected from ListObjects call inside the tests,
+	// This will be used in testCases and used for asserting the correctness of ListObjects output in the tests.
+
+	resultCases := []ListObjectsInfo{
+		{
+			IsTruncated: false,
+			Prefixes:    []string{"unique/folder/"},
+		},
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{Name: "unique/folder/1.txt"},
+			},
+		},
+		{
+			IsTruncated: false,
+			Objects:     []ObjectInfo{},
+		},
+	}
+
+	resultCasesV := []ListObjectVersionsInfo{
+		{
+			IsTruncated: false,
+			Prefixes:    []string{"unique/folder/"},
+		},
+		{
+			IsTruncated: false,
+			Objects: []ObjectInfo{
+				{
+					Name:         "unique/folder/",
+					DeleteMarker: true,
+				},
+				{
+					Name:         "unique/folder/",
+					DeleteMarker: false,
+				},
+				{
+					Name:         "unique/folder/1.txt",
+					DeleteMarker: false,
+				},
+			},
+		},
+	}
+
+	testCases := []struct {
+		// Inputs to ListObjects.
+		bucketName string
+		prefix     string
+		marker     string
+		delimiter  string
+		maxKeys    int
+		versioned  bool
+		// Expected output of ListObjects.
+		resultL ListObjectsInfo
+		resultV ListObjectVersionsInfo
+		err     error
+		// Flag indicating whether the test is expected to pass or not.
+		shouldPass bool
+	}{
+		{testBuckets[0], "unique/", "", "/", 1000, false, resultCases[0], ListObjectVersionsInfo{}, nil, true},
+		{testBuckets[0], "unique/folder", "", "/", 1000, false, resultCases[0], ListObjectVersionsInfo{}, nil, true},
+		{testBuckets[0], "unique/", "", "", 1000, false, resultCases[1], ListObjectVersionsInfo{}, nil, true},
+		{testBuckets[1], "unique/", "", "/", 1000, false, resultCases[0], ListObjectVersionsInfo{}, nil, true},
+		{testBuckets[1], "unique/folder/", "", "/", 1000, false, resultCases[2], ListObjectVersionsInfo{}, nil, true},
+		{testBuckets[0], "unique/", "", "/", 1000, true, ListObjectsInfo{}, resultCasesV[0], nil, true},
+		{testBuckets[0], "unique/", "", "", 1000, true, ListObjectsInfo{}, resultCasesV[1], nil, true},
+	}
+
+	for i, testCase := range testCases {
+		testCase := testCase
+		t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) {
+			t.Log("ListObjects, bucket:", testCase.bucketName, "prefix:",
+				testCase.prefix, "marker:", testCase.marker, "delimiter:",
+				testCase.delimiter, "maxkeys:", testCase.maxKeys)
+			var err error
+			var resultL ListObjectsInfo
+			var resultV ListObjectVersionsInfo
+			if testCase.versioned {
+				resultV, err = obj.ListObjectVersions(context.Background(), testCase.bucketName,
+					testCase.prefix, testCase.marker, "", testCase.delimiter, testCase.maxKeys)
+			} else {
+				resultL, err = obj.ListObjects(context.Background(), testCase.bucketName,
+					testCase.prefix, testCase.marker, testCase.delimiter, testCase.maxKeys)
+			}
+			if err != nil && testCase.shouldPass {
+				t.Errorf("Test %d: %s:  Expected to pass, but failed with: <ERROR> %s", i+1, instanceType, err.Error())
+			}
+			if err == nil && !testCase.shouldPass {
+				t.Errorf("Test %d: %s: Expected to fail with <ERROR> \"%s\", but passed instead", i+1, instanceType, testCase.err.Error())
+			}
+			// Failed as expected, but does it fail for the expected reason.
+			if err != nil && !testCase.shouldPass {
+				if !strings.Contains(err.Error(), testCase.err.Error()) {
+					t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error())
+				}
+			}
+			// Since there are cases for which ListObjects fails, this is
+			// necessary. Test passes as expected, but the output values
+			// are verified for correctness here.
+			if err == nil && testCase.shouldPass {
+				// The length of the expected ListObjectsResult.Objects
+				// should match in both expected result from test cases
+				// and in the output. On failure calling t.Fatalf,
+				// otherwise it may lead to index out of range error in
+				// assertion following this.
+				if !testCase.versioned {
+					if len(testCase.resultL.Objects) != len(resultL.Objects) {
+						t.Logf("want: %v", objInfoNames(testCase.resultL.Objects))
+						t.Logf("got: %v", objInfoNames(resultL.Objects))
+						t.Errorf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.resultL.Objects), len(resultL.Objects))
+					}
+					for j := 0; j < len(testCase.resultL.Objects); j++ {
+						if j >= len(resultL.Objects) {
+							t.Errorf("Test %d: %s: Expected object name to be \"%s\", but not nothing instead", i+1, instanceType, testCase.resultL.Objects[j].Name)
+							continue
+						}
+						if testCase.resultL.Objects[j].Name != resultL.Objects[j].Name {
+							t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultL.Objects[j].Name, resultL.Objects[j].Name)
+						}
+					}
+
+					if len(testCase.resultL.Prefixes) != len(resultL.Prefixes) {
+						t.Logf("want: %v", testCase.resultL.Prefixes)
+						t.Logf("got: %v", resultL.Prefixes)
+						t.Errorf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", i+1, instanceType, len(testCase.resultL.Prefixes), len(resultL.Prefixes))
+					}
+					for j := 0; j < len(testCase.resultL.Prefixes); j++ {
+						if j >= len(resultL.Prefixes) {
+							t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found no result", i+1, instanceType, testCase.resultL.Prefixes[j])
+							continue
+						}
+						if testCase.resultL.Prefixes[j] != resultL.Prefixes[j] {
+							t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultL.Prefixes[j], resultL.Prefixes[j])
+						}
+					}
+
+					if testCase.resultL.IsTruncated != resultL.IsTruncated {
+						// Allow an extra continuation token.
+						if !resultL.IsTruncated || len(resultL.Objects) == 0 {
+							t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.resultL.IsTruncated, resultL.IsTruncated)
+						}
+					}
+
+					if testCase.resultL.IsTruncated && resultL.NextMarker == "" {
+						t.Errorf("Test %d: %s: Expected NextMarker to contain a string since listing is truncated, but instead found it to be empty", i+1, instanceType)
+					}
+
+					if !testCase.resultL.IsTruncated && resultL.NextMarker != "" {
+						if !resultL.IsTruncated || len(resultL.Objects) == 0 {
+							t.Errorf("Test %d: %s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, resultL.NextMarker)
+						}
+					}
+				} else {
+					if len(testCase.resultV.Objects) != len(resultV.Objects) {
+						t.Logf("want: %v", objInfoNames(testCase.resultV.Objects))
+						t.Logf("got: %v", objInfoNames(resultV.Objects))
+						t.Errorf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.resultV.Objects), len(resultV.Objects))
+					}
+					for j := 0; j < len(testCase.resultV.Objects); j++ {
+						if j >= len(resultV.Objects) {
+							t.Errorf("Test %d: %s: Expected object name to be \"%s\", but not nothing instead", i+1, instanceType, testCase.resultV.Objects[j].Name)
+							continue
+						}
+						if testCase.resultV.Objects[j].Name != resultV.Objects[j].Name {
+							t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultV.Objects[j].Name, resultV.Objects[j].Name)
+						}
+					}
+
+					if len(testCase.resultV.Prefixes) != len(resultV.Prefixes) {
+						t.Logf("want: %v", testCase.resultV.Prefixes)
+						t.Logf("got: %v", resultV.Prefixes)
+						t.Errorf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", i+1, instanceType, len(testCase.resultV.Prefixes), len(resultV.Prefixes))
+					}
+					for j := 0; j < len(testCase.resultV.Prefixes); j++ {
+						if j >= len(resultV.Prefixes) {
+							t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found no result", i+1, instanceType, testCase.resultV.Prefixes[j])
+							continue
+						}
+						if testCase.resultV.Prefixes[j] != resultV.Prefixes[j] {
+							t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultV.Prefixes[j], resultV.Prefixes[j])
+						}
+					}
+
+					if testCase.resultV.IsTruncated != resultV.IsTruncated {
+						// Allow an extra continuation token.
+						if !resultV.IsTruncated || len(resultV.Objects) == 0 {
+							t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.resultV.IsTruncated, resultV.IsTruncated)
+						}
+					}
+
+					if testCase.resultV.IsTruncated && resultV.NextMarker == "" {
+						t.Errorf("Test %d: %s: Expected NextMarker to contain a string since listing is truncated, but instead found it to be empty", i+1, instanceType)
+					}
+
+					if !testCase.resultV.IsTruncated && resultV.NextMarker != "" {
+						if !resultV.IsTruncated || len(resultV.Objects) == 0 {
+							t.Errorf("Test %d: %s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, resultV.NextMarker)
+						}
+					}
+				}
+			}
+		})
+	}
+}
+
+// Wrapper for calling ListObjectsOnVersionedBuckets tests for both
+// Erasure multiple disks and single node setup.
+func TestListObjectsOnVersionedBuckets(t *testing.T) {
+	ExecObjectLayerTest(t, testListObjectsOnVersionedBuckets)
+}
+
+// Wrapper for calling ListObjects tests for both Erasure multiple
+// disks and single node setup.
 func TestListObjects(t *testing.T) {
 	ExecObjectLayerTest(t, testListObjects)
 }
 
-// Unit test for ListObjects in general.
+// Unit test for ListObjects on VersionedBucket.
+func testListObjectsOnVersionedBuckets(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
+	_testListObjects(obj, instanceType, t1, true)
+}
+
+// Unit test for ListObjects.
 func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
+	_testListObjects(obj, instanceType, t1, false)
+}
+
+func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, versioned bool) {
+	if instanceType == FSTestStr && versioned {
+		return
+	}
 	t, _ := t1.(*testing.T)
 	testBuckets := []string{
 		// This bucket is used for testing ListObject operations.
@@ -54,7 +337,9 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 		"test-bucket-max-keys-prefixes",
 	}
 	for _, bucket := range testBuckets {
-		err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{})
+		err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{
+			VersioningEnabled: versioned,
+		})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
@@ -92,15 +377,19 @@ func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
 	}
 	for _, object := range testObjects {
 		md5Bytes := md5.Sum([]byte(object.content))
-		_, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content),
-			int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{UserDefined: object.meta})
+		_, err = obj.PutObject(context.Background(), object.parentBucket, object.name,
+			mustGetPutObjReader(t, bytes.NewBufferString(object.content),
+				int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{
+				Versioned:   globalBucketVersioningSys.Enabled(object.parentBucket),
+				UserDefined: object.meta,
+			})
 		if err != nil {
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
 
 	}
 
-	// Formualting the result data set to be expected from ListObjects call inside the tests,
+	// Formulating the result data set to be expected from ListObjects call inside the tests,
 	// This will be used in testCases and used for asserting the correctness of ListObjects output in the tests.
 
 	resultCases := []ListObjectsInfo{
@@ -726,6 +1015,85 @@ func objInfoNames(o []ObjectInfo) []string {
 	return res
 }
 
+func TestDeleteObjectVersionMarker(t *testing.T) {
+	ExecObjectLayerTest(t, testDeleteObjectVersion)
+}
+
+func testDeleteObjectVersion(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
+	if instanceType == FSTestStr {
+		return
+	}
+
+	t, _ := t1.(*testing.T)
+
+	testBuckets := []string{
+		"bucket-suspended-version",
+		"bucket-suspended-version-id",
+	}
+	for _, bucket := range testBuckets {
+		err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{
+			VersioningEnabled: true,
+		})
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err)
+		}
+		meta, err := loadBucketMetadata(context.Background(), obj, bucket)
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err)
+		}
+		meta.VersioningConfigXML = []byte(`<VersioningConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Status>Suspended</Status></VersioningConfiguration>`)
+		if err := meta.Save(context.Background(), obj); err != nil {
+			t.Fatalf("%s : %s", instanceType, err)
+		}
+		globalBucketMetadataSys.Set(bucket, meta)
+		globalNotificationSys.LoadBucketMetadata(context.Background(), bucket)
+	}
+
+	testObjects := []struct {
+		parentBucket    string
+		name            string
+		content         string
+		meta            map[string]string
+		versionID       string
+		expectDelMarker bool
+	}{
+		{testBuckets[0], "delete-file", "contentstring", nil, "", true},
+		{testBuckets[1], "delete-file", "contentstring", nil, "null", false},
+	}
+	for _, object := range testObjects {
+		md5Bytes := md5.Sum([]byte(object.content))
+		_, err := obj.PutObject(context.Background(), object.parentBucket, object.name,
+			mustGetPutObjReader(t, bytes.NewBufferString(object.content),
+				int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{
+				Versioned:        globalBucketVersioningSys.Enabled(object.parentBucket),
+				VersionSuspended: globalBucketVersioningSys.Suspended(object.parentBucket),
+				UserDefined:      object.meta,
+			})
+		if err != nil {
+			t.Fatalf("%s : %s", instanceType, err)
+		}
+		obj, err := obj.DeleteObject(context.Background(), object.parentBucket, object.name, ObjectOptions{
+			Versioned:        globalBucketVersioningSys.Enabled(object.parentBucket),
+			VersionSuspended: globalBucketVersioningSys.Suspended(object.parentBucket),
+			VersionID:        object.versionID,
+		})
+		if err != nil {
+			if object.versionID != "" {
+				if !isErrVersionNotFound(err) {
+					t.Fatalf("%s : %s", instanceType, err)
+				}
+			} else {
+				if !isErrObjectNotFound(err) {
+					t.Fatalf("%s : %s", instanceType, err)
+				}
+			}
+		}
+		if obj.DeleteMarker != object.expectDelMarker {
+			t.Fatalf("%s : expected deleted marker %t, found %t", instanceType, object.expectDelMarker, obj.DeleteMarker)
+		}
+	}
+}
+
 // Wrapper for calling ListObjectVersions tests for both Erasure multiple disks and single node setup.
 func TestListObjectVersions(t *testing.T) {
 	ExecObjectLayerTest(t, testListObjectVersions)
@@ -733,6 +1101,10 @@ func TestListObjectVersions(t *testing.T) {
 
 // Unit test for ListObjectVersions
 func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHandler) {
+	if instanceType == FSTestStr {
+		return
+	}
+
 	t, _ := t1.(*testing.T)
 	testBuckets := []string{
 		// This bucket is used for testing ListObject operations.
@@ -752,10 +1124,6 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
 	for _, bucket := range testBuckets {
 		err := obj.MakeBucketWithLocation(context.Background(), bucket, BucketOptions{VersioningEnabled: true})
 		if err != nil {
-			if _, ok := err.(NotImplemented); ok {
-				// Skip test for FS mode.
-				continue
-			}
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
 	}
@@ -791,10 +1159,6 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
 		_, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content),
 			int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{UserDefined: object.meta})
 		if err != nil {
-			if _, ok := err.(BucketNotFound); ok {
-				// Skip test failure for FS mode.
-				continue
-			}
 			t.Fatalf("%s : %s", instanceType, err.Error())
 		}
 
@@ -1313,10 +1677,6 @@ func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHand
 		t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) {
 			result, err := obj.ListObjectVersions(context.Background(), testCase.bucketName,
 				testCase.prefix, testCase.marker, "", testCase.delimiter, int(testCase.maxKeys))
-			if _, ok := err.(NotImplemented); ok {
-				// Not implemented should be skipped
-				t.Skip()
-			}
 			if err != nil && testCase.shouldPass {
 				t.Errorf("%s:  Expected to pass, but failed with: <ERROR> %s", instanceType, err.Error())
 			}

cmd/object-handlers.go

@@ -282,7 +282,7 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 		case crypto.S3KMS:
 			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
-			w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.UserDefined[crypto.MetaKeyID])
+			w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.KMSKeyID())
 			if kmsCtx, ok := objInfo.UserDefined[crypto.MetaContext]; ok {
 				w.Header().Set(xhttp.AmzServerSideEncryptionKmsContext, kmsCtx)
 			}
@@ -481,7 +481,7 @@ func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI Obj
 			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 		case crypto.S3KMS:
 			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
-			w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.UserDefined[crypto.MetaKeyID])
+			w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.KMSKeyID())
 			if kmsCtx, ok := objInfo.UserDefined[crypto.MetaContext]; ok {
 				w.Header().Set(xhttp.AmzServerSideEncryptionKmsContext, kmsCtx)
 			}
@@ -729,7 +729,7 @@ func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI Ob
 			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES)
 		case crypto.S3KMS:
 			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
-			w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.UserDefined[crypto.MetaKeyID])
+			w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.KMSKeyID())
 			if kmsCtx, ok := objInfo.UserDefined[crypto.MetaContext]; ok {
 				w.Header().Set(xhttp.AmzServerSideEncryptionKmsContext, kmsCtx)
 			}
@@ -862,7 +862,7 @@ var (
 // Returns a minio-go Client configured to access remote host described by destDNSRecord
 // Applicable only in a federated deployment
 var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) {
-	cred := getReqAccessCred(r, globalServerRegion)
+	cred := getReqAccessCred(r, globalSite.Region)
 	// In a federated deployment, all the instances share config files
 	// and hence expected to have same credentials.
 	return miniogo.NewCore(host, &miniogo.Options{
@@ -1651,7 +1651,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 
 	case authTypePresigned, authTypeSigned:
-		if s3Err = reqSignatureV4Verify(r, globalServerRegion, serviceS3); s3Err != ErrNone {
+		if s3Err = reqSignatureV4Verify(r, globalSite.Region, serviceS3); s3Err != ErrNone {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 			return
 		}
@@ -1817,7 +1817,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 			objInfo.ETag, _ = DecryptETag(objectEncryptionKey, ObjectInfo{ETag: objInfo.ETag})
 		case crypto.S3KMS:
 			w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS)
-			w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.UserDefined[crypto.MetaKeyID])
+			w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.KMSKeyID())
 			if kmsCtx, ok := objInfo.UserDefined[crypto.MetaContext]; ok {
 				w.Header().Set(xhttp.AmzServerSideEncryptionKmsContext, kmsCtx)
 			}
@@ -1982,7 +1982,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		}
 
 	case authTypePresigned, authTypeSigned:
-		if s3Err = reqSignatureV4Verify(r, globalServerRegion, serviceS3); s3Err != ErrNone {
+		if s3Err = reqSignatureV4Verify(r, globalSite.Region, serviceS3); s3Err != ErrNone {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
 			return
 		}
@@ -2021,7 +2021,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		getObjectInfo = api.CacheAPI().GetObjectInfo
 	}
 
-	putObjectTar := func(reader io.Reader, info os.FileInfo, object string) {
+	putObjectTar := func(reader io.Reader, info os.FileInfo, object string) error {
 		size := info.Size()
 		metadata := map[string]string{
 			xhttp.AmzStorageClass: sc,
@@ -2035,8 +2035,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 
 			actualReader, err := hash.NewReader(reader, size, "", "", actualSize)
 			if err != nil {
-				writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-				return
+				return err
 			}
 
 			// Set compression metrics.
@@ -2048,8 +2047,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 
 		hashReader, err := hash.NewReader(reader, size, "", "", actualSize)
 		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
+			return err
 		}
 
 		rawReader := hashReader
@@ -2057,8 +2055,7 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 
 		if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() {
 			if s3Err = isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, iampolicy.ReplicateObjectAction); s3Err != ErrNone {
-				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
-				return
+				return err
 			}
 			metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String()
 			metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano)
@@ -2067,24 +2064,23 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		// get encryption options
 		opts, err := putOpts(ctx, r, bucket, object, metadata)
 		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
+			return err
 		}
 		opts.MTime = info.ModTime()
 
-		retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
-		if s3Err == ErrNone && retentionMode.Valid() {
+		retentionMode, retentionDate, legalHold, s3err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms)
+		if s3err == ErrNone && retentionMode.Valid() {
 			metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode)
 			metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = retentionDate.UTC().Format(iso8601TimeFormat)
 		}
 
-		if s3Err == ErrNone && legalHold.Status.Valid() {
+		if s3err == ErrNone && legalHold.Status.Valid() {
 			metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status)
 		}
 
-		if s3Err != ErrNone {
-			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL)
-			return
+		if s3err != ErrNone {
+			s3Err = s3err
+			return ObjectLocked{}
 		}
 
 		if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(ObjectInfo{
@@ -2099,14 +2095,12 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		if objectAPI.IsEncryptionSupported() {
 			if _, ok := crypto.IsRequested(r.Header); ok && !HasSuffix(object, SlashSeparator) { // handle SSE requests
 				if crypto.SSECopy.IsRequested(r.Header) {
-					writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL)
-					return
+					return errInvalidEncryptionParameters
 				}
 
 				reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata)
 				if err != nil {
-					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-					return
+					return err
 				}
 
 				wantSize := int64(-1)
@@ -2118,14 +2112,12 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 				// do not try to verify encrypted content
 				hashReader, err = hash.NewReader(etag.Wrap(reader, hashReader), wantSize, "", "", actualSize)
 				if err != nil {
-					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-					return
+					return err
 				}
 
 				pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey)
 				if err != nil {
-					writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-					return
+					return err
 				}
 			}
 		}
@@ -2136,20 +2128,34 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		// Create the object..
 		objInfo, err := putObject(ctx, bucket, object, pReader, opts)
 		if err != nil {
-			writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
-			return
+			return err
 		}
 
 		if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(ObjectInfo{
 			UserDefined: metadata,
 		}, replication.ObjectReplicationType, opts)); dsc.ReplicateAny() {
 			scheduleReplication(ctx, objInfo.Clone(), objectAPI, dsc, replication.ObjectReplicationType)
-
 		}
-
+		return nil
 	}
 
-	untar(hreader, putObjectTar)
+	if err = untar(hreader, putObjectTar); err != nil {
+		apiErr := errorCodes.ToAPIErr(s3Err)
+		// If not set, convert or use BadRequest
+		if s3Err == ErrNone {
+			apiErr = toAPIError(ctx, err)
+			if apiErr.Code == "InternalError" {
+				// Convert generic internal errors to bad requests.
+				apiErr = APIError{
+					Code:           "BadRequest",
+					Description:    err.Error(),
+					HTTPStatusCode: http.StatusBadRequest,
+				}
+			}
+		}
+		writeErrorResponse(ctx, w, apiErr, r.URL)
+		return
+	}
 
 	w.Header()[xhttp.ETag] = []string{`"` + hex.EncodeToString(hreader.MD5Current()) + `"`}
 	writeSuccessResponseHeadersOnly(w)
@@ -2739,7 +2745,7 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 			return
 		}
 	case authTypePresigned, authTypeSigned:
-		if s3Error = reqSignatureV4Verify(r, globalServerRegion, serviceS3); s3Error != ErrNone {
+		if s3Error = reqSignatureV4Verify(r, globalSite.Region, serviceS3); s3Error != ErrNone {
 			writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL)
 			return
 		}
@@ -3533,9 +3539,11 @@ func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r
 		return
 	}
 
-	legalHold, err := objectlock.ParseObjectLegalHold(r.Body)
+	legalHold, err := objectlock.ParseObjectLegalHold(io.LimitReader(r.Body, r.ContentLength))
 	if err != nil {
-		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
+		apiErr := errorCodes.ToAPIErr(ErrMalformedXML)
+		apiErr.Description = err.Error()
+		writeErrorResponse(ctx, w, apiErr, r.URL)
 		return
 	}
 

cmd/peer-rest-client.go

@@ -720,6 +720,11 @@ func (client *peerRESTClient) GetLocalDiskIDs(ctx context.Context) (diskIDs []st
 
 // GetMetacacheListing - get a new or existing metacache.
 func (client *peerRESTClient) GetMetacacheListing(ctx context.Context, o listPathOptions) (*metacache, error) {
+	if client == nil {
+		resp := localMetacacheMgr.getBucket(ctx, o.Bucket).findCache(o)
+		return &resp, nil
+	}
+
 	var reader bytes.Buffer
 	err := gob.NewEncoder(&reader).Encode(o)
 	if err != nil {
@@ -737,6 +742,9 @@ func (client *peerRESTClient) GetMetacacheListing(ctx context.Context, o listPat
 
 // UpdateMetacacheListing - update an existing metacache it will unconditionally be updated to the new state.
 func (client *peerRESTClient) UpdateMetacacheListing(ctx context.Context, m metacache) (metacache, error) {
+	if client == nil {
+		return localMetacacheMgr.updateCacheEntry(m)
+	}
 	b, err := m.MarshalMsg(nil)
 	if err != nil {
 		return m, err
@@ -1013,11 +1021,13 @@ func (client *peerRESTClient) GetPeerMetrics(ctx context.Context) (<-chan Metric
 	return ch, nil
 }
 
-func (client *peerRESTClient) Speedtest(ctx context.Context, size, concurrent int, duration time.Duration) (SpeedtestResult, error) {
+func (client *peerRESTClient) Speedtest(ctx context.Context, size,
+	concurrent int, duration time.Duration, storageClass string) (SpeedtestResult, error) {
 	values := make(url.Values)
 	values.Set(peerRESTSize, strconv.Itoa(size))
 	values.Set(peerRESTConcurrent, strconv.Itoa(concurrent))
 	values.Set(peerRESTDuration, duration.String())
+	values.Set(peerRESTStorageClass, storageClass)
 
 	respBody, err := client.callWithContext(context.Background(), peerRESTMethodSpeedtest, values, nil, -1)
 	if err != nil {

cmd/peer-rest-common.go

@@ -18,7 +18,7 @@
 package cmd
 
 const (
-	peerRESTVersion       = "v16" // Add new ServiceSignals.
+	peerRESTVersion       = "v17" // Add "storage-class" option for SpeedTest
 	peerRESTVersionPrefix = SlashSeparator + peerRESTVersion
 	peerRESTPrefix        = minioReservedBucketPath + "/peer"
 	peerRESTPath          = peerRESTPrefix + peerRESTVersionPrefix
@@ -89,6 +89,7 @@ const (
 	peerRESTSize           = "size"
 	peerRESTConcurrent     = "concurrent"
 	peerRESTDuration       = "duration"
+	peerRESTStorageClass   = "storage-class"
 
 	peerRESTListenBucket = "bucket"
 	peerRESTListenPrefix = "prefix"

cmd/peer-rest-server.go

@@ -38,6 +38,7 @@ import (
 	b "github.com/minio/minio/internal/bucket/bandwidth"
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/hash"
+	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/randreader"
 	"github.com/tinylib/msgp/msgp"
@@ -77,7 +78,7 @@ func (s *peerRESTServer) DeletePolicyHandler(w http.ResponseWriter, r *http.Requ
 		return
 	}
 
-	if err := globalIAMSys.DeletePolicy(r.Context(), policyName); err != nil {
+	if err := globalIAMSys.DeletePolicy(r.Context(), policyName, false); err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
@@ -156,7 +157,7 @@ func (s *peerRESTServer) DeleteServiceAccountHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if err := globalIAMSys.DeleteServiceAccount(r.Context(), accessKey); err != nil {
+	if err := globalIAMSys.DeleteServiceAccount(r.Context(), accessKey, false); err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
@@ -208,7 +209,7 @@ func (s *peerRESTServer) DeleteUserHandler(w http.ResponseWriter, r *http.Reques
 		return
 	}
 
-	if err := globalIAMSys.DeleteUser(r.Context(), accessKey); err != nil {
+	if err := globalIAMSys.DeleteUser(r.Context(), accessKey, false); err != nil {
 		s.writeErrorResponse(w, err)
 		return
 	}
@@ -1112,7 +1113,7 @@ func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request)
 
 	enc := gob.NewEncoder(w)
 
-	ch := ReportMetrics(r.Context(), GetGeneratorsForPeer)
+	ch := ReportMetrics(r.Context(), peerMetricsGroups)
 	for m := range ch {
 		if err := enc.Encode(m); err != nil {
 			s.writeErrorResponse(w, errors.New("Encoding metric failed: "+err.Error()))
@@ -1134,26 +1135,25 @@ func newRandomReader(size int) io.Reader {
 }
 
 // Runs the speedtest on local MinIO process.
-func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration) (SpeedtestResult, error) {
+func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration, storageClass string) (SpeedtestResult, error) {
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return SpeedtestResult{}, errServerNotInitialized
 	}
 
+	var errOnce sync.Once
 	var retError string
+	var wg sync.WaitGroup
+	var totalBytesWritten uint64
+	var totalBytesRead uint64
 
 	bucket := minioMetaSpeedTestBucket
 	objCountPerThread := make([]uint64, concurrent)
-
-	uploadsStopped := false
-	var totalBytesWritten uint64
 	uploadsCtx, uploadsCancel := context.WithCancel(context.Background())
-
-	var wg sync.WaitGroup
+	defer uploadsCancel()
 
 	go func() {
 		time.Sleep(duration)
-		uploadsStopped = true
 		uploadsCancel()
 	}()
 
@@ -1167,19 +1167,31 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 				hashReader, err := hash.NewReader(newRandomReader(size),
 					int64(size), "", "", int64(size))
 				if err != nil {
-					retError = err.Error()
-					logger.LogIf(ctx, err)
-					break
+					if !contextCanceled(uploadsCtx) {
+						errOnce.Do(func() {
+							retError = err.Error()
+							logger.LogIf(ctx, err)
+						})
+					}
+					uploadsCancel()
+					return
 				}
 				reader := NewPutObjReader(hashReader)
 				objInfo, err := objAPI.PutObject(uploadsCtx, bucket, fmt.Sprintf("%s.%d.%d",
-					objNamePrefix, i, objCountPerThread[i]), reader, ObjectOptions{})
-				if err != nil && !uploadsStopped {
-					retError = err.Error()
-					logger.LogIf(ctx, err)
-				}
+					objNamePrefix, i, objCountPerThread[i]), reader, ObjectOptions{
+					UserDefined: map[string]string{
+						xhttp.AmzStorageClass: storageClass,
+					},
+				})
 				if err != nil {
-					break
+					if !contextCanceled(uploadsCtx) {
+						errOnce.Do(func() {
+							retError = err.Error()
+							logger.LogIf(ctx, err)
+						})
+					}
+					uploadsCancel()
+					return
 				}
 				atomic.AddUint64(&totalBytesWritten, uint64(objInfo.Size))
 				objCountPerThread[i]++
@@ -1188,13 +1200,15 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 	}
 	wg.Wait()
 
-	downloadsStopped := false
-	var totalBytesRead uint64
-	downloadsCtx, downloadsCancel := context.WithCancel(context.Background())
+	// We already saw write failures, no need to proceed into read's
+	if retError != "" {
+		return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
+	}
 
+	downloadsCtx, downloadsCancel := context.WithCancel(context.Background())
+	defer downloadsCancel()
 	go func() {
 		time.Sleep(duration)
-		downloadsStopped = true
 		downloadsCancel()
 	}()
 
@@ -1212,12 +1226,15 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 				}
 				r, err := objAPI.GetObjectNInfo(downloadsCtx, bucket, fmt.Sprintf("%s.%d.%d",
 					objNamePrefix, i, j), nil, nil, noLock, ObjectOptions{})
-				if err != nil && !downloadsStopped {
-					retError = err.Error()
-					logger.LogIf(ctx, err)
-				}
 				if err != nil {
-					break
+					if !contextCanceled(downloadsCtx) {
+						errOnce.Do(func() {
+							retError = err.Error()
+							logger.LogIf(ctx, err)
+						})
+					}
+					downloadsCancel()
+					return
 				}
 				n, err := io.Copy(ioutil.Discard, r)
 				r.Close()
@@ -1227,18 +1244,22 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 					// reads etc.
 					atomic.AddUint64(&totalBytesRead, uint64(n))
 				}
-				if err != nil && !downloadsStopped {
-					retError = err.Error()
-					logger.LogIf(ctx, err)
-				}
 				if err != nil {
-					break
+					if !contextCanceled(downloadsCtx) {
+						errOnce.Do(func() {
+							retError = err.Error()
+							logger.LogIf(ctx, err)
+						})
+					}
+					downloadsCancel()
+					return
 				}
 				j++
 			}
 		}(i)
 	}
 	wg.Wait()
+
 	return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
 }
 
@@ -1257,6 +1278,7 @@ func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request
 	sizeStr := r.Form.Get(peerRESTSize)
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
+	storageClass := r.Form.Get(peerRESTStorageClass)
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -1275,7 +1297,7 @@ func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request
 
 	done := keepHTTPResponseAlive(w)
 
-	result, err := selfSpeedtest(r.Context(), size, concurrent, duration)
+	result, err := selfSpeedtest(r.Context(), size, concurrent, duration, storageClass)
 	if err != nil {
 		result.Error = err.Error()
 	}

cmd/prepare-storage.go

@@ -188,13 +188,17 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 	for i, err := range errs {
 		if err != nil {
 			if err == errDiskNotFound && retryCount >= 5 {
-				logger.Info("Unable to connect to %s: %v", endpoints[i], isServerResolvable(endpoints[i], time.Second))
+				logger.Error("Unable to connect to %s: %v", endpoints[i], isServerResolvable(endpoints[i], time.Second))
 			} else {
-				logger.Info("Unable to use the drive %s: %v", endpoints[i], err)
+				logger.Error("Unable to use the drive %s: %v", endpoints[i], err)
 			}
 		}
 	}
 
+	if err := checkDiskFatalErrs(errs); err != nil {
+		return nil, nil, err
+	}
+
 	// Attempt to load all `format.json` from all disks.
 	formatConfigs, sErrs := loadFormatErasureAll(storageDisks, false)
 	// Check if we have
@@ -202,7 +206,7 @@ func connectLoadInitFormats(retryCount int, firstDisk bool, endpoints Endpoints,
 		// print the error, nonetheless, which is perhaps unhandled
 		if sErr != errUnformattedDisk && sErr != errDiskNotFound && retryCount >= 5 {
 			if sErr != nil {
-				logger.Info("Unable to read 'format.json' from %s: %v\n", endpoints[i], sErr)
+				logger.Error("Unable to read 'format.json' from %s: %v\n", endpoints[i], sErr)
 			}
 		}
 	}

cmd/s3-zip-handlers.go

@@ -20,6 +20,7 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
@@ -142,8 +143,15 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 	var zipInfo []byte
 
 	if z, ok := zipObjInfo.UserDefined[archiveInfoMetadataKey]; ok {
-		zipInfo = []byte(z)
-	} else {
+		if globalIsErasure {
+			zipInfo = []byte(z)
+		} else {
+			zipInfo, err = base64.StdEncoding.DecodeString(z)
+			logger.LogIf(ctx, err)
+			// Will attempt to re-read...
+		}
+	}
+	if len(zipInfo) == 0 {
 		zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, opts)
 	}
 
@@ -151,7 +159,6 @@ func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context,
 		writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL)
 		return
 	}
-
 	file, err := zipindex.FindSerialized(zipInfo, object)
 	if err != nil {
 		if err == io.EOF {
@@ -491,7 +498,11 @@ func updateObjectMetadataWithZipInfo(ctx context.Context, objectAPI ObjectLayer,
 	}
 
 	srcInfo.UserDefined[archiveTypeMetadataKey] = archiveType
-	srcInfo.UserDefined[archiveInfoMetadataKey] = string(zipInfo)
+	if globalIsErasure {
+		srcInfo.UserDefined[archiveInfoMetadataKey] = string(zipInfo)
+	} else {
+		srcInfo.UserDefined[archiveInfoMetadataKey] = base64.StdEncoding.EncodeToString(zipInfo)
+	}
 	srcInfo.metadataOnly = true
 
 	// Always update the same version id & modtime

cmd/server-main.go

@@ -23,9 +23,9 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"log"
 	"math/rand"
-	"net"
 	"os"
 	"os/signal"
 	"strings"
@@ -64,6 +64,12 @@ var ServerFlags = []cli.Flag{
 		Name:  "console-address",
 		Usage: "bind to a specific ADDRESS:PORT for embedded Console UI, ADDRESS can be an IP or hostname",
 	},
+	cli.DurationFlag{
+		Name:   "shutdown-timeout",
+		Value:  xhttp.DefaultShutdownTimeout,
+		Usage:  "shutdown timeout to gracefully shutdown server",
+		Hidden: true,
+	},
 }
 
 var serverCmd = cli.Command{
@@ -412,12 +418,6 @@ func initConfigSubsystem(ctx context.Context, newObject ObjectLayer) ([]BucketIn
 	return buckets, nil
 }
 
-type nullWriter struct{}
-
-func (lw nullWriter) Write(b []byte) (int, error) {
-	return len(b), nil
-}
-
 // serverMain handler called for 'minio server' command.
 func serverMain(ctx *cli.Context) {
 	signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT)
@@ -460,10 +460,13 @@ func serverMain(ctx *cli.Context) {
 		}
 	}
 
-	if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
-		// Check for new updates from dl.min.io.
-		checkUpdate(getMinioMode())
-	}
+	// Check for updates in non-blocking manner.
+	go func() {
+		if !globalCLIContext.Quiet && !globalInplaceUpdateDisabled {
+			// Check for new updates from dl.min.io.
+			checkUpdate(getMinioMode())
+		}
+	}()
 
 	if !globalActiveCred.IsValid() && globalIsDistErasure {
 		globalActiveCred = auth.DefaultCredentials
@@ -492,12 +495,13 @@ func serverMain(ctx *cli.Context) {
 		addrs = append(addrs, globalMinioAddr)
 	}
 
-	httpServer := xhttp.NewServer(addrs, setCriticalErrorHandler(corsHandler(handler)), getCert)
-	httpServer.BaseContext = func(listener net.Listener) context.Context {
-		return GlobalContext
-	}
-	// Turn-off random logging by Go internally
-	httpServer.ErrorLog = log.New(&nullWriter{}, "", 0)
+	httpServer := xhttp.NewServer(addrs).
+		UseHandler(setCriticalErrorHandler(corsHandler(handler))).
+		UseTLSConfig(newTLSConfig(getCert)).
+		UseShutdownTimeout(ctx.Duration("shutdown-timeout")).
+		UseBaseContext(GlobalContext).
+		UseCustomLogger(log.New(ioutil.Discard, "", 0)) // Turn-off random logging by Go stdlib
+
 	go func() {
 		globalHTTPServerErrorCh <- httpServer.Start(GlobalContext)
 	}()
@@ -567,7 +571,7 @@ func serverMain(ctx *cli.Context) {
 	globalSiteReplicationSys.Init(GlobalContext, newObject)
 
 	// Initialize users credentials and policies in background right after config has initialized.
-	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval)
+	go globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalNotificationSys, globalRefreshIAMInterval)
 
 	// Initialize transition tier configuration manager
 	if globalIsErasure {
@@ -587,8 +591,9 @@ func serverMain(ctx *cli.Context) {
 		}
 	}
 
+	// initialize the new disk cache objects.
 	if globalCacheConfig.Enabled {
-		// initialize the new disk cache objects.
+		logStartupMessage(color.Yellow("WARNING: Disk caching is deprecated for single/multi drive MinIO setups. Please migrate to using MinIO S3 gateway instead of disk caching"))
 		var cacheAPI CacheObjectLayer
 		cacheAPI, err = newServerCacheObjects(GlobalContext, globalCacheConfig)
 		logger.FatalIf(err, "Unable to initialize disk caching")

cmd/server-startup-msg.go

@@ -126,7 +126,7 @@ func printServerCommonMsg(apiEndpoints []string) {
 	cred := globalActiveCred
 
 	// Get saved region.
-	region := globalServerRegion
+	region := globalSite.Region
 
 	apiEndpointStr := strings.Join(apiEndpoints, "  ")
 

cmd/signature-v4-utils.go

@@ -154,6 +154,11 @@ func checkKeyValid(r *http.Request, accessKey string) (auth.Credentials, bool, A
 		// Check if the access key is part of users credentials.
 		ucred, ok := globalIAMSys.GetUser(r.Context(), accessKey)
 		if !ok {
+			// Credentials will be invalid but and disabled
+			// return a different error in such a scenario.
+			if ucred.Status == auth.AccountOff {
+				return cred, false, ErrAccessKeyDisabled
+			}
 			return cred, false, ErrInvalidAccessKeyID
 		}
 		cred = ucred

cmd/signature-v4-utils_test.go

@@ -46,7 +46,7 @@ func TestCheckValid(t *testing.T) {
 
 	initConfigSubsystem(ctx, objLayer)
 
-	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second)
+	globalIAMSys.Init(ctx, objLayer, globalEtcdClient, globalNotificationSys, 2*time.Second)
 
 	req, err := newTestRequest(http.MethodGet, "http://example.com:9000/bucket/object", 0, nil)
 	if err != nil {
@@ -76,7 +76,7 @@ func TestCheckValid(t *testing.T) {
 		t.Fatalf("unable create credential, %s", err)
 	}
 
-	globalIAMSys.CreateUser(ctx, ucreds.AccessKey, madmin.UserInfo{
+	globalIAMSys.CreateUser(ctx, ucreds.AccessKey, madmin.AddOrUpdateUserReq{
 		SecretKey: ucreds.SecretKey,
 		Status:    madmin.AccountEnabled,
 	})

cmd/signature-v4.go

@@ -173,7 +173,7 @@ func compareSignatureV4(sig1, sig2 string) bool {
 // returns ErrNone if the signature matches.
 func doesPolicySignatureV4Match(formValues http.Header) (auth.Credentials, APIErrorCode) {
 	// Server region.
-	region := globalServerRegion
+	region := globalSite.Region
 
 	// Parse credential tag.
 	credHeader, s3Err := parseCredentialHeader("Credential="+formValues.Get(xhttp.AmzCredential), region, serviceS3)

cmd/signature-v4_test.go

@@ -107,7 +107,7 @@ func TestDoesPresignedSignatureMatch(t *testing.T) {
 	now := UTCNow()
 	credentialTemplate := "%s/%s/%s/s3/aws4_request"
 
-	region := globalServerRegion
+	region := globalSite.Region
 	accessKeyID := globalActiveCred.AccessKey
 	testCases := []struct {
 		queryParams map[string]string

cmd/site-replication.go

@@ -40,7 +40,6 @@ import (
 	"github.com/minio/minio-go/v7/pkg/set"
 	"github.com/minio/minio/internal/auth"
 	sreplication "github.com/minio/minio/internal/bucket/replication"
-	"github.com/minio/minio/internal/bucket/versioning"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
 	iampolicy "github.com/minio/pkg/iam/policy"
@@ -48,8 +47,7 @@ import (
 
 const (
 	srStatePrefix = minioConfigPrefix + "/site-replication"
-
-	srStateFile = "state.json"
+	srStateFile   = "state.json"
 )
 
 const (
@@ -57,11 +55,26 @@ const (
 )
 
 var (
-	errSRCannotJoin     = errors.New("this site is already configured for site-replication")
-	errSRDuplicateSites = errors.New("duplicate sites provided for site-replication")
-	errSRSelfNotFound   = errors.New("none of the given sites correspond to the current one")
-	errSRPeerNotFound   = errors.New("peer not found")
-	errSRNotEnabled     = errors.New("site replication is not enabled")
+	errSRCannotJoin = SRError{
+		Cause: errors.New("this site is already configured for site-replication"),
+		Code:  ErrSiteReplicationInvalidRequest,
+	}
+	errSRDuplicateSites = SRError{
+		Cause: errors.New("duplicate sites provided for site-replication"),
+		Code:  ErrSiteReplicationInvalidRequest,
+	}
+	errSRSelfNotFound = SRError{
+		Cause: errors.New("none of the given sites correspond to the current one"),
+		Code:  ErrSiteReplicationInvalidRequest,
+	}
+	errSRPeerNotFound = SRError{
+		Cause: errors.New("peer not found"),
+		Code:  ErrSiteReplicationInvalidRequest,
+	}
+	errSRNotEnabled = SRError{
+		Cause: errors.New("site replication is not enabled"),
+		Code:  ErrSiteReplicationInvalidRequest,
+	}
 )
 
 func errSRInvalidRequest(err error) SRError {
@@ -252,92 +265,132 @@ const (
 	siteReplicatorSvcAcc = "site-replicator-0"
 )
 
-// AddPeerClusters - add cluster sites for replication configuration.
-func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, sites []madmin.PeerSite) (madmin.ReplicateAddStatus, SRError) {
-	// If current cluster is already SR enabled, we fail.
-	if c.enabled {
-		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errSRCannotJoin)
-	}
+// PeerSiteInfo is a wrapper struct around madmin.PeerSite with extra info on site status
+type PeerSiteInfo struct {
+	madmin.PeerSite
+	self         bool
+	DeploymentID string
+	Replicated   bool // true if already participating in site replication
+	Empty        bool // true if cluster has no buckets
+}
 
-	// Only one of the clusters being added, can have any buckets (i.e. self
-	// here) - others must be empty.
-	selfIdx := -1
-	localHasBuckets := false
-	nonLocalPeerWithBuckets := ""
-	deploymentIDs := make([]string, 0, len(sites))
-	deploymentIDsSet := set.NewStringSet()
-	for i, v := range sites {
+// getSiteStatuses gathers more info on the sites being added
+func (c *SiteReplicationSys) getSiteStatuses(ctx context.Context, sites []madmin.PeerSite) (psi []PeerSiteInfo, err SRError) {
+	for _, v := range sites {
 		admClient, err := getAdminClient(v.Endpoint, v.AccessKey, v.SecretKey)
 		if err != nil {
-			return madmin.ReplicateAddStatus{}, errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err))
+			return psi, errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err))
 		}
 		info, err := admClient.ServerInfo(ctx)
 		if err != nil {
-			return madmin.ReplicateAddStatus{}, errSRPeerResp(fmt.Errorf("unable to fetch server info for %s: %w", v.Name, err))
+			return psi, errSRPeerResp(fmt.Errorf("unable to fetch server info for %s: %w", v.Name, err))
 		}
 
 		deploymentID := info.DeploymentID
-		if deploymentID == "" {
-			return madmin.ReplicateAddStatus{}, errSRPeerResp(fmt.Errorf("unable to fetch deploymentID for %s: value was empty!", v.Name))
+		pi := PeerSiteInfo{
+			PeerSite:     v,
+			DeploymentID: deploymentID,
+			Empty:        true,
 		}
 
-		deploymentIDs = append(deploymentIDs, deploymentID)
-
-		// deploymentIDs must be unique
-		if deploymentIDsSet.Contains(deploymentID) {
-			return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errSRDuplicateSites)
-		}
-		deploymentIDsSet.Add(deploymentID)
-
 		if deploymentID == globalDeploymentID {
-			selfIdx = i
 			objAPI := newObjectLayerFn()
 			if objAPI == nil {
-				return madmin.ReplicateAddStatus{}, errSRObjectLayerNotReady
+				return psi, errSRObjectLayerNotReady
 			}
 			res, err := objAPI.ListBuckets(ctx)
 			if err != nil {
-				return madmin.ReplicateAddStatus{}, errSRBackendIssue(err)
+				return psi, errSRBackendIssue(err)
 			}
 			if len(res) > 0 {
-				localHasBuckets = true
+				pi.Empty = false
 			}
+			pi.self = true
+		} else {
+			s3Client, err := getS3Client(v)
+			if err != nil {
+				return psi, errSRPeerResp(fmt.Errorf("unable to create s3 client for %s: %w", v.Name, err))
+			}
+			buckets, err := s3Client.ListBuckets(ctx)
+			if err != nil {
+				return psi, errSRPeerResp(fmt.Errorf("unable to list buckets for %s: %v", v.Name, err))
+			}
+			pi.Empty = len(buckets) == 0
+		}
+		psi = append(psi, pi)
+	}
+	return
+}
+
+// AddPeerClusters - add cluster sites for replication configuration.
+func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmin.PeerSite) (madmin.ReplicateAddStatus, error) {
+	sites, serr := c.getSiteStatuses(ctx, psites)
+	if serr.Cause != nil {
+		return madmin.ReplicateAddStatus{}, serr
+	}
+	var (
+		currSites            madmin.SiteReplicationInfo
+		currDeploymentIDsSet = set.NewStringSet()
+		err                  error
+	)
+	if c.enabled {
+		currSites, err = c.GetClusterInfo(ctx)
+		if err != nil {
+			return madmin.ReplicateAddStatus{}, errSRBackendIssue(err)
+		}
+		for _, v := range currSites.Sites {
+			currDeploymentIDsSet.Add(v.DeploymentID)
+		}
+	}
+	deploymentIDsSet := set.NewStringSet()
+	localHasBuckets := false
+	nonLocalPeerWithBuckets := ""
+	var selfIdx = -1
+	for i, v := range sites {
+		// deploymentIDs must be unique
+		if deploymentIDsSet.Contains(v.DeploymentID) {
+			return madmin.ReplicateAddStatus{}, errSRDuplicateSites
+		}
+		deploymentIDsSet.Add(v.DeploymentID)
+
+		if v.self {
+			selfIdx = i
+			localHasBuckets = !v.Empty
 			continue
 		}
-
-		s3Client, err := getS3Client(v)
-		if err != nil {
-			return madmin.ReplicateAddStatus{}, errSRPeerResp(fmt.Errorf("unable to create s3 client for %s: %w", v.Name, err))
-		}
-		buckets, err := s3Client.ListBuckets(ctx)
-		if err != nil {
-			return madmin.ReplicateAddStatus{}, errSRPeerResp(fmt.Errorf("unable to list buckets for %s: %v", v.Name, err))
-		}
-
-		if len(buckets) > 0 {
+		if !v.Empty && !currDeploymentIDsSet.Contains(v.DeploymentID) {
 			nonLocalPeerWithBuckets = v.Name
 		}
 	}
-
+	if c.enabled {
+		// If current cluster is already SR enabled and no new site being added ,fail.
+		if currDeploymentIDsSet.Equals(deploymentIDsSet) {
+			return madmin.ReplicateAddStatus{}, errSRCannotJoin
+		}
+		if len(currDeploymentIDsSet.Intersection(deploymentIDsSet)) != len(currDeploymentIDsSet) {
+			diffSlc := getMissingSiteNames(currDeploymentIDsSet, deploymentIDsSet, currSites.Sites)
+			return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("all existing replicated sites must be specified - missing %s", strings.Join(diffSlc, " ")))
+		}
+	}
 	// For this `add` API, either all clusters must be empty or the local
 	// cluster must be the only one having some buckets.
 
 	if localHasBuckets && nonLocalPeerWithBuckets != "" {
-		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errors.New("Only one cluster may have data when configuring site replication"))
+		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errors.New("only one cluster may have data when configuring site replication"))
 	}
 
 	if !localHasBuckets && nonLocalPeerWithBuckets != "" {
-		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("Please send your request to the cluster containing data/buckets: %s", nonLocalPeerWithBuckets))
+		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("please send your request to the cluster containing data/buckets: %s", nonLocalPeerWithBuckets))
 	}
 
 	// validate that all clusters are using the same (LDAP based)
 	// external IDP.
-	pass, verr := c.validateIDPSettings(ctx, sites, selfIdx)
-	if verr.Cause != nil {
-		return madmin.ReplicateAddStatus{}, verr
+	pass, err := c.validateIDPSettings(ctx, sites)
+	if err != nil {
+		return madmin.ReplicateAddStatus{}, err
 	}
 	if !pass {
-		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("All cluster sites must have the same (LDAP) IDP settings."))
+		return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errors.New("all cluster sites must have the same (LDAP) IDP settings"))
 	}
 
 	// FIXME: Ideally, we also need to check if there are any global IAM
@@ -352,60 +405,64 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, sites []madmin
 
 	// Create a local service account.
 
-	// Generate a secret key for the service account.
+	// Generate a secret key for the service account if not created already.
 	var secretKey string
-	_, secretKey, err := auth.GenerateCredentials()
-	if err != nil {
-		return madmin.ReplicateAddStatus{}, SRError{
-			Cause: err,
-			Code:  ErrInternalError,
+	svcCred, _, err := globalIAMSys.getServiceAccount(ctx, siteReplicatorSvcAcc)
+	switch {
+	case err == errNoSuchServiceAccount:
+		_, secretKey, err = auth.GenerateCredentials()
+		if err != nil {
+			return madmin.ReplicateAddStatus{}, errSRServiceAccount(fmt.Errorf("unable to create local service account: %w", err))
 		}
-	}
-
-	svcCred, err := globalIAMSys.NewServiceAccount(ctx, sites[selfIdx].AccessKey, nil, newServiceAccountOpts{
-		accessKey: siteReplicatorSvcAcc,
-		secretKey: secretKey,
-	})
-	if err != nil {
-		return madmin.ReplicateAddStatus{}, errSRServiceAccount(fmt.Errorf("unable to create local service account: %w", err))
-	}
-
-	// Notify all other Minio peers to reload user the service account
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadServiceAccount(svcCred.AccessKey) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
+		svcCred, err = globalIAMSys.NewServiceAccount(ctx, sites[selfIdx].AccessKey, nil, newServiceAccountOpts{
+			accessKey: siteReplicatorSvcAcc,
+			secretKey: secretKey,
+		})
+		if err != nil {
+			return madmin.ReplicateAddStatus{}, errSRServiceAccount(fmt.Errorf("unable to create local service account: %w", err))
 		}
+	case err == nil:
+		secretKey = svcCred.SecretKey
+	default:
+		return madmin.ReplicateAddStatus{}, errSRBackendIssue(err)
 	}
 
-	joinReq := madmin.SRInternalJoinReq{
+	joinReq := madmin.SRPeerJoinReq{
 		SvcAcctAccessKey: svcCred.AccessKey,
-		SvcAcctSecretKey: svcCred.SecretKey,
+		SvcAcctSecretKey: secretKey,
 		Peers:            make(map[string]madmin.PeerInfo),
 	}
-	for i, v := range sites {
-		joinReq.Peers[deploymentIDs[i]] = madmin.PeerInfo{
+
+	for _, v := range sites {
+		joinReq.Peers[v.DeploymentID] = madmin.PeerInfo{
 			Endpoint:     v.Endpoint,
 			Name:         v.Name,
-			DeploymentID: deploymentIDs[i],
+			DeploymentID: v.DeploymentID,
 		}
 	}
 
 	addedCount := 0
-	var peerAddErr SRError
-	for i, v := range sites {
-		if i == selfIdx {
+	var (
+		peerAddErr error
+		admClient  *madmin.AdminClient
+	)
+
+	for _, v := range sites {
+		if v.self {
 			continue
 		}
-		admClient, err := getAdminClient(v.Endpoint, v.AccessKey, v.SecretKey)
+		switch {
+		case currDeploymentIDsSet.Contains(v.DeploymentID):
+			admClient, err = c.getAdminClient(ctx, v.DeploymentID)
+		default:
+			admClient, err = getAdminClient(v.Endpoint, v.AccessKey, v.SecretKey)
+		}
 		if err != nil {
 			peerAddErr = errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err))
 			break
 		}
 		joinReq.SvcAcctParent = v.AccessKey
-		err = admClient.SRInternalJoin(ctx, joinReq)
+		err = admClient.SRPeerJoin(ctx, joinReq)
 		if err != nil {
 			peerAddErr = errSRPeerResp(fmt.Errorf("unable to link with peer %s: %w", v.Name, err))
 			break
@@ -413,7 +470,7 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, sites []madmin
 		addedCount++
 	}
 
-	if peerAddErr.Cause != nil {
+	if peerAddErr != nil {
 		if addedCount == 0 {
 			return madmin.ReplicateAddStatus{}, peerAddErr
 		}
@@ -425,7 +482,8 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, sites []madmin
 			Status:    madmin.ReplicateAddStatusPartial,
 			ErrDetail: peerAddErr.Error(),
 		}
-		return partial, SRError{}
+
+		return partial, nil
 	}
 
 	// Other than handling existing buckets, we can now save the cluster
@@ -435,33 +493,29 @@ func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, sites []madmin
 		Peers:                   joinReq.Peers,
 		ServiceAccountAccessKey: svcCred.AccessKey,
 	}
-	err = c.saveToDisk(ctx, state)
-	if err != nil {
+
+	if err = c.saveToDisk(ctx, state); err != nil {
 		return madmin.ReplicateAddStatus{
 			Status:    madmin.ReplicateAddStatusPartial,
 			ErrDetail: fmt.Sprintf("unable to save cluster-replication state on local: %v", err),
-		}, SRError{}
+		}, nil
 	}
 
 	result := madmin.ReplicateAddStatus{
 		Success: true,
 		Status:  madmin.ReplicateAddStatusSuccess,
 	}
-	initialSyncErr := c.syncLocalToPeers(ctx)
-	if initialSyncErr.Code != ErrNone {
-		result.InitialSyncErrorMessage = initialSyncErr.Error()
+
+	if err := c.syncLocalToPeers(ctx); err != nil {
+		result.InitialSyncErrorMessage = err.Error()
 	}
 
-	return result, SRError{}
+	return result, nil
 }
 
 // InternalJoinReq - internal API handler to respond to a peer cluster's request
 // to join.
-func (c *SiteReplicationSys) InternalJoinReq(ctx context.Context, arg madmin.SRInternalJoinReq) SRError {
-	if c.enabled {
-		return errSRInvalidRequest(errSRCannotJoin)
-	}
-
+func (c *SiteReplicationSys) InternalJoinReq(ctx context.Context, arg madmin.SRPeerJoinReq) error {
 	var ourName string
 	for d, p := range arg.Peers {
 		if d == globalDeploymentID {
@@ -470,37 +524,29 @@ func (c *SiteReplicationSys) InternalJoinReq(ctx context.Context, arg madmin.SRI
 		}
 	}
 	if ourName == "" {
-		return errSRInvalidRequest(errSRSelfNotFound)
+		return errSRSelfNotFound
 	}
 
-	svcCred, err := globalIAMSys.NewServiceAccount(ctx, arg.SvcAcctParent, nil, newServiceAccountOpts{
-		accessKey: arg.SvcAcctAccessKey,
-		secretKey: arg.SvcAcctSecretKey,
-	})
+	_, _, err := globalIAMSys.GetServiceAccount(ctx, arg.SvcAcctAccessKey)
+	if err == errNoSuchServiceAccount {
+		_, err = globalIAMSys.NewServiceAccount(ctx, arg.SvcAcctParent, nil, newServiceAccountOpts{
+			accessKey: arg.SvcAcctAccessKey,
+			secretKey: arg.SvcAcctSecretKey,
+		})
+	}
 	if err != nil {
 		return errSRServiceAccount(fmt.Errorf("unable to create service account on %s: %v", ourName, err))
 	}
 
-	// Notify all other Minio peers to reload the service account
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadServiceAccount(svcCred.AccessKey) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
-
 	state := srState{
 		Name:                    ourName,
 		Peers:                   arg.Peers,
 		ServiceAccountAccessKey: arg.SvcAcctAccessKey,
 	}
-	err = c.saveToDisk(ctx, state)
-	if err != nil {
+	if err = c.saveToDisk(ctx, state); err != nil {
 		return errSRBackendIssue(fmt.Errorf("unable to save cluster-replication state to disk on %s: %v", ourName, err))
 	}
-	return SRError{}
+	return nil
 }
 
 // GetIDPSettings returns info about the configured identity provider. It is
@@ -515,10 +561,10 @@ func (c *SiteReplicationSys) GetIDPSettings(ctx context.Context) madmin.IDPSetti
 	}
 }
 
-func (c *SiteReplicationSys) validateIDPSettings(ctx context.Context, peers []madmin.PeerSite, selfIdx int) (bool, SRError) {
+func (c *SiteReplicationSys) validateIDPSettings(ctx context.Context, peers []PeerSiteInfo) (bool, error) {
 	s := make([]madmin.IDPSettings, 0, len(peers))
-	for i, v := range peers {
-		if i == selfIdx {
+	for _, v := range peers {
+		if v.self {
 			s = append(s, c.GetIDPSettings(ctx))
 			continue
 		}
@@ -528,7 +574,7 @@ func (c *SiteReplicationSys) validateIDPSettings(ctx context.Context, peers []ma
 			return false, errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err))
 		}
 
-		is, err := admClient.SRInternalGetIDPSettings(ctx)
+		is, err := admClient.SRPeerGetIDPSettings(ctx)
 		if err != nil {
 			return false, errSRPeerResp(fmt.Errorf("unable to fetch IDP settings from %s: %v", v.Name, err))
 		}
@@ -537,15 +583,15 @@ func (c *SiteReplicationSys) validateIDPSettings(ctx context.Context, peers []ma
 
 	for _, v := range s {
 		if !v.IsLDAPEnabled {
-			return false, SRError{}
+			return false, nil
 		}
 	}
 	for i := 1; i < len(s); i++ {
 		if s[i] != s[0] {
-			return false, SRError{}
+			return false, nil
 		}
 	}
-	return true, SRError{}
+	return true, nil
 }
 
 // GetClusterInfo - returns site replication information.
@@ -588,7 +634,11 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 		optsMap["location"] = opts.Location
 	}
 	if opts.LockEnabled {
-		optsMap["lockEnabled"] = ""
+		optsMap["lockEnabled"] = "true"
+		optsMap["versioningEnabled"] = "true"
+	}
+	if opts.VersioningEnabled {
+		optsMap["versioningEnabled"] = "true"
 	}
 
 	// Create bucket and enable versioning on all peers.
@@ -604,7 +654,7 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 				return err
 			}
 
-			err = admClient.SRInternalBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap)
+			err = admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap)
 			logger.LogIf(ctx, c.annotatePeerErr(p.Name, "MakeWithVersioning", err))
 			return err
 		},
@@ -632,7 +682,7 @@ func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string,
 				return err
 			}
 
-			err = admClient.SRInternalBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil)
+			err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil)
 			logger.LogIf(ctx, c.annotatePeerErr(p.Name, "ConfigureRepl", err))
 			return err
 		},
@@ -669,7 +719,7 @@ func (c *SiteReplicationSys) DeleteBucketHook(ctx context.Context, bucket string
 			return wrapSRErr(err)
 		}
 
-		err = admClient.SRInternalBucketOps(ctx, bucket, op, nil)
+		err = admClient.SRPeerBucketOps(ctx, bucket, op, nil)
 		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "DeleteBucket", err))
 		return err
 	})
@@ -682,6 +732,7 @@ func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Con
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
+
 	err := objAPI.MakeBucketWithLocation(ctx, bucket, opts)
 	if err != nil {
 		// Check if this is a bucket exists error.
@@ -697,27 +748,25 @@ func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Con
 		globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 	}
 
-	// Enable versioning on the bucket.
-	config, err := globalBucketVersioningSys.Get(bucket)
+	meta, err := globalBucketMetadataSys.Get(bucket)
 	if err != nil {
+		logger.LogIf(ctx, c.annotateErr("MakeBucketErr on peer call", err))
 		return wrapSRErr(err)
 	}
-	if !config.Enabled() {
-		verConf := versioning.Versioning{
-			Status: versioning.Enabled,
-		}
-		// FIXME: need to confirm if skipping object lock and
-		// versioning-suspended state checks are valid here.
-		cfgData, err := xml.Marshal(verConf)
-		if err != nil {
-			return wrapSRErr(err)
-		}
-		err = globalBucketMetadataSys.Update(bucket, bucketVersioningConfig, cfgData)
-		if err != nil {
-			logger.LogIf(ctx, c.annotateErr("Versioning enabling error on peer call", err))
-			return wrapSRErr(err)
-		}
+
+	meta.VersioningConfigXML = enabledBucketVersioningConfig
+	if opts.LockEnabled {
+		meta.ObjectLockConfigXML = enabledBucketObjectLockConfig
 	}
+
+	if err := meta.Save(context.Background(), objAPI); err != nil {
+		return wrapSRErr(err)
+	}
+
+	globalBucketMetadataSys.Set(bucket, meta)
+
+	// Load updated bucket metadata into memory as new metadata updated.
+	globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket)
 	return nil
 }
 
@@ -811,25 +860,41 @@ func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context,
 				return err
 			}
 		}
-		err = replicationConfig.AddRule(replication.Options{
-			// Set the ID so we can identify the rule as being
-			// created for site-replication and include the
-			// destination cluster's deployment ID.
-			ID: fmt.Sprintf("site-repl-%s", d),
+		var (
+			ruleID  = fmt.Sprintf("site-repl-%s", d)
+			hasRule bool
+			opts    = replication.Options{
+				// Set the ID so we can identify the rule as being
+				// created for site-replication and include the
+				// destination cluster's deployment ID.
+				ID: ruleID,
 
-			// Use a helper to generate unique priority numbers.
-			Priority: fmt.Sprintf("%d", getPriorityHelper(replicationConfig)),
+				// Use a helper to generate unique priority numbers.
+				Priority: fmt.Sprintf("%d", getPriorityHelper(replicationConfig)),
 
-			Op:         replication.AddOption,
-			RuleStatus: "enable",
-			DestBucket: targetARN,
+				Op:         replication.AddOption,
+				RuleStatus: "enable",
+				DestBucket: targetARN,
+
+				// Replicate everything!
+				ReplicateDeletes:        "enable",
+				ReplicateDeleteMarkers:  "enable",
+				ReplicaSync:             "enable",
+				ExistingObjectReplicate: "enable",
+			}
+		)
+		for _, r := range replicationConfig.Rules {
+			if r.ID == ruleID {
+				hasRule = true
+			}
+		}
+		switch {
+		case hasRule:
+			err = replicationConfig.EditRule(opts)
+		default:
+			err = replicationConfig.AddRule(opts)
+		}
 
-			// Replicate everything!
-			ReplicateDeletes:        "enable",
-			ReplicateDeleteMarkers:  "enable",
-			ReplicaSync:             "enable",
-			ExistingObjectReplicate: "enable",
-		})
 		if err != nil {
 			logger.LogIf(ctx, c.annotatePeerErr(peer.Name, "Error adding bucket replication rule", err))
 			return err
@@ -883,14 +948,24 @@ func (c *SiteReplicationSys) PeerBucketDeleteHandler(ctx context.Context, bucket
 		return errSRNotEnabled
 	}
 
-	// FIXME: need to handle cases where globalDNSConfig is set.
-
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
 		return errServerNotInitialized
 	}
+
+	if globalDNSConfig != nil {
+		if err := globalDNSConfig.Delete(bucket); err != nil {
+			return err
+		}
+	}
+
 	err := objAPI.DeleteBucket(ctx, bucket, DeleteBucketOptions{Force: forceDelete})
 	if err != nil {
+		if globalDNSConfig != nil {
+			if err2 := globalDNSConfig.Put(bucket); err2 != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, please fix it manually", err2))
+			}
+		}
 		return err
 	}
 
@@ -935,8 +1010,8 @@ func (c *SiteReplicationSys) IAMChangeHook(ctx context.Context, item madmin.SRIA
 			return wrapSRErr(err)
 		}
 
-		err = admClient.SRInternalReplicateIAMItem(ctx, item)
-		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "SRInternalReplicateIAMItem", err))
+		err = admClient.SRPeerReplicateIAMItem(ctx, item)
+		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "SRPeerReplicateIAMItem", err))
 		return err
 	})
 	return cErr.summaryErr
@@ -947,39 +1022,21 @@ func (c *SiteReplicationSys) IAMChangeHook(ctx context.Context, item madmin.SRIA
 func (c *SiteReplicationSys) PeerAddPolicyHandler(ctx context.Context, policyName string, p *iampolicy.Policy) error {
 	var err error
 	if p == nil {
-		err = globalIAMSys.DeletePolicy(ctx, policyName)
+		err = globalIAMSys.DeletePolicy(ctx, policyName, true)
 	} else {
 		err = globalIAMSys.SetPolicy(ctx, policyName, *p)
 	}
 	if err != nil {
 		return wrapSRErr(err)
 	}
-
-	if p != nil {
-		if !globalIAMSys.HasWatcher() {
-			// Notify all other MinIO peers to reload policy
-			for _, nerr := range globalNotificationSys.LoadPolicy(policyName) {
-				if nerr.Err != nil {
-					logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-					logger.LogIf(ctx, nerr.Err)
-				}
-			}
-		}
-		return nil
-	}
-
-	// Notify all other MinIO peers to delete policy
-	for _, nerr := range globalNotificationSys.DeletePolicy(policyName) {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
 	return nil
 }
 
 // PeerSvcAccChangeHandler - copies service-account change to local.
-func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change madmin.SRSvcAccChange) error {
+func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change *madmin.SRSvcAccChange) error {
+	if change == nil {
+		return errSRInvalidRequest(errInvalidArgument)
+	}
 	switch {
 	case change.Create != nil:
 		var sp *iampolicy.Policy
@@ -997,20 +1054,11 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 			sessionPolicy: sp,
 			claims:        change.Create.Claims,
 		}
-		newCred, err := globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts)
+		_, err = globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts)
 		if err != nil {
 			return wrapSRErr(err)
 		}
 
-		// Notify all other Minio peers to reload the service account
-		if !globalIAMSys.HasWatcher() {
-			for _, nerr := range globalNotificationSys.LoadServiceAccount(newCred.AccessKey) {
-				if nerr.Err != nil {
-					logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-					logger.LogIf(ctx, nerr.Err)
-				}
-			}
-		}
 	case change.Update != nil:
 		var sp *iampolicy.Policy
 		var err error
@@ -1031,55 +1079,35 @@ func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change
 			return wrapSRErr(err)
 		}
 
-		// Notify all other Minio peers to reload the service account
-		if !globalIAMSys.HasWatcher() {
-			for _, nerr := range globalNotificationSys.LoadServiceAccount(change.Update.AccessKey) {
-				if nerr.Err != nil {
-					logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-					logger.LogIf(ctx, nerr.Err)
-				}
-			}
-		}
-
 	case change.Delete != nil:
-		err := globalIAMSys.DeleteServiceAccount(ctx, change.Delete.AccessKey)
+		err := globalIAMSys.DeleteServiceAccount(ctx, change.Delete.AccessKey, true)
 		if err != nil {
 			return wrapSRErr(err)
 		}
 
-		for _, nerr := range globalNotificationSys.DeleteServiceAccount(change.Delete.AccessKey) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-
 	}
 
 	return nil
 }
 
 // PeerPolicyMappingHandler - copies policy mapping to local.
-func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mapping madmin.SRPolicyMapping) error {
+func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mapping *madmin.SRPolicyMapping) error {
+	if mapping == nil {
+		return errSRInvalidRequest(errInvalidArgument)
+	}
 	err := globalIAMSys.PolicyDBSet(ctx, mapping.UserOrGroup, mapping.Policy, mapping.IsGroup)
 	if err != nil {
 		return wrapSRErr(err)
 	}
-
-	// Notify all other MinIO peers to reload policy
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadPolicyMapping(mapping.UserOrGroup, mapping.IsGroup) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
 	return nil
 }
 
 // PeerSTSAccHandler - replicates STS credential locally.
-func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred madmin.SRSTSCredential) error {
+func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *madmin.SRSTSCredential) error {
+	if stsCred == nil {
+		return errSRInvalidRequest(errInvalidArgument)
+	}
+
 	// Verify the session token of the stsCred
 	claims, err := auth.ExtractClaims(stsCred.SessionToken, globalActiveCred.SecretKey)
 	if err != nil {
@@ -1090,13 +1118,13 @@ func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred madm
 	mapClaims := claims.Map()
 	expiry, err := auth.ExpToInt64(mapClaims["exp"])
 	if err != nil {
-		return fmt.Errorf("Expiry claim was not found")
+		return fmt.Errorf("Expiry claim was not found: %v", mapClaims)
 	}
 
 	// Extract the username and lookup DN and groups in LDAP.
 	ldapUser, ok := claims.Lookup(ldapUserN)
 	if !ok {
-		return fmt.Errorf("Could not find LDAP username in claims")
+		return fmt.Errorf("Could not find LDAP username in claims: %v", mapClaims)
 	}
 
 	// Need to lookup the groups from LDAP.
@@ -1120,16 +1148,6 @@ func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred madm
 		return fmt.Errorf("unable to save STS credential: %v", err)
 	}
 
-	// Notify in-cluster peers to reload temp users.
-	if !globalIAMSys.HasWatcher() {
-		for _, nerr := range globalNotificationSys.LoadUser(cred.AccessKey, true) {
-			if nerr.Err != nil {
-				logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-				logger.LogIf(ctx, nerr.Err)
-			}
-		}
-	}
-
 	return nil
 }
 
@@ -1151,8 +1169,8 @@ func (c *SiteReplicationSys) BucketMetaHook(ctx context.Context, item madmin.SRB
 			return wrapSRErr(err)
 		}
 
-		err = admClient.SRInternalReplicateBucketMeta(ctx, item)
-		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "SRInternalReplicateBucketMeta", err))
+		err = admClient.SRPeerReplicateBucketMeta(ctx, item)
+		logger.LogIf(ctx, c.annotatePeerErr(p.Name, "SRPeerReplicateBucketMeta", err))
 		return err
 	})
 	return cErr.summaryErr
@@ -1263,7 +1281,7 @@ func (c *SiteReplicationSys) getAdminClient(ctx context.Context, deploymentID st
 func (c *SiteReplicationSys) getPeerCreds() (*auth.Credentials, error) {
 	creds, ok := globalIAMSys.store.GetUser(c.state.ServiceAccountAccessKey)
 	if !ok {
-		return nil, errors.New("site replication service account not found!")
+		return nil, errors.New("site replication service account not found")
 	}
 	return &creds, nil
 }
@@ -1271,7 +1289,7 @@ func (c *SiteReplicationSys) getPeerCreds() (*auth.Credentials, error) {
 // syncLocalToPeers is used when initially configuring site replication, to
 // copy existing buckets, their settings, service accounts and policies to all
 // new peers.
-func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) SRError {
+func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) error {
 	// If local has buckets, enable versioning on them, create them on peers
 	// and setup replication rules.
 	objAPI := newObjectLayerFn()
@@ -1484,8 +1502,10 @@ func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) SRError {
 			if err != nil {
 				return errSRBackendIssue(err)
 			}
-			if _, isLDAPAccount := claims[ldapUserN]; !isLDAPAccount {
-				continue
+			if claims != nil {
+				if _, isLDAPAccount := claims[ldapUserN]; !isLDAPAccount {
+					continue
+				}
 			}
 			_, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.AccessKey)
 			if err != nil {
@@ -1518,7 +1538,7 @@ func (c *SiteReplicationSys) syncLocalToPeers(ctx context.Context) SRError {
 		}
 	}
 
-	return SRError{}
+	return nil
 }
 
 // Concurrency helpers
@@ -1608,6 +1628,13 @@ func (c *SiteReplicationSys) annotatePeerErr(dstPeer string, annotation string,
 	return fmt.Errorf("%s->%s: %s: %v", c.state.Name, dstPeer, annotation, err)
 }
 
+// isEnabled returns true if site replication is enabled
+func (c *SiteReplicationSys) isEnabled() bool {
+	c.RLock()
+	defer c.RUnlock()
+	return c.enabled
+}
+
 // Other helpers
 
 // newRemoteClusterHTTPTransport returns a new http configuration
@@ -1652,3 +1679,16 @@ func getPriorityHelper(replicationConfig replication.Config) int {
 	// leave some gaps in priority numbers for flexibility
 	return maxPrio + 10
 }
+
+// returns a slice with site names participating in site replciation but unspecified while adding
+// a new site.
+func getMissingSiteNames(oldDeps, newDeps set.StringSet, currSites []madmin.PeerInfo) []string {
+	diff := oldDeps.Difference(newDeps)
+	var diffSlc []string
+	for _, v := range currSites {
+		if diff.Contains(v.DeploymentID) {
+			diffSlc = append(diffSlc, v.Name)
+		}
+	}
+	return diffSlc
+}

cmd/site-replication_test.go

@@ -0,0 +1,66 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"testing"
+
+	"github.com/minio/madmin-go"
+	"github.com/minio/minio-go/v7/pkg/set"
+)
+
+// TestGetMissingSiteNames
+func TestGetMissingSiteNames(t *testing.T) {
+	testCases := []struct {
+		currSites []madmin.PeerInfo
+		oldDepIDs set.StringSet
+		newDepIDs set.StringSet
+		expNames  []string
+	}{
+		// Test1: missing some sites in replicated setup
+		{
+			[]madmin.PeerInfo{{Endpoint: "minio1:9000", Name: "minio1", DeploymentID: "dep1"},
+				{Endpoint: "minio2:9000", Name: "minio2", DeploymentID: "dep2"},
+				{Endpoint: "minio3:9000", Name: "minio3", DeploymentID: "dep3"}},
+			set.CreateStringSet("dep1", "dep2", "dep3"),
+			set.CreateStringSet("dep1"),
+			[]string{"minio2", "minio3"},
+		},
+		// Test2: new site added that is not in replicated setup
+		{
+			[]madmin.PeerInfo{{Endpoint: "minio1:9000", Name: "minio1", DeploymentID: "dep1"}, {Endpoint: "minio2:9000", Name: "minio2", DeploymentID: "dep2"}, {Endpoint: "minio3:9000", Name: "minio3", DeploymentID: "dep3"}},
+			set.CreateStringSet("dep1", "dep2", "dep3"),
+			set.CreateStringSet("dep1", "dep2", "dep3", "dep4"),
+			[]string{},
+		},
+		// Test3: not currently under site replication.
+		{
+			[]madmin.PeerInfo{},
+			set.CreateStringSet(),
+			set.CreateStringSet("dep1", "dep2", "dep3", "dep4"),
+			[]string{},
+		},
+	}
+
+	for i, tc := range testCases {
+		names := getMissingSiteNames(tc.oldDepIDs, tc.newDepIDs, tc.currSites)
+		if len(names) != len(tc.expNames) {
+			t.Errorf("Test %d: Expected `%v`, got `%v`", i+1, tc.expNames, names)
+		}
+	}
+}